@inetafrica/open-claudia 3.4.23 → 3.5.0

package/test-provider-scheduler.js

@@ -6,6 +6,7 @@ const assert = require("assert");
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
+const { spawnSync } = require("child_process");
 
 const root = __dirname;
 const fixturesRoot = path.join(root, "test-fixtures", "migrations");
@@ -373,6 +374,7 @@ function baseJob(overrides = {}) {
 function schedulerHarness(createSchedulerService, seed, options = {}) {
   const jobsStore = options.jobsStore || memoryJobsStore(seed);
   const scheduled = [];
+  const crons = [];
   const admissions = [];
   const runCalls = [];
   const sends = [];
@@ -401,12 +403,24 @@ function schedulerHarness(createSchedulerService, seed, options = {}) {
     workspace: "/workspace",
     now: options.now || (() => 1_720_000_000_500),
     occurrenceIdFactory: () => `next-occ-${++occurrenceSequence}`,
-    setTimer(fn, delay) { scheduled.push({ fn, delay }); return { fn, delay }; },
+    setTimer(fn, delay) {
+      // Node's setTimeout fires at once past this; no path may ask for more.
+      assert.ok(delay <= 2 ** 31 - 1, `timer delay ${delay} overflows setTimeout`);
+      scheduled.push({ fn, delay });
+      return { fn, delay };
+    },
     clearTimer() {},
+    // Cron ticks are driven by hand: callback() is one tick of the schedule.
+    scheduleCron(expression, callback) {
+      const task = { expression, callback, stopped: false, stop() { task.stopped = true; } };
+      crons.push(task);
+      return task;
+    },
     runInChat: async (_ctx, fn) => fn(),
     getUserState: () => state,
     send: async (text) => { sends.push(text); if (options.onSend) await options.onSend(state); return { ok: true }; },
     admitRunContext(prompt, cwd, _replyTo, opts) {
+      if (options.admitError) throw options.admitError;
       const context = Object.freeze({
         runId: `scheduled-${admissions.length + 1}`,
         provider: opts.provider,
@@ -425,6 +439,7 @@ function schedulerHarness(createSchedulerService, seed, options = {}) {
     async runAgent(prompt, cwd, _replyTo, opts) {
       runCalls.push({ prompt, cwd, opts });
       const next = outcomes.shift();
+      if (next instanceof Error) throw next;
       return next && typeof next.then === "function" ? next : next;
     },
     providerStatus: options.providerStatus || (() => ({ availability: { state: "available" }, auth: { state: "authenticated" } })),
@@ -434,7 +449,7 @@ function schedulerHarness(createSchedulerService, seed, options = {}) {
     missedGraceMs: 60 * 60 * 1000,
     ...(options.runWatchCheck ? { runWatchCheck: options.runWatchCheck } : {}),
   });
-  return { service, jobsStore, scheduled, admissions, runCalls, sends, state };
+  return { service, jobsStore, scheduled, crons, admissions, runCalls, sends, state };
 }
 
 async function firetimeSelectionProbe(createSchedulerService) {
@@ -670,6 +685,472 @@ async function busyWatchNoticeProbe(createSchedulerService) {
   assert.strictEqual(w.sends.length, 0);
 }
 
+function succeededRun() {
+  return {
+    ok: true,
+    status: "succeeded",
+    persistence: { transcript: { ok: true }, usage: { ok: true }, session: { ok: true } },
+    delivery: { ok: true },
+  };
+}
+
+// One tick of a hand-driven cron, then let its fire or poll settle (the fakes
+// all resolve in microtasks).
+async function tick(task) {
+  task.callback();
+  for (let i = 0; i < 3; i++) await nextTurn();
+}
+
+function repeatingWatch(overrides = {}) {
+  return baseJob({
+    kind: "cron",
+    schedule: "*/15 * * * *",
+    fireAt: undefined,
+    check: "fixture-check",
+    repeat: true,
+    label: "EVENT BUS ALARM",
+    expiresAt: 1_720_000_000_000 + 30 * 24 * 60 * 60 * 1000,
+    ...overrides,
+  });
+}
+
+async function pendingDeliveryPollingProbe(createSchedulerService) {
+  // 24 Sep: the event-bus watch triggered at 04:15, a busy room deferred the
+  // delivery until 11:50, and the bus went unpolled the whole time.
+  const t0 = 1_720_000_000_000;
+  const at = (minutes) => t0 + minutes * 60_000;
+  const iso = (ms) => new Date(ms).toISOString();
+  let clock = t0;
+  let code = 0;
+  let checks = 0;
+  const h = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "bus", occurrenceId: "occ-bus" })], {
+    now: () => clock,
+    runWatchCheck: async () => { checks++; return { code, timedOut: code === 124 }; },
+    outcomes: [succeededRun()],
+  });
+  h.service.init();
+  assert.strictEqual(h.crons.length, 1);
+  const poll = h.crons[0];
+  h.state.runningProcess = { pid: 1 };
+  await tick(poll);
+  assert.strictEqual(h.jobsStore.get("bus").status, "deferred");
+  assert.strictEqual(poll.stopped, false, "a repeating watch keeps its poll schedule while the delivery waits");
+  assert.strictEqual(h.scheduled.length, 1, "the delivery waits on its own retry timer");
+
+  clock = at(15);
+  await tick(poll);
+  assert.strictEqual(checks, 2, "the watch polls while its delivery is pending");
+  let bus = h.jobsStore.get("bus");
+  assert.strictEqual(bus.watchExtraHits, 1, "a new hit folds into the pending delivery");
+  assert.strictEqual(bus.watchLastHitAt, at(15));
+  assert.strictEqual(bus.occurrenceId, "occ-bus");
+  assert.strictEqual(bus.watchTriggeredAt, t0, "the trigger time stays the first hit's");
+  assert.strictEqual(h.scheduled.length, 1, "no second delivery is queued");
+
+  clock = at(16);
+  await h.scheduled.at(-1).fn();
+  assert.deepStrictEqual(h.sends, ['⏳ Watch "EVENT BUS ALARM" triggered 16 min ago and is waiting for the current run to end. /stop ends that run so the watch can go.'],
+    "the 15-minute heads-up still goes out, counted from the first hit");
+  code = 1;
+  clock = at(30);
+  await tick(poll);
+  code = 124;
+  clock = at(40);
+  await tick(poll);
+  assert.strictEqual(h.jobsStore.get("bus").watchLastClearAt, at(30), "a timed-out poll is not a clear");
+  code = 0;
+  clock = at(45);
+  await tick(poll);
+  bus = h.jobsStore.get("bus");
+  assert.strictEqual(bus.watchExtraHits, 2);
+  assert.strictEqual(bus.watchLastClearAt, at(30));
+  assert.strictEqual(h.runCalls.length, 0);
+
+  h.state.runningProcess = null;
+  clock = at(46);
+  assert.strictEqual((await h.scheduled.at(-1).fn()).ok, true);
+  assert.strictEqual(h.runCalls.length, 1, "one delivery for the whole wait");
+  const prompt = h.runCalls[0].prompt;
+  assert.match(prompt, /^\[Watch triggered "EVENT BUS ALARM"/);
+  assert.ok(prompt.includes(`[This watch kept polling while this delivery waited (queued at ${iso(t0)}): the check passed 2 more times, the latest at ${iso(at(45))}; the condition was still true at the latest check.]`), prompt);
+  assert.strictEqual(h.sends.length, 2, "heads-up once, then the advisory");
+  bus = h.jobsStore.get("bus");
+  assert.strictEqual(bus.status, "pending");
+  assert.strictEqual(bus.watchExtraHits, 0, "the next occurrence starts its own count");
+  assert.strictEqual(bus.watchLastHitAt, null);
+  assert.strictEqual(bus.watchLastClearAt, null);
+  assert.strictEqual(h.crons.filter((task) => !task.stopped).length, 1, "one live poll schedule after the delivery");
+
+  // Third case: attempt 1 died with exit 143 after a /stop and polling never
+  // resumed. The watch keeps polling and the delivery is retried.
+  clock = t0;
+  code = 0;
+  const killed = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "bus2", occurrenceId: "occ-bus2" })], {
+    now: () => clock,
+    runWatchCheck: async () => ({ code }),
+    outcomes: [
+      { ok: false, status: "failed", exitCode: 143, error: { code: "PROVIDER_EXIT_NONZERO", message: "Provider exited with code 143" }, delivery: { ok: true } },
+      succeededRun(),
+    ],
+  });
+  killed.service.init();
+  const killedPoll = killed.crons[0];
+  await tick(killedPoll);
+  assert.strictEqual(killed.runCalls.length, 1);
+  assert.strictEqual(killed.jobsStore.get("bus2").status, "failed");
+  assert.strictEqual(killedPoll.stopped, false, "a killed delivery does not stop the polling");
+  assert.strictEqual(killed.scheduled.length, 1, "the pending delivery is retried");
+  code = 1;
+  clock = at(15);
+  await tick(killedPoll);
+  assert.strictEqual(killed.jobsStore.get("bus2").watchLastClearAt, at(15));
+  clock = at(16);
+  assert.strictEqual((await killed.scheduled.at(-1).fn()).ok, true);
+  assert.strictEqual(killed.runCalls.length, 2);
+  assert.ok(killed.runCalls[1].prompt.includes(`its latest run, at ${iso(at(15))}, did not pass, so the condition may have cleared`), killed.runCalls[1].prompt);
+
+  // The watch also polls while its delivery runs; a hit then reaches the retry
+  // of a run that did not finish.
+  clock = t0;
+  const inRun = deferred();
+  const during = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "bus3", occurrenceId: "occ-bus3" })], {
+    now: () => clock,
+    runWatchCheck: async () => ({ code: 0 }),
+    outcomes: [inRun.promise, succeededRun()],
+  });
+  during.service.init();
+  during.crons[0].callback();
+  await nextTurn();
+  assert.strictEqual(during.jobsStore.get("bus3").status, "running");
+  clock = at(15);
+  await tick(during.crons[0]);
+  assert.strictEqual(during.jobsStore.get("bus3").watchExtraHits, 1, "the watch polls while its delivery runs");
+  inRun.resolve({ ok: false, status: "cancelled", error: { code: "RUN_CANCELLED", message: "stopped" }, delivery: { ok: true } });
+  for (let i = 0; i < 3; i++) await nextTurn();
+  assert.strictEqual(during.jobsStore.get("bus3").status, "failed");
+  clock = at(16);
+  assert.strictEqual((await during.scheduled.at(-1).fn()).ok, true);
+  assert.ok(during.runCalls[1].prompt.includes(`the check passed 1 more time, the latest at ${iso(at(15))}`), during.runCalls[1].prompt);
+
+  // After a restart, a watch whose delivery is still queued polls again too.
+  clock = t0;
+  const bootStore = memoryJobsStore([repeatingWatch({
+    id: "bus4", occurrenceId: "occ-bus4", status: "deferred", nextAttemptAt: t0 + 30_000,
+    watchTriggeredOccurrenceId: "occ-bus4", watchTriggeredAt: t0,
+  })]);
+  const rebooted = schedulerHarness(createSchedulerService, [], { jobsStore: bootStore, now: () => clock, runWatchCheck: async () => ({ code: 0 }) });
+  rebooted.service.init();
+  assert.strictEqual(rebooted.crons.length, 1, "boot re-arms the poll schedule of a watch whose delivery is queued");
+  assert.deepStrictEqual(rebooted.scheduled.map((timer) => timer.delay), [30_000], "and its delivery retry");
+  clock = at(15);
+  await tick(rebooted.crons[0]);
+  assert.strictEqual(bootStore.get("bus4").watchExtraHits, 1);
+
+  // A retry with no latched trigger (state an older version could leave) is
+  // not a pending delivery: its tick polls nothing, the retry runs the check.
+  let legacyChecks = 0;
+  const legacy = schedulerHarness(createSchedulerService, [], {
+    jobsStore: memoryJobsStore([repeatingWatch({ id: "legacy", occurrenceId: "occ-legacy", status: "failed", nextAttemptAt: t0 + 30_000 })]),
+    now: () => clock,
+    runWatchCheck: async () => { legacyChecks++; return { code: 0 }; },
+  });
+  legacy.service.init();
+  await tick(legacy.crons[0]);
+  assert.strictEqual(legacyChecks, 0, "only a latched trigger has a delivery to fold into");
+
+  // A check that outlives the delivery must not leak into the next occurrence.
+  clock = t0;
+  let release = null;
+  const race = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "race", occurrenceId: "occ-race" })], {
+    now: () => clock,
+    runWatchCheck: () => (release === null ? Promise.resolve({ code: 0 }) : new Promise((resolve) => { release = resolve; })),
+    outcomes: [succeededRun()],
+  });
+  race.service.init();
+  race.state.runningProcess = { pid: 1 };
+  await tick(race.crons[0]);
+  release = () => {};
+  race.crons[0].callback();
+  await nextTurn();
+  race.state.runningProcess = null;
+  assert.strictEqual((await race.scheduled.at(-1).fn()).ok, true);
+  release({ code: 0 });
+  for (let i = 0; i < 3; i++) await nextTurn();
+  assert.strictEqual(race.jobsStore.get("race").occurrenceId, "next-occ-1");
+  assert.strictEqual(race.jobsStore.get("race").watchExtraHits, 0, "a late check result belongs to no occurrence");
+
+  // One-shot watches keep today's semantics: the delivery waits alone.
+  let onceChecks = 0;
+  const once = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "once", repeat: false, occurrenceId: "occ-once" })], {
+    now: () => clock,
+    runWatchCheck: async () => { onceChecks++; return { code: 0 }; },
+  });
+  once.service.init();
+  once.state.runningProcess = { pid: 1 };
+  await tick(once.crons[0]);
+  assert.strictEqual(once.crons[0].stopped, true, "a one-shot watch parks on its retry timer alone");
+  await tick(once.crons[0]);
+  assert.strictEqual(onceChecks, 1, "and is not polled while its delivery waits");
+}
+
+async function checkTimeoutAlarmProbe(createSchedulerService) {
+  // 24 Sep: the Phone health check timed out at 09:30 and 09:45 and was only
+  // logged idle. Three timeouts in a row alarm once; the watch keeps polling.
+  let clock = 1_720_000_000_000;
+  let result = { code: 124, timedOut: true };
+  const h = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "phone", label: "Phone health", occurrenceId: "occ-phone" })], {
+    now: () => clock,
+    runWatchCheck: async () => result,
+    outcomes: [succeededRun(), succeededRun(), succeededRun()],
+  });
+  const poll = () => { clock += 15 * 60_000; return h.service.fireJob("phone"); };
+  assert.strictEqual((await poll()).status, "watch-idle");
+  assert.strictEqual((await poll()).status, "watch-idle");
+  assert.strictEqual(h.jobsStore.get("phone").watchTimeoutStreak, 2);
+  result = { code: 1, timedOut: false };
+  await poll();
+  assert.strictEqual(h.jobsStore.get("phone").watchTimeoutStreak, 0, "a completed check resets the count");
+  result = { code: 124, timedOut: true };
+  await poll();
+  await poll();
+  assert.strictEqual(h.runCalls.length, 0, "two timeouts in a row stay quiet");
+  assert.strictEqual((await poll()).ok, true);
+  assert.strictEqual(h.runCalls.length, 1, "the third timeout in a row wakes the agent");
+  assert.match(h.runCalls[0].prompt, /^\[Watch check timing out "Phone health"/);
+  assert.match(h.runCalls[0].prompt, /timed out 3 times in a row \(each run is capped at 30 s\)/);
+  assert.ok(h.sends.includes('⚠️ Watch "Phone health": its check timed out 3 times in a row'), h.sends.join("\n"));
+  const log = fs.readFileSync(process.env.OPEN_CLAUDIA_WATCH_LOG_FILE, "utf8");
+  assert.match(log, /label="Phone health" idle \(check timed out, 2 in a row\)/);
+  assert.match(log, /label="Phone health" check timed out 3 times in a row — delivery queued/);
+  assert.strictEqual(h.jobsStore.get("phone").status, "pending", "the watch keeps polling");
+  assert.strictEqual(h.jobsStore.get("phone").watchCheckTimingOut, false);
+  for (let i = 0; i < 5; i++) await poll();
+  assert.strictEqual(h.runCalls.length, 1, "a check that stays hung alarms once, not on every poll");
+  result = { code: 1, timedOut: false };
+  await poll();
+  result = { code: 124, timedOut: true };
+  for (let i = 0; i < 3; i++) await poll();
+  assert.strictEqual(h.runCalls.length, 2, "after a completed check a new run of timeouts alarms again");
+  result = { code: 0, timedOut: false };
+  await poll();
+  assert.strictEqual(h.runCalls.length, 3);
+  assert.match(h.runCalls[2].prompt, /^\[Watch triggered "Phone health"/);
+  assert.strictEqual(h.jobsStore.get("phone").watchTimeoutStreak, 0, "a passing check resets the count too");
+
+  // A timeout alarm stuck behind a busy room gets the same one heads-up, and
+  // a check that completes while it waits resets the count.
+  let hungResult = { code: 124, timedOut: true };
+  const hung = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "hung", label: "Phone health", occurrenceId: "occ-hung", watchTimeoutStreak: 2 })], {
+    now: () => clock,
+    runWatchCheck: async () => hungResult,
+  });
+  hung.state.runningProcess = { pid: 1 };
+  const raisedAt = clock;
+  await hung.service.fireJob("hung");
+  clock = raisedAt + 16 * 60_000;
+  await hung.scheduled.at(-1).fn();
+  assert.deepStrictEqual(hung.sends, ['⏳ Watch "Phone health" raised a check-timeout alarm 16 min ago and is waiting for the current run to end. /stop ends that run so the watch can go.']);
+  hung.service.init();
+  hungResult = { code: 1, timedOut: false };
+  await tick(hung.crons.at(-1));
+  assert.strictEqual(hung.jobsStore.get("hung").watchTimeoutStreak, 0, "a completed poll while the alarm waits resets the count");
+
+  // A one-shot watch is not used up by a timeout alarm; a real hit still ends it.
+  let onceResult = { code: 124, timedOut: true };
+  const once = schedulerHarness(createSchedulerService, [repeatingWatch({ id: "once", repeat: false, label: "deploy done", occurrenceId: "occ-once" })], {
+    now: () => clock,
+    runWatchCheck: async () => onceResult,
+    outcomes: [succeededRun(), succeededRun()],
+  });
+  for (let i = 0; i < 3; i++) await once.service.fireJob("once");
+  assert.strictEqual(once.runCalls.length, 1);
+  assert.strictEqual(once.jobsStore.get("once")?.status, "pending", "a timeout alarm does not use up a one-shot watch");
+  onceResult = { code: 0, timedOut: false };
+  await once.service.fireJob("once");
+  assert.strictEqual(once.runCalls.length, 2);
+  assert.match(once.runCalls[1].prompt, /^\[Watch triggered "deploy done"/);
+  assert.strictEqual(once.jobsStore.get("once"), null, "the real trigger still ends a one-shot watch");
+}
+
+async function farWakeupProbe(createSchedulerService, MAX_TIMER_DELAY_MS) {
+  // 17 Sep: a reminder set for 2028 fired 61 s after it was set. setTimeout
+  // fires at once past 2^31-1 ms, so a far wakeup sleeps in chunks.
+  const DAY = 24 * 60 * 60 * 1000;
+  for (const [label, horizon] of [["thirty-day", 30 * DAY], ["two-year", 730 * DAY]]) {
+    const t0 = 1_720_000_000_000;
+    let clock = t0;
+    const h = schedulerHarness(createSchedulerService, [baseJob({ id: label, fireAt: t0 + horizon, occurrenceId: `occ-${label}` })], {
+      now: () => clock,
+      outcomes: [succeededRun()],
+    });
+    h.service.init();
+    let chunks = 0;
+    for (let timer = h.scheduled.at(-1); clock + timer.delay < t0 + horizon; timer = h.scheduled.at(-1)) {
+      clock += timer.delay;
+      timer.fn();
+      chunks += 1;
+      assert.strictEqual(h.runCalls.length, 0, `${label}: a chunk never fires the wakeup early`);
+      assert.strictEqual(h.jobsStore.get(label).status, "pending");
+    }
+    assert.strictEqual(chunks, Math.ceil(horizon / MAX_TIMER_DELAY_MS) - 1, `${label}: slept in whole chunks`);
+    const final = h.scheduled.at(-1);
+    clock += final.delay;
+    assert.strictEqual(clock, t0 + horizon, `${label}: the last timer lands on the fire time`);
+    assert.strictEqual((await final.fn()).ok, true);
+    assert.strictEqual(h.runCalls.length, 1, `${label}: fires once when due`);
+    assert.strictEqual(h.jobsStore.get(label), null, `${label}: consumed after it fires`);
+    const armed = h.scheduled.length;
+    h.service.init();
+    await final.fn();
+    assert.strictEqual(h.scheduled.length, armed, `${label}: nothing re-arms a consumed wakeup`);
+    assert.strictEqual(h.runCalls.length, 1, `${label}: never fires twice`);
+  }
+
+  const due = schedulerHarness(createSchedulerService, [baseJob({ id: "due", fireAt: 1_720_000_000_500 })], { outcomes: [succeededRun(), succeededRun()] });
+  due.service.init();
+  assert.deepStrictEqual(due.scheduled.map((timer) => timer.delay), [0]);
+  await due.scheduled[0].fn();
+  await due.scheduled[0].fn();
+  assert.strictEqual(due.runCalls.length, 1, "a due wakeup fires once");
+  assert.strictEqual(due.jobsStore.get("due"), null);
+
+  // With Node's own setTimeout: a 30-day wakeup must not fire now.
+  const runs = [];
+  const real = createSchedulerService({
+    jobsStore: memoryJobsStore([baseJob({ id: "real-far", fireAt: Date.now() + 30 * DAY })]),
+    adapters: [{ id: "telegram-fixture", type: "telegram" }],
+    workspace: "/workspace",
+    runInChat: async (_ctx, fn) => fn(),
+    getUserState: () => ({ userId: "fixture-user", settings: { backend: "claude" }, activeSessions: {}, completedCompactions: {} }),
+    send: async () => ({ ok: true }),
+    admitRunContext: (_prompt, cwd) => ({ cwd }),
+    runAgent: async () => { runs.push("fired"); return succeededRun(); },
+    providerStatus: () => ({ availability: { state: "available" }, auth: { state: "authenticated" } }),
+    fallbackProviders: [],
+  });
+  real.init();
+  await new Promise((resolve) => setTimeout(resolve, 50));
+  real.stopAll();
+  assert.deepStrictEqual(runs, [], "a real 30-day timer does not fire at once");
+}
+
+function wakeupHorizonCliProbe() {
+  // The CLI refuses a <when> more than 10 years out before it calls the bot.
+  const temp = fs.mkdtempSync(path.join(os.tmpdir(), "oc-wakeup-horizon-"));
+  const env = { PATH: process.env.PATH, HOME: temp, OPEN_CLAUDIA_CONFIG_DIR: temp };
+  const run = (when) => spawnSync(process.execPath, ["-e", `require("./bin/schedule").runScheduleWakeup([${JSON.stringify(when)}, "fixture reminder"])`], {
+    cwd: root, env, encoding: "utf8", timeout: 30000,
+  });
+  try {
+    const far = run("4000d");
+    assert.strictEqual(far.status, 2, far.stderr);
+    assert.match(far.stderr, /^<when> "4000d" is \d{4}-\d\d-\d\dT[\d:.]+Z, more than 10 years out\. Check the date; for a far reminder use a yearly cron-add\.\n$/);
+    const absurd = run("99999999999999999999d");
+    assert.strictEqual(absurd.status, 2, absurd.stderr);
+    assert.match(absurd.stderr, /is past any valid date, more than 10 years out/);
+    const near = run("3600d");
+    assert.strictEqual(near.status, 2);
+    assert.match(near.stderr, /No active chat context/, "a <when> inside 10 years passes the horizon check");
+  } finally {
+    fs.rmSync(temp, { recursive: true, force: true });
+  }
+}
+
+function listingProbe() {
+  // A fired job must not keep reading as a future one (17 Sep: a fired 2028
+  // reminder still listed "fires 2028-08-01"; 21 Sep: a triggered watch whose
+  // delivery was queued listed "EXPIRED (fires on next poll)" and was taken
+  // for a watch that never passed).
+  const { fmtJob, fmtWatch } = require("./bin/schedule");
+  const triggeredAt = Date.UTC(2026, 8, 21, 6, 46, 1);
+  const watch = {
+    id: "cron_x_hn854u", check: "pipeline-terminal 12933", schedule: "* * * * *", repeat: false,
+    label: "SDK tag pipeline", occurrenceId: "occ", expiresAt: triggeredAt + 60_000,
+  };
+  assert.match(fmtWatch({ ...watch, status: "pending" }, triggeredAt + 120_000), /polls "\* \* \* \* \*"  EXPIRED \(fires on next poll\)  check:/,
+    "an untriggered watch still shows its expiry");
+  const queued = fmtWatch({ ...watch, status: "deferred", watchTriggeredOccurrenceId: "occ", watchTriggeredAt: triggeredAt }, triggeredAt + 120_000);
+  assert.strictEqual(queued, 'cron_x_hn854u  one-shot  polls "* * * * *"  TRIGGERED 2026-09-21T06:46:01.000Z, delivery waiting for a busy room  check: pipeline-terminal 12933  → SDK tag pipeline');
+  assert.match(fmtWatch({ ...watch, status: "failed", watchTriggeredOccurrenceId: "occ" }), /polls "\* \* \* \* \*"  delivery failed \(error\), retrying  check:/,
+    "a latch without a trigger time still lists");
+  const wake = { id: "wakeup_x", kind: "wakeup", label: "rotate token", fireAt: Date.UTC(2028, 7, 1) };
+  assert.strictEqual(fmtJob({ ...wake, status: "pending" }), "wakeup_x  wakeup  rotate token  fires 2028-08-01T00:00:00.000Z");
+  assert.strictEqual(fmtJob({ ...wake, status: "running" }), "wakeup_x  wakeup  rotate token  fires 2028-08-01T00:00:00.000Z  [delivery running]");
+  assert.match(fmtJob({ ...wake, status: "failed", lastError: { code: "PROVIDER_EXIT_NONZERO" } }), /  \[delivery failed \(PROVIDER_EXIT_NONZERO\), retrying\]$/);
+  assert.match(fmtJob({ ...wake, status: "dead-letter", lastError: { code: "PROVIDER_UNAUTHENTICATED" } }), /  \[delivery failed \(PROVIDER_UNAUTHENTICATED\), DEAD: will not fire\]$/);
+}
+
+async function strandedDeliveryProbe(createSchedulerService) {
+  // 21 and 23 Sep: watch alarms that triggered during long turns were never
+  // delivered. A jobs-store write that throws (the disk was full on 15 Sep)
+  // must leave a retry armed, and the delivery goes at the next idle moment.
+  const diskFull = () => Object.assign(new Error("ENOSPC: no space left on device, write"), { code: "ENOSPC" });
+  const failedRun = { ok: false, status: "failed", exitCode: 1, error: { code: "PROVIDER_EXIT_NONZERO", message: "exit 1" }, delivery: { ok: true } };
+  const cases = [
+    ["busy deferral", (patch) => patch.status === "deferred" && !!patch.busyDeferralCount, { busy: true }],
+    ["attempt start", (patch) => patch.status === "running", {}],
+    ["failed run", (patch) => patch.status === "failed", { outcomes: [failedRun] }],
+    ["thrown run", (patch) => patch.status === "failed", { outcomes: [new Error("provider crashed")] }],
+    ["admission refusal", (patch) => patch.status === "failed", { admitError: new Error("Choose Claude Code or OpenAI Codex before starting a model turn") }],
+  ];
+  for (const [name, faulty, setup] of cases) {
+    let clock = 1_720_000_000_000;
+    const store = memoryJobsStore([repeatingWatch({ id: "w", repeat: false, occurrenceId: "occ-w" })]);
+    const update = store.update.bind(store);
+    let armed = true;
+    let faults = 0;
+    store.update = (id, patch) => {
+      if (armed && faulty(patch)) { armed = false; faults++; throw diskFull(); }
+      return update(id, patch);
+    };
+    const options = {
+      jobsStore: store,
+      now: () => clock,
+      runWatchCheck: async () => ({ code: 0 }),
+      outcomes: [...(setup.outcomes || []), succeededRun()],
+      admitError: setup.admitError || null,
+    };
+    const h = schedulerHarness(createSchedulerService, [], options);
+    if (setup.busy) h.state.runningProcess = { pid: 1 };
+    const first = await h.service.fireJob("w");
+    assert.strictEqual(faults, 1, `${name}: the write failed`);
+    assert.strictEqual(first.ok, false, name);
+    assert.strictEqual(h.scheduled.length, 1, `${name}: a retry stays armed after the failed write`);
+    h.state.runningProcess = null;
+    options.admitError = null;
+    clock += 60_000;
+    assert.strictEqual((await h.scheduled.at(-1).fn()).ok, true, `${name}: delivered at the next idle moment`);
+    assert.strictEqual(h.jobsStore.get("w"), null, name);
+  }
+  assert.match(fs.readFileSync(process.env.OPEN_CLAUDIA_WATCH_LOG_FILE, "utf8"), /delivery bookkeeping failed \(ENOSPC\) — retrying in \d+ s/);
+
+  // An admission refusal with a working store is a counted, retried failure.
+  const refused = schedulerHarness(createSchedulerService, [baseJob({ id: "refused" })], { admitError: new Error("no provider selected") });
+  const refusal = await refused.service.fireJob("refused");
+  assert.strictEqual(refusal.status, "failed");
+  assert.strictEqual(refused.jobsStore.get("refused").attemptCount, 1);
+  assert.strictEqual(refused.scheduled.length, 1);
+
+  // Pre-run failure records that cannot be written still keep a retry.
+  for (const [name, overrides, options] of [
+    ["unknown adapter", { adapter: "gone", adapterType: "gone" }, {}],
+    ["no runnable provider", {}, { providerStatus: () => ({ availability: { state: "missing" }, auth: { state: "not_checked" } }) }],
+  ]) {
+    const store = memoryJobsStore([baseJob({ id: "pre", ...overrides })]);
+    store.update = () => { throw diskFull(); };
+    const h = schedulerHarness(createSchedulerService, [], { jobsStore: store, ...options });
+    assert.strictEqual((await h.service.fireJob("pre")).status, "failed", name);
+    assert.strictEqual(h.scheduled.length, 1, `${name}: a retry stays armed`);
+    assert.strictEqual(h.runCalls.length, 0);
+  }
+
+  // A running lease left by an earlier process is looked at again when it
+  // lapses; fireJob already turns away a fire that is really in flight.
+  const leased = schedulerHarness(createSchedulerService, [baseJob({ id: "leased", status: "running", attemptCount: 1, nextAttemptAt: 1_720_000_000_500 + 5000 })]);
+  assert.strictEqual((await leased.service.fireJob("leased")).inFlight, true);
+  assert.deepStrictEqual(leased.scheduled.map((timer) => timer.delay), [5000]);
+}
+
 async function durablePendingProbe(createSchedulerService, jobsApi) {
   const configDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-pending-restart-"));
   let clock = 1_720_000_000_500;
@@ -706,7 +1187,7 @@ async function durablePendingProbe(createSchedulerService, jobsApi) {
       const terminal = deferred();
       const restarted = schedulerHarness(createSchedulerService, [], {
         jobsStore: reloaded, now: () => clock, outcomes: [terminal.promise, success],
-        runWatchCheck: async () => { throw new Error("pending watch must not poll after restart"); },
+        runWatchCheck: async () => { throw new Error("a queued delivery must not re-run its trigger check after restart"); },
       });
       restarted.service.init();
       const firing = restarted.scheduled[0].fn();
@@ -1063,6 +1544,18 @@ function staticSchedulingProbe() {
   assert.match(source["core/scheduler.js"], /watchExpired/, "scheduler must mark expiry fires distinctly");
   assert.match(source["core/loopback.js"], /expiresAt/, "loopback must pass the watch expiry through");
   assert.match(source["bin/schedule.js"], /--timeout/, "watch-add CLI must expose --timeout");
+
+  // Phase 0 (B2): a watch check must NOT inherit the bot's whole environment,
+  // must build an explicit hardened env, and must register a creator-scoped
+  // run around the poll.
+  assert.ok(!/env:\s*process\.env/.test(source["core/scheduler.js"]), "watch check must not spawn with the bot's whole process env");
+  assert.match(source["core/scheduler.js"], /buildWatchCheckEnv\(process\.env/, "watch check builds an explicit hardened env");
+  assert.match(source["core/scheduler.js"], /registerRun\(checkRunId/, "watch check registers a creator-scoped run");
+  assert.match(source["core/scheduler.js"], /endRun\(checkRunId\)/, "watch check ends its run when the check exits");
+  // Phase 0 (B2): the loopback binds a job's identity to the run-attested
+  // creator, not the payload's claims, and refuses an unattributed caller.
+  assert.match(source["core/loopback.js"], /JOB_CREATE_KINDS/, "loopback must bind job creation to the run-attested caller");
+  assert.match(source["core/loopback.js"], /createdBy/, "jobs must record their creator principal");
 }
 
 async function main() {
@@ -1072,6 +1565,8 @@ async function main() {
   assert.strictEqual(typeof jobsApi.migrateJobsDocument, "function");
   assert.strictEqual(typeof jobsApi.createJobsStore, "function");
   assert.strictEqual(typeof schedulerApi.createSchedulerService, "function");
+  assert.strictEqual(typeof schedulerApi.buildWatchCheckEnv, "function", "scheduler exports the watch-check env builder");
+  assert.strictEqual(typeof schedulerApi.watchCheckPrincipal, "function", "scheduler exports the watch-check principal resolver");
 
   migrationProbe(jobsApi);
   legacyDefaultsProbe(jobsApi);
@@ -1083,6 +1578,12 @@ async function main() {
   await busyRestartAndMissedProbe(schedulerApi.createSchedulerService);
   await durablePendingProbe(schedulerApi.createSchedulerService, jobsApi);
   await busyWatchNoticeProbe(schedulerApi.createSchedulerService);
+  await pendingDeliveryPollingProbe(schedulerApi.createSchedulerService);
+  await checkTimeoutAlarmProbe(schedulerApi.createSchedulerService);
+  await farWakeupProbe(schedulerApi.createSchedulerService, schedulerApi.MAX_TIMER_DELAY_MS);
+  wakeupHorizonCliProbe();
+  listingProbe();
+  await strandedDeliveryProbe(schedulerApi.createSchedulerService);
   await unavailableAndFallbackProbe(schedulerApi.createSchedulerService);
   await cronOccurrenceProbe(schedulerApi.createSchedulerService);
   await watchGateProbe(schedulerApi.createSchedulerService);

package/test-provider-tool-hooks.js

@@ -95,6 +95,17 @@ try {
   const claudeSettings = buildClaudeHookSettings();
   assert.strictEqual(claudeSettings.hooks.PreToolUse[0].matcher, "Bash");
   assert.match(claudeSettings.hooks.PreToolUse[0].hooks[0].command, /deny-gate-hook/);
+  // Claude's file-reading tools never pass through a shell, so the guarded
+  // read gate needs its own matcher on the same deny-gate entry.
+  const readHook = claudeSettings.hooks.PreToolUse.find((entry) => entry.matcher === "^(Read|Grep|Glob|NotebookRead|LS)$");
+  assert.ok(readHook, "Claude read tools are hooked");
+  assert.strictEqual(readHook.hooks[0].command, claudeSettings.hooks.PreToolUse[0].hooks[0].command);
+  for (const tool of ["Read", "Grep", "Glob", "NotebookRead", "LS"]) {
+    assert.ok(new RegExp(readHook.matcher).test(tool), `${tool} matches the read-gate matcher`);
+  }
+  for (const tool of ["Bash", "Write", "WebFetch", "ReadMcpResourceTool", "Task"]) {
+    assert.ok(!new RegExp(readHook.matcher).test(tool), `${tool} stays out of the read-gate matcher`);
+  }
   assert.match(claudeSettings.hooks.PreToolUse[0].hooks[0].command, new RegExp(process.execPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")));
   assert.strictEqual(ensureClaudeHookSettings({ settingsFile: claudeSettingsFile }), claudeSettingsFile);
   assert.deepStrictEqual(JSON.parse(fs.readFileSync(claudeSettingsFile, "utf8")), claudeSettings);

package/test-run-lock.js

@@ -87,6 +87,8 @@ const { acquireRunLock } = require("./core/state");
     'Queued behind the watch "Phone alarm" (running 42 min). /stop cancels it.');
   assert.strictEqual(queuedNotice({ admittedAt: t0, userPrompt: '[Scheduled wakeup "Check CI" fired at x]' }, t0 + 3 * 60_000),
     'Queued behind the wakeup "Check CI" (running 3 min). /stop cancels it.');
+  assert.strictEqual(queuedNotice({ admittedAt: t0, userPrompt: '[Watch check timing out "Phone health" fired at x; occurrence y]' }, t0 + 20 * 60_000),
+    'Queued behind the watch "Phone health" (running 20 min). /stop cancels it.', "a check-timeout alarm is a watch run too");
   const long = queuedNotice({ admittedAt: t0, userPrompt: `[Watch triggered "${"x".repeat(90)}" fired at x]` }, t0 + 3 * 60_000);
   assert.ok(long.includes(`"${"x".repeat(59)}…"`), "a long label is shortened to 60 characters");
 }

package/test-tool-guard.js

@@ -122,7 +122,9 @@ const deny = guard.runHook(JSON.stringify({ tool_name: "Bash", tool_input: { com
 assert.strictEqual(deny.exit, 2, "deny → exit 2 (blocking)");
 assert.ok(/deny-gate/.test(deny.stderr), "deny → stderr message for the model");
 assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Bash", tool_input: { command: "ls -la" } })).exit, 0, "benign Bash → allow");
-assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Read", tool_input: { file_path: "/etc/hosts" } })).exit, 0, "non-Bash tools are never gated");
+assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Read", tool_input: { file_path: "/etc/hosts" } })).exit, 0, "reads are never gated without channel context");
+// Owner-local (no channel context) fails OPEN; guarded turns fail closed
+// instead (pinned in test-read-guard.js).
 assert.strictEqual(guard.runHook("not json{{{").exit, 0, "unparseable payload fails OPEN");
 assert.strictEqual(guard.runHook(null).exit, 0, "null payload fails OPEN");
 assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Bash" })).exit, 0, "missing tool_input fails OPEN");
@@ -154,7 +156,7 @@ assert.strictEqual(
   guard.runHook(JSON.stringify({ tool_name: "apply_patch", tool_input: { command: "*** Begin Patch\n*** Add File: /tmp/evil.txt\n+x\n*** End Patch\n" } })).exit, 2,
   "Codex apply_patch equally gated on guarded turns (probed live: file edits arrive under this tool name)");
 assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Read", tool_input: { file_path: "/etc/hosts" } })).exit, 0,
-  "reads stay free on guarded turns");
+  "ordinary reads stay free on guarded turns (protected roots: test-read-guard.js)");
 assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Bash", tool_input: { command: "ls -la" } })).exit, 0,
   "benign Bash unaffected by the file gate (deny-gate rules unchanged)");
 
@@ -185,8 +187,11 @@ assert.strictEqual(guard.externalControlPlaneDenial(cpWrite), null, "no OC_CHANN
 assert.strictEqual(guard.runHook(JSON.stringify({ tool_name: "Bash", tool_input: { command: cpWrite } })).exit, 0,
   "owner-local shells keep full config access");
 process.exitCode = 0;
+delete process.env.OC_RUN_ATTEST; // hermetic even when the suite runs inside a live bot turn
 require("./bin/keyring").run(["path"]);
-assert.strictEqual(process.exitCode || 0, 0, "keyring CLI stays open owner-local (no channel env)");
+assert.strictEqual(process.exitCode, 1,
+  "keyring CLI is owner-only even with no channel env: no live owner run behind the process = unattributed (test-cli-owner-gates.js)");
+process.exitCode = 0;
 process.env.OC_CHANNEL_ADAPTER = "telegram";
 process.env.OC_CHANNEL_ID = "999000111";
 process.env.OC_CHANNEL_USER_ID = "999000111";
@@ -288,6 +293,10 @@ assert.strictEqual(pre[1].matcher, "^(Write|Edit|MultiEdit|NotebookEdit)$",
   "file-mutation tools are hooked for the external-speaker gate");
 assert.strictEqual(pre[1].hooks[0].command, pre[0].hooks[0].command,
   "file gate rides the same deny-gate-hook entry (one CLI, one audit path)");
+assert.strictEqual(pre[2].matcher, "^(Read|Grep|Glob|NotebookRead|LS)$",
+  "file-reading tools are hooked for the guarded read gate");
+assert.strictEqual(pre[2].hooks[0].command, pre[0].hooks[0].command,
+  "read gate rides the same deny-gate-hook entry");
 // second call with the file read-only: content-compare must skip the write
 fs.chmodSync(sPath, 0o444);
 assert.strictEqual(guard.ensureHookSettings(), sPath, "unchanged settings are not rewritten");