create-walle 0.10.1 → 0.10.2

package/template/coding-task-manager/lib/session-create/worker-wait-watcher.js

@@ -0,0 +1,315 @@
+'use strict';
+
+// WP-1 (worker-wait-watcher, D-3/D-4): while a lead's `walle_worker_wait` is parked in CTM,
+// a watcher tick asks the WORKER'S OWN pinned model whether the lead must act now -- the
+// worker is stuck or looping, repeats the same error, asks a question, or works outside its
+// brief. A `WAKE_LEAD` answer wakes the parked wait early with `woke_on: 'watcher'`.
+//
+// The watcher never delays a plain-code wake: it only ever calls `wake` from its own tick, so
+// terminal / generation / advisory / user_input / route_clear / timeout resolve exactly as
+// today, and a code wake that lands during an in-flight model call wins (stop() marks the
+// watcher dead; the late reply is dropped).
+//
+// Two consecutive failures (transport error, a reply with `ok: false`, a timeout, or a reply
+// that carries no decision) end the watcher with `watcher_unavailable` and the lead is back to
+// today's behaviour -- no retry loop, no second route (D-4).
+//
+// The module holds NO network import: the caller injects `callModel` (production wiring lives
+// in worker-observability.js) and the worker's pin is always the worker row's own
+// `route_alias` + ':' + `model` -- no vendor or model id appears here.
+
+const { redactSecrets } = require('../redact-secrets');
+
+const WATCH_INTERVAL_MS = 30000; // today's lead cadence
+const WATCH_TAIL_CHARS = 60000; // pilot watcher input size
+const WATCH_TIMEOUT_MS = 20000; // background-completion DEFAULT_TIMEOUT_MS (wall-e/http/background-completion-api.js:27)
+const WATCH_MAX_CONSECUTIVE_FAILURES = 2; // D-4
+const WATCH_MAX_CONCURRENT_CALLS = 4;
+const WATCH_SILENT_EVERY = 10;
+const WATCH_MODE_ON = 'on';
+const WATCH_MODE_SILENT = 'silent';
+const WATCH_MODE_OFF = 'off';
+const WATCH_MODES = new Set([WATCH_MODE_ON, WATCH_MODE_SILENT, WATCH_MODE_OFF]);
+const WATCH_FEATURE = 'lead-wait-watch';
+const WATCH_MAX_TOKENS = 300;
+const WATCH_SYSTEM = 'You are a watcher working for a lead coding agent. The lead is waiting on a delegated worker. '
+  + 'Read the worker status and the end of its transcript and decide whether the lead must act now (the worker is stuck '
+  + 'or looping, repeats the same error, asks a question or needs a decision, works outside its brief, or anything else '
+  + 'needs the lead) or whether it is progressing normally and the lead should keep waiting.\n'
+  + 'Answer with one JSON object only: {"decision":"KEEP_WAITING"|"WAKE_LEAD","reason":"<= 25 words"}';
+
+// The reply contract is a REGEX, not JSON.parse: a model that wraps its object in prose still
+// answers, and a reply with no decision at all is a failure (D-3/D-4).
+const WATCH_DECISION_RE = /"decision"\s*:\s*"(KEEP_WAITING|WAKE_LEAD)"/;
+const WATCH_REASON_RE = /"reason"\s*:\s*"([^"]{0,200})"/;
+
+// D-3: at most WATCH_MAX_CONCURRENT_CALLS model calls in flight across ALL parked waiters. A
+// tick that cannot get a slot is skipped, never queued.
+let _inFlightCalls = 0;
+function _tryAcquireSlot() {
+  if (_inFlightCalls >= WATCH_MAX_CONCURRENT_CALLS) return false;
+  _inFlightCalls += 1;
+  return true;
+}
+function _releaseSlot() {
+  if (_inFlightCalls > 0) _inFlightCalls -= 1;
+}
+
+// Test-only bookkeeping: every ARMED interval timer is tracked here (a fired timer removes
+// itself at fire time), so _resetForTests() can report how many watchers were still ticking
+// when everything was expected to be stopped.
+let _liveTimers = new Set();
+
+function parseWatchReply(text) {
+  if (typeof text !== 'string' || !text) return null;
+  const match = WATCH_DECISION_RE.exec(text);
+  if (!match) return null;
+  const reason = WATCH_REASON_RE.exec(text);
+  return { decision: match[1], reason: reason ? reason[1].trim() : '' };
+}
+
+// One compact status per tick: fixed keys, missing ones null, everything bounded to what the
+// watcher prompt needs (never the whole payload).
+function compactWorkerStatus(status) {
+  const source = status && typeof status === 'object' ? status : {};
+  const budget = source.request_budget && typeof source.request_budget === 'object' ? source.request_budget : null;
+  return {
+    status: typeof source.status === 'string' ? source.status : null,
+    turns: Number.isInteger(source.turns) ? source.turns : null,
+    last_activity_at: Number.isFinite(source.last_activity_at) ? source.last_activity_at : null,
+    request_budget: budget ? {
+      requestCount: Number.isInteger(budget.requestCount) ? budget.requestCount : null,
+      completedToolCalls: Number.isInteger(budget.completedToolCalls) ? budget.completedToolCalls : null,
+      incompleteStreams: Number.isInteger(budget.incomplete_streams) ? budget.incomplete_streams : (Number.isInteger(budget.incompleteStreams) ? budget.incompleteStreams : null),
+    } : null,
+    advisories: Array.isArray(source.advisories) ? source.advisories.slice(0, 8) : null,
+    repeated_error: source.repeated_error ?? null,
+    observation_repeats: Number.isInteger(source.observation_repeats) ? source.observation_repeats : null,
+  };
+}
+
+function errorText(value) {
+  if (!value) return 'watch_failed';
+  if (typeof value === 'string') return value.slice(0, 200);
+  return String(value.code || value.message || value).slice(0, 200);
+}
+
+// The worker's own pin: the row's route_alias + ':' + model, or null when the row carries no
+// pin (the lead keeps today's behaviour, D-3).
+function workerPin(workerRoute, workerId) {
+  if (typeof workerRoute !== 'function') return null;
+  let row = null;
+  try { row = workerRoute(workerId); }
+  catch { return null; /* expected: an unreadable row carries no pin -- the lead keeps today's behaviour */ }
+  if (!row || typeof row !== 'object') return null;
+  const route = typeof row.route === 'string' ? row.route.trim() : '';
+  const model = typeof row.model === 'string' ? row.model.trim() : '';
+  if (!route || !model) return null;
+  return `${route}:${model}`;
+}
+
+/**
+ * Start the per-waiter watcher. Returns { stop() } or null when nothing is watched.
+ *
+ * @param {object} o
+ * @param {string[]} o.workerIds
+ * @param {'on'|'silent'|'off'} [o.mode='on']
+ * @param {(why:'watcher'|'watcher_unavailable', detail:{worker_id:string, reason?:string, error?:string}) => void} o.wake
+ * @param {(d:{worker_id:string, at:number, decision:'KEEP_WAITING'|'WAKE_LEAD', silent:boolean, reason?:string}) => void} [o.onDecision]
+ * @param {(id:string) => Promise<object|null>|object|null} o.getStatus
+ * @param {(id:string, maxChars:number) => Promise<string>} o.readTail
+ * @param {(req:{feature:string, model:string, system:string, messages:object[], maxTokens:number, timeoutMs:number, sessionId:string}) => Promise<{ok:boolean, text?:string, error?:string}>} o.callModel
+ * @param {(id:string) => {route:string, model:string}|null} o.workerRoute
+ * @param {Function} [o.setTimer=setTimeout]
+ * @param {Function} [o.clearTimer=clearTimeout]
+ * @param {Function} [o.now=Date.now]
+ * @returns {{ stop(): void }|null}
+ */
+function startWaitWatcher({
+  workerIds, mode = WATCH_MODE_ON, wake, onDecision, getStatus, readTail, callModel, workerRoute,
+  setTimer = setTimeout, clearTimer = clearTimeout, now = Date.now,
+} = {}) {
+  if (!WATCH_MODES.has(mode) || mode === WATCH_MODE_OFF) return null;
+  if (typeof wake !== 'function' || typeof getStatus !== 'function' || typeof readTail !== 'function' || typeof callModel !== 'function') return null;
+  const silent = mode === WATCH_MODE_SILENT;
+  const targets = [];
+  for (const rawId of Array.isArray(workerIds) ? workerIds : []) {
+    const id = String(rawId || '');
+    if (!id) continue;
+    const pinnedModel = workerPin(workerRoute, id);
+    if (!pinnedModel) continue; // D-3: a worker with no route_alias/model row starts no watcher
+    targets.push({ id, pinnedModel });
+  }
+  if (!targets.length) return null;
+
+  const state = { dead: false, timer: null, ticks: 0, failures: new Map() };
+
+  function clearOwnTimer() {
+    if (state.timer == null) return;
+    clearTimer(state.timer);
+    _liveTimers.delete(state.timer);
+    state.timer = null;
+  }
+
+  function stop() {
+    if (state.dead) return;
+    state.dead = true; // a reply that lands after this is dropped
+    clearOwnTimer();
+  }
+
+  function recordFailure(target, error) {
+    if (state.dead) return;
+    const consecutive = (state.failures.get(target.id) || 0) + 1;
+    state.failures.set(target.id, consecutive);
+    if (consecutive < WATCH_MAX_CONSECUTIVE_FAILURES) return;
+    state.failures.set(target.id, 0);
+    wake('watcher_unavailable', { worker_id: target.id, error: errorText(error) });
+  }
+
+  // The call is bounded by WATCH_TIMEOUT_MS here as well as by the request's own timeoutMs: a
+  // wedged lane must resolve to watcher_unavailable, not park the tick forever.
+  async function callWithTimeout(target, status, tail) {
+    const request = {
+      feature: WATCH_FEATURE,
+      model: target.pinnedModel,
+      system: WATCH_SYSTEM,
+      messages: [{ role: 'user', content: `${JSON.stringify(compactWorkerStatus(status))}\n\nTranscript tail:\n${tail}` }],
+      maxTokens: WATCH_MAX_TOKENS,
+      timeoutMs: WATCH_TIMEOUT_MS,
+      // WP-4a (D-9): the ledger attribution key is the WATCHED WORKER's ctm_session_id. Each tick
+      // is per worker and every wait-watch outcome row already stores that worker as `worker_id`
+      // (WP-1), so the cell's spend is exactly the ledger rows of its own workers -- never the
+      // lead's (a lead parks several workers at once; the lead's id would blend their cells).
+      sessionId: target.id,
+    };
+    const call = Promise.resolve().then(() => callModel(request));
+    call.catch(() => { /* expected: the watchdog may settle first; a late rejection is never unhandled */ });
+    let watchdogId = null;
+    const watchdog = new Promise((_resolve, reject) => {
+      watchdogId = setTimer(() => reject(new Error('watch_timeout')), WATCH_TIMEOUT_MS);
+      if (watchdogId && typeof watchdogId.unref === 'function') watchdogId.unref();
+    });
+    try {
+      return await Promise.race([call, watchdog]);
+    } finally {
+      if (watchdogId != null) {
+        clearTimer(watchdogId);
+        watchdogId = null;
+      }
+    }
+  }
+
+  async function runWorkerTick(target) {
+    if (state.dead) return;
+    if (!_tryAcquireSlot()) return; // skipped, never queued
+    try {
+      let reply;
+      try {
+        const status = await getStatus(target.id);
+        if (state.dead) return;
+        const tail = redactSecrets(await readTail(target.id, WATCH_TAIL_CHARS));
+        if (state.dead) return;
+        reply = await callWithTimeout(target, status, tail);
+      } catch (error) {
+        recordFailure(target, error);
+        return;
+      }
+      if (state.dead) return;
+      if (!reply || reply.ok !== true) {
+        recordFailure(target, reply && reply.error);
+        return;
+      }
+      const parsed = parseWatchReply(reply.text);
+      if (!parsed) {
+        recordFailure(target, 'watch_reply_unparsed');
+        return;
+      }
+      state.failures.set(target.id, 0);
+      const decision = {
+        worker_id: target.id, at: now(), decision: parsed.decision, silent, reason: parsed.reason,
+      };
+      if (typeof onDecision === 'function') {
+        try { onDecision(decision); }
+        catch { /* expected: a decision sink failure never breaks the watcher */ }
+      }
+      // `silent` records its decisions and never wakes (D-3).
+      if (!silent && parsed.decision === 'WAKE_LEAD') wake('watcher', { worker_id: target.id, reason: parsed.reason });
+    } finally {
+      _releaseSlot();
+    }
+  }
+
+  function schedule() {
+    if (state.dead) return;
+    const id = setTimer(() => {
+      state.timer = null;
+      _liveTimers.delete(id);
+      tickCycle().catch(() => { /* expected: a tick never throws out; this is the last-resort net */ });
+    }, WATCH_INTERVAL_MS);
+    state.timer = id;
+    _liveTimers.add(id);
+    if (id && typeof id.unref === 'function') id.unref();
+  }
+
+  async function tickCycle() {
+    try {
+      if (state.dead) return;
+      state.ticks += 1;
+      // silent runs only every WATCH_SILENT_EVERY-th tick; every other tick is a no-op.
+      if (!silent || state.ticks % WATCH_SILENT_EVERY === 0) {
+        await Promise.all(targets.map((target) => runWorkerTick(target).catch(() => { /* expected: per-worker isolation; failures are counted inside */ })));
+      }
+    } finally {
+      schedule();
+    }
+  }
+
+  schedule();
+  return { stop };
+}
+
+// Production readTail: the worker's own provider transcript, tail-first, clipped to
+// WATCH_TAIL_CHARS characters (read through lib/transcript-source.js -- the only module
+// allowed to open a provider transcript).
+function createWaitWatcherReadTail({ dbModule } = {}) {
+  return async function readTail(workerId, maxChars = WATCH_TAIL_CHARS) {
+    const db = dbModule || require('../../db');
+    const agentRow = typeof db?.getAgentSessionByCtmId === 'function' ? db.getAgentSessionByCtmId(workerId) : null;
+    const transcriptPath = String((agentRow && agentRow.jsonl_path) || '');
+    if (!transcriptPath) return '';
+    const transcriptSource = require('../transcript-source');
+    const st = await transcriptSource.statTranscript(transcriptPath);
+    if (!st || st.exists === false) return '';
+    const size = Number(st.size) || 0;
+    const start = Math.max(0, size - maxChars * 4);
+    const buf = await transcriptSource.readTranscriptRange(transcriptPath, start, size);
+    return buf.toString('utf8').slice(-maxChars);
+  };
+}
+
+// Test-only: clear every armed interval timer and the module-level semaphore, returning how
+// many watchers were still ticking (0 when every watcher was stopped properly).
+function _resetForTests() {
+  const cleared = _liveTimers.size;
+  for (const id of _liveTimers) clearTimeout(id);
+  _liveTimers = new Set();
+  _inFlightCalls = 0;
+  return cleared;
+}
+
+module.exports = {
+  startWaitWatcher,
+  createWaitWatcherReadTail,
+  parseWatchReply,
+  compactWorkerStatus,
+  WATCH_MODES,
+  WATCH_FEATURE,
+  WATCH_INTERVAL_MS,
+  WATCH_TAIL_CHARS,
+  WATCH_TIMEOUT_MS,
+  WATCH_MAX_CONSECUTIVE_FAILURES,
+  WATCH_MAX_CONCURRENT_CALLS,
+  WATCH_SILENT_EVERY,
+  WATCH_SYSTEM,
+  _resetForTests,
+};

package/template/coding-task-manager/lib/session-geometry-authority-handler.js

@@ -118,8 +118,11 @@ function handleSessionGeometryClaim(ws, msg, replayOpts = {}) {
   let committedGrid = { cols: 0, rows: 0 };
   let queuedDueAt = 0;
   let broadcastRolesAfterAck = false;
+  // WP-2: the diagnostic below runs even when sanitizeGeometryClaim threw (or the claim never
+  // reached arbitration), so the sanitized claim is hoisted out of the try to be readable there.
+  let claim = null;
   try {
-    const claim = sanitizeGeometryClaim(msg, Number(ws.socketGeneration || 0));
+    claim = sanitizeGeometryClaim(msg, Number(ws.socketGeneration || 0));
     sessionId = claim.id;
     session = getSession(sessionId);
     let currentLease = _geometryLease(session, sessionId);
@@ -270,6 +273,12 @@ function handleSessionGeometryClaim(ws, msg, replayOpts = {}) {
   _recordTerminalGeometryClaimDiagnostic(session, sessionId, ws.clientId, {
     disposition, reason, intent: ['reconnect', 'activation', 'pointer', 'keyboard', 'split-focus', 'revalidate', 'selection'].includes(msg?.intent) ? msg.intent : 'invalid',
     ownerGeneration, measurementId, cols: committedGrid.cols, rows: committedGrid.rows, viewerRegistered,
+    // WP-2 (docs/codex-terminal-geometry-sticky-cols-2026-10-24.html §5): `cols` is the
+    // COMMITTED grid (pty truth); it is initialised from `session._ptyCols` and only moves on an
+    // accepted commit, so a queued/rejected claim for 196 against a 94-col pty logged `cols: 94`
+    // with no record of what the client asked for -- the exact hole in the live 94-cols incident.
+    claimCols: Math.max(0, Number(claim?.measurement?.cols) || 0),
+    claimRows: Math.max(0, Number(claim?.measurement?.rows) || 0),
   });
   _sendTerminalGeometryClaimAckV2(ws, {
     id: sessionId, socketGeneration: Number(ws.socketGeneration || 0), measurementId, expectedOwnerGeneration,