@kelceyp/caw-server 1.0.279 → 1.0.280

package/dist/pty-wrapper.mjs

@@ -25,6 +25,7 @@ import { tmpdir, homedir } from 'os';
 import { execSync } from 'child_process';
 import { readSentinelPayload, hasPendingWork } from './pty-deferred-teardown.mjs';
 import { emitResult as emitResultLine } from './pty-emit-result.mjs';
+import { TEARDOWN_REASONS } from './teardownReasons.mjs';
 
 // --- Arg parsing ---
 
@@ -386,25 +387,95 @@ let teardownReason = null;
 let startedAt = null;
 let endedAt = null;
 
+// Module-scope armed timestamp for the watchdog observability channel (As1012
+// Task 7). Written by `startOrResetWatchdog` on every call (initial arm and
+// each subsequent reset) and read by both `teardown()` and the setTimeout
+// expiry closure so `elapsedSinceArmMs` can be computed after the timer has
+// released. On a `reset` call the elapsed value is captured BEFORE `armedAt`
+// is overwritten so the emitted event carries the duration of the *previous*
+// armed interval (see item 4's ordering invariant).
+let armedAt = null;
+
+// Module-scope snapshot of the sentinel counts observed at the most recent
+// deferral, for the `WATCHDOG` observability channel (As1012 Task 13). Written
+// in the SIGUSR1 handler's defer branch BEFORE `startOrResetWatchdog()` so the
+// counts are in place if the watchdog subsequently fires; read from
+// `teardown()` when `reason === TEARDOWN_REASONS.WATCHDOG`. Lifted for the
+// same reason as `armedAt`: the SIGUSR1 handler's `bgCount` / `cronCount`
+// locals are block-scoped and out of scope inside the `setTimeout` closure
+// that eventually calls `teardown(TEARDOWN_REASONS.WATCHDOG)`. Overwritten on
+// every deferral, so it always holds the LAST deferral's counts — that is the
+// documented last-seen-snapshot semantic (design.md §Data on the wire).
+let lastDeferredCounts = null;
+
 // Forward declarations (assigned after ptyProcess is created)
 let ptyProcess = null;
 
+// --- Tagged event channel (As1012 Task 7) ---
+// Structured lifecycle events on stderr: `[caw:pty:event] <json>\n`. Consumed
+// downstream by JobRunner's `createStderrSplitter` (Task 6), which routes
+// matching lines through the operational tier (`caw:ops:pty:info`). Prose
+// stderr (every `log()` line above) is untouched — the splitter forwards
+// non-tagged bytes byte-identically to today's `stderrChunks` collector.
+// EPIPE is swallowed so a closed downstream pipe cannot crash the wrapper
+// (stderr is diagnostic; the tagged channel is best-effort).
+const EVENT_TAG = '[caw:pty:event] ';
+const emitEvent = (type, fields = {}) => {
+    const payload = {
+        type,
+        wrapperPid: process.pid,
+        claudePid: ptyProcess?.pid,
+        agentSessionId: effectiveSessionId,
+        ...fields
+    };
+    try {
+        process.stderr.write(`${EVENT_TAG}${JSON.stringify(payload)}\n`);
+    } catch (err) {
+        if (err && err.code !== 'EPIPE') throw err;
+    }
+};
+
 const startOrResetWatchdog = () => {
-    if (watchdogTimer) clearTimeout(watchdogTimer);
-    log(`Watchdog ${watchdogTimer ? 'reset' : 'armed'}: ${WATCHDOG_TIMEOUT_MS / 60000}m timeout`);
+    const wasArmed = watchdogTimer !== null;
+    if (wasArmed) clearTimeout(watchdogTimer);
+    log(`Watchdog ${wasArmed ? 'reset' : 'armed'}: ${WATCHDOG_TIMEOUT_MS / 60000}m timeout`);
+    // Ordering invariant (item 4): capture elapsedSinceArmMs BEFORE `armedAt`
+    // is overwritten so the `reset` event carries the duration of the previous
+    // armed interval. `armed` (first call) carries no elapsed value.
+    if (wasArmed) {
+        emitEvent('reset', { elapsedSinceArmMs: Date.now() - armedAt });
+    } else {
+        emitEvent('armed');
+    }
+    armedAt = Date.now();
     watchdogTimer = setTimeout(() => {
         log(`Watchdog expired: ${WATCHDOG_TIMEOUT_MS / 60000}m elapsed since last deferred Stop, force-killing claude`);
-        teardown('watchdog');
+        emitEvent('expired', { elapsedSinceArmMs: Date.now() - armedAt });
+        teardown(TEARDOWN_REASONS.WATCHDOG);
     }, WATCHDOG_TIMEOUT_MS);
 };
 
-const teardown = (reason = 'stop') => {
+const teardown = (reason = TEARDOWN_REASONS.STOP) => {
     if (teardownInitiated) return;
     teardownInitiated = true;
     endedAt = Date.now();
     teardownReason = reason;
     log(`Teardown (reason=${reason}): killing claude`);
     log(`Duration: ${startedAt !== null ? (endedAt - startedAt) : 'unknown'} ms (startedAt=${startedAt}, endedAt=${endedAt})`);
+    // As1012 Task 13 — `pendingBgCount` / `pendingCronCount` ride only on the
+    // `reason=watchdog` teardown, using the same conditional-spread idiom as
+    // `elapsedSinceArmMs`. No null-guard is needed: `startOrResetWatchdog()`
+    // has exactly one call site (the defer branch below), and that branch
+    // always writes `lastDeferredCounts` BEFORE arming the timer — so a
+    // `reason=watchdog` teardown is only reachable after at least one
+    // deferral, and at least one of the two counts is guaranteed ≥ 1.
+    emitEvent('teardown', {
+        reason,
+        ...(armedAt !== null ? { elapsedSinceArmMs: Date.now() - armedAt } : {}),
+        ...(reason === TEARDOWN_REASONS.WATCHDOG && lastDeferredCounts !== null
+            ? { pendingBgCount: lastDeferredCounts.bgCount, pendingCronCount: lastDeferredCounts.cronCount }
+            : {})
+    });
     if (watchdogTimer) { clearTimeout(watchdogTimer); watchdogTimer = null; }
     stopPolling();
     if (ptyProcess) ptyProcess.kill('SIGINT');
@@ -419,12 +490,12 @@ process.on('SIGUSR1', () => {
     const payload = readSentinelPayload(sentinelPath);
     if (!payload) {
         log(`Warning: failed to read sentinel, proceeding with teardown`);
-        teardown();
+        teardown(TEARDOWN_REASONS.SENTINEL_UNREADABLE);
         return;
     }
     if (payload.error) {
         log(`StopFailure received: error=${payload.error}`);
-        teardown();
+        teardown(TEARDOWN_REASONS.STOP_FAILURE);
         return;
     }
     const bgCount = Array.isArray(payload.background_tasks) ? payload.background_tasks.length : 0;
@@ -432,10 +503,14 @@ process.on('SIGUSR1', () => {
     log(`Sentinel: background_tasks=${bgCount}, session_crons=${cronCount}`);
     if (hasPendingWork(payload)) {
         log('Deferring teardown: pending background work');
+        // As1012 Task 13 — capture BEFORE arming so the counts are already in
+        // place if the timer subsequently fires. Members named to match the
+        // locals above; the lift is a zero-translation copy.
+        lastDeferredCounts = { bgCount, cronCount };
         startOrResetWatchdog();
         return;
     }
-    teardown();
+    teardown(TEARDOWN_REASONS.STOP);
 });
 
 writeFileSync(wrapperPidPath, String(process.pid));
@@ -740,6 +815,10 @@ ptyProcess.onExit(({ exitCode }) => {
     // teardown() never ran on this path (Stop hook did not fire), so `endedAt` is
     // unset and `durationMs` is deliberately omitted from the emission — the
     // service appends the pre-existing `'0'` duration sentinel (Decision D3).
+    // As1063 Task 1 item 9: track whether this branch was the one that set
+    // failureEmitted so the onExit-branch diagnostic below can distinguish
+    // `matched-prompt-too-long` from `emitted-failure` (both set the flag).
+    let matchedPromptTooLong = false;
     if (!resultEmitted && !failureEmitted && ttyBuffer.includes('Prompt is too long')) {
         log('Prompt-too-long: omitting durationMs (teardown() never ran, endedAt unset)');
         process.stdout.write(JSON.stringify({
@@ -748,11 +827,30 @@ ptyProcess.onExit(({ exitCode }) => {
             message: 'Prompt is too long'
         }) + '\n');
         failureEmitted = true;
+        matchedPromptTooLong = true;
     }
 
     // Clean up temp directory
     try { rmSync(tempDir, { recursive: true, force: true }); } catch {}
 
+    // As1063 Task 1 item 9: emit which onExit branch was taken along with
+    // the wrapper exit reason (teardownReason) and a bounded ttyBuffer tail.
+    // Bounded per lifecycle event by the `exited` guard at the top of onExit
+    // (no unbounded per-turn accumulation). Written to stderr via log() so
+    // JobRunner's stderr splitter captures it into the outputTail without
+    // affecting the wrapper's stdout emission contract.
+    const branchTaken = resultEmitted
+        ? 'emitted-result'
+        : matchedPromptTooLong
+            ? 'matched-prompt-too-long'
+            : failureEmitted
+                ? 'emitted-failure'
+                : 'no-emission';
+    // Cap the tty tail to 256 chars and escape newlines so the line stays on
+    // one log entry regardless of what claude last wrote.
+    const ttyTail = ttyBuffer.slice(-256).replace(/\n/g, '\\n').replace(/\r/g, '\\r');
+    log(`onExit-branch: ${branchTaken} exitCode=${exitCode} teardownReason=${teardownReason ?? 'unset'} ttyTail="${ttyTail}"`);
+
     log(`Claude exited (code: ${exitCode})`);
 
     if (failureEmitted) {

package/dist/teardownReasons.mjs

@@ -0,0 +1,100 @@
+/**
+ * teardownReasons.mjs — closed enum for the pty-wrapper's `teardown()` reason,
+ * shared between the observability path (As1012) and the classification path
+ * (As1033).
+ *
+ * Authored under As1012 as part of the S2 workshop's Q4 outcome — imported by
+ * `pty-wrapper.mjs` (emit site) and by `JobRunner.js`'s watchdog gate + the
+ * stream-handler metadata plumbing on the As1033 side.
+ *
+ * Non-forced-path contract (joint As1012 × As1033)
+ * ------------------------------------------------
+ *
+ *   1. `teardown()` runs on every graceful exit path. Four call sites; four
+ *      enum values: `WATCHDOG` (expiry), `SENTINEL_UNREADABLE`, `STOP_FAILURE`,
+ *      `STOP` (normal completion via the Stop hook). Normal completion is NOT a
+ *      no-teardown case — it is `teardown('stop')`. **`STOP` is a real value.**
+ *
+ *   2. `emitResult` always writes `metadata.teardownReason` when it runs, with
+ *      a value from this enum. The field is NOT optional on the graceful path —
+ *      consumers read it directly without an `undefined` check.
+ *
+ *   3. **`metadata.teardownReason` is omitted (not `null`) whenever the pty
+ *      child exits without `teardown()` firing.** Absence is the signal — the
+ *      enum does not carry a *no-teardown* value. Four non-teardown exit
+ *      paths, listed exhaustively so a future debugger does not misread the
+ *      gap:
+ *
+ *        a. **Child self-exit** — `claude` returns of its own accord before
+ *           the Stop hook fires (rare; e.g. an unhandled internal error).
+ *           `onExit` runs with a non-zero code; `emitResult` runs; the field
+ *           is omitted because `teardown()` never called `emitResult` with a
+ *           reason. Classifier reads a distinct exit class from `exitCode`.
+ *        b. **External signal to the wrapper** (SIGTERM/SIGINT delivered
+ *           externally, not from the wrapper's own deadline path). Wrapper
+ *           may or may not manage to emit anything depending on the signal
+ *           and its handler set at the moment.
+ *        c. **Prompt-too-long emission** — the wrapper's stdout emits its
+ *           result branch directly without routing through `teardown()`
+ *           (`pty-wrapper.mjs:733` and its neighbours; see
+ *           Easy-to-Miss #5 in As1012's design). Field omitted.
+ *        d. **As1006's F2 SIGKILL** — the wrapper-self-terminate path with a
+ *           process-group SIGKILL of the PTY child under the wrapper's own
+ *           deadline. SIGKILL is unmaskable; the wrapper is dead before it
+ *           can emit; the entire `type:"result"` JSONL line is absent.
+ *
+ *      **`As1006's F2 SIGKILL path is an OUT-OF-ENUM EXIT PATH`, as are the
+ *      other three above.** Do not read this enum as enumerating every wrapper
+ *      exit. An F2-killed job leaves NO wrapper-side breadcrumb via this
+ *      enum — that gap is by design, not by omission. As1033's dissent from
+ *      workshop round 3 is honoured by this note.
+ *
+ *      The classifier's contract for any of the four non-teardown cases:
+ *      `turnSignals.teardownReason === undefined` is a **distinct exit
+ *      class** — spawn failure / SIGKILL / crash / prompt-too-long / external
+ *      signal — not a teardown. Classification gate ordering (As1033):
+ *      `WATCHDOG` gate first (positive match on `reason === WATCHDOG`); other
+ *      reasons fall through unchanged; missing reason routes to the
+ *      spawn-failure / no-result branch.
+ *
+ *   4. Lifecycle events (`armed`, `reset`, `expired`, `teardown`) share the
+ *      enum only via the `teardown` event's `reason` field. `armed` / `reset`
+ *      / `expired` observability events carry `type`, not `reason` — no enum
+ *      value on those.
+ *
+ *   5. **The enum is closed at story-close.** Adding a value requires a
+ *      coordinated change to both this module AND the classification gate in
+ *      `JobRunner.js`. This is a maintenance rule, not a note — treat the set
+ *      below as fixed.
+ *
+ * Consumers
+ * ---------
+ *
+ *   - `pty-wrapper.mjs` — emit sites: `teardown(reason)` calls at the watchdog
+ *     expiry closure (`WATCHDOG`), sentinel-read failure (`SENTINEL_UNREADABLE`),
+ *     StopFailure hook (`STOP_FAILURE`), clean no-pending-work path (`STOP`).
+ *   - `pty-emit-result.mjs` — writes `metadata.teardownReason` on the result
+ *     JSONL line (As1033).
+ *   - `claude-session.js` — `getMetadata()` propagates the value into
+ *     `turnSignals.teardownReason` (As1033).
+ *   - `JobRunner.js` — watchdog classification gate reads
+ *     `turnSignals.teardownReason` and compares against `TEARDOWN_REASONS.WATCHDOG`
+ *     (As1033).
+ *   - `Log.js` observability — the `WATCHDOG event=teardown reason=<value>`
+ *     line format (As1012) references the enum values verbatim.
+ *
+ * Packaging
+ * ---------
+ *
+ * Copied verbatim into `dist/` by `server/package.json`'s `build:all` script,
+ * alongside `pty-wrapper.mjs`, `pty-emit-result.mjs`, and
+ * `pty-deferred-teardown.mjs`. Not bundled — the wrapper is executed as a
+ * standalone node process and imports this module by relative path.
+ */
+
+export const TEARDOWN_REASONS = Object.freeze({
+    WATCHDOG:              'watchdog',
+    STOP:                  'stop',
+    SENTINEL_UNREADABLE:   'sentinel-unreadable',
+    STOP_FAILURE:          'stop-failure',
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kelceyp/caw-server",
-  "version": "1.0.279",
+  "version": "1.0.280",
   "description": "CAW server - REST API, MCP server and core features",
   "type": "module",
   "main": "dist/main.js",
@@ -19,7 +19,7 @@
     "build": "bash scripts/bun-build.sh",
     "build:client": "cd ../client && bun run build",
     "build:cli": "cd ../cli && bun run build && cp dist/cli.js ../server/dist/",
-    "build:all": "bun run clean && bun run generate:applyPojo && bun run build:client && bun run build && cp src/core/services/embedding-worker.js dist/embedding-worker.js && cp src/core/agents/pty-wrapper.mjs dist/pty-wrapper.mjs && cp src/core/agents/pty-deferred-teardown.mjs dist/pty-deferred-teardown.mjs && cp src/core/agents/pty-emit-result.mjs dist/pty-emit-result.mjs && bun run build:cli",
+    "build:all": "bun run clean && bun run generate:applyPojo && bun run build:client && bun run build && cp src/core/services/embedding-worker.js dist/embedding-worker.js && cp src/core/agents/pty-wrapper.mjs dist/pty-wrapper.mjs && cp src/core/agents/pty-deferred-teardown.mjs dist/pty-deferred-teardown.mjs && cp src/core/agents/pty-emit-result.mjs dist/pty-emit-result.mjs && cp src/core/agents/teardownReasons.mjs dist/teardownReasons.mjs && bun run build:cli",
     "dev": "STATIC_PATH=../client/src bun --define __BUILTIN_POSTHOG_KEY__=null run src/main.js",
     "start": "node dist/main.js",
     "test": "bun test --define __BUILTIN_POSTHOG_KEY__=null test/unit test/integration test/component test/cli test/investigation && (cd ../client && bun test)",