create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/mainloop-observability.js

@@ -282,6 +282,19 @@ function createMainloopObservability(deps) {
       : _activeSyncSegment;
     const threshold = _attributedOp ? _lagThreshold : _unknownLagThreshold;
     const name = _attributedOp ? _attributedOp.name : '(unknown)';
+    // WP-23 (D-33): the watchdog worker's own measured suspension, read from shared memory.
+    // It is the only in-process observer that is frozen by a suspension but not by a
+    // main-thread block, so whenever it has ticked we decide sleep-vs-freeze from it and
+    // never from CPU or timed work. Absent (CTM_MAIN_WATCHDOG=0, or no tick yet) the legacy
+    // rule below applies unchanged.
+    //
+    // WP-2 (D-6): read it BEFORE the evidence build so the suspension is available as an
+    // input to the classifier — the ladder labels a mostly-suspended block `process-suspended`
+    // instead of the inferential `off-cpu-unattributed`/`main_descheduled`.
+    const _watchdog = _readWatchdogSuspension(lagIntervalMs);
+    const _suspendedMs = _watchdog
+      ? Math.max(0, Math.min(_watchdog.suspendedTo, now) - Math.max(_watchdog.suspendedFrom, now - blockedMs))
+      : 0;
     let _blockEvidence = null;
     try {
       if (blockedMs >= 200) {
@@ -293,17 +306,12 @@ function createMainloopObservability(deps) {
             mainThreadCpuDeltaMs: _mainThreadCpuDeltaMs,
             processCpuDeltaMs: _processCpuDeltaMs,
             activeSyncEvidence: _activeSyncEvidence,
+            suspendedMs: _suspendedMs,
           },
         );
       }
       _lagEvidenceBaseline = currentEvidence;
     } catch { /* expected: evidence collection must never break the detector */ }
-    // WP-23 (D-33): the watchdog worker's own measured suspension, read from shared memory.
-    // It is the only in-process observer that is frozen by a suspension but not by a
-    // main-thread block, so whenever it has ticked we decide sleep-vs-freeze from it and
-    // never from CPU or timed work. Absent (CTM_MAIN_WATCHDOG=0, or no tick yet) the legacy
-    // rule below applies unchanged.
-    const _watchdog = _readWatchdogSuspension(lagIntervalMs);
     const _suspectedSleep = perfTracker.isSuspectedSleepBlock(blockedMs, {
       idleDuringGap: _idleDuringGap,
       recentSyncEvidence: _recentSyncEvidence,
@@ -313,9 +321,6 @@ function createMainloopObservability(deps) {
       watchdog: _watchdog || undefined,
       gapStartMs: now - blockedMs,
     });
-    const _suspendedMs = _watchdog
-      ? Math.max(0, Math.min(_watchdog.suspendedTo, now) - Math.max(_watchdog.suspendedFrom, now - blockedMs))
-      : 0;
     let lagSample = null;
     try {
       lagSample = perfTracker.recordEventLoopLagSample(name, delta, {

package/template/coding-task-manager/lib/media-draft-owner.js

@@ -498,7 +498,11 @@ function createMediaDraftOwner(deps) {
     const next = stampDraftMutation(item, {
       ...item,
       state: input.state,
-      submittedInputId: String(input.submittedInputId || item.submittedInputId || ''),
+      // WP-2 (composer-image-chip-stuck-submission-recovery-2026-10-30.html §6): `''` still
+      // means "keep the current id" (unchanged default); the explicit clearSubmittedInputId
+      // flag is the ONLY way to drop a submittedInputId, and it exists for the stale-draft
+      // recovery that returns a frozen draft to attached.
+      submittedInputId: input.clearSubmittedInputId === true ? '' : String(input.submittedInputId || item.submittedInputId || ''),
       deliveryAttemptId: String(input.attemptId || item.deliveryAttemptId || ''),
       stateReason: String(input.reason || (input.state === 'ready' ? '' : item.stateReason || '')),
       updatedAt: new Date().toISOString(),
@@ -694,12 +698,21 @@ function createMediaDraftOwner(deps) {
       && Date.parse(item.updatedAt || item.createdAt || 0) < before);
     const recovered = [];
     for (const candidate of candidates) {
+      // R-3 (WP-2): a `frozen` draft whose delivery was CONFIRMED was interrupted before Enter,
+      // so the image is still in the agent composer -- `failed` would lie about it (and would
+      // re-deliver a token that is already there). It returns to `attached` with the stale
+      // submittedInputId cleared, so the next send freezes it again. A `delivery_pending`
+      // draft, or a `frozen` one the provider never acknowledged, keeps today's failure.
+      const interruptedBeforeEnter = candidate.state === 'frozen' && Boolean(candidate.deliveryConfirmedAt);
       const item = await setDraftState({
         sessionId,
         attachmentId: candidate.attachmentId,
         expectedRevision: candidate.revision,
-        state: 'failed',
-        reason: 'Image delivery was interrupted. Retry or remove it.',
+        state: interruptedBeforeEnter ? 'attached' : 'failed',
+        clearSubmittedInputId: interruptedBeforeEnter,
+        reason: interruptedBeforeEnter
+          ? 'Send was interrupted before Enter; image is still in the composer.'
+          : 'Image delivery was interrupted. Retry or remove it.',
       });
       recovered.push(item);
     }

package/template/coding-task-manager/lib/memory-join.js

@@ -0,0 +1,337 @@
+'use strict';
+
+// WP-4b (docs/shared-memory-every-machine-2026-10-31.html §5 D-6, §6.7): the HUB side of the join
+// state machine — a member whose vector does not name the hub origin (WP-3's 'join_required') takes
+// the hub's consistent snapshot in 4 MiB sha256-verified chunks and imports it, instead of idling.
+//
+//   advanceMemberJoin(member, ctx) -> { state, detail }
+//   state ∈ 'seed_pending' | 'join_snapshot' | 'join_transfer' | 'join_import' | 'join_done'
+//          | 'join_failed'
+//
+// D-6: (1) the hub takes a consistent snapshot of its own brain, (2) reads the snapshot's own
+// vector, (3) ships it in 4 MiB chunks, (4) the member imports the replicated tables, (5) the
+// member's vector becomes the snapshot's, so hub changes written during the transfer arrive through
+// the ordinary D-3 feed. P-1: the vector travels FROM THE SNAPSHOT FILE, never the live DB.
+//
+// The per-member progress Map is MODULE state by design, exactly like memory-sync.js's backoff
+// bookkeeping: a hub restart loses it and the transfer RESUMES from the member's own `received[]`
+// (P-3, checkpoints on both sides) — never from the hub's memory. A member restart mid-import is
+// Wall-E's job (WP-4a): this side only polls /join/status until it reads 'done' or 'failed'.
+
+const fsp = require('node:fs/promises');
+
+// §6.7: the transfer chunk size. Shared with lib/api/memory-sync-routes.js — the sender writes at
+// offset index*CHUNK_BYTES and the receiver reads at the same offset, so the two must never drift.
+const CHUNK_BYTES = 4 * 1024 * 1024;
+const DEFAULT_CHUNKS_PER_TICK = 32;
+// §6.7/D-6: a failed join is retried from step 1 after this long, never on every tick.
+const FAILED_RETRY_MS = 60 * 60 * 1000;
+const JOIN_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
+const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
+
+// The hub's LOCAL Wall-E join route (loopback + CTM token, like LOCAL_PATHS in memory-sync.js).
+const SNAPSHOT_PATH = '/api/walle/sync/v2/join/snapshot';
+// The member's CTM front routes (D-12): the hub dials the member's CTM, which forwards to its Wall-E.
+const MEMBER_JOIN_PATHS = Object.freeze({
+  chunk: '/api/sync/brain/v2/join/chunk',
+  transfer: '/api/sync/brain/v2/join/transfer',
+  import: '/api/sync/brain/v2/join/import',
+  status: '/api/sync/brain/v2/join/status',
+});
+
+// instanceId -> { joinId, snapshot, phase, memberPath, imported, restarted, error, nextAttemptAt }.
+const _progress = new Map();
+
+function _string(value, max = 200) {
+  if (typeof value === 'string') return value.trim().slice(0, max);
+  return value == null ? '' : String(value).trim().slice(0, max);
+}
+
+function _nowOf(ctx = {}) {
+  return typeof ctx.now === 'function' ? Number(ctx.now()) || Date.now() : Date.now();
+}
+
+function _chunkCount(bytes) {
+  const size = Math.max(0, Number(bytes) || 0);
+  return Math.max(1, Math.ceil(size / CHUNK_BYTES));
+}
+
+function _joinUrl(base, pathname) {
+  return `${String(base).replace(/\/+$/, '')}${pathname}`;
+}
+
+// One local Wall-E call. requestJson resolves { status, json } for any answer and rejects only on a
+// transport fault — the caller maps both (memory-sync.js's _walleRequest shape).
+async function _walleRequest(walle, pathname, { method = 'GET', body = null } = {}) {
+  try {
+    const res = await walle.requestJson(pathname, body == null ? { method } : { method, body });
+    return { status: Number(res && res.status) || 0, json: (res && res.json) || null };
+  } catch (err) {
+    return { status: 0, json: null, error: err?.message || String(err) };
+  }
+}
+
+// One member request, with the AbortController + unref'd timer shape memory-sync.js uses so a hung
+// member can never hold the tick open past the timeout. Never throws: a transport fault is reported
+// as `{ status: 0 }` and mapped by the caller.
+async function _memberRequest(fetchImpl, url, pathname, { method = 'GET', body = null, headers = {} } = {}) {
+  const fetchFn = fetchImpl || global.fetch;
+  if (typeof fetchFn !== 'function') return { status: 0, json: null, error: 'fetch_unavailable' };
+  const controller = typeof AbortController !== 'undefined' ? new AbortController() : null;
+  const timer = controller ? setTimeout(() => controller.abort(), DEFAULT_REQUEST_TIMEOUT_MS) : null;
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  try {
+    const payload = body == null ? null : JSON.stringify(body);
+    const res = await fetchFn(_joinUrl(url, pathname), {
+      method,
+      headers: {
+        ...headers,
+        ...(payload ? { 'Content-Type': 'application/json' } : {}),
+        Accept: 'application/json',
+      },
+      ...(payload ? { body: payload } : {}),
+      signal: controller ? controller.signal : undefined,
+    });
+    let json = null;
+    try { json = await res.json(); } catch { json = null; /* expected: a non-JSON body is degraded, mapped by the caller */ }
+    return { status: Number(res && res.status) || 0, json };
+  } catch (err) {
+    return { status: 0, json: null, error: err?.message || String(err) };
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+// The production chunk reader (the `readChunk` seam): a bounded read at an absolute offset. A short
+// read is a hard error — a truncated chunk would assemble to a wrong sha256 on the member, not to a
+// short file.
+async function _readChunkDefault(filePath, offset, length) {
+  const handle = await fsp.open(filePath, 'r');
+  try {
+    const buffer = Buffer.alloc(length);
+    const { bytesRead } = await handle.read(buffer, 0, length, offset);
+    if (bytesRead !== length) throw new Error(`short_read_${bytesRead}_of_${length}`);
+    return buffer;
+  } finally {
+    try { await handle.close(); } catch { /* expected: a close fault must not mask the read verdict */ }
+  }
+}
+
+// R-5 (§6.6): `freshSnapshot` rides on the failed record so the retry after the backoff asks the hub's
+// own Wall-E for a NEW snapshot instead of reusing the 6 h window's file the member already refused.
+function _fail(instanceId, nowMs, error, { freshSnapshot = false } = {}) {
+  _progress.set(instanceId, { phase: 'failed', error, nextAttemptAt: nowMs + FAILED_RETRY_MS, freshSnapshot });
+  return { state: 'join_failed', detail: { error } };
+}
+
+/**
+ * Advance one member's join by one tick. Called by memory-sync.js's `syncMemberBrainOnce` exactly
+ * where it used to answer the idle 'join_required' state.
+ *
+ * @param {{instanceId: string, urls?: string[], url?: string}} member
+ * @param {object} ctx { localInstanceId, walle, fetchImpl, headers, memberUrl, now, chunksPerTick, readChunk }
+ */
+async function advanceMemberJoin(member = {}, ctx = {}) {
+  const instanceId = _string(member.instanceId);
+  const walle = ctx.walle;
+  const fetchImpl = ctx.fetchImpl || global.fetch;
+  const headers = ctx.headers || {};
+  const memberUrl = _string(ctx.memberUrl, 1000);
+  const localInstanceId = _string(ctx.localInstanceId);
+  const chunksPerTick = Math.max(1, Number(ctx.chunksPerTick) || DEFAULT_CHUNKS_PER_TICK);
+  const readChunk = typeof ctx.readChunk === 'function' ? ctx.readChunk : _readChunkDefault;
+  const nowMs = _nowOf(ctx);
+
+  if (!instanceId) return { state: 'join_failed', detail: { error: 'member_instance_required' } };
+  if (!memberUrl) return { state: 'join_failed', detail: { error: 'member_has_no_url' } };
+  if (!walle || typeof walle.requestJson !== 'function') return { state: 'join_failed', detail: { error: 'walle_client_missing' } };
+
+  let p = _progress.get(instanceId) || null;
+  let fresh = false;
+  // A failed join is retried from step 1 only after the 1 h backoff (D-6/§6.7).
+  if (p && p.phase === 'failed') {
+    if (nowMs < p.nextAttemptAt) return { state: 'join_failed', detail: { error: p.error, retryAt: p.nextAttemptAt } };
+    // R-5: the member refused the previous snapshot — remember to ask for a fresh one (§6.6).
+    fresh = p.freshSnapshot === true;
+    _progress.delete(instanceId);
+    p = null;
+  }
+
+  // 1. snapshot — once per joinId. 409 clock_seed_pending (D-9: the hub's own clock walk is not
+  // done) parks the member for the next tick with no partial state. A retry after a refused import
+  // carries {fresh:true} so createJoinSnapshot skips the 6 h reuse instead of re-offering the same file.
+  if (!p) {
+    const snap = await _walleRequest(walle, SNAPSHOT_PATH, { method: 'POST', body: fresh ? { fresh: true } : {} });
+    if (snap.status === 409) return { state: 'seed_pending', detail: { error: 'clock_seed_pending' } };
+    if (snap.status === 0 || !snap.json || snap.status < 200 || snap.status >= 300) {
+      return { state: 'join_snapshot', detail: { error: snap.error || `snapshot_http_${snap.status}` } };
+    }
+    const joinId = _string(snap.json.joinId, 200);
+    const snapshotPath = _string(snap.json.path, 1000);
+    const bytes = Number(snap.json.bytes);
+    if (!JOIN_ID_RE.test(joinId) || !snapshotPath || !Number.isFinite(bytes) || bytes < 0) {
+      return { state: 'join_snapshot', detail: { error: 'snapshot_bad_reply' } };
+    }
+    p = {
+      joinId,
+      snapshot: { path: snapshotPath, bytes, sha256: _string(snap.json.sha256, 64), vector: (snap.json.vector && typeof snap.json.vector === 'object') ? snap.json.vector : {} },
+      phase: 'transfer',
+      memberPath: '',
+      imported: false,
+      restarted: false,
+    };
+    _progress.set(instanceId, p);
+  }
+
+  // 2. transfer — ask the member what it already holds (P-3: resume after a hub restart), then send
+  // up to `chunksPerTick` missing chunks in index order.
+  if (p.phase === 'transfer') {
+    const transfer = await _memberRequest(fetchImpl, memberUrl, `${MEMBER_JOIN_PATHS.transfer}?joinId=${encodeURIComponent(p.joinId)}`, { method: 'GET', headers });
+    if (transfer.status === 0) return { state: 'join_transfer', detail: { error: transfer.error || 'member_unreachable' } };
+    if (transfer.status < 200 || transfer.status >= 300 || !transfer.json) {
+      return { state: 'join_transfer', detail: { error: `transfer_http_${transfer.status}` } };
+    }
+    const declared = Number(transfer.json.total);
+    const total = Number.isInteger(declared) && declared > 0 ? declared : _chunkCount(p.snapshot.bytes);
+    if (transfer.json.complete === true && _string(transfer.json.path, 1000)) {
+      p.memberPath = _string(transfer.json.path, 1000);
+      p.phase = 'import';
+    } else {
+      const received = new Set((Array.isArray(transfer.json.received) ? transfer.json.received : []).map(Number).filter((n) => Number.isInteger(n) && n >= 0));
+      let sent = 0;
+      for (let index = 0; index < total && sent < chunksPerTick; index += 1) {
+        if (received.has(index)) continue;
+        const offset = index * CHUNK_BYTES;
+        const length = Math.min(CHUNK_BYTES, p.snapshot.bytes - offset);
+        if (length <= 0) continue;
+        let data;
+        try {
+          data = await readChunk(p.snapshot.path, offset, length, ctx);
+        } catch (err) {
+          return { state: 'join_transfer', detail: { error: `snapshot_read_failed: ${err?.message || String(err)}` } };
+        }
+        if (!Buffer.isBuffer(data) || data.length !== length) {
+          return { state: 'join_transfer', detail: { error: `snapshot_chunk_short_${index}` } };
+        }
+        const res = await _memberRequest(fetchImpl, memberUrl, MEMBER_JOIN_PATHS.chunk, {
+          method: 'POST',
+          headers,
+          body: {
+            joinId: p.joinId,
+            index,
+            total,
+            chunkBytes: length,
+            fileBytes: p.snapshot.bytes,
+            fileSha256: p.snapshot.sha256,
+            dataBase64: data.toString('base64'),
+          },
+        });
+        sent += 1;
+        if (res.status === 507 || (res.json && res.json.error === 'insufficient_storage')) {
+          return _fail(instanceId, nowMs, 'insufficient_storage');
+        }
+        if (res.status === 422 || (res.json && res.json.error === 'sha256_mismatch')) {
+          // The member deleted its .part + sidecar; restart the transfer from 0 exactly once.
+          if (!p.restarted) {
+            p.restarted = true;
+            p.memberPath = '';
+            return { state: 'join_transfer', detail: { error: 'sha256_mismatch', restarted: true } };
+          }
+          return _fail(instanceId, nowMs, 'sha256_mismatch');
+        }
+        if (res.status === 0) return { state: 'join_transfer', detail: { error: res.error || 'member_unreachable' } };
+        if (res.status < 200 || res.status >= 300 || !res.json) {
+          return { state: 'join_transfer', detail: { error: `chunk_http_${res.status}` } };
+        }
+        if (res.json.complete === true && _string(res.json.path, 1000)) {
+          p.memberPath = _string(res.json.path, 1000);
+          p.phase = 'import';
+          break;
+        }
+      }
+      if (p.phase !== 'import') return { state: 'join_transfer', detail: { sent, total } };
+    }
+  }
+
+  // 3. import + 4. poll — the member hands the file to its own brain owner (WP-4a); this side only
+  // drives and observes. The import POST is issued at most once per joinId.
+  if (p.phase === 'import') {
+    if (!p.imported) {
+      if (!p.memberPath) {
+        const again = await _memberRequest(fetchImpl, memberUrl, `${MEMBER_JOIN_PATHS.transfer}?joinId=${encodeURIComponent(p.joinId)}`, { method: 'GET', headers });
+        const path = _string(again.json && again.json.path, 1000);
+        if (again.status >= 200 && again.status < 300 && again.json && again.json.complete === true && path) {
+          p.memberPath = path;
+        } else {
+          p.phase = 'transfer';
+          return { state: 'join_transfer', detail: { error: 'member_path_missing' } };
+        }
+      }
+      const posted = await _memberRequest(fetchImpl, memberUrl, MEMBER_JOIN_PATHS.import, {
+        method: 'POST',
+        headers,
+        body: { joinId: p.joinId, path: p.memberPath, sha256: p.snapshot.sha256, homeInstanceId: localInstanceId },
+      });
+      if (posted.status === 0) return { state: 'join_import', detail: { error: posted.error || 'member_unreachable' } };
+      // 409 join_in_progress: a previous tick's import is already running — poll it, never re-post.
+      if (posted.status === 409) p.imported = true;
+      // R-9: a 4xx import reply is terminal for THIS attempt — fail with the status + the member's own
+      // error code instead of re-POSTing the same refused import on every round with no backoff.
+      else if (posted.status >= 400 && posted.status < 500) {
+        return _fail(instanceId, nowMs, `import_http_${posted.status}:${_string(posted.json && posted.json.error, 160)}`);
+      // 5xx / odd replies stay non-terminal — a member restart or a transient fault is retried next tick.
+      } else if (posted.status < 200 || posted.status >= 300 || !posted.json) {
+        return { state: 'join_import', detail: { error: `import_http_${posted.status}` } };
+      // R-4/R-5: a 2xx whose state is 'failed' means the member cannot resume THIS snapshot — ask for a
+      // fresh one on the retry instead of polling a file it has already refused.
+      } else if (_string(posted.json.state, 40) === 'failed') {
+        return _fail(instanceId, nowMs, 'member_join_not_resumable', { freshSnapshot: true });
+      } else p.imported = true;
+    }
+    const status = await _memberRequest(fetchImpl, memberUrl, MEMBER_JOIN_PATHS.status, { method: 'GET', headers });
+    if (status.status === 0) return { state: 'join_import', detail: { error: status.error || 'member_unreachable' } };
+    if (status.status < 200 || status.status >= 300 || !status.json) {
+      return { state: 'join_import', detail: { error: `status_http_${status.status}` } };
+    }
+    const memberState = _string(status.json.state, 40);
+    if (memberState === 'done') {
+      _progress.delete(instanceId);
+      return {
+        state: 'join_done',
+        detail: {
+          joinId: p.joinId,
+          memberPath: p.memberPath,
+          imported: Number(status.json.imported) || 0,
+          rekeyed: Number(status.json.rekeyed) || 0,
+          vector: p.snapshot.vector,
+        },
+      };
+    }
+    if (memberState === 'failed') return _fail(instanceId, nowMs, _string(status.json.error, 200) || 'join_import_failed');
+    return { state: 'join_import', detail: { joinId: p.joinId, memberState: memberState || 'none' } };
+  }
+
+  return { state: 'join_failed', detail: { error: 'join_phase_unknown' } };
+}
+
+/** Test seam: forget every member's join progress (module state survives a process). */
+function _resetMemberJoinStateForTests() {
+  _progress.clear();
+}
+
+/** Observability/test seam: the raw per-member progress record (never mutated by callers). */
+function memberJoinProgress(instanceId) {
+  const p = _progress.get(_string(instanceId));
+  return p ? { ...p } : null;
+}
+
+module.exports = {
+  advanceMemberJoin,
+  memberJoinProgress,
+  _resetMemberJoinStateForTests,
+  CHUNK_BYTES,
+  MEMBER_JOIN_PATHS,
+  SNAPSHOT_PATH,
+  FAILED_RETRY_MS,
+  JOIN_ID_RE,
+};