heyspex 0.1.0 → 0.2.1

package/vendor/heyspex-mcp/write-outbox.mjs

@@ -0,0 +1,512 @@
+/**
+ * Write outbox — "degrade, don't die" for vault WRITES in cloud reliance.
+ *
+ * Per [[heyspex-is-online-only-no-offline]] (2026-08-20 amendment): when the
+ * console is unreachable (status 0 / timeout) or answers 5xx, a spec write is
+ * queued durably and replayed on the next successful console call, with a
+ * truthful receipt ("Not written to the console yet; queued…") — never a lie
+ * that it landed, never a failed workflow.
+ *
+ * Modelled on capture-outbox.mjs (stash -> list -> replay -> remove, quarantine
+ * for what the console structurally refuses, vault-root path guard HS-AI1T).
+ *
+ * What is queued: spec create / update / ordered batch (promotions) and the
+ * append-only control writes (report_progress, record_wip, handoff_record).
+ * What is NEVER queued: claims and status flips — the Neon control plane is the
+ * mutual-exclusion / transition authority, so those keep failing loudly. 4xx
+ * and 401/403 are never queued either (they fail identically on replay, or need
+ * `hs login`).
+ *
+ * Layout: `<vaultRoot>/.heyspex/write-outbox/<ISO>__<NN>__<op>.json`.
+ *
+ * Replay safety:
+ *  - the STORED idempotency-key is replayed, never a fresh one ([[HS-2FQY]]);
+ *  - a stored `create` that failed with a timeout / 5xx first looks the spec up
+ *    BY NAME and skips the POST when it already landed (`landed_server_side`);
+ *  - a stored `update` is refused (quarantined, `cloud_newer`) when the cloud
+ *    copy changed after the pre-image the edit was based on.
+ */
+
+import {
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  existsSync,
+  renameSync,
+  rmSync,
+} from 'node:fs';
+import { basename, join, isAbsolute } from 'node:path';
+
+import { assertPathUnderRoot } from '../../scripts/lib/shared-path-helpers.mjs';
+
+export const DEFERRED_RECEIPT =
+  'Not written to the console yet; queued to .heyspex/write-outbox and replayed on the next successful console call.';
+
+/** Process-monotonic within-millisecond tiebreaker (see capture-outbox.mjs). */
+let stashSeq = 0;
+
+export function writeOutboxDir(vaultRoot) {
+  return join(vaultRoot, '.heyspex', 'write-outbox');
+}
+
+export function writeOutboxQuarantineDir(vaultRoot) {
+  return join(vaultRoot, '.heyspex', 'write-outbox', 'quarantine');
+}
+
+// ---------------------------------------------------------------------------
+// Default vault root — set once by the tool dispatcher so deep writers
+// (createSpecViaConsole & co.) that were never handed a vaultRoot can still
+// stash. With no root known, writers keep their old throw-loudly behaviour.
+// ---------------------------------------------------------------------------
+let defaultVaultRoot = null;
+
+export function setWriteOutboxRoot(vaultRoot) {
+  if (typeof vaultRoot === 'string' && vaultRoot.trim() && isAbsolute(vaultRoot)) {
+    defaultVaultRoot = vaultRoot;
+  }
+}
+
+export function getWriteOutboxRoot() {
+  return defaultVaultRoot;
+}
+
+/** Test helper. */
+export function resetWriteOutboxRoot() {
+  defaultVaultRoot = null;
+  drainOnceDone = false;
+}
+
+// ---------------------------------------------------------------------------
+// Classification
+// ---------------------------------------------------------------------------
+
+/**
+ * True for the OUTAGE class only: transport failure (status 0 / timeout / abort
+ * / network) or a 5xx answer. Sign-in problems, typed scope refusals, 401/403
+ * and every other 4xx are NOT outages — they never queue.
+ */
+export function isOutageClassResult(result) {
+  if (!result || result.ok) return false;
+  if (result.authProblem || result.refusal || result.urlMismatch) return false;
+  const status = typeof result.status === 'number' ? result.status : 0;
+  if (status === 401 || status === 403) return false;
+  if (status >= 400 && status < 500) return false;
+  if (status >= 500) return true;
+  // status 0 / missing: a genuine non-response.
+  return true;
+}
+
+// ---------------------------------------------------------------------------
+// Stash / list / remove / quarantine
+// ---------------------------------------------------------------------------
+
+/**
+ * Persist one write. `entry` carries
+ * `{ op, route, method, request_body, idempotency_key, acting_agent, session_id,
+ *    failure: {status, errorKind, message}, pre_image_updated? , steps? }`.
+ * Returns `{ outbox_id, path, entry }`, or `null` when no vault root is usable
+ * or the write failed (callers then fall back to their loud error).
+ */
+export function stashWrite(vaultRoot, entry) {
+  try {
+    const dir = writeOutboxDir(vaultRoot);
+    assertPathUnderRoot(vaultRoot, dir, 'write-outbox write');
+    mkdirSync(dir, { recursive: true });
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const seq = String(stashSeq++).padStart(6, '0');
+    const op = ['create', 'update', 'batch', 'control'].includes(entry.op) ? entry.op : 'control';
+    const outboxId = `${stamp}__${seq}__${op}`;
+    const record = {
+      op,
+      route: entry.route ?? null,
+      method: entry.method ?? null,
+      request_body: entry.request_body ?? null,
+      idempotency_key: entry.idempotency_key ?? null,
+      acting_agent: entry.acting_agent ?? null,
+      session_id: entry.session_id ?? null,
+      attempted_at: entry.attempted_at ?? new Date().toISOString(),
+      failure: entry.failure ?? null,
+      ...(entry.pre_image_updated !== undefined ? { pre_image_updated: entry.pre_image_updated } : {}),
+      ...(entry.steps ? { steps: entry.steps } : {}),
+      ...(entry.completed_steps ? { completed_steps: entry.completed_steps } : {}),
+      ...(entry.spec_id ? { spec_id: entry.spec_id } : {}),
+    };
+    const path = join(dir, `${outboxId}.json`);
+    writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`, 'utf8');
+    // A fresh entry re-arms the once-per-process first-success drain.
+    drainOnceDone = false;
+    return { outbox_id: outboxId, path, entry: record };
+  } catch {
+    return null;
+  }
+}
+
+/** Oldest-first list of parsable entries. Unreadable files are skipped. */
+export function listOutbox(vaultRoot) {
+  const dir = writeOutboxDir(vaultRoot);
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const d of readdirSync(dir, { withFileTypes: true })) {
+    if (!d.isFile() || !d.name.endsWith('.json')) continue;
+    try {
+      const entry = JSON.parse(readFileSync(join(dir, d.name), 'utf8'));
+      out.push({ ...entry, outbox_id: d.name.replace(/\.json$/, ''), path: join(dir, d.name) });
+    } catch {
+      /* skip unreadable */
+    }
+  }
+  return out.sort((a, b) => (a.outbox_id < b.outbox_id ? -1 : a.outbox_id > b.outbox_id ? 1 : 0));
+}
+
+export function outboxCount(vaultRoot) {
+  try {
+    return listOutbox(vaultRoot).length;
+  } catch {
+    return 0;
+  }
+}
+
+export function removeOutboxEntry(path) {
+  try {
+    rmSync(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Move an entry to quarantine/ with a `<name>.reason.json` sidecar. */
+export function quarantineOutboxEntry(vaultRoot, entryPath, metadata = {}) {
+  try {
+    const qdir = writeOutboxQuarantineDir(vaultRoot);
+    assertPathUnderRoot(vaultRoot, qdir, 'write-outbox write');
+    mkdirSync(qdir, { recursive: true });
+    const name = basename(entryPath);
+    renameSync(entryPath, join(qdir, name));
+    writeFileSync(
+      join(qdir, `${name}.reason.json`),
+      `${JSON.stringify(
+        {
+          reason: metadata.reason || 'console_refused',
+          status: metadata.status ?? null,
+          consoleMessage: metadata.consoleMessage ?? null,
+          quarantined_at: new Date().toISOString(),
+        },
+        null,
+        2,
+      )}\n`,
+      'utf8',
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function listQuarantinedOutbox(vaultRoot) {
+  const dir = writeOutboxQuarantineDir(vaultRoot);
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir, { withFileTypes: true })
+    .filter((d) => d.isFile() && d.name.endsWith('.json') && !d.name.endsWith('.reason.json'))
+    .map((d) => d.name)
+    .sort()
+    .map((f) => {
+      let reason = null;
+      try {
+        reason = JSON.parse(readFileSync(join(dir, `${f}.reason.json`), 'utf8'));
+      } catch {
+        /* sidecar may be absent */
+      }
+      return { file: f, path: join(dir, f), reason };
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Deferred response
+// ---------------------------------------------------------------------------
+
+/** The truthful tool response for a queued write. Never `isError`. */
+export function buildDeferredPayload(stashed, vaultRoot, extra = {}) {
+  return {
+    status: 'deferred',
+    outbox_id: stashed.outbox_id,
+    pending_sync: outboxCount(vaultRoot),
+    receipt: DEFERRED_RECEIPT,
+    ...extra,
+  };
+}
+
+/** MCP tool envelope for {@link buildDeferredPayload}. */
+export function deferredToolResponse(payload) {
+  return { content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }] };
+}
+
+/** Result descriptor for a failed consoleFetch, as stored in `failure`. */
+export function failureOf(result, fallbackMessage = null) {
+  return {
+    status: typeof result?.status === 'number' ? result.status : 0,
+    errorKind: result?.errorKind ?? null,
+    message: String(result?.error ?? fallbackMessage ?? '').slice(0, 500) || null,
+  };
+}
+
+/**
+ * Stash a single write described by a failed consoleFetch result, when (and only
+ * when) it is outage class and a vault root is known. Returns the stash record
+ * or null (caller then surfaces its original loud error).
+ */
+export function stashIfOutage(result, entry, vaultRoot = defaultVaultRoot) {
+  if (!vaultRoot || !isOutageClassResult(result)) return null;
+  return stashWrite(vaultRoot, { ...entry, failure: failureOf(result) });
+}
+
+// ---------------------------------------------------------------------------
+// Drain
+// ---------------------------------------------------------------------------
+
+let draining = false;
+let drainOnceDone = false;
+
+const sameName = (a, b) =>
+  typeof a === 'string' && typeof b === 'string' && a.trim().toLowerCase() === b.trim().toLowerCase();
+
+/** Find an already-landed spec by exact name (case-insensitive) within its type. */
+async function findSpecByName(consoleFetch, type, name) {
+  let after = null;
+  for (let i = 0; i < 20; i += 1) {
+    const params = new URLSearchParams();
+    if (type) params.set('type', type);
+    params.set('limit', '500');
+    if (after) params.set('after', after);
+    const res = await consoleFetch(`/api/specs?${params.toString()}`);
+    if (!res?.ok) return { error: res };
+    const rows = Array.isArray(res.body?.specs) ? res.body.specs : [];
+    const hit = rows.find((r) => sameName(r?.name, name) || sameName(r?.title, name));
+    if (hit) return { hit };
+    const next = typeof res.body?.next_cursor === 'string' && res.body.next_cursor ? res.body.next_cursor : null;
+    if (!next || next === after || rows.length === 0) break;
+    after = next;
+  }
+  return { hit: null };
+}
+
+function basenameOf(spec) {
+  const rel = spec?.relPath ?? spec?.path ?? null;
+  if (rel) return String(rel).split(/[\\/]/).pop().replace(/\.md$/i, '');
+  return spec?.basename ?? spec?.id ?? null;
+}
+
+const needsLandingCheck = (failure) =>
+  failure?.errorKind === 'timeout' || failure?.errorKind === 'abort' || (failure?.status ?? 0) >= 500;
+
+/**
+ * Replay one request. Returns
+ * `{ outcome: 'ok'|'skipped'|'quarantine'|'stop', result?, reason?, spec? }`.
+ * `outcome: 'stop'` means outage / sign-in: leave the entry (and everything
+ * after it) for the next successful call.
+ */
+async function replayRequest(consoleFetch, step, parentOp) {
+  if (parentOp === 'create' && needsLandingCheck(step.failure ?? null)) {
+    const found = await findSpecByName(consoleFetch, step.request_body?.type, step.request_body?.name);
+    if (found.error) {
+      return isOutageClassResult(found.error) || found.error.status === 401 || found.error.status === 403
+        ? { outcome: 'stop', result: found.error }
+        : { outcome: 'quarantine', reason: 'console_refused', result: found.error };
+    }
+    if (found.hit) return { outcome: 'skipped', landed_server_side: true, spec: found.hit };
+  }
+  if (parentOp === 'update' && step.pre_image_updated) {
+    const cur = await consoleFetch(step.route);
+    if (!cur?.ok) {
+      return isOutageClassResult(cur) || cur?.status === 401 || cur?.status === 403
+        ? { outcome: 'stop', result: cur }
+        : { outcome: 'quarantine', reason: 'console_refused', result: cur };
+    }
+    const cloudUpdated = cur.body?.spec?.updated ?? cur.body?.spec?.frontmatter?.updated ?? null;
+    const a = Date.parse(String(cloudUpdated));
+    const b = Date.parse(String(step.pre_image_updated));
+    if (Number.isFinite(a) && Number.isFinite(b) && a > b) {
+      return { outcome: 'quarantine', reason: 'cloud_newer', result: cur };
+    }
+  }
+  const headers = step.idempotency_key ? { 'idempotency-key': step.idempotency_key } : undefined;
+  const res = await consoleFetch(step.route, {
+    method: step.method || 'POST',
+    ...(step.request_body !== null && step.request_body !== undefined ? { body: step.request_body } : {}),
+    ...(headers ? { headers } : {}),
+    ...(parentOp === 'create' ? { retryOnTimeout: false } : {}),
+  });
+  if (res?.ok) return { outcome: 'ok', result: res };
+  if (isOutageClassResult(res) || res?.status === 401 || res?.status === 403 || res?.authProblem) {
+    return { outcome: 'stop', result: res };
+  }
+  return { outcome: 'quarantine', reason: 'console_refused', result: res };
+}
+
+/**
+ * Replay every queued write, oldest-first, with the STORED idempotency keys.
+ * Never stashes (recursion-guarded). 4xx -> quarantine and keep going; 5xx /
+ * status 0 / sign-in problem -> stop and leave the rest queued.
+ *
+ * @returns {Promise<{drained: object[], quarantined: object[], stopped_at: object|null}>}
+ */
+export async function drainWriteOutbox({ vaultRoot, consoleFetch } = {}) {
+  const out = { drained: [], quarantined: [], stopped_at: null };
+  if (draining || !vaultRoot || typeof consoleFetch !== 'function') return out;
+  draining = true;
+  try {
+    for (const entry of listOutbox(vaultRoot)) {
+      const stopAt = (res, step = null) => {
+        out.stopped_at = {
+          outbox_id: entry.outbox_id,
+          status: res?.status ?? 0,
+          errorKind: res?.errorKind ?? null,
+          ...(step ? { step_index: step } : {}),
+        };
+      };
+      const quarantine = (r, extra = {}) => {
+        quarantineOutboxEntry(vaultRoot, entry.path, {
+          reason: r.reason,
+          status: r.result?.status ?? null,
+          consoleMessage: r.result?.error ?? JSON.stringify(r.result?.body ?? {}).slice(0, 400),
+        });
+        out.quarantined.push({
+          outbox_id: entry.outbox_id,
+          reason: r.reason,
+          status: r.result?.status ?? null,
+          route: entry.route,
+          ...extra,
+        });
+      };
+
+      if (entry.op === 'batch') {
+        const steps = Array.isArray(entry.steps) ? [...entry.steps] : [];
+        let stopped = false;
+        let done = 0;
+        while (steps.length > 0) {
+          const step = steps[0];
+          const r = await replayRequest(consoleFetch, { ...step, failure: step.failure ?? (done === 0 ? entry.failure : null) }, step.kind ?? 'batch');
+          if (r.outcome === 'stop') {
+            stopAt(r.result, done);
+            stopped = true;
+            break;
+          }
+          if (r.outcome === 'quarantine') {
+            quarantine(r, { step_index: done });
+            stopped = 'quarantined';
+            break;
+          }
+          const spec = r.spec ?? r.result?.body?.spec ?? null;
+          out.drained.push({
+            outbox_id: entry.outbox_id,
+            spec_id: spec?.id ?? step.spec_id ?? null,
+            basename: basenameOf(spec),
+            route: step.route,
+            ...(r.landed_server_side ? { landed_server_side: true } : {}),
+          });
+          steps.shift();
+          done += 1;
+          if (steps.length > 0) {
+            // persist progress so a crash/outage never re-sends a landed step
+            try {
+              writeFileSync(entry.path, `${JSON.stringify({ ...stripRuntime(entry), steps }, null, 2)}\n`, 'utf8');
+            } catch {
+              /* best effort */
+            }
+          }
+        }
+        if (stopped === 'quarantined') continue;
+        if (stopped) break;
+        removeOutboxEntry(entry.path);
+        continue;
+      }
+
+      const r = await replayRequest(consoleFetch, entry, entry.op);
+      if (r.outcome === 'stop') {
+        stopAt(r.result);
+        break;
+      }
+      if (r.outcome === 'quarantine') {
+        quarantine(r);
+        continue;
+      }
+      const spec = r.spec ?? r.result?.body?.spec ?? null;
+      removeOutboxEntry(entry.path);
+      out.drained.push({
+        outbox_id: entry.outbox_id,
+        spec_id: spec?.id ?? entry.spec_id ?? null,
+        basename: basenameOf(spec),
+        route: entry.route,
+        ...(r.landed_server_side ? { landed_server_side: true } : {}),
+      });
+    }
+  } catch {
+    /* a drain never throws into a tool call */
+  } finally {
+    draining = false;
+  }
+  return out;
+}
+
+function stripRuntime(entry) {
+  const { outbox_id: _id, path: _p, ...rest } = entry;
+  return rest;
+}
+
+/**
+ * Drain and shape the result for a tool response: `{ drained, pending_sync }`
+ * fields only when there is something to say. Failure-safe.
+ */
+export async function drainAndSummarize(vaultRoot, consoleFetch) {
+  if (!vaultRoot) return {};
+  try {
+    if (outboxCount(vaultRoot) === 0) return {};
+    const d = await drainWriteOutbox({ vaultRoot, consoleFetch });
+    const pending = outboxCount(vaultRoot);
+    return {
+      ...(d.drained.length ? { drained: d.drained } : {}),
+      ...(d.quarantined.length ? { quarantined: d.quarantined } : {}),
+      ...(pending ? { pending_sync: pending } : {}),
+    };
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Once per process, after the first successful console call: drain in the
+ * background. Non-blocking and failure-safe.
+ */
+export function maybeDrainOnFirstSuccess(consoleFetch) {
+  if (drainOnceDone || !defaultVaultRoot) return;
+  drainOnceDone = true;
+  const root = defaultVaultRoot;
+  setImmediate(() => {
+    try {
+      if (outboxCount(root) === 0) return;
+      drainWriteOutbox({ vaultRoot: root, consoleFetch }).catch(() => {});
+    } catch {
+      /* swallow */
+    }
+  });
+}
+
+/**
+ * Control-plane append writes (report_progress, record_wip, handoff_record):
+ * on outage class in cloud reliance, queue as `op: 'control'` and return the
+ * `deferred` tool response; otherwise return null so the caller keeps its loud
+ * error envelope. `isCloud` is an injected async predicate (the caller's
+ * `resolveReadTier() === 'neon'`), keeping this module free of read-path.
+ * Claims and status flips must NEVER be routed through here.
+ */
+export async function deferControlWrite(result, entry, { vaultRoot = defaultVaultRoot, isCloud = async () => true } = {}) {
+  if (!vaultRoot || !isOutageClassResult(result)) return null;
+  try {
+    if (!(await isCloud())) return null;
+  } catch {
+    return null;
+  }
+  const stashed = stashWrite(vaultRoot, { ...entry, op: 'control', failure: failureOf(result) });
+  return stashed ? deferredToolResponse(buildDeferredPayload(stashed, vaultRoot)) : null;
+}