abmind 0.2.5-alpha.0 → 0.2.5

package/dist/src/sleep/orchestrator.js

@@ -52,6 +52,16 @@ export class SleepInitError extends Error {
 export class SleepTimeoutError extends Error {
     constructor(message) { super(message); this.name = "SleepTimeoutError"; }
 }
+/** Best-effort onStep invoker — swallow handler errors so a display callback
+ *  can never break the pipeline. */
+function fireOnStep(handler, e) {
+    if (!handler)
+        return;
+    try {
+        handler(e);
+    }
+    catch { /* host display only — never fail the cycle */ }
+}
 export function parseArgs(argv) {
     const args = argv.slice(2);
     const parsed = { dryRun: false, verbose: false, force: false };
@@ -136,6 +146,17 @@ function formatWiredResults(r) {
 // ── Transport ───────────────────────────────────────────────────────────────
 // (SleepRuntime is imported at the top of the file.)
 const MAX_RETRIES = 3;
+const TRANSPORT_MAX_RETRIES = 6;
+const DEFAULT_TRANSPORT_RETRY_WINDOW_MS = 8 * 60_000; // 8 minutes
+/** Thrown by sendWithRetry when runtime.complete() throws on every attempt within the retry window.
+ *  Extends LLMUnavailableError so buildDailySummary and extractFromDaily rethrow it naturally
+ *  (those functions check instanceof LLMUnavailableError to propagate errors up). */
+export class ModelUnavailableError extends LLMUnavailableError {
+    constructor(stepName) {
+        super(`Model unavailable for step "${stepName}" after retry window`);
+        this.name = "ModelUnavailableError";
+    }
+}
 /** Budget tracker — shared across all sendWithRetry calls in a sleep cycle. */
 class LlmBudget {
     state;
@@ -157,17 +178,28 @@ class LlmBudget {
     }
     get calls() { return this.state.llmCalls ?? 0; }
 }
-async function sendWithRetry(runtime, prompt, stepName, _verbose, budget, delayMs = 6000) {
-    for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
-        if (budget && !budget.consume()) {
-            logWarn(TAG, `[BUDGET] LLM call limit (${getAbmindEnv().sleepMaxLlmCalls}) reached at step ${stepName} — suspending`);
-            return null;
-        }
+async function sendWithRetry(runtime, prompt, stepName, _verbose, budget, delayMs = 6000, transportBackoffMs = (n) => Math.min(30_000 * Math.pow(2, n - 1), 120_000), transportRetryWindowMs = DEFAULT_TRANSPORT_RETRY_WINDOW_MS, nowFn = Date.now) {
+    const deadline = nowFn() + transportRetryWindowMs;
+    let emptyAttempts = 0;
+    let transportAttempts = 0;
+    // Budget-exhaustion pre-check: if we are already over the cap, bail immediately.
+    // This mirrors the old pre-loop consume() guard without burning a call.
+    if (budget?.exhausted) {
+        logWarn(TAG, `[BUDGET] LLM call limit (${getAbmindEnv().sleepMaxLlmCalls}) reached at step ${stepName} — suspending`);
+        return null;
+    }
+    while (true) {
         try {
             const result = await runtime.complete(prompt);
+            // Real call reached the model — count it now (success OR empty, never a throw).
+            if (budget && !budget.consume()) {
+                logWarn(TAG, `[BUDGET] LLM call limit (${getAbmindEnv().sleepMaxLlmCalls}) reached at step ${stepName} — suspending`);
+                return null;
+            }
             if (!result || !result.trim()) {
-                logWarn(TAG, `Step ${stepName} attempt ${attempt}/${MAX_RETRIES} returned empty response`);
-                if (attempt === MAX_RETRIES) {
+                emptyAttempts++;
+                logWarn(TAG, `Step ${stepName} attempt ${emptyAttempts}/${MAX_RETRIES} returned empty response`);
+                if (emptyAttempts >= MAX_RETRIES) {
                     logError(TAG, `Step ${stepName} failed after ${MAX_RETRIES} attempts (empty), skipping`);
                     return null;
                 }
@@ -178,17 +210,21 @@ async function sendWithRetry(runtime, prompt, stepName, _verbose, budget, delayM
             return result;
         }
         catch (err) {
+            // Transport failure — model unreachable. Do NOT consume budget (no tokens used).
+            transportAttempts++;
             const msg = err instanceof Error ? err.message : String(err);
-            logWarn(TAG, `Step ${stepName} attempt ${attempt}/${MAX_RETRIES} failed: ${msg}`);
-            if (attempt === MAX_RETRIES) {
-                logError(TAG, `Step ${stepName} failed after ${MAX_RETRIES} attempts, skipping`);
-                return null;
+            const backoff = transportBackoffMs(transportAttempts);
+            const timeRemaining = deadline - nowFn();
+            logWarn(TAG, `Step ${stepName} transport fail (${transportAttempts}/${TRANSPORT_MAX_RETRIES}): ${msg}`);
+            if (transportAttempts >= TRANSPORT_MAX_RETRIES || timeRemaining <= backoff) {
+                logError(TAG, `Step ${stepName} — model unavailable after ${transportAttempts} attempt(s), ${Math.round(transportRetryWindowMs / 60_000)}min window exhausted`);
+                throw new ModelUnavailableError(stepName);
             }
-            if (delayMs > 0)
-                await new Promise(r => setTimeout(r, delayMs));
+            logInfo(TAG, `[SLEEP] Model unreachable — waiting ${Math.round(backoff / 1000)}s before retry (step ${stepName})`);
+            if (backoff > 0)
+                await new Promise(r => setTimeout(r, backoff));
         }
     }
-    return null;
 }
 // ── Audit trail helpers ─────────────────────────────────────────────────────
 function buildSnapshotSummary(snapshot) {
@@ -372,7 +408,7 @@ function failedEssentials(state) {
     }
     return failed;
 }
-async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime, budget, retryDelayMs = 6000) {
+async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime, budget, retryDelayMs = 6000, onStep) {
     for (const lock of locks) {
         if (lock.ageDays > CATCHUP_MAX_AGE_DAYS) {
             logError(TAG, `[CATCH-UP] Abandoning stale lock ${basename(lock.path)} — ${lock.ageDays} days old, data unrecoverable`);
@@ -407,10 +443,22 @@ async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime,
                     lock.state.steps["daily-summary"] = { status: "skipped" };
                 }
                 logInfo(TAG, `[CATCH-UP] ✓ 04a-daily-summary for ${lock.dateStr} (${((Date.now() - start) / 1000).toFixed(1)}s)`);
+                // #895: emit terminal event for catch-up daily-summary
+                fireOnStep(onStep, {
+                    name: "daily-summary", filename: "catch-up",
+                    index: 0, total: 0,
+                    phase: summary ? "done" : "skipped",
+                });
             }
             catch (err) {
                 logWarn(TAG, `[CATCH-UP] ✗ 04a-daily-summary for ${lock.dateStr}: ${err instanceof Error ? err.message : String(err)}`);
                 lock.state.steps["daily-summary"] = { status: "failed", duration: Math.round((Date.now() - start) / 100) / 10 };
+                // #895: emit failed for catch-up daily-summary
+                fireOnStep(onStep, {
+                    name: "daily-summary", filename: "catch-up",
+                    index: 0, total: 0,
+                    phase: "failed",
+                });
             }
             writeStateFile(lock.path, lock.state);
         }
@@ -420,6 +468,12 @@ async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime,
             if (!existsSync(dailyPath)) {
                 logInfo(TAG, `[CATCH-UP] ⏭ 04b — no daily file for ${lock.dateStr}`);
                 lock.state.steps["extract-memories"] = { status: "skipped" };
+                // #895: emit skip when no daily file for catch-up
+                fireOnStep(onStep, {
+                    name: "extract-memories", filename: "catch-up",
+                    index: 0, total: 0,
+                    phase: "skipped",
+                });
             }
             else {
                 const start = Date.now();
@@ -429,10 +483,22 @@ async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime,
                         throw new LLMUnavailableError(); return r; }));
                     lock.state.steps["extract-memories"] = { status: "ok", duration: Math.round((Date.now() - start) / 100) / 10 };
                     logInfo(TAG, `[CATCH-UP] ✓ 04b-extract-memories for ${lock.dateStr} (${((Date.now() - start) / 1000).toFixed(1)}s) — ${result.slice(0, 80)}`);
+                    // #895: emit done for catch-up extract-memories
+                    fireOnStep(onStep, {
+                        name: "extract-memories", filename: "catch-up",
+                        index: 0, total: 0,
+                        phase: "done",
+                    });
                 }
                 catch (err) {
                     logWarn(TAG, `[CATCH-UP] ✗ 04b for ${lock.dateStr}: ${err instanceof Error ? err.message : String(err)}`);
                     lock.state.steps["extract-memories"] = { status: "failed", duration: Math.round((Date.now() - start) / 100) / 10 };
+                    // #895: emit failed for catch-up extract-memories
+                    fireOnStep(onStep, {
+                        name: "extract-memories", filename: "catch-up",
+                        index: 0, total: 0,
+                        phase: "failed",
+                    });
                 }
             }
             writeStateFile(lock.path, lock.state);
@@ -451,10 +517,22 @@ async function runCatchUp(locks, sleepData, memoryConfig, steps, flags, runtime,
             if (response) {
                 lock.state.steps[stepName] = { status: "ok", duration: Math.round((Date.now() - start) / 100) / 10 };
                 logInfo(TAG, `[CATCH-UP] ✓ ${stepName} (${((Date.now() - start) / 1000).toFixed(1)}s)`);
+                // #895: emit done for catch-up retrospective
+                fireOnStep(onStep, {
+                    name: stepName, filename: "catch-up",
+                    index: 0, total: 0,
+                    phase: "done",
+                });
             }
             else {
                 lock.state.steps[stepName] = { status: "failed", duration: Math.round((Date.now() - start) / 100) / 10 };
                 logWarn(TAG, `[CATCH-UP] ✗ ${stepName}`);
+                // #895: emit failed for catch-up retrospective
+                fireOnStep(onStep, {
+                    name: stepName, filename: "catch-up",
+                    index: 0, total: 0,
+                    phase: "failed",
+                });
             }
             writeStateFile(lock.path, lock.state);
         }
@@ -484,6 +562,8 @@ export async function runSleepCycle(opts) {
     const timeoutMs = opts.timeoutMs ?? getAbmindEnv().sleepTimeoutMin * 60 * 1000;
     const backoffMs = opts.backoffMs ?? ((n) => [10, 30, 60][Math.min(n, 2)] * 1000);
     const retryDelayMs = opts.retryDelayMs ?? 6000;
+    const transportBackoffMs = opts.transportBackoffMs ?? ((n) => Math.min(30_000 * Math.pow(2, n - 1), 120_000));
+    const transportRetryWindowMs = opts.transportRetryWindowMs ?? DEFAULT_TRANSPORT_RETRY_WINDOW_MS;
     const runtime = opts.runtime;
     if (flags.verbose) {
         logInfo(TAG, "Verbose mode enabled");
@@ -641,6 +721,13 @@ export async function runSleepCycle(opts) {
         const snapshotVars = buildSleepVars(snapshot);
         for (const [k, v] of Object.entries(snapshotVars))
             vars[k] = vars[k] ?? v;
+        // #895: Fire onCycleStart once before the first step — hosts use this to
+        // create a stepped card or reset per-night state. fireOnStep() already
+        // swallows handler errors so a misbehaving host can never break the cycle.
+        try {
+            opts.onCycleStart?.({ totalSteps: steps.length });
+        }
+        catch { /* host display only */ }
         // Progress protocol — emit PROGRESS:<pct>:<label> on stdout
         const totalSteps = steps.length;
         let stepIndex = 0;
@@ -754,7 +841,7 @@ export async function runSleepCycle(opts) {
             const previousLocks = scanPreviousLocks(sleepDir, dateStr);
             if (previousLocks.length > 0) {
                 logInfo(TAG, `[CATCH-UP] Found ${previousLocks.length} previous lock(s)`);
-                await runCatchUp(previousLocks, sleepData, memoryConfig, steps, flags, runtime, budget, retryDelayMs);
+                await runCatchUp(previousLocks, sleepData, memoryConfig, steps, flags, runtime, budget, retryDelayMs, opts.onStep);
             }
             // Housekeeping: move misplaced daily/consolidation_* to weekly/ (#640)
             try {
@@ -799,13 +886,19 @@ export async function runSleepCycle(opts) {
                 }
                 emitProgress(step.name);
                 stepIndex++;
+                // #895: "start" fires once per step (before skip checks) so every step
+                // emits exactly one start + one terminal (done/skipped/failed). Display-only,
+                // best-effort — a throwing handler never breaks the cycle.
+                fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "start" });
                 // Resume: skip already completed steps
                 if (isResume && existingState?.steps[step.name]?.status === "ok") {
                     logInfo(TAG, `[SLEEP] ⏭ ${step.name} — already done (resume)`);
+                    fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                     continue;
                 }
                 if (isResume && existingState?.steps[step.name]?.status === "skipped") {
                     logInfo(TAG, `[SLEEP] ⏭ ${step.name} — skipped (resume)`);
+                    fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                     continue;
                 }
                 // Skip logic (essential steps always run)
@@ -813,6 +906,7 @@ export async function runSleepCycle(opts) {
                     logInfo(TAG, `[SLEEP] ⏭ ${step.name} — skipped`);
                     state.steps[step.name] = { status: "skipped" };
                     writeStateFile(statePath, state);
+                    fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                     continue;
                 }
                 const start = Date.now();
@@ -829,7 +923,7 @@ export async function runSleepCycle(opts) {
                         const firstMsgTs = sleepData.getFirstMessageAfter(userId, watermarkTs);
                         const firstMsgDate = firstMsgTs ? new Date(firstMsgTs) : new Date(now());
                         const targetDate = `${firstMsgDate.getFullYear()}-${String(firstMsgDate.getMonth() + 1).padStart(2, "0")}-${String(firstMsgDate.getDate()).padStart(2, "0")}`;
-                        const summary = await buildDailySummary(sleepData.getDb(), (p) => sendWithRetry(runtime, p, "daily-summary", flags.verbose, budget, retryDelayMs).then(r => { if (r === null)
+                        const summary = await buildDailySummary(sleepData.getDb(), (p) => sendWithRetry(runtime, p, "daily-summary", flags.verbose, budget, retryDelayMs, transportBackoffMs, transportRetryWindowMs, now).then(r => { if (r === null)
                             throw new LLMUnavailableError(); return r; }), {
                             ctxWindow, memoryDir: memoryConfig.memoryDir, userId, watermarkTs,
                         });
@@ -846,9 +940,30 @@ export async function runSleepCycle(opts) {
                         logWarn(TAG, `[SLEEP] 04a failed: ${err instanceof Error ? err.message : String(err)}`);
                         state.steps[step.name] = { status: "failed", duration: Math.round((Date.now() - start) / 100) / 10 };
                         dreamySucceeded = false;
+                        writeStateFile(statePath, state);
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "failed" });
+                        if (err instanceof ModelUnavailableError) {
+                            logWarn(TAG, `[SLEEP] ${step.name} — model unavailable, stopping cycle (not advancing to next phase)`);
+                            break;
+                        }
+                    }
+                    if (state.steps[step.name]?.status !== "failed") {
+                        // success/skip path: persist state + log + fire event
+                        writeStateFile(statePath, state);
                     }
-                    writeStateFile(statePath, state);
                     logInfo(TAG, `[SLEEP] ${state.steps[step.name]?.status === "ok" ? "✓" : "✗"} ${step.name} (${((Date.now() - start) / 1000).toFixed(1)}s)`);
+                    // #895: emit terminal event for code-driven daily-summary (failure already fired in catch)
+                    if (state.steps[step.name]?.status !== "failed") {
+                        fireOnStep(opts.onStep, {
+                            name: step.name,
+                            filename: step.filename,
+                            index: stepIndex,
+                            total: totalSteps,
+                            phase: state.steps[step.name]?.status === "ok" ? "done"
+                                : state.steps[step.name]?.status === "skipped" ? "skipped"
+                                    : "failed",
+                        });
+                    }
                     continue;
                 }
                 if (step.name === "extract-memories") {
@@ -866,11 +981,13 @@ export async function runSleepCycle(opts) {
                         state.steps[step.name] = { status: "skipped" };
                         writeStateFile(statePath, state);
                         logInfo(TAG, `[SLEEP] ⏭ ${step.name} — no daily summary`);
+                        // #895: emit skip when there's no daily summary to extract from
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                         continue;
                     }
                     try {
                         const userId = sleepData.getPrimaryUserId();
-                        const result = await extractFromDaily(dailySummaryPath, userId, (p) => sendWithRetry(runtime, p, "04b-extract", flags.verbose, budget, retryDelayMs).then(r => { if (r === null)
+                        const result = await extractFromDaily(dailySummaryPath, userId, (p) => sendWithRetry(runtime, p, "04b-extract", flags.verbose, budget, retryDelayMs, transportBackoffMs, transportRetryWindowMs, now).then(r => { if (r === null)
                             throw new LLMUnavailableError(); return r; }));
                         state.steps[step.name] = { status: "ok", duration: Math.round((Date.now() - start) / 100) / 10 };
                         writeFileSync(join(stepLogDir, `${String(stepIndex).padStart(2, "0")}-${step.name}.md`), redactSecrets(result), "utf-8");
@@ -880,8 +997,24 @@ export async function runSleepCycle(opts) {
                         logWarn(TAG, `[SLEEP] 04b failed: ${err instanceof Error ? err.message : String(err)}`);
                         state.steps[step.name] = { status: "failed", duration: Math.round((Date.now() - start) / 100) / 10 };
                         dreamySucceeded = false;
+                        writeStateFile(statePath, state);
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "failed" });
+                        if (err instanceof ModelUnavailableError) {
+                            logWarn(TAG, `[SLEEP] ${step.name} — model unavailable, stopping cycle (not advancing to next phase)`);
+                            break;
+                        }
+                    }
+                    if (state.steps[step.name]?.status !== "failed") {
+                        writeStateFile(statePath, state);
+                        // #895: emit terminal event for code-driven extract-memories (failure already fired in catch)
+                        fireOnStep(opts.onStep, {
+                            name: step.name,
+                            filename: step.filename,
+                            index: stepIndex,
+                            total: totalSteps,
+                            phase: state.steps[step.name]?.status === "ok" ? "done" : "failed",
+                        });
                     }
-                    writeStateFile(statePath, state);
                     continue;
                 }
                 // Standard prompt-driven step — JIT substitution
@@ -896,6 +1029,8 @@ export async function runSleepCycle(opts) {
                             state.steps[step.name] = { status: "skipped" };
                             writeStateFile(statePath, state);
                             logInfo(TAG, `[SLEEP] ⏭ ${step.name} — no new extractions today`);
+                            // #895: emit skip when there's no work for contradiction-and-graph
+                            fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                             continue;
                         }
                         vars.NEW_EXTRACTIONS = newRows.map(r => `[id=${r.id}] (${r.memory_type}, trust=${r.trust}) ${r.content_en}`).join("\n");
@@ -925,6 +1060,8 @@ export async function runSleepCycle(opts) {
                         logWarn(TAG, `[SLEEP] contradiction-and-graph var prep failed: ${err instanceof Error ? err.message : String(err)}`);
                         state.steps[step.name] = { status: "skipped" };
                         writeStateFile(statePath, state);
+                        // #895: emit skip on var-prep failure
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                         continue;
                     }
                 }
@@ -937,6 +1074,8 @@ export async function runSleepCycle(opts) {
                             state.steps[step.name] = { status: "skipped" };
                             writeStateFile(statePath, state);
                             logInfo(TAG, `[SLEEP] ⏭ ${step.name} — not enough memories for REM`);
+                            // #895: emit skip when not enough memories for REM
+                            fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                             continue;
                         }
                         vars.REM_SAMPLE = sample.map(r => `[${r.memory_type}, ${new Date(r.created_at).toISOString().slice(0, 10)}] ${r.content_en}`).join("\n");
@@ -944,6 +1083,8 @@ export async function runSleepCycle(opts) {
                     catch {
                         state.steps[step.name] = { status: "skipped" };
                         writeStateFile(statePath, state);
+                        // #895: emit skip on var-prep failure for rem-synthesis
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "skipped" });
                         continue;
                     }
                 }
@@ -952,7 +1093,22 @@ export async function runSleepCycle(opts) {
                 if (soulPrefix)
                     soulPrefix = ""; // only prepend to first step
                 const ctxBefore = -1;
-                const response = await sendWithRetry(runtime, fullPrompt, step.name, flags.verbose, budget, retryDelayMs);
+                let response;
+                try {
+                    response = await sendWithRetry(runtime, fullPrompt, step.name, flags.verbose, budget, retryDelayMs, transportBackoffMs, transportRetryWindowMs, now);
+                }
+                catch (err) {
+                    if (err instanceof ModelUnavailableError) {
+                        const duration = Date.now() - start;
+                        logWarn(TAG, `[SLEEP] ${step.name} — model unavailable, stopping cycle (not advancing to next phase)`);
+                        state.steps[step.name] = { status: "failed", duration: Math.round(duration / 100) / 10, ctxBefore, ctxAfter: -1 };
+                        dreamySucceeded = false;
+                        writeStateFile(statePath, state);
+                        fireOnStep(opts.onStep, { name: step.name, filename: step.filename, index: stepIndex, total: totalSteps, phase: "failed" });
+                        break;
+                    }
+                    throw err;
+                }
                 const ctxAfter = -1;
                 const duration = Date.now() - start;
                 if (response) {
@@ -1005,6 +1161,14 @@ export async function runSleepCycle(opts) {
                     dreamySucceeded = false;
                 }
                 writeStateFile(statePath, state);
+                // #895: emit terminal event for the standard prompt-driven step
+                fireOnStep(opts.onStep, {
+                    name: step.name,
+                    filename: step.filename,
+                    index: stepIndex,
+                    total: totalSteps,
+                    phase: response ? "done" : "failed",
+                });
                 logInfo(TAG, `[SLEEP] ${response ? "✓" : "✗"} ${step.name} (${(duration / 1000).toFixed(1)}s, ${response?.length ?? 0} chars)`);
                 // Backoff between steps: 10s → 30s → 60s on consecutive failures, reset on success
                 if (response) {