@ddtcorex/dsh-maestro-supervisor 0.7.7 → 0.7.9

package/lib/health-poller.js

@@ -74,8 +74,21 @@ export async function pollHealth(opts = {}) {
     // (but otherwise fine) response was indistinguishable from a real crash, and
     // combined with a low down-threshold this caused a self-sustaining restart
     // loop (see Supervisor.downThreshold). 12s gives boot room without masking
-    // a genuinely dead process for long.
-    const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', opts.timeoutMs ?? 12000);
+    // a genuinely dead process for long. 20s (Option A 2026-09-03) tolerates
+    // high load (loadavg 12) stalls; configurable via domains.supervisor.pollTimeoutMs.
+    // If opts.timeoutMs is not injected, read from supervisor config (maestro settings).
+    let effectiveTimeout = opts.timeoutMs;
+    if (effectiveTimeout === undefined) {
+        try {
+            const { readSupervisorConfig } = await import('./config.js');
+            const cfg = await readSupervisorConfig();
+            if (typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
+                effectiveTimeout = cfg.pollTimeoutMs;
+        }
+        catch { }
+        effectiveTimeout ??= 20000;
+    }
+    const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', effectiveTimeout);
     const psAliveFn = opts.psAlive ?? defaultPsAlive;
     const logTailFn = opts.logTail ?? defaultLogTail;
     let httpCode;

package/lib/plugin.d.ts

@@ -73,6 +73,27 @@ export declare function runAutoResume(ctx: any, opts?: {
     logResume?: (entry: ResumeLogEntry) => void;
     config?: SupervisorPluginConfig;
 }): Promise<void>;
+export interface RecoveredRoute {
+    provider: string;
+    model: string;
+}
+/**
+ * Recover the provider/model route a resumed agent needs for its
+ * `{{model}}` persona variable: handle seam first, raw session log as
+ * fallback. Returns `undefined` only for genuinely corrupt/routeless
+ * sessions, which the caller skips instead of resuming.
+ */
+export declare function recoverAgentOptions(input: {
+    persistence: any;
+    sid: unknown;
+    sessionsRoot: string;
+    group: string;
+    sessionId: string;
+    logger?: {
+        warn?: (msg: string) => void;
+    };
+    id: string;
+}): Promise<RecoveredRoute | undefined>;
 export declare function resumeInterrupted(ctx: any, ids: string[], deps?: {
     readIntent?: (id: string) => RestartIntent | undefined;
     consumeIntent?: (id: string) => void;

package/lib/plugin.js

@@ -257,6 +257,98 @@ export async function runAutoResume(ctx, opts = {}) {
         catch { }
     }
 }
+function routeFromEvents(events) {
+    if (!Array.isArray(events))
+        return undefined;
+    const context = [...events].reverse().find((event) => event?.type === 'request/context')?.data;
+    if (typeof context?.provider === 'string' && typeof context?.model === 'string') {
+        return { provider: context.provider, model: context.model };
+    }
+    return undefined;
+}
+/**
+ * Route recovery through the handle seam (`open(id, 'read')` + `read(0)`).
+ * A `read` handle never takes ownership, so this works while another handle
+ * or process holds `write`. Replaces the removed `persistence.load(id)`
+ * API (gone since DSH's handle-based persistence seam) — calling `load`
+ * now resolves to `undefined` and every resumed agent would lose its
+ * provider/model.
+ */
+async function readRouteFromHandle(persistence, sid) {
+    const handle = await persistence?.open?.(sid, 'read');
+    if (handle === undefined || handle === null)
+        return undefined;
+    try {
+        return routeFromEvents(await handle.read(0));
+    }
+    finally {
+        try {
+            await handle.close?.();
+        }
+        catch { }
+    }
+}
+/**
+ * Raw-log fallback when the backend cannot open the session: stream the
+ * stored log for the first `request/context` line. `grep -m1` stops at the
+ * first match so huge logs cost one streamed pass, not a full decode.
+ */
+async function readRouteFromRawLog(sessionsRoot, group, sessionId) {
+    const zstdPath = path.join(sessionsRoot, group, sessionId, 'session.jsonl.zstd');
+    const jsonlPath = path.join(sessionsRoot, group, sessionId, 'session.jsonl');
+    try {
+        if (fs.existsSync(zstdPath)) {
+            const { execSync } = await import('node:child_process');
+            const out = execSync(`zstd -d -c ${JSON.stringify(zstdPath)} 2>/dev/null | grep -a -m1 '"type":"request/context"'`, { encoding: 'utf-8' });
+            const line = out.split('\n').find((l) => l.includes('request/context'));
+            if (line !== undefined) {
+                try {
+                    return routeFromEvents([JSON.parse(line)]);
+                }
+                catch { }
+            }
+            return undefined;
+        }
+        if (fs.existsSync(jsonlPath)) {
+            const content = fs.readFileSync(jsonlPath, 'utf-8');
+            for (const line of content.split('\n')) {
+                if (!line.includes('"request/context"'))
+                    continue;
+                try {
+                    const route = routeFromEvents([JSON.parse(line)]);
+                    if (route !== undefined)
+                        return route;
+                }
+                catch { }
+            }
+        }
+    }
+    catch { }
+    return undefined;
+}
+/**
+ * Recover the provider/model route a resumed agent needs for its
+ * `{{model}}` persona variable: handle seam first, raw session log as
+ * fallback. Returns `undefined` only for genuinely corrupt/routeless
+ * sessions, which the caller skips instead of resuming.
+ */
+export async function recoverAgentOptions(input) {
+    try {
+        const viaHandle = await readRouteFromHandle(input.persistence, input.sid);
+        if (viaHandle !== undefined)
+            return viaHandle;
+    }
+    catch (e) {
+        input.logger?.warn?.(`[supervisor] auto-resume: could not recover route for ${input.id}: ${e?.message ?? String(e)}`);
+    }
+    try {
+        return await readRouteFromRawLog(input.sessionsRoot, input.group, input.sessionId);
+    }
+    catch (e) {
+        input.logger?.warn?.(`[supervisor] auto-resume: could not recover route for ${input.id}: ${e?.message ?? String(e)}`);
+        return undefined;
+    }
+}
 export async function resumeInterrupted(ctx, ids, deps = {}) {
     const doReadIntent = deps.readIntent ?? readIntent;
     const doConsumeIntent = deps.consumeIntent ?? consumeIntent;
@@ -277,22 +369,22 @@ export async function resumeInterrupted(ctx, ids, deps = {}) {
                 const { SessionId } = await import('@deepseek-ai/dsh-session').catch(() => ({ SessionId: (s) => s }));
                 const sid = SessionId ? SessionId(sessionId) : sessionId;
                 const persistence = ctx.get?.('sessionPersistence') ?? ctx.sessionPersistence;
-                let agentOptions;
-                try {
-                    const loaded = await persistence?.load?.(sessionId);
-                    const context = Array.isArray(loaded?.events)
-                        ? [...loaded.events].reverse().find((event) => event?.type === 'request/context')?.data
-                        : undefined;
-                    if (typeof context?.provider === 'string' && typeof context?.model === 'string') {
-                        agentOptions = { provider: context.provider, model: context.model };
-                    }
-                }
-                catch (e) {
-                    ctx.logger?.warn?.(`[supervisor] auto-resume: could not recover route for ${id}: ${e?.message ?? String(e)}`);
+                const sessionsRoot = deps.config?.sessionLogRoot ?? path.join(os.homedir(), '.dsh', 'sessions');
+                const group = id.slice(0, Math.max(0, id.length - sessionId.length - 1));
+                const agentOptions = await recoverAgentOptions({ persistence, sid, sessionsRoot, group, sessionId, logger: ctx.logger, id });
+                if (agentOptions === undefined) {
+                    // Resuming without a provider/model builds an agent whose
+                    // {{model}} persona variable has no value, so the very next turn
+                    // fails with `prompt variable "{{model}}" has no value for this
+                    // assembly (section "deployment:persona")`. Skip corrupt/routeless
+                    // sessions instead of triggering a continue that can only fail.
+                    ctx.logger?.warn?.(`[supervisor] auto-resume: skipping ${id} — no provider/model recovered (corrupt or routeless session)`);
+                    doLog({ ts: Date.now(), sessionId, kind: 'resume-failed', error: 'missing provider/model: persistence has no request/context route — skipping corrupt/routeless session' });
+                    continue;
                 }
                 const handle = await agents?.resume?.({
                     resumeSessionId: sid,
-                    ...(agentOptions === undefined ? {} : { agentOptions }),
+                    agentOptions,
                 });
                 agent = handle?.agent;
                 if (agent !== undefined)

package/lib/supervisor.d.ts

@@ -84,6 +84,8 @@ export declare class Supervisor {
     private getResumeWithinMs;
     private getEffectiveIntervalMs;
     private getEffectiveDownThreshold;
+    private getEffectiveDegradedThreshold;
+    private getEffectivePollTimeoutMs;
     private findInterruptedRecent;
     private collectGitDiff;
     private attemptAutoResume;

package/lib/supervisor.js

@@ -211,7 +211,7 @@ export class Supervisor {
                 return cfg.intervalMs;
         }
         catch { }
-        return 3000;
+        return 5000;
     }
     async getEffectiveDownThreshold() {
         if (this.deps.downThreshold !== undefined)
@@ -222,8 +222,26 @@ export class Supervisor {
                 return cfg.downThreshold;
         }
         catch { }
+        return 6;
+    }
+    async getEffectiveDegradedThreshold() {
+        try {
+            const cfg = await readSupervisorConfig();
+            if (typeof cfg.degradedThreshold === 'number' && cfg.degradedThreshold > 0)
+                return cfg.degradedThreshold;
+        }
+        catch { }
         return 5;
     }
+    async getEffectivePollTimeoutMs() {
+        try {
+            const cfg = await readSupervisorConfig();
+            if (typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
+                return cfg.pollTimeoutMs;
+        }
+        catch { }
+        return 20000;
+    }
     async findInterruptedRecent(withinMs) {
         const ms = withinMs ?? this.getResumeWithinMs();
         // Prefer injected mock for testability
@@ -383,8 +401,8 @@ export class Supervisor {
                 }
             }
             const now = this.deps.getTime ? this.deps.getTime() : Date.now();
-            // degraded needs 3 consecutive (not downThreshold which tests set to 1) to avoid flapping
-            const degradedThreshold = 3;
+            // degraded needs consecutive hits to avoid flapping — Option A 2026-09-03: 3→5 (via config)
+            const degradedThreshold = await this.getEffectiveDegradedThreshold();
             this.consecutiveDegraded++;
             if (this.consecutiveDegraded < degradedThreshold) {
                 if (now - this.lastDegradedNotify < 60000)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ddtcorex/dsh-maestro-supervisor",
-  "version": "0.7.7",
+  "version": "0.7.9",
   "description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
   "type": "module",
   "main": "./lib/index.js",