dsh-decision-layer 0.5.2 → 0.5.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-decision-layer",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "A structured decision layer for the DeepSeek Harness agent loop: a danger-call gate, output self-check, tool narrowing, and loop guard backed by a pluggable adjudication model.",
   "keywords": [
     "dsh",

package/service/manager-api.mjs

@@ -45,6 +45,10 @@ export function createManagerRoutes({ path, sessions, backend, logStore, analyze
       const value = await methods[req.method](req);
       return respond(res, 200, { ok: true, value });
     } catch (error) {
+      // A genuine internal fault (log analysis / annotation write) is a 500; a
+      // tagged or untagged validation/config error stays a client 400. Neither
+      // body carries error internals.
+      if (error?.code === 'INTERNAL') return respond(res, 500, { ok: false, error: 'Internal error' });
       if (error?.code === 'BAD_REQUEST') return respond(res, 400, { ok: false, error: error.message });
       return respond(res, 400, { ok: false, error: 'Invalid request or backend unavailable' });
     }
@@ -81,7 +85,9 @@ export function createManagerRoutes({ path, sessions, backend, logStore, analyze
     route('logs', {
       GET: async req => {
         if (typeof analyze !== 'function') { const error = new Error('analysis unavailable'); error.code = 'BAD_REQUEST'; throw error; }
-        return analyze(sessionId(req));
+        const id = sessionId(req);
+        try { return await analyze(id); }
+        catch (error) { if (error?.code === 'BAD_REQUEST') throw error; const wrapped = new Error('analysis failed'); wrapped.code = 'INTERNAL'; throw wrapped; }
       },
     }),
     // Append one annotation event for a decision. Explicit user intent: validated
@@ -96,7 +102,10 @@ export function createManagerRoutes({ path, sessions, backend, logStore, analyze
         if (!target || target.length > 128 || !['good', 'bad', 'unsure'].includes(rating) || !session || session.length > 512) {
           const error = new Error('Invalid annotation'); error.code = 'BAD_REQUEST'; throw error;
         }
-        await logStore.appendAnnotation({ kind: 'annotation', target, rating, sessionId: session });
+        // A validated write that still fails is an internal fault (disk / IO), not
+        // bad input: surface it as 500 so the client does not read it as a 400.
+        try { await logStore.appendAnnotation({ kind: 'annotation', target, rating, sessionId: session }); }
+        catch { const error = new Error('annotation write failed'); error.code = 'INTERNAL'; throw error; }
         return { ok: true, target, rating };
       },
     }),

package/service/narrowing.mjs

@@ -1,6 +1,11 @@
 const isObject = value => value !== null && typeof value === 'object' && !Array.isArray(value);
 const probability = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
 const MAX_STATE = 8 * 1024;
+// The adjudication backend hard-rejects a request carrying more than 64 questions
+// (backend.mjs validateRequest). One relevance question is sent per optional tool,
+// so the candidate cap can never exceed this — otherwise evaluate() throws locally
+// before any network call and the failure is mislabeled as a backend error.
+const MAX_QUESTIONS = 64;
 
 // Core tools an agent almost always needs. They are never judged for relevance
 // and never dropped, so narrowing cannot strip the agent's fundamental
@@ -90,8 +95,11 @@ export function createNarrowing({ evaluate, sessions, settings, coreTools = DEFA
     const prefixes = Array.isArray(raw?.keepPrefixes)
       ? raw.keepPrefixes.filter(prefix => typeof prefix === 'string' && prefix)
       : defaultPrefixes;
-    // A configured optional-candidate ceiling overrides the constructor default.
-    const cap = Number.isSafeInteger(raw?.maxCandidates) && raw.maxCandidates >= 1 ? raw.maxCandidates : maxCandidates;
+    // A configured optional-candidate ceiling overrides the constructor default,
+    // clamped to the backend's per-request question limit so the judged set is
+    // never large enough to be rejected before it is sent.
+    const configuredCap = Number.isSafeInteger(raw?.maxCandidates) && raw.maxCandidates >= 1 ? raw.maxCandidates : maxCandidates;
+    const cap = Math.min(configuredCap, MAX_QUESTIONS);
     return { mode, threshold, prefixes, cap };
   };
   const record = (turn, outcome, detail = {}) => {
@@ -114,7 +122,6 @@ export function createNarrowing({ evaluate, sessions, settings, coreTools = DEFA
       const alwaysKeep = name => core.has(name) || prefixes.some(prefix => name.startsWith(prefix));
       const keptNames = names.filter(alwaysKeep);
       const optional = tools.filter(tool => !alwaysKeep(tool.name));
-      const optionalNames = optional.map(tool => tool.name);
       if (optional.length < minCandidates) return { ...skip, keep: names };
       const hasState = typeof turn?.state === 'string' ? turn.state.trim().length > 0
         : isObject(turn?.state) ? Object.keys(turn.state).length > 0 : false;
@@ -124,13 +131,18 @@ export function createNarrowing({ evaluate, sessions, settings, coreTools = DEFA
         try { if (sessions.snapshot(id).enabled === false) return { ...skip, keep: names }; }
         catch { return { ...skip, keep: names }; }
       }
-      // B: a large optional surface must not be enforced — only observed.
+      // B: a large optional surface must not be enforced — only observed. When
+      // over the cap we still evaluate a bounded subset (never more than the
+      // backend's question limit) so the pass is logged without throwing, but the
+      // result is observe-only and applies no restriction.
       const overCap = optional.length > cap;
       const effectiveMode = overCap ? 'observe' : mode;
+      const judged = overCap ? optional.slice(0, cap) : optional;
+      const judgedNames = judged.map(tool => tool.name);
       let result;
-      try { result = await evaluate(relevanceQuestions(turn.state, optional), { signal: turn.signal }); }
+      try { result = await evaluate(relevanceQuestions(turn.state, judged), { signal: turn.signal }); }
       catch (error) { record(turn, 'error', { reason: typeof error?.reason === 'string' ? error.reason : 'unreachable', mode: effectiveMode }); return { ...skip, keep: names, mode: effectiveMode }; }
-      const narrowed = narrowTools(optionalNames, result, { threshold });
+      const narrowed = narrowTools(judgedNames, result, { threshold });
       const keep = [...keptNames, ...narrowed.keep];
       const drop = narrowed.drop;
       // C: too few tools left over is a sign the verdict over-pruned; do not apply.

package/service/self-check.mjs

@@ -1,3 +1,5 @@
+import { createSteerOnce } from './steer-once.mjs';
+
 const MAX_OUTPUT = 24 * 1024;
 
 export const DEFAULT_RUBRIC = Object.freeze([
@@ -30,11 +32,7 @@ export function scoreQuestion(output, rubric = DEFAULT_RUBRIC) {
 // the framework UserMessage so this module stays free of DSH message types.
 export function createSelfCheck({ evaluate, sessions, steer, settings, minConfidence = 0.4 } = {}) {
   if (typeof evaluate !== 'function') throw new Error('Self-check requires an evaluator');
-  const steeredTurns = new Set();
-  const steerKey = turn => {
-    const id = turn?.agent?.session?.id;
-    return typeof id === 'string' && Number.isInteger(turn?.turn) ? `${id}:${turn.turn}` : undefined;
-  };
+  const steerOnce = createSteerOnce();
   const resolveSettings = async () => {
     let raw = {};
     try { raw = (typeof settings === 'function' ? await settings() : settings) ?? {}; } catch { raw = {}; }
@@ -82,20 +80,19 @@ export function createSelfCheck({ evaluate, sessions, steer, settings, minConfid
       // The trend run advances over every evaluated score; a high-confidence low
       // score is the only thing that grows it. Thresholds flow to session-state.
       const trend = { confident: lowScore && confidence !== undefined && confidence >= trendMinConfidence, trendRun, trendSevereRun };
+      // A low score the model is not confident about (confidence present but below
+      // the trend threshold) does not advance the quality-trend judgment. Flag it
+      // so the decision log can mark it as low-signal noise instead of a real dip.
+      if (lowScore && confidence !== undefined && !trend.confident) detail.lowConfidence = true;
       record(turn, lowScore ? 'low' : 'ok', { score: answer.score, ...detail }, trend);
       if (!lowScore) return { evaluated: true, lowScore: false, steered: false, score: answer.score };
       const confident = confidence !== undefined && confidence >= minConfidence;
-      if (mode === 'steer' && confident && typeof steer === 'function' && turn?.agent) {
-        const key = steerKey(turn);
-        if (key !== undefined && !steeredTurns.has(key)) {
-          steeredTurns.add(key);
-          if (steeredTurns.size > 4096) steeredTurns.delete(steeredTurns.values().next().value);
-          try { steer(turn.agent, STEER_TEXT); }
-          catch { return { evaluated: true, lowScore: true, steered: false, score: answer.score }; }
-          return { evaluated: true, lowScore: true, steered: true, score: answer.score };
-        }
-      }
-      return { evaluated: true, lowScore: true, steered: false, score: answer.score };
+      // The turn budget is consumed only on a successful steer (createSteerOnce),
+      // so a transient host failure does not permanently suppress this turn.
+      const steered = mode === 'steer' && confident && typeof steer === 'function' && turn?.agent
+        ? steerOnce.attempt(turn, () => steer(turn.agent, STEER_TEXT))
+        : false;
+      return { evaluated: true, lowScore: true, steered, score: answer.score };
     },
   };
 }

package/service/session-state.mjs

@@ -131,6 +131,7 @@ export function createSessionState({ maxSessions = 1024, maxLogEntries = 50, onL
       if (Number.isSafeInteger(entry.unsatisfied) && entry.unsatisfied >= 0) record.unsatisfied = entry.unsatisfied;
       if (Number.isSafeInteger(entry.insufficient) && entry.insufficient >= 0) record.insufficient = entry.insufficient;
       if (entry.steered === true) record.steered = true;
+      if (entry.lowConfidence === true) record.lowConfidence = true;
       if (Array.isArray(entry.tools)) {
         const names = entry.tools.filter(name => typeof name === 'string' && name.trim()).slice(0, 12).map(name => name.trim().slice(0, 64));
         if (names.length > 0) record.tools = names;

package/service/steer-once.mjs

@@ -0,0 +1,25 @@
+// Shared "steer at most once per turn" gate for the turn-stopping decision
+// points (self-check, task-completion). A steer opens a supplemental step, so it
+// must fire at most once per (session, turn) — but a transient host failure must
+// NOT burn that budget, or one flaky steer permanently suppresses the turn.
+//
+// `attempt(turn, run)` calls `run()` (which performs the host steer and may
+// throw) and records the turn key only after `run()` returns without throwing.
+// Returns true iff a steer was actually performed this call.
+export function createSteerOnce({ maxTurns = 4096 } = {}) {
+  const steeredTurns = new Set();
+  const keyOf = turn => {
+    const id = turn?.agent?.session?.id;
+    return typeof id === 'string' && Number.isInteger(turn?.turn) ? `${id}:${turn.turn}` : undefined;
+  };
+  return {
+    attempt(turn, run) {
+      const key = keyOf(turn);
+      if (key === undefined || steeredTurns.has(key)) return false;
+      try { run(); } catch { return false; }
+      steeredTurns.add(key);
+      if (steeredTurns.size > maxTurns) steeredTurns.delete(steeredTurns.values().next().value);
+      return true;
+    },
+  };
+}

package/service/task-completion.mjs

@@ -1,3 +1,5 @@
+import { createSteerOnce } from './steer-once.mjs';
+
 const MAX_STATE = 8 * 1024;
 const MAX_CONDITION = 512;
 const MAX_CONDITIONS = 20;
@@ -78,11 +80,7 @@ export function completionQuestions(conditions, evidence, answer) {
 // opens a supplemental step; it never rewrites the already-committed reply.
 export function createTaskCompletion({ evaluate, sessions, steer, settings, minConfidence = 0.4 } = {}) {
   if (typeof evaluate !== 'function') throw new Error('Task completion requires an evaluator');
-  const steeredTurns = new Set();
-  const steerKey = turn => {
-    const id = turn?.agent?.session?.id;
-    return typeof id === 'string' && Number.isInteger(turn?.turn) ? `${id}:${turn.turn}` : undefined;
-  };
+  const steerOnce = createSteerOnce();
   const resolveSettings = async () => {
     let raw = {};
     try { raw = (typeof settings === 'function' ? await settings() : settings) ?? {}; } catch { raw = {}; }
@@ -122,31 +120,31 @@ export function createTaskCompletion({ evaluate, sessions, steer, settings, minC
       }
       if (!answers) { record(turn, 'error', { reason: 'invalid-response' }); return { ...skip }; }
       let satisfied = 0, unsatisfied = 0, insufficient = 0;
-      let minConf = 1;
+      // Track the minimum confidence over the `unsatisfied` verdicts only: a steer
+      // fires on a confident unsatisfied finding, so a low-confidence satisfied /
+      // insufficient verdict on another condition must not suppress it.
+      let unsatisfiedMinConf = 1;
       for (let index = 0; index < conditions.length; index++) {
         const answer = answers[`c${index}`];
         if (!isObject(answer) || answer.type !== 'choice' || typeof answer.choice !== 'string') {
           insufficient++; continue;
         }
-        if (probability(answer.confidence)) minConf = Math.min(minConf, answer.confidence);
         if (answer.choice === 'satisfied') satisfied++;
-        else if (answer.choice === 'unsatisfied') unsatisfied++;
+        else if (answer.choice === 'unsatisfied') {
+          unsatisfied++;
+          if (probability(answer.confidence)) unsatisfiedMinConf = Math.min(unsatisfiedMinConf, answer.confidence);
+        }
         else insufficient++;
       }
       const outcome = unsatisfied > 0 ? 'unsatisfied' : 'ok';
       const detail = { conditions: conditions.length, satisfied, unsatisfied, insufficient };
       // A steer is only for a confident finding that a listed condition is not met.
-      const confident = unsatisfied > 0 && minConf >= minConfidence;
-      let steered = false;
-      if (mode === 'steer' && confident && typeof steer === 'function' && turn?.agent) {
-        const key = steerKey(turn);
-        if (key !== undefined && !steeredTurns.has(key)) {
-          steeredTurns.add(key);
-          if (steeredTurns.size > 4096) steeredTurns.delete(steeredTurns.values().next().value);
-          try { steer(turn.agent, STEER_TEXT); steered = true; }
-          catch { steered = false; }
-        }
-      }
+      const confident = unsatisfied > 0 && unsatisfiedMinConf >= minConfidence;
+      // The turn budget is consumed only on a successful steer (createSteerOnce),
+      // so a transient host failure does not permanently suppress this turn.
+      const steered = mode === 'steer' && confident && typeof steer === 'function' && turn?.agent
+        ? steerOnce.attempt(turn, () => steer(turn.agent, STEER_TEXT))
+        : false;
       if (steered) detail.steered = true;
       record(turn, outcome, detail);
       return { evaluated: true, steered, satisfied, unsatisfied, insufficient };