@dtlabs/ai-toolkit 0.17.3 → 0.17.5

package/CHANGELOG.md

@@ -1,4 +1,21 @@
 ## 0.1.4 (2026-07-21)
+## [0.17.5](https://github.com/dtlab-tech/ai-toolkit/compare/v0.17.4...v0.17.5) (2026-10-13)
+
+
+### Bug Fixes
+
+* **pm-phase1:** detect truncated Requirements/Tech-Spec instead of silently accepting them ([0783fe1](https://github.com/dtlab-tech/ai-toolkit/commit/0783fe1639258ea9701bffc647b9d0a54a85d3cd))
+* **workflow-control, pm-phase1:** retry transient API errors; catch truncated Gate 1 docs ([47d4f2c](https://github.com/dtlab-tech/ai-toolkit/commit/47d4f2c47f79ac62b5247b44aac13dd2c80a33ad))
+* **workflow-control:** retry worker dispatch on transient CLI/SDK API errors ([d75d02b](https://github.com/dtlab-tech/ai-toolkit/commit/d75d02b866a51ffde51bb276fdf21a60bd38c578))
+
+## [0.17.4](https://github.com/dtlab-tech/ai-toolkit/compare/v0.17.3...v0.17.4) (2026-10-13)
+
+
+### Bug Fixes
+
+* **workflow-control:** repair unescaped control chars in recovered JSON ([ebeddb5](https://github.com/dtlab-tech/ai-toolkit/commit/ebeddb5465ac988f94bd28d20b7f9dfdd967c2c9))
+* **workflow-control:** repair unescaped control chars in recovered JSON ([0ce2466](https://github.com/dtlab-tech/ai-toolkit/commit/0ce24666b6d4d09b25ca71b1ed3b9f0ba7c06d7a))
+
 ## [0.17.3](https://github.com/dtlab-tech/ai-toolkit/compare/v0.17.2...v0.17.3) (2026-10-12)
 
 

package/lib/workflow-control.js

@@ -169,28 +169,56 @@ function createControl(options) {
       });
       (options.log || (() => {}))('Worker evidence: ' + evidence.file);
       try {
-      const result = await (options.dispatch || spawnClaudeAgent)({
-        claudePath: options.claudePath, cwd: root, args: cliArgs, prompt,
-        taskTimeoutMs: options.taskTimeoutMs || 1800000,
-      });
+      // Retry a bounded number of times, but ONLY for a recognizable transient
+      // infrastructure failure — a CLI/SDK-level "API Error: ..." (stream idle
+      // timeout, overload, etc.), never a permission denial, schema violation,
+      // or any other deterministic content failure, which would just fail the
+      // same way again. Confirmed safe to retry: every agent dispatched through
+      // this path is read-only by design (no Write tool — see this function's
+      // own comment on why outputs are host-written, not agent-written; any
+      // Bash access granted is explicitly scoped to read-only assessment), so
+      // there is no partial side effect from a dead stream to clean up before
+      // retrying. Token usage from EVERY attempt is accumulated — a failed
+      // attempt still burns real output tokens before the stream dies — and
+      // each attempt's outcome is kept in evidence for diagnosing a recurring
+      // pattern like this one, rather than only ever seeing the last attempt.
+      let result;
+      let accumulatedTokens = 0;
+      const attempts = [];
+      for (let attempt = 1; attempt <= WORKER_MAX_ATTEMPTS; attempt++) {
+        result = await (options.dispatch || spawnClaudeAgent)({
+          claudePath: options.claudePath, cwd: root, args: cliArgs, prompt,
+          taskTimeoutMs: options.taskTimeoutMs || 1800000,
+        });
+        const usage = result.result?.usage;
+        const counts = usage && [usage.input_tokens, usage.output_tokens,
+          usage.cache_read_input_tokens ?? 0, usage.cache_creation_input_tokens ?? 0];
+        const tokens = counts && counts.every(n => Number.isInteger(n) && n >= 0)
+          ? counts.reduce((a, b) => a + b, 0) : null;
+        const attemptTokens = Number.isSafeInteger(tokens) && tokens > 0 ? tokens : null;
+        if (attemptTokens) accumulatedTokens += attemptTokens;
+        attempts.push({ attempt, exitCode: result.exitCode, isError: result.result?.is_error ?? null,
+          resultPreview: typeof result.result?.result === 'string' ? result.result.result.slice(0, 300) : null,
+          tokens: attemptTokens });
+        evidence.save({ attempts });
+        if (!_isTransientWorkerApiError(result) || attempt === WORKER_MAX_ATTEMPTS) break;
+        await _sleep(WORKER_RETRY_BACKOFF_MS[attempt - 1]);
+      }
       // Count usage before any business/protocol check, including denied tools.
-      const usage = result.result?.usage;
-      const counts = usage && [usage.input_tokens, usage.output_tokens,
-        usage.cache_read_input_tokens ?? 0, usage.cache_creation_input_tokens ?? 0];
-      const tokens = counts && counts.every(n => Number.isInteger(n) && n >= 0)
-        ? counts.reduce((a, b) => a + b, 0) : null;
-      measuredTokens = Number.isSafeInteger(tokens) && tokens > 0 ? tokens : null;
+      measuredTokens = accumulatedTokens > 0 ? accumulatedTokens : null;
       spent += measuredTokens || 0;
       tokenLedger.push({ agent: identity.nativeName, model: definition.model,
-        phase_delta_tokens: measuredTokens, usage: usage || null, evidence_path: evidence.file });
-      evidence.save({ response: result });
+        phase_delta_tokens: measuredTokens, usage: result.result?.usage || null, evidence_path: evidence.file });
+      evidence.save({ response: result, attempts });
       const denials = result.result?.permission_denials;
       if (Array.isArray(denials) && denials.length) {
         const tools = [...new Set(denials.map(d => d.tool_name || 'unknown'))].join(', ');
         throw new Error('Worker permission denied: ' + id + '; tools: ' + tools);
       }
       if (result.exitCode !== 0 || result.parseError || result.timedOut ||
-          !result.result || result.result.is_error !== false) throw new Error('Worker failed: ' + id);
+          !result.result || result.result.is_error !== false) {
+        throw new Error('Worker failed: ' + id + (attempts.length > 1 ? ' (after ' + attempts.length + ' attempts)' : ''));
+      }
       let payload = opts.schema ? result.result.structured_output : result.result.result;
       let usedTextFallback = false;
       // The CLI's structured-output extraction is turn-bound: it only populates
@@ -305,6 +333,65 @@ function createControl(options) {
   return api;
 }
 
+// worker()'s retry policy for transient CLI/SDK-level infrastructure failures (e.g. "API
+// Error: Stream idle timeout - no chunks received") — confirmed against a real pipeline run
+// that failed 3/3 identically with no automatic recovery, forcing the whole workflow to be
+// retried by hand from the CLI. 3 total attempts, waiting 5s then 20s between them: long
+// enough for a brief network stall to clear, short enough not to turn a real, deterministic
+// failure into a multi-minute stall before it's reported.
+const WORKER_MAX_ATTEMPTS = 3;
+const WORKER_RETRY_BACKOFF_MS = [5000, 20000];
+
+function _sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// Recognizes ONLY the CLI/SDK's own infrastructure-error reporting convention — never a
+// blanket "any failure is retryable" rule. A permission denial, a schema violation, or any
+// other content-level failure will fail the exact same way on a retry, so retrying those
+// would only burn tokens and time for no benefit; this checks specifically for the "API
+// Error:" prefix the CLI uses for its own transport/provider-level errors, which (unlike
+// those) are plausibly transient.
+function _isTransientWorkerApiError(result) {
+  return !!(result && result.result && result.result.is_error === true &&
+    typeof result.result.result === 'string' && /^API Error:/i.test(result.result.result.trim()));
+}
+
+// A model asked to emit a JSON blob as part of a chat reply (rather than through a real
+// serializer, i.e. whenever structured_output didn't get populated and extraction has to
+// recover the payload from free text) is prone to one specific, well-understood defect: a
+// long free-text field (a "report") gets its literal line breaks typed as real newlines
+// instead of the required `\n` escape. The JSON otherwise reads as entirely correct to a
+// human, but JSON.parse rejects it outright ("Bad control character in string literal") —
+// confirmed against a real run where the model's JSON was discarded for exactly this reason
+// despite carrying the correct ```json fence tag and schema-correct structure. This is a
+// narrow, mechanical repair for that one defect, not a lenient parser: it tracks whether
+// each character lies inside a double-quoted JSON string literal (honoring \" and \\
+// escapes) and re-escapes a raw newline/CR/tab ONLY there, leaving structural whitespace
+// between tokens untouched. Applied as a fallback after a plain JSON.parse fails, never in
+// place of one, so already-valid JSON is never second-guessed.
+function _repairUnescapedControlCharsInStrings(text) {
+  let result = '';
+  let inString = false;
+  let escaped = false;
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i];
+    if (inString) {
+      if (escaped) { result += ch; escaped = false; }
+      else if (ch === '\\') { result += ch; escaped = true; }
+      else if (ch === '"') { result += ch; inString = false; }
+      else if (ch === '\n') { result += '\\n'; }
+      else if (ch === '\r') { result += '\\r'; }
+      else if (ch === '\t') { result += '\\t'; }
+      else { result += ch; }
+    } else {
+      if (ch === '"') inString = true;
+      result += ch;
+    }
+  }
+  return result;
+}
+
 // The model is not consistent about which delimiter it uses to mark its final structured
 // answer inside otherwise-free text — observed empirically, same agent and schema, run to
 // run: an <structured_output> tag, a ```json fence, and an unprompted <json> tag. Growing
@@ -332,8 +419,15 @@ function extractDelimitedStructuredOutput(text, schema) {
   const last = text.lastIndexOf('}');
   if (first !== -1 && last > first) candidates.push(text.slice(first, last + 1));
   for (const raw of candidates) {
+    const trimmed = raw.trim();
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (e) {
+      try { parsed = JSON.parse(_repairUnescapedControlCharsInStrings(trimmed)); }
+      catch (e2) { continue; }
+    }
     try {
-      const parsed = JSON.parse(raw.trim());
       validateSchema(parsed, schema);
       return parsed;
     } catch (e) { /* try the next candidate, or fall through to null */ }
@@ -354,7 +448,18 @@ function extractDelimitedStructuredOutput(text, schema) {
 // remains the one thing that fails closed on it; this function only undoes wrapping, it is not
 // itself a validation gate.
 function extractJsonDocument(text) {
-  const tryParse = candidate => { try { JSON.parse(candidate); return candidate; } catch (e) { return null; } };
+  // Same "try the candidate as-is, then try it with raw control characters inside
+  // string literals re-escaped" fallback as extractDelimitedStructuredOutput, and
+  // for the identical reason: a long free-text field (e.g. a task description)
+  // typed with literal line breaks instead of `\n` is otherwise indistinguishable
+  // from genuinely malformed JSON, even though it's one mechanical fix away from
+  // valid. Returns the (possibly repaired) STRING to write to disk, never the
+  // parsed object — this function's contract is text recovery, not parsing.
+  const tryParse = candidate => {
+    try { JSON.parse(candidate); return candidate; } catch (e) {}
+    const repaired = _repairUnescapedControlCharsInStrings(candidate);
+    try { JSON.parse(repaired); return repaired; } catch (e) { return null; }
+  };
   const whole = tryParse(text.trim());
   if (whole != null) return whole;
   const fenceMatches = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dtlabs/ai-toolkit",
-  "version": "0.17.3",
+  "version": "0.17.5",
   "description": "AI Toolkit — reusable agents and procedures for developers to build AI-powered applications.",
   "files": [
     "docs/",

package/src/claude/workflows/pm-phase1.js

@@ -11,6 +11,24 @@ const force = args.includes('--force')
 return c.run(meta.name, featureDir, prefix, async () => {
   const file = suffix => `${featureDir}/${prefix}-${suffix}.md`
   const fresh = (output, inputs = [featurePath]) => c.exists(output) && c.read(output).trim().length > 0 && inputs.every(input => c.exists(input) && c.mtime(output) >= c.mtime(input))
+  // A long single-turn response (a feature complex enough to need many Use
+  // Cases/sections) can exceed the model's configured output-token ceiling
+  // mid-reply — confirmed against a real run where the CLI reported a clean
+  // exit (exitCode 0, is_error false, stop_reason "end_turn") with nothing in
+  // the response envelope signaling that anything was cut off. Only the
+  // document's own expected structure can catch this. Both Requirements (9
+  // top-level sections) and Tech-Spec (12) have a fixed, numbered skeleton
+  // (see their own agent definitions' "Output Structure"), so a missing
+  // "## N. " heading is a reliable, deterministic sign of truncation.
+  // Checked both right after generation (fail fast with an attributable
+  // error, not a silently-accepted half-document) and as part of the
+  // freshness test (so a previously truncated file left on disk from an
+  // earlier failed run is never mistaken for "fresh enough" on a later resume).
+  const missingSections = (content, count) => Array.from({ length: count }, (_, i) => i + 1).filter(n => !new RegExp(`^## ${n}\\.\\s`, 'm').test(content))
+  const assertCompleteSections = (label, content, count) => {
+    const missing = missingSections(content, count)
+    if (missing.length) throw new Error(`${label} output is incomplete: missing section(s) ${missing.join(', ')} of ${count} — likely truncated mid-generation (a model output-length ceiling was probably exceeded)`)
+  }
   const reqId = 'gaia.agent.planner.requirements'
   const specId = 'gaia.agent.planner.tech-spec'
   const valId = 'gaia.agent.planner.validate-feature-docs'
@@ -18,14 +36,16 @@ return c.run(meta.name, featureDir, prefix, async () => {
   const requirements = () => c.worker(reqId, featurePath, { outputs: [output('Requirements')] })
   const techSpec = () => c.worker(specId, featurePath, { outputs: [output('Tech-Spec', [featurePath, file('Requirements')])] })
   let changed = false
-  if (force || !fresh(file('Requirements'))) {
+  if (force || !fresh(file('Requirements')) || missingSections(c.read(file('Requirements')), 9).length) {
     await requirements()
     if (!c.exists(file('Requirements'))) throw new Error('Requirements output missing')
+    assertCompleteSections('Requirements', c.read(file('Requirements')), 9)
     changed = true
   }
-  if (force || changed || !fresh(file('Tech-Spec'), [featurePath, file('Requirements')])) {
+  if (force || changed || !fresh(file('Tech-Spec'), [featurePath, file('Requirements')]) || missingSections(c.read(file('Tech-Spec')), 12).length) {
     await techSpec()
     if (!c.exists(file('Tech-Spec'))) throw new Error('Tech-Spec output missing')
+    assertCompleteSections('Tech-Spec', c.read(file('Tech-Spec')), 12)
     changed = true
   }
   let validationSummary = 'skipped (fresh)'