@dtlabs/ai-toolkit 0.19.1 → 0.20.0

package/CHANGELOG.md

@@ -1,4 +1,20 @@
 ## 0.1.4 (2026-07-21)
+## [0.20.0](https://github.com/dtlab-tech/ai-toolkit/compare/v0.19.2...v0.20.0) (2026-10-19)
+
+
+### Features
+
+* **pm-phase1, pm-phase2:** resume-safe per-item generation (FTR-027) ([41e021f](https://github.com/dtlab-tech/ai-toolkit/commit/41e021f65aad694c18d51b6a7876e4a83373a4d9))
+* resume-safe per-item generation + component fileSlug normalization (FTR-027) ([e7e6b89](https://github.com/dtlab-tech/ai-toolkit/commit/e7e6b89596f267c4954c9be92bc2c4b3de8b9194))
+
+## [0.19.2](https://github.com/dtlab-tech/ai-toolkit/compare/v0.19.1...v0.19.2) (2026-10-19)
+
+
+### Bug Fixes
+
+* **workflow-control, tech-spec-assembler:** unwrap whole response fences in Markdown documents to prevent content loss ([fe88582](https://github.com/dtlab-tech/ai-toolkit/commit/fe885821bb7b813237a1d53ed827c70e09b6c9d0))
+* **workflow-control, tech-spec-assembler:** unwrap whole response fences in Markdown documents to prevent content loss ([358e484](https://github.com/dtlab-tech/ai-toolkit/commit/358e4847fe74c314c3c1158dbeda08669530c1f8))
+
 ## [0.19.1](https://github.com/dtlab-tech/ai-toolkit/compare/v0.19.0...v0.19.1) (2026-10-19)
 
 

package/lib/workflow-control.js

@@ -238,6 +238,13 @@ function createControl(options) {
         const candidate = extractDelimitedStructuredOutput(result.result.result, opts.schema);
         if (candidate != null) { payload = candidate; usedTextFallback = true; }
       }
+      // A non-schema worker's payload IS the document — unwrap a whole-response fence/preamble
+      // before anything downstream (the caller's own write, or this function's own outputTarget
+      // write below) ever sees it. See extractMarkdownDocument's own comment for the full story.
+      if (!opts.schema && typeof payload === 'string') {
+        const cleaned = extractMarkdownDocument(payload);
+        if (cleaned !== payload) { payload = cleaned; usedTextFallback = true; }
+      }
       if (payload == null || (typeof payload === 'string' && !payload.trim())) {
         if (opts.schema) {
           const fields = (opts.schema.required || []).join(', ') || '(see requested schema)';
@@ -476,6 +483,65 @@ function extractJsonDocument(text) {
   return text;
 }
 
+// Same problem extractJsonDocument fixes, for a Markdown document deliverable (a component/
+// use-case/task detail file, or a Requirements/Tech-Spec skeleton) instead of a JSON one: a
+// worker's entire response becomes the file verbatim (these agents have no Write tool, see
+// worker()'s own comment on why — the host writes whatever text they return), and a model that
+// wraps its deliverable in conversational prose and/or an outer code fence silently corrupts the
+// saved file. Found empirically in a real spec/components/*.md file: the agent's own real
+// "## Change"/"## Tests" headings ended up INSIDE an outer ```markdown fence and were (correctly)
+// treated as fenced, non-heading content by tech-spec-assembler.js's own fence-aware parser
+// (FTR-025) — the file wrote to disk with its real content unreachable, surfacing only much later
+// as an opaque "component file is missing its ## Change section" error during final assembly, with
+// no filename attached to say which one (see loadTechSpec's own fix for that half of the bug).
+//
+// Same "try as-is first, re-validate the candidate, never a blind substring guess, never throws"
+// discipline as extractJsonDocument: "valid" here means "has at least one unfenced Markdown
+// heading" — the same fence-length-aware scan tech-spec-assembler.js's own
+// _nonFencedHeadingIndexes uses, reimplemented here as a presence check rather than a line-index
+// list, for the identical "sandboxed runtime has no require()"-shaped reason these two files are
+// already siblings, not shared, elsewhere in this codebase. A document that legitimately ends in
+// its own small fenced code example (a real Tests section's last line) is never mistaken for a
+// whole-response wrap: its real headings already appear BEFORE that trailing fence, unfenced, so
+// the as-is candidate already satisfies this check and no unwrapping is even attempted.
+function _hasUnfencedHeading(text) {
+  let fenceLen = 0;
+  for (const line of text.split('\n')) {
+    const m = line.match(/^(`{3,})/);
+    if (m) { const n = m[1].length; fenceLen = fenceLen === 0 ? n : (n >= fenceLen ? 0 : fenceLen); continue; }
+    if (fenceLen === 0 && /^#{1,6}\s/.test(line)) return true;
+  }
+  return false;
+}
+// Line-range based, not regex-based: a regex scan for the "first ``` ... last ```" span would
+// wrongly stop at the first NESTED closing fence instead of the real outer one (exactly the shape
+// a Markdown document's own "## Data Model" section legitimately needs — a fenced JSON example
+// nested inside the model's outer ```markdown wrap). Only the LAST non-blank line being a bare
+// fence-close, matched back to the first fence-open of the SAME backtick-run length, correctly
+// identifies the true outer boundary regardless of what's nested between them.
+function _unwrapWholeResponseFence(text) {
+  const lines = text.split('\n');
+  let end = lines.length;
+  while (end > 0 && lines[end - 1].trim() === '') end--;
+  if (end === 0) return null;
+  const closeMatch = lines[end - 1].match(/^(`{3,})\s*$/);
+  if (!closeMatch) return null;
+  const fenceLen = closeMatch[1].length;
+  let start = -1;
+  for (let i = 0; i < end - 1; i++) {
+    const openMatch = lines[i].match(/^(`{3,})\S*\s*$/);
+    if (openMatch && openMatch[1].length === fenceLen) { start = i; break; }
+  }
+  if (start === -1) return null;
+  return lines.slice(start + 1, end - 1).join('\n').trim() + '\n';
+}
+function extractMarkdownDocument(text) {
+  if (_hasUnfencedHeading(text)) return text;
+  const unwrapped = _unwrapWholeResponseFence(text);
+  if (unwrapped != null && _hasUnfencedHeading(unwrapped)) return unwrapped;
+  return text;
+}
+
 function validateSchema(value, schema) {
   if (schema.type === 'object') {
     if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid structured object');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dtlabs/ai-toolkit",
-  "version": "0.19.1",
+  "version": "0.20.0",
   "description": "AI Toolkit — reusable agents and procedures for developers to build AI-powered applications.",
   "files": [
     "docs/",

package/src/claude/scripts/tech-spec-assembler.js

@@ -148,7 +148,18 @@ function loadTechSpec(inputPath) {
   for (const c of components) {
     const compPath = path.join(inputPath, 'components', c.fileSlug)
     if (!fs.existsSync(compPath)) throw new AssemblyError(`component "${c.filePath}" has no detail file at ${compPath}`)
-    detailByPath[c.filePath] = parseComponentDetail(fs.readFileSync(compPath, 'utf8'))
+    try {
+      detailByPath[c.filePath] = parseComponentDetail(fs.readFileSync(compPath, 'utf8'))
+    } catch (err) {
+      // parseComponentDetail's own error names only the symptom ("missing its ## Change
+      // section"), never which file — a real diagnosability bug, found when exactly this error
+      // surfaced during final assembly with nothing identifying which of ~60 component files was
+      // actually malformed (the real culprit here was a whole-response-wrapped-in-a-fence worker
+      // reply — see lib/workflow-control.js's extractMarkdownDocument for the fix to that other
+      // half of this same incident). Re-thrown with the file identified, never silently swallowed.
+      if (err instanceof AssemblyError) throw new AssemblyError(`${err.message} (component "${c.filePath}" at ${compPath})`)
+      throw err
+    }
   }
 
   let doc = rewriteSection9(indexContent, components)

package/src/claude/workflows/pm-phase1.js

@@ -90,6 +90,16 @@ return c.run(meta.name, featureDir, prefix, async () => {
     })
   }
 
+  // FTR-027: resume-by-existence must not trust mere presence — a use-case file that exists but
+  // is garbled/off-topic (e.g. echoes a different UC) should be regenerated, not skipped forever.
+  // Only the ID fragment is checked, never the free-text title after the colon: the title is
+  // prose the agent may legitimately phrase differently across a retry, and comparing it would
+  // reject a valid regeneration for no real reason.
+  function isValidUseCaseFile(content, ucId) {
+    const escaped = ucId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+    return new RegExp(`^###\\s+${escaped}:\\s`).test(content.trimStart())
+  }
+
   // Business Rules' "Applies to" / Acceptance Criteria's "Related UC" columns are the same
   // cross-cutting-reference shape: a comma-separated UC-ID list, or the literal "All UCs". Both
   // are 100% mechanically derivable from req/index.md's own tables — no LLM judgment needed,
@@ -185,7 +195,7 @@ return c.run(meta.name, featureDir, prefix, async () => {
       // If the skeleton itself was just (re)generated, every use case must regenerate too —
       // an existing use-case file would have been written against the OLD skeleton's row
       // (title/actor/priority may have changed) and cannot be trusted as-is.
-      if (!regenSkeleton && c.exists(ucPath)) continue // resume: skip, no dispatch
+      if (!regenSkeleton && c.exists(ucPath) && isValidUseCaseFile(c.read(ucPath), row.id)) continue // resume: skip, no dispatch
       const detail = await c.worker(reqUcId, buildUcPrompt(row, ucTable, featurePath, ucTemplate),
         { label: `requirements-use-case:${row.id}` })
       c.writeOutput(ucPath, detail.trimEnd() + ucCrossReferences(c.read(reqIndexPath), row.id))
@@ -241,6 +251,36 @@ return c.run(meta.name, featureDir, prefix, async () => {
     return entries
   }
 
+  // FTR-027: duplicates tech-spec-assembler.js's parseComponentDetail's own fence-aware
+  // "## Change"/"## Tests" heading scan (see that file for the full "why length-aware, not a
+  // boolean toggle" rationale) — returns a boolean instead of throwing/returning parsed sections,
+  // since every call site here only needs the yes/no "is this a structurally valid component
+  // file" signal, for resume-by-existence: a component file that exists but fails this check
+  // (e.g. a whole-response fence wrap, a truncated reply) must be regenerated, not skipped
+  // forever — found via a real incident where exactly this silently blocked every retry.
+  function _nonFencedHeadingIndexesLocal(lines, predicate) {
+    let fenceLen = 0
+    const indexes = []
+    for (let i = 0; i < lines.length; i++) {
+      const m = lines[i].match(/^(`{3,})/)
+      if (m) {
+        const runLen = m[1].length
+        if (fenceLen === 0) fenceLen = runLen
+        else if (runLen >= fenceLen) fenceLen = 0
+        continue
+      }
+      if (fenceLen !== 0) continue
+      if (predicate(lines[i])) indexes.push(i)
+    }
+    return indexes
+  }
+  function isValidComponentDetail(content) {
+    const lines = content.split('\n')
+    const changeIdx = _nonFencedHeadingIndexesLocal(lines, l => l.trim() === '## Change')[0] ?? -1
+    const testsIdx = _nonFencedHeadingIndexesLocal(lines, l => l.trim() === '## Tests')[0] ?? -1
+    return changeIdx !== -1 && testsIdx !== -1
+  }
+
   function buildComponentPrompt(row, components, featurePathArg, componentTemplate) {
     const compactIndex = components.map(c => `${c.filePath} (${c.domain})`).join('\n')
     const base = `${featurePathArg}\n\nGenerate the full technical design for this component:\n` +
@@ -254,6 +294,16 @@ return c.run(meta.name, featureDir, prefix, async () => {
   // FTR-020). Rewrites any colliding fileSlug's Spec-column link in place, right-to-left
   // prepending path segments until unique.
   //
+  // Also enforces the ".md" suffix itself, never trusting the model to have included it. The
+  // component template's own contract calls this Spec link "provisional, not final" specifically
+  // so the host can rewrite it — originally only exercised for collisions, but a real production
+  // run showed the model omits ".md" entirely a meaningful fraction of the time (confirmed: 20 of
+  // ~61 components in one real Tech-Spec skeleton), writing e.g. `components/Settings.tsx`
+  // instead of `components/Settings.tsx.md` — the host then faithfully wrote that component's
+  // Markdown CONTENT into a file literally named `Settings.tsx`. Normalized here before the
+  // collision check runs, so two rows that only differ by their original extension (already
+  // present in the Path column) still correctly collide-and-disambiguate on the normalized slug.
+  //
   // Operates per-LINE, not via a global string .replace() over the whole document — a real bug,
   // found by testing two colliding entries directly: when both rows share byte-identical Spec
   // text (the whole point of a collision), a global .replace() always rewrites the FIRST
@@ -270,6 +320,12 @@ return c.run(meta.name, featureDir, prefix, async () => {
       const m = cells[4].match(/\(components\/([^)]+)\)/)
       if (!m) continue // header/separator row, or a row with no valid Spec link yet
       let slug = m[1]
+      if (!slug.endsWith('.md')) {
+        const escaped = slug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+        const normalized = `${slug}.md`
+        lines[i] = lines[i].replace(new RegExp(`components/${escaped}`, 'g'), `components/${normalized}`)
+        slug = normalized
+      }
       if (seenSlugs.has(slug)) {
         const segments = filePath.split('/')
         let depth = 1
@@ -279,7 +335,7 @@ return c.run(meta.name, featureDir, prefix, async () => {
           candidate = segments.slice(-depth).join('-') + '.md'
         }
         const escaped = slug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-        lines[i] = line.replace(new RegExp(`components/${escaped}`, 'g'), `components/${candidate}`)
+        lines[i] = lines[i].replace(new RegExp(`components/${escaped}`, 'g'), `components/${candidate}`)
         slug = candidate
       }
       seenSlugs.add(slug)
@@ -321,7 +377,7 @@ return c.run(meta.name, featureDir, prefix, async () => {
     const componentTemplate = components.length ? readTemplate('gaia-component-template.md') : null
     for (const comp of components) {
       const compPath = `${specDir}/components/${comp.fileSlug}`
-      if (!regenSkeleton && c.exists(compPath)) continue // resume: skip, no dispatch
+      if (!regenSkeleton && c.exists(compPath) && isValidComponentDetail(c.read(compPath))) continue // resume: skip, no dispatch
       const detail = await c.worker(specComponentId, buildComponentPrompt(comp, components, featurePath, componentTemplate),
         { label: `tech-spec-component:${comp.fileSlug}` })
       c.writeOutput(compPath, detail)
@@ -352,8 +408,17 @@ return c.run(meta.name, featureDir, prefix, async () => {
   // for free by gating on the assembled document's own freshness (which could never look
   // "complete" while a referenced item was missing) — removing that gate (FTR-025) removed this
   // side effect along with it, so it needs restating explicitly here.
-  const anyUcFileMissing = indexContent => parseUseCaseIndexTable(indexContent).some(row => !c.exists(`${reqDir}/${row.file}`))
-  const anyComponentFileMissing = indexContent => parseComponentsTable(indexContent).some(comp => !c.exists(`${specDir}/components/${comp.fileSlug}`))
+  // FTR-027: "missing" now also means "present but invalid" — a structurally broken-but-existing
+  // file must trigger re-entry into generation the same way an absent one already does, otherwise
+  // the outer gate alone could never notice it (only the per-item loop would, one phase later).
+  const anyUcFileMissing = indexContent => parseUseCaseIndexTable(indexContent).some(row => {
+    const ucPath = `${reqDir}/${row.file}`
+    return !c.exists(ucPath) || !isValidUseCaseFile(c.read(ucPath), row.id)
+  })
+  const anyComponentFileMissing = indexContent => parseComponentsTable(indexContent).some(comp => {
+    const compPath = `${specDir}/components/${comp.fileSlug}`
+    return !c.exists(compPath) || !isValidComponentDetail(c.read(compPath))
+  })
 
   let changed = false
   if (force || !fresh(reqIndexPath) || missingSections(c.read(reqIndexPath), 9).length ||

package/src/claude/workflows/pm-phase2.js

@@ -234,6 +234,22 @@ function buildTaskTechSpecExcerpt(components, task, specDir, specIndexContent) {
   }).join('\n\n')
 }
 
+// FTR-027: duplicates wbs-assembler.js's own _parseVerificationCommands algorithm (last
+// occurrence of the marker wins — an illustrative example earlier in Implementation notes must
+// never be mistaken for the real section) — returns a boolean instead of the parsed array, since
+// resume-by-status only needs the yes/no "does this task file have at least one verification
+// command" signal, per the task template's own documented "Must be non-empty" contract. A task
+// file that exists and is marked `generated` but has none is regenerated, not skipped forever.
+const FENCE_RE_LOCAL = /```[^\n]*\n([\s\S]*?)```/g
+function hasVerificationCommands(content) {
+  const marker = '**Verification commands:**'
+  const idx = content.lastIndexOf(marker)
+  if (idx === -1) return false
+  const verifPart = content.slice(idx + marker.length)
+  FENCE_RE_LOCAL.lastIndex = 0
+  return FENCE_RE_LOCAL.test(verifPart)
+}
+
 // Compact prompt for the per-task detail-generation worker: this task's own skeleton entry,
 // the full skeleton's sibling task list (id + title + outcome only, for duplicate-
 // implementation awareness), and the Tech-Spec excerpt (FTR-026: already pre-sliced to this
@@ -351,7 +367,7 @@ return c.run(meta.name, featureDir, prefix, async () => {
     for (const taskId of phase.taskIds) {
       const task = index.tasks[taskId]
       const detailPath = `${wbsDir}/${task.detailFile}`
-      if (task.status === 'generated' && c.exists(detailPath)) continue // resume: skip, no dispatch
+      if (task.status === 'generated' && c.exists(detailPath) && hasVerificationCommands(c.read(detailPath))) continue // resume: skip, no dispatch
       const techSpecExcerpt = hasSpec
         ? buildTaskTechSpecExcerpt(components, task, specDir, c.read(specIndexPath))
         : c.read(file('Tech-Spec.md')) // legacy fallback: unchanged, full text