@bastani/atomic 0.5.17 → 0.5.18-0

package/src/sdk/workflows/builtin/deep-research-codebase/helpers/scratch.ts

@@ -0,0 +1,115 @@
+/**
+ * Deterministic synthesis of per-partition explorer scratch files.
+ *
+ * Each partition is investigated by four specialist sub-agents dispatched
+ * directly via the provider SDK's `agent` parameter:
+ *
+ *   - codebase-locator           → file index for the partition
+ *   - codebase-pattern-finder    → reusable code patterns in the partition
+ *   - codebase-analyzer          → how the most relevant impl files work
+ *   - codebase-online-researcher → external library docs (when central)
+ *
+ * Rather than spawn a fifth "synthesizer" LLM stage just to concatenate four
+ * markdown sections, we do that synthesis in plain TypeScript here. This keeps
+ * the per-partition cost at exactly four LLM calls and avoids burning tokens
+ * on a step whose output is fully determined by its inputs.
+ *
+ * The file we write is the canonical handoff to the aggregator — it MUST keep
+ * the heading shape that buildAggregatorPrompt() promises ("Scope / Files in
+ * Scope / How It Works / Patterns / External References / Out-of-Partition
+ * References"), or the aggregator will look for sections that don't exist.
+ */
+
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { PartitionUnit } from "./scout.ts";
+
+export type ExplorerSections = {
+  index: number;
+  total: number;
+  partition: PartitionUnit[];
+  /** Full assistant text from the codebase-locator sub-agent. */
+  locatorOutput: string;
+  /** Full assistant text from the codebase-pattern-finder sub-agent. */
+  patternsOutput: string;
+  /** Full assistant text from the codebase-analyzer sub-agent. */
+  analyzerOutput: string;
+  /** Full assistant text from the codebase-online-researcher sub-agent. */
+  onlineOutput: string;
+};
+
+/** Heuristic: detect the "no external research applicable" sentinel. */
+function isOnlineSkip(output: string): boolean {
+  return /\(\s*no external research applicable\s*\)/i.test(output);
+}
+
+/** Render the markdown body deterministically. */
+export function renderExplorerMarkdown(sections: ExplorerSections): string {
+  const scope = sections.partition
+    .map(
+      (u) =>
+        `\`${u.path}/\` (${u.fileCount} files, ${u.loc.toLocaleString()} LOC)`,
+    )
+    .join(", ");
+
+  const lines: string[] = [
+    `# Partition ${sections.index} of ${sections.total} — Findings`,
+    ``,
+    `## Scope`,
+    scope,
+    ``,
+    `## Files in Scope`,
+    `<!-- Source: codebase-locator sub-agent -->`,
+    sections.locatorOutput.trim() || "_(no files located)_",
+    ``,
+    `## How It Works`,
+    `<!-- Source: codebase-analyzer sub-agent -->`,
+    sections.analyzerOutput.trim() || "_(no analysis produced)_",
+    ``,
+    `## Patterns`,
+    `<!-- Source: codebase-pattern-finder sub-agent -->`,
+    sections.patternsOutput.trim() || "_(no patterns surfaced)_",
+    ``,
+  ];
+
+  // Only include the External References section when the online researcher
+  // actually returned external findings — its skip sentinel would otherwise
+  // pollute the aggregator's view of "evidence collected".
+  if (
+    sections.onlineOutput.trim().length > 0 &&
+    !isOnlineSkip(sections.onlineOutput)
+  ) {
+    lines.push(
+      `## External References`,
+      `<!-- Source: codebase-online-researcher sub-agent -->`,
+      sections.onlineOutput.trim(),
+      ``,
+    );
+  }
+
+  // Out-of-partition references live in the analyzer output already, but we
+  // surface a brief pointer for the aggregator's cross-stitching pass.
+  lines.push(
+    `## Out-of-Partition References`,
+    `Look for the **Out-of-Partition References** subsection inside the`,
+    `"How It Works" section above — that is where the analyzer flagged files`,
+    `outside this partition that other partitions should examine.`,
+    ``,
+  );
+
+  return lines.join("\n");
+}
+
+/**
+ * Write a partition's deterministic scratch file. Returns the absolute path so
+ * the caller can record it in the explorer manifest the aggregator reads.
+ */
+export async function writeExplorerScratchFile(
+  scratchPath: string,
+  sections: ExplorerSections,
+): Promise<string> {
+  const abs = path.resolve(scratchPath);
+  const md = renderExplorerMarkdown(sections);
+  await writeFile(abs, md, "utf8");
+  return abs;
+}

package/src/sdk/workflows/builtin/deep-research-codebase/opencode/index.ts

@@ -1,51 +1,29 @@
 /**
  * deep-research-codebase / opencode
  *
- * OpenCode replica of the Claude deep-research-codebase workflow. The Claude
- * version dispatches specialist sub-agents (codebase-locator, codebase-
- * analyzer, etc.) inside a single explorer session via `@"name (agent)"`
- * syntax — a Claude-specific feature. OpenCode sessions are bound to a
- * single agent for their lifetime, so we keep the SAME graph topology
- * (scout ∥ history → explorer-1..N → aggregator) but drive each explorer
- * through the locate → analyze → patterns → synthesize sequence inline using
- * the default agent's built-in file tools.
+ * OpenCode replica of the Claude deep-research-codebase workflow. Specialist
+ * sub-agents are dispatched as separate headless `ctx.stage()` calls — each
+ * call passes `agent: "<name>"` to `s.client.session.prompt()` directly,
+ * which is OpenCode's SDK-native way to route a turn to a sub-agent.
  *
- * Topology (identical to Claude version):
+ * OpenCode-specific concerns baked in (see references/failure-modes.md):
  *
- *           ┌─→ codebase-scout
- *   parent ─┤
- *           └─→ research-history
- *                     │
- *                     ▼
- *   ┌──────────────────────────────────────────────────┐
- *   │  explorer-1   explorer-2   ...   explorer-N      │   (Promise.all, headless)
- *   └──────────────────────────────────────────────────┘
- *                     │
- *                     ▼
- *                aggregator
+ *   • F5 — every `ctx.stage()` is a FRESH session. Each specialist receives
+ *     everything it needs (research question, scope, scout overview, and —
+ *     for layer-2 specialists — verbatim locator output) in its first prompt.
  *
- * Explorers run headless (in-process, no tmux window) — they are transparent
- * to the graph, so the visible topology is: [scout, history] → aggregator.
+ *   • F3 — `result.data!.parts` is a heterogenous array (text/tool/reasoning/
+ *     file parts). Use `extractResponseText()` to filter to text parts only;
+ *     concatenating raw `parts` produces `[object Object]` strings.
  *
- * OpenCode-specific concerns baked in:
+ *   • F6 — every prompt explicitly requires trailing prose so transcripts and
+ *     `extractResponseText()` reads are never empty.
  *
- *  • F5 — every `ctx.stage()` call is a FRESH session with no memory of
- *    prior stages. We forward the scout overview, history overview, and
- *    partition assignment explicitly into each explorer's first prompt.
+ *   • F9 — `s.save()` receives the unwrapped `{ info, parts }` payload from
+ *     `result.data!`; passing the full `result` or raw `result.data!.parts`
+ *     breaks downstream `transcript()` reads.
  *
- *  • F9 — `s.save()` receives the `{ info, parts }` payload from
- *    `s.client.session.prompt()` via `result.data!`. Passing the full
- *    `result` (with its wrapping) or raw `result.data.parts` breaks
- *    downstream `transcript()` reads.
- *
- *  • F6 — every prompt explicitly requires trailing prose AFTER any tool
- *    call so the rendered transcript has content. OpenCode's `parts` array
- *    mixes text/tool/reasoning/file parts; without trailing text the
- *    transcript extractor returns an empty string.
- *
- *  • F3 — transcript extraction relies on the runtime's text-only rendering
- *    of `result.data.parts`. The helpers call `ctx.transcript(handle)` which
- *    returns `{ path, content }` where content is already text-filtered.
+ * See claude/index.ts for the full design rationale and topology diagram.
  */
 
 import { defineWorkflow } from "../../../index.ts";
@@ -63,42 +41,59 @@ import {
 } from "../helpers/heuristic.ts";
 import {
   buildAggregatorPrompt,
-  buildExplorerPromptGeneric,
-  buildHistoryPromptGeneric,
+  buildAnalyzerPrompt,
+  buildHistoryAnalyzerPrompt,
+  buildHistoryLocatorPrompt,
+  buildLocatorPrompt,
+  buildOnlineResearcherPrompt,
+  buildPatternFinderPrompt,
   buildScoutPrompt,
   slugifyPrompt,
 } from "../helpers/prompts.ts";
+import { writeExplorerScratchFile } from "../helpers/scratch.ts";
+
+/** Filter for text parts only — non-text parts produce [object Object]. */
+function extractResponseText(
+  parts: Array<{ type: string; [key: string]: unknown }>,
+): string {
+  return parts
+    .filter((p) => p.type === "text")
+    .map((p) => (p as { type: string; text: string }).text)
+    .join("\n");
+}
 
 export default defineWorkflow({
-    name: "deep-research-codebase",
-    description:
-      "Deterministic deep codebase research: scout → LOC-driven parallel explorers → aggregator",
-    inputs: [
-      { name: "prompt", type: "text", required: true, description: "research question" },
-    ],
-  })
+  name: "deep-research-codebase",
+  description:
+    "Deterministic deep codebase research: scout → per-partition specialist sub-agents → aggregator",
+  inputs: [
+    {
+      name: "prompt",
+      type: "text",
+      required: true,
+      description: "research question",
+    },
+  ],
+})
   .for<"opencode">()
   .run(async (ctx) => {
-    // Destructure once so every stage below can close over a bare
-    // `inputs.prompt`; destructure once so every stage below can close
-    // over a bare `prompt` string without re-reaching into ctx.inputs.
     const prompt = ctx.inputs.prompt ?? "";
     const root = getCodebaseRoot();
     const startedAt = new Date();
     const isoDate = startedAt.toISOString().slice(0, 10);
     const slug = slugifyPrompt(prompt);
 
-    // ── Stages 1a + 1b: codebase-scout ∥ research-history ──────────────────
-    const [scout, history] = await Promise.all([
+    // ── Stage 1a: codebase-scout ‖ Stage 1b: research-history pipeline ────
+    const [scout, historyOverview] = await Promise.all([
       ctx.stage(
         {
           name: "codebase-scout",
-          description: "Map codebase, count LOC, partition for parallel explorers",
+          description:
+            "Map codebase, count LOC, partition for parallel specialists",
         },
         {},
         { title: "codebase-scout" },
         async (s) => {
-          // 1. Deterministic scouting (pure TypeScript — no LLM).
           const data = scoutCodebase(root);
           if (data.units.length === 0) {
             throw new Error(
@@ -107,13 +102,10 @@ export default defineWorkflow({
             );
           }
 
-          // 2. Heuristic decides explorer count (capped by available units).
           const targetCount = calculateExplorerCount(data.totalLoc);
           const partitions = partitionUnits(data.units, targetCount);
           const actualCount = partitions.length;
 
-          // 3. Scratch directory for explorer outputs (timestamped to avoid
-          //    collisions across runs).
           const scratchDir = path.join(
             root,
             "research",
@@ -122,9 +114,6 @@ export default defineWorkflow({
           );
           await mkdir(scratchDir, { recursive: true });
 
-          // 4. Short LLM call: architectural orientation for downstream
-          //    explorers. The prompt forbids the agent from answering the
-          //    research question — its only job here is to orient.
           const result = await s.client.session.prompt({
             sessionID: s.session.id,
             parts: [
@@ -156,103 +145,225 @@ export default defineWorkflow({
           };
         },
       ),
-      ctx.stage(
-        {
-          name: "research-history",
-          description: "Surface prior research from research/ directory",
-        },
-        {},
-        { title: "research-history" },
-        async (s) => {
-          // The generic history prompt drives a single default-agent session
-          // through locate → analyze → synthesize inline, instead of Claude's
-          // sub-agent dispatch.
-          const result = await s.client.session.prompt({
-            sessionID: s.session.id,
-            parts: [
-              {
-                type: "text",
-                text: buildHistoryPromptGeneric({
-                  question: prompt,
-                  root,
-                }),
-              },
-            ],
-          });
-          s.save(result.data!);
-        },
-      ),
-    ]);
-
-    const {
-      partitions,
-      explorerCount,
-      scratchDir,
-      totalLoc,
-      totalFiles,
-    } = scout.result;
-
-    // Pull both scout transcripts ONCE at the workflow level so every
-    // explorer + the aggregator can embed them in their prompts (F5). Both
-    // stages have completed here (we're past Promise.all), so these reads
-    // are safe (F13).
-    const scoutOverview = (await ctx.transcript(scout)).content;
-    const historyOverview = (await ctx.transcript(history)).content;
-
-    // ── Stage 2: parallel headless explorers ─────────────────────────────────
-    // Each explorer runs headless (in-process, no tmux pane) via Promise.all.
-    // They are invisible in the workflow graph but tracked by the background
-    // task counter in the statusline. Because each session is fresh (F5),
-    // every piece of context it needs — question, architectural orientation,
-    // historical context, partition assignment, scratch path — is injected
-    // into the first prompt via buildExplorerPromptGeneric.
-    const explorerHandles = await Promise.all(
-      partitions.map((partition, idx) => {
-        const i = idx + 1;
-        const scratchPath = path.join(scratchDir, `explorer-${i}.md`);
-        return ctx.stage(
+      // research-history pipeline: sequential locator → analyzer, both headless.
+      (async (): Promise<string> => {
+        const historyLocator = await ctx.stage(
           {
-            name: `explorer-${i}`,
+            name: "history-locator",
             headless: true,
-            description: `Explore ${partition
-              .map((u) => u.path)
-              .join(", ")} (${partition.reduce((s, u) => s + u.fileCount, 0)} files)`,
+            description: "Locate prior research docs (codebase-research-locator)",
           },
           {},
-          { title: `explorer-${i}` },
+          { title: "history-locator" },
           async (s) => {
             const result = await s.client.session.prompt({
               sessionID: s.session.id,
               parts: [
                 {
                   type: "text",
-                  text: buildExplorerPromptGeneric({
+                  text: buildHistoryLocatorPrompt({
                     question: prompt,
-                    index: i,
-                    total: explorerCount,
-                    partition,
-                    scoutOverview,
-                    historyOverview,
-                    scratchPath,
                     root,
                   }),
                 },
               ],
+              agent: "codebase-research-locator",
             });
             s.save(result.data!);
+            return extractResponseText(result.data!.parts);
+          },
+        );
 
-            // Returning structured metadata lets the aggregator stage reach
-            // each explorer's scratch path without re-parsing transcripts.
-            return { index: i, scratchPath, partition };
+        const historyAnalyzer = await ctx.stage(
+          {
+            name: "history-analyzer",
+            headless: true,
+            description: "Synthesize prior research (codebase-research-analyzer)",
+          },
+          {},
+          { title: "history-analyzer" },
+          async (s) => {
+            const result = await s.client.session.prompt({
+              sessionID: s.session.id,
+              parts: [
+                {
+                  type: "text",
+                  text: buildHistoryAnalyzerPrompt({
+                    question: prompt,
+                    locatorOutput: historyLocator.result,
+                    root,
+                  }),
+                },
+              ],
+              agent: "codebase-research-analyzer",
+            });
+            s.save(result.data!);
+            return extractResponseText(result.data!.parts);
           },
         );
+
+        return historyAnalyzer.result;
+      })(),
+    ]);
+
+    const { partitions, explorerCount, scratchDir, totalLoc, totalFiles } =
+      scout.result;
+
+    const scoutOverview = (await ctx.transcript(scout)).content;
+
+    // ── Stage 2: per-partition specialist fan-out ─────────────────────────
+    const explorerHandles = await Promise.all(
+      partitions.map(async (partition, idx) => {
+        const i = idx + 1;
+        const scratchPath = path.join(scratchDir, `explorer-${i}.md`);
+
+        // Layer 1: locator + pattern-finder run independently.
+        const [locator, patternFinder] = await Promise.all([
+          ctx.stage(
+            {
+              name: `locator-${i}`,
+              headless: true,
+              description: `codebase-locator over partition ${i}`,
+            },
+            {},
+            { title: `locator-${i}` },
+            async (s) => {
+              const result = await s.client.session.prompt({
+                sessionID: s.session.id,
+                parts: [
+                  {
+                    type: "text",
+                    text: buildLocatorPrompt({
+                      question: prompt,
+                      partition,
+                      root,
+                      scoutOverview,
+                      index: i,
+                      total: explorerCount,
+                    }),
+                  },
+                ],
+                agent: "codebase-locator",
+              });
+              s.save(result.data!);
+              return extractResponseText(result.data!.parts);
+            },
+          ),
+          ctx.stage(
+            {
+              name: `pattern-finder-${i}`,
+              headless: true,
+              description: `codebase-pattern-finder over partition ${i}`,
+            },
+            {},
+            { title: `pattern-finder-${i}` },
+            async (s) => {
+              const result = await s.client.session.prompt({
+                sessionID: s.session.id,
+                parts: [
+                  {
+                    type: "text",
+                    text: buildPatternFinderPrompt({
+                      question: prompt,
+                      partition,
+                      root,
+                      scoutOverview,
+                      index: i,
+                      total: explorerCount,
+                    }),
+                  },
+                ],
+                agent: "codebase-pattern-finder",
+              });
+              s.save(result.data!);
+              return extractResponseText(result.data!.parts);
+            },
+          ),
+        ]);
+
+        const locatorOutput = locator.result;
+        const patternsOutput = patternFinder.result;
+
+        // Layer 2: analyzer + online-researcher consume locator output.
+        const [analyzer, onlineResearcher] = await Promise.all([
+          ctx.stage(
+            {
+              name: `analyzer-${i}`,
+              headless: true,
+              description: `codebase-analyzer over partition ${i}`,
+            },
+            {},
+            { title: `analyzer-${i}` },
+            async (s) => {
+              const result = await s.client.session.prompt({
+                sessionID: s.session.id,
+                parts: [
+                  {
+                    type: "text",
+                    text: buildAnalyzerPrompt({
+                      question: prompt,
+                      partition,
+                      locatorOutput,
+                      root,
+                      scoutOverview,
+                      index: i,
+                      total: explorerCount,
+                    }),
+                  },
+                ],
+                agent: "codebase-analyzer",
+              });
+              s.save(result.data!);
+              return extractResponseText(result.data!.parts);
+            },
+          ),
+          ctx.stage(
+            {
+              name: `online-researcher-${i}`,
+              headless: true,
+              description: `codebase-online-researcher over partition ${i}`,
+            },
+            {},
+            { title: `online-researcher-${i}` },
+            async (s) => {
+              const result = await s.client.session.prompt({
+                sessionID: s.session.id,
+                parts: [
+                  {
+                    type: "text",
+                    text: buildOnlineResearcherPrompt({
+                      question: prompt,
+                      partition,
+                      locatorOutput,
+                      root,
+                      index: i,
+                      total: explorerCount,
+                    }),
+                  },
+                ],
+                agent: "codebase-online-researcher",
+              });
+              s.save(result.data!);
+              return extractResponseText(result.data!.parts);
+            },
+          ),
+        ]);
+
+        await writeExplorerScratchFile(scratchPath, {
+          index: i,
+          total: explorerCount,
+          partition,
+          locatorOutput,
+          patternsOutput,
+          analyzerOutput: analyzer.result,
+          onlineOutput: onlineResearcher.result,
+        });
+
+        return { index: i, scratchPath, partition };
       }),
     );
 
-    // ── Stage 3: aggregator ────────────────────────────────────────────────
-    // Reads explorer findings via FILE PATHS (filesystem-context skill) to
-    // keep the aggregator's own context lean — we deliberately do NOT inline
-    // N transcripts into the prompt. Token cost stays roughly constant in N.
+    // ── Stage 3: aggregator ───────────────────────────────────────────────
     const finalPath = path.join(
       root,
       "research",
@@ -263,7 +374,8 @@ export default defineWorkflow({
     await ctx.stage(
       {
         name: "aggregator",
-        description: "Synthesize explorer findings + history into final research doc",
+        description:
+          "Synthesize partition findings + history into final research doc",
       },
       {},
       { title: "aggregator" },
@@ -278,7 +390,7 @@ export default defineWorkflow({
                 totalLoc,
                 totalFiles,
                 explorerCount,
-                explorerFiles: explorerHandles.map((h) => h.result),
+                explorerFiles: explorerHandles,
                 finalPath,
                 scoutOverview,
                 historyOverview,