@pruddiman/hem 0.0.1-beta-5671db0

package/dist/orchestrator.js

@@ -0,0 +1,1169 @@
+/**
+ * OpenCode server lifecycle management and documentation generation for Hem.
+ *
+ * Starts the OpenCode server via `createOpencode()` from `@opencode-ai/sdk`,
+ * configures model and permission settings, provides a clean shutdown
+ * mechanism, and orchestrates parallel documentation generation across all
+ * file groups using `p-limit` for concurrency control.
+ *
+ * Architecture (v2):
+ *   - Agents that produce documents write them directly via the edit tool.
+ *   - Agents that return structured data return strict JSON only.
+ *   - The doc agent has full autonomy over file naming, structure, and
+ *     how many files to create per group.
+ *   - Permissions are scoped: `edit: "allow"` + `external_directory`
+ *     restricted to the destination directory for writing agents.
+ *
+ * Reference: FR-005, FR-006, FR-007.
+ */
+import { readFile } from "node:fs/promises";
+import { resolve, join } from "node:path";
+import pLimit from "p-limit";
+import fg from "fast-glob";
+import { createOpencode } from "@opencode-ai/sdk";
+import { AuthExpiredError } from "./auth.js";
+import { findFreePort } from "./server-utils.js";
+import { computeMaxConcurrency, describeResourceLimits } from "./resources.js";
+import { OpenCodeProvider } from "./providers/index.js";
+// Re-export permission constants from provider module for backward compatibility.
+export { READ_ONLY_BASH, ORG_AGENT_BASH } from "./providers/index.js";
+// ── Multi-Agent Exploration Constants ────────────────────────────────
+/**
+ * Total file count threshold above which multi-agent exploration is activated.
+ * Below this threshold, the existing 1-agent-per-group behavior is used.
+ */
+export const LARGE_PROJECT_THRESHOLD = 200;
+/**
+ * Hard ceiling on exploration sub-agents per group.
+ * Actual count is computed by `computeAgentsPerGroup()`.
+ */
+export const MAX_EXPLORATION_AGENTS_PER_GROUP = 8;
+/**
+ * Minimum files per sub-agent. Prevents creating sub-agents for trivially
+ * small partitions.
+ */
+const MIN_FILES_PER_AGENT = 3;
+/**
+ * Compute the number of sub-agents to use per exploration group based on
+ * total file count. Returns 1 (no splitting) when below the threshold.
+ *
+ * Scaling: starts at 4 agents at the threshold, grows to MAX at 2x the threshold.
+ */
+export function computeAgentsPerGroup(totalFiles) {
+    if (totalFiles < LARGE_PROJECT_THRESHOLD)
+        return 1;
+    const ratio = totalFiles / LARGE_PROJECT_THRESHOLD;
+    // Scale from 4 to MAX_EXPLORATION_AGENTS_PER_GROUP as ratio goes from 1 to 2+
+    const agents = Math.min(MAX_EXPLORATION_AGENTS_PER_GROUP, Math.max(4, Math.round(4 * ratio)));
+    return agents;
+}
+/**
+ * Partition a group's files into N sub-groups for parallel exploration.
+ * Uses round-robin assignment (files sorted by path for determinism).
+ * Each sub-group is a new `FileGroup` with id `{group.id}:sub-{n}`.
+ *
+ * Returns the original group as a single-element array if agentCount is 1
+ * or the group has fewer files than MIN_FILES_PER_AGENT * agentCount.
+ */
+export function partitionGroupFiles(group, agentCount) {
+    if (agentCount <= 1 || group.files.length < MIN_FILES_PER_AGENT * 2) {
+        return [group];
+    }
+    const effectiveAgents = Math.min(agentCount, Math.floor(group.files.length / MIN_FILES_PER_AGENT));
+    if (effectiveAgents <= 1)
+        return [group];
+    const sorted = [...group.files].sort((a, b) => a.path.localeCompare(b.path));
+    const subGroups = [];
+    for (let i = 0; i < effectiveAgents; i++) {
+        subGroups.push({
+            id: `${group.id}:sub-${i + 1}`,
+            label: `${group.label} (part ${i + 1}/${effectiveAgents})`,
+            type: group.type,
+            files: [],
+            directory: group.directory,
+        });
+    }
+    for (let i = 0; i < sorted.length; i++) {
+        subGroups[i % effectiveAgents].files.push(sorted[i]);
+    }
+    return subGroups;
+}
+/**
+ * Merge exploration findings from multiple sub-agents that explored
+ * different partitions of the same group.
+ *
+ * Strategy:
+ *   - `groupId`: use the original group ID (not the sub-agent IDs)
+ *   - `questions`: concatenate, deduplicate by `question` text
+ *   - `integrations`: concatenate, deduplicate by `name`; merge `sourceLocations` and `operationalQuestions` for duplicates
+ *   - `complexitySignals`: concatenate, deduplicate by `description`
+ *   - `crossGroupDependencies`: merge into a Set (unique strings)
+ *   - `styleGuides`: concatenate, deduplicate by `filePath`
+ *   - `summary`: join all summaries with newline separator
+ */
+export function mergeExplorationFindings(groupId, findingsArray) {
+    if (findingsArray.length === 0) {
+        return { groupId, text: "" };
+    }
+    if (findingsArray.length === 1) {
+        return { ...findingsArray[0], groupId };
+    }
+    const text = findingsArray
+        .map((f) => f.text.trim())
+        .filter((t) => t.length > 0)
+        .join("\n\n");
+    return { groupId, text };
+}
+// ── Main ────────────────────────────────────────────────────────────────
+/**
+ * Starts the OpenCode server and returns a client with a shutdown function.
+ *
+ * Delegates to {@link OpenCodeProvider} for server lifecycle management,
+ * permission configuration, and MCP tool setup.
+ *
+ * @param model   - The resolved model selection (provider + model ID).
+ * @param options - Orchestrator options including the destination path.
+ * @param factory - Optional `createOpencode` override for testing.
+ * @param findPort - Optional port finder override for testing.
+ * @returns An object with `client` and `shutdown`.
+ * @throws If the OpenCode server fails to start.
+ * @throws If `destinationPath` is empty or blank.
+ */
+export async function createOrchestrator(model, options, factory = createOpencode, findPort = findFreePort) {
+    const provider = await OpenCodeProvider.create({
+        model,
+        destinationPath: options.destinationPath,
+    }, factory, findPort);
+    return {
+        client: provider,
+        shutdown: () => provider.cleanup(),
+    };
+}
+// ── Existing Docs Detection ────────────────────────────────────────────
+/**
+ * Scans the destination directory for existing `.md` files and reads
+ * their content. Used to provide merge context to the doc agent.
+ *
+ * @param destinationPath - Absolute path to the documentation destination.
+ * @param verbose         - Optional verbose logger.
+ * @returns An array of `{ path, content }` for each existing `.md` file.
+ */
+export async function scanExistingDocs(destinationPath, verbose) {
+    const absoluteDestination = resolve(destinationPath);
+    if (verbose) {
+        verbose(`[orchestrator] Scanning for existing docs in: ${absoluteDestination}`);
+    }
+    let mdFiles;
+    try {
+        mdFiles = await fg("**/*.md", {
+            cwd: absoluteDestination,
+            absolute: false,
+            onlyFiles: true,
+            dot: false,
+        });
+    }
+    catch {
+        // Destination directory doesn't exist yet — no existing docs.
+        if (verbose) {
+            verbose("[orchestrator] Destination directory does not exist; skipping existing docs scan");
+        }
+        return [];
+    }
+    if (mdFiles.length === 0) {
+        if (verbose) {
+            verbose("[orchestrator] No existing .md files found in destination");
+        }
+        return [];
+    }
+    if (verbose) {
+        verbose(`[orchestrator] Found ${mdFiles.length} existing .md file(s)`);
+    }
+    const existingDocs = [];
+    for (const relPath of mdFiles) {
+        const absPath = join(absoluteDestination, relPath);
+        try {
+            const content = await readFile(absPath, "utf-8");
+            existingDocs.push({ path: relPath, content });
+        }
+        catch (err) {
+            if (verbose) {
+                const msg = err instanceof Error ? err.message : String(err);
+                verbose(`[orchestrator] Could not read existing doc "${relPath}": ${msg}`);
+            }
+        }
+    }
+    if (verbose) {
+        verbose(`[orchestrator] Successfully read ${existingDocs.length}/${mdFiles.length} existing doc(s)`);
+    }
+    return existingDocs;
+}
+// ── Per-group doc relevance filtering ─────────────────────────────────
+/** Maximum number of existing docs to include with full content per group. */
+const MAX_RELEVANT_DOCS = 5;
+/**
+ * Extracts keywords from a string by splitting on non-alphanumeric characters,
+ * lowercasing, and filtering out short/common words.
+ */
+function extractKeywords(text) {
+    const STOP_WORDS = new Set([
+        "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for",
+        "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
+        "has", "have", "had", "do", "does", "did", "this", "that", "it", "its",
+        "not", "no", "so", "if", "as", "we", "our", "can", "will", "may",
+        "should", "would", "could", "each", "all", "any", "both",
+    ]);
+    const words = text
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((w) => w.length > 2 && !STOP_WORDS.has(w));
+    return new Set(words);
+}
+/**
+ * Scores a single existing doc's relevance to a file group.
+ *
+ * Signals (additive):
+ *  1. **Path overlap**: shared directory segments between the doc path and
+ *     the group's file paths / common directory.
+ *  2. **Label match**: words from the group label appearing in the doc's
+ *     first 300 characters (headings/title area).
+ *  3. **Topic match**: keywords from the group's exploration findings
+ *     (summary, question texts, integration names) appearing in doc content.
+ */
+function scoreDoc(doc, group, topicKeywords) {
+    let score = 0;
+    // 1. Path overlap — shared directory segments
+    const docSegments = doc.path.toLowerCase().replace(/\.md$/, "").split(/[/\\]+/);
+    const groupDir = group.directory.toLowerCase();
+    const groupDirSegments = groupDir.split(/[/\\]+/).filter(Boolean);
+    const groupFileSegments = group.files
+        .flatMap((f) => f.path.toLowerCase().split(/[/\\]+/))
+        .filter(Boolean);
+    const pathPool = new Set([...groupDirSegments, ...groupFileSegments]);
+    for (const seg of docSegments) {
+        if (seg.length > 1 && pathPool.has(seg)) {
+            score += 3;
+        }
+    }
+    // 2. Label match — group label words in doc heading area
+    const headingArea = doc.content.slice(0, 300).toLowerCase();
+    const labelWords = group.label
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((w) => w.length > 2);
+    for (const word of labelWords) {
+        if (headingArea.includes(word)) {
+            score += 2;
+        }
+    }
+    // Group ID segments (e.g., "user-auth" → ["user", "auth"])
+    const idSegments = group.id
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((w) => w.length > 2);
+    for (const seg of idSegments) {
+        if (headingArea.includes(seg)) {
+            score += 2;
+        }
+    }
+    // 3. Topic match — exploration keywords in doc content
+    if (topicKeywords.size > 0) {
+        const docKeywords = extractKeywords(doc.content);
+        let topicHits = 0;
+        for (const kw of topicKeywords) {
+            if (docKeywords.has(kw)) {
+                topicHits++;
+            }
+        }
+        // Diminishing returns: first 5 hits are worth more
+        score += Math.min(topicHits, 5) * 1 + Math.max(0, topicHits - 5) * 0.3;
+    }
+    return score;
+}
+/**
+ * Builds topic keywords from a group's exploration findings for relevance scoring.
+ */
+function buildTopicKeywords(findings) {
+    if (!findings)
+        return new Set();
+    return extractKeywords(findings.text);
+}
+/**
+ * Filters the full set of existing docs down to the most relevant subset
+ * for a specific file group.
+ *
+ * Returns:
+ *  - `relevantDocs`: up to `MAX_RELEVANT_DOCS` docs with full content,
+ *     sorted by relevance score (highest first).
+ *  - `mentionedPaths`: paths of remaining docs (no content) so the agent
+ *     knows they exist.
+ *
+ * Relevance is scored by path overlap with the group's files/directory,
+ * label/ID keyword matches in the doc heading, and topic keyword overlap
+ * from the group's exploration findings.
+ */
+export function filterRelevantDocs(allDocs, group, groupFindings) {
+    if (allDocs.length <= MAX_RELEVANT_DOCS) {
+        return { relevantDocs: allDocs, mentionedPaths: [] };
+    }
+    const topicKeywords = buildTopicKeywords(groupFindings);
+    // Score each doc
+    const scored = allDocs.map((doc) => ({
+        doc,
+        score: scoreDoc(doc, group, topicKeywords),
+    }));
+    // Sort by score descending, stable on original order for ties
+    scored.sort((a, b) => b.score - a.score);
+    const MAX_DOC_CONTENT_CHARS = 3_072;
+    const relevantDocs = scored.slice(0, MAX_RELEVANT_DOCS).map((s) => ({
+        path: s.doc.path,
+        content: s.doc.content.length > MAX_DOC_CONTENT_CHARS
+            ? s.doc.content.slice(0, MAX_DOC_CONTENT_CHARS) + "\n... [truncated]"
+            : s.doc.content,
+    }));
+    const mentionedPaths = scored.slice(MAX_RELEVANT_DOCS).map((s) => s.doc.path);
+    return { relevantDocs, mentionedPaths };
+}
+// ── Index-based doc resolution ────────────────────────────────────────
+/** Maximum content chars to read from disk per relevant doc when using index. */
+const MAX_DOC_CONTENT_CHARS_INDEX = 3_072;
+/**
+ * Builds an FTS5 search query from a file group's label, ID, and exploration findings.
+ */
+function buildGroupSearchQuery(group, findings) {
+    const parts = [
+        group.label,
+        group.id.replace(/[-_]/g, " "),
+    ];
+    if (findings?.text) {
+        parts.push(findings.text.slice(0, 200));
+    }
+    return parts.join(" ");
+}
+/**
+ * Resolves the most relevant existing docs for a group using a SearchIndex.
+ *
+ * Uses FTS5 search to rank docs by keyword overlap with the group's label,
+ * ID, and exploration findings. Reads the top results from disk (capped at
+ * 3 072 chars each). All other indexed paths are returned as mentionedPaths.
+ */
+async function queryRelevantDocsFromIndex(index, destinationPath, group, groupFindings) {
+    const query = buildGroupSearchQuery(group, groupFindings);
+    const topResults = index.search(query, MAX_RELEVANT_DOCS);
+    const topPaths = new Set(topResults.map((r) => r.path));
+    const relevantDocs = await Promise.all(topResults.map(async ({ path }) => {
+        try {
+            const raw = await readFile(join(destinationPath, path), "utf-8");
+            const content = raw.length > MAX_DOC_CONTENT_CHARS_INDEX
+                ? raw.slice(0, MAX_DOC_CONTENT_CHARS_INDEX) + "\n... [truncated]"
+                : raw;
+            return { path, content };
+        }
+        catch {
+            return { path, content: "" };
+        }
+    }));
+    const mentionedPaths = [...index.getAllHashes().keys()].filter((p) => !topPaths.has(p));
+    return { relevantDocs: relevantDocs.filter((d) => d.content !== ""), mentionedPaths };
+}
+/**
+ * Resolves relevant existing docs for a group using either the search index
+ * (primary path) or the in-memory fallback (`filterRelevantDocs`).
+ */
+async function resolveRelevantDocs(searchIndex, destinationPath, existingDocs, group, groupFindings) {
+    if (searchIndex) {
+        return queryRelevantDocsFromIndex(searchIndex, destinationPath, group, groupFindings);
+    }
+    return filterRelevantDocs(existingDocs, group, groupFindings);
+}
+// ── Exploration Phase ─────────────────────────────────────────────────
+/**
+ * Runs the exploration phase for all groups in parallel.
+ *
+ * Each group gets its own ExplorationAgent session. Results are collected
+ * via `Promise.allSettled` — failed explorations are logged as warnings
+ * but do not abort the pipeline.
+ *
+ * @param explorationAgent - The exploration agent instance.
+ * @param groups           - The file groups to explore.
+ * @param options          - CLI options including source path and concurrency.
+ * @param onProgress       - Callback to update progress state.
+ * @param onGroupComplete  - Optional callback invoked each time a group's
+ *                           exploration finishes successfully. Used by the
+ *                           streaming pipeline to launch doc agents eagerly.
+ * @returns All successful `ExplorationFindings[]`.
+ */
+export async function runExploration(explorationAgent, groups, options, onProgress, onGroupComplete) {
+    const sourceRoot = resolve(options.source);
+    const verbose = options.verbose
+        ? (msg) => {
+            const ts = new Date().toISOString().slice(11, 23);
+            process.stderr.write(`[${ts}] ${msg}\n`);
+        }
+        : undefined;
+    const effectiveConcurrency = computeMaxConcurrency(options.concurrency);
+    if (verbose) {
+        verbose(`[orchestrator] Resource limits: ${describeResourceLimits(options.concurrency)}`);
+        verbose(`[orchestrator] Starting exploration: ${groups.length} groups, concurrency=${effectiveConcurrency}`);
+    }
+    // Build allGroups summary for cross-group awareness
+    const allGroups = groups.map((group) => ({
+        id: group.id,
+        label: group.label,
+        files: group.files.map((f) => f.path),
+    }));
+    // Compute total file count to determine if multi-agent exploration is needed
+    const totalFiles = groups.reduce((sum, g) => sum + g.files.length, 0);
+    const agentsPerGroup = computeAgentsPerGroup(totalFiles);
+    const isMultiAgent = agentsPerGroup > 1;
+    if (verbose && isMultiAgent) {
+        verbose(`[orchestrator] Large project detected: ${totalFiles} files → ${agentsPerGroup} agents per group`);
+    }
+    // Initialize per-group exploration status for the dashboard
+    const explorationStatuses = groups.map((group) => ({
+        groupId: group.id,
+        label: group.label,
+        status: "queued",
+        ...(isMultiAgent ? { subAgentProgress: { completed: 0, total: 0 } } : {}),
+    }));
+    onProgress({
+        phase: "exploration",
+        explorationStatuses: [...explorationStatuses],
+    });
+    const limit = pLimit(effectiveConcurrency);
+    // ── Single-agent path (existing behavior, totalFiles < threshold) ──
+    if (!isMultiAgent) {
+        const results = await Promise.allSettled(groups.map((group, i) => limit(async () => {
+            explorationStatuses[i] = { ...explorationStatuses[i], status: "generating" };
+            onProgress({
+                explorationStatuses: [...explorationStatuses],
+            });
+            try {
+                const findings = await explorationAgent.run(group, sourceRoot, allGroups, verbose);
+                explorationStatuses[i] = { ...explorationStatuses[i], status: "completed" };
+                onProgress({
+                    explorationStatuses: [...explorationStatuses],
+                });
+                onGroupComplete?.(group.id, findings);
+                return findings;
+            }
+            catch (firstError) {
+                if (firstError instanceof AuthExpiredError) {
+                    throw firstError;
+                }
+                const firstMessage = firstError instanceof Error ? firstError.message : String(firstError);
+                if (verbose) {
+                    verbose(`[orchestrator] Exploration failed for "${group.id}": ${firstMessage}`);
+                    verbose(`[orchestrator] Retrying exploration for "${group.id}" (attempt 2)...`);
+                }
+                // First attempt failed — retry once
+                explorationStatuses[i] = { ...explorationStatuses[i], status: "generating" };
+                onProgress({
+                    explorationStatuses: [...explorationStatuses],
+                });
+                try {
+                    const findings = await explorationAgent.run(group, sourceRoot, allGroups, verbose);
+                    explorationStatuses[i] = { ...explorationStatuses[i], status: "completed" };
+                    onProgress({
+                        explorationStatuses: [...explorationStatuses],
+                    });
+                    onGroupComplete?.(group.id, findings);
+                    return findings;
+                }
+                catch (retryError) {
+                    if (retryError instanceof AuthExpiredError) {
+                        throw retryError;
+                    }
+                    const retryMessage = retryError instanceof Error ? retryError.message : String(retryError);
+                    if (verbose) {
+                        verbose(`[orchestrator] Exploration retry failed for "${group.id}": ${retryMessage}`);
+                    }
+                    explorationStatuses[i] = {
+                        ...explorationStatuses[i],
+                        status: "failed",
+                        error: retryMessage,
+                    };
+                    onProgress({
+                        explorationStatuses: [...explorationStatuses],
+                    });
+                    throw retryError;
+                }
+            }
+        })));
+        // Check for AuthExpiredError in settled results
+        for (const result of results) {
+            if (result.status === "rejected" &&
+                result.reason instanceof AuthExpiredError) {
+                throw result.reason;
+            }
+        }
+        // Collect successful findings, log warnings for failures
+        const allFindings = [];
+        const warnings = [];
+        for (let i = 0; i < results.length; i++) {
+            const result = results[i];
+            if (result.status === "fulfilled") {
+                allFindings.push(result.value);
+            }
+            else {
+                const groupId = groups[i].id;
+                const errorMsg = result.reason instanceof Error
+                    ? result.reason.message
+                    : String(result.reason);
+                warnings.push(`Exploration failed for "${groupId}": ${errorMsg}`);
+            }
+        }
+        if (warnings.length > 0) {
+            onProgress({ warnings });
+            if (verbose) {
+                for (const warning of warnings) {
+                    verbose(`[orchestrator] WARNING: ${warning}`);
+                }
+            }
+        }
+        if (verbose) {
+            verbose(`[orchestrator] Exploration complete: ${allFindings.length}/${groups.length} groups explored successfully`);
+        }
+        onProgress({ explorationComplete: true });
+        return allFindings;
+    }
+    // ── Multi-agent path (totalFiles >= threshold) ─────────────────────
+    if (verbose) {
+        verbose(`[orchestrator] Multi-agent exploration: ${groups.length} groups × up to ${agentsPerGroup} agents`);
+    }
+    // For each group, partition files into sub-groups and run sub-agents.
+    // All sub-agents across all groups share the same p-limit instance.
+    const results = await Promise.allSettled(groups.map((group, i) => {
+        const subGroups = partitionGroupFiles(group, agentsPerGroup);
+        const subAgentCount = subGroups.length;
+        // Update status with sub-agent count
+        explorationStatuses[i] = {
+            ...explorationStatuses[i],
+            status: "generating",
+            subAgentProgress: { completed: 0, total: subAgentCount },
+        };
+        onProgress({
+            explorationStatuses: [...explorationStatuses],
+        });
+        if (verbose) {
+            verbose(`[orchestrator] Group "${group.id}": ${group.files.length} files → ${subAgentCount} sub-agent(s)`);
+        }
+        // Run all sub-agents for this group, each through p-limit
+        return (async () => {
+            let subAgentsCompleted = 0;
+            /**
+             * Run a single sub-agent with retry logic.
+             * Returns the findings on success, or throws on double failure.
+             */
+            const runSubAgent = async (subGroup) => {
+                try {
+                    return await explorationAgent.run(subGroup, sourceRoot, allGroups, verbose);
+                }
+                catch (firstError) {
+                    if (firstError instanceof AuthExpiredError)
+                        throw firstError;
+                    const firstMessage = firstError instanceof Error ? firstError.message : String(firstError);
+                    if (verbose) {
+                        verbose(`[orchestrator] Sub-agent failed for "${subGroup.id}": ${firstMessage}`);
+                        verbose(`[orchestrator] Retrying sub-agent "${subGroup.id}" (attempt 2)...`);
+                    }
+                    try {
+                        return await explorationAgent.run(subGroup, sourceRoot, allGroups, verbose);
+                    }
+                    catch (retryError) {
+                        if (retryError instanceof AuthExpiredError)
+                            throw retryError;
+                        const retryMessage = retryError instanceof Error ? retryError.message : String(retryError);
+                        if (verbose) {
+                            verbose(`[orchestrator] Sub-agent retry failed for "${subGroup.id}": ${retryMessage}`);
+                        }
+                        throw retryError;
+                    }
+                }
+            };
+            const subResults = await Promise.allSettled(subGroups.map((subGroup) => limit(async () => {
+                const findings = await runSubAgent(subGroup);
+                subAgentsCompleted++;
+                explorationStatuses[i] = {
+                    ...explorationStatuses[i],
+                    subAgentProgress: { completed: subAgentsCompleted, total: subAgentCount },
+                };
+                onProgress({
+                    explorationStatuses: [...explorationStatuses],
+                });
+                return findings;
+            })));
+            // Check for AuthExpiredError in sub-results
+            for (const sub of subResults) {
+                if (sub.status === "rejected" && sub.reason instanceof AuthExpiredError) {
+                    throw sub.reason;
+                }
+            }
+            // Collect successful sub-findings
+            const successfulFindings = [];
+            const subWarnings = [];
+            for (let j = 0; j < subResults.length; j++) {
+                const sub = subResults[j];
+                if (sub.status === "fulfilled") {
+                    successfulFindings.push(sub.value);
+                }
+                else {
+                    const subGroupId = subGroups[j].id;
+                    const errorMsg = sub.reason instanceof Error ? sub.reason.message : String(sub.reason);
+                    subWarnings.push(`Sub-agent failed for "${subGroupId}": ${errorMsg}`);
+                }
+            }
+            if (subWarnings.length > 0 && verbose) {
+                for (const w of subWarnings) {
+                    verbose(`[orchestrator] WARNING: ${w}`);
+                }
+            }
+            // If ALL sub-agents failed, mark the group as failed and throw
+            if (successfulFindings.length === 0) {
+                explorationStatuses[i] = {
+                    ...explorationStatuses[i],
+                    status: "failed",
+                    error: `All ${subAgentCount} sub-agents failed`,
+                };
+                onProgress({ explorationStatuses: [...explorationStatuses] });
+                throw new Error(`All ${subAgentCount} sub-agents failed for group "${group.id}"`);
+            }
+            // Merge sub-findings into one ExplorationFindings for this group
+            const merged = mergeExplorationFindings(group.id, successfulFindings);
+            // Mark group as completed
+            explorationStatuses[i] = {
+                ...explorationStatuses[i],
+                status: "completed",
+                subAgentProgress: { completed: subAgentsCompleted, total: subAgentCount },
+            };
+            onProgress({ explorationStatuses: [...explorationStatuses] });
+            // Notify streaming pipeline gate
+            onGroupComplete?.(group.id, merged);
+            if (verbose) {
+                verbose(`[orchestrator] Group "${group.id}": merged ${successfulFindings.length}/${subAgentCount} sub-findings → ` +
+                    `${merged.text.length.toLocaleString()} chars`);
+            }
+            return merged;
+        })();
+    }));
+    // Check for AuthExpiredError in group-level results
+    for (const result of results) {
+        if (result.status === "rejected" &&
+            result.reason instanceof AuthExpiredError) {
+            throw result.reason;
+        }
+    }
+    // Collect successful findings, log warnings for failures
+    const allFindings = [];
+    const warnings = [];
+    for (let i = 0; i < results.length; i++) {
+        const result = results[i];
+        if (result.status === "fulfilled") {
+            allFindings.push(result.value);
+        }
+        else {
+            const groupId = groups[i].id;
+            const errorMsg = result.reason instanceof Error
+                ? result.reason.message
+                : String(result.reason);
+            warnings.push(`Exploration failed for "${groupId}": ${errorMsg}`);
+        }
+    }
+    if (warnings.length > 0) {
+        onProgress({ warnings });
+        if (verbose) {
+            for (const warning of warnings) {
+                verbose(`[orchestrator] WARNING: ${warning}`);
+            }
+        }
+    }
+    if (verbose) {
+        verbose(`[orchestrator] Exploration complete: ${allFindings.length}/${groups.length} groups explored successfully`);
+    }
+    onProgress({ explorationComplete: true });
+    return allFindings;
+}
+// ── Documentation Generation Phase ────────────────────────────────────
+/**
+ * Runs the doc agent for a single file group.
+ *
+ * The agent writes documentation files directly via the edit tool.
+ * The pipeline discovers what was produced by scanning the destination
+ * directory afterward.
+ *
+ * @param agent           - The documentation agent instance.
+ * @param group           - The file group to document.
+ * @param context         - Shared generation context.
+ * @param verbose         - Optional verbose logger.
+ * @returns A `GenerationResult` indicating success or failure.
+ */
+export async function runDocAgent(agent, group, context, verbose) {
+    await agent.run(group, context, verbose);
+    if (verbose) {
+        verbose(`[orchestrator] Doc agent completed for "${group.id}"`);
+    }
+    return {
+        groupId: group.id,
+        status: "generated",
+        error: undefined,
+    };
+}
+/**
+ * Generates documentation for all file groups in parallel.
+ *
+ * Implements a streaming pipeline that overlaps exploration and documentation:
+ *   - **Exploration + existing docs scan** run in parallel.
+ *   - **Documentation** for each group starts as soon as its exploration
+ *     finishes, rather than waiting for all explorations to complete.
+ *   - Each doc agent receives accumulated exploration findings at launch time.
+ *
+ * When no `explorationAgent` is provided, all doc agents launch immediately.
+ *
+ * @param agent              - The documentation agent instance.
+ * @param groups             - The file groups to document.
+ * @param options            - CLI options including source/destination paths.
+ * @param onProgress         - Callback to update progress state.
+ * @param explorationAgent   - Optional exploration agent.
+ * @returns Results, exploration findings, and existing docs.
+ */
+export async function generateDocumentation(agent, groups, options, onProgress, explorationAgent, searchIndex) {
+    const destinationPath = resolve(options.destination);
+    const projectName = options.name ?? resolve(options.source).split("/").pop() ?? "project";
+    const verbose = options.verbose
+        ? (msg) => {
+            const ts = new Date().toISOString().slice(11, 23);
+            process.stderr.write(`[${ts}] ${msg}\n`);
+        }
+        : undefined;
+    // ── Per-group exploration gates ─────────────────────────────────
+    // Each group gets a promise that resolves when its exploration finishes
+    // (or immediately if no exploration agent is provided).
+    const groupGates = new Map();
+    const groupGatePromises = new Map();
+    for (const group of groups) {
+        const gate = {};
+        const promise = new Promise((res, rej) => {
+            gate.resolve = res;
+            gate.reject = rej;
+        });
+        // Attach a no-op catch so Node doesn't warn about unhandled rejection
+        // when a gate rejects before its doc agent starts awaiting. The consumer
+        // still awaits this promise and re-throws, so errors aren't lost.
+        promise.catch((err) => {
+            verbose?.(`[gate:${group.id}] rejected: ${err instanceof Error ? err.message : String(err)}`);
+        });
+        groupGates.set(group.id, gate);
+        groupGatePromises.set(group.id, promise);
+    }
+    // Accumulated findings — grows as explorations complete.
+    const allFindings = [];
+    // ── Launch exploration + existingDocs scan in parallel ───────────
+    let explorationPromise;
+    let existingDocsPromise;
+    if (explorationAgent) {
+        if (verbose) {
+            verbose(`[orchestrator] Exploration phase: ${groups.length} groups`);
+        }
+        explorationPromise = runExploration(explorationAgent, groups, options, onProgress, 
+        // Streaming callback: resolve the gate for each completed group
+        (groupId, findings) => {
+            allFindings.push(findings);
+            groupGates.get(groupId)?.resolve(findings);
+        });
+        // When exploration fully settles, resolve any remaining gates for groups
+        // whose exploration failed so their doc agents can proceed without findings.
+        // On AuthExpiredError, reject all remaining gates to abort doc agents.
+        explorationPromise.then(() => {
+            for (const gate of groupGates.values()) {
+                gate.resolve(undefined);
+            }
+        }, (err) => {
+            if (err instanceof AuthExpiredError) {
+                // Reject remaining gates — doc agents will propagate the error
+                for (const gate of groupGates.values()) {
+                    gate.reject(err);
+                }
+            }
+            else {
+                // Non-auth errors: still let doc agents proceed
+                for (const gate of groupGates.values()) {
+                    gate.resolve(undefined);
+                }
+            }
+        });
+    }
+    else {
+        // No exploration — resolve all gates immediately
+        for (const gate of groupGates.values()) {
+            gate.resolve(undefined);
+        }
+    }
+    // When a search index is available, skip the full-content scan.
+    // Relevant docs are fetched on demand (per group) via the index.
+    // Fall back to scanExistingDocs when no index is provided.
+    existingDocsPromise = searchIndex
+        ? Promise.resolve([])
+        : scanExistingDocs(destinationPath, verbose);
+    // ── Wait for existing docs (needed before launching any doc agent) ──
+    const existingDocs = await existingDocsPromise;
+    if (verbose) {
+        if (searchIndex) {
+            verbose("[orchestrator] Existing docs: using search index for per-group relevance queries");
+        }
+        else {
+            verbose(`[orchestrator] Existing docs: ${existingDocs.length} file(s) found` +
+                (existingDocs.length > 0 ? " — will provide to doc agents for merge" : ""));
+        }
+    }
+    // ── Documentation phase ─────────────────────────────────────────
+    if (verbose) {
+        verbose(`[orchestrator] Documentation phase: ${groups.length} groups, concurrency=${options.concurrency}`);
+    }
+    // Initialize per-group session tracking
+    const sessions = groups.map((group) => ({
+        sessionId: "",
+        groupId: group.id,
+        status: "pending",
+        startedAt: undefined,
+        completedAt: undefined,
+        retryCount: 0,
+        error: undefined,
+    }));
+    // Initialize per-group status for the dashboard
+    const groupStatuses = groups.map((group) => ({
+        groupId: group.id,
+        label: group.label,
+        status: "queued",
+    }));
+    let completedSessions = 0;
+    let failedSessions = 0;
+    onProgress({
+        phase: "generation",
+        groupStatuses: [...groupStatuses],
+        completedSessions: 0,
+        failedSessions: 0,
+    });
+    const docConcurrency = computeMaxConcurrency(options.concurrency);
+    // ── Multi-agent documentation detection ──────────────────────────
+    const totalFiles = groups.reduce((sum, g) => sum + g.files.length, 0);
+    const docAgentsPerGroup = computeAgentsPerGroup(totalFiles);
+    const isMultiAgentDoc = docAgentsPerGroup > 1;
+    if (verbose) {
+        verbose(`[orchestrator] Starting documentation: ${groups.length} groups, concurrency=${docConcurrency}` +
+            (isMultiAgentDoc ? `, multi-agent=${docAgentsPerGroup} agents/group` : ""));
+    }
+    const limit = pLimit(docConcurrency);
+    const results = await Promise.allSettled(groups.map((group, i) => {
+        if (!isMultiAgentDoc) {
+            // ── Single-agent path (existing behavior) ──────────────────
+            return limit(async () => {
+                // Wait for this group's exploration to finish (or resolve immediately)
+                const groupFindings = await groupGatePromises.get(group.id);
+                const session = sessions[i];
+                session.status = "active";
+                session.startedAt = Date.now();
+                groupStatuses[i] = { ...groupStatuses[i], status: "generating" };
+                onProgress({
+                    groupStatuses: [...groupStatuses],
+                });
+                // Build per-group context with accumulated findings at this point
+                // Resolve relevant docs via index (preferred) or in-memory filter
+                const { relevantDocs, mentionedPaths } = await resolveRelevantDocs(searchIndex, destinationPath, existingDocs, group, groupFindings);
+                const groupContext = {
+                    projectName,
+                    sourceRoot: resolve(options.source),
+                    destinationPath,
+                    allGroups: groups.map((g) => ({
+                        id: g.id,
+                        label: g.label,
+                        files: g.files.map((f) => f.path),
+                    })),
+                    explorationFindings: [...allFindings],
+                    existingDocs: relevantDocs,
+                    mentionedDocPaths: mentionedPaths,
+                    groupFindings,
+                };
+                try {
+                    const result = await runDocAgent(agent, group, groupContext, verbose);
+                    session.status = "completed";
+                    session.completedAt = Date.now();
+                    completedSessions++;
+                    groupStatuses[i] = { ...groupStatuses[i], status: "completed" };
+                    onProgress({
+                        groupStatuses: [...groupStatuses],
+                        completedSessions,
+                        failedSessions,
+                    });
+                    return result;
+                }
+                catch (firstError) {
+                    if (firstError instanceof AuthExpiredError) {
+                        throw firstError;
+                    }
+                    // First attempt failed — retry once
+                    session.status = "retrying";
+                    session.retryCount = 1;
+                    if (verbose) {
+                        verbose(`[orchestrator] Retrying "${group.id}" (attempt 2)...`);
+                    }
+                    groupStatuses[i] = { ...groupStatuses[i], status: "generating" };
+                    onProgress({
+                        groupStatuses: [...groupStatuses],
+                    });
+                    try {
+                        const result = await runDocAgent(agent, group, groupContext, verbose);
+                        session.status = "completed";
+                        session.completedAt = Date.now();
+                        completedSessions++;
+                        groupStatuses[i] = { ...groupStatuses[i], status: "completed" };
+                        onProgress({
+                            groupStatuses: [...groupStatuses],
+                            completedSessions,
+                            failedSessions,
+                        });
+                        return result;
+                    }
+                    catch (retryError) {
+                        const retryMessage = retryError instanceof Error
+                            ? retryError.message
+                            : String(retryError);
+                        session.status = "failed";
+                        session.completedAt = Date.now();
+                        session.error = retryMessage;
+                        failedSessions++;
+                        groupStatuses[i] = { ...groupStatuses[i], status: "failed" };
+                        onProgress({
+                            groupStatuses: [...groupStatuses],
+                            completedSessions,
+                            failedSessions,
+                        });
+                        return {
+                            groupId: group.id,
+                            status: "failed",
+                            error: retryMessage,
+                        };
+                    }
+                }
+            });
+        }
+        // ── Multi-agent path (totalFiles >= LARGE_PROJECT_THRESHOLD) ──
+        const subGroups = partitionGroupFiles(group, docAgentsPerGroup);
+        const subAgentCount = subGroups.length;
+        // Update status with sub-agent count
+        groupStatuses[i] = {
+            ...groupStatuses[i],
+            status: "generating",
+            subAgentProgress: { completed: 0, total: subAgentCount },
+        };
+        if (verbose) {
+            verbose(`[orchestrator] Doc group "${group.id}": ${group.files.length} files → ${subAgentCount} sub-agent(s)`);
+        }
+        return (async () => {
+            // Wait for this group's exploration to finish (or resolve immediately)
+            const groupFindings = await groupGatePromises.get(group.id);
+            const session = sessions[i];
+            session.status = "active";
+            session.startedAt = Date.now();
+            onProgress({
+                groupStatuses: [...groupStatuses],
+            });
+            // Build per-group context (shared across all sub-agents for this group)
+            const { relevantDocs, mentionedPaths } = await resolveRelevantDocs(searchIndex, destinationPath, existingDocs, group, groupFindings);
+            const groupContext = {
+                projectName,
+                sourceRoot: resolve(options.source),
+                destinationPath,
+                allGroups: groups.map((g) => ({
+                    id: g.id,
+                    label: g.label,
+                    files: g.files.map((f) => f.path),
+                })),
+                explorationFindings: [...allFindings],
+                existingDocs: relevantDocs,
+                mentionedDocPaths: mentionedPaths,
+                groupFindings,
+            };
+            let subAgentsCompleted = 0;
+            /**
+             * Run a single doc sub-agent with retry logic.
+             * Returns the result on success, or throws on double failure.
+             */
+            const runSubDocAgent = async (subGroup) => {
+                try {
+                    return await runDocAgent(agent, subGroup, groupContext, verbose);
+                }
+                catch (firstError) {
+                    if (firstError instanceof AuthExpiredError)
+                        throw firstError;
+                    if (verbose) {
+                        const msg = firstError instanceof Error ? firstError.message : String(firstError);
+                        verbose(`[orchestrator] Doc sub-agent failed for "${subGroup.id}": ${msg}`);
+                        verbose(`[orchestrator] Retrying doc sub-agent "${subGroup.id}" (attempt 2)...`);
+                    }
+                    try {
+                        return await runDocAgent(agent, subGroup, groupContext, verbose);
+                    }
+                    catch (retryError) {
+                        if (retryError instanceof AuthExpiredError)
+                            throw retryError;
+                        if (verbose) {
+                            const msg = retryError instanceof Error ? retryError.message : String(retryError);
+                            verbose(`[orchestrator] Doc sub-agent retry failed for "${subGroup.id}": ${msg}`);
+                        }
+                        throw retryError;
+                    }
+                }
+            };
+            const subResults = await Promise.allSettled(subGroups.map((subGroup) => limit(async () => {
+                const result = await runSubDocAgent(subGroup);
+                subAgentsCompleted++;
+                groupStatuses[i] = {
+                    ...groupStatuses[i],
+                    subAgentProgress: { completed: subAgentsCompleted, total: subAgentCount },
+                };
+                onProgress({
+                    groupStatuses: [...groupStatuses],
+                });
+                return result;
+            })));
+            // Check for AuthExpiredError in sub-results
+            for (const sub of subResults) {
+                if (sub.status === "rejected" && sub.reason instanceof AuthExpiredError) {
+                    throw sub.reason;
+                }
+            }
+            // Collect successful sub-results
+            const successfulResults = [];
+            const subWarnings = [];
+            for (let j = 0; j < subResults.length; j++) {
+                const sub = subResults[j];
+                if (sub.status === "fulfilled") {
+                    successfulResults.push(sub.value);
+                }
+                else {
+                    const subGroupId = subGroups[j].id;
+                    const errorMsg = sub.reason instanceof Error ? sub.reason.message : String(sub.reason);
+                    subWarnings.push(`Doc sub-agent failed for "${subGroupId}": ${errorMsg}`);
+                }
+            }
+            if (subWarnings.length > 0 && verbose) {
+                for (const w of subWarnings) {
+                    verbose(`[orchestrator] WARNING: ${w}`);
+                }
+            }
+            // If ALL sub-agents failed, mark the group as failed
+            if (successfulResults.length === 0) {
+                session.status = "failed";
+                session.completedAt = Date.now();
+                session.error = `All ${subAgentCount} doc sub-agents failed`;
+                failedSessions++;
+                groupStatuses[i] = {
+                    ...groupStatuses[i],
+                    status: "failed",
+                    error: `All ${subAgentCount} doc sub-agents failed`,
+                };
+                onProgress({
+                    groupStatuses: [...groupStatuses],
+                    completedSessions,
+                    failedSessions,
+                });
+                return {
+                    groupId: group.id,
+                    status: "failed",
+                    error: `All ${subAgentCount} doc sub-agents failed`,
+                };
+            }
+            // At least one sub-agent succeeded — mark the group as completed
+            session.status = "completed";
+            session.completedAt = Date.now();
+            completedSessions++;
+            groupStatuses[i] = {
+                ...groupStatuses[i],
+                status: "completed",
+                subAgentProgress: { completed: subAgentsCompleted, total: subAgentCount },
+            };
+            onProgress({
+                groupStatuses: [...groupStatuses],
+                completedSessions,
+                failedSessions,
+            });
+            if (verbose) {
+                verbose(`[orchestrator] Doc group "${group.id}": ${successfulResults.length}/${subAgentCount} sub-agents completed`);
+            }
+            return {
+                groupId: group.id,
+                status: "generated",
+                error: undefined,
+            };
+        })();
+    }));
+    // ── Wait for exploration to fully settle ────────────────────────
+    // (it may still be running for groups that didn't gate a doc agent,
+    //  e.g. groups whose exploration failed)
+    if (explorationPromise) {
+        try {
+            await explorationPromise;
+        }
+        catch (err) {
+            if (err instanceof AuthExpiredError) {
+                throw err;
+            }
+            // Non-auth exploration errors were already handled inside runExploration
+        }
+        if (verbose) {
+            verbose(`[orchestrator] Exploration complete: ${allFindings.length}/${groups.length} groups explored`);
+        }
+    }
+    // Resolve any remaining gates for groups whose exploration failed
+    // (they never got an onGroupComplete callback)
+    for (const gate of groupGates.values()) {
+        gate.resolve(undefined);
+    }
+    // Check for AuthExpiredError in settled results
+    for (const result of results) {
+        if (result.status === "rejected" &&
+            result.reason instanceof AuthExpiredError) {
+            throw result.reason;
+        }
+    }
+    // Collect results from settled promises
+    const generationResults = results.map((result) => {
+        if (result.status === "fulfilled") {
+            return result.value;
+        }
+        const error = result.reason instanceof Error
+            ? result.reason.message
+            : String(result.reason);
+        return {
+            groupId: "unknown",
+            status: "failed",
+            error,
+        };
+    });
+    // Emit warnings for failed groups
+    const failedWarnings = generationResults
+        .filter((r) => r.status === "failed")
+        .map((r) => `Failed to generate: ${r.groupId}${r.error ? ` (${r.error})` : ""}`);
+    if (failedWarnings.length > 0) {
+        onProgress({
+            warnings: failedWarnings,
+        });
+    }
+    return { results: generationResults, explorationFindings: allFindings, existingDocs };
+}
+// ── Exit Code ──────────────────────────────────────────────────────────
+/**
+ * Determine the exit code from documentation generation results.
+ *
+ * @param results - Generation results for all groups.
+ * @returns 0 if all succeeded, 1 if some failed, 2 if all failed or empty.
+ */
+export function getExitCode(results) {
+    if (results.length === 0)
+        return 2;
+    const failed = results.filter((r) => r.status === "failed").length;
+    if (failed === 0)
+        return 0;
+    if (failed === results.length)
+        return 2;
+    return 1;
+}