@aexol/spectral 0.6.2 → 0.6.4

package/dist/relay/auto-research.js

@@ -4,20 +4,82 @@
  *
  * Flow:
  *   1. Resolve project path from the SQLite store
- *   2. Spawn pi with --mode json -p running the auto-research agent
- *   3. Parse JSON-line output from the subprocess
- *   4. Map subprocess events to `auto_research_*` ServerEvent types
- *   5. Stream progress via the relay to the browser
- *   6. On completion, emit `auto_research_complete` with generated extensions
+ *   2. Load the auto-research agent definition (system prompt + model)
+ *   3. Write system prompt to a temp file (--append-system-prompt)
+ *   4. Spawn pi with --mode json -p --no-session --model <model> --append-system-prompt <tmp>
+ *   5. Pass the user task as a positional argument ("Task: ...")
+ *   6. Parse pi's JSON-line output: watch for message_end events on assistant
+ *      messages, extract the text, and interpret it as auto-research events
+ *      (progress / extension_generated / done / error)
+ *   7. Stream progress via the relay to the browser
+ *   8. On completion, emit `auto_research_complete` with generated extensions
  *
- * The subprocess runs with `cwd` set to the project root so all file
- * reads/writes are relative to the project. The agent definition markdown
- * file lives alongside other agent defs and gets loaded by pi's subagent
- * infrastructure.
+ * This mirrors the subagent extension's spawn pattern (agent/index.ts) so
+ * pi receives the task and system prompt in the same format it expects.
  */
 import { spawn } from "node:child_process";
 import * as fs from "node:fs";
+import * as os from "node:os";
 import * as path from "node:path";
+import { parseFrontmatter } from "@mariozechner/pi-coding-agent";
+/**
+ * Locate the auto-research agent definition markdown file.
+ * Searches project-level first, then user-level.
+ */
+function findAgentDef(projectPath) {
+    const candidates = [
+        path.join(projectPath, ".pi", "agents", "auto-research.md"),
+        path.join(os.homedir(), ".pi", "agent", "agents", "auto-research.md"),
+    ];
+    for (const filePath of candidates) {
+        try {
+            const content = fs.readFileSync(filePath, "utf-8");
+            const { frontmatter, body } = parseFrontmatter(content);
+            if (body.trim().length === 0)
+                continue;
+            return {
+                model: frontmatter.model ?? "claude-sonnet-4-5",
+                systemPrompt: body,
+            };
+        }
+        catch {
+            continue;
+        }
+    }
+    // Hardcoded fallback: system prompt for when the agent definition file
+    // is not found in the project or user agent directories. This ensures
+    // auto-research works out of the box on first use.
+    return getDefaultAgentDef();
+}
+/** Default agent definition used as a fallback when no agent .md file exists. */
+function getDefaultAgentDef() {
+    return {
+        model: "claude-sonnet-4-5",
+        systemPrompt: [
+            "You are an auto-research agent. Analyze the project and generate custom",
+            "pi coding agent extensions. Output ONLY JSON lines (one per line).",
+            "",
+            "## Process",
+            '1. Context: emit {"type":"progress","phase":"context_collecting","message":"..."}',
+            '2. Analysis: emit {"type":"progress","phase":"context_analyzing","message":"..."}',
+            '3. Generation: emit {"type":"progress","phase":"extension_generating","message":"..."}',
+            '4. Validation: emit {"type":"progress","phase":"extension_validating","message":"..."}',
+            "",
+            "## Extension categories",
+            "A. Workflow automation  B. Code gen  C. Project-specific tools",
+            "D. Quality/review  E. Documentation  F. LLM-powered  G. Stateful",
+            "",
+            'When you generate an extension, emit:',
+            '{"type":"extension_generated","name":"...","path":"...","description":"...","usesLLM":bool,"fileCount":n}',
+            "Extensions go under .pi/extensions/auto-research/",
+            "",
+            'When done, emit: {"type":"done","extensions":[...]}',
+            "",
+            "IMPORTANT: Output ONLY JSON lines. No markdown, no code blocks.",
+            "Each line must be a single valid JSON object.",
+        ].join("\n"),
+    };
+}
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -77,7 +139,6 @@ function killProcess(child) {
     try {
         if (!child.killed && child.exitCode === null) {
             child.kill("SIGTERM");
-            // Give it 2 seconds to clean up, then force-kill
             setTimeout(() => {
                 try {
                     if (!child.killed && child.exitCode === null) {
@@ -95,6 +156,21 @@ function killProcess(child) {
     }
 }
 // ---------------------------------------------------------------------------
+// Phase mapping
+// ---------------------------------------------------------------------------
+const VALID_PHASES = new Set([
+    "context_collecting",
+    "context_analyzing",
+    "extension_generating",
+    "extension_validating",
+]);
+function mapPhase(agentPhase) {
+    if (VALID_PHASES.has(agentPhase)) {
+        return agentPhase;
+    }
+    return "context_analyzing";
+}
+// ---------------------------------------------------------------------------
 // Handler
 // ---------------------------------------------------------------------------
 const AR_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes — generous for LLM-based analysis
@@ -107,7 +183,7 @@ const AR_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes — generous for LLM-based ana
  */
 export function handleAutoResearch(input, deps) {
     const { projectId, sessionId } = input;
-    const { store, relay } = deps;
+    const { store } = deps;
     const logger = deps.logger ?? console;
     // 1. Resolve the project from the store.
     const project = store.getProject(projectId);
@@ -120,25 +196,75 @@ export function handleAutoResearch(input, deps) {
         return;
     }
     const projectPath = project.path;
-    // 2. Emit start event
+    // 2. Load the auto-research agent definition.
+    const agentDef = findAgentDef(projectPath);
+    if (!agentDef) {
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: "Auto-research agent definition not found. " +
+                "Create .pi/agents/auto-research.md in the project or ~/.pi/agent/agents/auto-research.md.",
+        });
+        return;
+    }
+    // 3. Emit start event
     sendEvent(deps, sessionId, {
         type: "auto_research_start",
         projectId,
     });
-    // 3. Build the prompt for the auto-research agent.
-    //    The prompt instructs the agent to analyze the project and produce
-    //    JSON-line output describing suggested extensions.
-    const task = buildAutoResearchTask(projectPath, project.name);
-    // 4. Spawn pi subprocess.
-    //    We use --mode json for structured output, -p for non-interactive,
-    //    and pipe stdin/stdout.
-    const args = ["--mode", "json", "-p", "--no-session"];
+    // 4. Write the system prompt to a temp file so pi can load it via
+    //    --append-system-prompt (mirrors the subagent extension pattern).
+    let tmpDir = null;
+    let tmpPromptPath = null;
+    try {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "spectral-auto-research-"));
+        tmpPromptPath = path.join(tmpDir, "system-prompt.md");
+        fs.writeFileSync(tmpPromptPath, agentDef.systemPrompt, { encoding: "utf-8", mode: 0o600 });
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: `Failed to write system prompt temp file: ${msg}`,
+        });
+        if (tmpDir) {
+            try {
+                fs.rmSync(tmpDir, { recursive: true, force: true });
+            }
+            catch { /* ignore */ }
+        }
+        return;
+    }
+    // 5. Build the user task. This becomes a positional argument to pi,
+    //    matching how the subagent extension passes tasks.
+    const task = buildUserTask(projectPath, project.name);
+    // 6. Build spawn arguments (mirroring agent/index.ts subprocess spawn).
+    const args = [
+        "--mode", "json",
+        "-p",
+        "--no-session",
+        "--model", agentDef.model,
+        "--append-system-prompt", tmpPromptPath,
+    ];
+    // Add tools if defined in the agent definition. The auto-research agent
+    // uses read/grep/find/ls/bash/write/edit — its frontmatter should declare
+    // them so pi allows those tools.
+    // We don't extract tools from frontmatter currently because parseFrontmatter
+    // returns a generic Record. For now, auto-research always uses the default
+    // tool set (pi's full tool set is available by default in -p mode).
+    // TODO: when agent def frontmatter parsing is unified with agents.ts, also
+    //       pass --tools here.
+    // The user task is the last positional argument — pi treats it as the
+    // initial prompt in -p mode.
+    args.push(`Task: ${task}`);
+    // 7. Spawn pi subprocess.
     const invocation = getPiInvocation(args);
     let child;
     try {
         child = spawn(invocation.command, invocation.args, {
             cwd: projectPath,
-            stdio: ["pipe", "pipe", "pipe"],
+            stdio: ["ignore", "pipe", "pipe"], // stdin ignored — task is positional
             env: { ...process.env },
             shell: false,
         });
@@ -150,9 +276,10 @@ export function handleAutoResearch(input, deps) {
             projectId,
             message: `Failed to spawn pi subprocess: ${msg}`,
         });
+        cleanupTemp(tmpDir);
         return;
     }
-    // 5. Set up timeout
+    // 8. Set up timeout
     const timeout = setTimeout(() => {
         killProcess(child);
         sendEvent(deps, sessionId, {
@@ -160,8 +287,18 @@ export function handleAutoResearch(input, deps) {
             projectId,
             message: "Auto-research timed out after 5 minutes",
         });
+        cleanupTemp(tmpDir);
     }, AR_TIMEOUT_MS);
-    // 6. Collect stdout lines and parse as JSON events
+    // 9. Collect stdout and parse pi's JSON-line output format.
+    //    pi in --mode json emits one JSON line per event:
+    //      {"type":"message_start",...}
+    //      {"type":"text_delta","content":"..."}
+    //      {"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"..."}]}}
+    //      {"type":"agent_end",...}
+    //
+    //    The auto-research agent's output is inside assistant message_end
+    //    events. We extract the text and try to parse it as auto-research
+    //    event JSON (progress, extension_generated, done, error).
     let stdoutBuffer = "";
     const discoveredExtensions = [];
     let stderrBuffer = "";
@@ -169,34 +306,102 @@ export function handleAutoResearch(input, deps) {
         stdoutBuffer += chunk.toString("utf-8");
         // Process complete lines
         const lines = stdoutBuffer.split("\n");
-        // The last element may be an incomplete line — keep it in the buffer
         stdoutBuffer = lines.pop() ?? "";
         for (const rawLine of lines) {
             const line = rawLine.trim();
             if (!line)
                 continue;
-            let parsed;
+            let event;
             try {
-                parsed = JSON.parse(line);
+                event = JSON.parse(line);
             }
             catch {
-                // Non-JSON output — could be a log line. Skip.
+                // Non-JSON output — ignore.
                 continue;
             }
-            processArLine(parsed, discoveredExtensions, projectId, sessionId, deps);
+            // Only process message_end events from the assistant.
+            if (event.type !== "message_end" || !event.message)
+                continue;
+            if (event.message.role !== "assistant")
+                continue;
+            const content = event.message.content;
+            if (!content || !Array.isArray(content) || content.length === 0)
+                continue;
+            // Extract text blocks from the assistant message
+            for (const block of content) {
+                if (block.type !== "text" || typeof block.text !== "string")
+                    continue;
+                // Try to parse the assistant's text output as one or more JSON
+                // auto-research events. The agent may output multiple JSON objects
+                // in a single assistant message (separated by newlines or
+                // concatenated). We try parsing the full text first, then fall
+                // back to line-by-line.
+                const text = block.text.trim();
+                // First, try treating the entire text block as a single event
+                let parsed = null;
+                try {
+                    parsed = JSON.parse(text);
+                }
+                catch {
+                    // Not a single JSON object — try line-by-line
+                }
+                if (parsed && parsed.type) {
+                    processArEvent(parsed, discoveredExtensions, projectId, sessionId, deps);
+                }
+                else {
+                    // Try each line individually (the agent may emit multi-line
+                    // JSON event output, e.g. one event per line).
+                    for (const subLine of text.split("\n")) {
+                        const trimmed = subLine.trim();
+                        if (!trimmed)
+                            continue;
+                        try {
+                            const eventLine = JSON.parse(trimmed);
+                            if (eventLine && eventLine.type) {
+                                processArEvent(eventLine, discoveredExtensions, projectId, sessionId, deps);
+                            }
+                        }
+                        catch {
+                            // skip non-JSON lines
+                        }
+                    }
+                }
+            }
         }
     });
     child.stderr?.on("data", (chunk) => {
         stderrBuffer += chunk.toString("utf-8");
     });
-    // 7. Handle process exit
+    // 10. Handle process exit
     child.on("close", (code) => {
         clearTimeout(timeout);
-        // Process any remaining buffered stdout
+        // Process any remaining buffered stdout (drain the buffer)
         if (stdoutBuffer.trim()) {
             try {
-                const parsed = JSON.parse(stdoutBuffer.trim());
-                processArLine(parsed, discoveredExtensions, projectId, sessionId, deps);
+                const event = JSON.parse(stdoutBuffer.trim());
+                if (event.type === "message_end" && event.message?.role === "assistant") {
+                    const content = event.message?.content;
+                    if (Array.isArray(content)) {
+                        for (const block of content) {
+                            if (block.type === "text" && typeof block.text === "string") {
+                                for (const subLine of block.text.split("\n")) {
+                                    const trimmed = subLine.trim();
+                                    if (!trimmed)
+                                        continue;
+                                    try {
+                                        const ar = JSON.parse(trimmed);
+                                        if (ar && ar.type) {
+                                            processArEvent(ar, discoveredExtensions, projectId, sessionId, deps);
+                                        }
+                                    }
+                                    catch {
+                                        /* skip */
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
             }
             catch {
                 // ignore
@@ -212,6 +417,7 @@ export function handleAutoResearch(input, deps) {
                 projectId,
                 message: `Auto-research subprocess exited with code ${code}${errDetail}`,
             });
+            cleanupTemp(tmpDir);
             return;
         }
         // Emit completion with any discovered extensions (even partial — the
@@ -225,6 +431,7 @@ export function handleAutoResearch(input, deps) {
         if (stderrBuffer) {
             logger.error?.(`[auto-research] subprocess stderr (code=${code}): ${stderrBuffer.slice(0, 1000)}`);
         }
+        cleanupTemp(tmpDir);
     });
     child.on("error", (err) => {
         clearTimeout(timeout);
@@ -233,15 +440,13 @@ export function handleAutoResearch(input, deps) {
             projectId,
             message: `Auto-research subprocess error: ${err.message}`,
         });
+        cleanupTemp(tmpDir);
     });
-    // 8. Write the task to stdin and close
-    child.stdin?.write(task);
-    child.stdin?.end();
 }
 // ---------------------------------------------------------------------------
-// Line processor
+// Event processor
 // ---------------------------------------------------------------------------
-function processArLine(parsed, extensions, projectId, sessionId, deps) {
+function processArEvent(parsed, extensions, projectId, sessionId, deps) {
     const t = parsed.type;
     if (t === "progress") {
         const p = parsed;
@@ -262,6 +467,13 @@ function processArLine(parsed, extensions, projectId, sessionId, deps) {
             usesLLM: eg.usesLLM,
             fileCount: eg.fileCount,
         });
+        // Also emit as a progress update so the UI shows real-time activity
+        sendEvent(deps, sessionId, {
+            type: "auto_research_progress",
+            projectId,
+            phase: "extension_generating",
+            message: `Generated: ${eg.name}`,
+        });
     }
     else if (t === "done") {
         const d = parsed;
@@ -281,77 +493,38 @@ function processArLine(parsed, extensions, projectId, sessionId, deps) {
     // Unknown event types are silently ignored for forward compatibility
 }
 // ---------------------------------------------------------------------------
-// Phase mapping
-// ---------------------------------------------------------------------------
-const VALID_PHASES = new Set([
-    "context_collecting",
-    "context_analyzing",
-    "extension_generating",
-    "extension_validating",
-]);
-function mapPhase(agentPhase) {
-    if (VALID_PHASES.has(agentPhase)) {
-        return agentPhase;
-    }
-    return "context_analyzing";
-}
-// ---------------------------------------------------------------------------
 // Task builder
 // ---------------------------------------------------------------------------
 /**
- * Build the auto-research task prompt that will be sent to pi's stdin.
- *
- * The prompt instructs the auto-research subagent to:
- *   1. Collect context about the project
- *   2. Analyze what extensions would be useful
- *   3. Generate extension definitions
- *   4. Report results as structured JSON lines
+ * Build the user task prompt sent as positional argument to pi.
+ * The system prompt (from the agent definition) provides the detailed
+ * instructions; this is just the project-specific context.
  */
-function buildAutoResearchTask(projectPath, projectName) {
-    return JSON.stringify({
-        type: "auto_research",
-        projectPath,
-        projectName,
-        instructions: `You are an auto-research agent. Analyze the project at "${projectPath}" named "${projectName}" to determine what custom pi coding agent extensions would accelerate development.
-
-Follow these steps:
-
-1. **Context Collection** (emit progress: phase="context_collecting")
-   - Read the project's package.json, tsconfig.json, and any config files
-   - List the source directory structure (src/, packages/, etc.)
-   - Read existing .pi/extensions/ if any exist
-   - Note the tech stack (frameworks, databases, tools)
-   - Check git state (branch, recent commits) for workflow patterns
-
-2. **Analysis** (emit progress: phase="context_analyzing")
-   - Identify repetitive patterns in the codebase
-   - Find areas where automation would save developer time
-   - Consider common workflows: testing, linting, deployment, code review
-   - Evaluate which AI-powered extensions (LLM calls) would add value
-   - Prioritize extensions by impact vs. implementation complexity
-
-3. **Extension Generation** (emit progress: phase="extension_generating")
-   Generate .ts extension files under .pi/extensions/auto-research/ for each high-value extension. For each extension, emit:
-     {"type":"extension_generated","name":"...","path":"...","description":"...","usesLLM":bool,"fileCount":n}
-
-   Extension categories to consider:
-   A. Workflow automation (test runner, linter integration, git hooks)
-   B. Code generation (component scaffolding, API route generators)
-   C. Project-specific tools (database migrations, schema generators)
-   D. Quality & review (code reviewer, error explainer, type checker)
-   E. Documentation (doc generator, changelog, API reference)
-   F. LLM-powered (commit message generator, PR description, architecture advisor)
-   G. Stateful (session memory extensions, usage dashboard, adaptive guidelines)
-
-4. **Validation** (emit progress: phase="extension_validating")
-   - Verify generated extensions follow pi extension API patterns
-   - Ensure imports resolve against available npm packages
-
-5. **Completion**
-   When done, emit:
-     {"type":"done","extensions":[...]}
-   Then exit.
-
-IMPORTANT: Output ONLY JSON lines (one per line). Do not emit markdown, explanations, or any other text outside of JSON. Each line must be valid JSON on a single line. Your output will be parsed by a machine, not a human.`,
-    });
+function buildUserTask(projectPath, projectName) {
+    return [
+        `Analyze the project at "${projectPath}" named "${projectName}" to determine`,
+        `what custom pi coding agent extensions would accelerate development.`,
+        ``,
+        `Follow your system prompt instructions for the full process:`,
+        `1. Context collection — scan the project structure`,
+        `2. Analysis — identify patterns and automation opportunities`,
+        `3. Extension generation — create .ts extension files`,
+        `4. Validation — verify the extensions are correct`,
+        ``,
+        `Important: Read the template library at .pi/agents/auto-research-templates.md`,
+        `if it exists, for ready-to-adapt extension templates.`,
+    ].join("\n");
+}
+// ---------------------------------------------------------------------------
+// Temp file cleanup
+// ---------------------------------------------------------------------------
+function cleanupTemp(tmpDir) {
+    if (!tmpDir)
+        return;
+    try {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+    catch {
+        // best-effort cleanup
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aexol/spectral",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "description": "Always-on coding agent for Aexol — branded pi wrapper with relay-based browser access.",
   "type": "module",
   "private": false,