@aexol/spectral 0.6.0 → 0.6.2

package/dist/commands/serve.js

@@ -38,7 +38,7 @@ import { fileURLToPath } from "node:url";
 import { getConfigDir } from "../config.js";
 import { requireLogin } from "../preflight.js";
 import { RelayClient } from "../relay/client.js";
-import { detachAllSubscribers, handleCancelTurn, handleClientMessage, handleRestRequest, handleSubscribe, } from "../relay/dispatcher.js";
+import { detachAllSubscribers, handleAutoResearchFrame, handleCancelTurn, handleClientMessage, handleRestRequest, handleSubscribe, } from "../relay/dispatcher.js";
 import { ensureMachineRegistered } from "../relay/registration.js";
 import { SessionStreamManager } from "../server/session-stream.js";
 import { gracefulShutdown } from "../server/shutdown.js";
@@ -260,6 +260,15 @@ export async function runServe(opts = {}) {
             handleCancelTurn(frame, { manager });
             return;
         }
+        if (frame.kind === "auto_research") {
+            handleAutoResearchFrame(frame, {
+                store,
+                manager,
+                relay,
+                cwd,
+            });
+            return;
+        }
         // Other frames (error, machine_disconnected addressed to us, etc.)
         // are ignored at this layer. Future batches may surface them in
         // structured logs.

package/dist/relay/auto-research.js

@@ -0,0 +1,357 @@
+/**
+ * Auto-research handler — spawns an isolated pi subprocess to analyze a
+ * project and generate custom extensions.
+ *
+ * Flow:
+ *   1. Resolve project path from the SQLite store
+ *   2. Spawn pi with --mode json -p running the auto-research agent
+ *   3. Parse JSON-line output from the subprocess
+ *   4. Map subprocess events to `auto_research_*` ServerEvent types
+ *   5. Stream progress via the relay to the browser
+ *   6. On completion, emit `auto_research_complete` with generated extensions
+ *
+ * The subprocess runs with `cwd` set to the project root so all file
+ * reads/writes are relative to the project. The agent definition markdown
+ * file lives alongside other agent defs and gets loaded by pi's subagent
+ * infrastructure.
+ */
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Detect the pi/spectral binary for spawning subprocesses.
+ * Mirrors the logic in `agent/index.ts#getPiInvocation`.
+ */
+function getPiInvocation(subagentArgs) {
+    const currentScript = process.argv[1];
+    // Case 1: running via tsx with an existing script file
+    if (currentScript && fs.existsSync(currentScript)) {
+        return { command: process.execPath, args: [currentScript, ...subagentArgs] };
+    }
+    // Case 2: check if this is the spectral/aexol wrapper binary
+    const execName = path.basename(process.execPath).toLowerCase();
+    if (execName === "spectral" || execName === "aexol") {
+        return { command: process.execPath, args: subagentArgs };
+    }
+    // Case 3: generic node — try spectral first, then pi
+    function hasBin(name) {
+        const PATH = process.env.PATH || "";
+        const envPathSep = process.platform === "win32" ? ";" : ":";
+        for (const dir of PATH.split(envPathSep)) {
+            const candidate = path.join(dir, name);
+            try {
+                if (fs.existsSync(candidate))
+                    return true;
+            }
+            catch {
+                /* skip */
+            }
+        }
+        return false;
+    }
+    if (hasBin("spectral")) {
+        return { command: "spectral", args: subagentArgs };
+    }
+    if (hasBin("pi")) {
+        return { command: "pi", args: subagentArgs };
+    }
+    return { command: "pi", args: subagentArgs };
+}
+/**
+ * Send a ServerEvent to the browser via the relay on the auto-research session.
+ */
+function sendEvent(deps, sessionId, event) {
+    deps.relay.send({
+        kind: "ws_event",
+        sessionId,
+        event,
+    });
+}
+/**
+ * Safely kill a child process. Best-effort — errors are silently swallowed.
+ */
+function killProcess(child) {
+    try {
+        if (!child.killed && child.exitCode === null) {
+            child.kill("SIGTERM");
+            // Give it 2 seconds to clean up, then force-kill
+            setTimeout(() => {
+                try {
+                    if (!child.killed && child.exitCode === null) {
+                        child.kill("SIGKILL");
+                    }
+                }
+                catch {
+                    // ignore
+                }
+            }, 2000);
+        }
+    }
+    catch {
+        // ignore
+    }
+}
+// ---------------------------------------------------------------------------
+// Handler
+// ---------------------------------------------------------------------------
+const AR_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes — generous for LLM-based analysis
+/**
+ * Execute auto-research for a project.
+ *
+ * Caller (dispatcher) is fire-and-forget — this function is `void` and
+ * all errors are surfaced as `auto_research_error` events on the wire
+ * rather than thrown.
+ */
+export function handleAutoResearch(input, deps) {
+    const { projectId, sessionId } = input;
+    const { store, relay } = deps;
+    const logger = deps.logger ?? console;
+    // 1. Resolve the project from the store.
+    const project = store.getProject(projectId);
+    if (!project) {
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: `Project not found: ${projectId}`,
+        });
+        return;
+    }
+    const projectPath = project.path;
+    // 2. Emit start event
+    sendEvent(deps, sessionId, {
+        type: "auto_research_start",
+        projectId,
+    });
+    // 3. Build the prompt for the auto-research agent.
+    //    The prompt instructs the agent to analyze the project and produce
+    //    JSON-line output describing suggested extensions.
+    const task = buildAutoResearchTask(projectPath, project.name);
+    // 4. Spawn pi subprocess.
+    //    We use --mode json for structured output, -p for non-interactive,
+    //    and pipe stdin/stdout.
+    const args = ["--mode", "json", "-p", "--no-session"];
+    const invocation = getPiInvocation(args);
+    let child;
+    try {
+        child = spawn(invocation.command, invocation.args, {
+            cwd: projectPath,
+            stdio: ["pipe", "pipe", "pipe"],
+            env: { ...process.env },
+            shell: false,
+        });
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: `Failed to spawn pi subprocess: ${msg}`,
+        });
+        return;
+    }
+    // 5. Set up timeout
+    const timeout = setTimeout(() => {
+        killProcess(child);
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: "Auto-research timed out after 5 minutes",
+        });
+    }, AR_TIMEOUT_MS);
+    // 6. Collect stdout lines and parse as JSON events
+    let stdoutBuffer = "";
+    const discoveredExtensions = [];
+    let stderrBuffer = "";
+    child.stdout?.on("data", (chunk) => {
+        stdoutBuffer += chunk.toString("utf-8");
+        // Process complete lines
+        const lines = stdoutBuffer.split("\n");
+        // The last element may be an incomplete line — keep it in the buffer
+        stdoutBuffer = lines.pop() ?? "";
+        for (const rawLine of lines) {
+            const line = rawLine.trim();
+            if (!line)
+                continue;
+            let parsed;
+            try {
+                parsed = JSON.parse(line);
+            }
+            catch {
+                // Non-JSON output — could be a log line. Skip.
+                continue;
+            }
+            processArLine(parsed, discoveredExtensions, projectId, sessionId, deps);
+        }
+    });
+    child.stderr?.on("data", (chunk) => {
+        stderrBuffer += chunk.toString("utf-8");
+    });
+    // 7. Handle process exit
+    child.on("close", (code) => {
+        clearTimeout(timeout);
+        // Process any remaining buffered stdout
+        if (stdoutBuffer.trim()) {
+            try {
+                const parsed = JSON.parse(stdoutBuffer.trim());
+                processArLine(parsed, discoveredExtensions, projectId, sessionId, deps);
+            }
+            catch {
+                // ignore
+            }
+        }
+        if (code !== 0 && discoveredExtensions.length === 0) {
+            // Subprocess exited with error and no extensions were generated
+            const errDetail = stderrBuffer
+                ? ` (stderr: ${stderrBuffer.slice(0, 500)})`
+                : "";
+            sendEvent(deps, sessionId, {
+                type: "auto_research_error",
+                projectId,
+                message: `Auto-research subprocess exited with code ${code}${errDetail}`,
+            });
+            return;
+        }
+        // Emit completion with any discovered extensions (even partial — the
+        // UI can show what was generated before the process died).
+        sendEvent(deps, sessionId, {
+            type: "auto_research_complete",
+            projectId,
+            extensions: discoveredExtensions,
+        });
+        // Log any stderr for debugging
+        if (stderrBuffer) {
+            logger.error?.(`[auto-research] subprocess stderr (code=${code}): ${stderrBuffer.slice(0, 1000)}`);
+        }
+    });
+    child.on("error", (err) => {
+        clearTimeout(timeout);
+        sendEvent(deps, sessionId, {
+            type: "auto_research_error",
+            projectId,
+            message: `Auto-research subprocess error: ${err.message}`,
+        });
+    });
+    // 8. Write the task to stdin and close
+    child.stdin?.write(task);
+    child.stdin?.end();
+}
+// ---------------------------------------------------------------------------
+// Line processor
+// ---------------------------------------------------------------------------
+function processArLine(parsed, extensions, projectId, sessionId, deps) {
+    const t = parsed.type;
+    if (t === "progress") {
+        const p = parsed;
+        const wirePhase = mapPhase(p.phase);
+        sendEvent(deps, sessionId, {
+            type: "auto_research_progress",
+            projectId,
+            phase: wirePhase,
+            message: p.message,
+        });
+    }
+    else if (t === "extension_generated") {
+        const eg = parsed;
+        extensions.push({
+            name: eg.name,
+            path: eg.path,
+            description: eg.description,
+            usesLLM: eg.usesLLM,
+            fileCount: eg.fileCount,
+        });
+    }
+    else if (t === "done") {
+        const d = parsed;
+        for (const ext of d.extensions) {
+            extensions.push(ext);
+        }
+    }
+    else if (t === "error") {
+        const e = parsed;
+        sendEvent(deps, sessionId, {
+            type: "auto_research_progress",
+            projectId,
+            phase: "extension_validating",
+            message: `Error: ${e.message}`,
+        });
+    }
+    // Unknown event types are silently ignored for forward compatibility
+}
+// ---------------------------------------------------------------------------
+// Phase mapping
+// ---------------------------------------------------------------------------
+const VALID_PHASES = new Set([
+    "context_collecting",
+    "context_analyzing",
+    "extension_generating",
+    "extension_validating",
+]);
+function mapPhase(agentPhase) {
+    if (VALID_PHASES.has(agentPhase)) {
+        return agentPhase;
+    }
+    return "context_analyzing";
+}
+// ---------------------------------------------------------------------------
+// Task builder
+// ---------------------------------------------------------------------------
+/**
+ * Build the auto-research task prompt that will be sent to pi's stdin.
+ *
+ * The prompt instructs the auto-research subagent to:
+ *   1. Collect context about the project
+ *   2. Analyze what extensions would be useful
+ *   3. Generate extension definitions
+ *   4. Report results as structured JSON lines
+ */
+function buildAutoResearchTask(projectPath, projectName) {
+    return JSON.stringify({
+        type: "auto_research",
+        projectPath,
+        projectName,
+        instructions: `You are an auto-research agent. Analyze the project at "${projectPath}" named "${projectName}" to determine what custom pi coding agent extensions would accelerate development.
+
+Follow these steps:
+
+1. **Context Collection** (emit progress: phase="context_collecting")
+   - Read the project's package.json, tsconfig.json, and any config files
+   - List the source directory structure (src/, packages/, etc.)
+   - Read existing .pi/extensions/ if any exist
+   - Note the tech stack (frameworks, databases, tools)
+   - Check git state (branch, recent commits) for workflow patterns
+
+2. **Analysis** (emit progress: phase="context_analyzing")
+   - Identify repetitive patterns in the codebase
+   - Find areas where automation would save developer time
+   - Consider common workflows: testing, linting, deployment, code review
+   - Evaluate which AI-powered extensions (LLM calls) would add value
+   - Prioritize extensions by impact vs. implementation complexity
+
+3. **Extension Generation** (emit progress: phase="extension_generating")
+   Generate .ts extension files under .pi/extensions/auto-research/ for each high-value extension. For each extension, emit:
+     {"type":"extension_generated","name":"...","path":"...","description":"...","usesLLM":bool,"fileCount":n}
+
+   Extension categories to consider:
+   A. Workflow automation (test runner, linter integration, git hooks)
+   B. Code generation (component scaffolding, API route generators)
+   C. Project-specific tools (database migrations, schema generators)
+   D. Quality & review (code reviewer, error explainer, type checker)
+   E. Documentation (doc generator, changelog, API reference)
+   F. LLM-powered (commit message generator, PR description, architecture advisor)
+   G. Stateful (session memory extensions, usage dashboard, adaptive guidelines)
+
+4. **Validation** (emit progress: phase="extension_validating")
+   - Verify generated extensions follow pi extension API patterns
+   - Ensure imports resolve against available npm packages
+
+5. **Completion**
+   When done, emit:
+     {"type":"done","extensions":[...]}
+   Then exit.
+
+IMPORTANT: Output ONLY JSON lines (one per line). Do not emit markdown, explanations, or any other text outside of JSON. Each line must be valid JSON on a single line. Your output will be parsed by a machine, not a human.`,
+    });
+}

package/dist/relay/dispatcher.js

@@ -41,8 +41,9 @@
 import { BadRequestError, NotFoundError } from "../server/handlers/errors.js";
 import { handlePathAutocomplete } from "../server/handlers/paths-autocomplete.js";
 import { handleCreateProject, handleDeleteProject, handleListProjects, handleListSessionsByProject, handleUpdateProject, } from "../server/handlers/projects.js";
-import { handleCompactSession, handleCreateSession, handleDeleteSession, handleForkSession, handleGetSessionDetail, handleGetSessionMemoryStatus, handleUpdateSession, } from "../server/handlers/sessions.js";
+import { handleCompactSession, handleCreateSession, handleDeleteSession, handleForkSession, handleGetSessionDetail, handleGetSessionMemoryDetails, handleGetSessionMemoryStatus, handleUpdateSession, } from "../server/handlers/sessions.js";
 import { shutdownState } from "../server/shutdown.js";
+import { handleAutoResearch } from "./auto-research.js";
 /**
  * Inline path matcher. Returns `null` for any path/method combination we
  * don't recognise; the caller turns that into a `404 Unknown route`.
@@ -103,13 +104,15 @@ export function matchRoute(method, path) {
             return { route: "delete_session", id };
         return null;
     }
-    // /api/sessions/:id/memory and /api/sessions/:id/compact
-    const sessionActionMatch = /^\/api\/sessions\/([^/]+)\/(memory|compact)$/.exec(cleanPath);
+    // /api/sessions/:id/memory/details and /api/sessions/:id/memory and /api/sessions/:id/compact
+    const sessionActionMatch = /^\/api\/sessions\/([^/]+)\/(memory(?:\/details)?|compact)$/.exec(cleanPath);
     if (sessionActionMatch) {
         const id = decodeURIComponent(sessionActionMatch[1]);
         const action = sessionActionMatch[2];
         if (action === "memory" && method === "GET")
             return { route: "get_session_memory", id };
+        if (action === "memory/details" && method === "GET")
+            return { route: "get_session_memory_details", id };
         if (action === "compact" && method === "POST")
             return { route: "compact_session", id };
         return null;
@@ -269,6 +272,8 @@ async function dispatchRoute(match, body, deps) {
             return handleGetSessionDetail(store, id);
         case "get_session_memory":
             return handleGetSessionMemoryStatus(store, manager, id);
+        case "get_session_memory_details":
+            return handleGetSessionMemoryDetails(store, manager, id);
         case "update_session": {
             const session = handleUpdateSession(store, id, asObject(body));
             safePublish(publishMetaEvent, logger, {
@@ -627,3 +632,22 @@ export function detachAllSubscribers(manager, subscribers) {
     }
     subscribers.clear();
 }
+/**
+ * Dispatch an `auto_research` frame. Spawns an isolated pi subprocess that:
+ *   1. Collects project context (files, existing extensions, git state)
+ *   2. Analyzes the project using an LLM (via subagent)
+ *   3. Generates extension `.ts` files under `.pi/extensions/auto-research/`
+ *   4. Hot-reloads pi so new extensions take effect immediately
+ *
+ * Progress is streamed back as `ws_event` frames carrying
+ * `auto_research_*` ServerEvent types on the `sessionId` channel.
+ *
+ * Errors are surfaced as `auto_research_error` events; the underlying
+ * subprocess is killed on error to prevent zombie processes.
+ */
+export function handleAutoResearchFrame(frame, deps) {
+    handleAutoResearch({
+        projectId: frame.projectId,
+        sessionId: frame.sessionId,
+    }, deps);
+}

package/dist/server/handlers/sessions.js

@@ -46,6 +46,12 @@ export function handleGetSessionMemoryStatus(store, manager, id) {
         throw new NotFoundError("Session not found");
     return manager.getSessionMemoryStatus(id);
 }
+export function handleGetSessionMemoryDetails(store, manager, id) {
+    const detail = store.getSession(id);
+    if (!detail)
+        throw new NotFoundError("Session not found");
+    return manager.getSessionMemoryDetails(id);
+}
 export async function handleCompactSession(store, manager, id) {
     const detail = store.getSession(id);
     if (!detail)

package/dist/server/session-stream.js

@@ -42,7 +42,7 @@ import { getMemoryState, isSourceEntry, rawTokensSinceLastBound, rawTokensSinceL
 import { observationPoolTokens, renderSummary } from "../memory/compaction.js";
 import { loadConfig } from "../memory/config.js";
 import { estimateStringTokens } from "../memory/tokens.js";
-import { reflectionContent } from "../memory/types.js";
+import { reflectionContent, reflectionId } from "../memory/types.js";
 import { generateSessionTitle, isDefaultTitle, } from "./title-generator.js";
 const DEFAULT_BRIDGE_FACTORY = (args) => new PiBridge(args);
 /** Safety limit for autonomous loop iterations per session. */
@@ -373,6 +373,61 @@ export class SessionStreamManager {
         }
         return status;
     }
+    getSessionMemoryDetails(sessionId) {
+        const detail = this.store.getSession(sessionId);
+        if (!detail)
+            throw new Error(`Unknown sessionId: ${sessionId}`);
+        const stream = this.streams.get(sessionId);
+        const entries = stream?.bridge.getSessionBranch?.();
+        let observations = [];
+        let pendingObs = [];
+        let reflections = [];
+        let summary = null;
+        if (entries) {
+            const memoryState = getMemoryState(entries);
+            observations = memoryState.committedObs;
+            pendingObs = memoryState.pendingObs;
+            reflections = memoryState.reflections;
+            summary = renderSummary(reflections, [...observations, ...pendingObs]);
+        }
+        else {
+            const snapshot = this.store.getSessionMemorySnapshot(sessionId);
+            if (snapshot) {
+                observations = snapshot.details.observations;
+                reflections = snapshot.details.reflections;
+                summary = snapshot.summary;
+            }
+        }
+        const pendingIds = new Set(pendingObs.map((o) => o.id));
+        const wireObservations = [
+            ...observations.map((o) => ({
+                id: o.id,
+                content: o.content,
+                timestamp: o.timestamp,
+                relevance: o.relevance,
+                pending: false,
+            })),
+            ...pendingObs.map((o) => ({
+                id: o.id,
+                content: o.content,
+                timestamp: o.timestamp,
+                relevance: o.relevance,
+                pending: true,
+            })),
+        ];
+        const wireReflections = reflections.map((r) => ({
+            id: reflectionId(r) ?? "legacy",
+            content: reflectionContent(r),
+            supportingObservationIds: typeof r === "object" && "supportingObservationIds" in r
+                ? r.supportingObservationIds
+                : [],
+        }));
+        return {
+            summary,
+            observations: wireObservations,
+            reflections: wireReflections,
+        };
+    }
     async compactSession(sessionId) {
         if (this.disposed)
             throw new Error("SessionStreamManager disposed");
@@ -839,14 +894,23 @@ export class SessionStreamManager {
         const parts = [];
         if (stream.loopGoal) {
             parts.push(`[GOAL]: ${stream.loopGoal}`);
-            parts.push("");
         }
         parts.push(stream.loopOriginalPrompt);
-        if (stream.loopGoal) {
-            parts.push("");
-            parts.push("After completing your work, evaluate whether the goal has been FULLY achieved. " +
-                'Include <LOOP_DONE> in your response ONLY if you are confident the goal is completely met.');
-        }
+        parts.push("");
+        parts.push("--- LOOP INSTRUCTIONS ---\n" +
+            "You are running in an iterative loop. Each iteration you see your previous changes\n" +
+            "and can improve them further.\n" +
+            (stream.loopGoal
+                ? "The [GOAL] above defines what you need to achieve.\n"
+                : "\n") +
+            "CRITICAL: After EVERY response you MUST self-evaluate:\n" +
+            "- If the task IS FULLY COMPLETE (nothing meaningful left to do), " +
+            "respond with <LOOP_DONE> as the VERY FIRST text in your reply,\n" +
+            "  followed by a brief summary of what was accomplished.\n" +
+            "- If more work remains, continue working WITHOUT <LOOP_DONE>.\n" +
+            "- <LOOP_DONE> is the ONLY way to stop the loop. If you do not emit it,\n" +
+            "  you WILL be prompted again to keep working.\n" +
+            "Be decisive: if the task is done, stop. Do not keep polishing.");
         return parts.join("\n");
     }
     async sendNextLoopIteration(stream) {
@@ -1090,7 +1154,10 @@ export class SessionStreamManager {
             // iteratively improves the solution.
             if (stream.loopActive && stream.loopOriginalPrompt) {
                 const finishedAssistantText = finishedTurn?.assistantText ?? "";
-                if (finishedAssistantText.includes(LOOP_DONE_MARKER)) {
+                // Check for the loop-done marker in the first portion of the
+                // response (the prompt instructs the agent to emit it first).
+                const head = finishedAssistantText.slice(0, 500);
+                if (head.includes(LOOP_DONE_MARKER)) {
                     console.log(`[loop] completion marker detected after ${stream.loopIterationCount} iteration(s), stopping`);
                     const completedIterations = stream.loopIterationCount;
                     stream.loopActive = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aexol/spectral",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Always-on coding agent for Aexol — branded pi wrapper with relay-based browser access.",
   "type": "module",
   "private": false,