@neotx/core 0.1.0-alpha.0 → 0.1.0-alpha.1

package/dist/index.d.ts

@@ -132,6 +132,7 @@ declare const globalConfigSchema: z.ZodObject<{
         port: z.ZodDefault<z.ZodNumber>;
         secret: z.ZodOptional<z.ZodString>;
         idleIntervalMs: z.ZodDefault<z.ZodNumber>;
+        idleSkipMax: z.ZodDefault<z.ZodNumber>;
         heartbeatTimeoutMs: z.ZodDefault<z.ZodNumber>;
         maxConsecutiveFailures: z.ZodDefault<z.ZodNumber>;
         maxEventsPerSec: z.ZodDefault<z.ZodNumber>;
@@ -195,6 +196,7 @@ declare const neoConfigSchema: z.ZodObject<{
         port: z.ZodDefault<z.ZodNumber>;
         secret: z.ZodOptional<z.ZodString>;
         idleIntervalMs: z.ZodDefault<z.ZodNumber>;
+        idleSkipMax: z.ZodDefault<z.ZodNumber>;
         heartbeatTimeoutMs: z.ZodDefault<z.ZodNumber>;
         maxConsecutiveFailures: z.ZodDefault<z.ZodNumber>;
         maxEventsPerSec: z.ZodDefault<z.ZodNumber>;
@@ -867,6 +869,11 @@ declare class Orchestrator extends NeoEventEmitter {
     private preDispatchChecks;
     private buildDispatchContext;
     private executeStep;
+    /**
+     * Push the branch, then remove the worktree.
+     * Runs in `finally` so it executes on both success and failure.
+     */
+    private finalizeWorktree;
     private runAgentSession;
     private finalizeDispatch;
     private emitCostEvents;
@@ -1012,6 +1019,7 @@ declare const supervisorDaemonStateSchema: z.ZodObject<{
     totalCostUsd: z.ZodDefault<z.ZodNumber>;
     todayCostUsd: z.ZodDefault<z.ZodNumber>;
     costResetDate: z.ZodOptional<z.ZodString>;
+    idleSkipCount: z.ZodDefault<z.ZodNumber>;
     status: z.ZodDefault<z.ZodEnum<{
         running: "running";
         draining: "draining";
@@ -1072,7 +1080,7 @@ type QueuedEvent = {
 };
 
 declare class ActivityLog {
-    private readonly filePath;
+    readonly filePath: string;
     private readonly dir;
     constructor(dir: string);
     /**
@@ -1121,6 +1129,16 @@ declare class SupervisorDaemon {
 interface EventQueueOptions {
     maxEventsPerSec: number;
 }
+interface GroupedMessage {
+    text: string;
+    from: string;
+    count: number;
+}
+interface GroupedEvents {
+    messages: GroupedMessage[];
+    webhooks: QueuedEvent[];
+    runCompletions: QueuedEvent[];
+}
 /**
  * In-memory event queue with deduplication, rate limiting, and file watching.
  *
@@ -1151,6 +1169,11 @@ declare class EventQueue {
      * Drain all queued events and return them. Clears the queue.
      */
     drain(): QueuedEvent[];
+    /**
+     * Drain and group events: deduplicates messages by content,
+     * keeps webhooks and run completions separate.
+     */
+    drainAndGroup(): GroupedEvents;
     size(): number;
     /**
      * Start watching inbox.jsonl and events.jsonl for new entries.
@@ -1224,7 +1247,7 @@ declare class HeartbeatLoop {
     private loadInstructions;
     /** Route a single SDK stream message to the appropriate log handler. */
     private logStreamMessage;
-    /** Log thinking and plan blocks from assistant content. */
+    /** Log thinking and plan blocks from assistant content — no truncation. */
     private logContentBlocks;
     /** Log tool use events — distinguish MCP tools from built-in tools. */
     private logToolUse;
@@ -1235,17 +1258,16 @@ declare class HeartbeatLoop {
 
 /**
  * Load the supervisor memory from disk.
- * Returns empty string if no memory file exists yet.
+ * Migrates from legacy memory.md if needed.
  */
 declare function loadMemory(dir: string): Promise<string>;
 /**
  * Save the supervisor memory to disk (full overwrite).
+ * Automatically compacts if needed.
  */
 declare function saveMemory(dir: string, content: string): Promise<void>;
 /**
  * Extract memory content from Claude's response using <memory>...</memory> tags.
- * Handles both JSON and markdown content inside the tags.
- * Returns null if no memory block is found.
  */
 declare function extractMemoryFromResponse(response: string): string | null;
 /**
@@ -1259,8 +1281,9 @@ declare function checkMemorySize(content: string): {
 interface HeartbeatPromptOptions {
     repos: RepoConfig[];
     memory: string;
+    knowledge: string;
     memorySizeKB: number;
-    events: QueuedEvent[];
+    grouped: GroupedEvents;
     budgetStatus: {
         todayUsd: number;
         capUsd: number;

package/dist/index.js

@@ -510,6 +510,7 @@ var globalConfigSchema = z2.object({
     port: z2.number().default(7777),
     secret: z2.string().optional(),
     idleIntervalMs: z2.number().default(6e4),
+    idleSkipMax: z2.number().default(20),
     heartbeatTimeoutMs: z2.number().default(3e5),
     maxConsecutiveFailures: z2.number().default(3),
     maxEventsPerSec: z2.number().default(10),
@@ -518,6 +519,7 @@ var globalConfigSchema = z2.object({
   }).default({
     port: 7777,
     idleIntervalMs: 6e4,
+    idleSkipMax: 20,
     heartbeatTimeoutMs: 3e5,
     maxConsecutiveFailures: 3,
     maxEventsPerSec: 10,
@@ -846,6 +848,9 @@ function getBranchName(config, runId) {
   const sanitized = runId.toLowerCase().replace(/[^a-z0-9-]/g, "-");
   return `${prefix}/run-${sanitized}`;
 }
+async function pushWorktreeBranch(worktreePath, branch, remote) {
+  await withGitLock(worktreePath, () => git(worktreePath, ["push", "-u", remote, branch]));
+}
 
 // src/isolation/sandbox.ts
 import { resolve as resolve2 } from "path";
@@ -1261,7 +1266,14 @@ async function runSession(options) {
     let output = "";
     let costUsd = 0;
     let turnCount = 0;
-    const stream = sdk.query({ prompt, options: queryOptions });
+    const fullPrompt = agent.definition.prompt ? `${agent.definition.prompt}
+
+---
+
+## Task
+
+${prompt}` : prompt;
+    const stream = sdk.query({ prompt: fullPrompt, options: queryOptions });
     for await (const message of stream) {
       checkAborted(abortController.signal);
       const msg = message;
@@ -1779,6 +1791,9 @@ var Orchestrator = class extends NeoEventEmitter {
         attempt: 1
       };
     } finally {
+      if (worktreePath) {
+        await this.finalizeWorktree(worktreePath, ctx);
+      }
       this.semaphore.release(sessionId);
       this._activeSessions.delete(sessionId);
       this.abortControllers.delete(sessionId);
@@ -1788,6 +1803,24 @@ var Orchestrator = class extends NeoEventEmitter {
       }
     }
   }
+  /**
+   * Push the branch, then remove the worktree.
+   * Runs in `finally` so it executes on both success and failure.
+   */
+  async finalizeWorktree(worktreePath, ctx) {
+    const { runId, repoConfig } = ctx;
+    const branch = getBranchName(repoConfig, runId);
+    const remote = repoConfig.pushRemote ?? "origin";
+    try {
+      await pushWorktreeBranch(worktreePath, branch, remote).catch(() => {
+      });
+    } catch {
+    }
+    try {
+      await removeWorktree(worktreePath);
+    } catch {
+    }
+  }
   async runAgentSession(ctx, worktreePath) {
     const { input, runId, sessionId, stepName, stepDef, agent, activeSession } = ctx;
     const sandboxConfig = buildSandboxConfig(agent, worktreePath);
@@ -2141,6 +2174,7 @@ var supervisorDaemonStateSchema = z4.object({
   totalCostUsd: z4.number().default(0),
   todayCostUsd: z4.number().default(0),
   costResetDate: z4.string().optional(),
+  idleSkipCount: z4.number().default(0),
   status: z4.enum(["running", "draining", "stopped"]).default("running")
 });
 var webhookIncomingEventSchema = z4.object({
@@ -2302,6 +2336,36 @@ var EventQueue = class {
     this.queue.length = 0;
     return events;
   }
+  /**
+   * Drain and group events: deduplicates messages by content,
+   * keeps webhooks and run completions separate.
+   */
+  drainAndGroup() {
+    const events = this.drain();
+    const messageMap = /* @__PURE__ */ new Map();
+    const webhooks = [];
+    const runCompletions = [];
+    for (const event of events) {
+      if (event.kind === "message") {
+        const key = event.data.text.trim().toLowerCase();
+        const existing = messageMap.get(key);
+        if (existing) {
+          existing.count++;
+        } else {
+          messageMap.set(key, { text: event.data.text, from: event.data.from, count: 1 });
+        }
+      } else if (event.kind === "webhook") {
+        webhooks.push(event);
+      } else {
+        runCompletions.push(event);
+      }
+    }
+    return {
+      messages: [...messageMap.values()],
+      webhooks,
+      runCompletions
+    };
+  }
   size() {
     return this.queue.length;
   }
@@ -2458,21 +2522,74 @@ import { randomUUID as randomUUID3 } from "crypto";
 import { readFile as readFile9, writeFile as writeFile5 } from "fs/promises";
 import { homedir as homedir2 } from "os";
 import path12 from "path";
+import { fileURLToPath } from "url";
 
 // src/supervisor/memory.ts
-import { readFile as readFile8, writeFile as writeFile4 } from "fs/promises";
+import { appendFile as appendFile5, readFile as readFile8, rename as rename2, writeFile as writeFile4 } from "fs/promises";
 import path11 from "path";
-var MEMORY_FILE = "memory.md";
-var MAX_SIZE_KB = 10;
+var MEMORY_FILE = "memory.json";
+var KNOWLEDGE_FILE = "knowledge.json";
+var ARCHIVE_FILE = "memory-archive.jsonl";
+var LEGACY_FILE = "memory.md";
+var MAX_SIZE_KB = 6;
+var MAX_DECISIONS = 10;
+function parseStructuredMemory(raw) {
+  if (!raw.trim()) {
+    return emptyMemory();
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    return {
+      activeWork: parsed.activeWork ?? [],
+      blockers: parsed.blockers ?? [],
+      repoNotes: parsed.repoNotes ?? {},
+      recentDecisions: parsed.recentDecisions ?? [],
+      trackerSync: parsed.trackerSync ?? {},
+      notes: parsed.notes ?? ""
+    };
+  } catch {
+    return { ...emptyMemory(), notes: raw };
+  }
+}
+function emptyMemory() {
+  return {
+    activeWork: [],
+    blockers: [],
+    repoNotes: {},
+    recentDecisions: [],
+    trackerSync: {},
+    notes: ""
+  };
+}
+async function loadKnowledge(dir) {
+  try {
+    return await readFile8(path11.join(dir, KNOWLEDGE_FILE), "utf-8");
+  } catch {
+    return "";
+  }
+}
+async function saveKnowledge(dir, content) {
+  await writeFile4(path11.join(dir, KNOWLEDGE_FILE), content, "utf-8");
+}
 async function loadMemory(dir) {
   try {
     return await readFile8(path11.join(dir, MEMORY_FILE), "utf-8");
   } catch {
-    return "";
   }
+  try {
+    const legacy = await readFile8(path11.join(dir, LEGACY_FILE), "utf-8");
+    if (legacy.trim()) {
+      await writeFile4(path11.join(dir, MEMORY_FILE), legacy, "utf-8");
+      await rename2(path11.join(dir, LEGACY_FILE), path11.join(dir, `${LEGACY_FILE}.bak`));
+      return legacy;
+    }
+  } catch {
+  }
+  return "";
 }
 async function saveMemory(dir, content) {
-  await writeFile4(path11.join(dir, MEMORY_FILE), content, "utf-8");
+  const compacted = await compactMemory(dir, content);
+  await writeFile4(path11.join(dir, MEMORY_FILE), compacted, "utf-8");
 }
 function extractMemoryFromResponse(response) {
   const match = /<memory>([\s\S]*?)<\/memory>/i.exec(response);
@@ -2487,10 +2604,53 @@ function extractMemoryFromResponse(response) {
   }
   return content;
 }
+function extractKnowledgeFromResponse(response) {
+  const match = /<knowledge>([\s\S]*?)<\/knowledge>/i.exec(response);
+  if (!match?.[1]) return null;
+  return match[1].trim();
+}
 function checkMemorySize(content) {
   const sizeKB = Buffer.byteLength(content, "utf-8") / 1024;
   return { ok: sizeKB <= MAX_SIZE_KB, sizeKB: Math.round(sizeKB * 10) / 10 };
 }
+async function compactMemory(dir, content) {
+  if (!content.startsWith("{")) return content;
+  let parsed;
+  try {
+    parsed = parseStructuredMemory(content);
+  } catch {
+    return content;
+  }
+  let changed = false;
+  if (parsed.recentDecisions.length > MAX_DECISIONS) {
+    const toArchive = parsed.recentDecisions.slice(0, -MAX_DECISIONS);
+    parsed.recentDecisions = parsed.recentDecisions.slice(-MAX_DECISIONS);
+    changed = true;
+    const archivePath = path11.join(dir, ARCHIVE_FILE);
+    const entry = {
+      type: "decisions_archived",
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      decisions: toArchive
+    };
+    await appendFile5(archivePath, `${JSON.stringify(entry)}
+`, "utf-8");
+  }
+  const result = changed ? JSON.stringify(parsed, null, 2) : content;
+  const sizeKB = Buffer.byteLength(result, "utf-8") / 1024;
+  if (sizeKB > MAX_SIZE_KB && parsed.notes.length > 200) {
+    const archivePath = path11.join(dir, ARCHIVE_FILE);
+    const entry = {
+      type: "notes_archived",
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      notes: parsed.notes
+    };
+    await appendFile5(archivePath, `${JSON.stringify(entry)}
+`, "utf-8");
+    parsed.notes = "(archived \u2014 see memory-archive.jsonl)";
+    return JSON.stringify(parsed, null, 2);
+  }
+  return result;
+}
 
 // src/supervisor/prompt-builder.ts
 function buildHeartbeatPrompt(opts) {
@@ -2510,7 +2670,12 @@ Available commands (via bash):
   neo cost --short [--all]                         check budget
   neo agents                                       list available agents
 
-IMPORTANT: Always include a <memory>...</memory> block at the end of your response with your updated memory.`);
+IMPORTANT: Always include a <memory>...</memory> block at the end of your response with your updated memory.
+
+## Reporting
+Use the \`mcp__neo__report_progress\` tool to log your decisions, actions and blockers.
+Always report what you're doing and why \u2014 these logs are your audit trail.
+Types: "decision" (what you chose), "action" (what you did), "blocker" (what's stuck), "progress" (status update).`);
   if (opts.customInstructions) {
     sections.push(`## Custom instructions
 ${opts.customInstructions}`);
@@ -2539,15 +2704,33 @@ You can use these tools directly to query external systems.`
     sections.push(`## Active runs
 ${opts.activeRuns.map((r) => `- ${r}`).join("\n")}`);
   }
-  if (opts.events.length > 0) {
-    const eventDescriptions = opts.events.map(formatEvent);
-    sections.push(`## Pending events (${opts.events.length})
-${eventDescriptions.join("\n\n")}`);
+  const { messages, webhooks, runCompletions } = opts.grouped;
+  const totalEvents = messages.length + webhooks.length + runCompletions.length;
+  if (totalEvents > 0) {
+    const parts = [];
+    for (const msg of messages) {
+      const countSuffix = msg.count > 1 ? ` (\xD7${msg.count})` : "";
+      parts.push(`**Message from ${msg.from}${countSuffix}**: ${msg.text}`);
+    }
+    for (const evt of webhooks) {
+      parts.push(formatEvent(evt));
+    }
+    for (const evt of runCompletions) {
+      parts.push(formatEvent(evt));
+    }
+    sections.push(`## Pending events (${totalEvents})
+${parts.join("\n\n")}`);
   } else {
     sections.push(
       "## Pending events\nNo new events. This is an idle heartbeat \u2014 check on active runs if any, or wait."
     );
   }
+  if (opts.knowledge) {
+    sections.push(`## Reference knowledge (read-only)
+${opts.knowledge}
+
+To update knowledge, output a \`<knowledge>...</knowledge>\` block. Only update when reference data changes (API IDs, workspace config, etc.).`);
+  }
   sections.push(buildMemorySection(opts.memory, opts.memorySizeKB));
   return sections.join("\n\n---\n\n");
 }
@@ -2582,7 +2765,7 @@ function formatEvent(event) {
     case "webhook":
       return `**Webhook** [${event.data.source ?? "unknown"}] ${event.data.event ?? ""}
 \`\`\`json
-${JSON.stringify(event.data.payload ?? {}, null, 2).slice(0, 2e3)}
+${JSON.stringify(event.data.payload ?? {}, null, 2)}
 \`\`\``;
     case "message":
       return `**Message from ${event.data.from}**: ${event.data.text}`;
@@ -2660,15 +2843,27 @@ var HeartbeatLoop = class {
       await this.sleep(this.config.supervisor.idleIntervalMs);
       return;
     }
-    const events = this.eventQueue.drain();
+    const grouped = this.eventQueue.drainAndGroup();
+    const totalEventCount = grouped.messages.length + grouped.webhooks.length + grouped.runCompletions.length;
+    const idleSkipCount = state?.idleSkipCount ?? 0;
+    if (totalEventCount === 0 && idleSkipCount < this.config.supervisor.idleSkipMax) {
+      await this.updateState({ idleSkipCount: idleSkipCount + 1 });
+      await this.activityLog.log("heartbeat", `Idle skip #${idleSkipCount + 1} \u2014 no events`);
+      return;
+    }
+    if (idleSkipCount > 0) {
+      await this.updateState({ idleSkipCount: 0 });
+    }
     const memory = await loadMemory(this.supervisorDir);
+    const knowledge = await loadKnowledge(this.supervisorDir);
     const memoryCheck = checkMemorySize(memory);
     const mcpServerNames = this.config.mcpServers ? Object.keys(this.config.mcpServers) : [];
     const prompt = buildHeartbeatPrompt({
       repos: this.config.repos,
       memory,
+      knowledge,
       memorySizeKB: memoryCheck.sizeKB,
-      events,
+      grouped,
       budgetStatus: {
         todayUsd: todayCost,
         capUsd: this.config.supervisor.dailyCapUsd,
@@ -2682,8 +2877,10 @@ var HeartbeatLoop = class {
     });
     await this.activityLog.log("heartbeat", `Heartbeat #${state?.heartbeatCount ?? 0} starting`, {
       heartbeatId,
-      eventCount: events.length,
-      triggeredBy: events.map((e) => e.kind)
+      eventCount: totalEventCount,
+      messages: grouped.messages.length,
+      webhooks: grouped.webhooks.length,
+      runCompletions: grouped.runCompletions.length
     });
     const abortController = new AbortController();
     this.activeAbort = abortController;
@@ -2695,16 +2892,33 @@ var HeartbeatLoop = class {
     let turnCount = 0;
     try {
       const sdk = await import("@anthropic-ai/claude-agent-sdk");
+      const allowedTools = ["Bash", "Read", "mcp__neo__*"];
+      if (this.config.mcpServers) {
+        for (const name of Object.keys(this.config.mcpServers)) {
+          allowedTools.push(`mcp__${name}__*`);
+        }
+      }
+      const mcpInternalPath = path12.join(
+        path12.dirname(fileURLToPath(import.meta.url)),
+        "mcp-internal.js"
+      );
+      const mcpServers = {
+        neo: {
+          type: "stdio",
+          command: "node",
+          args: [mcpInternalPath],
+          env: { NEO_ACTIVITY_PATH: this.activityLog.filePath }
+        },
+        ...this.config.mcpServers ?? {}
+      };
       const queryOptions = {
         cwd: homedir2(),
         maxTurns: 50,
-        allowedTools: ["Bash", "Read"],
+        allowedTools,
         permissionMode: "bypassPermissions",
-        allowDangerouslySkipPermissions: true
+        allowDangerouslySkipPermissions: true,
+        mcpServers
       };
-      if (this.config.mcpServers) {
-        queryOptions.mcpServers = this.config.mcpServers;
-      }
       const stream = sdk.query({ prompt, options: queryOptions });
       for await (const message of stream) {
         if (abortController.signal.aborted) break;
@@ -2727,6 +2941,10 @@ var HeartbeatLoop = class {
     if (newMemory) {
       await saveMemory(this.supervisorDir, newMemory);
     }
+    const newKnowledge = extractKnowledgeFromResponse(output);
+    if (newKnowledge) {
+      await saveKnowledge(this.supervisorDir, newKnowledge);
+    }
     const durationMs = Date.now() - startTime;
     await this.updateState({
       sessionId: this.sessionId,
@@ -2745,7 +2963,7 @@ var HeartbeatLoop = class {
         durationMs,
         turnCount,
         memoryUpdated: !!newMemory,
-        responseSummary: output.slice(0, 500)
+        responseSummary: output
       }
     );
   }
@@ -2799,16 +3017,16 @@ var HeartbeatLoop = class {
       await this.logToolResult(msg, heartbeatId);
     }
   }
-  /** Log thinking and plan blocks from assistant content. */
+  /** Log thinking and plan blocks from assistant content — no truncation. */
   async logContentBlocks(msg, heartbeatId) {
     const content = msg.message?.content;
     if (!content) return;
     for (const block of content) {
       if (block.type === "thinking" && block.thinking) {
-        await this.activityLog.log("thinking", block.thinking.slice(0, 500), { heartbeatId });
+        await this.activityLog.log("thinking", block.thinking, { heartbeatId });
       }
       if (block.type === "text" && block.text) {
-        await this.activityLog.log("plan", block.text.slice(0, 500), { heartbeatId });
+        await this.activityLog.log("plan", block.text, { heartbeatId });
         break;
       }
     }
@@ -2841,7 +3059,7 @@ var HeartbeatLoop = class {
 
 // src/supervisor/webhook-server.ts
 import { timingSafeEqual } from "crypto";
-import { appendFile as appendFile5 } from "fs/promises";
+import { appendFile as appendFile6 } from "fs/promises";
 import { createServer } from "http";
 var MAX_BODY_SIZE = 1024 * 1024;
 var WebhookServer = class {
@@ -2925,7 +3143,7 @@ var WebhookServer = class {
       payload: parsed.payload ?? parsed,
       receivedAt: (/* @__PURE__ */ new Date()).toISOString()
     };
-    await appendFile5(this.eventsPath, `${JSON.stringify(event)}
+    await appendFile6(this.eventsPath, `${JSON.stringify(event)}
 `, "utf-8");
     this.onEvent(event);
     this.sendJson(res, 200, { ok: true, id: event.id });
@@ -2984,8 +3202,8 @@ var SupervisorDaemon = class {
     }
     const tempLock = `${lockPath}.${process.pid}`;
     await writeFile6(tempLock, String(process.pid), "utf-8");
-    const { rename: rename2 } = await import("fs/promises");
-    await rename2(tempLock, lockPath);
+    const { rename: rename3 } = await import("fs/promises");
+    await rename3(tempLock, lockPath);
     const existingState = await this.readState();
     if (existingState?.sessionId && existingState.status !== "stopped") {
       this.sessionId = existingState.sessionId;
@@ -3022,6 +3240,7 @@ var SupervisorDaemon = class {
       totalCostUsd: existingState?.totalCostUsd ?? 0,
       todayCostUsd: existingState?.todayCostUsd ?? 0,
       costResetDate: existingState?.costResetDate,
+      idleSkipCount: existingState?.idleSkipCount ?? 0,
       status: "running"
     });
     const shutdown = () => {