agent-sh 0.7.0 → 0.8.0

package/dist/agent/conversation-state.js

@@ -1,10 +1,22 @@
-/**
- * Manages the OpenAI chat messages array for the agent loop.
- * Separate from ContextManager — this is the LLM conversation,
- * not the shell history.
- */
+import { getSettings } from "../settings.js";
+import { toNuclearEntries, formatNuclearLine, isReadOnly, READ_ONLY_TOOLS, WRITE_TOOLS, } from "./nuclear-form.js";
 export class ConversationState {
+    // ── Tier 1: Active context ────────────────────────────────────
     messages = [];
+    // ── Tier 2: Nuclear memory ────────────────────────────────────
+    nuclearEntries = [];
+    recallArchive = new Map();
+    // ── Tier 3 reference ──────────────────────────────────────────
+    historyFile;
+    // ── Shared state ──────────────────────────────────────────────
+    nextSeq = 1;
+    constructor(historyFile) {
+        this.historyFile = historyFile ?? null;
+    }
+    get instanceId() {
+        return this.historyFile?.instanceId ?? "0000";
+    }
+    // ── Message API (unchanged) ───────────────────────────────────
     addUserMessage(text) {
         this.messages.push({ role: "user", content: text });
     }
@@ -31,29 +43,336 @@ export class ConversationState {
             content,
         });
     }
-    /** Inject a system-level note into the conversation (e.g. context change). */
     addSystemNote(text) {
         this.messages.push({ role: "user", content: text });
     }
     getMessages() {
         return this.messages;
     }
+    // ── Token estimation ──────────────────────────────────────────
+    estimateTokens() {
+        return Math.ceil(JSON.stringify(this.messages).length / 4);
+    }
+    // ── Tier 1 → Tier 2: Compaction ───────────────────────────────
+    /**
+     * Priority-based compaction. Evicts lowest-priority turns, replacing
+     * them with nuclear one-liner summaries that stay in the conversation.
+     * Read-only tool results are dropped entirely.
+     */
+    compact(targetTokens, recentTurnsToKeep = 10) {
+        const before = this.estimateTokens();
+        if (before <= targetTokens)
+            return null;
+        const turns = this.parseTurns();
+        if (turns.length <= 2)
+            return null;
+        // Assign priorities
+        const pinnedCount = Math.min(recentTurnsToKeep, turns.length - 1);
+        for (const turn of turns) {
+            turn.priority = this.inferPriority(turn.messages);
+        }
+        turns[0].priority = 4 /* Priority.PINNED */;
+        for (let i = turns.length - pinnedCount; i < turns.length; i++) {
+            turns[i].priority = 4 /* Priority.PINNED */;
+        }
+        // Sort candidates: lowest priority first, then oldest
+        const candidates = turns
+            .map((t, idx) => ({ turn: t, idx }))
+            .filter((c) => c.turn.priority !== 4 /* Priority.PINNED */)
+            .sort((a, b) => a.turn.priority - b.turn.priority || a.idx - b.idx);
+        // Evict until under budget
+        const evictedIndices = new Set();
+        let currentTokens = this.estimateTokens();
+        for (const c of candidates) {
+            if (currentTokens <= targetTokens)
+                break;
+            const turnTokens = Math.ceil(JSON.stringify(c.turn.messages).length / 4);
+            evictedIndices.add(c.idx);
+            currentTokens -= turnTokens;
+            // Generate nuclear entries from this turn
+            const entries = toNuclearEntries(c.turn.messages, this.nextSeq, this.instanceId);
+            this.nextSeq += entries.length;
+            for (const entry of entries) {
+                if (isReadOnly(entry)) {
+                    // Read-only: archive only (dropped from conversation), agent can re-read
+                    this.recallArchive.set(entry.seq, c.turn.messages);
+                }
+                else {
+                    // State-changing: keep nuclear one-liner in conversation + archive
+                    this.nuclearEntries.push(entry);
+                    this.recallArchive.set(entry.seq, c.turn.messages);
+                }
+            }
+        }
+        if (evictedIndices.size === 0)
+            return null;
+        // Rebuild: first turn + nuclear summary block + remaining turns
+        const rebuilt = [];
+        let insertedNuclearBlock = false;
+        for (let i = 0; i < turns.length; i++) {
+            if (evictedIndices.has(i)) {
+                if (!insertedNuclearBlock) {
+                    rebuilt.push(this.buildNuclearBlock());
+                    insertedNuclearBlock = true;
+                }
+            }
+            else {
+                rebuilt.push(...turns[i].messages);
+            }
+        }
+        // If no nuclear block was inserted but we have entries from prior compactions,
+        // update the existing nuclear block
+        if (!insertedNuclearBlock && this.nuclearEntries.length > 0) {
+            this.updateNuclearBlockInMessages(rebuilt);
+        }
+        this.messages = rebuilt;
+        return { before, after: this.estimateTokens() };
+    }
+    // ── Tier 2 → Tier 3: Flush ───────────────────────────────────
     /**
-     * Simple compaction — drop oldest turns, keeping the first user message
-     * (original task context) and the most recent turns.
+     * Flush oldest nuclear entries to the history file when the
+     * in-context nuclear block grows too large.
      */
-    compact(maxTurns) {
-        if (this.messages.length <= maxTurns * 2)
+    async flush() {
+        const maxEntries = getSettings().nuclearMaxEntries;
+        if (this.nuclearEntries.length <= maxEntries)
             return;
-        const first = this.messages[0];
-        const recent = this.messages.slice(-(maxTurns * 2));
-        this.messages = [
-            first,
-            { role: "user", content: "[Earlier conversation turns omitted for context space]" },
-            ...recent,
-        ];
+        const flushCount = this.nuclearEntries.length - maxEntries;
+        const toFlush = this.nuclearEntries.slice(0, flushCount);
+        // Write to history file
+        if (this.historyFile) {
+            await this.historyFile.append(toFlush);
+        }
+        // Remove flushed entries from memory
+        for (const entry of toFlush) {
+            this.recallArchive.delete(entry.seq);
+        }
+        this.nuclearEntries = this.nuclearEntries.slice(flushCount);
+        // Update the nuclear block in messages
+        this.updateNuclearBlockInMessages(this.messages);
+    }
+    // ── Startup: Load prior history ───────────────────────────────
+    /**
+     * Inject prior session history from the history file as a context note.
+     */
+    loadPriorHistory(entries) {
+        if (entries.length === 0)
+            return;
+        // Update nextSeq to avoid collisions
+        const maxSeq = Math.max(...entries.map((e) => e.seq));
+        if (maxSeq >= this.nextSeq)
+            this.nextSeq = maxSeq + 1;
+        const lines = entries.map(formatNuclearLine);
+        this.messages.push({
+            role: "user",
+            content: `[Prior session history — loaded from ~/.agent-sh/history]\n${lines.join("\n")}`,
+        });
+    }
+    // ── Conversation recall ───────────────────────────────────────
+    /** Search Tier 2 archive + Tier 3 history file. */
+    async search(query) {
+        if (!query.trim())
+            return "No query provided.";
+        const parts = [];
+        // Search Tier 2 (in-memory archive)
+        const archiveResults = this.searchArchive(query);
+        if (archiveResults)
+            parts.push(archiveResults);
+        // Search Tier 3 (history file)
+        if (this.historyFile) {
+            const fileResults = await this.historyFile.search(query);
+            if (fileResults.length > 0) {
+                parts.push(`History file matches (${fileResults.length}):`);
+                for (const r of fileResults.slice(0, 20)) {
+                    parts.push(`  ${r.line}`);
+                }
+            }
+        }
+        if (parts.length === 0)
+            return `No results found for "${query}".`;
+        return parts.join("\n\n");
+    }
+    /** Expand full content of a nuclear entry by seq number. */
+    async expand(seq) {
+        // Check Tier 2 archive first
+        const archived = this.recallArchive.get(seq);
+        if (archived) {
+            const entry = this.nuclearEntries.find((e) => e.seq === seq);
+            const header = entry ? formatNuclearLine(entry) : `#${seq}`;
+            return `${header}\n\n${this.turnToText(archived)}`;
+        }
+        return `Entry #${seq}: not found in recall archive (may have been flushed to history file).`;
+    }
+    /** Browse nuclear entries (Tier 2) + recent history (Tier 3). */
+    async browse() {
+        const parts = [];
+        if (this.nuclearEntries.length > 0) {
+            parts.push("In-context nuclear entries:");
+            for (const e of this.nuclearEntries) {
+                parts.push(`  ${formatNuclearLine(e)}`);
+            }
+        }
+        if (this.historyFile) {
+            const recent = await this.historyFile.readRecent(25);
+            if (recent.length > 0) {
+                parts.push("\nRecent history file entries:");
+                for (const e of recent) {
+                    parts.push(`  ${formatNuclearLine(e)}`);
+                }
+            }
+        }
+        if (parts.length === 0)
+            return "No conversation history.";
+        return parts.join("\n");
+    }
+    // ── Stats ─────────────────────────────────────────────────────
+    getNuclearEntryCount() {
+        return this.nuclearEntries.length;
+    }
+    getRecallArchiveSize() {
+        return this.recallArchive.size;
     }
+    // ── Clear ─────────────────────────────────────────────────────
     clear() {
         this.messages = [];
+        this.nuclearEntries = [];
+        this.recallArchive.clear();
+    }
+    // ── Internal: Nuclear block management ────────────────────────
+    buildNuclearBlock() {
+        const lines = this.nuclearEntries.map(formatNuclearLine);
+        return {
+            role: "user",
+            content: `[Conversation history — use conversation_recall to expand any entry]\n${lines.join("\n")}`,
+        };
+    }
+    updateNuclearBlockInMessages(messages) {
+        if (this.nuclearEntries.length === 0)
+            return;
+        const marker = "[Conversation history — use conversation_recall";
+        for (let i = 0; i < messages.length; i++) {
+            const msg = messages[i];
+            if (msg.role === "user" && typeof msg.content === "string" && msg.content.startsWith(marker)) {
+                messages[i] = this.buildNuclearBlock();
+                return;
+            }
+        }
+        // No existing block found — insert after the first turn
+        if (messages.length > 0) {
+            // Find end of first turn (next user message or end)
+            let insertIdx = 1;
+            for (let i = 1; i < messages.length; i++) {
+                if (messages[i].role === "user") {
+                    insertIdx = i;
+                    break;
+                }
+                insertIdx = i + 1;
+            }
+            messages.splice(insertIdx, 0, this.buildNuclearBlock());
+        }
+    }
+    // ── Internal: Turn parsing and priority ───────────────────────
+    parseTurns() {
+        const turns = [];
+        let current = [];
+        for (const msg of this.messages) {
+            if (msg.role === "user" && current.length > 0) {
+                turns.push({ messages: current, priority: 2 /* Priority.MEDIUM */ });
+                current = [];
+            }
+            current.push(msg);
+        }
+        if (current.length > 0) {
+            turns.push({ messages: current, priority: 2 /* Priority.MEDIUM */ });
+        }
+        return turns;
+    }
+    inferPriority(messages) {
+        let hasError = false;
+        let hasWriteTool = false;
+        let allReadOnly = true;
+        let hasToolResult = false;
+        for (const msg of messages) {
+            if (msg.role === "user")
+                return 3 /* Priority.HIGH */;
+            if (msg.role === "tool") {
+                hasToolResult = true;
+                const content = typeof msg.content === "string" ? msg.content : "";
+                if (content.startsWith("Error:") || content.includes("error")) {
+                    hasError = true;
+                }
+            }
+            if (msg.role === "assistant" && "tool_calls" in msg && msg.tool_calls) {
+                for (const tc of msg.tool_calls) {
+                    const fn = "function" in tc ? tc.function : undefined;
+                    if (!fn)
+                        continue;
+                    const name = fn.name;
+                    if (WRITE_TOOLS.has(name))
+                        hasWriteTool = true;
+                    if (!READ_ONLY_TOOLS.has(name))
+                        allReadOnly = false;
+                }
+            }
+        }
+        if (hasError)
+            return 3 /* Priority.HIGH */;
+        if (hasWriteTool)
+            return 2 /* Priority.MEDIUM */;
+        if (hasToolResult && allReadOnly)
+            return 0 /* Priority.LOWEST */;
+        if (hasToolResult)
+            return 1 /* Priority.LOW */;
+        return 2 /* Priority.MEDIUM */;
+    }
+    // ── Internal: Search helpers ──────────────────────────────────
+    searchArchive(query) {
+        if (this.recallArchive.size === 0)
+            return null;
+        let regex;
+        try {
+            regex = new RegExp(query, "i");
+        }
+        catch {
+            const words = query.split(/\s+/).filter((w) => w.length > 0);
+            const pattern = words.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
+            regex = new RegExp(pattern, "i");
+        }
+        const matches = [];
+        for (const [seq, msgs] of this.recallArchive) {
+            const text = this.turnToText(msgs);
+            if (regex.test(text)) {
+                const entry = this.nuclearEntries.find((e) => e.seq === seq);
+                matches.push(entry ? formatNuclearLine(entry) : `#${seq}`);
+            }
+        }
+        if (matches.length === 0)
+            return null;
+        return `Recall archive matches (${matches.length}):\n${matches.map((m) => `  ${m}`).join("\n")}`;
+    }
+    turnToText(messages) {
+        const lines = [];
+        for (const msg of messages) {
+            if (msg.role === "user") {
+                lines.push(`[user] ${typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)}`);
+            }
+            else if (msg.role === "assistant") {
+                if (typeof msg.content === "string" && msg.content) {
+                    lines.push(`[assistant] ${msg.content}`);
+                }
+                if ("tool_calls" in msg && msg.tool_calls) {
+                    for (const tc of msg.tool_calls) {
+                        if ("function" in tc) {
+                            lines.push(`[tool_call] ${tc.function.name}(${tc.function.arguments.slice(0, 200)})`);
+                        }
+                    }
+                }
+            }
+            else if (msg.role === "tool") {
+                const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content);
+                lines.push(`[tool_result] ${content.slice(0, 500)}`);
+            }
+        }
+        return lines.join("\n");
     }
 }

package/dist/agent/history-file.d.ts

@@ -0,0 +1,36 @@
+import { type NuclearEntry } from "./nuclear-form.js";
+export declare class HistoryFile {
+    readonly instanceId: string;
+    private filePath;
+    constructor(opts?: {
+        filePath?: string;
+        instanceId?: string;
+    });
+    /**
+     * Append entries atomically. Uses O_APPEND for concurrency safety.
+     * Triggers truncation check after writing.
+     */
+    append(entries: NuclearEntry[]): Promise<void>;
+    /**
+     * Read the most recent N entries from the history file.
+     */
+    readRecent(maxEntries?: number): Promise<NuclearEntry[]>;
+    /**
+     * Search history entries by regex/keyword.
+     */
+    search(query: string): Promise<{
+        entry: NuclearEntry;
+        line: string;
+    }[]>;
+    /**
+     * Get file size in bytes. Returns 0 if file doesn't exist.
+     */
+    getSize(): Promise<number>;
+    /**
+     * Truncate from the front if file exceeds historyMaxBytes.
+     * Uses a lock file for the rewrite operation.
+     */
+    private maybeTruncate;
+    private acquireLock;
+    private releaseLock;
+}

package/dist/agent/history-file.js

@@ -0,0 +1,167 @@
+/**
+ * Persistent history file — Tier 3 of the three-tier history system.
+ *
+ * Append-only JSONL file at ~/.agent-sh/history. Multiple agent-sh
+ * instances can write concurrently — each line is under PIPE_BUF so
+ * O_APPEND writes are atomic. Only truncation (which rewrites the file)
+ * uses a lock file for safety.
+ */
+import * as fs from "node:fs/promises";
+import * as fss from "node:fs";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+import { CONFIG_DIR, getSettings } from "../settings.js";
+import { serializeEntry, deserializeEntry, formatNuclearLine, } from "./nuclear-form.js";
+const HISTORY_PATH = path.join(CONFIG_DIR, "history");
+const LOCK_PATH = HISTORY_PATH + ".lock";
+const LOCK_STALE_MS = 10_000; // consider lock stale after 10s
+export class HistoryFile {
+    instanceId;
+    filePath;
+    constructor(opts) {
+        this.filePath = opts?.filePath ?? HISTORY_PATH;
+        this.instanceId = opts?.instanceId ?? crypto.randomBytes(2).toString("hex");
+    }
+    /**
+     * Append entries atomically. Uses O_APPEND for concurrency safety.
+     * Triggers truncation check after writing.
+     */
+    async append(entries) {
+        if (entries.length === 0)
+            return;
+        const lines = entries.map((e) => serializeEntry(e) + "\n").join("");
+        await fs.appendFile(this.filePath, lines, { flag: "a" });
+        await this.maybeTruncate();
+    }
+    /**
+     * Read the most recent N entries from the history file.
+     */
+    async readRecent(maxEntries) {
+        maxEntries ??= getSettings().historyStartupEntries;
+        let content;
+        try {
+            content = await fs.readFile(this.filePath, "utf-8");
+        }
+        catch {
+            return [];
+        }
+        const lines = content.trim().split("\n").filter(Boolean);
+        const recent = lines.slice(-maxEntries);
+        const entries = [];
+        for (const line of recent) {
+            const entry = deserializeEntry(line);
+            if (entry)
+                entries.push(entry);
+        }
+        return entries;
+    }
+    /**
+     * Search history entries by regex/keyword.
+     */
+    async search(query) {
+        if (!query.trim())
+            return [];
+        let regex;
+        try {
+            regex = new RegExp(query, "i");
+        }
+        catch {
+            const words = query.split(/\s+/).filter((w) => w.length > 0);
+            const pattern = words.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
+            regex = new RegExp(pattern, "i");
+        }
+        let content;
+        try {
+            content = await fs.readFile(this.filePath, "utf-8");
+        }
+        catch {
+            return [];
+        }
+        const results = [];
+        for (const line of content.trim().split("\n")) {
+            const entry = deserializeEntry(line);
+            if (entry && regex.test(entry.sum)) {
+                results.push({ entry, line: formatNuclearLine(entry) });
+            }
+        }
+        return results;
+    }
+    /**
+     * Get file size in bytes. Returns 0 if file doesn't exist.
+     */
+    async getSize() {
+        try {
+            const stat = await fs.stat(this.filePath);
+            return stat.size;
+        }
+        catch {
+            return 0;
+        }
+    }
+    // ── Truncation ──────────────────────────────────────────────────
+    /**
+     * Truncate from the front if file exceeds historyMaxBytes.
+     * Uses a lock file for the rewrite operation.
+     */
+    async maybeTruncate() {
+        const maxBytes = getSettings().historyMaxBytes;
+        const size = await this.getSize();
+        // Only truncate when significantly over (150%) to avoid frequent rewrites
+        if (size <= maxBytes * 1.5)
+            return;
+        const acquired = await this.acquireLock();
+        if (!acquired)
+            return; // another process is truncating
+        try {
+            let content;
+            try {
+                content = await fs.readFile(this.filePath, "utf-8");
+            }
+            catch {
+                return;
+            }
+            const lines = content.split("\n").filter(Boolean);
+            // Drop oldest lines until under maxBytes
+            let totalBytes = Buffer.byteLength(content, "utf-8");
+            let dropCount = 0;
+            while (totalBytes > maxBytes && dropCount < lines.length - 1) {
+                totalBytes -= Buffer.byteLength(lines[dropCount] + "\n", "utf-8");
+                dropCount++;
+            }
+            if (dropCount === 0)
+                return;
+            const remaining = lines.slice(dropCount).join("\n") + "\n";
+            // Atomic rewrite: write temp → rename
+            const tmpPath = this.filePath + ".tmp." + process.pid;
+            await fs.writeFile(tmpPath, remaining);
+            await fs.rename(tmpPath, this.filePath);
+        }
+        finally {
+            await this.releaseLock();
+        }
+    }
+    async acquireLock() {
+        try {
+            // Check for stale lock
+            try {
+                const stat = await fs.stat(LOCK_PATH);
+                if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
+                    await fs.unlink(LOCK_PATH).catch(() => { });
+                }
+            }
+            catch {
+                // Lock doesn't exist — good
+            }
+            // O_EXCL ensures atomicity
+            const fd = await fs.open(LOCK_PATH, fss.constants.O_CREAT | fss.constants.O_EXCL | fss.constants.O_WRONLY);
+            await fd.close();
+            return true;
+        }
+        catch {
+            return false; // lock held by another process
+        }
+    }
+    async releaseLock() {
+        await fs.unlink(LOCK_PATH).catch(() => { });
+    }
+}

package/dist/agent/nuclear-form.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Nuclear form — compact one-liner summaries of conversation actions.
+ *
+ * Used by the three-tier history system:
+ *   Tier 1 (full content) → compacts into → Tier 2 (nuclear one-liners)
+ *   Tier 2 → flushes to → Tier 3 (history file on disk)
+ *
+ * Nuclear entries are the currency of Tier 2 and Tier 3.
+ */
+import type { ChatCompletionMessageParam } from "../utils/llm-client.js";
+export interface NuclearEntry {
+    /** Global sequence number. */
+    seq: number;
+    /** Timestamp (Date.now()). */
+    ts: number;
+    /** Instance ID — 4-char hex identifying the agent-sh process. */
+    iid: string;
+    /** Entry kind. */
+    kind: "user" | "agent" | "tool" | "error";
+    /** Tool name (for kind=tool or kind=error). */
+    tool?: string;
+    /** The one-liner summary. */
+    sum: string;
+}
+/** Read-only tools whose results are dropped at Tier 1→2 (agent can re-read). */
+export declare const READ_ONLY_TOOLS: Set<string>;
+/** State-changing tools whose summaries are kept in nuclear memory. */
+export declare const WRITE_TOOLS: Set<string>;
+/**
+ * Generate nuclear entries from a logical turn (a sequence of messages
+ * starting with a user message, followed by assistant + tool messages).
+ */
+export declare function toNuclearEntries(messages: ChatCompletionMessageParam[], startSeq: number, instanceId: string): NuclearEntry[];
+/** Format a nuclear entry as a display line (for in-context injection). */
+export declare function formatNuclearLine(entry: NuclearEntry): string;
+/** Serialize a nuclear entry to a JSONL line. */
+export declare function serializeEntry(entry: NuclearEntry): string;
+/** Deserialize a JSONL line to a nuclear entry. Returns null on parse failure. */
+export declare function deserializeEntry(line: string): NuclearEntry | null;
+/** Check if a nuclear entry represents a read-only action (should be dropped). */
+export declare function isReadOnly(entry: NuclearEntry): boolean;