@aion0/forge 0.9.18 → 0.9.19

package/lib/chat/session-store.ts

@@ -265,6 +265,55 @@ export function listMessages(session_id: string, opts?: { limit?: number; after_
   return rows.map(rowToMessage);
 }
 
+/** Last N messages in chronological order — used by agent-loop to cap LLM context. */
+export function listRecentMessages(session_id: string, limit: number): Message[] {
+  ensureSchema();
+  const rows = db().prepare(`
+    SELECT * FROM chat_messages WHERE session_id = ?
+    ORDER BY ts DESC LIMIT ?
+  `).all(session_id, limit) as MessageRow[];
+  return rows.map(rowToMessage).reverse();
+}
+
+/**
+ * Take the most recent messages, stopping when either the message-count
+ * budget OR the token-estimate budget would be exceeded. Walks
+ * newest-first so the most recent dialogue is always kept; returns
+ * chronological order for the LLM history slot.
+ *
+ * msgBudget is enforced via SQL LIMIT (cheap). tokenBudget is enforced
+ * via the caller-supplied estimator (decoupled to avoid pulling the
+ * token-estimator into the storage layer).
+ */
+export function listMessagesCapped(
+  session_id: string,
+  msgBudget: number,
+  tokenBudget: number,
+  estimateTokens: (m: Message) => number,
+): Message[] {
+  ensureSchema();
+  const cap = Math.max(1, Math.floor(msgBudget));
+  // Pull newest-first via SQL — bounded by msgBudget so we never load
+  // more rows than we could possibly keep.
+  const rows = db().prepare(`
+    SELECT * FROM chat_messages WHERE session_id = ?
+    ORDER BY ts DESC LIMIT ?
+  `).all(session_id, cap) as MessageRow[];
+  const newestFirst = rows.map(rowToMessage);
+
+  // Now apply tokenBudget walking newest → oldest. Always keep at
+  // least one (so an oversized last message doesn't strand the loop).
+  const kept: Message[] = [];
+  let used = 0;
+  for (const m of newestFirst) {
+    const cost = estimateTokens(m);
+    if (kept.length > 0 && used + cost > tokenBudget) break;
+    kept.push(m);
+    used += cost;
+  }
+  return kept.reverse();
+}
+
 export function deleteMessage(id: string): boolean {
   ensureSchema();
   const r = db().prepare(`DELETE FROM chat_messages WHERE id = ?`).run(id);

package/lib/chat-standalone.ts

@@ -144,6 +144,12 @@ async function handleSessionDelete(_req: IncomingMessage, res: ServerResponse, i
 }
 
 async function handleSessionClearMessages(_req: IncomingMessage, res: ServerResponse, id: string): Promise<void> {
+  // Intent: "Clear chat" only drops chat_messages rows. memory_store
+  // blocks (cursor / health / summary / facts) stay — once a fact has
+  // been extracted into long-term memory it should survive clearing
+  // the conversation it came from. Users can delete memory explicitly
+  // from the memory tab if they really want to forget. See
+  // forge-chat-memory-summarizer-design.md §11 decision 3.
   const session = getSession(id);
   if (!session) return sendJson(res, 404, { error: 'session not found' });
   const removed = clearSessionMessages(id);

package/lib/init.ts

@@ -249,6 +249,7 @@ export function ensureInitialized() {
   startWorkspaceProcess(); // spawns workspace-standalone
   startBrowserBridgeProcess(); // spawns browser-bridge-standalone
   startChatProcess(); // spawns chat-standalone
+  startMemoryProcess(); // spawns memory-standalone
 
   const settings = loadSettings();
   if (settings.tunnelAutoStart) {
@@ -402,3 +403,18 @@ function startBrowserBridgeProcess() {
   });
   tester.listen(bridgePort);
 }
+
+let memoryChild: ReturnType<typeof spawn> | null = null;
+
+function startMemoryProcess() {
+  if (memoryChild) return;
+  // No HTTP port — pure background poller. Just spawn-if-not-running.
+  const script = join(process.cwd(), 'lib', 'memory-standalone.ts');
+  memoryChild = spawn('npx', ['tsx', script], {
+    stdio: ['ignore', 'inherit', 'inherit'],
+    env: { ...process.env },
+    detached: false,
+  });
+  memoryChild.on('exit', () => { memoryChild = null; });
+  console.log('[memory] Started standalone (pid:', memoryChild.pid, ')');
+}

package/lib/memory/compress-messages.ts

@@ -0,0 +1,65 @@
+/**
+ * Compact chat messages into a summarizer-friendly transcript.
+ *
+ * Raw tool_use / tool_result blocks can each carry kilobytes of JSON
+ * (stack traces, encoded args, HTML responses). Feeding those into the
+ * summarizer LLM wastes input tokens and crowds out actual dialogue.
+ *
+ * This module flattens each Message into one or more text lines:
+ *   - text blocks pass through (truncated to MAX_TEXT_CHARS)
+ *   - tool_use → `tool[name](key1, key2, …)`
+ *   - tool_result → `→ ok: <first line>` or `→ err: <first line>`
+ *
+ * Output is a plain string ready to drop into the summarizer prompt.
+ */
+
+import type { ContentBlock, Message, ToolResultBlock, ToolUseBlock } from '../chat/types';
+
+const MAX_TEXT_CHARS = 1200;
+const MAX_TOOL_RESULT_CHARS = 200;
+const MAX_INPUT_KEYS = 8;
+
+export function compressMessagesForSummarizer(messages: Message[]): string {
+  const lines: string[] = [];
+  for (const m of messages) {
+    for (const block of m.blocks) {
+      const rendered = renderBlock(m.role, block);
+      if (rendered) lines.push(rendered);
+    }
+  }
+  return lines.join('\n');
+}
+
+function renderBlock(role: 'user' | 'assistant', block: ContentBlock): string | null {
+  if (block.type === 'text') {
+    const text = truncate(block.text.trim(), MAX_TEXT_CHARS);
+    if (!text) return null;
+    return `${role}: ${text}`;
+  }
+  if (block.type === 'tool_use') {
+    return `${role}: ${renderToolUse(block)}`;
+  }
+  if (block.type === 'tool_result') {
+    return `${role}: ${renderToolResult(block)}`;
+  }
+  return null;
+}
+
+function renderToolUse(block: ToolUseBlock): string {
+  const keys = block.input && typeof block.input === 'object'
+    ? Object.keys(block.input as Record<string, unknown>).slice(0, MAX_INPUT_KEYS)
+    : [];
+  const argsStr = keys.length > 0 ? `(${keys.join(', ')})` : '()';
+  return `tool[${block.name}]${argsStr}`;
+}
+
+function renderToolResult(block: ToolResultBlock): string {
+  const firstLine = (block.content ?? '').split(/\r?\n/, 1)[0] ?? '';
+  const head = truncate(firstLine, MAX_TOOL_RESULT_CHARS);
+  return block.is_error ? `→ err: ${head}` : `→ ok: ${head}`;
+}
+
+function truncate(s: string, max: number): string {
+  if (s.length <= max) return s;
+  return s.slice(0, max) + '…';
+}

package/lib/memory/keys.ts

@@ -0,0 +1,82 @@
+/**
+ * Key-naming conventions for the chat memory summarizer.
+ *
+ * MemoryStore is a flat key/value API; this module encodes the
+ * design's "scope + subject" classification into deterministic key
+ * strings so that:
+ *   - cursor / health blocks land on a single stable key (upsert covers)
+ *   - repeated ingest of the same fact maps to the same key (upsert
+ *     replaces, not appends — this is what "memory reinforcement" looks
+ *     like at the storage layer)
+ *   - buildMemoryContext can post-filter by prefix to keep internal
+ *     bookkeeping blocks (cursor / health) out of the LLM prompt
+ *
+ * See forge-chat-memory-summarizer-design.md §4.2 for the full table.
+ */
+
+import { createHash } from 'node:crypto';
+
+/** Stable 12-char content hash. Used in fact keys so re-ingesting the
+ *  same fact maps to the same memory block (upsert = reinforcement). */
+export function stableHash(input: string): string {
+  return createHash('sha256').update(input).digest('hex').slice(0, 12);
+}
+
+/** Session summary at a given cursor end-ts. */
+export function summaryKey(sessionId: string, toTs: number): string {
+  return `chat:${sessionId}:summary:${toTs}`;
+}
+
+/** Long-term fact. scope/subject classify it; hash keys re-ingest. */
+export function factKey(scope: string, subject: string, contentHash: string): string {
+  return `fact:${scope}:${subject}:${contentHash}`;
+}
+
+/** Per-session ingest progress cursor. One row per session. */
+export function cursorKey(sessionId: string): string {
+  return `forge.summarizer.cursor:${sessionId}`;
+}
+
+/** Per-session summarizer health (last_run, errors, counts). */
+export function healthKey(sessionId: string): string {
+  return `forge.summarizer.health:${sessionId}`;
+}
+
+/** Prefixes buildMemoryContext should exclude when rendering context —
+ *  bookkeeping blocks the LLM shouldn't see. */
+export const INTERNAL_KEY_PREFIXES: readonly string[] = [
+  'forge.summarizer.cursor:',
+  'forge.summarizer.health:',
+];
+
+export interface CursorValue {
+  last_ingested_ts: number;
+  last_run_ts: number;
+  ingest_count: number;
+}
+
+export interface SummaryValue {
+  text: string;
+  from_ts: number;
+  to_ts: number;
+  message_count: number;
+  model: string;
+  provider: string;
+  ingest_ts: number;
+}
+
+export interface FactValue {
+  content: string;
+  subject_kind: string;
+  subject: string;
+  source_ref: string;
+  confidence: number | null;
+  extracted_by: 'summarizer' | string;
+}
+
+export interface HealthValue {
+  last_run_ts: number;
+  error: string | null;
+  ingest_count: number;
+  last_token_estimate: number;
+}