@soleri/core 9.4.0 → 9.5.0

package/src/runtime/shutdown-registry.test.ts

@@ -0,0 +1,151 @@
+/**
+ * Unit tests for ShutdownRegistry — centralized cleanup for agent runtime.
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import { ShutdownRegistry } from './shutdown-registry.js';
+
+describe('ShutdownRegistry', () => {
+  it('starts with zero entries and not closed', () => {
+    const registry = new ShutdownRegistry();
+    expect(registry.size).toBe(0);
+    expect(registry.isClosed).toBe(false);
+  });
+
+  it('tracks registered entries', () => {
+    const registry = new ShutdownRegistry();
+    registry.register('a', vi.fn());
+    registry.register('b', vi.fn());
+    expect(registry.size).toBe(2);
+  });
+
+  it('calls callbacks in LIFO order on closeAll', async () => {
+    const order: string[] = [];
+    const registry = new ShutdownRegistry();
+    registry.register('first', () => {
+      order.push('first');
+    });
+    registry.register('second', () => {
+      order.push('second');
+    });
+    registry.register('third', () => {
+      order.push('third');
+    });
+
+    await registry.closeAll();
+    expect(order).toEqual(['third', 'second', 'first']);
+  });
+
+  it('calls callbacks in LIFO order on closeAllSync', () => {
+    const order: string[] = [];
+    const registry = new ShutdownRegistry();
+    registry.register('first', () => {
+      order.push('first');
+    });
+    registry.register('second', () => {
+      order.push('second');
+    });
+    registry.register('third', () => {
+      order.push('third');
+    });
+
+    registry.closeAllSync();
+    expect(order).toEqual(['third', 'second', 'first']);
+  });
+
+  it('is idempotent — second closeAll is a no-op', async () => {
+    const callback = vi.fn();
+    const registry = new ShutdownRegistry();
+    registry.register('test', callback);
+
+    await registry.closeAll();
+    await registry.closeAll();
+
+    expect(callback).toHaveBeenCalledTimes(1);
+    expect(registry.isClosed).toBe(true);
+  });
+
+  it('is idempotent — second closeAllSync is a no-op', () => {
+    const callback = vi.fn();
+    const registry = new ShutdownRegistry();
+    registry.register('test', callback);
+
+    registry.closeAllSync();
+    registry.closeAllSync();
+
+    expect(callback).toHaveBeenCalledTimes(1);
+  });
+
+  it('handles async callbacks in closeAll', async () => {
+    const order: string[] = [];
+    const registry = new ShutdownRegistry();
+    registry.register('sync', () => {
+      order.push('sync');
+    });
+    registry.register('async', async () => {
+      await new Promise((r) => setTimeout(r, 5));
+      order.push('async');
+    });
+
+    await registry.closeAll();
+    expect(order).toEqual(['async', 'sync']);
+  });
+
+  it('continues on error — remaining callbacks still execute', async () => {
+    const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true);
+    const order: string[] = [];
+    const registry = new ShutdownRegistry();
+    registry.register('first', () => {
+      order.push('first');
+    });
+    registry.register('failing', () => {
+      throw new Error('boom');
+    });
+    registry.register('third', () => {
+      order.push('third');
+    });
+
+    await registry.closeAll();
+
+    // third runs first (LIFO), failing throws but first still runs
+    expect(order).toEqual(['third', 'first']);
+    expect(stderrSpy).toHaveBeenCalledWith(expect.stringContaining('boom'));
+    stderrSpy.mockRestore();
+  });
+
+  it('closeAllSync swallows errors silently', () => {
+    const order: string[] = [];
+    const registry = new ShutdownRegistry();
+    registry.register('first', () => {
+      order.push('first');
+    });
+    registry.register('failing', () => {
+      throw new Error('boom');
+    });
+    registry.register('third', () => {
+      order.push('third');
+    });
+
+    // Should not throw
+    registry.closeAllSync();
+    expect(order).toEqual(['third', 'first']);
+  });
+
+  it('ignores registrations after close', async () => {
+    const registry = new ShutdownRegistry();
+    await registry.closeAll();
+
+    const callback = vi.fn();
+    registry.register('late', callback);
+    expect(registry.size).toBe(0);
+  });
+
+  it('clears entries after closeAll', async () => {
+    const registry = new ShutdownRegistry();
+    registry.register('test', vi.fn());
+    expect(registry.size).toBe(1);
+
+    await registry.closeAll();
+    expect(registry.size).toBe(0);
+  });
+});

package/src/runtime/shutdown-registry.ts

@@ -0,0 +1,85 @@
+/**
+ * Shutdown Registry — centralized cleanup for agent runtime resources.
+ *
+ * Modules register their cleanup callbacks (clear timers, close watchers,
+ * kill child processes). On shutdown, callbacks run in LIFO order so
+ * dependents close before their dependencies.
+ *
+ * Idempotent: calling `closeAll()` multiple times is safe.
+ */
+
+export type ShutdownCallback = () => void | Promise<void>;
+
+interface ShutdownEntry {
+  name: string;
+  callback: ShutdownCallback;
+}
+
+export class ShutdownRegistry {
+  private entries: ShutdownEntry[] = [];
+  private closed = false;
+
+  /**
+   * Register a named cleanup callback.
+   * Callbacks are invoked in LIFO order (last registered = first closed).
+   */
+  register(name: string, callback: ShutdownCallback): void {
+    if (this.closed) return;
+    this.entries.push({ name, callback });
+  }
+
+  /**
+   * Run all registered cleanup callbacks in LIFO order.
+   * Idempotent — subsequent calls are no-ops.
+   * Errors in individual callbacks are caught and logged to stderr
+   * so that remaining callbacks still execute.
+   */
+  async closeAll(): Promise<void> {
+    if (this.closed) return;
+    this.closed = true;
+
+    // LIFO order
+    for (let i = this.entries.length - 1; i >= 0; i--) {
+      const entry = this.entries[i];
+      try {
+        await entry.callback();
+      } catch (err) {
+        // Log but don't throw — remaining cleanups must still run
+        const msg = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[shutdown] ${entry.name}: ${msg}\n`);
+      }
+    }
+
+    this.entries = [];
+  }
+
+  /**
+   * Synchronous close — best-effort for non-async callbacks.
+   * Use when you can't await (e.g. process.on('exit')).
+   */
+  closeAllSync(): void {
+    if (this.closed) return;
+    this.closed = true;
+
+    for (let i = this.entries.length - 1; i >= 0; i--) {
+      const entry = this.entries[i];
+      try {
+        entry.callback();
+      } catch {
+        // Best-effort — swallow errors in sync path
+      }
+    }
+
+    this.entries = [];
+  }
+
+  /** Number of registered callbacks. */
+  get size(): number {
+    return this.entries.length;
+  }
+
+  /** Whether closeAll() has been called. */
+  get isClosed(): boolean {
+    return this.closed;
+  }
+}

package/src/runtime/types.ts

@@ -36,6 +36,7 @@ import type { PipelineRunner } from '../queue/pipeline-runner.js';
 import type { OperatorProfileStore } from '../operator/operator-profile.js';
 import type { OperatorContextStore } from '../operator/operator-context-store.js';
 import type { ContextHealthMonitor } from './context-health.js';
+import type { ShutdownRegistry } from './shutdown-registry.js';
 
 /**
  * Configuration for creating an agent runtime.
@@ -130,8 +131,10 @@ export interface AgentRuntime {
   personaInstructions: import('../persona/types.js').PersonaSystemInstructions;
   /** Context health monitor — tracks tool call volume and context window fill. */
   contextHealth: ContextHealthMonitor;
+  /** Shutdown registry — centralized cleanup for timers, watchers, child processes. */
+  shutdownRegistry: ShutdownRegistry;
   /** Timestamp (ms since epoch) when this runtime was created. */
   createdAt: number;
-  /** Close the vault database connection. Call on shutdown. */
+  /** Close all runtime resources (vault, timers, watchers). Call on shutdown. */
   close(): void;
 }

package/src/transport/http-server.ts

@@ -151,7 +151,18 @@ export class HttpMcpServer {
       const sessionId = req.headers['mcp-session-id'] as string | undefined;
 
       if (method === 'POST') {
-        const body = await this.readBody(req);
+        let body: unknown;
+        try {
+          body = await this.readBody(req);
+        } catch (err) {
+          const statusCode = (err as { statusCode?: number }).statusCode;
+          if (statusCode === 413) {
+            this.sendJSON(res, 413, { error: 'Request body too large' });
+            return;
+          }
+          this.sendJSON(res, 400, { error: 'Failed to read request body' });
+          return;
+        }
 
         if (sessionId) {
           const session = this.sessions.get(sessionId);
@@ -241,10 +252,41 @@ export class HttpMcpServer {
   }
 
   private readBody(req: IncomingMessage): Promise<unknown> {
+    const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10 MB
+    const BODY_TIMEOUT = 30_000; // 30 seconds
+
     return new Promise((resolve, reject) => {
+      let size = 0;
       const chunks: Buffer[] = [];
-      req.on('data', (chunk: Buffer) => chunks.push(chunk));
+      let settled = false;
+
+      const timer = setTimeout(() => {
+        if (!settled) {
+          settled = true;
+          req.destroy();
+          reject(new Error('Request body timeout'));
+        }
+      }, BODY_TIMEOUT);
+
+      const cleanup = () => clearTimeout(timer);
+
+      req.on('data', (chunk: Buffer) => {
+        size += chunk.length;
+        if (size > MAX_BODY_SIZE) {
+          if (!settled) {
+            settled = true;
+            cleanup();
+            req.destroy();
+            reject(Object.assign(new Error('Request body too large'), { statusCode: 413 }));
+          }
+          return;
+        }
+        chunks.push(chunk);
+      });
       req.on('end', () => {
+        if (settled) return;
+        settled = true;
+        cleanup();
         try {
           const text = Buffer.concat(chunks).toString('utf-8');
           resolve(text.length > 0 ? JSON.parse(text) : {});
@@ -252,7 +294,12 @@ export class HttpMcpServer {
           reject(e);
         }
       });
-      req.on('error', reject);
+      req.on('error', (e) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        reject(e);
+      });
     });
   }
 

package/src/transport/ws-server.ts

@@ -230,11 +230,19 @@ export class WsMcpServer {
 
     // Set up frame reader
     const maxSize = this.config.maxMessageSize ?? DEFAULT_MAX_MESSAGE_SIZE;
+    const maxBufferSize = maxSize; // raw buffer limit matches max message size (1 MB default)
     let buffer = Buffer.alloc(0);
 
     socket.on('data', (chunk: Buffer) => {
       buffer = Buffer.concat([buffer, chunk]);
 
+      // Guard against unbounded buffer growth (e.g. slow-drip DoS with no complete frames)
+      if (buffer.length > maxBufferSize) {
+        this.sendClose(socket, 1009, 'Buffer exceeded max size');
+        socket.destroy();
+        return;
+      }
+
       // Process all complete frames in the buffer
       while (buffer.length >= 2) {
         const frame = this.parseFrame(buffer);

package/src/vault/linking.test.ts

@@ -68,6 +68,12 @@ class LinkingMockDB implements PersistenceProvider {
   get<T>(sql: string, params?: unknown[]): T | undefined {
     const p = params ?? [];
     if (sql.includes('COUNT(*)')) {
+      if (sql.includes('NOT IN')) {
+        // Count orphan entries (no links)
+        const linkedIds = new Set(this.links.flatMap((l) => [l.source_id, l.target_id]));
+        const count = this.entries.filter((e) => !linkedIds.has(e.id)).length;
+        return { count } as T;
+      }
       const id = p[0] as string;
       const count = this.links.filter((l) => l.source_id === id || l.target_id === id).length;
       return { count } as T;
@@ -102,6 +108,12 @@ class LinkingMockDB implements PersistenceProvider {
       const ids = new Set(p.slice(0, half) as string[]);
       return this.links.filter((l) => ids.has(l.source_id) || ids.has(l.target_id)) as T[];
     }
+    if (sql.includes('FROM entries WHERE id IN')) {
+      const ids = new Set(p as string[]);
+      return this.entries
+        .filter((e) => ids.has(e.id))
+        .map((e) => ({ id: e.id, title: e.title, type: e.type, domain: e.domain })) as T[];
+    }
     if (sql.includes('NOT IN')) {
       const limit = p[0] as number;
       const linkedIds = new Set(this.links.flatMap((l) => [l.source_id, l.target_id]));

package/src/vault/linking.ts

@@ -145,6 +145,7 @@ export class LinkManager {
   /**
    * Walk the link graph from a starting entry up to `depth` hops.
    * BFS — walks both outgoing and incoming links (undirected).
+   * Batch-loads all links per frontier level to avoid N+1 queries.
    */
   traverse(entryId: string, depth: number = 2): LinkedEntry[] {
     const visited = new Set<string>([entryId]);
@@ -152,9 +153,47 @@ export class LinkManager {
     let frontier = [entryId];
 
     for (let d = 0; d < depth && frontier.length > 0; d++) {
+      // Batch-load all links for entire frontier in one query
+      const allLinks = this.getAllLinksForEntries(frontier);
+
+      // Collect unvisited neighbor IDs
+      const neighborMap = new Map<
+        string,
+        { link: VaultLink; direction: 'outgoing' | 'incoming' }
+      >();
+      for (const link of allLinks) {
+        // Outgoing: source is in frontier, target is the neighbor
+        if (frontier.includes(link.sourceId) && !visited.has(link.targetId)) {
+          if (!neighborMap.has(link.targetId)) {
+            neighborMap.set(link.targetId, { link, direction: 'outgoing' });
+          }
+        }
+        // Incoming: target is in frontier, source is the neighbor
+        if (frontier.includes(link.targetId) && !visited.has(link.sourceId)) {
+          if (!neighborMap.has(link.sourceId)) {
+            neighborMap.set(link.sourceId, { link, direction: 'incoming' });
+          }
+        }
+      }
+
+      if (neighborMap.size === 0) break;
+
+      // Batch-load entry metadata for all neighbors in one query
+      const neighborIds = [...neighborMap.keys()];
+      const metaMap = this.getEntryMetaBatch(neighborIds);
+
       const nextFrontier: string[] = [];
-      for (const currentId of frontier) {
-        this.collectNeighbors(currentId, visited, nextFrontier, result);
+      for (const [neighborId, { link, direction }] of neighborMap) {
+        visited.add(neighborId);
+        nextFrontier.push(neighborId);
+        const entry = metaMap.get(neighborId);
+        if (!entry) continue;
+        result.push({
+          ...entry,
+          linkType: link.linkType,
+          linkDirection: direction,
+          linkNote: link.note,
+        });
       }
       frontier = nextFrontier;
     }
@@ -162,43 +201,6 @@ export class LinkManager {
     return result;
   }
 
-  /** Collect unvisited outgoing and incoming neighbors for BFS. */
-  private collectNeighbors(
-    currentId: string,
-    visited: Set<string>,
-    nextFrontier: string[],
-    result: LinkedEntry[],
-  ): void {
-    for (const link of this.getLinks(currentId)) {
-      this.visitNeighbor(link.targetId, link, 'outgoing', visited, nextFrontier, result);
-    }
-    for (const link of this.getBacklinks(currentId)) {
-      this.visitNeighbor(link.sourceId, link, 'incoming', visited, nextFrontier, result);
-    }
-  }
-
-  /** Visit a single neighbor node if not already visited. */
-  private visitNeighbor(
-    neighborId: string,
-    link: VaultLink,
-    direction: 'outgoing' | 'incoming',
-    visited: Set<string>,
-    nextFrontier: string[],
-    result: LinkedEntry[],
-  ): void {
-    if (visited.has(neighborId)) return;
-    visited.add(neighborId);
-    nextFrontier.push(neighborId);
-    const entry = this.getEntryMeta(neighborId);
-    if (!entry) return;
-    result.push({
-      ...entry,
-      linkType: link.linkType,
-      linkDirection: direction,
-      linkNote: link.note,
-    });
-  }
-
   // ── Bulk Queries ────────────────────────────────────────────────────
 
   /** Get all links where either source or target is in the given ID set. */
@@ -317,20 +319,43 @@ export class LinkManager {
     const batchSize = opts?.batchSize ?? 50;
     const start = Date.now();
 
-    const orphans = this.getOrphans(10000);
     let processed = 0;
     let linksCreated = 0;
     const preview: Array<{ sourceId: string; targetId: string; linkType: string; score: number }> =
       [];
 
-    for (let i = 0; i < orphans.length; i += batchSize) {
-      const batch = orphans.slice(i, i + batchSize);
-      for (const entry of batch) {
+    // Estimate total for progress reporting (single COUNT query)
+    let totalEstimate = 0;
+    try {
+      const countRow = this.provider.get<{ count: number }>(
+        `SELECT COUNT(*) as count FROM entries
+         WHERE id NOT IN (SELECT source_id FROM vault_links)
+           AND id NOT IN (SELECT target_id FROM vault_links)`,
+      );
+      totalEstimate = countRow?.count ?? 0;
+    } catch {
+      // fall through with 0
+    }
+
+    // Process orphans in batches of batchSize instead of loading all at once.
+    // After each batch, successfully linked entries are no longer orphans,
+    // so the next getOrphans() call returns the next set.
+    // For dry-run mode, we must track processed IDs to avoid re-fetching the same orphans.
+    const processedIds = new Set<string>();
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const batch = this.getOrphans(batchSize);
+      // Filter out already-processed entries (relevant for dry-run where orphan status doesn't change)
+      const unprocessed = batch.filter((e) => !processedIds.has(e.id));
+      if (unprocessed.length === 0) break;
+
+      for (const entry of unprocessed) {
+        processedIds.add(entry.id);
         const created = this.processOrphan(entry.id, threshold, maxLinks, dryRun, preview);
         linksCreated += created;
         processed++;
       }
-      opts?.onProgress?.({ processed, total: orphans.length, linksCreated });
+      opts?.onProgress?.({ processed, total: totalEstimate, linksCreated });
     }
 
     return {
@@ -381,6 +406,27 @@ export class LinkManager {
       return null;
     }
   }
+
+  /** Batch-load entry metadata for multiple IDs in a single query. */
+  private getEntryMetaBatch(
+    entryIds: string[],
+  ): Map<string, Omit<LinkedEntry, 'linkType' | 'linkDirection' | 'linkNote'>> {
+    const result = new Map<string, Omit<LinkedEntry, 'linkType' | 'linkDirection' | 'linkNote'>>();
+    if (entryIds.length === 0) return result;
+    try {
+      const placeholders = entryIds.map(() => '?').join(',');
+      const rows = this.provider.all<{ id: string; title: string; type: string; domain: string }>(
+        `SELECT id, title, type, domain FROM entries WHERE id IN (${placeholders})`,
+        entryIds,
+      );
+      for (const row of rows) {
+        result.set(row.id, row);
+      }
+    } catch {
+      // graceful degradation
+    }
+    return result;
+  }
 }
 
 // ── Free-standing helpers ─────────────────────────────────────────────

package/src/vault/vault-maintenance.ts

@@ -111,25 +111,18 @@ export function archive(
   const reason = options.reason ?? `Archived: older than ${options.olderThanDays} days`;
 
   return provider.transaction(() => {
-    const candidates = provider.all<{ id: string }>('SELECT id FROM entries WHERE updated_at < ?', [
-      cutoff,
-    ]);
-
-    if (candidates.length === 0) return { archived: 0 };
-
-    let archived = 0;
-    for (const { id } of candidates) {
-      provider.run(
-        `INSERT OR IGNORE INTO entries_archive (id, type, domain, title, severity, description, context, example, counter_example, why, tags, applies_to, created_at, updated_at, valid_from, valid_until, archive_reason)
-         SELECT id, type, domain, title, severity, description, context, example, counter_example, why, tags, applies_to, created_at, updated_at, valid_from, valid_until, ?
-         FROM entries WHERE id = ?`,
-        [reason, id],
-      );
-      const result = provider.run('DELETE FROM entries WHERE id = ?', [id]);
-      archived += result.changes;
-    }
+    // Bulk INSERT INTO ... SELECT — copies all matching entries to archive in one query
+    provider.run(
+      `INSERT OR IGNORE INTO entries_archive (id, type, domain, title, severity, description, context, example, counter_example, why, tags, applies_to, created_at, updated_at, valid_from, valid_until, archive_reason)
+       SELECT id, type, domain, title, severity, description, context, example, counter_example, why, tags, applies_to, created_at, updated_at, valid_from, valid_until, ?
+       FROM entries WHERE updated_at < ?`,
+      [reason, cutoff],
+    );
+
+    // Bulk DELETE — removes all archived entries in one query
+    const result = provider.run('DELETE FROM entries WHERE updated_at < ?', [cutoff]);
 
-    return { archived };
+    return { archived: result.changes };
   });
 }
 

package/src/vault/vault-memories.ts

@@ -299,21 +299,29 @@ export function memoryTopics(
 export function memoriesByProject(
   provider: PersistenceProvider,
 ): Array<{ project: string; count: number; memories: Memory[] }> {
-  const rows = provider.all<{ project: string; count: number }>(
-    'SELECT project_path as project, COUNT(*) as count FROM memories WHERE archived_at IS NULL GROUP BY project_path ORDER BY count DESC',
+  // Single query fetching all non-archived memories, grouped client-side by project
+  const allRows = provider.all<Record<string, unknown>>(
+    'SELECT * FROM memories WHERE archived_at IS NULL ORDER BY project_path, created_at DESC',
   );
 
-  return rows.map((row) => {
-    const mems = provider.all<Record<string, unknown>>(
-      'SELECT * FROM memories WHERE project_path = ? AND archived_at IS NULL ORDER BY created_at DESC',
-      [row.project],
-    );
-    return {
-      project: row.project,
-      count: row.count,
-      memories: mems.map(rowToMemory),
-    };
-  });
+  const projectMap = new Map<string, Memory[]>();
+  for (const row of allRows) {
+    const memory = rowToMemory(row);
+    const project = memory.projectPath;
+    if (!projectMap.has(project)) {
+      projectMap.set(project, []);
+    }
+    projectMap.get(project)!.push(memory);
+  }
+
+  // Sort by count descending (matching original behavior)
+  return [...projectMap.entries()]
+    .map(([project, memories]) => ({
+      project,
+      count: memories.length,
+      memories,
+    }))
+    .sort((a, b) => b.count - a.count);
 }
 
 // ── Helper ──────────────────────────────────────────────────────────────

package/src/vault/vault-schema.ts

@@ -27,6 +27,7 @@ export function initializeSchema(provider: PersistenceProvider): void {
   migrateOriginColumn(provider);
   migrateContentHash(provider);
   migrateTierColumn(provider);
+  migratePerformanceIndexes(provider);
 }
 
 function createCoreTables(provider: PersistenceProvider): void {
@@ -236,3 +237,23 @@ function migrateTierColumn(provider: PersistenceProvider): void {
     'CREATE INDEX IF NOT EXISTS idx_entries_tier ON entries(tier) WHERE tier IS NOT NULL',
   );
 }
+
+export function migratePerformanceIndexes(provider: PersistenceProvider): void {
+  provider.execSql(`
+    CREATE INDEX IF NOT EXISTS idx_memories_archived_at ON memories(archived_at);
+    CREATE INDEX IF NOT EXISTS idx_entries_updated_at ON entries(updated_at);
+    CREATE INDEX IF NOT EXISTS idx_brain_feedback_entry_id ON brain_feedback(entry_id);
+    CREATE INDEX IF NOT EXISTS idx_entries_valid_until ON entries(valid_until) WHERE valid_until IS NOT NULL;
+    CREATE INDEX IF NOT EXISTS idx_entries_valid_from ON entries(valid_from) WHERE valid_from IS NOT NULL;
+  `);
+
+  // brain_sessions may not exist yet if intelligence module hasn't initialized
+  try {
+    provider.execSql(`
+      CREATE INDEX IF NOT EXISTS idx_brain_sessions_plan_id ON brain_sessions(plan_id) WHERE plan_id IS NOT NULL;
+      CREATE INDEX IF NOT EXISTS idx_brain_sessions_started_at ON brain_sessions(started_at);
+    `);
+  } catch {
+    /* brain_sessions table doesn't exist yet — indexes will be created on next init */
+  }
+}