macro-agent 0.0.16 → 0.1.0

package/src/store/event-store.ts

@@ -9,8 +9,8 @@
  */
 
 import { createStore, Store } from 'tinybase';
-import { createFilePersister } from 'tinybase/persisters/persister-file';
-import { createCustomPersister } from 'tinybase/persisters';
+import { createCustomPersister, createCustomSqlitePersister } from 'tinybase/persisters';
+import type { DatabasePersisterConfig } from 'tinybase/persisters';
 import Database from 'better-sqlite3';
 import { nanoid } from 'nanoid';
 import * as path from 'path';
@@ -65,38 +65,80 @@ import type { StorageBackend, ExportedEvent } from './backends/types.js';
 import { createTinyBaseBackend } from './backends/tinybase-backend.js';
 
 // ─────────────────────────────────────────────────────────────────────────────
-// Custom better-sqlite3 Persister
+// Tabular better-sqlite3 Persister
 // ─────────────────────────────────────────────────────────────────────────────
 
 /**
- * Creates a custom TinyBase persister for better-sqlite3.
- * TinyBase's built-in createSqlite3Persister expects node-sqlite3 (async/callback API),
- * but we use better-sqlite3 (sync API). This custom persister bridges the gap.
+ * Build tabular config for the 10 TinyBase tables.
+ * Identity mapping: TinyBase table name === SQLite table name.
  */
-function createBetterSqlite3Persister(
+function getTabularConfig(): DatabasePersisterConfig {
+  const tableNames = [
+    'events', 'agents', 'tasks', 'messages',
+    'sessions', 'conversations', 'turns',
+    'threads', 'subscriptions', 'participants',
+  ];
+
+  const load: Record<string, string> = {};
+  const save: Record<string, string> = {};
+  for (const t of tableNames) {
+    load[t] = t;   // SQLite table -> TinyBase table (same name)
+    save[t] = t;   // TinyBase table -> SQLite table (same name)
+  }
+
+  return {
+    mode: 'tabular',
+    tables: { load, save },
+    autoLoadIntervalSeconds: 1,
+  };
+}
+
+/**
+ * Creates a tabular TinyBase persister backed by better-sqlite3.
+ *
+ * Unlike the old JSON blob approach (entire store as one JSON string in a
+ * single row), tabular mode maps each TinyBase table to a real SQLite table
+ * and only writes changed rows on autoSave — making persistence O(delta)
+ * instead of O(total).
+ */
+function createTabularBetterSqlite3Persister(
   store: Store,
   db: ReturnType<typeof Database>,
-  tableName: string = 'tinybase_store'
 ) {
-  // Create table if not exists
-  db.exec(`CREATE TABLE IF NOT EXISTS ${tableName} (_id TEXT PRIMARY KEY, store TEXT)`);
+  // Wrap better-sqlite3's sync API as the async DatabaseExecuteCommand.
+  // TinyBase generates SQL with $1, $2, ... placeholders (PostgreSQL-style),
+  // but better-sqlite3 uses ? for positional array binding. Convert them.
+  const executeCommand = async (sql: string, params?: any[]): Promise<Record<string, any>[]> => {
+    const convertedSql = sql.replace(/\$\d+/g, '?');
+    const trimmed = convertedSql.trimStart().toUpperCase();
+    const stmt = db.prepare(convertedSql);
+    if (trimmed.startsWith('SELECT') || trimmed.startsWith('PRAGMA')) {
+      return (params ? stmt.all(...params) : stmt.all()) as Record<string, any>[];
+    }
+    params ? stmt.run(...params) : stmt.run();
+    return [];
+  };
 
-  return createCustomPersister(
+  return createCustomSqlitePersister(
     store,
-    // getPersisted - load from SQLite
-    async () => {
-      const row = db.prepare(`SELECT store FROM ${tableName} WHERE _id = '_'`).get() as { store: string } | undefined;
-      return row ? JSON.parse(row.store) : undefined;
-    },
-    // setPersisted - save to SQLite
-    async (getContent) => {
-      const json = JSON.stringify(getContent());
-      db.prepare(`INSERT OR REPLACE INTO ${tableName} (_id, store) VALUES ('_', ?)`).run(json);
-    },
-    // addPersisterListener - poll for external changes (cross-process)
-    (listener) => setInterval(listener, 1000),
-    // delPersisterListener - cleanup polling
-    (interval: ReturnType<typeof setInterval>) => clearInterval(interval),
+    getTabularConfig(),
+    executeCommand,
+    // addChangeListener — better-sqlite3 has no native change events
+    (_listener: (tableName: string) => void) => null as any,
+    // delChangeListener — no-op
+    (_handle: any) => {},
+    // onSqlCommand
+    undefined,
+    // onIgnoredError
+    (error: any) => console.warn('[EventStore] Persister error:', error),
+    // destroy
+    () => db.close(),
+    // persist mode (1 = StoreOnly)
+    1 as any,
+    // thing (the db instance)
+    db,
+    // getThing accessor name
+    'getDb',
   );
 }
 
@@ -161,6 +203,7 @@ export interface EventStore {
   // Agent view
   getAgent(agentId: AgentId): Agent | null;
   listAgents(filter?: { state?: AgentState; parent?: AgentId | null }): Agent[];
+  updateAgentPlan(agentId: AgentId, plan: Array<{ content: string; priority: string; status: string }>): void;
 
   // Task view
   getTask(taskId: TaskId): Task | null;
@@ -252,7 +295,15 @@ export function parseDuration(duration: string): number {
 export async function createEventStore(config: StoreConfig = {}): Promise<EventStore> {
   // Resolve instance configuration
   const resolved = resolveInstancePath(config);
-  const { instanceId, instancePath, namespace, isNew, isLegacy, backendType } = resolved;
+  const { instanceId, instancePath, namespace, isNew, backendType } = resolved;
+
+  // Reject deprecated legacy path option
+  if (config.path) {
+    throw new Error(
+      '[macro-agent] The `path` option has been removed. ' +
+        'Use `instanceId` and `baseDir` instead for per-instance isolation.'
+    );
+  }
 
   // Track baseDir for MCP subprocess communication
   const baseDir = config.baseDir ?? path.join(os.homedir(), '.multiagent');
@@ -261,15 +312,6 @@ export async function createEventStore(config: StoreConfig = {}): Promise<EventS
   const peerVisibility: PeerVisibilityConfig =
     config.peerVisibility ?? DEFAULT_PEER_VISIBILITY;
 
-  // Emit deprecation warning for legacy path option
-  if (isLegacy) {
-    console.warn(
-      '[macro-agent] DEPRECATION WARNING: The `path` option is deprecated and will be removed in a future version. ' +
-        'Use `instanceId` and `baseDir` instead for per-instance isolation. ' +
-        'See documentation for migration guide.'
-    );
-  }
-
   // Emit warning for in-memory mode (only in non-test environments)
   if (config.inMemory && process.env.NODE_ENV !== 'test') {
     console.warn(
@@ -282,31 +324,53 @@ export async function createEventStore(config: StoreConfig = {}): Promise<EventS
   const store = createStore();
 
   // Set up persister based on backend type
-  let persister: ReturnType<typeof createFilePersister> | ReturnType<typeof createBetterSqlite3Persister> | null = null;
+  let persister: ReturnType<typeof createTabularBetterSqlite3Persister> | null = null;
   let db: ReturnType<typeof Database> | null = null;
 
   if (!config.inMemory) {
-    if (isLegacy) {
-      // Legacy mode: Use JSON file persister at the specified path
-      const dir = path.dirname(instancePath);
-      if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true });
-      }
-      persister = createFilePersister(store, instancePath);
-    } else {
-      // New instances: Use SQLite with custom better-sqlite3 persister
-      ensureInstanceDir(instancePath);
-      const dbPath = path.join(instancePath, 'store.sqlite');
-      db = new Database(dbPath);
-      db.pragma('journal_mode = WAL');
-      db.pragma('busy_timeout = 5000');
-      persister = createBetterSqlite3Persister(store, db);
+    ensureInstanceDir(instancePath);
+    const dbPath = path.join(instancePath, 'store.sqlite');
+    db = new Database(dbPath);
+    db.pragma('journal_mode = WAL');
+    db.pragma('busy_timeout = 5000');
+
+    // Migration: if old JSON blob table exists, load data from it first
+    const oldTableExists = db.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='tinybase_store'"
+    ).get();
+
+    if (oldTableExists) {
+      const oldPersister = createCustomPersister(
+        store,
+        async () => {
+          const row = db!.prepare("SELECT store FROM tinybase_store WHERE _id = '_'").get() as { store: string } | undefined;
+          return row ? JSON.parse(row.store) : undefined;
+        },
+        async () => {},
+        (listener) => setInterval(listener, 1000),
+        (interval: ReturnType<typeof setInterval>) => clearInterval(interval),
+      );
+      await oldPersister.load();
+      oldPersister.destroy();
     }
+
+    persister = createTabularBetterSqlite3Persister(store, db);
+
+    if (oldTableExists) {
+      // Save migrated data to new tabular format, then drop old table
+      await persister.save();
+      db.exec('DROP TABLE IF EXISTS tinybase_store');
+    }
+
     await persister.load();
+    // Auto-save: persist to disk whenever the in-memory store changes.
+    // Without this, emit() only writes to TinyBase's in-memory store and
+    // data is lost if the server is killed before an explicit persist().
+    await persister.startAutoSave();
   }
 
   // Initialize/update instance metadata
-  if (!config.inMemory && !isLegacy) {
+  if (!config.inMemory) {
     if (isNew) {
       const meta = createInstanceMeta(resolved, config);
       writeInstanceMeta(instancePath, meta);
@@ -456,6 +520,25 @@ export async function createEventStore(config: StoreConfig = {}): Promise<EventS
     return agents;
   }
 
+  /**
+   * Update an agent's plan entries (persisted to SQLite via TinyBase)
+   */
+  function updateAgentPlan(
+    agentId: AgentId,
+    plan: Array<{ content: string; priority: string; status: string }>,
+  ): void {
+    const row = store.getRow('agents', agentId);
+    if (!row.id) return;
+
+    store.setPartialRow('agents', agentId, {
+      plan: JSON.stringify(plan),
+      last_activity_at: Date.now(),
+    });
+
+    const agent = rowToAgent(store.getRow('agents', agentId));
+    notifyAgentChange(agentId, agent);
+  }
+
   /**
    * Get task by ID
    */
@@ -876,11 +959,6 @@ export async function createEventStore(config: StoreConfig = {}): Promise<EventS
    * Get archives directory path
    */
   function getArchivesDir(): string {
-    // For legacy mode, use parent of the file path
-    // For new mode, use the instance directory
-    if (isLegacy) {
-      return path.join(path.dirname(instancePath), 'archives');
-    }
     return path.join(instancePath, 'archives');
   }
 
@@ -1145,6 +1223,7 @@ export async function createEventStore(config: StoreConfig = {}): Promise<EventS
     // Views
     getAgent,
     listAgents,
+    updateAgentPlan,
     getTask,
     listTasks,
     getMessages,
@@ -1211,6 +1290,17 @@ function initializeTables(store: Store): void {
  * Rebuild materialized views from the event log
  */
 function rebuildViews(store: Store): void {
+  // Preserve out-of-band agent fields that aren't derived from events.
+  // `plan` is written directly via updateAgentPlan(), not through events,
+  // so it would be lost when we clear and replay.
+  const savedAgentPlan = new Map<string, string>();
+  for (const rowId of store.getRowIds('agents')) {
+    const row = store.getRow('agents', rowId);
+    if (row.plan && row.plan !== '[]') {
+      savedAgentPlan.set(rowId, row.plan as string);
+    }
+  }
+
   // Clear existing views
   for (const rowId of store.getRowIds('agents')) {
     store.delRow('agents', rowId);
@@ -1266,6 +1356,14 @@ function rebuildViews(store: Store): void {
   for (const event of events) {
     applyEventToViews(store, event, noop, noop, noop, noop, noop, noop);
   }
+
+  // Restore out-of-band agent fields preserved before the wipe
+  for (const [agentId, plan] of savedAgentPlan) {
+    const row = store.getRow('agents', agentId);
+    if (row.id) {
+      store.setPartialRow('agents', agentId, { plan });
+    }
+  }
 }
 
 /**
@@ -1364,6 +1462,7 @@ function applySpawnEvent(
     role: payload.role ?? '',
     config: JSON.stringify(payload.config ?? {}),
     cwd: payload.cwd ?? process.cwd(),
+    plan: '[]',
     created_at: event.timestamp,
     started_at: 0,
     stopped_at: 0,
@@ -1712,6 +1811,7 @@ function rowToAgent(row: Record<string, unknown>): Agent {
     role: (row.role as string) || undefined,
     config: row.config ? JSON.parse(row.config as string) : {},
     cwd: (row.cwd as string) || process.cwd(),
+    plan: row.plan ? JSON.parse(row.plan as string) : [],
     created_at: row.created_at as Timestamp,
     started_at: (row.started_at as number) || undefined,
     stopped_at: (row.stopped_at as number) || undefined,

package/src/store/instance.ts

@@ -213,9 +213,6 @@ export interface ResolvedInstance {
   /** Whether this is a new instance */
   isNew: boolean;
 
-  /** Whether this is using legacy path mode */
-  isLegacy: boolean;
-
   /** Backend type to use */
   backendType: string;
 }
@@ -300,25 +297,11 @@ export function resolveInstancePath(config: StoreConfig): ResolvedInstance {
       instancePath: ':memory:',
       namespace,
       isNew: true,
-      isLegacy: false,
       backendType: 'memory',
     };
   }
 
-  // Legacy path takes precedence for backward compat
-  if (config.path) {
-    const legacyInstanceId = deriveInstanceIdFromPath(config.path);
-    return {
-      instanceId: legacyInstanceId,
-      instancePath: config.path,
-      namespace,
-      isNew: !fs.existsSync(config.path),
-      isLegacy: true,
-      backendType: 'json',
-    };
-  }
-
-  // New instance resolution
+  // Instance resolution
   const instanceId = config.instanceId ?? generateInstanceId();
 
   if (!isValidInstanceId(instanceId)) {
@@ -339,21 +322,10 @@ export function resolveInstancePath(config: StoreConfig): ResolvedInstance {
     instancePath,
     namespace,
     isNew,
-    isLegacy: false,
     backendType,
   };
 }
 
-/**
- * Derive an instance ID from a legacy path
- */
-function deriveInstanceIdFromPath(legacyPath: string): string {
-  // Use the filename without extension as the instance ID
-  const basename = path.basename(legacyPath, path.extname(legacyPath));
-  // Sanitize to valid instance ID
-  return basename.replace(/[^a-zA-Z0-9_-]/g, '_');
-}
-
 // ─────────────────────────────────────────────────────────────────────────────
 // Instance Directory Management
 // ─────────────────────────────────────────────────────────────────────────────

package/src/store/types/agents.ts

@@ -40,6 +40,7 @@ export interface Agent {
   role?: string;
   config: AgentConfig;
   cwd: string;
+  plan: Array<{ content: string; priority: string; status: string }>;
   created_at: Timestamp;
   started_at?: Timestamp;
   stopped_at?: Timestamp;