agent-working-memory 0.7.17 → 0.8.6

package/src/cli/migrate.ts

@@ -0,0 +1,307 @@
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * `awm migrate` — Convert a SQLite AWM database to a PGlite store.
+ *
+ * Usage:
+ *   awm migrate --from <sqlite.db> --to <pglite-dir> [--dry-run]
+ *
+ * Reads each table from the source SQLite database and writes it to a fresh
+ * PGlite directory. Embedding BLOBs are converted from Float32Array bytes to
+ * pgvector text format (`[v1,v2,...]`). Integer booleans (0/1) become
+ * Postgres BOOLEAN. Timestamps are passed through as ISO strings.
+ *
+ * Tables migrated:
+ *   - engrams + their derived FTS index (regenerated by PGlite trigger)
+ *   - associations
+ *   - agents
+ *   - activation_events
+ *   - staging_events
+ *   - retrieval_feedback
+ *   - episodes
+ *   - conscious_state
+ */
+
+import Database from 'better-sqlite3';
+import { PGlite } from '@electric-sql/pglite';
+import { vector } from '@electric-sql/pglite/vector';
+import { PGLITE_SCHEMA_DDL } from '../storage/pglite-schema.js';
+import { existsSync, mkdirSync } from 'node:fs';
+
+function bufferToFloat32Array(buf: Buffer): Float32Array {
+  const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+  return new Float32Array(ab);
+}
+
+function vectorLiteral(values: number[] | null): string | null {
+  if (!values || values.length === 0) return null;
+  return '[' + values.join(',') + ']';
+}
+
+function intToBool(v: unknown): boolean {
+  return v === 1 || v === true || v === '1' || v === 'true';
+}
+
+interface MigrateOptions {
+  from: string;
+  to: string;
+  dryRun?: boolean;
+  batchSize?: number;
+  verbose?: boolean;
+}
+
+interface MigrateStats {
+  engrams: number;
+  associations: number;
+  agents: number;
+  activationEvents: number;
+  stagingEvents: number;
+  retrievalFeedback: number;
+  episodes: number;
+  consciousState: number;
+}
+
+export async function migrate(opts: MigrateOptions): Promise<MigrateStats> {
+  const { from, to, dryRun = false, batchSize = 500, verbose = false } = opts;
+
+  if (!existsSync(from)) {
+    throw new Error(`Source SQLite database not found: ${from}`);
+  }
+
+  if (!dryRun && !existsSync(to)) {
+    mkdirSync(to, { recursive: true });
+  }
+
+  const src = new Database(from, { readonly: true });
+  const dst = dryRun ? null : await PGlite.create(to, { extensions: { vector } });
+
+  if (dst) {
+    await dst.exec(PGLITE_SCHEMA_DDL);
+  }
+
+  const stats: MigrateStats = {
+    engrams: 0,
+    associations: 0,
+    agents: 0,
+    activationEvents: 0,
+    stagingEvents: 0,
+    retrievalFeedback: 0,
+    episodes: 0,
+    consciousState: 0,
+  };
+
+  // ── engrams ─────────────────────────────────────────────
+  const engramRows = src.prepare(`SELECT * FROM engrams`).all() as any[];
+  if (verbose) console.log(`engrams: ${engramRows.length} rows to migrate`);
+  if (dst && engramRows.length > 0) {
+    for (let i = 0; i < engramRows.length; i += batchSize) {
+      const batch = engramRows.slice(i, i + batchSize);
+      for (const r of batch) {
+        const embedding = r.embedding
+          ? vectorLiteral(Array.from(bufferToFloat32Array(r.embedding)))
+          : null;
+        await dst.query(
+          `INSERT INTO engrams (
+             id, agent_id, concept, content, embedding, embedding_model,
+             confidence, salience, access_count, last_accessed, created_at,
+             salience_features, reason_codes, stage, ttl,
+             retracted, retracted_by, retracted_at, tags, memory_type,
+             memory_class, superseded_by, supersedes, episode_id,
+             task_status, task_priority, blocked_by, sequence, references_json
+           ) VALUES (
+             $1, $2, $3, $4, $5::vector, $6, $7, $8, $9, $10, $11,
+             $12, $13, $14, $15, $16, $17, $18, $19, $20,
+             $21, $22, $23, $24, $25, $26, $27, $28, $29
+           )`,
+          [
+            r.id, r.agent_id, r.concept, r.content, embedding, r.embedding_model,
+            r.confidence, r.salience, r.access_count, r.last_accessed, r.created_at,
+            r.salience_features ?? '{}', r.reason_codes ?? '[]',
+            r.stage ?? 'active', r.ttl,
+            intToBool(r.retracted), r.retracted_by, r.retracted_at,
+            r.tags ?? '[]', r.memory_type ?? 'unclassified',
+            r.memory_class ?? 'working', r.superseded_by, r.supersedes, r.episode_id,
+            r.task_status, r.task_priority, r.blocked_by, r.sequence, r.references_json,
+          ],
+        );
+        stats.engrams++;
+      }
+      if (verbose) process.stdout.write(`  ${Math.min(i + batchSize, engramRows.length)}/${engramRows.length}\r`);
+    }
+    if (verbose) console.log(`  engrams: done (${stats.engrams})`);
+  } else if (dryRun) {
+    stats.engrams = engramRows.length;
+  }
+
+  // ── associations ────────────────────────────────────────
+  const assocRows = src.prepare(`SELECT * FROM associations`).all() as any[];
+  if (verbose) console.log(`associations: ${assocRows.length} rows`);
+  if (dst && assocRows.length > 0) {
+    for (let i = 0; i < assocRows.length; i += batchSize) {
+      const batch = assocRows.slice(i, i + batchSize);
+      for (const r of batch) {
+        await dst.query(
+          `INSERT INTO associations (
+             id, from_engram_id, to_engram_id, weight, confidence, type,
+             activation_count, created_at, last_activated
+           ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
+          [
+            r.id, r.from_engram_id, r.to_engram_id, r.weight, r.confidence,
+            r.type ?? 'hebbian', r.activation_count ?? 0,
+            r.created_at, r.last_activated,
+          ],
+        );
+        stats.associations++;
+      }
+    }
+  } else if (dryRun) {
+    stats.associations = assocRows.length;
+  }
+
+  // ── agents ──────────────────────────────────────────────
+  const agentRows = src.prepare(`SELECT * FROM agents`).all() as any[];
+  if (verbose) console.log(`agents: ${agentRows.length} rows`);
+  if (dst && agentRows.length > 0) {
+    for (const r of agentRows) {
+      await dst.query(
+        `INSERT INTO agents (id, name, created_at, config) VALUES ($1,$2,$3,$4)
+         ON CONFLICT (id) DO NOTHING`,
+        [r.id, r.name, r.created_at, r.config ?? '{}'],
+      );
+      stats.agents++;
+    }
+  } else if (dryRun) {
+    stats.agents = agentRows.length;
+  }
+
+  // ── activation_events ───────────────────────────────────
+  const aeRows = src.prepare(`SELECT * FROM activation_events`).all() as any[];
+  if (verbose) console.log(`activation_events: ${aeRows.length} rows`);
+  if (dst && aeRows.length > 0) {
+    for (let i = 0; i < aeRows.length; i += batchSize) {
+      for (const r of aeRows.slice(i, i + batchSize)) {
+        await dst.query(
+          `INSERT INTO activation_events
+             (id, agent_id, timestamp, context, results_returned, top_score, latency_ms, engram_ids)
+           VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`,
+          [r.id, r.agent_id, r.timestamp, r.context, r.results_returned,
+            r.top_score, r.latency_ms, r.engram_ids ?? '[]'],
+        );
+        stats.activationEvents++;
+      }
+    }
+  } else if (dryRun) {
+    stats.activationEvents = aeRows.length;
+  }
+
+  // ── staging_events ──────────────────────────────────────
+  const seRows = src.prepare(`SELECT * FROM staging_events`).all() as any[];
+  if (verbose) console.log(`staging_events: ${seRows.length} rows`);
+  if (dst && seRows.length > 0) {
+    for (const r of seRows) {
+      await dst.query(
+        `INSERT INTO staging_events (engram_id, agent_id, action, resonance_score, timestamp, age_ms)
+         VALUES ($1,$2,$3,$4,$5,$6)`,
+        [r.engram_id, r.agent_id, r.action, r.resonance_score, r.timestamp, r.age_ms],
+      );
+      stats.stagingEvents++;
+    }
+  } else if (dryRun) {
+    stats.stagingEvents = seRows.length;
+  }
+
+  // ── retrieval_feedback ──────────────────────────────────
+  const rfRows = src.prepare(`SELECT * FROM retrieval_feedback`).all() as any[];
+  if (verbose) console.log(`retrieval_feedback: ${rfRows.length} rows`);
+  if (dst && rfRows.length > 0) {
+    for (const r of rfRows) {
+      await dst.query(
+        `INSERT INTO retrieval_feedback
+           (id, activation_event_id, engram_id, useful, context, timestamp)
+         VALUES ($1,$2,$3,$4,$5,$6)`,
+        [r.id, r.activation_event_id, r.engram_id, intToBool(r.useful), r.context, r.timestamp],
+      );
+      stats.retrievalFeedback++;
+    }
+  } else if (dryRun) {
+    stats.retrievalFeedback = rfRows.length;
+  }
+
+  // ── episodes ────────────────────────────────────────────
+  // The episodes table may not exist in older SQLite databases.
+  try {
+    const epRows = src.prepare(`SELECT * FROM episodes`).all() as any[];
+    if (verbose) console.log(`episodes: ${epRows.length} rows`);
+    if (dst && epRows.length > 0) {
+      for (const r of epRows) {
+        const embedding = r.embedding
+          ? vectorLiteral(Array.from(bufferToFloat32Array(r.embedding)))
+          : null;
+        await dst.query(
+          `INSERT INTO episodes
+             (id, agent_id, label, embedding, engram_count, start_time, end_time, created_at)
+           VALUES ($1,$2,$3,$4::vector,$5,$6,$7,$8)`,
+          [r.id, r.agent_id, r.label, embedding, r.engram_count ?? 0,
+            r.start_time, r.end_time, r.created_at],
+        );
+        stats.episodes++;
+      }
+    } else if (dryRun) {
+      stats.episodes = epRows.length;
+    }
+  } catch {
+    if (verbose) console.log(`episodes: table not present in source (skipping)`);
+  }
+
+  // ── conscious_state ─────────────────────────────────────
+  try {
+    const csRows = src.prepare(`SELECT * FROM conscious_state`).all() as any[];
+    if (verbose) console.log(`conscious_state: ${csRows.length} rows`);
+    if (dst && csRows.length > 0) {
+      for (const r of csRows) {
+        await dst.query(
+          `INSERT INTO conscious_state (
+             agent_id, last_write_id, last_recall_context, last_recall_ids,
+             last_activity_at, write_count_since_consolidation,
+             recall_count_since_consolidation, execution_state, checkpoint_at,
+             last_consolidation_at, last_mini_consolidation_at,
+             consolidation_cycle_count, updated_at
+           ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)
+           ON CONFLICT (agent_id) DO NOTHING`,
+          [
+            r.agent_id, r.last_write_id, r.last_recall_context,
+            r.last_recall_ids ?? '[]', r.last_activity_at,
+            r.write_count_since_consolidation ?? 0,
+            r.recall_count_since_consolidation ?? 0,
+            r.execution_state, r.checkpoint_at,
+            r.last_consolidation_at, r.last_mini_consolidation_at,
+            r.consolidation_cycle_count ?? 0, r.updated_at,
+          ],
+        );
+        stats.consciousState++;
+      }
+    } else if (dryRun) {
+      stats.consciousState = csRows.length;
+    }
+  } catch {
+    if (verbose) console.log(`conscious_state: table not present in source (skipping)`);
+  }
+
+  src.close();
+  if (dst) await dst.close();
+
+  return stats;
+}
+
+export function printStats(stats: MigrateStats, dryRun: boolean): void {
+  const verb = dryRun ? 'would migrate' : 'migrated';
+  console.log(`\nMigration ${dryRun ? 'plan' : 'complete'}:`);
+  console.log(`  ${verb} ${stats.engrams} engrams`);
+  console.log(`  ${verb} ${stats.associations} associations`);
+  console.log(`  ${verb} ${stats.agents} agents`);
+  console.log(`  ${verb} ${stats.activationEvents} activation events`);
+  console.log(`  ${verb} ${stats.stagingEvents} staging events`);
+  console.log(`  ${verb} ${stats.retrievalFeedback} retrieval feedback records`);
+  console.log(`  ${verb} ${stats.episodes} episodes`);
+  console.log(`  ${verb} ${stats.consciousState} conscious_state rows`);
+}