agent-working-memory 0.7.16 → 0.8.5

package/src/storage/factory.ts

@@ -0,0 +1,147 @@
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Storage backend factory.
+ *
+ * Picks between SQLite (mature, full features) and PGlite (portable, async,
+ * pgvector). Both backends satisfy the IEngramStore contract — the cognitive
+ * engines work with either through `await`.
+ *
+ *   AWM_STORE_BACKEND=sqlite  — better-sqlite3 + FTS5 + BLOB embeddings
+ *   AWM_STORE_BACKEND=pglite  — PGlite + pgvector + tsvector
+ *
+ * **Backend selection (v0.8.5):** auto-detect on disk when the env var is
+ * unset, so upgrading users on existing `memory.db` files keep working
+ * without setting anything. Order of precedence:
+ *
+ *   1. `AWM_STORE_BACKEND` env var (explicit override — always wins).
+ *   2. Auto-detect: if `memory-pglite/` exists on disk → pglite.
+ *   3. Auto-detect: else if `memory.db` exists on disk → sqlite.
+ *   4. Fall back to sqlite (the default for fresh installs).
+ *
+ * If `AWM_STORE_BACKEND` is set but the on-disk state disagrees (e.g. env
+ * says pglite but only `memory.db` is present), `openStore` prints a one-line
+ * warning suggesting `awm migrate`. The env var still wins — we never
+ * silently switch backends behind the user's back.
+ *
+ * The `AWM_DB_PATH` env var carries the file path (SQLite) or directory
+ * path (PGlite). When unset, default is `memory.db` (sqlite) or
+ * `memory-pglite/` (pglite).
+ *
+ * **Feature parity gaps** (see `docs/pglite-feature-parity.md`):
+ * a few legacy code paths reach for SQLite-specific methods
+ * (coordination plugin needs `store.getDb()`, slim-cache warming, hot
+ * backups). PGlite-backed AWM is fully functional for the cognitive
+ * engines (write, recall, consolidation, retraction, eviction). Opt-in
+ * `AWM_STORE_BACKEND=pglite` skips the SQLite-only extras.
+ */
+
+import { existsSync, statSync } from 'node:fs';
+import type { IEngramStore } from './store.js';
+
+export type StoreBackend = 'sqlite' | 'pglite';
+
+/**
+ * Auto-detect the backend from on-disk state. Used when `AWM_STORE_BACKEND`
+ * is unset. Looks in the current working directory; honors `AWM_DB_PATH`
+ * if it points at a known shape.
+ */
+function detectBackendFromDisk(): StoreBackend | null {
+  // If AWM_DB_PATH is set, infer from its shape.
+  const explicitPath = process.env.AWM_DB_PATH;
+  if (explicitPath && existsSync(explicitPath)) {
+    try {
+      const stat = statSync(explicitPath);
+      if (stat.isDirectory()) return 'pglite'; // PGlite uses a directory
+      if (stat.isFile()) return 'sqlite';      // SQLite is a single file
+    } catch { /* fall through */ }
+  }
+
+  // Otherwise look for the conventional defaults in cwd.
+  // PGlite directory wins over SQLite file when both exist — assume the
+  // user actively migrated and forgot to set the env var. We warn below.
+  if (existsSync('memory-pglite')) return 'pglite';
+  if (existsSync('memory.db')) return 'sqlite';
+  return null;
+}
+
+export function getConfiguredBackend(): StoreBackend {
+  const raw = process.env.AWM_STORE_BACKEND;
+  if (raw !== undefined && raw !== '') {
+    const normalized = raw.toLowerCase();
+    if (normalized === 'pglite') return 'pglite';
+    if (normalized === 'sqlite') return 'sqlite';
+    console.warn(`Unknown AWM_STORE_BACKEND=${raw}; falling back to sqlite`);
+    return 'sqlite';
+  }
+  // Env unset → auto-detect from on-disk state.
+  const detected = detectBackendFromDisk();
+  return detected ?? 'sqlite';
+}
+
+export function getConfiguredPath(): string {
+  if (process.env.AWM_DB_PATH) return process.env.AWM_DB_PATH;
+  return getConfiguredBackend() === 'pglite' ? 'memory-pglite' : 'memory.db';
+}
+
+/**
+ * Print a one-line warning to stderr when the configured backend disagrees
+ * with what's actually on disk. We never fail or silently switch backends
+ * — the explicit configuration always wins. The warning helps users notice
+ * that they may have stranded data on the other backend.
+ */
+function warnIfBackendDisagreesWithDisk(backend: StoreBackend, path: string): void {
+  if (process.env.AWM_SUPPRESS_BACKEND_WARNINGS === '1') return;
+
+  // Only warn when env var was explicit (auto-detect already follows disk).
+  if (!process.env.AWM_STORE_BACKEND) return;
+
+  const otherPath = backend === 'pglite' ? 'memory.db' : 'memory-pglite';
+  const otherBackend = backend === 'pglite' ? 'sqlite' : 'pglite';
+
+  // The "real" check: configured target doesn't exist yet (fresh / empty) AND
+  // the other backend's conventional file exists with data. Likely stranded.
+  const configuredExists = existsSync(path);
+  const otherExists = existsSync(otherPath);
+
+  if (!configuredExists && otherExists) {
+    console.warn(
+      `[awm] AWM_STORE_BACKEND=${backend} but no data at "${path}". ` +
+      `Existing ${otherBackend} data at "${otherPath}" — run \`awm migrate\` ` +
+      `to convert it to ${backend}, or unset AWM_STORE_BACKEND to use ${otherBackend}. ` +
+      `Suppress with AWM_SUPPRESS_BACKEND_WARNINGS=1.`,
+    );
+  }
+}
+
+/**
+ * Open a store using the env-configured (or auto-detected) backend and path.
+ * Returns the concrete store class (cast to IEngramStore at call sites
+ * that need the async contract; SQLite callers can keep the concrete class
+ * to retain access to SQLite-specific methods like getDb()).
+ *
+ * Never fails on backend/disk mismatch — only warns. Existing users
+ * upgrading without setting env vars get auto-detect; users on the legacy
+ * `memory.db` keep working without touching anything.
+ */
+export async function openStore(): Promise<{
+  store: IEngramStore;
+  backend: StoreBackend;
+  path: string;
+}> {
+  const backend = getConfiguredBackend();
+  const path = getConfiguredPath();
+
+  warnIfBackendDisagreesWithDisk(backend, path);
+
+  if (backend === 'pglite') {
+    const { PGliteEngramStore } = await import('./pglite.js');
+    const store = new PGliteEngramStore(path);
+    await store.ready();
+    return { store: store as unknown as IEngramStore, backend, path };
+  }
+
+  const { EngramStore } = await import('./sqlite.js');
+  const store = new EngramStore(path);
+  return { store: store as unknown as IEngramStore, backend, path };
+}

package/src/storage/index.ts

@@ -1,3 +1,3 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-export * from './sqlite.js';
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+export * from './sqlite.js';

package/src/storage/pglite-schema.ts

@@ -0,0 +1,166 @@
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * PGlite schema DDL — translated from the SQLite schema in sqlite.ts:init().
+ *
+ * Translation notes:
+ *   - SQLite TEXT → Postgres TEXT
+ *   - SQLite REAL → Postgres DOUBLE PRECISION
+ *   - SQLite INTEGER (0/1 booleans) → Postgres BOOLEAN
+ *   - SQLite BLOB embedding → pgvector VECTOR(384) (native)
+ *   - SQLite FTS5 virtual table + triggers → Postgres tsvector column with
+ *     a GIN index, kept up-to-date via a BEFORE INSERT/UPDATE trigger.
+ *
+ * All tables and indexes use IF NOT EXISTS so re-running is idempotent.
+ *
+ * The vector extension MUST be loaded before this schema runs:
+ *   await db.exec('CREATE EXTENSION IF NOT EXISTS vector')
+ * PGlite's vector extension does this automatically when imported.
+ */
+
+export const PGLITE_VECTOR_DIMENSIONS = parseInt(process.env.AWM_EMBED_DIMS ?? '384', 10);
+
+export const PGLITE_SCHEMA_DDL = `
+  CREATE EXTENSION IF NOT EXISTS vector;
+
+  CREATE TABLE IF NOT EXISTS engrams (
+    id                   TEXT PRIMARY KEY,
+    agent_id             TEXT NOT NULL,
+    concept              TEXT NOT NULL,
+    content              TEXT NOT NULL,
+    embedding            VECTOR(${PGLITE_VECTOR_DIMENSIONS}),
+    embedding_model      TEXT,
+    confidence           DOUBLE PRECISION NOT NULL DEFAULT 0.5,
+    salience             DOUBLE PRECISION NOT NULL DEFAULT 0.5,
+    access_count         INTEGER NOT NULL DEFAULT 0,
+    last_accessed        TEXT NOT NULL,
+    created_at           TEXT NOT NULL,
+    salience_features    TEXT NOT NULL DEFAULT '{}',
+    reason_codes         TEXT NOT NULL DEFAULT '[]',
+    stage                TEXT NOT NULL DEFAULT 'active',
+    ttl                  INTEGER,
+    retracted            BOOLEAN NOT NULL DEFAULT FALSE,
+    retracted_by         TEXT,
+    retracted_at         TEXT,
+    tags                 TEXT NOT NULL DEFAULT '[]',
+    memory_type          TEXT NOT NULL DEFAULT 'unclassified',
+    memory_class         TEXT NOT NULL DEFAULT 'working',
+    superseded_by        TEXT,
+    supersedes           TEXT,
+    episode_id           TEXT,
+    task_status          TEXT,
+    task_priority        TEXT,
+    blocked_by           TEXT,
+    sequence             INTEGER,
+    references_json      TEXT,
+    fts                  TSVECTOR
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_engrams_agent           ON engrams(agent_id);
+  CREATE INDEX IF NOT EXISTS idx_engrams_stage           ON engrams(agent_id, stage);
+  CREATE INDEX IF NOT EXISTS idx_engrams_concept         ON engrams(concept);
+  CREATE INDEX IF NOT EXISTS idx_engrams_retracted       ON engrams(agent_id, retracted);
+  CREATE INDEX IF NOT EXISTS idx_engrams_task            ON engrams(agent_id, task_status);
+  CREATE INDEX IF NOT EXISTS idx_engrams_agent_sequence  ON engrams(agent_id, sequence) WHERE sequence IS NOT NULL;
+  CREATE INDEX IF NOT EXISTS idx_engrams_fts             ON engrams USING GIN(fts);
+
+  CREATE INDEX IF NOT EXISTS idx_engrams_embedding
+    ON engrams USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
+
+  CREATE OR REPLACE FUNCTION engrams_fts_update() RETURNS TRIGGER AS $$
+  BEGIN
+    NEW.fts := to_tsvector('english',
+      COALESCE(NEW.concept, '') || ' ' ||
+      COALESCE(NEW.content, '') || ' ' ||
+      COALESCE(NEW.tags, ''));
+    RETURN NEW;
+  END;
+  $$ LANGUAGE plpgsql;
+
+  DROP TRIGGER IF EXISTS engrams_fts_trigger ON engrams;
+  CREATE TRIGGER engrams_fts_trigger
+    BEFORE INSERT OR UPDATE OF concept, content, tags ON engrams
+    FOR EACH ROW EXECUTE FUNCTION engrams_fts_update();
+
+  CREATE TABLE IF NOT EXISTS associations (
+    id                TEXT PRIMARY KEY,
+    from_engram_id    TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
+    to_engram_id      TEXT NOT NULL REFERENCES engrams(id) ON DELETE CASCADE,
+    weight            DOUBLE PRECISION NOT NULL DEFAULT 0.1,
+    confidence        DOUBLE PRECISION NOT NULL DEFAULT 0.5,
+    type              TEXT NOT NULL DEFAULT 'hebbian',
+    activation_count  INTEGER NOT NULL DEFAULT 0,
+    created_at        TEXT NOT NULL,
+    last_activated    TEXT NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_assoc_from        ON associations(from_engram_id);
+  CREATE INDEX IF NOT EXISTS idx_assoc_to          ON associations(to_engram_id);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_assoc_pair ON associations(from_engram_id, to_engram_id);
+
+  CREATE TABLE IF NOT EXISTS agents (
+    id          TEXT PRIMARY KEY,
+    name        TEXT NOT NULL,
+    created_at  TEXT NOT NULL,
+    config      TEXT NOT NULL DEFAULT '{}'
+  );
+
+  CREATE TABLE IF NOT EXISTS activation_events (
+    id                 TEXT PRIMARY KEY,
+    agent_id           TEXT NOT NULL,
+    timestamp          TEXT NOT NULL,
+    context            TEXT NOT NULL,
+    results_returned   INTEGER NOT NULL,
+    top_score          DOUBLE PRECISION,
+    latency_ms         DOUBLE PRECISION NOT NULL,
+    engram_ids         TEXT NOT NULL DEFAULT '[]'
+  );
+
+  CREATE TABLE IF NOT EXISTS staging_events (
+    engram_id        TEXT NOT NULL,
+    agent_id         TEXT NOT NULL,
+    action           TEXT NOT NULL,
+    resonance_score  DOUBLE PRECISION,
+    timestamp        TEXT NOT NULL,
+    age_ms           INTEGER NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS retrieval_feedback (
+    id                   TEXT PRIMARY KEY,
+    activation_event_id  TEXT,
+    engram_id            TEXT NOT NULL,
+    useful               BOOLEAN NOT NULL,
+    context              TEXT,
+    timestamp            TEXT NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS episodes (
+    id             TEXT PRIMARY KEY,
+    agent_id       TEXT NOT NULL,
+    label          TEXT NOT NULL,
+    embedding      VECTOR(${PGLITE_VECTOR_DIMENSIONS}),
+    engram_count   INTEGER NOT NULL DEFAULT 0,
+    start_time     TEXT NOT NULL,
+    end_time       TEXT NOT NULL,
+    created_at     TEXT NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id);
+  CREATE INDEX IF NOT EXISTS idx_episodes_time  ON episodes(agent_id, end_time);
+
+  CREATE TABLE IF NOT EXISTS conscious_state (
+    agent_id                            TEXT PRIMARY KEY,
+    last_write_id                       TEXT,
+    last_recall_context                 TEXT,
+    last_recall_ids                     TEXT NOT NULL DEFAULT '[]',
+    last_activity_at                    TEXT NOT NULL DEFAULT (to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
+    write_count_since_consolidation     INTEGER NOT NULL DEFAULT 0,
+    recall_count_since_consolidation    INTEGER NOT NULL DEFAULT 0,
+    execution_state                     TEXT,
+    checkpoint_at                       TEXT,
+    last_consolidation_at               TEXT,
+    last_mini_consolidation_at          TEXT,
+    consolidation_cycle_count           INTEGER NOT NULL DEFAULT 0,
+    updated_at                          TEXT NOT NULL DEFAULT (to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))
+  );
+`;