mixdog 0.7.17 → 0.8.0

package/src/runtime/memory/lib/memory.mjs

@@ -0,0 +1,418 @@
+const __mixdogMemoryStderrWrite = process.stderr.write.bind(process.stderr);
+function __mixdogMemoryLog(...args) {
+  if (process.env.MIXDOG_QUIET_MEMORY_LOG) return true;
+  return __mixdogMemoryStderrWrite(...args);
+}
+
+// Native-PG-backed memory store. Schema, helpers, and lifecycle.
+
+import { ensurePgInstance, closePgInstance, withSchemaBootstrapLock } from './pg/adapter.mjs'
+import { mkdirSync } from 'fs'
+import { resolve } from 'path'
+import { cleanMemoryText } from './memory-extraction.mjs'
+
+const dbs = new Map()
+const opening = new Map()
+
+export { cleanMemoryText }
+
+export const VALID_CATEGORY = new Set([
+  'rule', 'constraint', 'decision', 'fact', 'goal', 'preference', 'task', 'issue',
+])
+
+export async function init(db, dims) {
+  const dimCount = Number(dims)
+  if (!Number.isInteger(dimCount) || dimCount <= 0) {
+    throw new Error(`init: dims must be a positive integer, got ${dims}`)
+  }
+
+  // Extensions are created once by pg-adapter.bootstrapInstance; skip here.
+
+  // Status as a real ENUM type — DB-level enforcement, B-tree friendly.
+  // PG has no CREATE TYPE IF NOT EXISTS; guard via pg_type lookup so a partial
+  // bootstrap (crash after CREATE TYPE but before boot.schema_bootstrap_complete)
+  // can re-run init() on the next boot without colliding on the existing type.
+  await db.exec(`
+    DO $$
+    BEGIN
+      IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'entry_status') THEN
+        CREATE TYPE entry_status AS ENUM ('pending', 'active', 'archived');
+      END IF;
+    END
+    $$
+  `)
+
+  // Per-category score parameters (lookup table for the score function).
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS category_score_params (
+      category TEXT PRIMARY KEY,
+      grade    REAL NOT NULL,
+      decay    REAL NOT NULL
+    )
+  `)
+  await db.query(`
+    INSERT INTO category_score_params(category, grade, decay) VALUES
+      ('rule', 2.0, 0.0),
+      ('constraint', 1.9, 0.06),
+      ('decision', 1.8, 0.15),
+      ('fact', 1.6, 0.25),
+      ('goal', 1.5, 0.30),
+      ('preference', 1.4, 0.35),
+      ('task', 1.1, 0.45),
+      ('issue', 1.0, 0.50)
+    ON CONFLICT (category) DO NOTHING
+  `)
+
+  // SQL function mirrors src/memory/lib/memory-score.mjs computeEntryScore.
+  // STABLE (not IMMUTABLE) because the function reads category_score_params.
+  // IMMUTABLE would let the planner cache results across rows where params
+  // could legitimately differ if the table is updated.
+  await db.exec(`
+    CREATE OR REPLACE FUNCTION compute_entry_score(
+      category_p TEXT,
+      last_seen_at_p BIGINT,
+      now_ms_p BIGINT
+    ) RETURNS REAL LANGUAGE sql STABLE AS $$
+      SELECT CASE
+        WHEN p.grade IS NULL OR last_seen_at_p IS NULL OR now_ms_p IS NULL THEN NULL::REAL
+        WHEN p.decay = 0 THEN p.grade
+        ELSE LEAST(
+          p.grade,
+          p.grade / POWER(
+            1 + (GREATEST(0, (now_ms_p - last_seen_at_p)) / 86400000.0) * p.decay / 30,
+            0.3
+          )
+        )::REAL
+      END
+      FROM category_score_params p
+      WHERE p.category = category_p
+    $$
+  `)
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS entries (
+      id            BIGSERIAL PRIMARY KEY,
+      ts            BIGINT NOT NULL,
+      role          TEXT NOT NULL,
+      content       TEXT NOT NULL,
+      source_ref    TEXT NOT NULL UNIQUE,
+      session_id    TEXT,
+      project_id    TEXT,
+      source_turn   INTEGER,
+      chunk_root    BIGINT REFERENCES entries(id) ON DELETE SET NULL,
+      is_root       SMALLINT NOT NULL DEFAULT 0,
+      element       TEXT,
+      category      TEXT,
+      summary       TEXT,
+      core_summary  TEXT,
+      status        entry_status,
+      score         REAL,
+      last_seen_at  BIGINT,
+      reviewed_at   BIGINT,
+      promoted_at   BIGINT,
+      error_count   INTEGER NOT NULL DEFAULT 0,
+      embedding     halfvec(${dimCount}),
+      summary_hash  TEXT,
+      search_tsv    tsvector GENERATED ALWAYS AS (
+        setweight(to_tsvector('simple',  coalesce(element, '')), 'A') ||
+        setweight(to_tsvector('simple',  coalesce(summary, '')), 'B') ||
+        setweight(to_tsvector('simple',  coalesce(content, '')), 'C') ||
+        setweight(to_tsvector('english', coalesce(element, '')), 'A') ||
+        setweight(to_tsvector('english', coalesce(summary, '')), 'B') ||
+        setweight(to_tsvector('english', coalesce(content, '')), 'C')
+      ) STORED
+    )
+  `)
+
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_chunk_root  ON entries(chunk_root) WHERE chunk_root IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_ts_desc     ON entries(ts DESC)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_session_ts  ON entries(session_id, ts DESC) WHERE session_id IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_root_status_score ON entries(status, score DESC) WHERE is_root = 1`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_root_category     ON entries(category, status)   WHERE is_root = 1`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_pending     ON entries(ts DESC, id DESC) WHERE chunk_root IS NULL AND session_id IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_roots_active        ON entries(status, last_seen_at ASC, score DESC) WHERE is_root = 1 AND status = 'active'`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_project     ON entries(project_id) WHERE project_id IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_reviewed_at ON entries(reviewed_at ASC) WHERE is_root = 1`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_phase_sweep ON entries(status, is_root, error_count, reviewed_at, id)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_promoted_at ON entries(promoted_at) WHERE promoted_at IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_tsv         ON entries USING GIN (search_tsv)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_content_trgm ON entries USING GIN (content gin_trgm_ops) WHERE is_root = 1`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_element_trgm ON entries USING GIN (element gin_trgm_ops) WHERE is_root = 1 AND element IS NOT NULL`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS idx_entries_embedding_hnsw ON entries USING hnsw (embedding halfvec_cosine_ops) WHERE is_root = 1 AND embedding IS NOT NULL`)
+
+  // BEFORE INSERT/UPDATE trigger keeps score in sync with category + last_seen_at
+  // automatically; cycle code no longer needs to UPDATE entries SET score = ...
+  await db.exec(`
+    CREATE OR REPLACE FUNCTION trg_entry_score_recalc() RETURNS trigger LANGUAGE plpgsql AS $$
+    BEGIN
+      IF NEW.is_root = 1 AND NEW.category IS NOT NULL THEN
+        -- NOW()-to-ms conversion is intentional schema-level work; the
+        -- "no EXTRACT(EPOCH …)" rule applies to ms-stored BIGINT timestamp
+        -- COLUMNS, not to the trigger reading the current wall clock.
+        NEW.score := compute_entry_score(
+          NEW.category,
+          COALESCE(NEW.last_seen_at, NEW.ts),
+          (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
+        );
+      END IF;
+      RETURN NEW;
+    END;
+    $$
+  `)
+  await db.exec(`DROP TRIGGER IF EXISTS trg_entries_score ON entries`)
+  await db.exec(`
+    CREATE TRIGGER trg_entries_score
+    BEFORE INSERT OR UPDATE OF category, last_seen_at, promoted_at, is_root ON entries
+    FOR EACH ROW
+    EXECUTE FUNCTION trg_entry_score_recalc()
+  `)
+
+  await db.exec(`
+    CREATE OR REPLACE FUNCTION trg_entry_embedding_invalidate() RETURNS trigger LANGUAGE plpgsql AS $$
+    BEGIN
+      IF NEW.is_root = 1 AND (
+        NEW.content IS DISTINCT FROM OLD.content OR
+        NEW.summary IS DISTINCT FROM OLD.summary OR
+        NEW.element IS DISTINCT FROM OLD.element
+      ) THEN
+        NEW.embedding := NULL;
+        NEW.summary_hash := NULL;
+      END IF;
+      RETURN NEW;
+    END;
+    $$
+  `)
+  await db.exec(`DROP TRIGGER IF EXISTS trg_entries_embedding_invalidate ON entries`)
+  await db.exec(`
+    CREATE TRIGGER trg_entries_embedding_invalidate
+    BEFORE UPDATE OF content, summary, element ON entries
+    FOR EACH ROW EXECUTE FUNCTION trg_entry_embedding_invalidate()
+  `)
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS core_entries (
+      id          BIGSERIAL PRIMARY KEY,
+      element     TEXT NOT NULL,
+      summary     TEXT NOT NULL,
+      category    TEXT NOT NULL,
+      project_id  TEXT,
+      embedding   halfvec(${dimCount}),
+      created_at  BIGINT NOT NULL,
+      updated_at  BIGINT NOT NULL
+    )
+  `)
+  await db.exec(`CREATE INDEX IF NOT EXISTS core_entries_project_idx ON core_entries(project_id)`)
+  await db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS core_entries_unique_proj_elem ON core_entries (project_id, element) NULLS NOT DISTINCT`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS core_entries_embedding_hnsw ON core_entries USING hnsw (embedding halfvec_cosine_ops) WHERE embedding IS NOT NULL`)
+
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS meta (
+      key    TEXT PRIMARY KEY,
+      value  JSONB NOT NULL
+    )
+  `)
+
+  // Operational view — used by /health and dashboards. One round-trip,
+  // covers the metrics that previously needed 6+ COUNT queries.
+  await db.exec(`
+    CREATE OR REPLACE VIEW v_cycle_state AS
+    SELECT
+      COUNT(*) FILTER (WHERE is_root = 1) AS roots,
+      COUNT(*) FILTER (WHERE is_root = 1 AND status = 'pending')  AS pending,
+      COUNT(*) FILTER (WHERE is_root = 1 AND status = 'active')   AS active,
+      COUNT(*) FILTER (WHERE is_root = 1 AND status = 'archived') AS archived,
+      COUNT(*) FILTER (WHERE chunk_root IS NULL)                  AS unclassified,
+      COUNT(*) AS total
+    FROM entries
+  `)
+
+  // Hot active set — recall hot path uses the materialized copy. Refresh hook
+  // is owned by cycle2 (after promotion/archival). Created WITH NO DATA so
+  // bootstrap is fast; first refresh happens on the first cycle2 run.
+  await db.exec(`
+    CREATE MATERIALIZED VIEW IF NOT EXISTS mv_hot_active AS
+    SELECT id, element, summary, category, status, score, last_seen_at, promoted_at,
+           project_id, embedding, search_tsv
+    FROM entries
+    WHERE is_root = 1 AND status = 'active' AND embedding IS NOT NULL
+    WITH NO DATA
+  `)
+  await db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS mv_hot_active_id ON mv_hot_active(id)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_hnsw ON mv_hot_active USING hnsw (embedding halfvec_cosine_ops)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_tsv  ON mv_hot_active USING GIN (search_tsv)`)
+  await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_score ON mv_hot_active(score DESC)`)
+
+  await db.query(
+    `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+     ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value`,
+    ['embedding.current_dims', JSON.stringify(dimCount)],
+  )
+  await db.query(
+    `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+     ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value`,
+    ['boot.schema_bootstrap_complete', JSON.stringify('1')],
+  )
+}
+
+// Validate that the halfvec column dimension stored in the DB matches
+// dimCount from the current model config. Call after schema is confirmed
+// complete and before any embedding operations.
+export async function validateEmbeddingDims(db, dimCount) {
+  const r = await db.query(`
+    SELECT atttypmod
+    FROM pg_attribute a
+    JOIN pg_class c ON c.oid = a.attrelid
+    WHERE c.relname = 'entries'
+      AND a.attname = 'embedding'
+      AND a.attnum > 0
+      AND NOT a.attisdropped
+  `)
+  const row = r.rows[0]
+  if (!row) return // column absent — pre-schema DB; bootstrapSchema will handle
+  // pgvector halfvec stores dimension as atttypmod directly (unlike varchar which uses dims+4).
+  const colDims = row.atttypmod
+  if (colDims !== dimCount) {
+    throw new Error(
+      `Embedding dimension mismatch: DB column halfvec(${colDims}) vs model config ${dimCount} dims. ` +
+      `Reconfigure the embedding model or rebuild the memory store before booting.`
+    )
+  }
+}
+
+export async function ensureCurrentSchemaExtensions(db, dims) {
+  // core_entries gained an embedding column for cross-table semantic dedup
+  // between user-curated rows and cycle2-promoted entries. ALTER + index are
+  // idempotent and define the current runtime schema.
+  if (Number.isInteger(dims) && dims > 0) {
+    await db.exec(`ALTER TABLE core_entries ADD COLUMN IF NOT EXISTS embedding halfvec(${dims})`)
+    await db.exec(`CREATE INDEX IF NOT EXISTS core_entries_embedding_hnsw ON core_entries USING hnsw (embedding halfvec_cosine_ops) WHERE embedding IS NOT NULL`)
+  }
+  await db.exec(`ALTER TABLE entries ADD COLUMN IF NOT EXISTS core_summary text`)
+
+
+
+  // Dedupe core_entries before creating the unique index — keeps the row with
+  // the most recent updated_at (id breaks ties), drops the rest.
+  const dedupe = await db.query(`
+    WITH ranked AS (
+      SELECT id,
+             row_number() OVER (
+               PARTITION BY project_id, element
+               ORDER BY updated_at DESC NULLS LAST, id DESC
+             ) AS rn
+      FROM core_entries
+    ), deleted AS (
+      DELETE FROM core_entries c
+      USING ranked r
+      WHERE c.id = r.id AND r.rn > 1
+      RETURNING c.id
+    )
+    SELECT count(*)::int AS n FROM deleted
+  `)
+  const deduped = Number(dedupe.rows?.[0]?.n ?? 0)
+  if (deduped > 0) {
+    __mixdogMemoryLog(`[memory] ensureCurrentSchemaExtensions: removed ${deduped} duplicate core_entries before unique index creation\n`)
+  }
+  try {
+    await db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS core_entries_unique_proj_elem ON core_entries (project_id, element) NULLS NOT DISTINCT`)
+  } catch (err) {
+    __mixdogMemoryLog(`[memory] ensureCurrentSchemaExtensions: core_entries_unique_proj_elem creation failed — duplicate rows must be deduplicated before this index can be created: ${err?.message || err}\n`)
+    throw err
+  }
+}
+
+export async function openDatabase(dataDir, dims) {
+  const key = resolve(dataDir)
+
+  // Fast path — already resolved.
+  if (dbs.get(key)) return dbs.get(key)
+
+  // Dedupe concurrent callers — return the in-flight Promise if one exists.
+  if (opening.has(key)) return opening.get(key)
+
+  const promise = (async () => {
+    mkdirSync(key, { recursive: true })
+
+    const { db, pool } = await ensurePgInstance(dataDir, { schema: 'memory' })
+
+    if (!(await isBootstrapComplete(db))) {
+      // Serialize the schema/CREATE TYPE bootstrap across concurrent first-boot
+      // processes with a cluster-global advisory lock. Re-check completion once
+      // the lock is held (double-checked locking) so a worker that lost the
+      // race skips the redundant DDL instead of re-running init().
+      await withSchemaBootstrapLock(pool, async () => {
+        if (!(await isBootstrapComplete(db))) {
+          await init(db, dims)
+        }
+      })
+    }
+    await ensureCurrentSchemaExtensions(db, Number(dims))
+    await validateEmbeddingDims(db, Number(dims))
+
+    dbs.set(key, db)
+    return db
+  })()
+
+  opening.set(key, promise)
+  try {
+    return await promise
+  } finally {
+    opening.delete(key)
+  }
+}
+
+export function getDatabase(dataDir) {
+  if (!dataDir) return null
+  const key = resolve(dataDir)
+  return dbs.get(key) ?? null
+}
+
+export async function closeDatabase(dataDir) {
+  const key = resolve(dataDir)
+  const db = dbs.get(key)
+  if (!db) return
+  try { await db.close() } catch {}
+  dbs.delete(key)
+  // Evict pg-adapter's instance cache too: db.close() ends the pool, but the
+  // adapter still holds `instances.get(key)` pointing at the ended pool. A
+  // same-process reopen would then return the dead handle. closePgInstance
+  // drops the cache entry (and re-ends the pool, which is a safe no-op on
+  // an already-ended pool) so the next ensurePgInstance rebuilds fresh.
+  try { await closePgInstance(dataDir, { schema: 'memory' }) } catch {}
+}
+
+export async function isBootstrapComplete(db) {
+  try {
+    const r = await db.query(`SELECT 1 FROM meta WHERE key = 'boot.schema_bootstrap_complete'`)
+    return r.rows.length > 0
+  } catch {
+    return false
+  }
+}
+
+// Returns the raw JSON-encoded string stored in meta.value. Callers JSON.parse
+// it themselves; preserves API parity with the prior TEXT column.
+export async function getMetaValue(db, key, fallback = null) {
+  try {
+    const r = await db.query(`SELECT value::text AS v FROM meta WHERE key = $1`, [key])
+    if (r.rows.length === 0) return fallback
+    return r.rows[0].v ?? fallback
+  } catch {
+    return fallback
+  }
+}
+
+// Caller passes a JSON-encoded string (e.g. JSON.stringify(obj) or a quoted
+// scalar like '"v1"'). Stored verbatim into the JSONB column.
+export async function setMetaValue(db, key, value) {
+  await db.query(
+    `INSERT INTO meta(key, value) VALUES ($1, $2::jsonb)
+     ON CONFLICT(key) DO UPDATE SET value = EXCLUDED.value`,
+    [key, value == null ? 'null' : String(value)],
+  )
+}
+
+export function embeddingToSql(arr) {
+  if (!arr || !Array.isArray(arr)) return null
+  return `[${arr.map((n) => Number(n).toFixed(6)).join(',')}]`
+}