mixdog 0.7.1

package/src/memory/lib/model-profile.mjs

@@ -0,0 +1,85 @@
+/**
+ * model-profile.mjs — Lightweight ONNX model profiling.
+ *
+ * Writes one JSONL line per measurement point to
+ * `history/model-profile.jsonl` (mirrors `retrieval-trace.jsonl`). Meant for
+ * the embedding provider. Fire-and-forget: any I/O error is swallowed so
+ * instrumentation never breaks a live bridge or memory-cycle call.
+ *
+ * Schema:
+ *   {
+ *     ts, phase, model, device, dtype,
+ *     rssBytes, externalBytes, heapUsedBytes,
+ *     cpuUserUs, cpuSystemUs, wallMs,
+ *     docsScored?, note?
+ *   }
+ *
+ * `phase` values used by the providers:
+ *   - 'baseline'      (pre-load RSS / CPU snapshot)
+ *   - 'load'          (after pipeline/model load resolves)
+ *   - 'warmup'        (after the first forward pass)
+ *   - 'steady'        (sampled steady-state per-query)
+ *   - 'post-idle'     (after the idle dispose fires)
+ *
+ * No behaviour change when nobody reads the JSONL — this is telemetry only.
+ */
+
+import { appendFile, mkdir } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { resolvePluginData } from '../../shared/plugin-paths.mjs';
+
+const DATA_DIR = resolvePluginData();
+
+const PROFILE_PATH = join(DATA_DIR, 'history', 'model-profile.jsonl');
+
+let _mkdirPromise = null;
+async function ensureDir() {
+  if (!_mkdirPromise) {
+    _mkdirPromise = mkdir(dirname(PROFILE_PATH), { recursive: true }).catch(() => {});
+  }
+  return _mkdirPromise;
+}
+
+/**
+ * Collect the current RSS + CPU snapshot plus caller metadata. Fire-and-forget
+ * writer; errors are swallowed. Callers should pass `wallMs` measured with
+ * `performance.now()` around the operation being profiled.
+ *
+ * @param {object} record
+ * @param {string} record.phase     — 'baseline' | 'load' | 'warmup' | 'steady' | 'post-idle' | ...
+ * @param {string} record.model     — model id (e.g. 'Xenova/bge-m3')
+ * @param {string} [record.device]  — 'cpu' | 'dml' | 'webgpu' | …
+ * @param {string} [record.dtype]
+ * @param {number} [record.wallMs]
+ * @param {number} [record.docsScored]
+ * @param {string} [record.note]
+ */
+export function writeProfilePoint(record) {
+  try {
+    const mem = process.memoryUsage();
+    const cpu = process.resourceUsage();
+    const line = JSON.stringify({
+      ts: new Date().toISOString(),
+      phase: record.phase,
+      model: record.model,
+      device: record.device ?? null,
+      dtype: record.dtype ?? null,
+      rssBytes: mem.rss,
+      externalBytes: mem.external,
+      heapUsedBytes: mem.heapUsed,
+      arrayBufferBytes: mem.arrayBuffers,
+      cpuUserUs: cpu.userCPUTime,
+      cpuSystemUs: cpu.systemCPUTime,
+      wallMs: typeof record.wallMs === 'number' ? record.wallMs : null,
+      docsScored: typeof record.docsScored === 'number' ? record.docsScored : undefined,
+      note: record.note ?? undefined,
+    });
+    // Fire-and-forget: any error is swallowed so instrumentation never
+    // breaks a live call path.
+    ensureDir()
+      .then(() => appendFile(PROFILE_PATH, line + '\n'))
+      .catch(() => {});
+  } catch {
+    // never throw from instrumentation
+  }
+}

package/src/memory/lib/pg/adapter.mjs

@@ -0,0 +1,308 @@
+// pg-adapter.mjs — PG connection manager for mixdog 0.4.0
+// Single owner: supervisor-pg.ensurePgInstance(dataDir) starts PG.
+// pg-adapter calls supervisor-pg — never pg-process directly.
+//
+// Public API:
+//   ensurePgInstance(dataDir, { schema? }) → { db, pool, host, port, runtimeDir, pgdataDir }
+//   closePgInstance(dataDir)               → void
+//
+// The returned `db` exposes the native PG query surface:
+//   db.query(sql, params?)          → { rows, rowCount }
+//   db.exec(sql)                    → multi-statement; resolves on completion
+//   db.transaction(async tx => …)  → auto BEGIN/COMMIT, ROLLBACK on throw
+
+import { resolve } from 'path'
+import { ensurePgInstance as supervisorEnsure } from './supervisor.mjs'
+
+// ---------------------------------------------------------------------------
+// One-shot bootstrap guard — keyed by resolved dataDir (cluster-level, not schema)
+// ---------------------------------------------------------------------------
+
+const _bootstrapped = new Set()
+
+// ---------------------------------------------------------------------------
+// Cross-process advisory-lock keys (two-int4 form: classid + objid).
+//
+// pg advisory locks are CLUSTER-global regardless of which database the session
+// is connected to, so a single shared classid namespaces all mixdog first-boot
+// locks; distinct objids separate the CREATE DATABASE race from the per-cluster
+// extension/schema DDL race. Fixed app-specific keys → concurrent first-boot
+// workers serialize on the exact same lock rather than racing the DDL path.
+// ---------------------------------------------------------------------------
+
+const ADVISORY_LOCK_CLASSID = 0x6d696478 // "midx" — mixdog bootstrap namespace
+const ADVISORY_OBJID_CREATE_DB = 1       // CREATE DATABASE mixdog
+export const ADVISORY_OBJID_SCHEMA_BOOTSTRAP = 2 // extensions + schema DDL
+
+// ---------------------------------------------------------------------------
+// In-process cache
+// ---------------------------------------------------------------------------
+
+const instances = new Map() // `${dataDir}|${schema}` → instance handle
+const opening   = new Map() // same key → Promise (dedupe concurrent calls)
+
+// ---------------------------------------------------------------------------
+// Per-connection init — WeakSet-guarded so settings run exactly once per client
+// ---------------------------------------------------------------------------
+
+const _clientInited = new WeakSet()
+
+async function _initClient(client, schema) {
+  if (_clientInited.has(client)) return
+  // Set search_path so unqualified table names resolve to the correct schema.
+  //
+  // CROSS-SCHEMA QUERY RULE: search_path is per-connection and biases every
+  // unqualified table reference toward the connection's primary schema. Any
+  // query that intentionally touches the OTHER schema (e.g. orphan sweep on
+  // memory.entries from a trace-schema connection, or recall × trace JOIN
+  // analytics) MUST use fully-qualified names (memory.entries / trace.trace_events).
+  // Relying on search_path silently in cross-schema code = bug magnet.
+  const sp = schema === 'trace' ? 'trace, public' : 'memory, public'
+  await client.query(`SET search_path = ${sp}`)
+  // pg_trgm similarity threshold: session-local, must be set per connection.
+  await client.query(`SELECT set_limit(0.10)`)
+  await client.query(`SET default_transaction_isolation TO 'read committed'`)
+  // Mark seen only after all init statements succeed; failure leaves client
+  // unmarked so the next checkout retries init.
+  _clientInited.add(client)
+}
+
+// Wrapper around pool.connect() that runs per-client init before returning.
+// Exported so external callers (e.g. trace-store.insertBridgeCalls) can obtain
+// a connection with search_path already set; raw _pool.connect() leaves it at
+// the PG default and unqualified table lookups resolve in the wrong schema.
+export async function checkedConnect(pgPool, schema) {
+  const client = await pgPool.connect()
+  try {
+    await _initClient(client, schema)
+  } catch (e) {
+    client.release()
+    throw e
+  }
+  return client
+}
+const _checkedConnect = checkedConnect
+
+// ---------------------------------------------------------------------------
+// native PG db shim
+// ---------------------------------------------------------------------------
+
+function makeCompatDb(pgPool, schema) {
+  const db = {
+    // query: use pool directly for single-statement queries
+    query: async (sql, params) => {
+      const client = await _checkedConnect(pgPool, schema)
+      try {
+        return await client.query(sql, params)
+      } finally {
+        client.release()
+      }
+    },
+
+    // exec: multi-statement SQL (semicolon-separated); single client for session state
+    exec: async (sql) => {
+      const client = await _checkedConnect(pgPool, schema)
+      try {
+        await client.query(sql)
+      } finally {
+        client.release()
+      }
+    },
+
+    // transaction: check out one client, BEGIN, run callback(tx), COMMIT or ROLLBACK
+    transaction: async (fn) => {
+      const client = await _checkedConnect(pgPool, schema)
+      try {
+        await client.query('BEGIN')
+        const tx = {
+          query: (sql, params) => client.query(sql, params),
+          exec:  (sql)         => client.query(sql),
+        }
+        const result = await fn(tx)
+        await client.query('COMMIT')
+        return result
+      } catch (err) {
+        try { await client.query('ROLLBACK') } catch {}
+        throw err
+      } finally {
+        client.release()
+      }
+    },
+
+    // close: drain pool
+    close: () => pgPool.end(),
+
+    // Internal access for callers that need raw pool
+    _pool: pgPool,
+  }
+  return db
+}
+
+// ---------------------------------------------------------------------------
+// Instance bootstrap — extensions + schemas (idempotent)
+// ---------------------------------------------------------------------------
+
+// Run `fn` while holding the cluster-global schema-bootstrap advisory lock on a
+// dedicated client. Concurrent first-boot workers across processes serialize on
+// the fixed key so the CREATE TYPE / CREATE EXTENSION / schema DDL path is never
+// raced. The lock is session-scoped (held by this client) and released in
+// finally; `fn`'s own DDL may run on other pool connections — the lock only has
+// to provide cross-process mutual exclusion, not cover the same session.
+export async function withSchemaBootstrapLock(pgPool, fn) {
+  const client = await pgPool.connect()
+  // Tracks whether the advisory lock is provably released. Stays false until a
+  // successful pg_advisory_unlock; any failure/uncertainty keeps it false so we
+  // never return a still-locked session to the pool.
+  let unlockErr = new Error('schema-bootstrap advisory lock release not attempted')
+  try {
+    await client.query(`SELECT pg_advisory_lock($1, $2)`, [
+      ADVISORY_LOCK_CLASSID, ADVISORY_OBJID_SCHEMA_BOOTSTRAP,
+    ])
+    try {
+      return await fn()
+    } finally {
+      try {
+        await client.query(`SELECT pg_advisory_unlock($1, $2)`, [
+          ADVISORY_LOCK_CLASSID, ADVISORY_OBJID_SCHEMA_BOOTSTRAP,
+        ])
+        unlockErr = null // lock provably released → client is clean to reuse
+      } catch (err) {
+        unlockErr = err instanceof Error ? err : new Error(String(err))
+      }
+    }
+  } finally {
+    // A client whose advisory-lock state is uncertain must never be reused:
+    // release(truthy) makes node-pg destroy it instead of pooling it.
+    client.release(unlockErr || undefined)
+  }
+}
+
+async function bootstrapInstance(pgPool, dataDirKey) {
+  if (_bootstrapped.has(dataDirKey)) return
+  // Serialize the cluster-level DDL across concurrent first-boot processes.
+  await withSchemaBootstrapLock(pgPool, async () => {
+    // Use a raw client bypassing per-client schema settings (bootstrap targets
+    // the cluster level, not a specific schema).
+    const client = await pgPool.connect()
+    try {
+      await client.query(`CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public`)
+      await client.query(`CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public`)
+      await client.query(`CREATE SCHEMA IF NOT EXISTS memory`)
+      await client.query(`CREATE SCHEMA IF NOT EXISTS trace`)
+    } finally {
+      client.release()
+    }
+  })
+  _bootstrapped.add(dataDirKey)
+}
+
+// ---------------------------------------------------------------------------
+// ensurePgInstance — public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Ensure a live PG instance and return a native PG db handle.
+ *
+ * @param {string} dataDir      Plugin data directory.
+ * @param {{ schema?: 'memory' | 'trace' }} [opts]
+ * @returns {Promise<{ db, pool, host, port, runtimeDir, pgdataDir }>}
+ */
+export async function ensurePgInstance(dataDir, opts = {}) {
+  const schema = opts.schema ?? 'memory'
+  const key    = `${resolve(dataDir)}|${schema}`
+
+  if (instances.has(key)) return instances.get(key)
+  if (opening.has(key))   return opening.get(key)
+
+  const promise = (async () => {
+    // 1. Let supervisor-pg own PG startup and health-checking.
+    //    Returns { host, port, runtimeDir, pgdataDir }.
+    const { host, port, runtimeDir, pgdataDir } = await supervisorEnsure(dataDir)
+
+    // 2. Connect via node-postgres; auto-create the mixdog database if absent.
+    const { default: pg } = await import('pg')
+
+    const PG_USER = 'postgres'
+    const PG_DB   = 'mixdog'
+
+    const adminPool = new pg.Pool({
+      host, port, user: PG_USER, database: 'postgres',
+      password: '', max: 1, idleTimeoutMillis: 5_000,
+    })
+    try {
+      // CREATE DATABASE cannot run inside a transaction, so guard the
+      // check-then-create with a session-level advisory lock held on a single
+      // dedicated client. Concurrent first-boot workers block on the same
+      // cluster-global key here and only one issues the CREATE; the rest see
+      // the row present after the holder releases. Released in finally.
+      const adminClient = await adminPool.connect()
+      // Same invariant as withSchemaBootstrapLock: only reuse the client if the
+      // advisory lock is provably released.
+      let unlockErr = new Error('create-db advisory lock release not attempted')
+      try {
+        await adminClient.query(`SELECT pg_advisory_lock($1, $2)`, [
+          ADVISORY_LOCK_CLASSID, ADVISORY_OBJID_CREATE_DB,
+        ])
+        try {
+          const r = await adminClient.query(`SELECT 1 FROM pg_database WHERE datname = $1`, [PG_DB])
+          if (r.rows.length === 0) {
+            await adminClient.query(`CREATE DATABASE ${PG_DB}`)
+            process.stderr.write(`[pg-adapter] created database ${PG_DB}\n`)
+          }
+        } finally {
+          try {
+            await adminClient.query(`SELECT pg_advisory_unlock($1, $2)`, [
+              ADVISORY_LOCK_CLASSID, ADVISORY_OBJID_CREATE_DB,
+            ])
+            unlockErr = null // lock provably released → client is clean to reuse
+          } catch (err) {
+            unlockErr = err instanceof Error ? err : new Error(String(err))
+          }
+        }
+      } finally {
+        // Uncertain advisory-lock state → destroy the client (truthy release arg)
+        // so a still-locked session never re-enters the admin pool.
+        adminClient.release(unlockErr || undefined)
+      }
+    } finally {
+      await adminPool.end()
+    }
+
+    // 3. Production pool.
+    const pgPool = new pg.Pool({
+      host, port, user: PG_USER, database: PG_DB,
+      password: '', max: 5, idleTimeoutMillis: 30_000,
+      connectionTimeoutMillis: 10_000,
+    })
+
+    // 4. Bootstrap extensions + schemas once (idempotent).
+    await bootstrapInstance(pgPool, resolve(dataDir))
+
+    // 5. Build the compat db shim.
+    const db = makeCompatDb(pgPool, schema)
+
+    const result = { db, pool: pgPool, host, port, runtimeDir, pgdataDir }
+    instances.set(key, result)
+    return result
+  })()
+
+  opening.set(key, promise)
+  try {
+    return await promise
+  } finally {
+    opening.delete(key)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// closePgInstance — drain pool
+// ---------------------------------------------------------------------------
+
+export async function closePgInstance(dataDir, opts = {}) {
+  const schema = opts.schema ?? 'memory'
+  const key    = `${resolve(dataDir)}|${schema}`
+  const inst   = instances.get(key)
+  if (!inst) return
+  try { await inst.pool.end() } catch {}
+  instances.delete(key)
+}