mirror-mind 0.0.1 → 0.32.1

package/ts/src/db/decode.ts

@@ -0,0 +1,66 @@
+// Core decoders for on-disk value shapes (originally CV22.DS2.TS2).
+//
+// These two functions turn the exact on-disk shapes the Python core writes
+// (little-endian float32 embedding BLOBs, ISO timestamps) into the values the
+// ranker consumes. They live in the db layer because both the production core
+// (search/ranker.ts) and the parity harness depend on them — the dependency
+// flows downward into the core, never from core into parity (CR006).
+
+/**
+ * Decode a little-endian IEEE-754 float32 BLOB into a `Float32Array`.
+ *
+ * The Python core stores embeddings as `np.float32(...).tobytes()`
+ * (see `memory.intelligence.embeddings.embedding_to_bytes`), i.e. packed
+ * little-endian 4-byte floats. We decode via `DataView` with an explicit
+ * little-endian flag so the result is independent of both the platform byte
+ * order and the byte alignment of the incoming view (a base64/Buffer decode can
+ * hand us a non-4-aligned `byteOffset`, which the `new Float32Array(buffer, …)`
+ * view constructor rejects).
+ *
+ * Trailing bytes that do not complete a 4-byte float are ignored, matching
+ * `np.frombuffer(data, dtype=np.float32)`.
+ */
+export function blobToFloat32(bytes: Uint8Array): Float32Array {
+  const count = Math.floor(bytes.byteLength / 4);
+  const out = new Float32Array(count);
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  for (let i = 0; i < count; i++) {
+    out[i] = view.getFloat32(i * 4, /* littleEndian */ true);
+  }
+  return out;
+}
+
+/**
+ * Encode a vector as a little-endian IEEE-754 float32 BLOB -- the counterpart
+ * to `blobToFloat32` and a byte-for-byte match of Python's
+ * `embedding_to_bytes` (`np.float32(...).tobytes()`). Shared by every writer
+ * that stores a freshly generated embedding (conversation extraction,
+ * cultivation's `merge` action), so the encode step has exactly one
+ * implementation, matching `blobToFloat32`'s single decode implementation.
+ */
+export function embeddingToBytes(values: readonly number[]): Uint8Array {
+  const buffer = Buffer.alloc(values.length * 4);
+  values.forEach((value, index) => {
+    buffer.writeFloatLE(value, index * 4);
+  });
+  return buffer;
+}
+
+/**
+ * Parse a stored ISO timestamp into epoch milliseconds (UTC), or `null`.
+ *
+ * Mirrors `memory.intelligence.search._parse_datetime_utc`: naive strings are
+ * treated as UTC, while Z-suffixed and offset-aware strings are honored as
+ * written. Returns `null` for empty/unparseable input (the Python helper returns
+ * `None`, which the ranker treats as a neutral default).
+ *
+ * Millisecond precision is intentional: JS `Date` is millisecond-resolution, so
+ * the golden generator records epoch-ms references truncated to the same
+ * granularity.
+ */
+export function parseUtcMs(value: string | null | undefined): number | null {
+  if (!value) return null;
+  const hasTz = /[zZ]$|[+-]\d\d:\d\d$/.test(value);
+  const ms = Date.parse(hasTz ? value : `${value}Z`);
+  return Number.isNaN(ms) ? null : ms;
+}

package/ts/src/db/ftsIntegrity.ts

@@ -0,0 +1,41 @@
+// FTS5 integrity assertion for the database seam (CR021).
+//
+// `memories_fts` is an external-content FTS5 table kept in sync by triggers on
+// `memories`. A write that fires those triggers can, in principle, leave the
+// index inconsistent (the external-content "ghost row" failure mode). Nothing
+// in the system ran FTS5's own integrity check; this exposes it so the
+// write-parity harness can grade the FTS side-effect and a health surface can
+// verify it.
+
+import type { WritableDatabase } from "./database.ts";
+
+/** Raised when the FTS5 integrity check reports a corrupt/inconsistent index. */
+export class FtsIntegrityError extends Error {}
+
+const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+function tableExists(db: WritableDatabase, name: string): boolean {
+  return (
+    db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) !==
+    undefined
+  );
+}
+
+/**
+ * Run FTS5's `integrity-check` for `ftsTable`, throwing `FtsIntegrityError` if
+ * the index is inconsistent with its content table. A no-op when the table is
+ * absent, so callers can invoke it unconditionally on any copy. Requires a
+ * writable handle (the check is issued as an FTS5 command INSERT).
+ */
+export function assertFtsIntegrity(db: WritableDatabase, ftsTable = "memories_fts"): void {
+  if (!IDENTIFIER.test(ftsTable)) {
+    throw new FtsIntegrityError(`unsafe FTS table identifier: ${ftsTable}`);
+  }
+  if (!tableExists(db, ftsTable)) return;
+  try {
+    db.prepare(`INSERT INTO ${ftsTable}(${ftsTable}) VALUES('integrity-check')`).run();
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new FtsIntegrityError(`FTS integrity check failed for ${ftsTable}: ${message}`);
+  }
+}

package/ts/src/db/migrateOnOpen.ts

@@ -0,0 +1,214 @@
+// Migrate-on-open — CV22.DS6.US3 (D2/D3).
+//
+// The front door's activation seam for forward migrations on an *existing*
+// database. `bootstrapDatabaseIfMissing` only runs on a *missing* file, so
+// without this seam an existing database never receives a new migration -- the
+// column stays dormant, merely tolerated by `assertSchemaState`.
+//
+// CV22.DS10.TS5 made TypeScript the SOLE custodian (D-025). The seam was built
+// for two engines: Python's MIGRATIONS list stopped at 016, so this applied
+// only the TS-authored tail and DEFERRED whenever a Python-authored migration
+// was pending, rather than let TypeScript silently apply work Python still
+// owned. That deferral was honest while Python existed. Its verdict was not --
+// `runtime migrate` printed `nothing pending`, exit 0, and the updater's
+// migrate stage passed on a database with pending work nothing had applied.
+//
+// There is no second engine to defer to now, so the split is gone and every
+// pending known migration is applied here. What remains as `declined` is
+// structural and cannot be resolved by another engine: a file with no
+// `_migrations` table, or a database carrying migrations this core does not
+// know. Those fail loudly.
+//
+// Migrations, THEN the bootstrap schema (TS5 handoff review, finding B1).
+// Python's `get_connection` ran both on every open, and the second half is the
+// only thing that ever created the objects the schema has and no migration
+// does: `_ext_migrations`, `_ext_bindings` and its index (v0.8.0), and
+// `journey_mutation_receipts` (v0.31.12). Taking over the migrations without
+// the schema step left a v0.7.0 home at "current (17/17)" and failing in
+// Mirror Mode on `no such table: _ext_bindings`. So a slow-path open composes
+// the two exactly as `bootstrapDatabase` does for a new file. Every Python-era
+// database arrives with `017` pending and takes this path once. A database
+// already at every migration but missing those objects gets no repair here;
+// only TS5-era development copies can be in that state.
+//
+// Discipline (see this story's plan):
+//   - D2: called before serving on BOTH read and write opens, but the steady
+//     state is a single `_migrations` read -- no lock, no backup -- so the hot
+//     path of an already-current database pays almost nothing.
+//   - D3: the backup is conditional and owned here (not the per-write
+//     `openLiveWriteDatabase` snapshot), taken only once we have committed to applying a migration.
+//   - Backup-first, then lock, then idempotent `runMigrations`; the pending set
+//     is re-checked under the lock so concurrent openers cannot double-apply or
+//     double-backup (the loser sees nothing pending and no-ops).
+
+import { chmodSync, existsSync, mkdirSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { PROGRAM } from "#util/program.ts";
+
+import { acquireBootstrapLock, type BootstrapLockOptions } from "./bootstrapLock.ts";
+import {
+  type OpenOptions,
+  openDatabaseForBootstrap,
+  openDatabaseReadOnly,
+  snapshotDatabaseTo,
+} from "./database.ts";
+import { runMigrations } from "./migrations.ts";
+import { createSchema } from "./schema.ts";
+import { KNOWN_MIGRATION_IDS } from "./schemaState.ts";
+
+const BACKUP_DIR_NAME = "backups";
+const MIGRATION_BACKUP_FILE_NAME = "frontdoor-pre-migration-backup.db";
+
+/** Where migrate-on-open snapshots the database before applying a migration.
+ * A distinct name from the per-write backup so a migration snapshot never
+ * clobbers (or is clobbered by) a routine pre-write snapshot. */
+export function migrationBackupPathFor(dbPath: string): string {
+  return join(dirname(dbPath), BACKUP_DIR_NAME, MIGRATION_BACKUP_FILE_NAME);
+}
+
+/**
+ * What migrate-on-open did. THREE verdicts, not two (CV22.DS10.TS5, D-025).
+ *
+ * `declined` used to be `deferredToPython`, and it was honest while Python
+ * existed: the work was real, and another engine could do it. What was not
+ * honest was the VERDICT -- `runMigrate` reported `nothing pending` with exit
+ * 0, and the updater read that as a passing stage on a database with pending
+ * work nothing had applied.
+ *
+ * After this story there is nothing to defer to, so a decline is structural:
+ * the file is not a Mirror database, or it carries migrations this core does
+ * not know. Both are real refusals that must fail loudly rather than pass
+ * quietly, on the one command whose job is to leave the database correct after
+ * an update.
+ */
+export type MigrateVerdict = "applied" | "nothing_pending" | "declined";
+
+export interface MigrateOnOpenResult {
+  /** Which of the three things happened. */
+  verdict: MigrateVerdict;
+  /** True only when this call actually applied one or more migrations. */
+  migrated: boolean;
+  /** The migration ids applied (empty unless `migrated`). */
+  appliedIds: readonly string[];
+  /** The pre-migration snapshot path, when a migration was applied. */
+  backupPath?: string;
+  /** Why the work was declined, when it was. Never empty for `declined`. */
+  declinedReason?: string;
+}
+
+export interface MigrateOnOpenOptions extends OpenOptions, BootstrapLockOptions {}
+
+const NOTHING_PENDING: MigrateOnOpenResult = {
+  verdict: "nothing_pending",
+  migrated: false,
+  appliedIds: [],
+};
+
+function declined(reason: string): MigrateOnOpenResult {
+  return { verdict: "declined", migrated: false, appliedIds: [], declinedReason: reason };
+}
+
+/**
+ * Read the applied migration ids. Returns `null` when the database has no
+ * `_migrations` table — i.e. it is not a bootstrapped Mirror database. That is
+ * `assertSchemaState`'s error to raise, not ours: migrate-on-open simply
+ * declines rather than trying to bootstrap an unknown file.
+ */
+function appliedMigrationIds(dbPath: string, options: OpenOptions): Set<string> | null {
+  const db = openDatabaseReadOnly(dbPath, options);
+  try {
+    const rows = db.prepare("SELECT id FROM _migrations").all() as { id: string }[];
+    return new Set(rows.map((row) => row.id));
+  } catch {
+    return null;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * The known migrations this database has not applied, in order.
+ *
+ * No split any more. Until CV22.DS10.TS5 this separated the TS-authored tail
+ * from Python-authored ids and declined the whole thing whenever a Python one
+ * was pending -- correct while Python could still apply them. TypeScript
+ * carries all seventeen and is the only engine left, so every pending known
+ * migration is simply pending.
+ */
+function pendingMigrations(applied: Set<string>): string[] {
+  return KNOWN_MIGRATION_IDS.filter((id) => !applied.has(id));
+}
+
+/** Migrations the database carries that this core does not know about. */
+function unknownMigrations(applied: Set<string>): string[] {
+  const known = new Set(KNOWN_MIGRATION_IDS);
+  return [...applied].filter((id) => !known.has(id)).sort();
+}
+
+function takeBackup(dbPath: string): string {
+  const backupPath = migrationBackupPathFor(dbPath);
+  mkdirSync(dirname(backupPath), { recursive: true, mode: 0o700 });
+  rmSync(backupPath, { force: true });
+  snapshotDatabaseTo(dbPath, backupPath);
+  chmodSync(backupPath, 0o600);
+  return backupPath;
+}
+
+/**
+ * Bring an existing `dbPath` to this core's schema: apply every pending known
+ * migration, then the bootstrap schema, backup first and under the
+ * cross-process bootstrap lock. Cheap and side-effect-free when nothing is
+ * pending, which is every open after the first. Safe to call on every open.
+ * See the module header for the full contract.
+ */
+export function ensureMigratedOnOpen(
+  dbPath: string,
+  options: MigrateOnOpenOptions = {},
+): MigrateOnOpenResult {
+  // A missing file is not this seam's business: `bootstrapDatabaseIfMissing`
+  // owns creation, and `runMigrate` reports the absence with its own message.
+  if (!existsSync(dbPath)) return NOTHING_PENDING;
+
+  // Cheap steady-state pre-check — no lock, no backup.
+  const applied = appliedMigrationIds(dbPath, options);
+  if (applied === null) {
+    return declined("database has no _migrations table — not a bootstrapped Mirror database");
+  }
+  const unknown = unknownMigrations(applied);
+  if (unknown.length > 0) {
+    // The database is AHEAD of this core. Applying anything now would write
+    // with older code against a newer schema; declining is the safe answer,
+    // and saying so is the honest one.
+    return declined(
+      `database carries migrations this core does not know (${unknown.join(", ")}) — ` +
+        `update this Mirror installation: ${PROGRAM} runtime update`,
+    );
+  }
+  if (pendingMigrations(applied).length === 0) return NOTHING_PENDING;
+
+  // Slow path: serialize with concurrent openers and re-decide under the lock so
+  // a race cannot double-apply or double-backup.
+  const lock = acquireBootstrapLock(dbPath, options);
+  try {
+    const appliedNow = appliedMigrationIds(dbPath, options);
+    if (appliedNow === null) {
+      return declined("database has no _migrations table — not a bootstrapped Mirror database");
+    }
+    const pending = pendingMigrations(appliedNow);
+    // Another opener migrated between the pre-check and the lock.
+    if (pending.length === 0) return NOTHING_PENDING;
+
+    const backupPath = takeBackup(dbPath);
+    const db = openDatabaseForBootstrap(dbPath, options);
+    try {
+      runMigrations(db);
+      // The half of Python's open that only the schema performed (B1).
+      createSchema(db);
+    } finally {
+      db.close();
+    }
+    return { verdict: "applied", migrated: true, appliedIds: pending, backupPath };
+  } finally {
+    lock.release();
+  }
+}