memorix 1.8.8 → 1.9.0

package/src/sync/policy.ts

@@ -0,0 +1,37 @@
+import type { Observation } from '../types.js';
+
+export interface SyncEligibility {
+  eligible: boolean;
+  reason?: string;
+}
+
+/**
+ * The first sync contract is deliberately conservative. A remote relay is not
+ * an agent-facing visibility boundary, so only durable project memory crosses
+ * it. Personal, team, candidate, and ephemeral records stay local.
+ */
+export function syncEligibility(observation: Observation, projectId: string): SyncEligibility {
+  if (observation.projectId !== projectId) return { eligible: false, reason: 'other-project' };
+  if ((observation.visibility ?? 'project') !== 'project') return { eligible: false, reason: 'non-project-visibility' };
+  if ((observation.sharedWithAgentIds?.length ?? 0) > 0) {
+    return { eligible: false, reason: 'agent-targeted' };
+  }
+  if (observation.admissionState === 'candidate' || observation.admissionState === 'ephemeral') {
+    return { eligible: false, reason: 'unqualified-observation' };
+  }
+  if (observation.valueCategory === 'ephemeral') return { eligible: false, reason: 'ephemeral-value' };
+  return { eligible: true };
+}
+
+export function eligibleObservations(observations: Observation[], projectId: string): {
+  eligible: Observation[];
+  excluded: number;
+} {
+  const eligible: Observation[] = [];
+  let excluded = 0;
+  for (const observation of observations) {
+    if (syncEligibility(observation, projectId).eligible) eligible.push(observation);
+    else excluded++;
+  }
+  return { eligible, excluded };
+}

package/src/sync/remote-factory.ts

@@ -0,0 +1,49 @@
+/**
+ * Resolves a `SyncRemote` from configuration. Provider choice is data, not
+ * code: the caller picks a provider name and the matching adapter is
+ * lazy-imported so unused providers never load their optional SDKs.
+ *
+ * Configuration is env-driven and opt-in. Sync is OFF unless
+ * `MEMORIX_SYNC_PROVIDER` is set (or an explicit provider is passed).
+ */
+
+import type { SyncRemote } from './types.js';
+
+export type SyncProvider = 'fs' | 'github' | 's3' | 'postgres';
+
+export interface SyncConfig {
+  enabled: boolean;
+  provider?: SyncProvider;
+}
+
+export function resolveSyncConfig(env: NodeJS.ProcessEnv = process.env): SyncConfig {
+  const provider = env.MEMORIX_SYNC_PROVIDER as SyncProvider | undefined;
+  if (!provider) return { enabled: false };
+  if (provider !== 'fs' && provider !== 'github' && provider !== 's3' && provider !== 'postgres') {
+    throw new Error(`[memorix] unknown MEMORIX_SYNC_PROVIDER "${provider}" (expected fs|github|s3|postgres)`);
+  }
+  return { enabled: true, provider };
+}
+
+export async function createRemote(
+  provider: SyncProvider,
+  namespace: string,
+  env: NodeJS.ProcessEnv = process.env,
+): Promise<SyncRemote> {
+  if (provider === 'fs') {
+    const root = env.MEMORIX_SYNC_FS_ROOT;
+    if (!root) throw new Error('[memorix] fs sync requires MEMORIX_SYNC_FS_ROOT');
+    const { FsRemote } = await import('./adapters/fs.js');
+    return new FsRemote({ root, namespace });
+  }
+  if (provider === 'github') {
+    const { createGitHubRemote } = await import('./adapters/github.js');
+    return createGitHubRemote(namespace, env);
+  }
+  if (provider === 's3') {
+    const { createS3ObjectStore, ObjectStoreRemote } = await import('./adapters/object-store.js');
+    return new ObjectStoreRemote(await createS3ObjectStore(env), namespace, env.MEMORIX_SYNC_S3_PREFIX ?? '');
+  }
+  const { createPgSqlClient, PostgresSyncRemote } = await import('./adapters/postgres.js');
+  return new PostgresSyncRemote(await createPgSqlClient(env), namespace);
+}

package/src/sync/store-port.ts

@@ -0,0 +1,288 @@
+/**
+ * Binds the sync engine's `SyncStorePort` to the real Memorix store: the
+ * canonical `ObservationStore` for rows, plus the additive `sync_row_state` /
+ * `sync_meta` tables (created here, never altering `observations`).
+ *
+ * This is the only place the engine touches persistence in production.
+ */
+import os from 'node:os';
+import { randomUUID } from 'node:crypto';
+import type { Observation } from '../types.js';
+import type { ObservationStore } from '../store/obs-store.js';
+import { getDatabase } from '../store/sqlite-db.js';
+import { CREATE_SYNC_TABLES } from './journal.js';
+import { syncNamespace } from './namespace.js';
+import type { SyncStorePort } from './engine.js';
+import type { SyncConflict, SyncCursor, SyncPendingBatch, SyncRowState } from './types.js';
+
+const SEQ_KEY = 'local_sequence';
+const DEVICE_KEY = 'device_id';
+const FINGERPRINT_KEY = 'device_fingerprint';
+const LEGACY_PROJECT_ID = '';
+
+export interface SqliteSyncStore extends SyncStorePort {
+  /** Stable device id for this store (generated once, persisted in sync_meta). */
+  deviceId(): string;
+}
+
+export function createSqliteSyncStore(dataDir: string, store: ObservationStore, projectId: string): SqliteSyncStore {
+  const db = getDatabase(dataDir);
+  ensureSyncSchema(db, projectId);
+  const getMeta = db.prepare('SELECT value FROM sync_meta WHERE key = ?');
+  const setMeta = db.prepare('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)');
+  const selState = db.prepare('SELECT projectId, syncKey, obsId, revision, writer, kind, contentHash, shippedSeq FROM sync_row_state WHERE projectId = ?');
+  const upState = db.prepare(
+    `INSERT INTO sync_row_state (projectId, syncKey, obsId, revision, writer, kind, contentHash, shippedSeq)
+     VALUES (@projectId, @syncKey, @obsId, @revision, @writer, @kind, @contentHash, @shippedSeq)
+     ON CONFLICT(projectId, syncKey) DO UPDATE SET
+       obsId = excluded.obsId,
+       revision = excluded.revision,
+       writer = excluded.writer,
+       kind = excluded.kind,
+       contentHash = excluded.contentHash,
+       shippedSeq = excluded.shippedSeq`,
+  );
+  const insertConflict = db.prepare(
+    `INSERT OR IGNORE INTO sync_conflicts (id, projectId, syncKey, winnerJson, loserJson, createdAt, reason)
+     VALUES (@id, @projectId, @syncKey, @winnerJson, @loserJson, @createdAt, @reason)`,
+  );
+  const selPending = db.prepare(
+    `SELECT batchJson, statesJson FROM sync_outbox
+     WHERE projectId = ? AND namespace = ? AND deviceId = ? ORDER BY sequence ASC`,
+  );
+  const insertPending = db.prepare(
+    `INSERT OR IGNORE INTO sync_outbox
+     (projectId, namespace, deviceId, sequence, batchJson, statesJson, createdAt)
+     VALUES (@projectId, @namespace, @deviceId, @sequence, @batchJson, @statesJson, @createdAt)`,
+  );
+  const deletePending = db.prepare(
+    `DELETE FROM sync_outbox WHERE projectId = ? AND namespace = ? AND deviceId = ? AND sequence = ?`,
+  );
+  const pendingForDevice = db.prepare(
+    `SELECT projectId, namespace, sequence, batchJson
+     FROM sync_outbox WHERE deviceId = ? ORDER BY projectId, namespace, sequence`,
+  );
+  const rewritePendingDevice = db.prepare(
+    `UPDATE sync_outbox
+        SET deviceId = ?, batchJson = ?
+      WHERE projectId = ? AND namespace = ? AND deviceId = ? AND sequence = ?`,
+  );
+
+  const namespace = syncNamespace(projectId);
+  const deviceFingerprint = process.env.MEMORIX_SYNC_DEVICE_FINGERPRINT?.trim()
+    || `${process.platform}|${os.hostname()}|${os.homedir()}`;
+
+  function ensureDevice(): string {
+    const row = getMeta.get(DEVICE_KEY) as { value: string } | undefined;
+    if (row?.value) return row.value;
+    const id = `dev_${randomUUID().replaceAll('-', '')}`;
+    setMeta.run(DEVICE_KEY, id);
+    setMeta.run(FINGERPRINT_KEY, deviceFingerprint);
+    return id;
+  }
+
+  function assertCanPublish(): void {
+    const device = getMeta.get(DEVICE_KEY) as { value: string } | undefined;
+    if (!device?.value) {
+      ensureDevice();
+      return;
+    }
+    const row = getMeta.get(FINGERPRINT_KEY) as { value: string } | undefined;
+    if (!row?.value) throw new Error('[memorix] sync device identity is unknown; run `memorix sync store device rotate` before publishing');
+    if (row?.value && row.value !== deviceFingerprint) {
+      throw new Error(
+        '[memorix] sync device clone detected; run `memorix sync store device rotate` before publishing from this data directory',
+      );
+    }
+  }
+
+  function cursorKey(): string {
+    return `cursor:${namespace}`;
+  }
+
+  return {
+    projectId: () => projectId,
+    namespace: () => namespace,
+    deviceId: ensureDevice,
+    assertCanPublish,
+    rotateDevice: () => {
+      const previous = ensureDevice();
+      const id = `dev_${randomUUID().replaceAll('-', '')}`;
+      const pending = pendingForDevice.all(previous) as Array<{
+        projectId: string;
+        namespace: string;
+        sequence: number;
+        batchJson: string;
+      }>;
+      const rotate = db.transaction(() => {
+        setMeta.run(DEVICE_KEY, id);
+        setMeta.run(FINGERPRINT_KEY, deviceFingerprint);
+        // Pending batches were created before the clone was detected. Keep
+        // them and move their transport identity to the new replica; deleting
+        // them here would silently lose successfully recorded local memory.
+        for (const item of pending) {
+          const batch = JSON.parse(item.batchJson) as { deviceId?: string } & Record<string, unknown>;
+          rewritePendingDevice.run(
+            id,
+            JSON.stringify({ ...batch, deviceId: id }),
+            item.projectId,
+            item.namespace,
+            previous,
+            item.sequence,
+          );
+        }
+      });
+      rotate();
+      return id;
+    },
+
+    async loadCursor(): Promise<SyncCursor> {
+      const row = getMeta.get(cursorKey()) as { value: string } | undefined;
+      if (!row) return { applied: {} };
+      try {
+        const value = JSON.parse(row.value) as SyncCursor;
+        return value && typeof value.applied === 'object' ? value : { applied: {} };
+      } catch {
+        return { applied: {} };
+      }
+    },
+
+    async saveCursor(cursor: SyncCursor): Promise<void> {
+      setMeta.run(cursorKey(), JSON.stringify(cursor));
+    },
+
+    async loadPendingBatches(): Promise<SyncPendingBatch[]> {
+      return (selPending.all(projectId, namespace, ensureDevice()) as Array<{ batchJson: string; statesJson: string }>).map((row) => ({
+        batch: JSON.parse(row.batchJson),
+        states: JSON.parse(row.statesJson),
+      } as SyncPendingBatch));
+    },
+
+    async enqueueBatch(pending: SyncPendingBatch): Promise<void> {
+      insertPending.run({
+        projectId,
+        namespace,
+        deviceId: pending.batch.deviceId,
+        sequence: pending.batch.sequence,
+        batchJson: JSON.stringify(pending.batch),
+        statesJson: JSON.stringify(pending.states),
+        createdAt: pending.batch.producedAt,
+      });
+    },
+
+    async markBatchShipped(sequence: number): Promise<void> {
+      deletePending.run(projectId, namespace, ensureDevice(), sequence);
+    },
+
+    async loadAll(): Promise<Observation[]> {
+      return store.loadByProject(projectId);
+    },
+
+    async loadState(): Promise<Map<string, SyncRowState>> {
+      const rows = selState.all(projectId) as SyncRowState[];
+      const map = new Map<string, SyncRowState>();
+      for (const r of rows) map.set(r.syncKey, r);
+      return map;
+    },
+
+    async saveState(rows: SyncRowState[]): Promise<void> {
+      const tx = db.transaction((batch: SyncRowState[]) => {
+        for (const r of batch) upState.run(r);
+      });
+      tx(rows);
+    },
+
+    async allocateId(): Promise<number> {
+      return store.atomic(async (tx) => {
+        const nextId = await tx.loadIdCounter();
+        await tx.saveIdCounter(nextId + 1);
+        return nextId;
+      });
+    },
+
+    async applyInsert(row: Observation, state: SyncRowState): Promise<void> {
+      await store.atomic(async (tx) => {
+        const existing = await tx.getById(row.id);
+        if (existing) {
+          if (existing.projectId !== row.projectId) {
+            throw new Error('[memorix] sync refused to overwrite an observation from another project');
+          }
+          throw new Error('[memorix] sync insert collided with an existing local observation id');
+        }
+        await tx.insert(row);
+        upState.run(state);
+      });
+    },
+
+    async applyUpdate(row: Observation, state: SyncRowState): Promise<void> {
+      await store.atomic(async (tx) => {
+        const existing = await tx.getById(row.id);
+        if (existing && existing.projectId !== row.projectId) {
+          throw new Error('[memorix] sync refused to overwrite an observation from another project');
+        }
+        if (existing) await tx.update(row);
+        else await tx.insert(row);
+        upState.run(state);
+      });
+    },
+
+    async applyRemove(id: number, state: SyncRowState): Promise<void> {
+      await store.atomic(async (tx) => {
+        const existing = await tx.getById(id);
+        if (existing && existing.projectId !== state.projectId) {
+          throw new Error('[memorix] sync refused to remove an observation from another project');
+        }
+        await tx.remove(id);
+        upState.run(state);
+      });
+    },
+
+    async getById(id: number): Promise<Observation | undefined> {
+      return store.getById(id);
+    },
+
+    async recordConflict(conflict: SyncConflict): Promise<void> {
+      insertConflict.run({
+        id: conflict.id,
+        projectId: conflict.projectId,
+        syncKey: conflict.syncKey,
+        winnerJson: JSON.stringify(conflict.winner),
+        loserJson: JSON.stringify({ version: conflict.loser, kind: conflict.loserKind, row: conflict.loserRow }),
+        createdAt: conflict.createdAt,
+        reason: conflict.reason,
+      });
+    },
+
+    async nextSequence(): Promise<number> {
+      const row = getMeta.get(SEQ_KEY) as { value: string } | undefined;
+      const next = (row ? parseInt(row.value, 10) : 0) + 1;
+      setMeta.run(SEQ_KEY, String(next));
+      return next;
+    },
+  };
+}
+
+function ensureSyncSchema(db: any, projectId: string): void {
+  db.exec(CREATE_SYNC_TABLES);
+  const columns = db.prepare('PRAGMA table_info(sync_row_state)').all() as Array<{ name: string }>;
+  if (!columns.some((column) => column.name === 'projectId')) {
+    // #277 created this table with syncKey as the only primary key. Adding a
+    // nullable-by-default project column keeps old stores readable while the
+    // composite unique index enables the scoped v3 writer.
+    db.exec("ALTER TABLE sync_row_state ADD COLUMN projectId TEXT NOT NULL DEFAULT ''");
+  }
+  db.exec('CREATE UNIQUE INDEX IF NOT EXISTS sync_row_state_project_key ON sync_row_state(projectId, syncKey)');
+  db.exec('CREATE INDEX IF NOT EXISTS sync_row_state_project_idx ON sync_row_state(projectId)');
+  db.exec('CREATE INDEX IF NOT EXISTS sync_outbox_project_idx ON sync_outbox(projectId, namespace, deviceId, sequence)');
+
+  // Attach legacy live rows to the project they already belong to. Legacy
+  // tombstones have no surviving observation from which a project can be
+  // inferred, so they remain quarantined under the empty project ID.
+  const legacy = db.prepare(
+    `SELECT s.syncKey, s.obsId FROM sync_row_state s
+     JOIN observations o ON o.id = s.obsId
+     WHERE s.projectId = '' AND o.projectId = ?`,
+  ).all(projectId) as Array<{ syncKey: string; obsId: number }>;
+  const update = db.prepare('UPDATE sync_row_state SET projectId = ? WHERE syncKey = ? AND projectId = ?');
+  for (const row of legacy) update.run(projectId, row.syncKey, LEGACY_PROJECT_ID);
+}

package/src/sync/types.ts

@@ -0,0 +1,183 @@
+/**
+ * Multi-device store sync — shared contracts.
+ *
+ * One feature, interchangeable remotes. The merge engine never talks to a
+ * vendor; it talks to `SyncRemote`. A remote can be a local directory, an
+ * S3-compatible object store, or a Postgres hub — chosen by configuration.
+ *
+ * Local-first is preserved: the canonical store stays local SQLite on every
+ * machine. Sync is opt-in, asynchronous, and off the hot path.
+ *
+ * Identity & versioning (why this converges):
+ *   - Rows are merged by a stable `syncKey`, not by the replica-local integer
+ *     id. Two machines that independently insert new rows never collide, and a
+ *     topic-keyed observation carries the same key everywhere.
+ *   - A row's version is a per-row logical clock `(revision, writer)`. Merge is
+ *     a single total order over that tuple, so replaying batches in any order
+ *     reaches the same final state (last-writer-wins, convergent).
+ */
+import type { Observation } from '../types.js';
+
+/** Kind of change carried in the journal. */
+export type ChangeKind = 'upsert' | 'tombstone';
+
+/**
+ * A per-row logical version. Larger wins under last-writer-wins. Comparison is
+ * lexicographic over `(revision, writer)` and is a strict total order, which is
+ * what makes replay convergent regardless of delivery order.
+ */
+export interface RowVersion {
+  /** Per-row Lamport counter: max revision observed for this row + 1. */
+  revision: number;
+  /** Stable id of the device that produced this version (deterministic tiebreak). */
+  writer: string;
+  /** Informational ISO timestamp; never used in comparison. */
+  at?: string;
+}
+
+/** One replicated change for a single logical row, addressed by `syncKey`. */
+export interface ChangeEntry {
+  /** Stable cross-device identity of the row (see module doc). */
+  syncKey: string;
+  kind: ChangeKind;
+  version: RowVersion;
+  /** SHA-256 of the canonical row body; empty for tombstones. */
+  contentHash: string;
+  /** Present when kind === 'upsert'; omitted for tombstones. */
+  row?: Observation;
+}
+
+/** A batch of changes shipped in one direction. */
+export interface ChangeBatch {
+  /** Schema version of the batch envelope. */
+  formatVersion: 3;
+  /** Hash-like remote namespace for one canonical Git project. */
+  namespace: string;
+  /** Canonical Memorix project ID. */
+  projectId: string;
+  /** Device that produced the batch. */
+  deviceId: string;
+  /** Monotonic sequence for this device; lets remotes dedupe/order. */
+  sequence: number;
+  /** ISO timestamp the batch was produced. */
+  producedAt: string;
+  entries: ChangeEntry[];
+}
+
+/**
+ * Authoritative local record of the version we currently hold for a row,
+ * keyed by `syncKey`. It serves double duty: change-detection basis for push
+ * and the merge basis for pull, so a locally applied remote row is never
+ * re-shipped in a loop. Persisted in the additive `sync_row_state` table.
+ */
+export interface SyncRowState {
+  /** Canonical project that owns this row. */
+  projectId: string;
+  syncKey: string;
+  /** Local integer id of the row, or null once tombstoned. */
+  obsId: number | null;
+  revision: number;
+  writer: string;
+  kind: ChangeKind;
+  /** Content fingerprint of the row body (excludes replica-local fields). */
+  contentHash: string;
+  /** Sequence of the batch that last shipped this state locally. */
+  shippedSeq: number;
+}
+
+/**
+ * Opaque per-remote cursor. Maps deviceId -> last applied sequence, so both
+ * sides know what the other has already seen. Remotes persist and return it.
+ */
+export interface SyncCursor {
+  /** deviceId -> highest applied sequence from that device. */
+  applied: Record<string, number>;
+}
+
+export function emptyCursor(): SyncCursor {
+  return { applied: {} };
+}
+
+/**
+ * The only surface the engine uses to reach a backend. Every adapter (fs,
+ * object-store, postgres, ...) implements exactly this. No adapter is
+ * privileged; provider choice is configuration, not code.
+ */
+export interface SyncRemote {
+  /** Namespace this remote instance is allowed to read and write. */
+  readonly namespace: string;
+  /** Human-readable id for logs and status (e.g. "fs", "s3", "postgres"). */
+  readonly kind: string;
+  /** Open connections / ensure containers exist. Safe to call repeatedly. */
+  init(options?: { create?: boolean }): Promise<void>;
+  /** Upload one batch of local changes. Idempotent by (deviceId, sequence). */
+  push(batch: ChangeBatch): Promise<void>;
+  /**
+   * Download batches this device has not applied yet. `since` maps
+   * deviceId -> highest sequence already applied locally; the remote returns
+   * only newer batches, oldest first.
+   */
+  pull(since: Record<string, number>, limit: number, pageToken?: string): Promise<SyncPullPage>;
+  /** Explicitly remove events that every operator-supplied device cursor passed. */
+  compact(through: Record<string, number>, options?: { dryRun?: boolean }): Promise<SyncCompactReport>;
+  /** Release connections / flush. Safe to call when never initialized. */
+  close(): Promise<void>;
+}
+
+export interface SyncCompactReport {
+  candidates: number;
+  deleted: number;
+}
+
+export interface SyncPullPage {
+  batches: ChangeBatch[];
+  hasMore: boolean;
+  /** Opaque provider page token for the next call in the same run. */
+  nextPageToken?: string;
+}
+
+export interface SyncPendingBatch {
+  batch: ChangeBatch;
+  states: SyncRowState[];
+}
+
+export interface SyncConflict {
+  id: string;
+  projectId: string;
+  syncKey: string;
+  winner: RowVersion;
+  loser: RowVersion;
+  loserKind: ChangeKind;
+  loserRow?: Observation;
+  createdAt: string;
+  reason: string;
+}
+
+/** Result of a push/pull/status run, for CLI reporting. */
+export interface SyncReport {
+  remote: string;
+  namespace: string;
+  projectId: string;
+  deviceId: string;
+  pushed: number;
+  pulledBatches: number;
+  applied: number;
+  skipped: number;
+  tombstoned: number;
+  eligible: number;
+  excluded: number;
+  conflicts: number;
+  pending: number;
+  dryRun: boolean;
+  /** Per-change decisions, populated on dry runs and verbose runs. */
+  decisions: SyncDecision[];
+}
+
+export type SyncOutcome = 'apply' | 'skip-older' | 'skip-equal';
+
+export interface SyncDecision {
+  syncKey: string;
+  kind: ChangeKind;
+  outcome: SyncOutcome;
+  reason: string;
+}