memorix 1.9.3 → 1.9.5

package/src/store/sqlite-store.ts

@@ -28,6 +28,7 @@ import {
 import type { LexicalSearchHit } from './obs-store.js';
 import path from 'node:path';
 import fs from 'node:fs';
+import { withFileLock } from './file-lock.js';
 
 // ── Row ↔ Observation serialization ────────────────────────────────
 
@@ -294,49 +295,53 @@ export class SqliteBackend implements ObservationStore {
   // ── Migration ────────────────────────────────────────────────────
 
   private async migrateFromJsonIfNeeded(): Promise<void> {
-    // Only migrate if table is empty
-    const count = this.db.prepare(`SELECT COUNT(*) AS cnt FROM observations`).get();
-    if (count.cnt > 0) return;
-
     const jsonPath = path.join(this.dataDir, 'observations.json');
     if (!fs.existsSync(jsonPath)) return;
 
-    try {
-      const raw = fs.readFileSync(jsonPath, 'utf-8');
-      const observations: Observation[] = JSON.parse(raw);
+    await withFileLock(this.dataDir, async () => {
+      // Re-check under the cross-process lock. Another initializer may have
+      // completed the migration while this process was waiting.
+      const count = this.db.prepare(`SELECT COUNT(*) AS cnt FROM observations`).get();
+      if (count.cnt > 0) return;
 
+      const raw = fs.readFileSync(jsonPath, 'utf-8');
+      const parsed: unknown = JSON.parse(raw);
+      if (!Array.isArray(parsed)) {
+        throw new Error('[memorix] observations.json migration refused: expected a JSON array');
+      }
+      const observations = parsed as Observation[];
       if (observations.length === 0) return;
+      if (observations.some((obs) => !obs || !Number.isSafeInteger(obs.id) || obs.id < 1 || typeof obs.projectId !== 'string')) {
+        throw new Error('[memorix] observations.json migration refused: invalid observation record');
+      }
 
       console.error(`[memorix] Migrating ${observations.length} observations from JSON to SQLite...`);
 
+      // The SQLite transaction makes the migration all-or-nothing. The JSON
+      // source remains intact until an operator explicitly removes it.
       const insertMany = this.db.transaction((obsList: Observation[]) => {
         for (const obs of obsList) {
           this.stmtInsert.run(obsToRow(obs));
         }
-      });
-      insertMany(observations);
 
-      // Migrate counter
-      const counterPath = path.join(this.dataDir, 'counter.json');
-      if (fs.existsSync(counterPath)) {
-        try {
-          const counterData = JSON.parse(fs.readFileSync(counterPath, 'utf-8'));
-          const nextId = counterData.nextId ?? counterData.next_id ?? (Math.max(...observations.map(o => o.id)) + 1);
-          this.rawSaveIdCounter(nextId);
-        } catch {
-          // Fallback: derive from max observation ID
-          this.rawSaveIdCounter(Math.max(...observations.map(o => o.id)) + 1);
+        const counterPath = path.join(this.dataDir, 'counter.json');
+        let counterNextId: number | undefined;
+        if (fs.existsSync(counterPath)) {
+          const counterData = JSON.parse(fs.readFileSync(counterPath, 'utf-8')) as { nextId?: unknown; next_id?: unknown };
+          const candidate = counterData.nextId ?? counterData.next_id;
+          if (candidate !== undefined && (!Number.isSafeInteger(candidate) || Number(candidate) < 1)) {
+            throw new Error('[memorix] counter.json migration refused: nextId must be a positive integer');
+          }
+          counterNextId = candidate === undefined ? undefined : Number(candidate);
         }
-      } else {
-        this.rawSaveIdCounter(Math.max(...observations.map(o => o.id)) + 1);
-      }
-
-      this.bumpGeneration();
+        const maxId = Math.max(...obsList.map((obs) => obs.id));
+        this.rawSaveIdCounter(Math.max(counterNextId ?? 1, maxId + 1));
+        this.bumpGeneration();
+      });
+      insertMany(observations);
 
       console.error(`[memorix] Migration complete. ${observations.length} observations now in SQLite.`);
-    } catch (err) {
-      console.error(`[memorix] JSON→SQLite migration failed (non-fatal, data preserved in JSON): ${err}`);
-    }
+    });
   }
 
   // ── Public read ──────────────────────────────────────────────────

package/src/sync/engine.ts

@@ -9,7 +9,7 @@
 import type { Observation } from '../types.js';
 import { computeChanges, contentHash, nowIso } from './journal.js';
 import { compareVersion, mergeOne } from './merge.js';
-import { eligibleObservations } from './policy.js';
+import { eligibleObservations, type SyncScope } from './policy.js';
 import { isSafeSyncDeviceId } from './namespace.js';
 import type {
   ChangeBatch,
@@ -29,6 +29,7 @@ import { createHash } from 'node:crypto';
  */
 export interface SyncStorePort {
   projectId(): string;
+  scope(): SyncScope;
   namespace(): string;
   /** Stable device id for this replica. */
   deviceId(): string;
@@ -83,6 +84,7 @@ export async function runSync(
     remote: remote.kind,
     namespace: remote.namespace,
     projectId: store.projectId(),
+    scope: store.scope(),
     deviceId: opts.deviceId,
     pushed: 0,
     pulledBatches: 0,
@@ -116,7 +118,7 @@ export async function runSync(
       report.pending = pending.length;
       if (!opts.dryRun) {
         for (const pendingBatch of pending) {
-          validateBatches([pendingBatch.batch], store.projectId(), remote.namespace);
+          validateBatches([pendingBatch.batch], store.projectId(), remote.namespace, store.scope());
           if (pendingBatch.batch.deviceId !== opts.deviceId) {
             throw new Error('[memorix] sync outbox rejected: batch belongs to another device');
           }
@@ -128,7 +130,7 @@ export async function runSync(
         report.pending = 0;
       }
       const current = await store.loadAll();
-      const filtered = eligibleObservations(current, store.projectId());
+      const filtered = eligibleObservations(current, store.projectId(), store.scope());
       report.eligible = filtered.eligible.length;
       report.excluded = filtered.excluded;
       const state = await store.loadState();
@@ -151,7 +153,7 @@ export async function runSync(
             producedAt: nowIso(),
             entries: chunk.map((c) => c.entry),
           };
-          if (!opts.dryRun) validateBatches([batch], store.projectId(), remote.namespace);
+          if (!opts.dryRun) validateBatches([batch], store.projectId(), remote.namespace, store.scope());
           report.pushed += chunk.length;
           if (!opts.dryRun) {
             const states = chunk.map((c) => ({ ...c.nextState, shippedSeq: sequence }));
@@ -177,7 +179,7 @@ export async function runSync(
         // can skip later rows after page one.
         const page = await remote.pull(pullSince, PULL_PAGE_SIZE, pageToken);
         report.pulledBatches += page.batches.length;
-        validateBatches(page.batches, store.projectId(), remote.namespace);
+        validateBatches(page.batches, store.projectId(), remote.namespace, store.scope());
 
         for (const batch of page.batches) {
           if (batch.deviceId === opts.deviceId) {
@@ -254,7 +256,7 @@ export async function runSync(
   }
 }
 
-function validateBatches(batches: ChangeBatch[], projectId: string, namespace: string): void {
+function validateBatches(batches: ChangeBatch[], projectId: string, namespace: string, scope: SyncScope = 'project'): void {
   for (const batch of batches) {
     if (batch.formatVersion !== 3 || batch.projectId !== projectId || batch.namespace !== namespace) {
       throw new Error('[memorix] sync batch rejected: wrong format, project, or remote namespace');
@@ -284,7 +286,7 @@ function validateBatches(batches: ChangeBatch[], projectId: string, namespace: s
       }
       if (
         entry.kind === 'upsert'
-        && (!Number.isSafeInteger(entry.row!.id) || entry.row!.id < 1 || !eligibleObservations([entry.row!], projectId).eligible.length)
+        && (!Number.isSafeInteger(entry.row!.id) || entry.row!.id < 1 || !eligibleObservations([entry.row!], projectId, scope).eligible.length)
       ) {
         throw new Error('[memorix] sync batch rejected: ineligible observation payload');
       }

package/src/sync/namespace.ts

@@ -1,10 +1,33 @@
 import { createHash } from 'node:crypto';
 
+/** Sentinel project id for user-global store sync envelopes and local state. */
+export const USER_SYNC_SCOPE_ID = '__user__';
+
+/** Prefix used for explicit user-global remote namespaces. */
+export const USER_SYNC_NAMESPACE = 'user-';
+
 /** Stable, path-safe namespace for one canonical Git project. */
 export function syncNamespace(projectId: string): string {
   return `project-${createHash('sha256').update(projectId).digest('hex').slice(0, 32)}`;
 }
 
+/**
+ * Remote namespace for `--scope user`. The operator must choose a stable
+ * namespace and reuse it on the same user's devices. Requiring the value is
+ * intentional: a fixed default would allow two operators sharing a relay to
+ * exchange otherwise project-visible memories.
+ */
+export function userSyncNamespace(env: NodeJS.ProcessEnv = process.env): string {
+  const override = env.MEMORIX_SYNC_USER_NAMESPACE?.trim();
+  if (!override) {
+    throw new Error('[memorix] MEMORIX_SYNC_USER_NAMESPACE is required for --scope user; use one stable per-user namespace across devices');
+  }
+  if (!isSafeSyncDeviceId(override)) {
+    throw new Error('[memorix] MEMORIX_SYNC_USER_NAMESPACE must match [A-Za-z0-9][A-Za-z0-9._-]{0,127}');
+  }
+  return `${USER_SYNC_NAMESPACE}${override}`;
+}
+
 export function projectSyncPrefix(projectId: string): string {
   return `p:${encodeURIComponent(projectId)}:`;
 }

package/src/sync/policy.ts

@@ -1,5 +1,7 @@
 import type { Observation } from '../types.js';
 
+export type SyncScope = 'project' | 'user';
+
 export interface SyncEligibility {
   eligible: boolean;
   reason?: string;
@@ -9,9 +11,18 @@ export interface SyncEligibility {
  * The first sync contract is deliberately conservative. A remote relay is not
  * an agent-facing visibility boundary, so only durable project memory crosses
  * it. Personal, team, candidate, and ephemeral records stay local.
+ *
+ * `--scope project` (default) also drops other-project rows. `--scope user`
+ * keeps those rows so one local SQLite can replicate every project.
  */
-export function syncEligibility(observation: Observation, projectId: string): SyncEligibility {
-  if (observation.projectId !== projectId) return { eligible: false, reason: 'other-project' };
+export function syncEligibility(
+  observation: Observation,
+  scopeId: string,
+  scope: SyncScope = 'project',
+): SyncEligibility {
+  if (scope === 'project' && observation.projectId !== scopeId) {
+    return { eligible: false, reason: 'other-project' };
+  }
   if ((observation.visibility ?? 'project') !== 'project') return { eligible: false, reason: 'non-project-visibility' };
   if ((observation.sharedWithAgentIds?.length ?? 0) > 0) {
     return { eligible: false, reason: 'agent-targeted' };
@@ -23,14 +34,18 @@ export function syncEligibility(observation: Observation, projectId: string): Sy
   return { eligible: true };
 }
 
-export function eligibleObservations(observations: Observation[], projectId: string): {
+export function eligibleObservations(
+  observations: Observation[],
+  scopeId: string,
+  scope: SyncScope = 'project',
+): {
   eligible: Observation[];
   excluded: number;
 } {
   const eligible: Observation[] = [];
   let excluded = 0;
   for (const observation of observations) {
-    if (syncEligibility(observation, projectId).eligible) eligible.push(observation);
+    if (syncEligibility(observation, scopeId, scope).eligible) eligible.push(observation);
     else excluded++;
   }
   return { eligible, excluded };

package/src/sync/store-port.ts

@@ -11,9 +11,10 @@ import type { Observation } from '../types.js';
 import type { ObservationStore } from '../store/obs-store.js';
 import { getDatabase } from '../store/sqlite-db.js';
 import { CREATE_SYNC_TABLES } from './journal.js';
-import { syncNamespace } from './namespace.js';
+import { syncNamespace, userSyncNamespace, USER_SYNC_SCOPE_ID } from './namespace.js';
 import type { SyncStorePort } from './engine.js';
 import type { SyncConflict, SyncCursor, SyncPendingBatch, SyncRowState } from './types.js';
+import type { SyncScope } from './policy.js';
 
 const SEQ_KEY = 'local_sequence';
 const DEVICE_KEY = 'device_id';
@@ -25,9 +26,16 @@ export interface SqliteSyncStore extends SyncStorePort {
   deviceId(): string;
 }
 
-export function createSqliteSyncStore(dataDir: string, store: ObservationStore, projectId: string): SqliteSyncStore {
+export function createSqliteSyncStore(
+  dataDir: string,
+  store: ObservationStore,
+  projectId: string,
+  options: { scope?: SyncScope } = {},
+): SqliteSyncStore {
   const db = getDatabase(dataDir);
-  ensureSyncSchema(db, projectId);
+  const scope: SyncScope = options.scope ?? 'project';
+  const scopeId = scope === 'user' ? USER_SYNC_SCOPE_ID : projectId;
+  ensureSyncSchema(db, scope === 'user' ? LEGACY_PROJECT_ID : projectId);
   const getMeta = db.prepare('SELECT value FROM sync_meta WHERE key = ?');
   const setMeta = db.prepare('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)');
   const selState = db.prepare('SELECT projectId, syncKey, obsId, revision, writer, kind, contentHash, shippedSeq FROM sync_row_state WHERE projectId = ?');
@@ -68,7 +76,7 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
       WHERE projectId = ? AND namespace = ? AND deviceId = ? AND sequence = ?`,
   );
 
-  const namespace = syncNamespace(projectId);
+  const namespace = scope === 'user' ? userSyncNamespace() : syncNamespace(projectId);
   const deviceFingerprint = process.env.MEMORIX_SYNC_DEVICE_FINGERPRINT?.trim()
     || `${process.platform}|${os.hostname()}|${os.homedir()}`;
 
@@ -101,7 +109,8 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
   }
 
   return {
-    projectId: () => projectId,
+    projectId: () => scopeId,
+    scope: () => scope,
     namespace: () => namespace,
     deviceId: ensureDevice,
     assertCanPublish,
@@ -152,7 +161,7 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
     },
 
     async loadPendingBatches(): Promise<SyncPendingBatch[]> {
-      return (selPending.all(projectId, namespace, ensureDevice()) as Array<{ batchJson: string; statesJson: string }>).map((row) => ({
+      return (selPending.all(scopeId, namespace, ensureDevice()) as Array<{ batchJson: string; statesJson: string }>).map((row) => ({
         batch: JSON.parse(row.batchJson),
         states: JSON.parse(row.statesJson),
       } as SyncPendingBatch));
@@ -160,7 +169,7 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
 
     async enqueueBatch(pending: SyncPendingBatch): Promise<void> {
       insertPending.run({
-        projectId,
+        projectId: scopeId,
         namespace,
         deviceId: pending.batch.deviceId,
         sequence: pending.batch.sequence,
@@ -171,15 +180,15 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
     },
 
     async markBatchShipped(sequence: number): Promise<void> {
-      deletePending.run(projectId, namespace, ensureDevice(), sequence);
+      deletePending.run(scopeId, namespace, ensureDevice(), sequence);
     },
 
     async loadAll(): Promise<Observation[]> {
-      return store.loadByProject(projectId);
+      return scope === 'user' ? store.loadAll() : store.loadByProject(projectId);
     },
 
     async loadState(): Promise<Map<string, SyncRowState>> {
-      const rows = selState.all(projectId) as SyncRowState[];
+      const rows = selState.all(scopeId) as SyncRowState[];
       const map = new Map<string, SyncRowState>();
       for (const r of rows) map.set(r.syncKey, r);
       return map;
@@ -195,8 +204,13 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
     async allocateId(): Promise<number> {
       return store.atomic(async (tx) => {
         const nextId = await tx.loadIdCounter();
-        await tx.saveIdCounter(nextId + 1);
-        return nextId;
+        // Direct imports and legacy migrations may not have advanced the
+        // counter. Reconcile it with the table while the same write lock is
+        // held so a sync pull cannot reuse an occupied local id.
+        const maxId = Number(db.prepare('SELECT COALESCE(MAX(id), 0) AS maxId FROM observations').get()?.maxId ?? 0);
+        const allocated = Math.max(nextId, maxId + 1);
+        await tx.saveIdCounter(allocated + 1);
+        return allocated;
       });
     },
 
@@ -229,7 +243,13 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
     async applyRemove(id: number, state: SyncRowState): Promise<void> {
       await store.atomic(async (tx) => {
         const existing = await tx.getById(id);
-        if (existing && existing.projectId !== state.projectId) {
+        const expectedProjectId = scope === 'user'
+          ? sourceProjectIdFromSyncKey(state.syncKey)
+          : state.projectId;
+        if (!expectedProjectId) {
+          throw new Error('[memorix] sync refused to remove an observation without a project-scoped sync key');
+        }
+        if (existing && existing.projectId !== expectedProjectId) {
           throw new Error('[memorix] sync refused to remove an observation from another project');
         }
         await tx.remove(id);
@@ -262,6 +282,16 @@ export function createSqliteSyncStore(dataDir: string, store: ObservationStore,
   };
 }
 
+function sourceProjectIdFromSyncKey(syncKey: string): string | undefined {
+  const match = /^p:([^:]+):/.exec(syncKey);
+  if (!match) return undefined;
+  try {
+    return decodeURIComponent(match[1]);
+  } catch {
+    return undefined;
+  }
+}
+
 function ensureSyncSchema(db: any, projectId: string): void {
   db.exec(CREATE_SYNC_TABLES);
   const columns = db.prepare('PRAGMA table_info(sync_row_state)').all() as Array<{ name: string }>;

package/src/sync/types.ts

@@ -17,6 +17,7 @@
  *     reaches the same final state (last-writer-wins, convergent).
  */
 import type { Observation } from '../types.js';
+import type { SyncScope } from './policy.js';
 
 /** Kind of change carried in the journal. */
 export type ChangeKind = 'upsert' | 'tombstone';
@@ -158,6 +159,7 @@ export interface SyncReport {
   remote: string;
   namespace: string;
   projectId: string;
+  scope: SyncScope;
   deviceId: string;
   pushed: number;
   pulledBatches: number;

package/src/types.ts

@@ -670,7 +670,7 @@ export interface MemoryRef {
 /** MCP config format adapter interface */
 export interface MCPConfigAdapter {
   /** Setup-only adapters may support a host that does not participate in workspace sync. */
-  readonly source: AgentTarget | 'codebuddy' | 'dsh' | 'workbuddy';
+  readonly source: AgentTarget | 'codebuddy' | 'dsh' | 'workbuddy' | 'grok';
   /** Parse MCP server entries from a config file */
   parse(content: string): MCPServerEntry[];
   /** Generate config file content from MCP server entries */

package/src/workspace/mcp-adapters/grok.ts

@@ -0,0 +1,32 @@
+import type { MCPConfigAdapter, MCPServerEntry } from '../../types.js';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { CodexMCPAdapter } from './codex.js';
+
+/**
+ * Grok Build MCP config adapter.
+ * Format: TOML at $GROK_HOME/config.toml (default ~/.grok/config.toml) or
+ * .grok/config.toml (project-level). Same `[mcp_servers.<name>]` shape as Codex.
+ *
+ * Default setup leaves this file host-owned. Explicit `memorix setup --mcp http`
+ * writes a URL-only Memorix block.
+ */
+export class GrokMCPAdapter implements MCPConfigAdapter {
+  readonly source = 'grok' as const;
+  private readonly toml = new CodexMCPAdapter();
+
+  parse(content: string): MCPServerEntry[] {
+    return this.toml.parse(content);
+  }
+
+  generate(servers: MCPServerEntry[]): string {
+    return this.toml.generate(servers);
+  }
+
+  getConfigPath(_projectRoot?: string): string {
+    // Grok reads user-level ~/.grok/config.toml (or $GROK_HOME). Project
+    // `.grok/` is the hooks directory, not MCP config.
+    const grokHome = process.env.GROK_HOME?.trim() || join(homedir(), '.grok');
+    return join(grokHome, 'config.toml');
+  }
+}