memorix 1.8.9 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +7 -0
  3. package/README.zh-CN.md +6 -0
  4. package/dist/cli/index.js +3261 -1348
  5. package/dist/index.js +324 -187
  6. package/dist/maintenance-runner.js +285 -103
  7. package/dist/media-audio-runner.js +285 -103
  8. package/dist/media-video-runner.js +285 -103
  9. package/dist/memcode-runtime/CHANGELOG.md +18 -0
  10. package/dist/sdk.js +323 -186
  11. package/dist/vector-backfill-runner.d.ts +4 -1
  12. package/dist/vector-backfill-runner.js +288 -104
  13. package/docs/API_REFERENCE.md +14 -0
  14. package/docs/CONFIGURATION.md +46 -0
  15. package/docs/DEVELOPMENT.md +2 -2
  16. package/docs/MULTI-DEVICE-SYNC-RESEARCH.md +344 -0
  17. package/docs/PERFORMANCE.md +32 -0
  18. package/docs/README.md +4 -2
  19. package/llms-full.txt +1 -0
  20. package/llms.txt +1 -0
  21. package/package.json +1 -1
  22. package/plugins/antigravity/memorix/plugin.json +1 -1
  23. package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
  24. package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
  25. package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
  26. package/plugins/copilot/memorix/plugin.json +1 -1
  27. package/plugins/gemini/memorix/gemini-extension.json +1 -1
  28. package/plugins/hermes/memorix/plugin.yaml +1 -1
  29. package/plugins/omp/memorix/package.json +1 -1
  30. package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
  31. package/plugins/pi/memorix/package.json +1 -1
  32. package/server.json +2 -2
  33. package/src/cli/command-guide.ts +9 -1
  34. package/src/cli/commands/serve-http.ts +11 -0
  35. package/src/cli/commands/sync-store.ts +191 -0
  36. package/src/cli/commands/sync.ts +8 -0
  37. package/src/memory/observations.ts +15 -2
  38. package/src/runtime/maintenance-targets.ts +6 -0
  39. package/src/runtime/vector-backfill-runner.ts +185 -7
  40. package/src/store/orama-store.ts +85 -102
  41. package/src/sync/adapters/fs.ts +179 -0
  42. package/src/sync/adapters/github.ts +258 -0
  43. package/src/sync/adapters/object-store.ts +267 -0
  44. package/src/sync/adapters/postgres.ts +269 -0
  45. package/src/sync/engine.ts +353 -0
  46. package/src/sync/journal.ts +162 -0
  47. package/src/sync/merge.ts +116 -0
  48. package/src/sync/namespace.ts +15 -0
  49. package/src/sync/paging.ts +45 -0
  50. package/src/sync/policy.ts +37 -0
  51. package/src/sync/remote-factory.ts +49 -0
  52. package/src/sync/store-port.ts +288 -0
  53. package/src/sync/types.ts +183 -0
@@ -0,0 +1,269 @@
1
+ import type { ChangeBatch, SyncCompactReport, SyncPullPage, SyncRemote } from '../types.js';
2
+ import { decodePageCursor, encodePageCursor } from '../paging.js';
3
+ import { isSafeSyncDeviceId } from '../namespace.js';
4
+
5
+ export interface SqlClient {
6
+ query<T>(sql: string, params: unknown[]): Promise<T[]>;
7
+ exec(sql: string, params: unknown[]): Promise<void>;
8
+ close?(): Promise<void>;
9
+ }
10
+
11
+ interface BatchRow {
12
+ device_id: string;
13
+ sequence: number | string;
14
+ produced_at: string;
15
+ payload: ChangeBatch | string;
16
+ }
17
+
18
+ export class PostgresSyncRemote implements SyncRemote {
19
+ readonly kind = 'postgres';
20
+ readonly namespace: string;
21
+ private available = false;
22
+
23
+ constructor(private readonly sql: SqlClient, namespace: string) {
24
+ this.namespace = namespace;
25
+ }
26
+
27
+ async init(options: { create?: boolean } = {}): Promise<void> {
28
+ if (options.create === false) {
29
+ const [row] = await this.sql.query<{ name?: string | null }>(
30
+ `SELECT to_regclass('memorix_sync_batches') AS name`,
31
+ [],
32
+ );
33
+ this.available = typeof row?.name === 'string' && row.name.length > 0;
34
+ return;
35
+ }
36
+ await this.sql.exec(
37
+ `CREATE TABLE IF NOT EXISTS memorix_sync_batches (
38
+ namespace text NOT NULL,
39
+ device_id text NOT NULL,
40
+ sequence bigint NOT NULL,
41
+ produced_at text NOT NULL,
42
+ payload jsonb NOT NULL,
43
+ PRIMARY KEY (namespace, device_id, sequence)
44
+ )`,
45
+ [],
46
+ );
47
+ // Upgrade the pre-v3 #277 table without touching its payloads. Legacy rows
48
+ // stay quarantined under a non-project namespace; a fresh scoped sync never
49
+ // accidentally imports them.
50
+ await this.sql.exec(
51
+ `ALTER TABLE memorix_sync_batches
52
+ ADD COLUMN IF NOT EXISTS namespace text NOT NULL DEFAULT 'legacy'`,
53
+ [],
54
+ );
55
+ await this.sql.exec(
56
+ `CREATE UNIQUE INDEX IF NOT EXISTS memorix_sync_batches_scope_key
57
+ ON memorix_sync_batches(namespace, device_id, sequence)`,
58
+ [],
59
+ );
60
+ this.available = true;
61
+ }
62
+
63
+ async push(batch: ChangeBatch): Promise<void> {
64
+ assertBatchNamespace(batch, this.namespace);
65
+ if (!this.available) throw new Error('[memorix] Postgres sync remote is not initialized for writes');
66
+ await this.sql.exec(
67
+ `INSERT INTO memorix_sync_batches (namespace, device_id, sequence, produced_at, payload)
68
+ VALUES ($1, $2, $3, $4, $5::jsonb)
69
+ ON CONFLICT (namespace, device_id, sequence) DO NOTHING`,
70
+ [this.namespace, batch.deviceId, batch.sequence, batch.producedAt, JSON.stringify(batch)],
71
+ );
72
+ const [stored] = await this.sql.query<{ payload: ChangeBatch | string }>(
73
+ `SELECT payload FROM memorix_sync_batches
74
+ WHERE namespace = $1 AND device_id = $2 AND sequence = $3`,
75
+ [this.namespace, batch.deviceId, batch.sequence],
76
+ );
77
+ const payload = typeof stored?.payload === 'string'
78
+ ? JSON.parse(stored.payload) as ChangeBatch
79
+ : stored?.payload;
80
+ if (!payload || stableJson(payload) !== stableJson(batch)) {
81
+ throw new Error('[memorix] Postgres relay event path already contains a different payload');
82
+ }
83
+ }
84
+
85
+ async pull(since: Record<string, number>, limit: number, pageToken?: string): Promise<SyncPullPage> {
86
+ assertLimit(limit);
87
+ if (!this.available) return { batches: [], hasMore: false };
88
+ const after = decodePageCursor(pageToken);
89
+ const sinceEntries = Object.entries(since);
90
+ const params: unknown[] = [this.namespace];
91
+ const sinceClauses: string[] = [];
92
+ for (const [deviceId, sequence] of sinceEntries) {
93
+ params.push(deviceId, sequence);
94
+ const deviceParam = `$${params.length - 1}`;
95
+ const sequenceParam = `$${params.length}`;
96
+ sinceClauses.push(`(device_id = ${deviceParam} AND sequence > ${sequenceParam})`);
97
+ }
98
+ if (sinceEntries.length > 0) {
99
+ params.push(...sinceEntries.map(([deviceId]) => deviceId));
100
+ const placeholders = sinceEntries.map((_, index) => `$${params.length - sinceEntries.length + index + 1}`);
101
+ sinceClauses.push(`device_id NOT IN (${placeholders.join(', ')})`);
102
+ } else {
103
+ sinceClauses.push('TRUE');
104
+ }
105
+ const filters = [`(${sinceClauses.join(' OR ')})`];
106
+ if (after) {
107
+ params.push(after.deviceId, after.sequence);
108
+ const deviceParam = `$${params.length - 1}`;
109
+ const sequenceParam = `$${params.length}`;
110
+ filters.push(`(device_id > ${deviceParam} OR (device_id = ${deviceParam} AND sequence > ${sequenceParam}))`);
111
+ }
112
+ params.push(limit + 1);
113
+ const rows = await this.sql.query<BatchRow>(
114
+ `SELECT device_id, sequence, produced_at, payload
115
+ FROM memorix_sync_batches
116
+ WHERE namespace = $1 AND ${filters.join(' AND ')}
117
+ ORDER BY device_id, sequence
118
+ LIMIT $${params.length}`,
119
+ params,
120
+ );
121
+
122
+ const batches = rows
123
+ .map((row) => {
124
+ const payload = typeof row.payload === 'string'
125
+ ? JSON.parse(row.payload) as ChangeBatch
126
+ : row.payload;
127
+ return {
128
+ ...payload,
129
+ deviceId: row.device_id,
130
+ sequence: Number(row.sequence),
131
+ producedAt: row.produced_at,
132
+ };
133
+ })
134
+ .sort((left, right) => {
135
+ if (left.deviceId < right.deviceId) return -1;
136
+ if (left.deviceId > right.deviceId) return 1;
137
+ return left.sequence - right.sequence;
138
+ });
139
+ const hasMore = batches.length > limit;
140
+ const page = batches.slice(0, limit);
141
+ return {
142
+ batches: page,
143
+ hasMore,
144
+ nextPageToken: hasMore && page.length > 0
145
+ ? encodePageCursor(page[page.length - 1])
146
+ : undefined,
147
+ };
148
+ }
149
+
150
+ async compact(through: Record<string, number>, options: { dryRun?: boolean } = {}): Promise<SyncCompactReport> {
151
+ if (!this.available) return { candidates: 0, deleted: 0 };
152
+ const entries = Object.entries(through).filter(([, value]) => Number.isSafeInteger(value) && value >= 0);
153
+ if (entries.length === 0) return { candidates: 0, deleted: 0 };
154
+ const clauses: string[] = [];
155
+ const params: unknown[] = [this.namespace];
156
+ for (const [deviceId, sequence] of entries) {
157
+ params.push(deviceId, sequence);
158
+ clauses.push(`(device_id = $${params.length - 1} AND sequence <= $${params.length})`);
159
+ }
160
+ const where = `namespace = $1 AND (${clauses.join(' OR ')})`;
161
+ const [row] = await this.sql.query<{ count: number | string }>(`SELECT COUNT(*) AS count FROM memorix_sync_batches WHERE ${where}`, params);
162
+ const candidates = Number(row?.count ?? 0);
163
+ if (!options.dryRun && candidates > 0) await this.sql.exec(`DELETE FROM memorix_sync_batches WHERE ${where}`, params);
164
+ return { candidates, deleted: options.dryRun ? 0 : candidates };
165
+ }
166
+
167
+ async close(): Promise<void> {
168
+ await this.sql.close?.();
169
+ }
170
+ }
171
+
172
+ function assertBatchNamespace(batch: ChangeBatch, namespace: string): void {
173
+ if (batch.namespace !== namespace) {
174
+ throw new Error('[memorix] sync batch namespace does not match the Postgres relay');
175
+ }
176
+ if (!isSafeSyncDeviceId(batch.deviceId)) {
177
+ throw new Error('[memorix] sync batch has an invalid device id');
178
+ }
179
+ }
180
+
181
+ function assertLimit(limit: number): void {
182
+ if (!Number.isSafeInteger(limit) || limit < 1) {
183
+ throw new Error('[memorix] sync pull limit must be a positive integer');
184
+ }
185
+ }
186
+
187
+ function stableJson(value: unknown): string {
188
+ if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
189
+ if (value && typeof value === 'object') {
190
+ const record = value as Record<string, unknown>;
191
+ return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`;
192
+ }
193
+ return JSON.stringify(value) ?? 'null';
194
+ }
195
+
196
+ interface PgPool {
197
+ query(sql: string, params: unknown[]): Promise<{ rows: unknown[] }>;
198
+ end?(): Promise<void>;
199
+ }
200
+
201
+ interface PgModule {
202
+ Pool?: new (config: Record<string, unknown>) => PgPool;
203
+ default?: {
204
+ Pool?: new (config: Record<string, unknown>) => PgPool;
205
+ };
206
+ }
207
+
208
+ export async function createPgSqlClient(env: NodeJS.ProcessEnv = process.env): Promise<SqlClient> {
209
+ const connectionString = env.MEMORIX_SYNC_PG_URL;
210
+ let config: Record<string, unknown>;
211
+
212
+ if (connectionString) {
213
+ config = { connectionString };
214
+ } else {
215
+ const required = [
216
+ 'MEMORIX_SYNC_PG_HOST',
217
+ 'MEMORIX_SYNC_PG_PORT',
218
+ 'MEMORIX_SYNC_PG_DB',
219
+ 'MEMORIX_SYNC_PG_USER',
220
+ 'MEMORIX_SYNC_PG_PASSWORD',
221
+ ] as const;
222
+ const missing = required.filter((name) => !env[name]);
223
+ if (missing.length > 0) {
224
+ throw new Error(
225
+ `Postgres sync requires MEMORIX_SYNC_PG_URL or all discrete settings; missing ${missing.join(', ')}`,
226
+ );
227
+ }
228
+
229
+ const port = Number(env.MEMORIX_SYNC_PG_PORT);
230
+ if (!Number.isInteger(port) || port <= 0) {
231
+ throw new Error('MEMORIX_SYNC_PG_PORT must be a positive integer');
232
+ }
233
+ config = {
234
+ host: env.MEMORIX_SYNC_PG_HOST,
235
+ port,
236
+ database: env.MEMORIX_SYNC_PG_DB,
237
+ user: env.MEMORIX_SYNC_PG_USER,
238
+ password: env.MEMORIX_SYNC_PG_PASSWORD,
239
+ };
240
+ }
241
+
242
+ // pg is optional by design, so a static import would make every install require it.
243
+ const packageName = 'pg';
244
+ let pg: PgModule;
245
+ try {
246
+ pg = await import(packageName) as PgModule;
247
+ } catch (cause) {
248
+ throw new Error('Postgres sync requires the optional "pg" package; install it in your application', { cause });
249
+ }
250
+
251
+ const Pool = pg.Pool ?? pg.default?.Pool;
252
+ if (!Pool) {
253
+ throw new Error('The installed "pg" package does not export Pool');
254
+ }
255
+ const pool = new Pool(config);
256
+
257
+ return {
258
+ async query<T>(sql: string, params: unknown[]): Promise<T[]> {
259
+ const result = await pool.query(sql, params);
260
+ return result.rows as T[];
261
+ },
262
+ async exec(sql: string, params: unknown[]): Promise<void> {
263
+ await pool.query(sql, params);
264
+ },
265
+ async close(): Promise<void> {
266
+ await pool.end?.();
267
+ },
268
+ };
269
+ }
@@ -0,0 +1,353 @@
1
+ /**
2
+ * Sync engine — orchestrates push and pull against any `SyncRemote`.
3
+ *
4
+ * Local-first and off the hot path: the engine reads the local store, ships a
5
+ * change journal, pulls remote batches, and applies them through the pure merge
6
+ * policy. All identity/versioning lives in `sync_row_state`; the engine only
7
+ * moves data between that state, the observation store, and the remote.
8
+ */
9
+ import type { Observation } from '../types.js';
10
+ import { computeChanges, contentHash, nowIso } from './journal.js';
11
+ import { compareVersion, mergeOne } from './merge.js';
12
+ import { eligibleObservations } from './policy.js';
13
+ import { isSafeSyncDeviceId } from './namespace.js';
14
+ import type {
15
+ ChangeBatch,
16
+ SyncConflict,
17
+ SyncDecision,
18
+ SyncCursor,
19
+ SyncPendingBatch,
20
+ SyncRemote,
21
+ SyncReport,
22
+ SyncRowState,
23
+ } from './types.js';
24
+ import { createHash } from 'node:crypto';
25
+
26
+ /**
27
+ * Everything the engine needs from the local side. The SQLite binding lives in
28
+ * `store-port.ts`; tests supply an in-memory fake.
29
+ */
30
+ export interface SyncStorePort {
31
+ projectId(): string;
32
+ namespace(): string;
33
+ /** Stable device id for this replica. */
34
+ deviceId(): string;
35
+ /** Refuse writes when this data directory was copied to another machine. */
36
+ assertCanPublish(): void;
37
+ /** Rotate a cloned installation onto a new device identity. */
38
+ rotateDevice(): string;
39
+ /** Current local cursor for this project/remote namespace. */
40
+ loadCursor(): Promise<SyncCursor>;
41
+ saveCursor(cursor: SyncCursor): Promise<void>;
42
+ /** Current project observations only. */
43
+ loadAll(): Promise<Observation[]>;
44
+ /** Recorded sync state, keyed by syncKey. */
45
+ loadState(): Promise<Map<string, SyncRowState>>;
46
+ /** Upsert sync-state rows. */
47
+ saveState(rows: SyncRowState[]): Promise<void>;
48
+ /** Reserve and return a fresh local observation id (for imports). */
49
+ allocateId(): Promise<number>;
50
+ /** Insert an imported observation and its sync state atomically. */
51
+ applyInsert(row: Observation, state: SyncRowState): Promise<void>;
52
+ /** Update an existing observation and its sync state atomically. */
53
+ applyUpdate(row: Observation, state: SyncRowState): Promise<void>;
54
+ /** Remove an observation and persist its tombstone state atomically. */
55
+ applyRemove(id: number, state: SyncRowState): Promise<void>;
56
+ getById(id: number): Promise<Observation | undefined>;
57
+ recordConflict(conflict: SyncConflict): Promise<void>;
58
+ loadPendingBatches(): Promise<SyncPendingBatch[]>;
59
+ enqueueBatch(pending: SyncPendingBatch): Promise<void>;
60
+ markBatchShipped(sequence: number): Promise<void>;
61
+ /** Next monotonic batch sequence for this device. */
62
+ nextSequence(): Promise<number>;
63
+ }
64
+
65
+ const PULL_PAGE_SIZE = 100;
66
+ const MAX_BATCH_BYTES = 750_000;
67
+
68
+ export interface RunOptions {
69
+ deviceId: string;
70
+ dryRun: boolean;
71
+ /** When false, only pull (no local changes shipped). */
72
+ push: boolean;
73
+ /** When false, only push (no remote changes applied). */
74
+ pull: boolean;
75
+ }
76
+
77
+ export async function runSync(
78
+ store: SyncStorePort,
79
+ remote: SyncRemote,
80
+ opts: RunOptions,
81
+ ): Promise<SyncReport> {
82
+ const report: SyncReport = {
83
+ remote: remote.kind,
84
+ namespace: remote.namespace,
85
+ projectId: store.projectId(),
86
+ deviceId: opts.deviceId,
87
+ pushed: 0,
88
+ pulledBatches: 0,
89
+ applied: 0,
90
+ skipped: 0,
91
+ tombstoned: 0,
92
+ eligible: 0,
93
+ excluded: 0,
94
+ conflicts: 0,
95
+ pending: 0,
96
+ dryRun: opts.dryRun,
97
+ decisions: [],
98
+ };
99
+
100
+ if (remote.namespace !== store.namespace()) {
101
+ throw new Error('[memorix] sync refused: store and remote namespaces do not match');
102
+ }
103
+ if (store.deviceId() !== opts.deviceId) {
104
+ throw new Error('[memorix] sync refused: requested device does not match the local replica identity');
105
+ }
106
+ try {
107
+ // Check the local replica before a remote init can create a branch, table,
108
+ // or bucket prefix. A copied data directory must not gain write access by
109
+ // merely starting a sync run.
110
+ if (opts.push && !opts.dryRun) store.assertCanPublish();
111
+ await remote.init({ create: opts.push && !opts.dryRun });
112
+
113
+ // ── Push ──────────────────────────────────────────────────────────
114
+ if (opts.push) {
115
+ const pending = await store.loadPendingBatches();
116
+ report.pending = pending.length;
117
+ if (!opts.dryRun) {
118
+ for (const pendingBatch of pending) {
119
+ validateBatches([pendingBatch.batch], store.projectId(), remote.namespace);
120
+ if (pendingBatch.batch.deviceId !== opts.deviceId) {
121
+ throw new Error('[memorix] sync outbox rejected: batch belongs to another device');
122
+ }
123
+ await remote.push(pendingBatch.batch);
124
+ await store.saveState(pendingBatch.states);
125
+ await store.markBatchShipped(pendingBatch.batch.sequence);
126
+ report.pushed += pendingBatch.batch.entries.length;
127
+ }
128
+ report.pending = 0;
129
+ }
130
+ const current = await store.loadAll();
131
+ const filtered = eligibleObservations(current, store.projectId());
132
+ report.eligible = filtered.eligible.length;
133
+ report.excluded = filtered.excluded;
134
+ const state = await store.loadState();
135
+ const changes = computeChanges({
136
+ current: filtered.eligible,
137
+ state,
138
+ deviceId: opts.deviceId,
139
+ projectId: store.projectId(),
140
+ });
141
+ if (changes.length > 0) {
142
+ const chunks = chunkChanges(changes);
143
+ for (const chunk of chunks) {
144
+ const sequence = opts.dryRun ? 0 : await store.nextSequence();
145
+ const batch: ChangeBatch = {
146
+ formatVersion: 3,
147
+ namespace: remote.namespace,
148
+ projectId: store.projectId(),
149
+ deviceId: opts.deviceId,
150
+ sequence,
151
+ producedAt: nowIso(),
152
+ entries: chunk.map((c) => c.entry),
153
+ };
154
+ if (!opts.dryRun) validateBatches([batch], store.projectId(), remote.namespace);
155
+ report.pushed += chunk.length;
156
+ if (!opts.dryRun) {
157
+ const states = chunk.map((c) => ({ ...c.nextState, shippedSeq: sequence }));
158
+ await store.enqueueBatch({ batch, states });
159
+ await remote.push(batch);
160
+ await store.saveState(states);
161
+ await store.markBatchShipped(sequence);
162
+ }
163
+ }
164
+ }
165
+ }
166
+
167
+ // ── Pull ──────────────────────────────────────────────────────────
168
+ if (opts.pull) {
169
+ const cursor = await store.loadCursor();
170
+ const state = await store.loadState();
171
+ const pullSince = { ...cursor.applied };
172
+ let pageToken: string | undefined;
173
+ let hasMore = true;
174
+ while (hasMore) {
175
+ // Keep the baseline fixed for every page. Adapters use pageToken to
176
+ // advance through one logical pull; passing the mutating cursor here
177
+ // can skip later rows after page one.
178
+ const page = await remote.pull(pullSince, PULL_PAGE_SIZE, pageToken);
179
+ report.pulledBatches += page.batches.length;
180
+ validateBatches(page.batches, store.projectId(), remote.namespace);
181
+
182
+ for (const batch of page.batches) {
183
+ if (batch.deviceId === opts.deviceId) {
184
+ cursor.applied[batch.deviceId] = Math.max(cursor.applied[batch.deviceId] ?? 0, batch.sequence);
185
+ continue;
186
+ }
187
+ for (const entry of batch.entries) {
188
+ const local = state.get(entry.syncKey);
189
+ const incomingHash = entry.row ? contentHash(entry.row) : '';
190
+ if (entry.contentHash !== incomingHash) {
191
+ throw new Error('[memorix] sync batch rejected: content hash mismatch');
192
+ }
193
+ if (
194
+ local
195
+ && compareVersion(entry.version, { revision: local.revision, writer: local.writer }) === 0
196
+ && local.contentHash !== incomingHash
197
+ ) {
198
+ throw new Error('[memorix] sync batch rejected: same version has different content');
199
+ }
200
+ const { decision, action } = mergeOne({ incoming: entry, local, incomingHash });
201
+ recordDecision(report, decision);
202
+ if (local && entry.version.revision === local.revision && entry.version.writer !== local.writer) {
203
+ const currentRow = local.obsId == null ? undefined : await store.getById(local.obsId);
204
+ if (opts.dryRun) report.conflicts++;
205
+ else await recordConcurrentConflict(store, report, local, entry, action.type === 'noop' ? entry.row : currentRow);
206
+ }
207
+ if (opts.dryRun || action.type === 'noop') {
208
+ if (!opts.dryRun && action.type === 'noop' && decision.outcome === 'apply') {
209
+ const next: SyncRowState = {
210
+ projectId: store.projectId(),
211
+ syncKey: entry.syncKey,
212
+ obsId: null,
213
+ revision: entry.version.revision,
214
+ writer: entry.version.writer,
215
+ kind: 'tombstone',
216
+ contentHash: '',
217
+ shippedSeq: local?.shippedSeq ?? 0,
218
+ };
219
+ state.set(entry.syncKey, next);
220
+ await store.saveState([next]);
221
+ }
222
+ continue;
223
+ }
224
+
225
+ let next: SyncRowState;
226
+ if (action.type === 'insert') {
227
+ const newId = await store.allocateId();
228
+ const row = { ...entry.row!, id: newId };
229
+ next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: newId, revision: action.version.revision, writer: action.version.writer, kind: 'upsert', contentHash: action.contentHash, shippedSeq: local?.shippedSeq ?? 0 };
230
+ await store.applyInsert(row, next);
231
+ } else if (action.type === 'update') {
232
+ const row = { ...entry.row!, id: action.obsId };
233
+ next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: action.obsId, revision: action.version.revision, writer: action.version.writer, kind: 'upsert', contentHash: action.contentHash, shippedSeq: local?.shippedSeq ?? 0 };
234
+ await store.applyUpdate(row, next);
235
+ } else {
236
+ next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: null, revision: action.version.revision, writer: action.version.writer, kind: 'tombstone', contentHash: '', shippedSeq: local?.shippedSeq ?? 0 };
237
+ await store.applyRemove(action.obsId, next);
238
+ }
239
+ state.set(entry.syncKey, next);
240
+ await store.saveState([next]);
241
+ }
242
+ cursor.applied[batch.deviceId] = Math.max(cursor.applied[batch.deviceId] ?? 0, batch.sequence);
243
+ }
244
+ hasMore = page.hasMore;
245
+ pageToken = page.nextPageToken;
246
+ if (!opts.dryRun && page.batches.length > 0) await store.saveCursor(cursor);
247
+ if (page.batches.length === 0 && !page.nextPageToken) hasMore = false;
248
+ if (hasMore && !page.nextPageToken) throw new Error('[memorix] sync remote returned an unpaged continuation');
249
+ }
250
+ }
251
+ return report;
252
+ } finally {
253
+ await remote.close();
254
+ }
255
+ }
256
+
257
+ function validateBatches(batches: ChangeBatch[], projectId: string, namespace: string): void {
258
+ for (const batch of batches) {
259
+ if (batch.formatVersion !== 3 || batch.projectId !== projectId || batch.namespace !== namespace) {
260
+ throw new Error('[memorix] sync batch rejected: wrong format, project, or remote namespace');
261
+ }
262
+ if (
263
+ !isSafeSyncDeviceId(batch.deviceId)
264
+ || !Number.isSafeInteger(batch.sequence)
265
+ || batch.sequence < 1
266
+ || !Array.isArray(batch.entries)
267
+ || Buffer.byteLength(JSON.stringify(batch), 'utf8') > MAX_BATCH_BYTES
268
+ ) {
269
+ throw new Error('[memorix] sync batch rejected: malformed envelope');
270
+ }
271
+ for (const entry of batch.entries) {
272
+ if (
273
+ !entry.syncKey
274
+ || (entry.kind !== 'upsert' && entry.kind !== 'tombstone')
275
+ || !entry.version
276
+ || !Number.isSafeInteger(entry.version.revision)
277
+ || entry.version.revision < 1
278
+ || !entry.version.writer
279
+ || typeof entry.contentHash !== 'string'
280
+ || (entry.kind === 'tombstone' && (entry.row !== undefined || entry.contentHash !== ''))
281
+ || (entry.kind === 'upsert' && (!/^[a-f0-9]{64}$/.test(entry.contentHash) || entry.row === undefined))
282
+ ) {
283
+ throw new Error('[memorix] sync batch rejected: malformed change entry');
284
+ }
285
+ if (
286
+ entry.kind === 'upsert'
287
+ && (!Number.isSafeInteger(entry.row!.id) || entry.row!.id < 1 || !eligibleObservations([entry.row!], projectId).eligible.length)
288
+ ) {
289
+ throw new Error('[memorix] sync batch rejected: ineligible observation payload');
290
+ }
291
+ const incomingHash = entry.row ? contentHash(entry.row) : '';
292
+ if (entry.contentHash !== incomingHash) {
293
+ throw new Error('[memorix] sync batch rejected: content hash mismatch');
294
+ }
295
+ }
296
+ }
297
+ }
298
+
299
+ function chunkChanges<T extends { entry: ChangeBatch['entries'][number] }>(changes: T[]): T[][] {
300
+ const chunks: T[][] = [];
301
+ let current: T[] = [];
302
+ let currentBytes = 2;
303
+ for (const change of changes) {
304
+ const bytes = Buffer.byteLength(JSON.stringify(change.entry), 'utf8') + 1;
305
+ if (bytes + 2 > MAX_BATCH_BYTES) throw new Error('[memorix] one sync event is too large for the relay');
306
+ if (current.length > 0 && currentBytes + bytes > MAX_BATCH_BYTES) {
307
+ chunks.push(current);
308
+ current = [];
309
+ currentBytes = 2;
310
+ }
311
+ current.push(change);
312
+ currentBytes += bytes;
313
+ }
314
+ if (current.length > 0) chunks.push(current);
315
+ return chunks;
316
+ }
317
+
318
+ async function recordConcurrentConflict(
319
+ store: SyncStorePort,
320
+ report: SyncReport,
321
+ local: SyncRowState,
322
+ incoming: ChangeBatch['entries'][number],
323
+ localRow: Observation | undefined,
324
+ ): Promise<void> {
325
+ const incomingWins = compareVersion(incoming.version, { revision: local.revision, writer: local.writer }) > 0;
326
+ const loser = incomingWins ? { revision: local.revision, writer: local.writer } : incoming.version;
327
+ const loserRow = incomingWins ? localRow : incoming.row;
328
+ const id = createHash('sha256')
329
+ .update(`${store.projectId()}|${incoming.syncKey}|${local.revision}|${local.writer}|${incoming.version.revision}|${incoming.version.writer}`)
330
+ .digest('hex');
331
+ await store.recordConflict({
332
+ id,
333
+ projectId: store.projectId(),
334
+ syncKey: incoming.syncKey,
335
+ winner: incomingWins ? incoming.version : { revision: local.revision, writer: local.writer },
336
+ loser,
337
+ loserKind: incomingWins ? local.kind : incoming.kind,
338
+ loserRow,
339
+ createdAt: nowIso(),
340
+ reason: 'concurrent versions shared the same logical revision',
341
+ });
342
+ report.conflicts++;
343
+ }
344
+
345
+ function recordDecision(report: SyncReport, decision: SyncDecision): void {
346
+ report.decisions.push(decision);
347
+ if (decision.outcome === 'apply') {
348
+ if (decision.kind === 'tombstone') report.tombstoned++;
349
+ else report.applied++;
350
+ } else {
351
+ report.skipped++;
352
+ }
353
+ }