memorix 1.8.9 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +7 -0
- package/README.zh-CN.md +6 -0
- package/dist/cli/index.js +3261 -1348
- package/dist/index.js +324 -187
- package/dist/maintenance-runner.js +285 -103
- package/dist/media-audio-runner.js +285 -103
- package/dist/media-video-runner.js +285 -103
- package/dist/memcode-runtime/CHANGELOG.md +18 -0
- package/dist/sdk.js +323 -186
- package/dist/vector-backfill-runner.d.ts +4 -1
- package/dist/vector-backfill-runner.js +288 -104
- package/docs/API_REFERENCE.md +14 -0
- package/docs/CONFIGURATION.md +46 -0
- package/docs/DEVELOPMENT.md +2 -2
- package/docs/MULTI-DEVICE-SYNC-RESEARCH.md +344 -0
- package/docs/PERFORMANCE.md +32 -0
- package/docs/README.md +4 -2
- package/llms-full.txt +1 -0
- package/llms.txt +1 -0
- package/package.json +1 -1
- package/plugins/antigravity/memorix/plugin.json +1 -1
- package/plugins/claude/memorix/.claude-plugin/plugin.json +1 -1
- package/plugins/codebuddy/memorix/.codebuddy-plugin/plugin.json +1 -1
- package/plugins/codex/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/copilot/memorix/plugin.json +1 -1
- package/plugins/gemini/memorix/gemini-extension.json +1 -1
- package/plugins/hermes/memorix/plugin.yaml +1 -1
- package/plugins/omp/memorix/package.json +1 -1
- package/plugins/openclaw/memorix/.codex-plugin/plugin.json +1 -1
- package/plugins/pi/memorix/package.json +1 -1
- package/server.json +2 -2
- package/src/cli/command-guide.ts +9 -1
- package/src/cli/commands/serve-http.ts +11 -0
- package/src/cli/commands/sync-store.ts +191 -0
- package/src/cli/commands/sync.ts +8 -0
- package/src/memory/observations.ts +15 -2
- package/src/runtime/maintenance-targets.ts +6 -0
- package/src/runtime/vector-backfill-runner.ts +185 -7
- package/src/store/orama-store.ts +85 -102
- package/src/sync/adapters/fs.ts +179 -0
- package/src/sync/adapters/github.ts +258 -0
- package/src/sync/adapters/object-store.ts +267 -0
- package/src/sync/adapters/postgres.ts +269 -0
- package/src/sync/engine.ts +353 -0
- package/src/sync/journal.ts +162 -0
- package/src/sync/merge.ts +116 -0
- package/src/sync/namespace.ts +15 -0
- package/src/sync/paging.ts +45 -0
- package/src/sync/policy.ts +37 -0
- package/src/sync/remote-factory.ts +49 -0
- package/src/sync/store-port.ts +288 -0
- package/src/sync/types.ts +183 -0
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import type { ChangeBatch, SyncCompactReport, SyncPullPage, SyncRemote } from '../types.js';
|
|
2
|
+
import { decodePageCursor, encodePageCursor } from '../paging.js';
|
|
3
|
+
import { isSafeSyncDeviceId } from '../namespace.js';
|
|
4
|
+
|
|
5
|
+
export interface SqlClient {
|
|
6
|
+
query<T>(sql: string, params: unknown[]): Promise<T[]>;
|
|
7
|
+
exec(sql: string, params: unknown[]): Promise<void>;
|
|
8
|
+
close?(): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface BatchRow {
|
|
12
|
+
device_id: string;
|
|
13
|
+
sequence: number | string;
|
|
14
|
+
produced_at: string;
|
|
15
|
+
payload: ChangeBatch | string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class PostgresSyncRemote implements SyncRemote {
|
|
19
|
+
readonly kind = 'postgres';
|
|
20
|
+
readonly namespace: string;
|
|
21
|
+
private available = false;
|
|
22
|
+
|
|
23
|
+
constructor(private readonly sql: SqlClient, namespace: string) {
|
|
24
|
+
this.namespace = namespace;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async init(options: { create?: boolean } = {}): Promise<void> {
|
|
28
|
+
if (options.create === false) {
|
|
29
|
+
const [row] = await this.sql.query<{ name?: string | null }>(
|
|
30
|
+
`SELECT to_regclass('memorix_sync_batches') AS name`,
|
|
31
|
+
[],
|
|
32
|
+
);
|
|
33
|
+
this.available = typeof row?.name === 'string' && row.name.length > 0;
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
await this.sql.exec(
|
|
37
|
+
`CREATE TABLE IF NOT EXISTS memorix_sync_batches (
|
|
38
|
+
namespace text NOT NULL,
|
|
39
|
+
device_id text NOT NULL,
|
|
40
|
+
sequence bigint NOT NULL,
|
|
41
|
+
produced_at text NOT NULL,
|
|
42
|
+
payload jsonb NOT NULL,
|
|
43
|
+
PRIMARY KEY (namespace, device_id, sequence)
|
|
44
|
+
)`,
|
|
45
|
+
[],
|
|
46
|
+
);
|
|
47
|
+
// Upgrade the pre-v3 #277 table without touching its payloads. Legacy rows
|
|
48
|
+
// stay quarantined under a non-project namespace; a fresh scoped sync never
|
|
49
|
+
// accidentally imports them.
|
|
50
|
+
await this.sql.exec(
|
|
51
|
+
`ALTER TABLE memorix_sync_batches
|
|
52
|
+
ADD COLUMN IF NOT EXISTS namespace text NOT NULL DEFAULT 'legacy'`,
|
|
53
|
+
[],
|
|
54
|
+
);
|
|
55
|
+
await this.sql.exec(
|
|
56
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS memorix_sync_batches_scope_key
|
|
57
|
+
ON memorix_sync_batches(namespace, device_id, sequence)`,
|
|
58
|
+
[],
|
|
59
|
+
);
|
|
60
|
+
this.available = true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async push(batch: ChangeBatch): Promise<void> {
|
|
64
|
+
assertBatchNamespace(batch, this.namespace);
|
|
65
|
+
if (!this.available) throw new Error('[memorix] Postgres sync remote is not initialized for writes');
|
|
66
|
+
await this.sql.exec(
|
|
67
|
+
`INSERT INTO memorix_sync_batches (namespace, device_id, sequence, produced_at, payload)
|
|
68
|
+
VALUES ($1, $2, $3, $4, $5::jsonb)
|
|
69
|
+
ON CONFLICT (namespace, device_id, sequence) DO NOTHING`,
|
|
70
|
+
[this.namespace, batch.deviceId, batch.sequence, batch.producedAt, JSON.stringify(batch)],
|
|
71
|
+
);
|
|
72
|
+
const [stored] = await this.sql.query<{ payload: ChangeBatch | string }>(
|
|
73
|
+
`SELECT payload FROM memorix_sync_batches
|
|
74
|
+
WHERE namespace = $1 AND device_id = $2 AND sequence = $3`,
|
|
75
|
+
[this.namespace, batch.deviceId, batch.sequence],
|
|
76
|
+
);
|
|
77
|
+
const payload = typeof stored?.payload === 'string'
|
|
78
|
+
? JSON.parse(stored.payload) as ChangeBatch
|
|
79
|
+
: stored?.payload;
|
|
80
|
+
if (!payload || stableJson(payload) !== stableJson(batch)) {
|
|
81
|
+
throw new Error('[memorix] Postgres relay event path already contains a different payload');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async pull(since: Record<string, number>, limit: number, pageToken?: string): Promise<SyncPullPage> {
|
|
86
|
+
assertLimit(limit);
|
|
87
|
+
if (!this.available) return { batches: [], hasMore: false };
|
|
88
|
+
const after = decodePageCursor(pageToken);
|
|
89
|
+
const sinceEntries = Object.entries(since);
|
|
90
|
+
const params: unknown[] = [this.namespace];
|
|
91
|
+
const sinceClauses: string[] = [];
|
|
92
|
+
for (const [deviceId, sequence] of sinceEntries) {
|
|
93
|
+
params.push(deviceId, sequence);
|
|
94
|
+
const deviceParam = `$${params.length - 1}`;
|
|
95
|
+
const sequenceParam = `$${params.length}`;
|
|
96
|
+
sinceClauses.push(`(device_id = ${deviceParam} AND sequence > ${sequenceParam})`);
|
|
97
|
+
}
|
|
98
|
+
if (sinceEntries.length > 0) {
|
|
99
|
+
params.push(...sinceEntries.map(([deviceId]) => deviceId));
|
|
100
|
+
const placeholders = sinceEntries.map((_, index) => `$${params.length - sinceEntries.length + index + 1}`);
|
|
101
|
+
sinceClauses.push(`device_id NOT IN (${placeholders.join(', ')})`);
|
|
102
|
+
} else {
|
|
103
|
+
sinceClauses.push('TRUE');
|
|
104
|
+
}
|
|
105
|
+
const filters = [`(${sinceClauses.join(' OR ')})`];
|
|
106
|
+
if (after) {
|
|
107
|
+
params.push(after.deviceId, after.sequence);
|
|
108
|
+
const deviceParam = `$${params.length - 1}`;
|
|
109
|
+
const sequenceParam = `$${params.length}`;
|
|
110
|
+
filters.push(`(device_id > ${deviceParam} OR (device_id = ${deviceParam} AND sequence > ${sequenceParam}))`);
|
|
111
|
+
}
|
|
112
|
+
params.push(limit + 1);
|
|
113
|
+
const rows = await this.sql.query<BatchRow>(
|
|
114
|
+
`SELECT device_id, sequence, produced_at, payload
|
|
115
|
+
FROM memorix_sync_batches
|
|
116
|
+
WHERE namespace = $1 AND ${filters.join(' AND ')}
|
|
117
|
+
ORDER BY device_id, sequence
|
|
118
|
+
LIMIT $${params.length}`,
|
|
119
|
+
params,
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
const batches = rows
|
|
123
|
+
.map((row) => {
|
|
124
|
+
const payload = typeof row.payload === 'string'
|
|
125
|
+
? JSON.parse(row.payload) as ChangeBatch
|
|
126
|
+
: row.payload;
|
|
127
|
+
return {
|
|
128
|
+
...payload,
|
|
129
|
+
deviceId: row.device_id,
|
|
130
|
+
sequence: Number(row.sequence),
|
|
131
|
+
producedAt: row.produced_at,
|
|
132
|
+
};
|
|
133
|
+
})
|
|
134
|
+
.sort((left, right) => {
|
|
135
|
+
if (left.deviceId < right.deviceId) return -1;
|
|
136
|
+
if (left.deviceId > right.deviceId) return 1;
|
|
137
|
+
return left.sequence - right.sequence;
|
|
138
|
+
});
|
|
139
|
+
const hasMore = batches.length > limit;
|
|
140
|
+
const page = batches.slice(0, limit);
|
|
141
|
+
return {
|
|
142
|
+
batches: page,
|
|
143
|
+
hasMore,
|
|
144
|
+
nextPageToken: hasMore && page.length > 0
|
|
145
|
+
? encodePageCursor(page[page.length - 1])
|
|
146
|
+
: undefined,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
async compact(through: Record<string, number>, options: { dryRun?: boolean } = {}): Promise<SyncCompactReport> {
|
|
151
|
+
if (!this.available) return { candidates: 0, deleted: 0 };
|
|
152
|
+
const entries = Object.entries(through).filter(([, value]) => Number.isSafeInteger(value) && value >= 0);
|
|
153
|
+
if (entries.length === 0) return { candidates: 0, deleted: 0 };
|
|
154
|
+
const clauses: string[] = [];
|
|
155
|
+
const params: unknown[] = [this.namespace];
|
|
156
|
+
for (const [deviceId, sequence] of entries) {
|
|
157
|
+
params.push(deviceId, sequence);
|
|
158
|
+
clauses.push(`(device_id = $${params.length - 1} AND sequence <= $${params.length})`);
|
|
159
|
+
}
|
|
160
|
+
const where = `namespace = $1 AND (${clauses.join(' OR ')})`;
|
|
161
|
+
const [row] = await this.sql.query<{ count: number | string }>(`SELECT COUNT(*) AS count FROM memorix_sync_batches WHERE ${where}`, params);
|
|
162
|
+
const candidates = Number(row?.count ?? 0);
|
|
163
|
+
if (!options.dryRun && candidates > 0) await this.sql.exec(`DELETE FROM memorix_sync_batches WHERE ${where}`, params);
|
|
164
|
+
return { candidates, deleted: options.dryRun ? 0 : candidates };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async close(): Promise<void> {
|
|
168
|
+
await this.sql.close?.();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function assertBatchNamespace(batch: ChangeBatch, namespace: string): void {
|
|
173
|
+
if (batch.namespace !== namespace) {
|
|
174
|
+
throw new Error('[memorix] sync batch namespace does not match the Postgres relay');
|
|
175
|
+
}
|
|
176
|
+
if (!isSafeSyncDeviceId(batch.deviceId)) {
|
|
177
|
+
throw new Error('[memorix] sync batch has an invalid device id');
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function assertLimit(limit: number): void {
|
|
182
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
183
|
+
throw new Error('[memorix] sync pull limit must be a positive integer');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function stableJson(value: unknown): string {
|
|
188
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
|
|
189
|
+
if (value && typeof value === 'object') {
|
|
190
|
+
const record = value as Record<string, unknown>;
|
|
191
|
+
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`;
|
|
192
|
+
}
|
|
193
|
+
return JSON.stringify(value) ?? 'null';
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
interface PgPool {
|
|
197
|
+
query(sql: string, params: unknown[]): Promise<{ rows: unknown[] }>;
|
|
198
|
+
end?(): Promise<void>;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
interface PgModule {
|
|
202
|
+
Pool?: new (config: Record<string, unknown>) => PgPool;
|
|
203
|
+
default?: {
|
|
204
|
+
Pool?: new (config: Record<string, unknown>) => PgPool;
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export async function createPgSqlClient(env: NodeJS.ProcessEnv = process.env): Promise<SqlClient> {
|
|
209
|
+
const connectionString = env.MEMORIX_SYNC_PG_URL;
|
|
210
|
+
let config: Record<string, unknown>;
|
|
211
|
+
|
|
212
|
+
if (connectionString) {
|
|
213
|
+
config = { connectionString };
|
|
214
|
+
} else {
|
|
215
|
+
const required = [
|
|
216
|
+
'MEMORIX_SYNC_PG_HOST',
|
|
217
|
+
'MEMORIX_SYNC_PG_PORT',
|
|
218
|
+
'MEMORIX_SYNC_PG_DB',
|
|
219
|
+
'MEMORIX_SYNC_PG_USER',
|
|
220
|
+
'MEMORIX_SYNC_PG_PASSWORD',
|
|
221
|
+
] as const;
|
|
222
|
+
const missing = required.filter((name) => !env[name]);
|
|
223
|
+
if (missing.length > 0) {
|
|
224
|
+
throw new Error(
|
|
225
|
+
`Postgres sync requires MEMORIX_SYNC_PG_URL or all discrete settings; missing ${missing.join(', ')}`,
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const port = Number(env.MEMORIX_SYNC_PG_PORT);
|
|
230
|
+
if (!Number.isInteger(port) || port <= 0) {
|
|
231
|
+
throw new Error('MEMORIX_SYNC_PG_PORT must be a positive integer');
|
|
232
|
+
}
|
|
233
|
+
config = {
|
|
234
|
+
host: env.MEMORIX_SYNC_PG_HOST,
|
|
235
|
+
port,
|
|
236
|
+
database: env.MEMORIX_SYNC_PG_DB,
|
|
237
|
+
user: env.MEMORIX_SYNC_PG_USER,
|
|
238
|
+
password: env.MEMORIX_SYNC_PG_PASSWORD,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// pg is optional by design, so a static import would make every install require it.
|
|
243
|
+
const packageName = 'pg';
|
|
244
|
+
let pg: PgModule;
|
|
245
|
+
try {
|
|
246
|
+
pg = await import(packageName) as PgModule;
|
|
247
|
+
} catch (cause) {
|
|
248
|
+
throw new Error('Postgres sync requires the optional "pg" package; install it in your application', { cause });
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const Pool = pg.Pool ?? pg.default?.Pool;
|
|
252
|
+
if (!Pool) {
|
|
253
|
+
throw new Error('The installed "pg" package does not export Pool');
|
|
254
|
+
}
|
|
255
|
+
const pool = new Pool(config);
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
async query<T>(sql: string, params: unknown[]): Promise<T[]> {
|
|
259
|
+
const result = await pool.query(sql, params);
|
|
260
|
+
return result.rows as T[];
|
|
261
|
+
},
|
|
262
|
+
async exec(sql: string, params: unknown[]): Promise<void> {
|
|
263
|
+
await pool.query(sql, params);
|
|
264
|
+
},
|
|
265
|
+
async close(): Promise<void> {
|
|
266
|
+
await pool.end?.();
|
|
267
|
+
},
|
|
268
|
+
};
|
|
269
|
+
}
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync engine — orchestrates push and pull against any `SyncRemote`.
|
|
3
|
+
*
|
|
4
|
+
* Local-first and off the hot path: the engine reads the local store, ships a
|
|
5
|
+
* change journal, pulls remote batches, and applies them through the pure merge
|
|
6
|
+
* policy. All identity/versioning lives in `sync_row_state`; the engine only
|
|
7
|
+
* moves data between that state, the observation store, and the remote.
|
|
8
|
+
*/
|
|
9
|
+
import type { Observation } from '../types.js';
|
|
10
|
+
import { computeChanges, contentHash, nowIso } from './journal.js';
|
|
11
|
+
import { compareVersion, mergeOne } from './merge.js';
|
|
12
|
+
import { eligibleObservations } from './policy.js';
|
|
13
|
+
import { isSafeSyncDeviceId } from './namespace.js';
|
|
14
|
+
import type {
|
|
15
|
+
ChangeBatch,
|
|
16
|
+
SyncConflict,
|
|
17
|
+
SyncDecision,
|
|
18
|
+
SyncCursor,
|
|
19
|
+
SyncPendingBatch,
|
|
20
|
+
SyncRemote,
|
|
21
|
+
SyncReport,
|
|
22
|
+
SyncRowState,
|
|
23
|
+
} from './types.js';
|
|
24
|
+
import { createHash } from 'node:crypto';
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Everything the engine needs from the local side. The SQLite binding lives in
|
|
28
|
+
* `store-port.ts`; tests supply an in-memory fake.
|
|
29
|
+
*/
|
|
30
|
+
export interface SyncStorePort {
|
|
31
|
+
projectId(): string;
|
|
32
|
+
namespace(): string;
|
|
33
|
+
/** Stable device id for this replica. */
|
|
34
|
+
deviceId(): string;
|
|
35
|
+
/** Refuse writes when this data directory was copied to another machine. */
|
|
36
|
+
assertCanPublish(): void;
|
|
37
|
+
/** Rotate a cloned installation onto a new device identity. */
|
|
38
|
+
rotateDevice(): string;
|
|
39
|
+
/** Current local cursor for this project/remote namespace. */
|
|
40
|
+
loadCursor(): Promise<SyncCursor>;
|
|
41
|
+
saveCursor(cursor: SyncCursor): Promise<void>;
|
|
42
|
+
/** Current project observations only. */
|
|
43
|
+
loadAll(): Promise<Observation[]>;
|
|
44
|
+
/** Recorded sync state, keyed by syncKey. */
|
|
45
|
+
loadState(): Promise<Map<string, SyncRowState>>;
|
|
46
|
+
/** Upsert sync-state rows. */
|
|
47
|
+
saveState(rows: SyncRowState[]): Promise<void>;
|
|
48
|
+
/** Reserve and return a fresh local observation id (for imports). */
|
|
49
|
+
allocateId(): Promise<number>;
|
|
50
|
+
/** Insert an imported observation and its sync state atomically. */
|
|
51
|
+
applyInsert(row: Observation, state: SyncRowState): Promise<void>;
|
|
52
|
+
/** Update an existing observation and its sync state atomically. */
|
|
53
|
+
applyUpdate(row: Observation, state: SyncRowState): Promise<void>;
|
|
54
|
+
/** Remove an observation and persist its tombstone state atomically. */
|
|
55
|
+
applyRemove(id: number, state: SyncRowState): Promise<void>;
|
|
56
|
+
getById(id: number): Promise<Observation | undefined>;
|
|
57
|
+
recordConflict(conflict: SyncConflict): Promise<void>;
|
|
58
|
+
loadPendingBatches(): Promise<SyncPendingBatch[]>;
|
|
59
|
+
enqueueBatch(pending: SyncPendingBatch): Promise<void>;
|
|
60
|
+
markBatchShipped(sequence: number): Promise<void>;
|
|
61
|
+
/** Next monotonic batch sequence for this device. */
|
|
62
|
+
nextSequence(): Promise<number>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const PULL_PAGE_SIZE = 100;
|
|
66
|
+
const MAX_BATCH_BYTES = 750_000;
|
|
67
|
+
|
|
68
|
+
export interface RunOptions {
|
|
69
|
+
deviceId: string;
|
|
70
|
+
dryRun: boolean;
|
|
71
|
+
/** When false, only pull (no local changes shipped). */
|
|
72
|
+
push: boolean;
|
|
73
|
+
/** When false, only push (no remote changes applied). */
|
|
74
|
+
pull: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export async function runSync(
|
|
78
|
+
store: SyncStorePort,
|
|
79
|
+
remote: SyncRemote,
|
|
80
|
+
opts: RunOptions,
|
|
81
|
+
): Promise<SyncReport> {
|
|
82
|
+
const report: SyncReport = {
|
|
83
|
+
remote: remote.kind,
|
|
84
|
+
namespace: remote.namespace,
|
|
85
|
+
projectId: store.projectId(),
|
|
86
|
+
deviceId: opts.deviceId,
|
|
87
|
+
pushed: 0,
|
|
88
|
+
pulledBatches: 0,
|
|
89
|
+
applied: 0,
|
|
90
|
+
skipped: 0,
|
|
91
|
+
tombstoned: 0,
|
|
92
|
+
eligible: 0,
|
|
93
|
+
excluded: 0,
|
|
94
|
+
conflicts: 0,
|
|
95
|
+
pending: 0,
|
|
96
|
+
dryRun: opts.dryRun,
|
|
97
|
+
decisions: [],
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
if (remote.namespace !== store.namespace()) {
|
|
101
|
+
throw new Error('[memorix] sync refused: store and remote namespaces do not match');
|
|
102
|
+
}
|
|
103
|
+
if (store.deviceId() !== opts.deviceId) {
|
|
104
|
+
throw new Error('[memorix] sync refused: requested device does not match the local replica identity');
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
// Check the local replica before a remote init can create a branch, table,
|
|
108
|
+
// or bucket prefix. A copied data directory must not gain write access by
|
|
109
|
+
// merely starting a sync run.
|
|
110
|
+
if (opts.push && !opts.dryRun) store.assertCanPublish();
|
|
111
|
+
await remote.init({ create: opts.push && !opts.dryRun });
|
|
112
|
+
|
|
113
|
+
// ── Push ──────────────────────────────────────────────────────────
|
|
114
|
+
if (opts.push) {
|
|
115
|
+
const pending = await store.loadPendingBatches();
|
|
116
|
+
report.pending = pending.length;
|
|
117
|
+
if (!opts.dryRun) {
|
|
118
|
+
for (const pendingBatch of pending) {
|
|
119
|
+
validateBatches([pendingBatch.batch], store.projectId(), remote.namespace);
|
|
120
|
+
if (pendingBatch.batch.deviceId !== opts.deviceId) {
|
|
121
|
+
throw new Error('[memorix] sync outbox rejected: batch belongs to another device');
|
|
122
|
+
}
|
|
123
|
+
await remote.push(pendingBatch.batch);
|
|
124
|
+
await store.saveState(pendingBatch.states);
|
|
125
|
+
await store.markBatchShipped(pendingBatch.batch.sequence);
|
|
126
|
+
report.pushed += pendingBatch.batch.entries.length;
|
|
127
|
+
}
|
|
128
|
+
report.pending = 0;
|
|
129
|
+
}
|
|
130
|
+
const current = await store.loadAll();
|
|
131
|
+
const filtered = eligibleObservations(current, store.projectId());
|
|
132
|
+
report.eligible = filtered.eligible.length;
|
|
133
|
+
report.excluded = filtered.excluded;
|
|
134
|
+
const state = await store.loadState();
|
|
135
|
+
const changes = computeChanges({
|
|
136
|
+
current: filtered.eligible,
|
|
137
|
+
state,
|
|
138
|
+
deviceId: opts.deviceId,
|
|
139
|
+
projectId: store.projectId(),
|
|
140
|
+
});
|
|
141
|
+
if (changes.length > 0) {
|
|
142
|
+
const chunks = chunkChanges(changes);
|
|
143
|
+
for (const chunk of chunks) {
|
|
144
|
+
const sequence = opts.dryRun ? 0 : await store.nextSequence();
|
|
145
|
+
const batch: ChangeBatch = {
|
|
146
|
+
formatVersion: 3,
|
|
147
|
+
namespace: remote.namespace,
|
|
148
|
+
projectId: store.projectId(),
|
|
149
|
+
deviceId: opts.deviceId,
|
|
150
|
+
sequence,
|
|
151
|
+
producedAt: nowIso(),
|
|
152
|
+
entries: chunk.map((c) => c.entry),
|
|
153
|
+
};
|
|
154
|
+
if (!opts.dryRun) validateBatches([batch], store.projectId(), remote.namespace);
|
|
155
|
+
report.pushed += chunk.length;
|
|
156
|
+
if (!opts.dryRun) {
|
|
157
|
+
const states = chunk.map((c) => ({ ...c.nextState, shippedSeq: sequence }));
|
|
158
|
+
await store.enqueueBatch({ batch, states });
|
|
159
|
+
await remote.push(batch);
|
|
160
|
+
await store.saveState(states);
|
|
161
|
+
await store.markBatchShipped(sequence);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// ── Pull ──────────────────────────────────────────────────────────
|
|
168
|
+
if (opts.pull) {
|
|
169
|
+
const cursor = await store.loadCursor();
|
|
170
|
+
const state = await store.loadState();
|
|
171
|
+
const pullSince = { ...cursor.applied };
|
|
172
|
+
let pageToken: string | undefined;
|
|
173
|
+
let hasMore = true;
|
|
174
|
+
while (hasMore) {
|
|
175
|
+
// Keep the baseline fixed for every page. Adapters use pageToken to
|
|
176
|
+
// advance through one logical pull; passing the mutating cursor here
|
|
177
|
+
// can skip later rows after page one.
|
|
178
|
+
const page = await remote.pull(pullSince, PULL_PAGE_SIZE, pageToken);
|
|
179
|
+
report.pulledBatches += page.batches.length;
|
|
180
|
+
validateBatches(page.batches, store.projectId(), remote.namespace);
|
|
181
|
+
|
|
182
|
+
for (const batch of page.batches) {
|
|
183
|
+
if (batch.deviceId === opts.deviceId) {
|
|
184
|
+
cursor.applied[batch.deviceId] = Math.max(cursor.applied[batch.deviceId] ?? 0, batch.sequence);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
for (const entry of batch.entries) {
|
|
188
|
+
const local = state.get(entry.syncKey);
|
|
189
|
+
const incomingHash = entry.row ? contentHash(entry.row) : '';
|
|
190
|
+
if (entry.contentHash !== incomingHash) {
|
|
191
|
+
throw new Error('[memorix] sync batch rejected: content hash mismatch');
|
|
192
|
+
}
|
|
193
|
+
if (
|
|
194
|
+
local
|
|
195
|
+
&& compareVersion(entry.version, { revision: local.revision, writer: local.writer }) === 0
|
|
196
|
+
&& local.contentHash !== incomingHash
|
|
197
|
+
) {
|
|
198
|
+
throw new Error('[memorix] sync batch rejected: same version has different content');
|
|
199
|
+
}
|
|
200
|
+
const { decision, action } = mergeOne({ incoming: entry, local, incomingHash });
|
|
201
|
+
recordDecision(report, decision);
|
|
202
|
+
if (local && entry.version.revision === local.revision && entry.version.writer !== local.writer) {
|
|
203
|
+
const currentRow = local.obsId == null ? undefined : await store.getById(local.obsId);
|
|
204
|
+
if (opts.dryRun) report.conflicts++;
|
|
205
|
+
else await recordConcurrentConflict(store, report, local, entry, action.type === 'noop' ? entry.row : currentRow);
|
|
206
|
+
}
|
|
207
|
+
if (opts.dryRun || action.type === 'noop') {
|
|
208
|
+
if (!opts.dryRun && action.type === 'noop' && decision.outcome === 'apply') {
|
|
209
|
+
const next: SyncRowState = {
|
|
210
|
+
projectId: store.projectId(),
|
|
211
|
+
syncKey: entry.syncKey,
|
|
212
|
+
obsId: null,
|
|
213
|
+
revision: entry.version.revision,
|
|
214
|
+
writer: entry.version.writer,
|
|
215
|
+
kind: 'tombstone',
|
|
216
|
+
contentHash: '',
|
|
217
|
+
shippedSeq: local?.shippedSeq ?? 0,
|
|
218
|
+
};
|
|
219
|
+
state.set(entry.syncKey, next);
|
|
220
|
+
await store.saveState([next]);
|
|
221
|
+
}
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
let next: SyncRowState;
|
|
226
|
+
if (action.type === 'insert') {
|
|
227
|
+
const newId = await store.allocateId();
|
|
228
|
+
const row = { ...entry.row!, id: newId };
|
|
229
|
+
next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: newId, revision: action.version.revision, writer: action.version.writer, kind: 'upsert', contentHash: action.contentHash, shippedSeq: local?.shippedSeq ?? 0 };
|
|
230
|
+
await store.applyInsert(row, next);
|
|
231
|
+
} else if (action.type === 'update') {
|
|
232
|
+
const row = { ...entry.row!, id: action.obsId };
|
|
233
|
+
next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: action.obsId, revision: action.version.revision, writer: action.version.writer, kind: 'upsert', contentHash: action.contentHash, shippedSeq: local?.shippedSeq ?? 0 };
|
|
234
|
+
await store.applyUpdate(row, next);
|
|
235
|
+
} else {
|
|
236
|
+
next = { projectId: store.projectId(), syncKey: entry.syncKey, obsId: null, revision: action.version.revision, writer: action.version.writer, kind: 'tombstone', contentHash: '', shippedSeq: local?.shippedSeq ?? 0 };
|
|
237
|
+
await store.applyRemove(action.obsId, next);
|
|
238
|
+
}
|
|
239
|
+
state.set(entry.syncKey, next);
|
|
240
|
+
await store.saveState([next]);
|
|
241
|
+
}
|
|
242
|
+
cursor.applied[batch.deviceId] = Math.max(cursor.applied[batch.deviceId] ?? 0, batch.sequence);
|
|
243
|
+
}
|
|
244
|
+
hasMore = page.hasMore;
|
|
245
|
+
pageToken = page.nextPageToken;
|
|
246
|
+
if (!opts.dryRun && page.batches.length > 0) await store.saveCursor(cursor);
|
|
247
|
+
if (page.batches.length === 0 && !page.nextPageToken) hasMore = false;
|
|
248
|
+
if (hasMore && !page.nextPageToken) throw new Error('[memorix] sync remote returned an unpaged continuation');
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return report;
|
|
252
|
+
} finally {
|
|
253
|
+
await remote.close();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function validateBatches(batches: ChangeBatch[], projectId: string, namespace: string): void {
|
|
258
|
+
for (const batch of batches) {
|
|
259
|
+
if (batch.formatVersion !== 3 || batch.projectId !== projectId || batch.namespace !== namespace) {
|
|
260
|
+
throw new Error('[memorix] sync batch rejected: wrong format, project, or remote namespace');
|
|
261
|
+
}
|
|
262
|
+
if (
|
|
263
|
+
!isSafeSyncDeviceId(batch.deviceId)
|
|
264
|
+
|| !Number.isSafeInteger(batch.sequence)
|
|
265
|
+
|| batch.sequence < 1
|
|
266
|
+
|| !Array.isArray(batch.entries)
|
|
267
|
+
|| Buffer.byteLength(JSON.stringify(batch), 'utf8') > MAX_BATCH_BYTES
|
|
268
|
+
) {
|
|
269
|
+
throw new Error('[memorix] sync batch rejected: malformed envelope');
|
|
270
|
+
}
|
|
271
|
+
for (const entry of batch.entries) {
|
|
272
|
+
if (
|
|
273
|
+
!entry.syncKey
|
|
274
|
+
|| (entry.kind !== 'upsert' && entry.kind !== 'tombstone')
|
|
275
|
+
|| !entry.version
|
|
276
|
+
|| !Number.isSafeInteger(entry.version.revision)
|
|
277
|
+
|| entry.version.revision < 1
|
|
278
|
+
|| !entry.version.writer
|
|
279
|
+
|| typeof entry.contentHash !== 'string'
|
|
280
|
+
|| (entry.kind === 'tombstone' && (entry.row !== undefined || entry.contentHash !== ''))
|
|
281
|
+
|| (entry.kind === 'upsert' && (!/^[a-f0-9]{64}$/.test(entry.contentHash) || entry.row === undefined))
|
|
282
|
+
) {
|
|
283
|
+
throw new Error('[memorix] sync batch rejected: malformed change entry');
|
|
284
|
+
}
|
|
285
|
+
if (
|
|
286
|
+
entry.kind === 'upsert'
|
|
287
|
+
&& (!Number.isSafeInteger(entry.row!.id) || entry.row!.id < 1 || !eligibleObservations([entry.row!], projectId).eligible.length)
|
|
288
|
+
) {
|
|
289
|
+
throw new Error('[memorix] sync batch rejected: ineligible observation payload');
|
|
290
|
+
}
|
|
291
|
+
const incomingHash = entry.row ? contentHash(entry.row) : '';
|
|
292
|
+
if (entry.contentHash !== incomingHash) {
|
|
293
|
+
throw new Error('[memorix] sync batch rejected: content hash mismatch');
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function chunkChanges<T extends { entry: ChangeBatch['entries'][number] }>(changes: T[]): T[][] {
|
|
300
|
+
const chunks: T[][] = [];
|
|
301
|
+
let current: T[] = [];
|
|
302
|
+
let currentBytes = 2;
|
|
303
|
+
for (const change of changes) {
|
|
304
|
+
const bytes = Buffer.byteLength(JSON.stringify(change.entry), 'utf8') + 1;
|
|
305
|
+
if (bytes + 2 > MAX_BATCH_BYTES) throw new Error('[memorix] one sync event is too large for the relay');
|
|
306
|
+
if (current.length > 0 && currentBytes + bytes > MAX_BATCH_BYTES) {
|
|
307
|
+
chunks.push(current);
|
|
308
|
+
current = [];
|
|
309
|
+
currentBytes = 2;
|
|
310
|
+
}
|
|
311
|
+
current.push(change);
|
|
312
|
+
currentBytes += bytes;
|
|
313
|
+
}
|
|
314
|
+
if (current.length > 0) chunks.push(current);
|
|
315
|
+
return chunks;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async function recordConcurrentConflict(
|
|
319
|
+
store: SyncStorePort,
|
|
320
|
+
report: SyncReport,
|
|
321
|
+
local: SyncRowState,
|
|
322
|
+
incoming: ChangeBatch['entries'][number],
|
|
323
|
+
localRow: Observation | undefined,
|
|
324
|
+
): Promise<void> {
|
|
325
|
+
const incomingWins = compareVersion(incoming.version, { revision: local.revision, writer: local.writer }) > 0;
|
|
326
|
+
const loser = incomingWins ? { revision: local.revision, writer: local.writer } : incoming.version;
|
|
327
|
+
const loserRow = incomingWins ? localRow : incoming.row;
|
|
328
|
+
const id = createHash('sha256')
|
|
329
|
+
.update(`${store.projectId()}|${incoming.syncKey}|${local.revision}|${local.writer}|${incoming.version.revision}|${incoming.version.writer}`)
|
|
330
|
+
.digest('hex');
|
|
331
|
+
await store.recordConflict({
|
|
332
|
+
id,
|
|
333
|
+
projectId: store.projectId(),
|
|
334
|
+
syncKey: incoming.syncKey,
|
|
335
|
+
winner: incomingWins ? incoming.version : { revision: local.revision, writer: local.writer },
|
|
336
|
+
loser,
|
|
337
|
+
loserKind: incomingWins ? local.kind : incoming.kind,
|
|
338
|
+
loserRow,
|
|
339
|
+
createdAt: nowIso(),
|
|
340
|
+
reason: 'concurrent versions shared the same logical revision',
|
|
341
|
+
});
|
|
342
|
+
report.conflicts++;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function recordDecision(report: SyncReport, decision: SyncDecision): void {
|
|
346
|
+
report.decisions.push(decision);
|
|
347
|
+
if (decision.outcome === 'apply') {
|
|
348
|
+
if (decision.kind === 'tombstone') report.tombstoned++;
|
|
349
|
+
else report.applied++;
|
|
350
|
+
} else {
|
|
351
|
+
report.skipped++;
|
|
352
|
+
}
|
|
353
|
+
}
|