eklavya 1.48.3 → 1.48.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/cli-memory.js +11 -2
- package/dist/cli-memory.js.map +1 -1
- package/dist/memory/sync.js +392 -44
- package/dist/memory/sync.js.map +1 -1
- package/dist/migrations/022_sync_supersessions.sql +20 -0
- package/dist/migrations/023_sync_received.sql +34 -0
- package/dist/plugin/.claude-plugin/plugin.json +1 -1
- package/dist/plugin/skills/memory/SKILL.md +5 -2
- package/dist/tools/memory_read_tools.js +17 -3
- package/dist/tools/memory_read_tools.js.map +1 -1
- package/dist/tools/memory_write_tools.js +69 -36
- package/dist/tools/memory_write_tools.js.map +1 -1
- package/package.json +1 -1
package/dist/memory/sync.js
CHANGED
|
@@ -24,6 +24,14 @@ import { deleteEntry, indexVector } from './store.js';
|
|
|
24
24
|
* not cover, such as a cloud client materialising a partial file under the real
|
|
25
25
|
* name.
|
|
26
26
|
*
|
|
27
|
+
* **Files arrive out of order.** A synced folder can deliver revision 2 before
|
|
28
|
+
* revision 1, so a reader's high-water mark is a floor -- every revision at or
|
|
29
|
+
* below it is accounted for -- and never jumps a gap. A revision applied above
|
|
30
|
+
* the floor is remembered individually (`sync_received`) until the gap fills.
|
|
31
|
+
* Some gaps never fill: a revision staged and then superseded before its file
|
|
32
|
+
* was written is gone. The writer says so in `void.json` beside its records, and
|
|
33
|
+
* the reader's floor steps over a voided revision as if it had been applied.
|
|
34
|
+
*
|
|
27
35
|
* **What crosses, and what must not.** Memory entries, their tags and their
|
|
28
36
|
* tombstones. Not attempts, not mastery, not gates, not receipts, not raw
|
|
29
37
|
* `evidence_events` (PRD SEC-02). A team that points every member at one shared
|
|
@@ -40,7 +48,10 @@ export const SYNC_FORMAT_VERSION = 1;
|
|
|
40
48
|
* An allowlist rather than `SELECT *`, so a later migration adding a column to
|
|
41
49
|
* `memory_entries` does not silently start publishing it. `session_id`,
|
|
42
50
|
* `batch_id` and `superseded_by` are left out for a different reason: they are
|
|
43
|
-
* local row ids and mean nothing on the other device.
|
|
51
|
+
* local row ids and mean nothing on the other device. A correction still
|
|
52
|
+
* crosses: the record carries the replacement's `entry_uid` beside the payload
|
|
53
|
+
* (`SyncRecord.superseded_by`), and the receiving device resolves it to its own
|
|
54
|
+
* row id.
|
|
44
55
|
*/
|
|
45
56
|
const ENTRY_COLUMNS = [
|
|
46
57
|
'project',
|
|
@@ -56,6 +67,18 @@ const ENTRY_COLUMNS = [
|
|
|
56
67
|
'created_at',
|
|
57
68
|
'import_source',
|
|
58
69
|
];
|
|
70
|
+
/**
|
|
71
|
+
* The columns a version is staged from, with `superseded_by` translated to the
|
|
72
|
+
* replacement's uid: a resolved link reads it from the replacement's row, and
|
|
73
|
+
* a link still waiting for its replacement to arrive reads it from
|
|
74
|
+
* `sync_supersessions`. Either way this device states the correction it holds,
|
|
75
|
+
* so a push never publishes a corrected entry as live again.
|
|
76
|
+
*/
|
|
77
|
+
const STAGE_COLUMNS = `e.entry_uid, e.deleted_at, ${ENTRY_COLUMNS.map((c) => `e.${c}`).join(', ')},
|
|
78
|
+
COALESCE(
|
|
79
|
+
(SELECT r.entry_uid FROM memory_entries r WHERE r.id = e.superseded_by),
|
|
80
|
+
(SELECT p.replacement_uid FROM sync_supersessions p WHERE p.entry_uid = e.entry_uid)
|
|
81
|
+
) AS superseded_by_uid`;
|
|
59
82
|
const DEVICE_ID_KEY = 'sync_device_id';
|
|
60
83
|
/**
|
|
61
84
|
* This install's identity, generated once.
|
|
@@ -109,12 +132,22 @@ function guard(db, config, override) {
|
|
|
109
132
|
// ---------------------------------------------------------------------------
|
|
110
133
|
// Hashing and the version vector
|
|
111
134
|
// ---------------------------------------------------------------------------
|
|
112
|
-
/**
|
|
113
|
-
|
|
135
|
+
/**
|
|
136
|
+
* Content identity of a version. Equal hashes mean nothing has to be sent.
|
|
137
|
+
*
|
|
138
|
+
* A correction is part of the content: superseding an entry changes its hash,
|
|
139
|
+
* so the push scan stages it and the peer learns of it. The replacement's uid
|
|
140
|
+
* is hashed only when there is one, so an uncorrected entry keeps the hash it
|
|
141
|
+
* had before corrections travelled and an upgrade re-sends nothing else.
|
|
142
|
+
*/
|
|
143
|
+
function payloadHash(op, entry, tags, supersededBy = null) {
|
|
114
144
|
const h = crypto.createHash('sha256');
|
|
115
145
|
// Canonical by construction: ENTRY_COLUMNS is ordered and tags are sorted, so
|
|
116
146
|
// two devices holding the same content always agree on the hash.
|
|
117
|
-
|
|
147
|
+
const parts = [op, entry ? ENTRY_COLUMNS.map((c) => entry[c] ?? null) : null, [...tags].sort()];
|
|
148
|
+
if (supersededBy)
|
|
149
|
+
parts.push(supersededBy);
|
|
150
|
+
h.update(JSON.stringify(parts));
|
|
118
151
|
return h.digest('hex').slice(0, 32);
|
|
119
152
|
}
|
|
120
153
|
function ref(device, revision) {
|
|
@@ -146,7 +179,14 @@ function writeRecord(db, rec) {
|
|
|
146
179
|
op = excluded.op, hash = excluded.hash, updated_at = excluded.updated_at`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, rec.op, rec.hash, nowIso());
|
|
147
180
|
}
|
|
148
181
|
function tombstone(entryUid) {
|
|
149
|
-
return {
|
|
182
|
+
return {
|
|
183
|
+
entry_uid: entryUid,
|
|
184
|
+
op: 'delete',
|
|
185
|
+
entry: null,
|
|
186
|
+
tags: [],
|
|
187
|
+
superseded_by: null,
|
|
188
|
+
hash: payloadHash('delete', null, []),
|
|
189
|
+
};
|
|
150
190
|
}
|
|
151
191
|
function tagsOf(db, entryUid) {
|
|
152
192
|
return db
|
|
@@ -158,16 +198,22 @@ function stage(db, row) {
|
|
|
158
198
|
return tombstone(row.entry_uid);
|
|
159
199
|
const entry = Object.fromEntries(ENTRY_COLUMNS.map((c) => [c, row[c] ?? null]));
|
|
160
200
|
const tags = tagsOf(db, row.entry_uid);
|
|
161
|
-
|
|
201
|
+
const supersededBy = row.superseded_by_uid;
|
|
202
|
+
return {
|
|
203
|
+
entry_uid: row.entry_uid,
|
|
204
|
+
op: 'upsert',
|
|
205
|
+
entry,
|
|
206
|
+
tags,
|
|
207
|
+
superseded_by: supersededBy,
|
|
208
|
+
hash: payloadHash('upsert', entry, tags, supersededBy),
|
|
209
|
+
};
|
|
162
210
|
}
|
|
163
211
|
/**
|
|
164
212
|
* What this device holds for one entry right now, whether or not it has been
|
|
165
213
|
* staged. Null when the entry is unknown here in every sense.
|
|
166
214
|
*/
|
|
167
215
|
function localContent(db, entryUid) {
|
|
168
|
-
const row = db
|
|
169
|
-
.prepare(`SELECT entry_uid, deleted_at, ${ENTRY_COLUMNS.join(', ')} FROM memory_entries WHERE entry_uid = ?`)
|
|
170
|
-
.get(entryUid);
|
|
216
|
+
const row = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e WHERE e.entry_uid = ?`).get(entryUid);
|
|
171
217
|
if (row)
|
|
172
218
|
return stage(db, row);
|
|
173
219
|
// No row, but a record that outlived it: a hard delete. Still a tombstone.
|
|
@@ -183,9 +229,7 @@ function localContent(db, entryUid) {
|
|
|
183
229
|
* shows up in a profile.
|
|
184
230
|
*/
|
|
185
231
|
function scanLocal(db) {
|
|
186
|
-
const rows = db
|
|
187
|
-
.prepare(`SELECT entry_uid, deleted_at, ${ENTRY_COLUMNS.join(', ')} FROM memory_entries`)
|
|
188
|
-
.all();
|
|
232
|
+
const rows = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e`).all();
|
|
189
233
|
const out = rows
|
|
190
234
|
.map((row) => stage(db, row))
|
|
191
235
|
.filter((s) => localRecord(db, s.entry_uid)?.hash !== s.hash);
|
|
@@ -209,6 +253,7 @@ function deviceDir(target, device) {
|
|
|
209
253
|
// segment under `<target>/devices`, and a peer's directory name arrives from
|
|
210
254
|
// `readdirSync` rather than from a record, but a single assertion at the one
|
|
211
255
|
// place paths are built is cheaper than trusting both.
|
|
256
|
+
/* c8 ignore next -- both sources are already checked: `deviceId` for ours, `peerDevices` for theirs */
|
|
212
257
|
if (!DEVICE_ID.test(device))
|
|
213
258
|
throw new Error(`unsafe device id: ${JSON.stringify(device)}`);
|
|
214
259
|
return path.join(target, 'devices', device);
|
|
@@ -223,6 +268,113 @@ function writeAtomic(file, text) {
|
|
|
223
268
|
fs.writeFileSync(tmp, text, 'utf8');
|
|
224
269
|
fs.renameSync(tmp, file);
|
|
225
270
|
}
|
|
271
|
+
const RECORD_NAME = /^\d{12}\.json$/;
|
|
272
|
+
/** iCloud Drive's stand-in for a file it has evicted to save space: still there. */
|
|
273
|
+
const EVICTED_NAME = /^\.(\d{12}\.json)\.icloud$/;
|
|
274
|
+
const VOID_FILE = 'void.json';
|
|
275
|
+
/**
|
|
276
|
+
* Revisions this device handed out and will never write: no file in the
|
|
277
|
+
* target, and no record left to build one from.
|
|
278
|
+
*
|
|
279
|
+
* That happens when an entry is staged, the push is interrupted before its file
|
|
280
|
+
* lands, and the entry changes again -- the next push stages a new revision and
|
|
281
|
+
* the old one's record is overwritten. The content it carried is superseded
|
|
282
|
+
* here, so nothing is lost by declaring it; what matters is that a reader stops
|
|
283
|
+
* waiting for it. The set only grows: a revision number is never handed out
|
|
284
|
+
* twice, and a file is only ever written for a record that still exists.
|
|
285
|
+
*/
|
|
286
|
+
function abandonedRevisions(db, target, device) {
|
|
287
|
+
let names;
|
|
288
|
+
try {
|
|
289
|
+
names = fs.readdirSync(deviceDir(target, device));
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
/* c8 ignore next -- push creates the directory before this runs, unless the target vanished mid-push */
|
|
293
|
+
return [];
|
|
294
|
+
}
|
|
295
|
+
const present = new Set();
|
|
296
|
+
for (const n of names) {
|
|
297
|
+
const name = EVICTED_NAME.exec(n)?.[1] ?? n;
|
|
298
|
+
if (RECORD_NAME.test(name))
|
|
299
|
+
present.add(Number(name.slice(0, 12)));
|
|
300
|
+
}
|
|
301
|
+
const held = new Set(db.prepare('SELECT revision FROM sync_records WHERE device_id = ?').all(device).map((r) => r.revision));
|
|
302
|
+
const out = [];
|
|
303
|
+
for (let r = 1; r <= highWater(db, device); r += 1) {
|
|
304
|
+
if (!present.has(r) && !held.has(r))
|
|
305
|
+
out.push(r);
|
|
306
|
+
}
|
|
307
|
+
return out;
|
|
308
|
+
}
|
|
309
|
+
/** `[1, 2, 3, 7]` as `[[1, 3], [7, 7]]`: a void list stays small however old the device. */
|
|
310
|
+
function toRanges(revisions) {
|
|
311
|
+
const ranges = [];
|
|
312
|
+
for (const r of revisions) {
|
|
313
|
+
const last = ranges[ranges.length - 1];
|
|
314
|
+
if (last && last[1] === r - 1)
|
|
315
|
+
last[1] = r;
|
|
316
|
+
else
|
|
317
|
+
ranges.push([r, r]);
|
|
318
|
+
}
|
|
319
|
+
return ranges;
|
|
320
|
+
}
|
|
321
|
+
/** Publishes the void list, rewriting it only when it changed. */
|
|
322
|
+
function writeVoids(target, device, revisions) {
|
|
323
|
+
if (!revisions.length)
|
|
324
|
+
return;
|
|
325
|
+
const file = path.join(deviceDir(target, device), VOID_FILE);
|
|
326
|
+
const text = `${JSON.stringify({ v: SYNC_FORMAT_VERSION, device_id: device, revisions: toRanges(revisions) })}\n`;
|
|
327
|
+
try {
|
|
328
|
+
if (fs.readFileSync(file, 'utf8') === text)
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
// Not written yet.
|
|
333
|
+
}
|
|
334
|
+
writeAtomic(file, text);
|
|
335
|
+
}
|
|
336
|
+
/** Larger than any honest list: a void list holds ranges, not one line per record. */
|
|
337
|
+
const VOID_MAX_BYTES = 1024 * 1024;
|
|
338
|
+
/**
|
|
339
|
+
* A peer's voided revisions above `floor`, as sorted, merged ranges, or none if
|
|
340
|
+
* its list is missing, oversized or unreadable. A torn list is harmless: every
|
|
341
|
+
* revision it does name is void, and one it fails to name is only waited for a
|
|
342
|
+
* little longer. Ranges stay ranges -- a list claiming `[1, 1e12]` costs one
|
|
343
|
+
* pair, not a trillion set entries.
|
|
344
|
+
*/
|
|
345
|
+
function readVoids(target, device, floor) {
|
|
346
|
+
const file = path.join(deviceDir(target, device), VOID_FILE);
|
|
347
|
+
let parsed;
|
|
348
|
+
try {
|
|
349
|
+
if (fs.statSync(file).size > VOID_MAX_BYTES)
|
|
350
|
+
return [];
|
|
351
|
+
parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
return [];
|
|
355
|
+
}
|
|
356
|
+
if (parsed?.v !== SYNC_FORMAT_VERSION || parsed.device_id !== device || !Array.isArray(parsed.revisions)) {
|
|
357
|
+
return [];
|
|
358
|
+
}
|
|
359
|
+
const ranges = [];
|
|
360
|
+
for (const range of parsed.revisions) {
|
|
361
|
+
if (!Array.isArray(range) || !Number.isSafeInteger(range[0]) || !Number.isSafeInteger(range[1]))
|
|
362
|
+
continue;
|
|
363
|
+
const lo = Math.max(range[0], floor + 1);
|
|
364
|
+
if (lo <= range[1])
|
|
365
|
+
ranges.push([lo, range[1]]);
|
|
366
|
+
}
|
|
367
|
+
ranges.sort((a, b) => a[0] - b[0]);
|
|
368
|
+
const merged = [];
|
|
369
|
+
for (const [lo, hi] of ranges) {
|
|
370
|
+
const last = merged[merged.length - 1];
|
|
371
|
+
if (last && lo <= last[1] + 1)
|
|
372
|
+
last[1] = Math.max(last[1], hi);
|
|
373
|
+
else
|
|
374
|
+
merged.push([lo, hi]);
|
|
375
|
+
}
|
|
376
|
+
return merged;
|
|
377
|
+
}
|
|
226
378
|
// ---------------------------------------------------------------------------
|
|
227
379
|
// Push
|
|
228
380
|
// ---------------------------------------------------------------------------
|
|
@@ -268,8 +420,8 @@ export function push(db, config, opts = {}) {
|
|
|
268
420
|
const body = payloads.get(rec.entry_uid);
|
|
269
421
|
// A record whose payload this run did not rebuild predates the interruption
|
|
270
422
|
// that lost its file; it is rebuilt from the database below, and dropped if
|
|
271
|
-
// the entry
|
|
272
|
-
//
|
|
423
|
+
// the entry has changed since -- the peer waits on the gap until the next
|
|
424
|
+
// push re-stages the entry and the old revision is declared void.
|
|
273
425
|
const rebuilt = body ?? rebuildPayload(db, rec);
|
|
274
426
|
if (!rebuilt)
|
|
275
427
|
continue;
|
|
@@ -284,28 +436,26 @@ export function push(db, config, opts = {}) {
|
|
|
284
436
|
written_at: nowIso(),
|
|
285
437
|
entry: rebuilt.entry,
|
|
286
438
|
tags: rebuilt.tags,
|
|
439
|
+
...(rebuilt.superseded_by ? { superseded_by: rebuilt.superseded_by } : {}),
|
|
287
440
|
};
|
|
288
441
|
writeAtomic(file, `${JSON.stringify(record)}\n`);
|
|
289
442
|
written += 1;
|
|
290
443
|
}
|
|
444
|
+
writeVoids(g.target, g.device, abandonedRevisions(db, g.target, g.device));
|
|
291
445
|
return { ok: true, device_id: g.device, target: g.target, staged: staged.length, written, already };
|
|
292
446
|
}
|
|
293
447
|
function rebuildPayload(db, rec) {
|
|
294
448
|
if (rec.op === 'delete')
|
|
295
|
-
return { entry: null, tags: [] };
|
|
296
|
-
const row = db
|
|
297
|
-
.prepare(`SELECT ${ENTRY_COLUMNS.join(', ')} FROM memory_entries WHERE entry_uid = ?`)
|
|
298
|
-
.get(rec.entry_uid);
|
|
449
|
+
return { entry: null, tags: [], superseded_by: null };
|
|
450
|
+
const row = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e WHERE e.entry_uid = ?`).get(rec.entry_uid);
|
|
299
451
|
/* c8 ignore next -- unreachable: an upsert record whose row is gone is re-staged as an orphan tombstone first */
|
|
300
452
|
if (!row)
|
|
301
453
|
return null;
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const entry = Object.fromEntries(ENTRY_COLUMNS.map((c) => [c, row[c] ?? null]));
|
|
306
|
-
if (payloadHash('upsert', entry, tags) !== rec.hash)
|
|
454
|
+
const staged = stage(db, row);
|
|
455
|
+
// A row deleted since stages as a tombstone, which is not this record either.
|
|
456
|
+
if (staged.op !== rec.op || staged.hash !== rec.hash)
|
|
307
457
|
return null;
|
|
308
|
-
return { entry, tags };
|
|
458
|
+
return { entry: staged.entry, tags: staged.tags, superseded_by: staged.superseded_by };
|
|
309
459
|
}
|
|
310
460
|
// ---------------------------------------------------------------------------
|
|
311
461
|
// Pull
|
|
@@ -342,9 +492,10 @@ function readRecord(file, device, revision) {
|
|
|
342
492
|
const entry = rec.op === 'delete' ? null : sanitizeEntry(rec.entry);
|
|
343
493
|
if (rec.op === 'upsert' && !entry)
|
|
344
494
|
return null;
|
|
345
|
-
|
|
495
|
+
const supersededBy = rec.op === 'upsert' && typeof rec.superseded_by === 'string' && rec.superseded_by ? rec.superseded_by : null;
|
|
496
|
+
if (payloadHash(rec.op, entry, tags, supersededBy) !== rec.hash)
|
|
346
497
|
return null;
|
|
347
|
-
return { ...rec, entry, tags, base: rec.base ?? null };
|
|
498
|
+
return { ...rec, entry, tags, superseded_by: supersededBy, base: rec.base ?? null };
|
|
348
499
|
}
|
|
349
500
|
/**
|
|
350
501
|
* The inbound half of the allowlist. Anything a peer put in the payload that is
|
|
@@ -365,7 +516,8 @@ function peerDevices(target, local) {
|
|
|
365
516
|
try {
|
|
366
517
|
return fs
|
|
367
518
|
.readdirSync(root, { withFileTypes: true })
|
|
368
|
-
|
|
519
|
+
// A directory whose name is not a device id was not written by Eklavya.
|
|
520
|
+
.filter((d) => d.isDirectory() && d.name !== local && DEVICE_ID.test(d.name))
|
|
369
521
|
.map((d) => d.name)
|
|
370
522
|
.sort();
|
|
371
523
|
}
|
|
@@ -377,48 +529,183 @@ function pendingRecordFiles(target, device, after) {
|
|
|
377
529
|
let names;
|
|
378
530
|
try {
|
|
379
531
|
names = fs.readdirSync(deviceDir(target, device));
|
|
532
|
+
/* c8 ignore start -- `peerDevices` just listed this directory; only a folder removed mid-pull lands here */
|
|
380
533
|
}
|
|
381
534
|
catch {
|
|
382
535
|
return [];
|
|
383
536
|
}
|
|
537
|
+
/* c8 ignore stop */
|
|
384
538
|
return names
|
|
385
|
-
.filter((n) =>
|
|
539
|
+
.filter((n) => RECORD_NAME.test(n))
|
|
386
540
|
.map((n) => ({ revision: Number(n.slice(0, 12)), file: path.join(deviceDir(target, device), n) }))
|
|
387
541
|
.filter((r) => r.revision > after)
|
|
388
542
|
.sort((a, b) => a.revision - b.revision);
|
|
389
543
|
}
|
|
544
|
+
/** Revisions from `device` applied above its floor, waiting for the gap below to fill. */
|
|
545
|
+
function receivedAbove(db, device, floor) {
|
|
546
|
+
return new Set(db.prepare('SELECT revision FROM sync_received WHERE device_id = ? AND revision > ?').all(device, floor).map((r) => r.revision));
|
|
547
|
+
}
|
|
548
|
+
/** The void range holding `revision`, by binary search over sorted, disjoint ranges. */
|
|
549
|
+
function voidAt(voids, revision) {
|
|
550
|
+
let lo = 0;
|
|
551
|
+
let hi = voids.length - 1;
|
|
552
|
+
while (lo <= hi) {
|
|
553
|
+
const mid = (lo + hi) >> 1;
|
|
554
|
+
const [a, b] = voids[mid];
|
|
555
|
+
if (revision < a)
|
|
556
|
+
hi = mid - 1;
|
|
557
|
+
else if (revision > b)
|
|
558
|
+
lo = mid + 1;
|
|
559
|
+
else
|
|
560
|
+
return voids[mid];
|
|
561
|
+
}
|
|
562
|
+
return undefined;
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Moves a peer's floor up through every revision now accounted for, forgets
|
|
566
|
+
* the individual receipts it covers, and counts what is still owed above it.
|
|
567
|
+
*
|
|
568
|
+
* Accounted for means applied, or voided by its writer with no file present: a
|
|
569
|
+
* file that exists but could not be read yet beats the writer's claim that it
|
|
570
|
+
* never will, because stepping over it would lose it for good. Everything here
|
|
571
|
+
* is arithmetic over the receipts, the unread files and the void ranges -- never
|
|
572
|
+
* a walk over revision numbers, which a stray `999999999999.json` would make a
|
|
573
|
+
* walk of a trillion steps.
|
|
574
|
+
*/
|
|
575
|
+
function advanceFloor(db, device, floor, received, unread, voids) {
|
|
576
|
+
let newest = floor;
|
|
577
|
+
for (const r of received)
|
|
578
|
+
newest = Math.max(newest, r);
|
|
579
|
+
for (const r of unread)
|
|
580
|
+
newest = Math.max(newest, r);
|
|
581
|
+
// A void past the newest revision anyone has seen cannot be owed, so it is not
|
|
582
|
+
// stepped over either: a later file would otherwise land below the floor.
|
|
583
|
+
const clamped = voids.filter(([a]) => a <= newest).map(([a, b]) => [a, Math.min(b, newest)]);
|
|
584
|
+
const blockers = [...unread].sort((a, b) => a - b);
|
|
585
|
+
let next = floor;
|
|
586
|
+
for (;;) {
|
|
587
|
+
if (received.has(next + 1)) {
|
|
588
|
+
next += 1;
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
const range = voidAt(clamped, next + 1);
|
|
592
|
+
if (!range || unread.has(next + 1))
|
|
593
|
+
break;
|
|
594
|
+
const blocker = blockers.find((u) => u > next + 1 && u <= range[1]);
|
|
595
|
+
next = blocker ? blocker - 1 : range[1];
|
|
596
|
+
}
|
|
597
|
+
// Owed = every revision in (next, newest] that is not covered, where covered
|
|
598
|
+
// is received, or void with no unread file in its place.
|
|
599
|
+
const above = (r) => r > next && r <= newest;
|
|
600
|
+
let covered = 0;
|
|
601
|
+
for (const r of received)
|
|
602
|
+
if (above(r))
|
|
603
|
+
covered += 1;
|
|
604
|
+
for (const [a, b] of clamped) {
|
|
605
|
+
const lo = Math.max(a, next + 1);
|
|
606
|
+
if (lo <= b)
|
|
607
|
+
covered += b - lo + 1;
|
|
608
|
+
}
|
|
609
|
+
for (const r of [...received, ...unread])
|
|
610
|
+
if (above(r) && voidAt(clamped, r))
|
|
611
|
+
covered -= 1;
|
|
612
|
+
const missing = newest - next - covered;
|
|
613
|
+
db.transaction(() => {
|
|
614
|
+
setHighWater(db, device, next);
|
|
615
|
+
db.prepare('DELETE FROM sync_received WHERE device_id = ? AND revision <= ?').run(device, next);
|
|
616
|
+
db.prepare('UPDATE sync_state SET outstanding = ? WHERE device_id = ?').run(missing, device);
|
|
617
|
+
})();
|
|
618
|
+
// The floor stops only at a revision that is not covered, so it is the first owed.
|
|
619
|
+
return { missing, first: missing ? next + 1 : 0 };
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* The one-time recovery for an install upgraded from a release whose pull
|
|
623
|
+
* stepped over late records: every file at or below the mark it had then is
|
|
624
|
+
* read once more. Only what is unambiguous is applied -- an entry this device
|
|
625
|
+
* has never seen, or a plain fast-forward of the version it holds. Anything that
|
|
626
|
+
* would need a person to judge is left exactly as it is, because a record read
|
|
627
|
+
* now cannot be told from one the old pull already quarantined or superseded.
|
|
628
|
+
* It runs once: the old pull stopped at an unreadable record rather than
|
|
629
|
+
* stepping over it, so nothing below its mark is still waiting to be written.
|
|
630
|
+
*/
|
|
631
|
+
function replayEarlier(db, target, device) {
|
|
632
|
+
const through = db.prepare('SELECT replay_through FROM sync_state WHERE device_id = ?').get(device)?.replay_through;
|
|
633
|
+
if (!through)
|
|
634
|
+
return { applied: 0, tombstones: 0 };
|
|
635
|
+
let applied = 0;
|
|
636
|
+
let tombstones = 0;
|
|
637
|
+
for (const { revision, file } of pendingRecordFiles(target, device, 0)) {
|
|
638
|
+
if (revision > through)
|
|
639
|
+
break;
|
|
640
|
+
const rec = readRecord(file, device, revision);
|
|
641
|
+
if (!rec)
|
|
642
|
+
continue;
|
|
643
|
+
const outcome = db.transaction(() => applyRecord(db, rec, true))();
|
|
644
|
+
if (outcome === 'applied')
|
|
645
|
+
applied += 1;
|
|
646
|
+
else if (outcome === 'tombstone') {
|
|
647
|
+
applied += 1;
|
|
648
|
+
tombstones += 1;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
db.prepare('UPDATE sync_state SET replay_through = 0 WHERE device_id = ?').run(device);
|
|
652
|
+
return { applied, tombstones };
|
|
653
|
+
}
|
|
390
654
|
/**
|
|
391
655
|
* Applies what the peers wrote, one record at a time, in revision order.
|
|
392
656
|
*
|
|
393
|
-
* Each record is applied in its own transaction together with the
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
*
|
|
397
|
-
*
|
|
398
|
-
*
|
|
657
|
+
* Each record is applied in its own transaction together with the note that it
|
|
658
|
+
* was received, so an interrupted pull leaves a consistent database and resumes
|
|
659
|
+
* without applying anything twice. Order of arrival does not matter: a record
|
|
660
|
+
* that turns up after a later one is still read, because the floor never moves
|
|
661
|
+
* past a revision that has neither been applied nor voided by its writer. An
|
|
662
|
+
* unreadable record -- most likely one still being written -- is left for the
|
|
663
|
+
* next pull, and the records after it are still applied.
|
|
399
664
|
*/
|
|
400
665
|
export function pull(db, config, opts = {}) {
|
|
401
666
|
const g = guard(db, config, opts.target);
|
|
402
667
|
if (!g.ok) {
|
|
403
|
-
return {
|
|
668
|
+
return {
|
|
669
|
+
ok: false,
|
|
670
|
+
reason: g.reason,
|
|
671
|
+
applied: 0,
|
|
672
|
+
tombstones: 0,
|
|
673
|
+
skipped: 0,
|
|
674
|
+
conflicts: 0,
|
|
675
|
+
recovered: 0,
|
|
676
|
+
stalled: [],
|
|
677
|
+
outstanding: [],
|
|
678
|
+
};
|
|
404
679
|
}
|
|
405
680
|
let applied = 0;
|
|
406
681
|
let tombstones = 0;
|
|
407
682
|
let skipped = 0;
|
|
408
683
|
let conflicts = 0;
|
|
684
|
+
let recovered = 0;
|
|
409
685
|
const stalled = [];
|
|
686
|
+
const outstanding = [];
|
|
410
687
|
for (const device of peerDevices(g.target, g.device)) {
|
|
411
|
-
|
|
688
|
+
const replay = replayEarlier(db, g.target, device);
|
|
689
|
+
recovered += replay.applied;
|
|
690
|
+
applied += replay.applied;
|
|
691
|
+
tombstones += replay.tombstones;
|
|
692
|
+
const floor = highWater(db, device);
|
|
693
|
+
const received = receivedAbove(db, device, floor);
|
|
694
|
+
const unread = new Set();
|
|
695
|
+
for (const { revision, file } of pendingRecordFiles(g.target, device, floor)) {
|
|
696
|
+
if (received.has(revision))
|
|
697
|
+
continue;
|
|
412
698
|
const rec = readRecord(file, device, revision);
|
|
413
699
|
if (!rec) {
|
|
414
|
-
|
|
415
|
-
|
|
700
|
+
unread.add(revision);
|
|
701
|
+
continue;
|
|
416
702
|
}
|
|
417
703
|
const outcome = db.transaction(() => {
|
|
418
704
|
const result = applyRecord(db, rec);
|
|
419
|
-
|
|
705
|
+
db.prepare('INSERT OR IGNORE INTO sync_received (device_id, revision) VALUES (?, ?)').run(device, revision);
|
|
420
706
|
return result;
|
|
421
707
|
})();
|
|
708
|
+
received.add(revision);
|
|
422
709
|
if (outcome === 'applied')
|
|
423
710
|
applied += 1;
|
|
424
711
|
else if (outcome === 'tombstone') {
|
|
@@ -430,8 +717,24 @@ export function pull(db, config, opts = {}) {
|
|
|
430
717
|
else
|
|
431
718
|
skipped += 1;
|
|
432
719
|
}
|
|
720
|
+
if (unread.size)
|
|
721
|
+
stalled.push(device);
|
|
722
|
+
const owed = advanceFloor(db, device, floor, received, unread, readVoids(g.target, device, floor));
|
|
723
|
+
if (owed.missing)
|
|
724
|
+
outstanding.push({ device_id: device, ...owed });
|
|
433
725
|
}
|
|
434
|
-
return {
|
|
726
|
+
return {
|
|
727
|
+
ok: true,
|
|
728
|
+
device_id: g.device,
|
|
729
|
+
target: g.target,
|
|
730
|
+
applied,
|
|
731
|
+
tombstones,
|
|
732
|
+
skipped,
|
|
733
|
+
conflicts,
|
|
734
|
+
recovered,
|
|
735
|
+
stalled,
|
|
736
|
+
outstanding,
|
|
737
|
+
};
|
|
435
738
|
}
|
|
436
739
|
/**
|
|
437
740
|
* The merge rule, and the only place a remote version reaches local state.
|
|
@@ -448,10 +751,27 @@ export function pull(db, config, opts = {}) {
|
|
|
448
751
|
* edited the same entry, which no timestamp can adjudicate: the local version
|
|
449
752
|
* stays live and the incoming one is quarantined whole, for `repairConflict`.
|
|
450
753
|
*/
|
|
451
|
-
function applyRecord(db, rec) {
|
|
754
|
+
function applyRecord(db, rec, recovering = false) {
|
|
452
755
|
const held = localRecord(db, rec.entry_uid);
|
|
453
756
|
const local = localContent(db, rec.entry_uid);
|
|
757
|
+
// A version this one already moved past, delivered late: an earlier revision
|
|
758
|
+
// from the device that wrote what is held, or the version held one replaced.
|
|
759
|
+
// Checked first, so a late arrival can neither quarantine nor rename anything.
|
|
760
|
+
if (held &&
|
|
761
|
+
((held.device_id === rec.device_id && rec.revision <= held.revision) ||
|
|
762
|
+
held.base === ref(rec.device_id, rec.revision))) {
|
|
763
|
+
return 'skipped';
|
|
764
|
+
}
|
|
765
|
+
// Recovery applies only what needs no judgement: an entry with a quarantine
|
|
766
|
+
// open is already waiting on a person, and settling it here would retire that.
|
|
767
|
+
if (recovering &&
|
|
768
|
+
db.prepare('SELECT 1 FROM sync_conflicts WHERE entry_uid = ? AND resolved_at IS NULL').get(rec.entry_uid)) {
|
|
769
|
+
return 'skipped';
|
|
770
|
+
}
|
|
454
771
|
if (local && local.hash === rec.hash) {
|
|
772
|
+
// Renaming the agreed version to an old one is not unambiguous.
|
|
773
|
+
if (recovering && held)
|
|
774
|
+
return 'skipped';
|
|
455
775
|
// The two devices reached the same content independently. Adopting the
|
|
456
776
|
// incoming version name costs nothing and lines up the next fast-forward.
|
|
457
777
|
settle(db, rec);
|
|
@@ -462,9 +782,9 @@ function applyRecord(db, rec) {
|
|
|
462
782
|
const fastForward = !dirty &&
|
|
463
783
|
(rec.base === ref(held.device_id, held.revision) ||
|
|
464
784
|
(held.device_id === rec.device_id && rec.revision > held.revision));
|
|
465
|
-
if (!dirty && held.device_id === rec.device_id && rec.revision <= held.revision)
|
|
466
|
-
return 'skipped';
|
|
467
785
|
if (!fastForward) {
|
|
786
|
+
if (recovering)
|
|
787
|
+
return 'skipped';
|
|
468
788
|
db.prepare(`INSERT INTO sync_conflicts (entry_uid, device_id, revision, base, local_ref, payload, detected_at)
|
|
469
789
|
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, ref(held.device_id, held.revision), JSON.stringify(rec), nowIso());
|
|
470
790
|
return 'conflict';
|
|
@@ -473,6 +793,8 @@ function applyRecord(db, rec) {
|
|
|
473
793
|
else if (local) {
|
|
474
794
|
// No agreed version, but content here already: two independent creations
|
|
475
795
|
// under one uid. Rare enough to be suspicious, so it is not merged either.
|
|
796
|
+
if (recovering)
|
|
797
|
+
return 'skipped';
|
|
476
798
|
db.prepare(`INSERT INTO sync_conflicts (entry_uid, device_id, revision, base, local_ref, payload, detected_at)
|
|
477
799
|
VALUES (?, ?, ?, ?, 'local:unstaged', ?, ?)`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, JSON.stringify(rec), nowIso());
|
|
478
800
|
return 'conflict';
|
|
@@ -506,6 +828,7 @@ function materialize(db, rec) {
|
|
|
506
828
|
// peer that still holds the entry from putting it back.
|
|
507
829
|
if (existing)
|
|
508
830
|
deleteEntry(db, existing.id);
|
|
831
|
+
db.prepare('DELETE FROM sync_supersessions WHERE entry_uid = ?').run(rec.entry_uid);
|
|
509
832
|
return 'tombstone';
|
|
510
833
|
}
|
|
511
834
|
const entry = rec.entry;
|
|
@@ -520,11 +843,34 @@ function materialize(db, rec) {
|
|
|
520
843
|
const tag = db.prepare('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)');
|
|
521
844
|
for (const t of rec.tags)
|
|
522
845
|
tag.run(id, t.toLowerCase());
|
|
846
|
+
linkSupersession(db, id, rec.entry_uid, rec.superseded_by ?? null);
|
|
523
847
|
indexVector(db, id,
|
|
524
848
|
/* c8 ignore next -- narrative is NOT NULL, so a null one fails the insert above before reaching here */
|
|
525
849
|
[entry.title, entry.narrative ?? '', entry.facts ?? '', entry.files ?? ''].join('\n'));
|
|
526
850
|
return 'applied';
|
|
527
851
|
}
|
|
852
|
+
/**
|
|
853
|
+
* Applies the correction a record carries, in both directions.
|
|
854
|
+
*
|
|
855
|
+
* This entry's own link: to the replacement's local row when it is here, or
|
|
856
|
+
* held in `sync_supersessions` until it arrives. A record with no link clears
|
|
857
|
+
* one, because the record is the version this device now agrees on. Then the
|
|
858
|
+
* other direction: originals that arrived before this entry and were waiting
|
|
859
|
+
* for it leave retrieval now.
|
|
860
|
+
*/
|
|
861
|
+
function linkSupersession(db, id, entryUid, replacementUid) {
|
|
862
|
+
db.prepare('DELETE FROM sync_supersessions WHERE entry_uid = ?').run(entryUid);
|
|
863
|
+
const replacement = replacementUid
|
|
864
|
+
? db.prepare('SELECT id FROM memory_entries WHERE entry_uid = ?').get(replacementUid)
|
|
865
|
+
: undefined;
|
|
866
|
+
if (replacementUid && !replacement) {
|
|
867
|
+
db.prepare('INSERT INTO sync_supersessions (entry_uid, replacement_uid) VALUES (?, ?)').run(entryUid, replacementUid);
|
|
868
|
+
}
|
|
869
|
+
db.prepare('UPDATE memory_entries SET superseded_by = ? WHERE id = ?').run(replacement?.id ?? null, id);
|
|
870
|
+
db.prepare(`UPDATE memory_entries SET superseded_by = ?
|
|
871
|
+
WHERE entry_uid IN (SELECT entry_uid FROM sync_supersessions WHERE replacement_uid = ?)`).run(id, entryUid);
|
|
872
|
+
db.prepare('DELETE FROM sync_supersessions WHERE replacement_uid = ?').run(entryUid);
|
|
873
|
+
}
|
|
528
874
|
export function conflicts(db, includeResolved = false) {
|
|
529
875
|
const sql = includeResolved
|
|
530
876
|
? 'SELECT * FROM sync_conflicts ORDER BY id'
|
|
@@ -585,7 +931,9 @@ export function syncStatus(db, config, opts = {}) {
|
|
|
585
931
|
if (!target)
|
|
586
932
|
return { ...base, reason: 'no-target' };
|
|
587
933
|
const device = deviceId(db, config);
|
|
588
|
-
const peers = db
|
|
934
|
+
const peers = db
|
|
935
|
+
.prepare('SELECT device_id, last_revision, last_sync_at, outstanding FROM sync_state ORDER BY device_id')
|
|
936
|
+
.all().filter((p) => p.device_id !== device);
|
|
589
937
|
return {
|
|
590
938
|
...base,
|
|
591
939
|
device_id: device,
|