eklavya 1.48.3 → 1.48.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,14 @@ import { deleteEntry, indexVector } from './store.js';
24
24
  * not cover, such as a cloud client materialising a partial file under the real
25
25
  * name.
26
26
  *
27
+ * **Files arrive out of order.** A synced folder can deliver revision 2 before
28
+ * revision 1, so a reader's high-water mark is a floor -- every revision at or
29
+ * below it is accounted for -- and never jumps a gap. A revision applied above
30
+ * the floor is remembered individually (`sync_received`) until the gap fills.
31
+ * Some gaps never fill: a revision staged and then superseded before its file
32
+ * was written is gone. The writer says so in `void.json` beside its records, and
33
+ * the reader's floor steps over a voided revision as if it had been applied.
34
+ *
27
35
  * **What crosses, and what must not.** Memory entries, their tags and their
28
36
  * tombstones. Not attempts, not mastery, not gates, not receipts, not raw
29
37
  * `evidence_events` (PRD SEC-02). A team that points every member at one shared
@@ -40,7 +48,10 @@ export const SYNC_FORMAT_VERSION = 1;
40
48
  * An allowlist rather than `SELECT *`, so a later migration adding a column to
41
49
  * `memory_entries` does not silently start publishing it. `session_id`,
42
50
  * `batch_id` and `superseded_by` are left out for a different reason: they are
43
- * local row ids and mean nothing on the other device.
51
+ * local row ids and mean nothing on the other device. A correction still
52
+ * crosses: the record carries the replacement's `entry_uid` beside the payload
53
+ * (`SyncRecord.superseded_by`), and the receiving device resolves it to its own
54
+ * row id.
44
55
  */
45
56
  const ENTRY_COLUMNS = [
46
57
  'project',
@@ -56,6 +67,18 @@ const ENTRY_COLUMNS = [
56
67
  'created_at',
57
68
  'import_source',
58
69
  ];
70
+ /**
71
+ * The columns a version is staged from, with `superseded_by` translated to the
72
+ * replacement's uid: a resolved link reads it from the replacement's row, and
73
+ * a link still waiting for its replacement to arrive reads it from
74
+ * `sync_supersessions`. Either way this device states the correction it holds,
75
+ * so a push never publishes a corrected entry as live again.
76
+ */
77
+ const STAGE_COLUMNS = `e.entry_uid, e.deleted_at, ${ENTRY_COLUMNS.map((c) => `e.${c}`).join(', ')},
78
+ COALESCE(
79
+ (SELECT r.entry_uid FROM memory_entries r WHERE r.id = e.superseded_by),
80
+ (SELECT p.replacement_uid FROM sync_supersessions p WHERE p.entry_uid = e.entry_uid)
81
+ ) AS superseded_by_uid`;
59
82
  const DEVICE_ID_KEY = 'sync_device_id';
60
83
  /**
61
84
  * This install's identity, generated once.
@@ -109,12 +132,22 @@ function guard(db, config, override) {
109
132
  // ---------------------------------------------------------------------------
110
133
  // Hashing and the version vector
111
134
  // ---------------------------------------------------------------------------
112
- /** Content identity of a version. Equal hashes mean nothing has to be sent. */
113
- function payloadHash(op, entry, tags) {
135
+ /**
136
+ * Content identity of a version. Equal hashes mean nothing has to be sent.
137
+ *
138
+ * A correction is part of the content: superseding an entry changes its hash,
139
+ * so the push scan stages it and the peer learns of it. The replacement's uid
140
+ * is hashed only when there is one, so an uncorrected entry keeps the hash it
141
+ * had before corrections travelled and an upgrade re-sends nothing else.
142
+ */
143
+ function payloadHash(op, entry, tags, supersededBy = null) {
114
144
  const h = crypto.createHash('sha256');
115
145
  // Canonical by construction: ENTRY_COLUMNS is ordered and tags are sorted, so
116
146
  // two devices holding the same content always agree on the hash.
117
- h.update(JSON.stringify([op, entry ? ENTRY_COLUMNS.map((c) => entry[c] ?? null) : null, [...tags].sort()]));
147
+ const parts = [op, entry ? ENTRY_COLUMNS.map((c) => entry[c] ?? null) : null, [...tags].sort()];
148
+ if (supersededBy)
149
+ parts.push(supersededBy);
150
+ h.update(JSON.stringify(parts));
118
151
  return h.digest('hex').slice(0, 32);
119
152
  }
120
153
  function ref(device, revision) {
@@ -146,7 +179,14 @@ function writeRecord(db, rec) {
146
179
  op = excluded.op, hash = excluded.hash, updated_at = excluded.updated_at`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, rec.op, rec.hash, nowIso());
147
180
  }
148
181
  function tombstone(entryUid) {
149
- return { entry_uid: entryUid, op: 'delete', entry: null, tags: [], hash: payloadHash('delete', null, []) };
182
+ return {
183
+ entry_uid: entryUid,
184
+ op: 'delete',
185
+ entry: null,
186
+ tags: [],
187
+ superseded_by: null,
188
+ hash: payloadHash('delete', null, []),
189
+ };
150
190
  }
151
191
  function tagsOf(db, entryUid) {
152
192
  return db
@@ -158,16 +198,22 @@ function stage(db, row) {
158
198
  return tombstone(row.entry_uid);
159
199
  const entry = Object.fromEntries(ENTRY_COLUMNS.map((c) => [c, row[c] ?? null]));
160
200
  const tags = tagsOf(db, row.entry_uid);
161
- return { entry_uid: row.entry_uid, op: 'upsert', entry, tags, hash: payloadHash('upsert', entry, tags) };
201
+ const supersededBy = row.superseded_by_uid;
202
+ return {
203
+ entry_uid: row.entry_uid,
204
+ op: 'upsert',
205
+ entry,
206
+ tags,
207
+ superseded_by: supersededBy,
208
+ hash: payloadHash('upsert', entry, tags, supersededBy),
209
+ };
162
210
  }
163
211
  /**
164
212
  * What this device holds for one entry right now, whether or not it has been
165
213
  * staged. Null when the entry is unknown here in every sense.
166
214
  */
167
215
  function localContent(db, entryUid) {
168
- const row = db
169
- .prepare(`SELECT entry_uid, deleted_at, ${ENTRY_COLUMNS.join(', ')} FROM memory_entries WHERE entry_uid = ?`)
170
- .get(entryUid);
216
+ const row = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e WHERE e.entry_uid = ?`).get(entryUid);
171
217
  if (row)
172
218
  return stage(db, row);
173
219
  // No row, but a record that outlived it: a hard delete. Still a tombstone.
@@ -183,9 +229,7 @@ function localContent(db, entryUid) {
183
229
  * shows up in a profile.
184
230
  */
185
231
  function scanLocal(db) {
186
- const rows = db
187
- .prepare(`SELECT entry_uid, deleted_at, ${ENTRY_COLUMNS.join(', ')} FROM memory_entries`)
188
- .all();
232
+ const rows = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e`).all();
189
233
  const out = rows
190
234
  .map((row) => stage(db, row))
191
235
  .filter((s) => localRecord(db, s.entry_uid)?.hash !== s.hash);
@@ -209,6 +253,7 @@ function deviceDir(target, device) {
209
253
  // segment under `<target>/devices`, and a peer's directory name arrives from
210
254
  // `readdirSync` rather than from a record, but a single assertion at the one
211
255
  // place paths are built is cheaper than trusting both.
256
+ /* c8 ignore next -- both sources are already checked: `deviceId` for ours, `peerDevices` for theirs */
212
257
  if (!DEVICE_ID.test(device))
213
258
  throw new Error(`unsafe device id: ${JSON.stringify(device)}`);
214
259
  return path.join(target, 'devices', device);
@@ -223,6 +268,113 @@ function writeAtomic(file, text) {
223
268
  fs.writeFileSync(tmp, text, 'utf8');
224
269
  fs.renameSync(tmp, file);
225
270
  }
271
+ const RECORD_NAME = /^\d{12}\.json$/;
272
+ /** iCloud Drive's stand-in for a file it has evicted to save space: still there. */
273
+ const EVICTED_NAME = /^\.(\d{12}\.json)\.icloud$/;
274
+ const VOID_FILE = 'void.json';
275
+ /**
276
+ * Revisions this device handed out and will never write: no file in the
277
+ * target, and no record left to build one from.
278
+ *
279
+ * That happens when an entry is staged, the push is interrupted before its file
280
+ * lands, and the entry changes again -- the next push stages a new revision and
281
+ * the old one's record is overwritten. The content it carried is superseded
282
+ * here, so nothing is lost by declaring it; what matters is that a reader stops
283
+ * waiting for it. The set only grows: a revision number is never handed out
284
+ * twice, and a file is only ever written for a record that still exists.
285
+ */
286
+ function abandonedRevisions(db, target, device) {
287
+ let names;
288
+ try {
289
+ names = fs.readdirSync(deviceDir(target, device));
290
+ }
291
+ catch {
292
+ /* c8 ignore next -- push creates the directory before this runs, unless the target vanished mid-push */
293
+ return [];
294
+ }
295
+ const present = new Set();
296
+ for (const n of names) {
297
+ const name = EVICTED_NAME.exec(n)?.[1] ?? n;
298
+ if (RECORD_NAME.test(name))
299
+ present.add(Number(name.slice(0, 12)));
300
+ }
301
+ const held = new Set(db.prepare('SELECT revision FROM sync_records WHERE device_id = ?').all(device).map((r) => r.revision));
302
+ const out = [];
303
+ for (let r = 1; r <= highWater(db, device); r += 1) {
304
+ if (!present.has(r) && !held.has(r))
305
+ out.push(r);
306
+ }
307
+ return out;
308
+ }
309
+ /** `[1, 2, 3, 7]` as `[[1, 3], [7, 7]]`: a void list stays small however old the device. */
310
+ function toRanges(revisions) {
311
+ const ranges = [];
312
+ for (const r of revisions) {
313
+ const last = ranges[ranges.length - 1];
314
+ if (last && last[1] === r - 1)
315
+ last[1] = r;
316
+ else
317
+ ranges.push([r, r]);
318
+ }
319
+ return ranges;
320
+ }
321
+ /** Publishes the void list, rewriting it only when it changed. */
322
+ function writeVoids(target, device, revisions) {
323
+ if (!revisions.length)
324
+ return;
325
+ const file = path.join(deviceDir(target, device), VOID_FILE);
326
+ const text = `${JSON.stringify({ v: SYNC_FORMAT_VERSION, device_id: device, revisions: toRanges(revisions) })}\n`;
327
+ try {
328
+ if (fs.readFileSync(file, 'utf8') === text)
329
+ return;
330
+ }
331
+ catch {
332
+ // Not written yet.
333
+ }
334
+ writeAtomic(file, text);
335
+ }
336
+ /** Larger than any honest list: a void list holds ranges, not one line per record. */
337
+ const VOID_MAX_BYTES = 1024 * 1024;
338
+ /**
339
+ * A peer's voided revisions above `floor`, as sorted, merged ranges, or none if
340
+ * its list is missing, oversized or unreadable. A torn list is harmless: every
341
+ * revision it does name is void, and one it fails to name is only waited for a
342
+ * little longer. Ranges stay ranges -- a list claiming `[1, 1e12]` costs one
343
+ * pair, not a trillion set entries.
344
+ */
345
+ function readVoids(target, device, floor) {
346
+ const file = path.join(deviceDir(target, device), VOID_FILE);
347
+ let parsed;
348
+ try {
349
+ if (fs.statSync(file).size > VOID_MAX_BYTES)
350
+ return [];
351
+ parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
352
+ }
353
+ catch {
354
+ return [];
355
+ }
356
+ if (parsed?.v !== SYNC_FORMAT_VERSION || parsed.device_id !== device || !Array.isArray(parsed.revisions)) {
357
+ return [];
358
+ }
359
+ const ranges = [];
360
+ for (const range of parsed.revisions) {
361
+ if (!Array.isArray(range) || !Number.isSafeInteger(range[0]) || !Number.isSafeInteger(range[1]))
362
+ continue;
363
+ const lo = Math.max(range[0], floor + 1);
364
+ if (lo <= range[1])
365
+ ranges.push([lo, range[1]]);
366
+ }
367
+ ranges.sort((a, b) => a[0] - b[0]);
368
+ const merged = [];
369
+ for (const [lo, hi] of ranges) {
370
+ const last = merged[merged.length - 1];
371
+ if (last && lo <= last[1] + 1)
372
+ last[1] = Math.max(last[1], hi);
373
+ else
374
+ merged.push([lo, hi]);
375
+ }
376
+ return merged;
377
+ }
226
378
  // ---------------------------------------------------------------------------
227
379
  // Push
228
380
  // ---------------------------------------------------------------------------
@@ -268,8 +420,8 @@ export function push(db, config, opts = {}) {
268
420
  const body = payloads.get(rec.entry_uid);
269
421
  // A record whose payload this run did not rebuild predates the interruption
270
422
  // that lost its file; it is rebuilt from the database below, and dropped if
271
- // the entry is gone -- a missing revision file is a gap the peer's reader
272
- // stops at, which is correct: it will be filled on the next push.
423
+ // the entry has changed since -- the peer waits on the gap until the next
424
+ // push re-stages the entry and the old revision is declared void.
273
425
  const rebuilt = body ?? rebuildPayload(db, rec);
274
426
  if (!rebuilt)
275
427
  continue;
@@ -284,28 +436,26 @@ export function push(db, config, opts = {}) {
284
436
  written_at: nowIso(),
285
437
  entry: rebuilt.entry,
286
438
  tags: rebuilt.tags,
439
+ ...(rebuilt.superseded_by ? { superseded_by: rebuilt.superseded_by } : {}),
287
440
  };
288
441
  writeAtomic(file, `${JSON.stringify(record)}\n`);
289
442
  written += 1;
290
443
  }
444
+ writeVoids(g.target, g.device, abandonedRevisions(db, g.target, g.device));
291
445
  return { ok: true, device_id: g.device, target: g.target, staged: staged.length, written, already };
292
446
  }
293
447
  function rebuildPayload(db, rec) {
294
448
  if (rec.op === 'delete')
295
- return { entry: null, tags: [] };
296
- const row = db
297
- .prepare(`SELECT ${ENTRY_COLUMNS.join(', ')} FROM memory_entries WHERE entry_uid = ?`)
298
- .get(rec.entry_uid);
449
+ return { entry: null, tags: [], superseded_by: null };
450
+ const row = db.prepare(`SELECT ${STAGE_COLUMNS} FROM memory_entries e WHERE e.entry_uid = ?`).get(rec.entry_uid);
299
451
  /* c8 ignore next -- unreachable: an upsert record whose row is gone is re-staged as an orphan tombstone first */
300
452
  if (!row)
301
453
  return null;
302
- const tags = db
303
- .prepare('SELECT tag FROM memory_entry_tags t JOIN memory_entries e ON e.id = t.entry_id WHERE e.entry_uid = ? ORDER BY tag')
304
- .all(rec.entry_uid).map((t) => t.tag);
305
- const entry = Object.fromEntries(ENTRY_COLUMNS.map((c) => [c, row[c] ?? null]));
306
- if (payloadHash('upsert', entry, tags) !== rec.hash)
454
+ const staged = stage(db, row);
455
+ // A row deleted since stages as a tombstone, which is not this record either.
456
+ if (staged.op !== rec.op || staged.hash !== rec.hash)
307
457
  return null;
308
- return { entry, tags };
458
+ return { entry: staged.entry, tags: staged.tags, superseded_by: staged.superseded_by };
309
459
  }
310
460
  // ---------------------------------------------------------------------------
311
461
  // Pull
@@ -342,9 +492,10 @@ function readRecord(file, device, revision) {
342
492
  const entry = rec.op === 'delete' ? null : sanitizeEntry(rec.entry);
343
493
  if (rec.op === 'upsert' && !entry)
344
494
  return null;
345
- if (payloadHash(rec.op, entry, tags) !== rec.hash)
495
+ const supersededBy = rec.op === 'upsert' && typeof rec.superseded_by === 'string' && rec.superseded_by ? rec.superseded_by : null;
496
+ if (payloadHash(rec.op, entry, tags, supersededBy) !== rec.hash)
346
497
  return null;
347
- return { ...rec, entry, tags, base: rec.base ?? null };
498
+ return { ...rec, entry, tags, superseded_by: supersededBy, base: rec.base ?? null };
348
499
  }
349
500
  /**
350
501
  * The inbound half of the allowlist. Anything a peer put in the payload that is
@@ -365,7 +516,8 @@ function peerDevices(target, local) {
365
516
  try {
366
517
  return fs
367
518
  .readdirSync(root, { withFileTypes: true })
368
- .filter((d) => d.isDirectory() && d.name !== local)
519
+ // A directory whose name is not a device id was not written by Eklavya.
520
+ .filter((d) => d.isDirectory() && d.name !== local && DEVICE_ID.test(d.name))
369
521
  .map((d) => d.name)
370
522
  .sort();
371
523
  }
@@ -377,48 +529,183 @@ function pendingRecordFiles(target, device, after) {
377
529
  let names;
378
530
  try {
379
531
  names = fs.readdirSync(deviceDir(target, device));
532
+ /* c8 ignore start -- `peerDevices` just listed this directory; only a folder removed mid-pull lands here */
380
533
  }
381
534
  catch {
382
535
  return [];
383
536
  }
537
+ /* c8 ignore stop */
384
538
  return names
385
- .filter((n) => /^\d{12}\.json$/.test(n))
539
+ .filter((n) => RECORD_NAME.test(n))
386
540
  .map((n) => ({ revision: Number(n.slice(0, 12)), file: path.join(deviceDir(target, device), n) }))
387
541
  .filter((r) => r.revision > after)
388
542
  .sort((a, b) => a.revision - b.revision);
389
543
  }
544
+ /** Revisions from `device` applied above its floor, waiting for the gap below to fill. */
545
+ function receivedAbove(db, device, floor) {
546
+ return new Set(db.prepare('SELECT revision FROM sync_received WHERE device_id = ? AND revision > ?').all(device, floor).map((r) => r.revision));
547
+ }
548
+ /** The void range holding `revision`, by binary search over sorted, disjoint ranges. */
549
+ function voidAt(voids, revision) {
550
+ let lo = 0;
551
+ let hi = voids.length - 1;
552
+ while (lo <= hi) {
553
+ const mid = (lo + hi) >> 1;
554
+ const [a, b] = voids[mid];
555
+ if (revision < a)
556
+ hi = mid - 1;
557
+ else if (revision > b)
558
+ lo = mid + 1;
559
+ else
560
+ return voids[mid];
561
+ }
562
+ return undefined;
563
+ }
564
+ /**
565
+ * Moves a peer's floor up through every revision now accounted for, forgets
566
+ * the individual receipts it covers, and counts what is still owed above it.
567
+ *
568
+ * Accounted for means applied, or voided by its writer with no file present: a
569
+ * file that exists but could not be read yet beats the writer's claim that it
570
+ * never will, because stepping over it would lose it for good. Everything here
571
+ * is arithmetic over the receipts, the unread files and the void ranges -- never
572
+ * a walk over revision numbers, which a stray `999999999999.json` would make a
573
+ * walk of a trillion steps.
574
+ */
575
+ function advanceFloor(db, device, floor, received, unread, voids) {
576
+ let newest = floor;
577
+ for (const r of received)
578
+ newest = Math.max(newest, r);
579
+ for (const r of unread)
580
+ newest = Math.max(newest, r);
581
+ // A void past the newest revision anyone has seen cannot be owed, so it is not
582
+ // stepped over either: a later file would otherwise land below the floor.
583
+ const clamped = voids.filter(([a]) => a <= newest).map(([a, b]) => [a, Math.min(b, newest)]);
584
+ const blockers = [...unread].sort((a, b) => a - b);
585
+ let next = floor;
586
+ for (;;) {
587
+ if (received.has(next + 1)) {
588
+ next += 1;
589
+ continue;
590
+ }
591
+ const range = voidAt(clamped, next + 1);
592
+ if (!range || unread.has(next + 1))
593
+ break;
594
+ const blocker = blockers.find((u) => u > next + 1 && u <= range[1]);
595
+ next = blocker ? blocker - 1 : range[1];
596
+ }
597
+ // Owed = every revision in (next, newest] that is not covered, where covered
598
+ // is received, or void with no unread file in its place.
599
+ const above = (r) => r > next && r <= newest;
600
+ let covered = 0;
601
+ for (const r of received)
602
+ if (above(r))
603
+ covered += 1;
604
+ for (const [a, b] of clamped) {
605
+ const lo = Math.max(a, next + 1);
606
+ if (lo <= b)
607
+ covered += b - lo + 1;
608
+ }
609
+ for (const r of [...received, ...unread])
610
+ if (above(r) && voidAt(clamped, r))
611
+ covered -= 1;
612
+ const missing = newest - next - covered;
613
+ db.transaction(() => {
614
+ setHighWater(db, device, next);
615
+ db.prepare('DELETE FROM sync_received WHERE device_id = ? AND revision <= ?').run(device, next);
616
+ db.prepare('UPDATE sync_state SET outstanding = ? WHERE device_id = ?').run(missing, device);
617
+ })();
618
+ // The floor stops only at a revision that is not covered, so it is the first owed.
619
+ return { missing, first: missing ? next + 1 : 0 };
620
+ }
621
+ /**
622
+ * The one-time recovery for an install upgraded from a release whose pull
623
+ * stepped over late records: every file at or below the mark it had then is
624
+ * read once more. Only what is unambiguous is applied -- an entry this device
625
+ * has never seen, or a plain fast-forward of the version it holds. Anything that
626
+ * would need a person to judge is left exactly as it is, because a record read
627
+ * now cannot be told from one the old pull already quarantined or superseded.
628
+ * It runs once: the old pull stopped at an unreadable record rather than
629
+ * stepping over it, so nothing below its mark is still waiting to be written.
630
+ */
631
+ function replayEarlier(db, target, device) {
632
+ const through = db.prepare('SELECT replay_through FROM sync_state WHERE device_id = ?').get(device)?.replay_through;
633
+ if (!through)
634
+ return { applied: 0, tombstones: 0 };
635
+ let applied = 0;
636
+ let tombstones = 0;
637
+ for (const { revision, file } of pendingRecordFiles(target, device, 0)) {
638
+ if (revision > through)
639
+ break;
640
+ const rec = readRecord(file, device, revision);
641
+ if (!rec)
642
+ continue;
643
+ const outcome = db.transaction(() => applyRecord(db, rec, true))();
644
+ if (outcome === 'applied')
645
+ applied += 1;
646
+ else if (outcome === 'tombstone') {
647
+ applied += 1;
648
+ tombstones += 1;
649
+ }
650
+ }
651
+ db.prepare('UPDATE sync_state SET replay_through = 0 WHERE device_id = ?').run(device);
652
+ return { applied, tombstones };
653
+ }
390
654
  /**
391
655
  * Applies what the peers wrote, one record at a time, in revision order.
392
656
  *
393
- * Each record is applied in its own transaction together with the high-water
394
- * mark it advances, so an interrupted pull leaves a consistent database and
395
- * resumes exactly where it stopped. An unreadable record stops that device's
396
- * stream rather than being skipped past: skipping would advance the mark over a
397
- * file that is merely still being written, and the record would never be read
398
- * again.
657
+ * Each record is applied in its own transaction together with the note that it
658
+ * was received, so an interrupted pull leaves a consistent database and resumes
659
+ * without applying anything twice. Order of arrival does not matter: a record
660
+ * that turns up after a later one is still read, because the floor never moves
661
+ * past a revision that has neither been applied nor voided by its writer. An
662
+ * unreadable record -- most likely one still being written -- is left for the
663
+ * next pull, and the records after it are still applied.
399
664
  */
400
665
  export function pull(db, config, opts = {}) {
401
666
  const g = guard(db, config, opts.target);
402
667
  if (!g.ok) {
403
- return { ok: false, reason: g.reason, applied: 0, tombstones: 0, skipped: 0, conflicts: 0, stalled: [] };
668
+ return {
669
+ ok: false,
670
+ reason: g.reason,
671
+ applied: 0,
672
+ tombstones: 0,
673
+ skipped: 0,
674
+ conflicts: 0,
675
+ recovered: 0,
676
+ stalled: [],
677
+ outstanding: [],
678
+ };
404
679
  }
405
680
  let applied = 0;
406
681
  let tombstones = 0;
407
682
  let skipped = 0;
408
683
  let conflicts = 0;
684
+ let recovered = 0;
409
685
  const stalled = [];
686
+ const outstanding = [];
410
687
  for (const device of peerDevices(g.target, g.device)) {
411
- for (const { revision, file } of pendingRecordFiles(g.target, device, highWater(db, device))) {
688
+ const replay = replayEarlier(db, g.target, device);
689
+ recovered += replay.applied;
690
+ applied += replay.applied;
691
+ tombstones += replay.tombstones;
692
+ const floor = highWater(db, device);
693
+ const received = receivedAbove(db, device, floor);
694
+ const unread = new Set();
695
+ for (const { revision, file } of pendingRecordFiles(g.target, device, floor)) {
696
+ if (received.has(revision))
697
+ continue;
412
698
  const rec = readRecord(file, device, revision);
413
699
  if (!rec) {
414
- stalled.push(device);
415
- break;
700
+ unread.add(revision);
701
+ continue;
416
702
  }
417
703
  const outcome = db.transaction(() => {
418
704
  const result = applyRecord(db, rec);
419
- setHighWater(db, device, revision);
705
+ db.prepare('INSERT OR IGNORE INTO sync_received (device_id, revision) VALUES (?, ?)').run(device, revision);
420
706
  return result;
421
707
  })();
708
+ received.add(revision);
422
709
  if (outcome === 'applied')
423
710
  applied += 1;
424
711
  else if (outcome === 'tombstone') {
@@ -430,8 +717,24 @@ export function pull(db, config, opts = {}) {
430
717
  else
431
718
  skipped += 1;
432
719
  }
720
+ if (unread.size)
721
+ stalled.push(device);
722
+ const owed = advanceFloor(db, device, floor, received, unread, readVoids(g.target, device, floor));
723
+ if (owed.missing)
724
+ outstanding.push({ device_id: device, ...owed });
433
725
  }
434
- return { ok: true, device_id: g.device, target: g.target, applied, tombstones, skipped, conflicts, stalled };
726
+ return {
727
+ ok: true,
728
+ device_id: g.device,
729
+ target: g.target,
730
+ applied,
731
+ tombstones,
732
+ skipped,
733
+ conflicts,
734
+ recovered,
735
+ stalled,
736
+ outstanding,
737
+ };
435
738
  }
436
739
  /**
437
740
  * The merge rule, and the only place a remote version reaches local state.
@@ -448,10 +751,27 @@ export function pull(db, config, opts = {}) {
448
751
  * edited the same entry, which no timestamp can adjudicate: the local version
449
752
  * stays live and the incoming one is quarantined whole, for `repairConflict`.
450
753
  */
451
- function applyRecord(db, rec) {
754
+ function applyRecord(db, rec, recovering = false) {
452
755
  const held = localRecord(db, rec.entry_uid);
453
756
  const local = localContent(db, rec.entry_uid);
757
+ // A version this one already moved past, delivered late: an earlier revision
758
+ // from the device that wrote what is held, or the version held one replaced.
759
+ // Checked first, so a late arrival can neither quarantine nor rename anything.
760
+ if (held &&
761
+ ((held.device_id === rec.device_id && rec.revision <= held.revision) ||
762
+ held.base === ref(rec.device_id, rec.revision))) {
763
+ return 'skipped';
764
+ }
765
+ // Recovery applies only what needs no judgement: an entry with a quarantine
766
+ // open is already waiting on a person, and settling it here would retire that.
767
+ if (recovering &&
768
+ db.prepare('SELECT 1 FROM sync_conflicts WHERE entry_uid = ? AND resolved_at IS NULL').get(rec.entry_uid)) {
769
+ return 'skipped';
770
+ }
454
771
  if (local && local.hash === rec.hash) {
772
+ // Renaming the agreed version to an old one is not unambiguous.
773
+ if (recovering && held)
774
+ return 'skipped';
455
775
  // The two devices reached the same content independently. Adopting the
456
776
  // incoming version name costs nothing and lines up the next fast-forward.
457
777
  settle(db, rec);
@@ -462,9 +782,9 @@ function applyRecord(db, rec) {
462
782
  const fastForward = !dirty &&
463
783
  (rec.base === ref(held.device_id, held.revision) ||
464
784
  (held.device_id === rec.device_id && rec.revision > held.revision));
465
- if (!dirty && held.device_id === rec.device_id && rec.revision <= held.revision)
466
- return 'skipped';
467
785
  if (!fastForward) {
786
+ if (recovering)
787
+ return 'skipped';
468
788
  db.prepare(`INSERT INTO sync_conflicts (entry_uid, device_id, revision, base, local_ref, payload, detected_at)
469
789
  VALUES (?, ?, ?, ?, ?, ?, ?)`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, ref(held.device_id, held.revision), JSON.stringify(rec), nowIso());
470
790
  return 'conflict';
@@ -473,6 +793,8 @@ function applyRecord(db, rec) {
473
793
  else if (local) {
474
794
  // No agreed version, but content here already: two independent creations
475
795
  // under one uid. Rare enough to be suspicious, so it is not merged either.
796
+ if (recovering)
797
+ return 'skipped';
476
798
  db.prepare(`INSERT INTO sync_conflicts (entry_uid, device_id, revision, base, local_ref, payload, detected_at)
477
799
  VALUES (?, ?, ?, ?, 'local:unstaged', ?, ?)`).run(rec.entry_uid, rec.device_id, rec.revision, rec.base, JSON.stringify(rec), nowIso());
478
800
  return 'conflict';
@@ -506,6 +828,7 @@ function materialize(db, rec) {
506
828
  // peer that still holds the entry from putting it back.
507
829
  if (existing)
508
830
  deleteEntry(db, existing.id);
831
+ db.prepare('DELETE FROM sync_supersessions WHERE entry_uid = ?').run(rec.entry_uid);
509
832
  return 'tombstone';
510
833
  }
511
834
  const entry = rec.entry;
@@ -520,11 +843,34 @@ function materialize(db, rec) {
520
843
  const tag = db.prepare('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)');
521
844
  for (const t of rec.tags)
522
845
  tag.run(id, t.toLowerCase());
846
+ linkSupersession(db, id, rec.entry_uid, rec.superseded_by ?? null);
523
847
  indexVector(db, id,
524
848
  /* c8 ignore next -- narrative is NOT NULL, so a null one fails the insert above before reaching here */
525
849
  [entry.title, entry.narrative ?? '', entry.facts ?? '', entry.files ?? ''].join('\n'));
526
850
  return 'applied';
527
851
  }
852
+ /**
853
+ * Applies the correction a record carries, in both directions.
854
+ *
855
+ * This entry's own link: to the replacement's local row when it is here, or
856
+ * held in `sync_supersessions` until it arrives. A record with no link clears
857
+ * one, because the record is the version this device now agrees on. Then the
858
+ * other direction: originals that arrived before this entry and were waiting
859
+ * for it leave retrieval now.
860
+ */
861
+ function linkSupersession(db, id, entryUid, replacementUid) {
862
+ db.prepare('DELETE FROM sync_supersessions WHERE entry_uid = ?').run(entryUid);
863
+ const replacement = replacementUid
864
+ ? db.prepare('SELECT id FROM memory_entries WHERE entry_uid = ?').get(replacementUid)
865
+ : undefined;
866
+ if (replacementUid && !replacement) {
867
+ db.prepare('INSERT INTO sync_supersessions (entry_uid, replacement_uid) VALUES (?, ?)').run(entryUid, replacementUid);
868
+ }
869
+ db.prepare('UPDATE memory_entries SET superseded_by = ? WHERE id = ?').run(replacement?.id ?? null, id);
870
+ db.prepare(`UPDATE memory_entries SET superseded_by = ?
871
+ WHERE entry_uid IN (SELECT entry_uid FROM sync_supersessions WHERE replacement_uid = ?)`).run(id, entryUid);
872
+ db.prepare('DELETE FROM sync_supersessions WHERE replacement_uid = ?').run(entryUid);
873
+ }
528
874
  export function conflicts(db, includeResolved = false) {
529
875
  const sql = includeResolved
530
876
  ? 'SELECT * FROM sync_conflicts ORDER BY id'
@@ -585,7 +931,9 @@ export function syncStatus(db, config, opts = {}) {
585
931
  if (!target)
586
932
  return { ...base, reason: 'no-target' };
587
933
  const device = deviceId(db, config);
588
- const peers = db.prepare('SELECT * FROM sync_state ORDER BY device_id').all().filter((p) => p.device_id !== device);
934
+ const peers = db
935
+ .prepare('SELECT device_id, last_revision, last_sync_at, outstanding FROM sync_state ORDER BY device_id')
936
+ .all().filter((p) => p.device_id !== device);
589
937
  return {
590
938
  ...base,
591
939
  device_id: device,