@quereus/sync 4.3.2 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/dist/src/create-sync-module.d.ts +11 -1
- package/dist/src/create-sync-module.d.ts.map +1 -1
- package/dist/src/create-sync-module.js +2 -2
- package/dist/src/create-sync-module.js.map +1 -1
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +9 -3
- package/dist/src/index.js.map +1 -1
- package/dist/src/metadata/change-log.d.ts +39 -3
- package/dist/src/metadata/change-log.d.ts.map +1 -1
- package/dist/src/metadata/change-log.js +43 -8
- package/dist/src/metadata/change-log.js.map +1 -1
- package/dist/src/metadata/column-version.d.ts +62 -9
- package/dist/src/metadata/column-version.d.ts.map +1 -1
- package/dist/src/metadata/column-version.js +79 -25
- package/dist/src/metadata/column-version.js.map +1 -1
- package/dist/src/metadata/index.d.ts +1 -0
- package/dist/src/metadata/index.d.ts.map +1 -1
- package/dist/src/metadata/index.js +1 -0
- package/dist/src/metadata/index.js.map +1 -1
- package/dist/src/metadata/keys.d.ts +88 -22
- package/dist/src/metadata/keys.d.ts.map +1 -1
- package/dist/src/metadata/keys.js +134 -77
- package/dist/src/metadata/keys.js.map +1 -1
- package/dist/src/metadata/pk-identity.d.ts +60 -0
- package/dist/src/metadata/pk-identity.d.ts.map +1 -0
- package/dist/src/metadata/pk-identity.js +94 -0
- package/dist/src/metadata/pk-identity.js.map +1 -0
- package/dist/src/metadata/tombstones.d.ts +41 -11
- package/dist/src/metadata/tombstones.d.ts.map +1 -1
- package/dist/src/metadata/tombstones.js +58 -33
- package/dist/src/metadata/tombstones.js.map +1 -1
- package/dist/src/sync/change-applicator.d.ts +30 -1
- package/dist/src/sync/change-applicator.d.ts.map +1 -1
- package/dist/src/sync/change-applicator.js +161 -40
- package/dist/src/sync/change-applicator.js.map +1 -1
- package/dist/src/sync/maintenance.d.ts +70 -0
- package/dist/src/sync/maintenance.d.ts.map +1 -0
- package/dist/src/sync/maintenance.js +84 -0
- package/dist/src/sync/maintenance.js.map +1 -0
- package/dist/src/sync/protocol.d.ts +61 -10
- package/dist/src/sync/protocol.d.ts.map +1 -1
- package/dist/src/sync/protocol.js +15 -0
- package/dist/src/sync/protocol.js.map +1 -1
- package/dist/src/sync/snapshot-identity.d.ts +62 -0
- package/dist/src/sync/snapshot-identity.d.ts.map +1 -0
- package/dist/src/sync/snapshot-identity.js +88 -0
- package/dist/src/sync/snapshot-identity.js.map +1 -0
- package/dist/src/sync/snapshot-stream.d.ts +8 -1
- package/dist/src/sync/snapshot-stream.d.ts.map +1 -1
- package/dist/src/sync/snapshot-stream.js +166 -67
- package/dist/src/sync/snapshot-stream.js.map +1 -1
- package/dist/src/sync/snapshot.d.ts +1 -1
- package/dist/src/sync/snapshot.d.ts.map +1 -1
- package/dist/src/sync/snapshot.js +75 -44
- package/dist/src/sync/snapshot.js.map +1 -1
- package/dist/src/sync/staged-transaction-metadata.d.ts +75 -0
- package/dist/src/sync/staged-transaction-metadata.d.ts.map +1 -0
- package/dist/src/sync/staged-transaction-metadata.js +100 -0
- package/dist/src/sync/staged-transaction-metadata.js.map +1 -0
- package/dist/src/sync/store-adapter.d.ts +10 -4
- package/dist/src/sync/store-adapter.d.ts.map +1 -1
- package/dist/src/sync/store-adapter.js +244 -21
- package/dist/src/sync/store-adapter.js.map +1 -1
- package/dist/src/sync/sync-context.d.ts +54 -2
- package/dist/src/sync/sync-context.d.ts.map +1 -1
- package/dist/src/sync/sync-context.js +66 -0
- package/dist/src/sync/sync-context.js.map +1 -1
- package/dist/src/sync/sync-manager-impl.d.ts +70 -4
- package/dist/src/sync/sync-manager-impl.d.ts.map +1 -1
- package/dist/src/sync/sync-manager-impl.js +206 -36
- package/dist/src/sync/sync-manager-impl.js.map +1 -1
- package/dist/src/sync/wire.d.ts +47 -6
- package/dist/src/sync/wire.d.ts.map +1 -1
- package/dist/src/sync/wire.js +62 -11
- package/dist/src/sync/wire.js.map +1 -1
- package/package.json +6 -6
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Delegates to focused sub-modules for snapshot, streaming, and change application.
|
|
6
6
|
*/
|
|
7
7
|
import type { KVStore } from '@quereus/store';
|
|
8
|
-
import type { Database, LensDeploymentSnapshot, TransactionCommitBatch } from '@quereus/quereus';
|
|
8
|
+
import type { Database, KeyNormalizerResolver, LensDeploymentSnapshot, TransactionCommitBatch } from '@quereus/quereus';
|
|
9
9
|
import type { GetTableSchemaCallback, TransactionCommitSource } from '../create-sync-module.js';
|
|
10
10
|
import { HLCManager, type HLC } from '../clock/hlc.js';
|
|
11
11
|
import { type SiteId } from '../clock/site.js';
|
|
@@ -15,6 +15,7 @@ import { SchemaMigrationStore } from '../metadata/schema-migration.js';
|
|
|
15
15
|
import { ChangeLogStore } from '../metadata/change-log.js';
|
|
16
16
|
import { QuarantineStore } from '../metadata/quarantine.js';
|
|
17
17
|
import { BasisLifecycleStore, type BasisTableLifecycleRecord } from '../metadata/basis-lifecycle.js';
|
|
18
|
+
import { type PkKeying } from '../metadata/keys.js';
|
|
18
19
|
import type { SyncManager, SnapshotCheckpoint } from './manager.js';
|
|
19
20
|
import type { SyncConfig, ChangeSet, ApplyResult, Snapshot, SnapshotChunk, SnapshotProgress, ApplyToStoreCallback, DropLocalTableCallback, UnknownTableDisposition } from './protocol.js';
|
|
20
21
|
import { SyncEventEmitterImpl } from './events.js';
|
|
@@ -49,6 +50,13 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
49
50
|
private readonly getTableSchema?;
|
|
50
51
|
/** Reclaim-by-name callback for the eviction sweep; absent ⇒ sweep is a no-op. */
|
|
51
52
|
private readonly dropLocalTable?;
|
|
53
|
+
/**
|
|
54
|
+
* Per-table pk-identity keying resolver (see `metadata/pk-identity.ts`),
|
|
55
|
+
* threaded into every metadata store so `cv:`/`tb:`/`cl:` keys file under the
|
|
56
|
+
* row's collation/transform-normalized identity. Raw keying when no
|
|
57
|
+
* `getTableSchema` oracle was wired (relay-only deployment).
|
|
58
|
+
*/
|
|
59
|
+
private readonly pkKeying;
|
|
52
60
|
private commitChain;
|
|
53
61
|
private readonly lastBasisHash;
|
|
54
62
|
private unknownTableIgnored;
|
|
@@ -72,10 +80,16 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
72
80
|
* @param getTableSchema - Optional callback for getting table schema by name
|
|
73
81
|
* @param dropLocalTable - Optional reclaim-by-name callback for the basis-table
|
|
74
82
|
* eviction sweep; when absent (e.g. a relay-only coordinator) the sweep is a no-op.
|
|
83
|
+
* @param keyNormalizerResolver - Optional collation-name → key-normalizer resolver
|
|
84
|
+
* (pass the engine's `db.getKeyNormalizerResolver()`), used to build pk-identity
|
|
85
|
+
* keys that agree with the database's row identity. When absent, a built-ins-only
|
|
86
|
+
* resolver (BINARY/NOCASE/RTRIM) is used, which throws on custom collation names.
|
|
75
87
|
*/
|
|
76
|
-
static create(kv: KVStore, transactionSource: TransactionCommitSource | undefined, config: SyncConfig, syncEvents: SyncEventEmitterImpl, applyToStore?: ApplyToStoreCallback, getTableSchema?: GetTableSchemaCallback, dropLocalTable?: DropLocalTableCallback): Promise<SyncManagerImpl>;
|
|
88
|
+
static create(kv: KVStore, transactionSource: TransactionCommitSource | undefined, config: SyncConfig, syncEvents: SyncEventEmitterImpl, applyToStore?: ApplyToStoreCallback, getTableSchema?: GetTableSchemaCallback, dropLocalTable?: DropLocalTableCallback, keyNormalizerResolver?: KeyNormalizerResolver): Promise<SyncManagerImpl>;
|
|
77
89
|
getSiteId(): SiteId;
|
|
78
90
|
getCurrentHLC(): HLC;
|
|
91
|
+
/** See {@link SyncContext.getPkKeying}. */
|
|
92
|
+
getPkKeying(schema: string, table: string): PkKeying;
|
|
79
93
|
/**
|
|
80
94
|
* Basis-membership oracle for unknown-table detection. `getTableSchema(s,t)
|
|
81
95
|
* === undefined` ⇒ the table is outside the local basis. When no oracle was
|
|
@@ -164,6 +178,45 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
164
178
|
* HLC clock state — lands in a single KV batch.
|
|
165
179
|
*/
|
|
166
180
|
private handleTransactionCommit;
|
|
181
|
+
/**
|
|
182
|
+
* Drop the data events of any table that is no longer in the local basis — the
|
|
183
|
+
* schema oracle does not know it, typically because this same transaction dropped
|
|
184
|
+
* it and the schema is already gone by the time the commit group is delivered.
|
|
185
|
+
* A schemaless table has no sound pk identity (see `metadata/pk-identity.ts`), so
|
|
186
|
+
* its rows cannot be filed.
|
|
187
|
+
*
|
|
188
|
+
* Local capture is therefore best-effort at TABLE granularity: one unknown table
|
|
189
|
+
* costs only its own rows, never the rest of the transaction (which is what the
|
|
190
|
+
* handler's top-level catch would do — it aborts recording of every other table's
|
|
191
|
+
* rows AND the transaction's schema migrations).
|
|
192
|
+
*
|
|
193
|
+
* Runs BEFORE the HLC tick, so a transaction whose data is entirely skipped and
|
|
194
|
+
* that carries no schema events consumes no clock and emits no local change,
|
|
195
|
+
* mirroring the all-remote echo early-out. Filtering up front also means no event
|
|
196
|
+
* is ever half-recorded: the throw it replaces could fire *after* a tombstone had
|
|
197
|
+
* already been staged into the shared `WriteBatch`.
|
|
198
|
+
*
|
|
199
|
+
* The gate is basis MEMBERSHIP, not "this transaction dropped the table":
|
|
200
|
+
* `drop t; create t; insert into t` carries a drop event for `t` yet leaves it in
|
|
201
|
+
* basis, and its rows must still be captured. `localSchema` is consulted only to
|
|
202
|
+
* classify the log line — an expected skip reads differently from a table that
|
|
203
|
+
* vanished for some other reason.
|
|
204
|
+
*
|
|
205
|
+
* ONLY the unknown-table case is skipped. Any other keying failure — a pk column
|
|
206
|
+
* whose collation the wired `keyNormalizerResolver` does not know, say — still
|
|
207
|
+
* throws out of `recordDataEvent` and fails the transaction loudly; silently
|
|
208
|
+
* discarding those rows would bury a real misconfiguration.
|
|
209
|
+
*
|
|
210
|
+
* A relay-only manager (no `getTableSchema` oracle) reports every table in basis,
|
|
211
|
+
* so nothing is skipped there.
|
|
212
|
+
*/
|
|
213
|
+
private filterCapturableDataEvents;
|
|
214
|
+
/**
|
|
215
|
+
* Report skipped tables: ONE line per table with its change count, never one per
|
|
216
|
+
* row. A table this same transaction dropped is expected and logs informationally;
|
|
217
|
+
* anything else is a warning — the table vanished for some other reason.
|
|
218
|
+
*/
|
|
219
|
+
private logSkippedTables;
|
|
167
220
|
/**
|
|
168
221
|
* Record one local schema-change event as a migration. Allocates an `opSeq`
|
|
169
222
|
* (via `nextHlc`) only when the event maps to a tracked migration, so
|
|
@@ -174,6 +227,11 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
174
227
|
/**
|
|
175
228
|
* Record one local data-change event: a tombstone (delete) or the changed
|
|
176
229
|
* column versions (insert/update). Each recorded fact consumes one `opSeq`.
|
|
230
|
+
*
|
|
231
|
+
* All reads of the row's current metadata go overlay-first (`staged`), so a
|
|
232
|
+
* transaction that touches one row more than once records the same metadata
|
|
233
|
+
* the equivalent sequence of separate transactions would — and every write
|
|
234
|
+
* staged into `batch` is noted back into the overlay.
|
|
177
235
|
*/
|
|
178
236
|
private recordDataEvent;
|
|
179
237
|
private recordColumnVersions;
|
|
@@ -234,8 +292,8 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
234
292
|
* Resolve a change-log entry to the live {@link Change} it references, or
|
|
235
293
|
* `null` when the underlying column version / tombstone is gone — a stale log
|
|
236
294
|
* entry (e.g. a column overwritten, or a row deleted after the entry was
|
|
237
|
-
* written). The authoritative HLC
|
|
238
|
-
* the log key.
|
|
295
|
+
* written). The authoritative HLC, value AND pk come from the current version
|
|
296
|
+
* record, not the log key — the key carries only the lossy pk identity.
|
|
239
297
|
*/
|
|
240
298
|
private resolveLogEntry;
|
|
241
299
|
/**
|
|
@@ -315,6 +373,14 @@ export declare class SyncManagerImpl implements SyncManager, SyncContext {
|
|
|
315
373
|
getPeerSyncState(peerSiteId: SiteId): Promise<HLC | undefined>;
|
|
316
374
|
updatePeerSentState(peerSiteId: SiteId, hlc: HLC): Promise<void>;
|
|
317
375
|
getPeerSentState(peerSiteId: SiteId): Promise<HLC | undefined>;
|
|
376
|
+
/**
|
|
377
|
+
* Drop tombstones past the retention horizon, together with the change-log
|
|
378
|
+
* `delete` entry each one backs. The entry is only an index pointer at the
|
|
379
|
+
* tombstone ({@link resolveLogEntry} returns null once the tombstone is gone),
|
|
380
|
+
* so leaving it would accumulate one dead entry per delete the replica has ever
|
|
381
|
+
* seen — the delete-side twin of the column cleanup in
|
|
382
|
+
* {@link deleteRowVersionsAndLogEntries}. Both deletions share this batch.
|
|
383
|
+
*/
|
|
318
384
|
pruneTombstones(): Promise<number>;
|
|
319
385
|
/**
|
|
320
386
|
* GC quarantined out-of-basis straggler changes past the retention horizon —
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-manager-impl.d.ts","sourceRoot":"","sources":["../../../src/sync/sync-manager-impl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAc,MAAM,gBAAgB,CAAC;AAC1D,OAAO,KAAK,EAGX,QAAQ,EACR,sBAAsB,EACtB,sBAAsB,EAGtB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,KAAK,GAAG,EAAwD,MAAM,iBAAiB,CAAC;AAC7G,OAAO,EAEN,KAAK,MAAM,EAKX,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"sync-manager-impl.d.ts","sourceRoot":"","sources":["../../../src/sync/sync-manager-impl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAc,MAAM,gBAAgB,CAAC;AAC1D,OAAO,KAAK,EAGX,QAAQ,EACR,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EAGtB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,KAAK,GAAG,EAAwD,MAAM,iBAAiB,CAAC;AAC7G,OAAO,EAEN,KAAK,MAAM,EAKX,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAoD,MAAM,+BAA+B,CAAC;AACrH,OAAO,EAAE,cAAc,EAAwB,MAAM,2BAA2B,CAAC;AAEjF,OAAO,EAAE,oBAAoB,EAAwB,MAAM,iCAAiC,CAAC;AAC7F,OAAO,EAAE,cAAc,EAAuB,MAAM,2BAA2B,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EACN,mBAAmB,EAOnB,KAAK,yBAAyB,EAE9B,MAAM,gCAAgC,CAAC;AAExC,OAAO,EASN,KAAK,QAAQ,EACb,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACpE,OAAO,KAAK,EACX,UAAU,EACV,SAAS,EAIT,WAAW,EACX,QAAQ,EAGR,aAAa,EACb,gBAAgB,EAChB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAarD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAOtD;AA4BD;;;;;GAKG;AACH,qBAAa,eAAgB,YAAW,WAAW,EAAE,WAAW;IAC/D,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,kBAAkB,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,gBAAgB,EAAE,oBAAoB,CAAC;IAChD,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,cAAc,EAAE,mBAAmB,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAC;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,oBAAoB,CAAC;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAyB;IACzD,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAyB;IACzD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;IAO5C,OAAO,CAAC,WAAW,CAAoC;IAIvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA6B;IAI3D,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,qBAAqB,CAAK;IAMlC,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAA6B;IAEjE,OAAO;IA2BP;;;;;;;;;;;;;;;;;;;OAmBG;WACU,MAAM,CAClB,EAAE,EAAE,OAAO,EACX,iBAAiB,EAAE,uBAAuB,GAAG,SAAS,EACtD,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,oBAAoB,EAChC,YAAY,CAAC,EAAE,oBAAoB,EACnC,cAAc,CAAC,EAAE,sBAAsB,EACvC,cAAc,CAAC,EAAE,sBAAsB,EACvC,qBAAqB,CAAC,EAAE,qBAAqB,GAC3C,OAAO,CAAC,eAAe,CAAC;IA0E3B,SAAS,IAAI,MAAM;IAInB,aAAa,IAAI,GAAG;IAIpB,2CAA2C;IAC3C,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,QAAQ;IAIpD;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO;IAItD;;;;;;OAMG;IACH,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,GAAG,SAAS;IAIjF;;;OAGG;IACH,kBAAkB,CACjB,WAAW,EAAE,uBAAuB,EACpC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GACjB,IAAI;IAcP,oBAAoB,IAAI;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAclI;;;;;;;;;;;;;OAaG;IACG,oBAAoB,CACzB,EAAE,EAAE,QAAQ,EACZ,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC;IA8MV,sBAAsB,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC;IAIpE;;;;;;;;;;;;;;OAcG;IACG,uBAAuB,CAAC,GAAG,GAAE,MAAmB,GAAG,OAAO,CAAC,MAAM,CAAC;IA2CxE;;;;;;;;OAQG;IACH,wBAAwB,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI;IAM7D;;;;OAIG;IACH,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAInC;;;;;;;;;;;OAWG;YACW,uBAAuB;IA8DrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,OAAO,CAAC,0BAA0B;IAmBlC;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAqBxB;;;;;OAKG;YACW,qBAAqB;IAgDnC;;;;;;;;OAQG;YACW,eAAe;YA8Df,oBAAoB;IA2ElC;;;;;;;OAOG;IACG,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwC/E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;YACW,mBAAmB;IAyCjC;;;;;;OAMG;YACW,eAAe;IA6C7B;;;;;;;;;;;OAWG;YACW,iBAAiB;IA8C/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsCG;YACW,yBAAyB;IAmBvC;;;;;;;;;;;;;OAaG;YACW,uBAAuB;IA+B/B,YAAY,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IAIxD,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIlE,YAAY,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC;IAoBjE,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAIhC,aAAa,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhD,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAK9D,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAIpE;;;;;;;OAOG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IA2BxC;;;;;OAKG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAKxC,eAAe,IAAI,oBAAoB;IAQhC,iBAAiB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,aAAa,CAAC;IAIpE,mBAAmB,CACxB,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,EACpC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAC/C,OAAO,CAAC,IAAI,CAAC;IAIV,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAIjF,oBAAoB,CAAC,UAAU,EAAE,kBAAkB,GAAG,aAAa,CAAC,aAAa,CAAC;CAGzF"}
|
|
@@ -14,8 +14,10 @@ import { SchemaMigrationStore, deserializeMigration } from '../metadata/schema-m
|
|
|
14
14
|
import { ChangeLogStore } from '../metadata/change-log.js';
|
|
15
15
|
import { QuarantineStore } from '../metadata/quarantine.js';
|
|
16
16
|
import { BasisLifecycleStore, classifyBasisLifecycle, basisLifecycleRecordChanged, splitRelKey, parseEvictPolicyTag, isEvictable, quietSince, } from '../metadata/basis-lifecycle.js';
|
|
17
|
-
import { SYNC_KEY_PREFIX, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, parseColumnVersionKey, parseTombstoneKey, parseSchemaMigrationKey, } from '../metadata/keys.js';
|
|
18
|
-
import {
|
|
17
|
+
import { SYNC_KEY_PREFIX, SYNC_METADATA_FORMAT_VERSION, buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, parseColumnVersionKey, parseTombstoneKey, parseSchemaMigrationKey, } from '../metadata/keys.js';
|
|
18
|
+
import { createPkKeyingResolver } from '../metadata/pk-identity.js';
|
|
19
|
+
import { persistHLCStateBatch, toError, deleteRowVersionsAndLogEntries } from './sync-context.js';
|
|
20
|
+
import { StagedTransactionMetadata } from './staged-transaction-metadata.js';
|
|
19
21
|
import { applyChanges as applyChangesImpl, drainHeldChanges as drainHeldChangesImpl, drainReappearedTables } from './change-applicator.js';
|
|
20
22
|
import { buildTransactionChangeSets } from './change-grouping.js';
|
|
21
23
|
import { getSnapshot as getSnapshotImpl, applySnapshot as applySnapshotImpl } from './snapshot.js';
|
|
@@ -77,6 +79,13 @@ export class SyncManagerImpl {
|
|
|
77
79
|
getTableSchema;
|
|
78
80
|
/** Reclaim-by-name callback for the eviction sweep; absent ⇒ sweep is a no-op. */
|
|
79
81
|
dropLocalTable;
|
|
82
|
+
/**
|
|
83
|
+
* Per-table pk-identity keying resolver (see `metadata/pk-identity.ts`),
|
|
84
|
+
* threaded into every metadata store so `cv:`/`tb:`/`cl:` keys file under the
|
|
85
|
+
* row's collation/transform-normalized identity. Raw keying when no
|
|
86
|
+
* `getTableSchema` oracle was wired (relay-only deployment).
|
|
87
|
+
*/
|
|
88
|
+
pkKeying;
|
|
80
89
|
// Tail-promise chain serializing commit recording. Each commit chains onto the
|
|
81
90
|
// prior handler's completion so N+1's dedup reads see N's durable writes,
|
|
82
91
|
// preserving the change-log ordering invariant (docs/sync.md § Transaction-Based
|
|
@@ -98,7 +107,7 @@ export class SyncManagerImpl {
|
|
|
98
107
|
// relay activity, not with distinct stragglers. Bumped in collectForwardableChanges.
|
|
99
108
|
unknownTableRelayed = 0;
|
|
100
109
|
unknownTableByTable = new Map();
|
|
101
|
-
constructor(kv, config, hlcManager, syncEvents, applyToStore, getTableSchema, dropLocalTable) {
|
|
110
|
+
constructor(kv, config, hlcManager, syncEvents, applyToStore, getTableSchema, dropLocalTable, keyNormalizerResolver) {
|
|
102
111
|
this.kv = kv;
|
|
103
112
|
this.config = config;
|
|
104
113
|
this.hlcManager = hlcManager;
|
|
@@ -106,10 +115,11 @@ export class SyncManagerImpl {
|
|
|
106
115
|
this.applyToStore = applyToStore;
|
|
107
116
|
this.getTableSchema = getTableSchema;
|
|
108
117
|
this.dropLocalTable = dropLocalTable;
|
|
109
|
-
this.
|
|
110
|
-
this.
|
|
118
|
+
this.pkKeying = createPkKeyingResolver(getTableSchema, keyNormalizerResolver);
|
|
119
|
+
this.columnVersions = new ColumnVersionStore(kv, this.pkKeying);
|
|
120
|
+
this.tombstones = new TombstoneStore(kv, config.retentionHorizonMs, this.pkKeying);
|
|
111
121
|
this.peerStates = new PeerStateStore(kv);
|
|
112
|
-
this.changeLog = new ChangeLogStore(kv);
|
|
122
|
+
this.changeLog = new ChangeLogStore(kv, this.pkKeying);
|
|
113
123
|
this.schemaMigrations = new SchemaMigrationStore(kv);
|
|
114
124
|
this.quarantine = new QuarantineStore(kv);
|
|
115
125
|
this.basisLifecycle = new BasisLifecycleStore(kv);
|
|
@@ -129,12 +139,35 @@ export class SyncManagerImpl {
|
|
|
129
139
|
* @param getTableSchema - Optional callback for getting table schema by name
|
|
130
140
|
* @param dropLocalTable - Optional reclaim-by-name callback for the basis-table
|
|
131
141
|
* eviction sweep; when absent (e.g. a relay-only coordinator) the sweep is a no-op.
|
|
142
|
+
* @param keyNormalizerResolver - Optional collation-name → key-normalizer resolver
|
|
143
|
+
* (pass the engine's `db.getKeyNormalizerResolver()`), used to build pk-identity
|
|
144
|
+
* keys that agree with the database's row identity. When absent, a built-ins-only
|
|
145
|
+
* resolver (BINARY/NOCASE/RTRIM) is used, which throws on custom collation names.
|
|
132
146
|
*/
|
|
133
|
-
static async create(kv, transactionSource, config, syncEvents, applyToStore, getTableSchema, dropLocalTable) {
|
|
147
|
+
static async create(kv, transactionSource, config, syncEvents, applyToStore, getTableSchema, dropLocalTable, keyNormalizerResolver) {
|
|
134
148
|
// Load or create site identity
|
|
135
149
|
const siteIdKey = new TextEncoder().encode(SITE_ID_KEY);
|
|
136
150
|
let siteId;
|
|
137
151
|
const existingIdentity = await kv.get(siteIdKey);
|
|
152
|
+
// Sync-metadata format gate: per-row records are keyed by pk IDENTITY since
|
|
153
|
+
// format 2 (see SYNC_METADATA_FORMAT_VERSION); older metadata is unreadable
|
|
154
|
+
// under this layout, and writing new-format records beside it would corrupt
|
|
155
|
+
// both. A replica with existing identity but a missing/mismatched version
|
|
156
|
+
// must re-bootstrap from a peer snapshot (docs/sync.md § Metadata format
|
|
157
|
+
// version) — refuse loudly rather than silently mis-keying.
|
|
158
|
+
const formatData = await kv.get(SYNC_KEY_PREFIX.FORMAT_VERSION);
|
|
159
|
+
const storedFormat = formatData ? parseInt(new TextDecoder().decode(formatData), 10) : undefined;
|
|
160
|
+
if (storedFormat === undefined) {
|
|
161
|
+
if (existingIdentity) {
|
|
162
|
+
throw new QuereusError(`Sync metadata predates format version ${SYNC_METADATA_FORMAT_VERSION} (pk-identity keys). `
|
|
163
|
+
+ `Clear this replica's sync metadata and re-bootstrap from a peer snapshot (docs/sync.md § Metadata format version).`, StatusCode.ERROR);
|
|
164
|
+
}
|
|
165
|
+
await kv.put(SYNC_KEY_PREFIX.FORMAT_VERSION, new TextEncoder().encode(String(SYNC_METADATA_FORMAT_VERSION)));
|
|
166
|
+
}
|
|
167
|
+
else if (storedFormat !== SYNC_METADATA_FORMAT_VERSION) {
|
|
168
|
+
throw new QuereusError(`Sync metadata format version ${storedFormat} does not match this build's ${SYNC_METADATA_FORMAT_VERSION}. `
|
|
169
|
+
+ `Clear this replica's sync metadata and re-bootstrap from a peer snapshot (docs/sync.md § Metadata format version).`, StatusCode.ERROR);
|
|
170
|
+
}
|
|
138
171
|
if (existingIdentity) {
|
|
139
172
|
const identity = deserializeSiteIdentity(existingIdentity);
|
|
140
173
|
siteId = identity.siteId;
|
|
@@ -159,7 +192,7 @@ export class SyncManagerImpl {
|
|
|
159
192
|
};
|
|
160
193
|
}
|
|
161
194
|
const hlcManager = new HLCManager(siteId, hlcState);
|
|
162
|
-
const manager = new SyncManagerImpl(kv, config, hlcManager, syncEvents, applyToStore, getTableSchema, dropLocalTable);
|
|
195
|
+
const manager = new SyncManagerImpl(kv, config, hlcManager, syncEvents, applyToStore, getTableSchema, dropLocalTable, keyNormalizerResolver);
|
|
163
196
|
// Capture local changes at the engine transaction boundary: one grouped
|
|
164
197
|
// `onTransactionCommit` batch ⇒ one transaction ⇒ one HLC. The store's
|
|
165
198
|
// per-table emitter is below the transaction boundary and cannot group a
|
|
@@ -179,6 +212,10 @@ export class SyncManagerImpl {
|
|
|
179
212
|
getCurrentHLC() {
|
|
180
213
|
return this.hlcManager.now();
|
|
181
214
|
}
|
|
215
|
+
/** See {@link SyncContext.getPkKeying}. */
|
|
216
|
+
getPkKeying(schema, table) {
|
|
217
|
+
return this.pkKeying(schema, table);
|
|
218
|
+
}
|
|
182
219
|
/**
|
|
183
220
|
* Basis-membership oracle for unknown-table detection. `getTableSchema(s,t)
|
|
184
221
|
* === undefined` ⇒ the table is outside the local basis. When no oracle was
|
|
@@ -535,7 +572,7 @@ export class SyncManagerImpl {
|
|
|
535
572
|
// (local + remote in one transaction — unusual) records only its local
|
|
536
573
|
// facts; opSeq is assigned only to recorded facts so they stay contiguous.
|
|
537
574
|
const localSchema = batch.schemaEvents.filter(e => !e.remote);
|
|
538
|
-
const localData = batch.dataEvents.filter(e => !e.remote);
|
|
575
|
+
const localData = this.filterCapturableDataEvents(batch.dataEvents.filter(e => !e.remote), localSchema);
|
|
539
576
|
if (localSchema.length === 0 && localData.length === 0)
|
|
540
577
|
return;
|
|
541
578
|
// ONE tick per committed transaction. tick() returns opSeq 0; the closure
|
|
@@ -550,13 +587,18 @@ export class SyncManagerImpl {
|
|
|
550
587
|
};
|
|
551
588
|
const kvBatch = this.kv.batch();
|
|
552
589
|
const changes = [];
|
|
590
|
+
// Read-your-own-writes overlay: everything staged into kvBatch is noted
|
|
591
|
+
// here so a later event of the SAME transaction reads its own staged
|
|
592
|
+
// metadata instead of pre-transaction storage (see
|
|
593
|
+
// staged-transaction-metadata.ts). Lives for this one commit.
|
|
594
|
+
const staged = new StagedTransactionMetadata((schema, table) => this.getPkKeying(schema, table));
|
|
553
595
|
// DDL before DML: migrations take the lowest opSeqs.
|
|
554
596
|
const versionCounters = new Map();
|
|
555
597
|
for (const event of localSchema) {
|
|
556
598
|
await this.recordSchemaMigration(kvBatch, event, nextHlc, versionCounters);
|
|
557
599
|
}
|
|
558
600
|
for (const event of localData) {
|
|
559
|
-
await this.recordDataEvent(kvBatch, event, nextHlc, changes);
|
|
601
|
+
await this.recordDataEvent(kvBatch, event, nextHlc, changes, staged);
|
|
560
602
|
}
|
|
561
603
|
// Persist HLC clock state (wallTime/counter only — opSeq is never persisted).
|
|
562
604
|
persistHLCStateBatch(this, kvBatch);
|
|
@@ -575,6 +617,75 @@ export class SyncManagerImpl {
|
|
|
575
617
|
});
|
|
576
618
|
}
|
|
577
619
|
}
|
|
620
|
+
/**
|
|
621
|
+
* Drop the data events of any table that is no longer in the local basis — the
|
|
622
|
+
* schema oracle does not know it, typically because this same transaction dropped
|
|
623
|
+
* it and the schema is already gone by the time the commit group is delivered.
|
|
624
|
+
* A schemaless table has no sound pk identity (see `metadata/pk-identity.ts`), so
|
|
625
|
+
* its rows cannot be filed.
|
|
626
|
+
*
|
|
627
|
+
* Local capture is therefore best-effort at TABLE granularity: one unknown table
|
|
628
|
+
* costs only its own rows, never the rest of the transaction (which is what the
|
|
629
|
+
* handler's top-level catch would do — it aborts recording of every other table's
|
|
630
|
+
* rows AND the transaction's schema migrations).
|
|
631
|
+
*
|
|
632
|
+
* Runs BEFORE the HLC tick, so a transaction whose data is entirely skipped and
|
|
633
|
+
* that carries no schema events consumes no clock and emits no local change,
|
|
634
|
+
* mirroring the all-remote echo early-out. Filtering up front also means no event
|
|
635
|
+
* is ever half-recorded: the throw it replaces could fire *after* a tombstone had
|
|
636
|
+
* already been staged into the shared `WriteBatch`.
|
|
637
|
+
*
|
|
638
|
+
* The gate is basis MEMBERSHIP, not "this transaction dropped the table":
|
|
639
|
+
* `drop t; create t; insert into t` carries a drop event for `t` yet leaves it in
|
|
640
|
+
* basis, and its rows must still be captured. `localSchema` is consulted only to
|
|
641
|
+
* classify the log line — an expected skip reads differently from a table that
|
|
642
|
+
* vanished for some other reason.
|
|
643
|
+
*
|
|
644
|
+
* ONLY the unknown-table case is skipped. Any other keying failure — a pk column
|
|
645
|
+
* whose collation the wired `keyNormalizerResolver` does not know, say — still
|
|
646
|
+
* throws out of `recordDataEvent` and fails the transaction loudly; silently
|
|
647
|
+
* discarding those rows would bury a real misconfiguration.
|
|
648
|
+
*
|
|
649
|
+
* A relay-only manager (no `getTableSchema` oracle) reports every table in basis,
|
|
650
|
+
* so nothing is skipped there.
|
|
651
|
+
*/
|
|
652
|
+
filterCapturableDataEvents(events, localSchema) {
|
|
653
|
+
// NOTE: one oracle lookup per ROW (a schema-map hit today, and dwarfed by the
|
|
654
|
+
// per-row record work below). If `getTableSchema` ever becomes expensive, memo
|
|
655
|
+
// the verdict per table for the duration of the transaction.
|
|
656
|
+
const skipCounts = new Map();
|
|
657
|
+
const kept = events.filter(event => {
|
|
658
|
+
if (this.isTableInBasis(event.schemaName, event.tableName))
|
|
659
|
+
return true;
|
|
660
|
+
const tableKey = `${event.schemaName}.${event.tableName}`;
|
|
661
|
+
skipCounts.set(tableKey, (skipCounts.get(tableKey) ?? 0) + 1);
|
|
662
|
+
return false;
|
|
663
|
+
});
|
|
664
|
+
if (skipCounts.size > 0)
|
|
665
|
+
this.logSkippedTables(skipCounts, localSchema);
|
|
666
|
+
return kept;
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Report skipped tables: ONE line per table with its change count, never one per
|
|
670
|
+
* row. A table this same transaction dropped is expected and logs informationally;
|
|
671
|
+
* anything else is a warning — the table vanished for some other reason.
|
|
672
|
+
*/
|
|
673
|
+
logSkippedTables(skipCounts, localSchema) {
|
|
674
|
+
const droppedHere = new Set(localSchema
|
|
675
|
+
.filter(e => e.objectType === 'table' && e.type === 'drop')
|
|
676
|
+
.map(e => `${e.schemaName}.${e.objectName}`));
|
|
677
|
+
// NOTE: the only `console.log` in this package (everything else is warn/error) —
|
|
678
|
+
// an expected skip is not a warning. If the package ever gains a real logger
|
|
679
|
+
// abstraction, this is the call site that wants an `info` level.
|
|
680
|
+
for (const [tableKey, count] of skipCounts) {
|
|
681
|
+
if (droppedHere.has(tableKey)) {
|
|
682
|
+
console.log(`[Sync] Skipped ${count} change(s) for ${tableKey} — table dropped by the same transaction`);
|
|
683
|
+
}
|
|
684
|
+
else {
|
|
685
|
+
console.warn(`[Sync] Skipped ${count} change(s) for ${tableKey} — table is outside the local basis, so sync pk identity is unresolvable`);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
578
689
|
/**
|
|
579
690
|
* Record one local schema-change event as a migration. Allocates an `opSeq`
|
|
580
691
|
* (via `nextHlc`) only when the event maps to a tracked migration, so
|
|
@@ -586,6 +697,24 @@ export class SyncManagerImpl {
|
|
|
586
697
|
if (!migrationType)
|
|
587
698
|
return;
|
|
588
699
|
const { schemaName, objectName, ddl } = event;
|
|
700
|
+
// `alter_column` is the only tracked type that legitimately reaches here with
|
|
701
|
+
// no DDL — create/drop table and add/drop index all carry canonical text (see
|
|
702
|
+
// the store module's emit sites). The migration below still records and still
|
|
703
|
+
// advances the schema version (destructiveness comparison depends on it), but
|
|
704
|
+
// nothing here reaches a peer, so a device that alters a table stays silently
|
|
705
|
+
// diverged unless an operator sees this.
|
|
706
|
+
// See tickets/backlog/feat-sync-replicate-alter-table.md.
|
|
707
|
+
if (migrationType === 'alter_column' && !ddl?.trim()) {
|
|
708
|
+
console.warn(`[Sync] ${schemaName}.${objectName}: recording an ${migrationType} migration with no DDL — `
|
|
709
|
+
+ `this table alteration will NOT reach other synced devices`);
|
|
710
|
+
}
|
|
711
|
+
// NOTE: for an index migration `objectName` is the INDEX name, so this key
|
|
712
|
+
// is `<schema>.<index name>` — index-vs-index it is unambiguous only because
|
|
713
|
+
// `SchemaManager.createIndex` enforces index names unique per schema. Two
|
|
714
|
+
// same-named indexes on different tables would otherwise share one version
|
|
715
|
+
// counter and suppress each other's migrations. (An index name colliding
|
|
716
|
+
// with a *table* name still shares a counter — the key carries no object
|
|
717
|
+
// kind; tracked separately as bug-sync-migration-version-key-ignores-object-kind.)
|
|
589
718
|
const counterKey = `${schemaName}.${objectName}`;
|
|
590
719
|
let version = versionCounters.get(counterKey);
|
|
591
720
|
if (version === undefined) {
|
|
@@ -603,8 +732,13 @@ export class SyncManagerImpl {
|
|
|
603
732
|
/**
|
|
604
733
|
* Record one local data-change event: a tombstone (delete) or the changed
|
|
605
734
|
* column versions (insert/update). Each recorded fact consumes one `opSeq`.
|
|
735
|
+
*
|
|
736
|
+
* All reads of the row's current metadata go overlay-first (`staged`), so a
|
|
737
|
+
* transaction that touches one row more than once records the same metadata
|
|
738
|
+
* the equivalent sequence of separate transactions would — and every write
|
|
739
|
+
* staged into `batch` is noted back into the overlay.
|
|
606
740
|
*/
|
|
607
|
-
async recordDataEvent(batch, event, nextHlc, changes) {
|
|
741
|
+
async recordDataEvent(batch, event, nextHlc, changes, staged) {
|
|
608
742
|
const { schemaName, tableName, type, oldRow, newRow } = event;
|
|
609
743
|
const pk = event.key;
|
|
610
744
|
if (!pk) {
|
|
@@ -613,14 +747,16 @@ export class SyncManagerImpl {
|
|
|
613
747
|
}
|
|
614
748
|
if (type === 'delete') {
|
|
615
749
|
// Dedupe the change-log delete entry the same way columns are deduped: if a
|
|
616
|
-
// prior tombstone exists for this pk (a delete→reinsert→delete key reuse
|
|
617
|
-
//
|
|
618
|
-
//
|
|
619
|
-
//
|
|
620
|
-
// on the
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
this.
|
|
750
|
+
// prior tombstone exists for this pk (a delete→reinsert→delete key reuse —
|
|
751
|
+
// staged by this same transaction, or committed), its stale delete entry
|
|
752
|
+
// must go, so at most one delete entry survives per pk with HLC equal to
|
|
753
|
+
// the current tombstone. This keeps collectChangesSince's boundary
|
|
754
|
+
// detection (keyed on the log HLC) in lockstep with grouping (keyed on the
|
|
755
|
+
// resolved tombstone HLC) — see its LOAD-BEARING INVARIANT.
|
|
756
|
+
const existingHlc = staged.tombstoneHlc(schemaName, tableName, pk)
|
|
757
|
+
?? (await this.tombstones.getTombstone(schemaName, tableName, pk))?.hlc;
|
|
758
|
+
if (existingHlc) {
|
|
759
|
+
this.changeLog.deleteEntryBatch(batch, existingHlc, 'delete', schemaName, tableName, pk);
|
|
624
760
|
}
|
|
625
761
|
const hlc = nextHlc();
|
|
626
762
|
// Carry the row's last-known image (the engine `oldRow`) as a best-effort
|
|
@@ -630,7 +766,17 @@ export class SyncManagerImpl {
|
|
|
630
766
|
const priorRow = oldRow ? [...oldRow] : undefined;
|
|
631
767
|
this.tombstones.setTombstoneBatch(batch, schemaName, tableName, pk, hlc, priorRow);
|
|
632
768
|
this.changeLog.recordDeletionBatch(batch, hlc, schemaName, tableName, pk);
|
|
633
|
-
|
|
769
|
+
// The row's column versions die here, so their change-log entries must die
|
|
770
|
+
// with them — otherwise every deleted row leaves one orphan entry per column
|
|
771
|
+
// behind forever (see deleteRowVersionsAndLogEntries). The cleanup stages
|
|
772
|
+
// into this same `batch`, covering cells the transaction itself staged
|
|
773
|
+
// (batch ordering makes these later removals win), and consults the row's
|
|
774
|
+
// overlay state — captured BEFORE the notes below reset it.
|
|
775
|
+
await deleteRowVersionsAndLogEntries(this, batch, schemaName, tableName, pk, {
|
|
776
|
+
staged: staged.rowState(schemaName, tableName, pk),
|
|
777
|
+
});
|
|
778
|
+
staged.noteRowCleared(schemaName, tableName, pk);
|
|
779
|
+
staged.noteTombstone(schemaName, tableName, pk, hlc);
|
|
634
780
|
const change = {
|
|
635
781
|
type: 'delete',
|
|
636
782
|
schema: schemaName,
|
|
@@ -642,21 +788,27 @@ export class SyncManagerImpl {
|
|
|
642
788
|
changes.push(change);
|
|
643
789
|
}
|
|
644
790
|
else if (newRow) {
|
|
645
|
-
await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, nextHlc, changes);
|
|
791
|
+
await this.recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, nextHlc, changes, staged);
|
|
646
792
|
}
|
|
647
793
|
}
|
|
648
|
-
async recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, nextHlc, changes) {
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
}
|
|
794
|
+
async recordColumnVersions(batch, schemaName, tableName, pk, oldRow, newRow, nextHlc, changes, staged) {
|
|
795
|
+
// `col_${i}` fallback below is the live path for a RELAY-ONLY manager (no
|
|
796
|
+
// schema oracle at all). An oracle-wired manager can no longer reach here for
|
|
797
|
+
// a table the oracle does not know — filterCapturableDataEvents skipped it.
|
|
798
|
+
const columnNames = this.getTableSchema?.(schemaName, tableName)?.columns?.map(c => c.name);
|
|
654
799
|
for (let i = 0; i < newRow.length; i++) {
|
|
655
800
|
const oldValue = oldRow?.[i];
|
|
656
801
|
const newValue = newRow[i];
|
|
657
802
|
if (!oldRow || oldValue !== newValue) {
|
|
658
803
|
const column = columnNames?.[i] ?? `col_${i}`;
|
|
659
|
-
|
|
804
|
+
// Overlay first: a cell this same transaction rewrote reads its staged
|
|
805
|
+
// version (so the staged entry is the one deduped, and the before-image
|
|
806
|
+
// chains correctly); one it deleted reads as absent (a reinsert after a
|
|
807
|
+
// delete records no before-image, exactly like a first write).
|
|
808
|
+
const stagedCell = staged.columnVersion(schemaName, tableName, pk, column);
|
|
809
|
+
const oldVersion = stagedCell === null
|
|
810
|
+
? undefined // staged as deleted
|
|
811
|
+
: stagedCell ?? await this.columnVersions.getColumnVersion(schemaName, tableName, pk, column);
|
|
660
812
|
if (oldVersion) {
|
|
661
813
|
this.changeLog.deleteEntryBatch(batch, oldVersion.hlc, 'column', schemaName, tableName, pk, column);
|
|
662
814
|
}
|
|
@@ -670,6 +822,7 @@ export class SyncManagerImpl {
|
|
|
670
822
|
const version = { hlc, value: newValue, ...prior };
|
|
671
823
|
this.columnVersions.setColumnVersionBatch(batch, schemaName, tableName, pk, column, version);
|
|
672
824
|
this.changeLog.recordColumnChangeBatch(batch, hlc, schemaName, tableName, pk, column);
|
|
825
|
+
staged.noteColumnVersion(schemaName, tableName, pk, column, version);
|
|
673
826
|
const change = {
|
|
674
827
|
type: 'column',
|
|
675
828
|
schema: schemaName,
|
|
@@ -807,19 +960,19 @@ export class SyncManagerImpl {
|
|
|
807
960
|
* Resolve a change-log entry to the live {@link Change} it references, or
|
|
808
961
|
* `null` when the underlying column version / tombstone is gone — a stale log
|
|
809
962
|
* entry (e.g. a column overwritten, or a row deleted after the entry was
|
|
810
|
-
* written). The authoritative HLC
|
|
811
|
-
* the log key.
|
|
963
|
+
* written). The authoritative HLC, value AND pk come from the current version
|
|
964
|
+
* record, not the log key — the key carries only the lossy pk identity.
|
|
812
965
|
*/
|
|
813
966
|
async resolveLogEntry(logEntry) {
|
|
814
967
|
if (logEntry.entryType === 'column') {
|
|
815
|
-
const cv = await this.columnVersions.
|
|
968
|
+
const cv = await this.columnVersions.getColumnVersionByIdentity(logEntry.schema, logEntry.table, logEntry.identity, logEntry.column);
|
|
816
969
|
if (!cv)
|
|
817
970
|
return null;
|
|
818
971
|
const columnChange = {
|
|
819
972
|
type: 'column',
|
|
820
973
|
schema: logEntry.schema,
|
|
821
974
|
table: logEntry.table,
|
|
822
|
-
pk:
|
|
975
|
+
pk: cv.pk,
|
|
823
976
|
column: logEntry.column,
|
|
824
977
|
value: cv.value,
|
|
825
978
|
hlc: cv.hlc,
|
|
@@ -829,14 +982,14 @@ export class SyncManagerImpl {
|
|
|
829
982
|
};
|
|
830
983
|
return columnChange;
|
|
831
984
|
}
|
|
832
|
-
const tombstone = await this.tombstones.
|
|
985
|
+
const tombstone = await this.tombstones.getTombstoneByIdentity(logEntry.schema, logEntry.table, logEntry.identity);
|
|
833
986
|
if (!tombstone)
|
|
834
987
|
return null;
|
|
835
988
|
const deletion = {
|
|
836
989
|
type: 'delete',
|
|
837
990
|
schema: logEntry.schema,
|
|
838
991
|
table: logEntry.table,
|
|
839
|
-
pk:
|
|
992
|
+
pk: tombstone.pk,
|
|
840
993
|
hlc: tombstone.hlc,
|
|
841
994
|
// Re-emit the stored before-image (spread only when present) so a
|
|
842
995
|
// receiver/relay forwards the origin's last-known row image unchanged.
|
|
@@ -870,7 +1023,7 @@ export class SyncManagerImpl {
|
|
|
870
1023
|
type: 'column',
|
|
871
1024
|
schema: parsed.schema,
|
|
872
1025
|
table: parsed.table,
|
|
873
|
-
pk:
|
|
1026
|
+
pk: cv.pk,
|
|
874
1027
|
column: parsed.column,
|
|
875
1028
|
value: cv.value,
|
|
876
1029
|
hlc: cv.hlc,
|
|
@@ -890,7 +1043,7 @@ export class SyncManagerImpl {
|
|
|
890
1043
|
type: 'delete',
|
|
891
1044
|
schema: parsed.schema,
|
|
892
1045
|
table: parsed.table,
|
|
893
|
-
pk:
|
|
1046
|
+
pk: tombstone.pk,
|
|
894
1047
|
hlc: tombstone.hlc,
|
|
895
1048
|
...(tombstone.priorRow !== undefined ? { priorRow: tombstone.priorRow } : {}),
|
|
896
1049
|
};
|
|
@@ -1041,6 +1194,14 @@ export class SyncManagerImpl {
|
|
|
1041
1194
|
async getPeerSentState(peerSiteId) {
|
|
1042
1195
|
return this.peerStates.getPeerSentState(peerSiteId);
|
|
1043
1196
|
}
|
|
1197
|
+
/**
|
|
1198
|
+
* Drop tombstones past the retention horizon, together with the change-log
|
|
1199
|
+
* `delete` entry each one backs. The entry is only an index pointer at the
|
|
1200
|
+
* tombstone ({@link resolveLogEntry} returns null once the tombstone is gone),
|
|
1201
|
+
* so leaving it would accumulate one dead entry per delete the replica has ever
|
|
1202
|
+
* seen — the delete-side twin of the column cleanup in
|
|
1203
|
+
* {@link deleteRowVersionsAndLogEntries}. Both deletions share this batch.
|
|
1204
|
+
*/
|
|
1044
1205
|
async pruneTombstones() {
|
|
1045
1206
|
const now = Date.now();
|
|
1046
1207
|
let count = 0;
|
|
@@ -1050,6 +1211,15 @@ export class SyncManagerImpl {
|
|
|
1050
1211
|
const tombstone = deserializeTombstone(entry.value);
|
|
1051
1212
|
if (now - tombstone.createdAt > this.config.retentionHorizonMs) {
|
|
1052
1213
|
batch.delete(entry.key);
|
|
1214
|
+
const parsed = parseTombstoneKey(entry.key);
|
|
1215
|
+
if (parsed) {
|
|
1216
|
+
// Identity-based delete: the parsed key already carries the identity, so
|
|
1217
|
+
// no schema (which a since-retired table may no longer have) is needed.
|
|
1218
|
+
this.changeLog.deleteEntryByIdentityBatch(batch, tombstone.hlc, 'delete', parsed.schema, parsed.table, parsed.identity);
|
|
1219
|
+
}
|
|
1220
|
+
else {
|
|
1221
|
+
console.warn('[Sync] Unparseable tombstone key during prune — its change-log entry is left orphaned');
|
|
1222
|
+
}
|
|
1053
1223
|
count++;
|
|
1054
1224
|
}
|
|
1055
1225
|
}
|