@syncular/client 0.78.1 → 0.78.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apply.d.ts +4 -0
- package/dist/apply.js +7 -5
- package/dist/blob.js +2 -3
- package/dist/client.d.ts +2 -1
- package/dist/client.js +71 -66
- package/dist/previous-version.js +2 -3
- package/dist/schema.d.ts +1 -0
- package/dist/schema.js +42 -20
- package/dist/state.d.ts +8 -1
- package/dist/state.js +13 -3
- package/package.json +3 -3
- package/src/apply.ts +14 -5
- package/src/blob.ts +2 -5
- package/src/client.ts +90 -80
- package/src/previous-version.ts +2 -5
- package/src/schema.ts +49 -32
- package/src/state.ts +24 -3
package/dist/apply.d.ts
CHANGED
|
@@ -94,6 +94,10 @@ export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledC
|
|
|
94
94
|
* to the next (§5.3).
|
|
95
95
|
*/
|
|
96
96
|
readonly chunkRows?: Map<string, number>;
|
|
97
|
+
/** Rows of this image an earlier attempt committed (§5.6). */
|
|
98
|
+
readonly appliedRows?: number | undefined;
|
|
99
|
+
/** Runs inside each chunk's transaction with the rows applied so far. */
|
|
100
|
+
readonly onChunk?: (appliedRows: number) => void;
|
|
97
101
|
}): Promise<number>;
|
|
98
102
|
/**
|
|
99
103
|
* Apply a decoded rows segment: each block in one local transaction
|
package/dist/apply.js
CHANGED
|
@@ -362,8 +362,9 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
362
362
|
}
|
|
363
363
|
return [boundary, predicates.join(' AND ') || 'true', params];
|
|
364
364
|
};
|
|
365
|
-
|
|
366
|
-
let
|
|
365
|
+
const resumed = options.appliedRows ?? 0;
|
|
366
|
+
let [boundary, where, params] = range(resumed > 0 ? range(undefined, resumed)[0] : undefined, 1024);
|
|
367
|
+
let applied = resumed;
|
|
367
368
|
// §5.3: a table's first chunk holds 1,024 rows; a later range image
|
|
368
369
|
// starts at the size its previous range reached. A later chunk holds at least
|
|
369
370
|
// 1,024 rows, at most twice the rows of the one before, at most 65,536,
|
|
@@ -371,7 +372,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
371
372
|
// to its COMMIT; a replaying chunk holds one statement.
|
|
372
373
|
for (let size = (!options.clearFirst && options.chunkRows?.get(table.name)) || 1024;;) {
|
|
373
374
|
if (options.isCurrent?.() === false)
|
|
374
|
-
return applied;
|
|
375
|
+
return applied - resumed;
|
|
375
376
|
const started = now();
|
|
376
377
|
let chunk = 0;
|
|
377
378
|
const first = applied === 0;
|
|
@@ -399,7 +400,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
399
400
|
const skipped = new Set();
|
|
400
401
|
const landing = rows.filter((row) => lands(db, table, String(row[table.primaryKey]), Number(row._syncular_version), kept, skipped));
|
|
401
402
|
for (;;) {
|
|
402
|
-
db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where}
|
|
403
|
+
db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} AS image WHERE ${where} AND NOT EXISTS(SELECT 1 FROM ${quoteIdent(table.name)} AS held WHERE held.${primaryKey}=image.${primaryKey} AND held.${version}>=image._syncular_version) ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`, params);
|
|
403
404
|
const copied = Number(db.query(`SELECT count(*)AS n FROM ${source} WHERE ${where}`, params)[0]?.n);
|
|
404
405
|
applied += copied;
|
|
405
406
|
chunk += copied;
|
|
@@ -418,6 +419,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
418
419
|
rowValueToJson(row[column.name]),
|
|
419
420
|
])), Number(row._syncular_version));
|
|
420
421
|
}
|
|
422
|
+
options.onChunk?.(applied);
|
|
421
423
|
return rebaseScope(scope, kept, skipped);
|
|
422
424
|
});
|
|
423
425
|
const duration = now() - started;
|
|
@@ -432,7 +434,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
432
434
|
imageInvalid('image applied row count does not match descriptor');
|
|
433
435
|
// A file replica's WAL keeps its high-water size until truncated.
|
|
434
436
|
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
435
|
-
return applied;
|
|
437
|
+
return applied - resumed;
|
|
436
438
|
});
|
|
437
439
|
}
|
|
438
440
|
/**
|
package/dist/blob.js
CHANGED
|
@@ -8,7 +8,7 @@ export const BLOB_COMMIT_REFS = '_syncular_blob_commit_refs';
|
|
|
8
8
|
*/
|
|
9
9
|
import { parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
10
10
|
import { ClientSyncError } from './errors.js';
|
|
11
|
-
import { quoteIdent } from './schema.js';
|
|
11
|
+
import { quoteIdent, tableExists } from './schema.js';
|
|
12
12
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
13
13
|
export async function computeBlobId(bytes) {
|
|
14
14
|
const input = bytes.buffer instanceof ArrayBuffer
|
|
@@ -21,8 +21,7 @@ export async function computeBlobId(bytes) {
|
|
|
21
21
|
return `sha256:${hex}`;
|
|
22
22
|
}
|
|
23
23
|
export function ensureBlobSchema(db) {
|
|
24
|
-
|
|
25
|
-
if (existing.length > 0) {
|
|
24
|
+
if (tableExists(db, '_syncular_blobs')) {
|
|
26
25
|
const columns = db
|
|
27
26
|
.query('PRAGMA table_info(_syncular_blobs)')
|
|
28
27
|
.map((row) => row.name)
|
package/dist/client.d.ts
CHANGED
|
@@ -187,7 +187,8 @@ export interface SyncClientConfig {
|
|
|
187
187
|
* `transport.timeout` (`causeKind` `heartbeat`).
|
|
188
188
|
*/
|
|
189
189
|
readonly realtimeHeartbeatMs?: number;
|
|
190
|
-
/** Timer seam for the heartbeat watchdog
|
|
190
|
+
/** Timer seam for the heartbeat watchdog and the `syncUntilIdle` round
|
|
191
|
+
* bound; defaults to `setTimeout`. */
|
|
191
192
|
readonly schedule?: (callback: () => void, delayMs: number) => () => void;
|
|
192
193
|
/** Stable per-device id (§1.5); defaults to a persisted random UUID. */
|
|
193
194
|
readonly clientId?: string;
|
package/dist/client.js
CHANGED
|
@@ -30,7 +30,7 @@ import { appendOutboxCommit, chainsOn, deleteOutboxCommit, encodeOutboxCommit, l
|
|
|
30
30
|
import { activeFailureRecords, isStoredOperation, listCommitOutcomes, persistCommitOutcomeResolution, persistedCommitOutcome, pruneCommitOutcomes, commitOutcome as readCommitOutcome, recordCommitOutcome, scrubPurgedOutcomes, } from './outcomes.js';
|
|
31
31
|
import { buildPreviousVersionAudit, capturePreviousVersion, clearPreviousVersionAudit, clearPreviousVersionRefusal, dropPreviousVersionContainer, loadLocalSchemaDescriptor, PREVIOUS_VERSION_CONTAINER_NAME, PREVIOUS_VERSION_MAX_LIMIT, readPreviousVersionContainer, readPreviousVersionRows, setLocalSchemaDescriptor, storedPreviousVersionAudit, storedPreviousVersionRefusal, withPreviousVersionContainer, writePreviousVersionAudit, writePreviousVersionRefusal, } from './previous-version.js';
|
|
32
32
|
import { bytesToHex, compileClientSchema, hexToBytes, matchesLocalScopes, resetLocalReplica, ensureLocalBookkeepingSchema, ensureLocalSyncedSchema, fromSqlValue, coerceSqlRepresentation, localColumnType, jsonToRowValue, LOCAL_SCHEMA_VERSION_KEY, normalizeRecordKeys, OPTIMISTIC_VERSION, quoteIdent, recordToRowValues, rowValueToJson, SEALED_COLUMN, SEALED_KEY_COLUMN, SYNC_VERSION_COLUMN, stripSyncColumns, toSqlValue, } from './schema.js';
|
|
33
|
-
import { bootstrapping, bumpLocalRevision, covers, deleteUnit, ECHO_DIGEST_KEY, getLocalRevision, getMeta, getUnit, loadUnits, META_TABLE, owning, pairScopes, pruneUnknownUnits, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY, replicaCursor, saveUnit, saveUnitScopeSchema, setMeta, unitOwners, } from './state.js';
|
|
33
|
+
import { bootstrapping, bumpLocalRevision, covers, deleteUnit, ECHO_DIGEST_KEY, getLocalRevision, getMeta, getUnit, loadUnits, META_TABLE, owning, pairScopes, pruneUnknownUnits, recordAppliedRows, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY, replicaCursor, saveUnit, saveUnitScopeSchema, setMeta, unitOwners, } from './state.js';
|
|
34
34
|
import { deletePendingEviction, loadPendingEvictions, loadWindowUnits, savePendingEviction, windowBaseKey, windowRefs, WINDOWS_TABLE, windowUnitScopes, } from './window.js';
|
|
35
35
|
/** Stable client-local error while protected operations are preflight-gated. */
|
|
36
36
|
export const SECURITY_PREFLIGHT_REQUIRED_CODE = 'client.security_preflight_required';
|
|
@@ -71,8 +71,6 @@ async function sha256Hex(bytes) {
|
|
|
71
71
|
}
|
|
72
72
|
const invalidHostResponse = (message) => new ClientSyncError('client.invalid_host_response', message);
|
|
73
73
|
const outcomeNotFound = () => new ClientSyncError('sync.outcome_not_found', 'no durable outcome exists for the commit');
|
|
74
|
-
/** The position a unit presents in a pull (§5.6 applied-segment rule). */
|
|
75
|
-
const position = (unit) => JSON.stringify([unit.id, unit.cursor, unit.bootstrapState]);
|
|
76
74
|
function emptySummary(pushed) {
|
|
77
75
|
return {
|
|
78
76
|
pushed,
|
|
@@ -198,13 +196,10 @@ export class SyncClient {
|
|
|
198
196
|
#pendingRound;
|
|
199
197
|
#needsPull = false;
|
|
200
198
|
#progress = new ProgressEmitter();
|
|
201
|
-
/**
|
|
202
|
-
* §5.6 applied-segment rule: unit position → the segments applied in
|
|
203
|
-
* sections whose SUB_END did not persist.
|
|
204
|
-
*/
|
|
205
|
-
#appliedSegments = new Map();
|
|
206
199
|
/** §5.3: a table's next image chunk size, carried across its ranges. */
|
|
207
200
|
#chunkRows = new Map();
|
|
201
|
+
/** §7.8: restarts the running drive's round bound when rows apply. */
|
|
202
|
+
#roundProgressed;
|
|
208
203
|
#progressAttempt = 0;
|
|
209
204
|
/**
|
|
210
205
|
* True from the synchronous entry of `sync()` until its serialized round
|
|
@@ -3345,26 +3340,16 @@ export class SyncClient {
|
|
|
3345
3340
|
deferredCommits: 0,
|
|
3346
3341
|
schemaFloor: undefined,
|
|
3347
3342
|
};
|
|
3348
|
-
const result = (budgetExhausted) =>
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
revoked: aggregate.revoked,
|
|
3359
|
-
failed: aggregate.failed,
|
|
3360
|
-
...(aggregate.deferredCommits > 0
|
|
3361
|
-
? { deferredCommits: aggregate.deferredCommits }
|
|
3362
|
-
: {}),
|
|
3363
|
-
...(aggregate.schemaFloor !== undefined
|
|
3364
|
-
? { schemaFloor: aggregate.schemaFloor }
|
|
3365
|
-
: {}),
|
|
3366
|
-
budgetExhausted,
|
|
3367
|
-
});
|
|
3343
|
+
const result = (budgetExhausted) => {
|
|
3344
|
+
const { deferredCommits, schemaFloor, ...summary } = aggregate;
|
|
3345
|
+
return {
|
|
3346
|
+
...summary,
|
|
3347
|
+
...(deferredCommits > 0 ? { deferredCommits } : {}),
|
|
3348
|
+
...(schemaFloor !== undefined ? { schemaFloor } : {}),
|
|
3349
|
+
budgetExhausted,
|
|
3350
|
+
};
|
|
3351
|
+
};
|
|
3352
|
+
const states = () => JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]));
|
|
3368
3353
|
const startedAtMs = this.#now();
|
|
3369
3354
|
let completed = false;
|
|
3370
3355
|
for (let round = 0; round < budget; round++) {
|
|
@@ -3372,57 +3357,69 @@ export class SyncClient {
|
|
|
3372
3357
|
// like an exhausted round budget; the next drive resumes.
|
|
3373
3358
|
if (completed && this.#now() - startedAtMs >= timeoutMs)
|
|
3374
3359
|
return result(true);
|
|
3375
|
-
const before =
|
|
3376
|
-
// Each round is bounded by `timeoutMs`
|
|
3377
|
-
// every transport call of the round.
|
|
3360
|
+
const before = states();
|
|
3361
|
+
// Each round is bounded by `timeoutMs` from its start or its last
|
|
3362
|
+
// applied rows. One signal reaches every transport call of the round.
|
|
3378
3363
|
const controller = new AbortController();
|
|
3379
|
-
let
|
|
3364
|
+
let cancel;
|
|
3380
3365
|
const sync = this.#sync(controller.signal);
|
|
3381
3366
|
const last = await (timeoutMs === Number.POSITIVE_INFINITY
|
|
3382
3367
|
? sync
|
|
3383
3368
|
: Promise.race([
|
|
3384
3369
|
sync,
|
|
3385
3370
|
new Promise((_, reject) => {
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
3393
|
-
|
|
3371
|
+
this.#roundProgressed = () => {
|
|
3372
|
+
cancel?.();
|
|
3373
|
+
cancel = this.#schedule(() => {
|
|
3374
|
+
const error = new ClientSyncError('sync.idle_timeout', 'syncUntilIdle exceeded its deadline', true, { timeoutMs, rounds: round });
|
|
3375
|
+
// The transport may never answer; release the round so the
|
|
3376
|
+
// client can run another one after the caller sees the error.
|
|
3377
|
+
controller.abort(error);
|
|
3378
|
+
this.#abortPendingRound(error);
|
|
3379
|
+
reject(error);
|
|
3380
|
+
}, timeoutMs);
|
|
3381
|
+
};
|
|
3382
|
+
this.#roundProgressed();
|
|
3394
3383
|
}),
|
|
3395
|
-
])).finally(() =>
|
|
3384
|
+
])).finally(() => {
|
|
3385
|
+
cancel?.();
|
|
3386
|
+
this.#roundProgressed = undefined;
|
|
3387
|
+
});
|
|
3396
3388
|
completed = true;
|
|
3397
3389
|
// Historical counters accumulate; current readiness (`bootstrapping`,
|
|
3398
3390
|
// `deferredCommits`, `schemaFloor`) describes the latest round.
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3391
|
+
for (const key of [
|
|
3392
|
+
'pushed',
|
|
3393
|
+
'commitsApplied',
|
|
3394
|
+
'segmentRowsApplied',
|
|
3395
|
+
])
|
|
3396
|
+
aggregate[key] += last[key];
|
|
3397
|
+
for (const key of [
|
|
3398
|
+
'applied',
|
|
3399
|
+
'rejected',
|
|
3400
|
+
'retryable',
|
|
3401
|
+
'resets',
|
|
3402
|
+
'revoked',
|
|
3403
|
+
'failed',
|
|
3404
|
+
])
|
|
3405
|
+
aggregate[key].push(...last[key]);
|
|
3403
3406
|
aggregate.conflicts.push(...last.conflicts);
|
|
3404
|
-
aggregate.commitsApplied += last.commitsApplied;
|
|
3405
|
-
aggregate.segmentRowsApplied += last.segmentRowsApplied;
|
|
3406
3407
|
aggregate.bootstrapping = [...last.bootstrapping];
|
|
3407
|
-
aggregate.resets.push(...last.resets);
|
|
3408
|
-
aggregate.revoked.push(...last.revoked);
|
|
3409
|
-
aggregate.failed.push(...last.failed);
|
|
3410
3408
|
aggregate.deferredCommits = last.deferredCommits ?? 0;
|
|
3411
3409
|
aggregate.schemaFloor = last.schemaFloor;
|
|
3412
3410
|
// A disabled transport or a schema floor ends the loop, not the budget.
|
|
3413
|
-
if (!this.#transportEnabled ||
|
|
3411
|
+
if (!this.#transportEnabled || aggregate.schemaFloor)
|
|
3414
3412
|
return result(false);
|
|
3415
3413
|
if (maxRounds === undefined &&
|
|
3416
3414
|
last.segmentRowsApplied > 0 &&
|
|
3417
3415
|
last.bootstrapping.length > 0 &&
|
|
3418
|
-
before !==
|
|
3419
|
-
JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState])))
|
|
3416
|
+
before !== states())
|
|
3420
3417
|
round = -1;
|
|
3421
3418
|
if (last.commitsApplied === 0 &&
|
|
3422
3419
|
last.segmentRowsApplied === 0 &&
|
|
3423
3420
|
last.bootstrapping.length === 0 &&
|
|
3424
3421
|
last.resets.length === 0 &&
|
|
3425
|
-
|
|
3422
|
+
!aggregate.deferredCommits &&
|
|
3426
3423
|
!this.#needsPull) {
|
|
3427
3424
|
return result(false);
|
|
3428
3425
|
}
|
|
@@ -3918,7 +3915,7 @@ export class SyncClient {
|
|
|
3918
3915
|
else if (frame.type === 'SEGMENT_REF' &&
|
|
3919
3916
|
open !== undefined &&
|
|
3920
3917
|
this.#config.segments !== undefined &&
|
|
3921
|
-
|
|
3918
|
+
open.applied?.[frame.segmentId] !== frame.rowCount) {
|
|
3922
3919
|
queued.push([frame, open.scopes]);
|
|
3923
3920
|
}
|
|
3924
3921
|
}
|
|
@@ -4022,15 +4019,17 @@ export class SyncClient {
|
|
|
4022
4019
|
image.rowCursor !== undefined ||
|
|
4023
4020
|
image.nextRowCursor !== undefined))
|
|
4024
4021
|
throw invalidRequest('unadvertised or paged sqlite segment (§4.2, §5.3)');
|
|
4025
|
-
const applied = this.#appliedSegments.get(position(section.unit)) ??
|
|
4026
|
-
new Set();
|
|
4027
|
-
if (frame.type === 'SEGMENT_REF' && applied.has(frame.segmentId)) {
|
|
4028
|
-
// §5.6: applied in an earlier attempt at this position.
|
|
4029
|
-
section.seeded.add(frame.table);
|
|
4030
|
-
break;
|
|
4031
|
-
}
|
|
4032
4022
|
let bytes;
|
|
4023
|
+
let appliedRows;
|
|
4033
4024
|
if (frame.type === 'SEGMENT_REF') {
|
|
4025
|
+
// §5.6: rows an earlier attempt at this position applied. A
|
|
4026
|
+
// partly applied image already ran its table's first-page clear.
|
|
4027
|
+
appliedRows = section.unit.applied?.[frame.segmentId];
|
|
4028
|
+
if (appliedRows !== undefined) {
|
|
4029
|
+
section.seeded.add(frame.table);
|
|
4030
|
+
if (appliedRows === frame.rowCount)
|
|
4031
|
+
break;
|
|
4032
|
+
}
|
|
4034
4033
|
this.#progress.update({
|
|
4035
4034
|
...NO_PAYLOAD,
|
|
4036
4035
|
phase: 'download',
|
|
@@ -4083,11 +4082,15 @@ export class SyncClient {
|
|
|
4083
4082
|
? image.rowCount > 0
|
|
4084
4083
|
: segment.blocks.some((block) => block.length > 0), resetEpoch, image !== undefined);
|
|
4085
4084
|
return image !== undefined
|
|
4086
|
-
? applySqliteSegment(this.#db, this.#schema, table, bytes, image,
|
|
4085
|
+
? applySqliteSegment(this.#db, this.#schema, table, bytes, image, {
|
|
4086
|
+
...options,
|
|
4087
|
+
appliedRows,
|
|
4088
|
+
onChunk: (rows) => recordAppliedRows(this.#db, unitId, image.segmentId, rows),
|
|
4089
|
+
})
|
|
4087
4090
|
: applyRowsSegment(this.#db, this.#schema, table, segment, options, this.#encryption);
|
|
4088
4091
|
});
|
|
4089
4092
|
if (frame.type === 'SEGMENT_REF' && !section.skip)
|
|
4090
|
-
this.#
|
|
4093
|
+
recordAppliedRows(this.#db, unitId, frame.segmentId, frame.rowCount);
|
|
4091
4094
|
break;
|
|
4092
4095
|
}
|
|
4093
4096
|
case 'SUB_END': {
|
|
@@ -4098,7 +4101,6 @@ export class SyncClient {
|
|
|
4098
4101
|
!section.skip &&
|
|
4099
4102
|
this.#localResetEpoch === resetEpoch) {
|
|
4100
4103
|
this.#finishSection(section, frame, summary);
|
|
4101
|
-
this.#appliedSegments.delete(position(section.unit));
|
|
4102
4104
|
}
|
|
4103
4105
|
section = undefined;
|
|
4104
4106
|
// The counters stay: `complete` and `failed` report them.
|
|
@@ -4504,7 +4506,10 @@ export class SyncClient {
|
|
|
4504
4506
|
effective,
|
|
4505
4507
|
owners,
|
|
4506
4508
|
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
4507
|
-
onProgress: (rowsProcessed) =>
|
|
4509
|
+
onProgress: (rowsProcessed) => {
|
|
4510
|
+
this.#progress.update({ rowsProcessed });
|
|
4511
|
+
this.#roundProgressed?.();
|
|
4512
|
+
},
|
|
4508
4513
|
transaction: (fn) => {
|
|
4509
4514
|
const started = performance.now();
|
|
4510
4515
|
let written = started;
|
package/dist/previous-version.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ClientSyncError } from './errors.js';
|
|
2
2
|
import { INVALID_HOST_RESPONSE_CODE } from './local-rebootstrap.js';
|
|
3
|
-
import { fromSqlValue, jsonToRowValue, localColumnType, quoteIdent, rowValueToJson, } from './schema.js';
|
|
3
|
+
import { fromSqlValue, jsonToRowValue, localColumnType, quoteIdent, rowValueToJson, tableExists, } from './schema.js';
|
|
4
4
|
import { getMeta, setMeta, META_TABLE } from './state.js';
|
|
5
5
|
/** `_syncular_meta` key holding the persisted {@link LocalSchemaDescriptor}. */
|
|
6
6
|
const LOCAL_SCHEMA_DESCRIPTOR_KEY = 'localSchemaDescriptor';
|
|
@@ -228,8 +228,7 @@ export function withPreviousVersionContainer(db, fn, removeFile = false) {
|
|
|
228
228
|
}
|
|
229
229
|
/** Read the container's own metadata record; `undefined` when absent/corrupt. */
|
|
230
230
|
export function readPreviousVersionContainer(db) {
|
|
231
|
-
|
|
232
|
-
if (meta.length === 0)
|
|
231
|
+
if (!tableExists(db, CONTAINER_META_TABLE))
|
|
233
232
|
return undefined;
|
|
234
233
|
const raw = db.query(`SELECT record FROM ${quoteIdent(CONTAINER_META_TABLE)} WHERE id=1`)[0]?.record;
|
|
235
234
|
if (raw === undefined)
|
package/dist/schema.d.ts
CHANGED
|
@@ -110,6 +110,7 @@ export declare function matchesLocalScopes(table: CompiledClientTable, scopes: R
|
|
|
110
110
|
*/
|
|
111
111
|
export declare function pkSqlValue(table: CompiledClientTable, rowId: string): SqlValue;
|
|
112
112
|
export declare function quoteIdent(name: string): string;
|
|
113
|
+
export declare function tableExists(db: ClientDatabase, name: string): boolean;
|
|
113
114
|
/**
|
|
114
115
|
* §5.11: the app-side type of a column for local (plaintext) storage. For an
|
|
115
116
|
* encrypted column this is `declaredType` — the local mirror stays plaintext,
|
package/dist/schema.js
CHANGED
|
@@ -216,6 +216,11 @@ export function pkSqlValue(table, rowId) {
|
|
|
216
216
|
export function quoteIdent(name) {
|
|
217
217
|
return `"${name.replaceAll('"', '""')}"`;
|
|
218
218
|
}
|
|
219
|
+
export function tableExists(db, name) {
|
|
220
|
+
return (db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
|
|
221
|
+
name,
|
|
222
|
+
]).length > 0);
|
|
223
|
+
}
|
|
219
224
|
/**
|
|
220
225
|
* §5.11: the app-side type of a column for local (plaintext) storage. For an
|
|
221
226
|
* encrypted column this is `declaredType` — the local mirror stays plaintext,
|
|
@@ -288,21 +293,22 @@ const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
|
|
|
288
293
|
* Limitation: because a BEFORE trigger runs before SQLite resolves the
|
|
289
294
|
* conflict, an `OR IGNORE` write whose row is dropped still executes these
|
|
290
295
|
* displacement DELETEs, transiently removing the surviving row's projection
|
|
291
|
-
* entry.
|
|
292
|
-
*
|
|
296
|
+
* entry. The same holds for an upsert whose `DO UPDATE ... WHERE` skips the
|
|
297
|
+
* row, so Syncular's mirror writes never offer a row that will not replace
|
|
298
|
+
* (§5.3); see the guard-block comment below for the full rationale. An open
|
|
299
|
+
* rebuilds a projection whose mapping holds a different row count than its
|
|
300
|
+
* source table.
|
|
293
301
|
*/
|
|
294
302
|
function createFtsProjection(db, table, index) {
|
|
295
303
|
const projection = quoteIdent(index.name);
|
|
296
304
|
const sourceTable = quoteIdent(table.name);
|
|
297
305
|
const primaryKey = quoteIdent(table.primaryKey);
|
|
298
|
-
const existed = db
|
|
306
|
+
const existed = tableExists(db, index.name);
|
|
299
307
|
const indexedColumns = index.columns.map(quoteIdent);
|
|
300
308
|
const tokenizer = index.tokenize.replaceAll("'", "''");
|
|
301
309
|
db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${projection} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED,${indexedColumns.join(', ')},tokenize='${tokenizer}')`);
|
|
302
310
|
const mapping = quoteIdent(`_syncular_fts_${index.name}`);
|
|
303
|
-
const mapped = db.
|
|
304
|
-
`_syncular_fts_${index.name}`,
|
|
305
|
-
]).length > 0;
|
|
311
|
+
const mapped = tableExists(db, `_syncular_fts_${index.name}`);
|
|
306
312
|
db.exec(`CREATE TABLE IF NOT EXISTS ${mapping}(id INTEGER PRIMARY KEY,source_id TEXT NOT NULL UNIQUE)`);
|
|
307
313
|
const sourceId = `CAST(${primaryKey} AS TEXT)`;
|
|
308
314
|
const newSourceId = `CAST(new.${primaryKey} AS TEXT)`;
|
|
@@ -332,16 +338,19 @@ function createFtsProjection(db, table, index) {
|
|
|
332
338
|
// `UPDATE OR REPLACE` needs no BEFORE guard: the new pk equals the displaced
|
|
333
339
|
// row's pk, so the AFTER UPDATE delete of the new source id already clears it.
|
|
334
340
|
//
|
|
335
|
-
// Known limitation
|
|
336
|
-
//
|
|
337
|
-
//
|
|
338
|
-
//
|
|
339
|
-
// SURVIVING row's projection entry (a
|
|
340
|
-
// rewritten). Moving cleanup to AFTER
|
|
341
|
-
// reintroduce the REPLACE-doesn't-fire-
|
|
342
|
-
// per write, so the guards stay
|
|
343
|
-
//
|
|
344
|
-
//
|
|
341
|
+
// Known limitation: a BEFORE trigger fires before SQLite resolves the
|
|
342
|
+
// conflict, and SQLite exposes no signal for the eventual resolution. So an
|
|
343
|
+
// `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is dropped on conflict,
|
|
344
|
+
// and an upsert whose `DO UPDATE ... WHERE` skips the row, still run these
|
|
345
|
+
// displacement DELETEs, removing the SURVIVING row's projection entry (a
|
|
346
|
+
// false negative until that row is next rewritten). Moving cleanup to AFTER
|
|
347
|
+
// triggers would dodge the no-op but reintroduce the REPLACE-doesn't-fire-
|
|
348
|
+
// DELETE gap and a full-projection scan per write, so the guards stay
|
|
349
|
+
// BEFORE. Syncular's own mirror writes use an unconditional
|
|
350
|
+
// `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts); the image copy filters
|
|
351
|
+
// held rows at the same or a newer version out of its SELECT instead of a
|
|
352
|
+
// `DO UPDATE ... WHERE`. Only hand-written `OR IGNORE` against a mirror
|
|
353
|
+
// table is exposed.
|
|
345
354
|
for (const suffix of ['bi', 'ai', 'ad', 'au', 'bu']) {
|
|
346
355
|
db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(`${index.name}_${suffix}`)}`);
|
|
347
356
|
}
|
|
@@ -380,10 +389,16 @@ function createFtsProjection(db, table, index) {
|
|
|
380
389
|
const updateGuardBody = displacedByUnique.map(deleteDisplaced).join('; ');
|
|
381
390
|
createTrigger('bu', 'BEFORE UPDATE', updateGuardBody, updateGuardCondition);
|
|
382
391
|
}
|
|
383
|
-
|
|
392
|
+
// Every source row has exactly one mapping row, so a count mismatch marks a
|
|
393
|
+
// projection an earlier release damaged; the open rebuilds it.
|
|
394
|
+
const stale = !existed ||
|
|
395
|
+
(mapped &&
|
|
396
|
+
Number(db.query(`SELECT (SELECT count(*) FROM ${mapping})=(SELECT count(*) FROM ${sourceTable}) AS whole`)[0]?.whole) !== 1);
|
|
397
|
+
if (stale) {
|
|
398
|
+
db.exec(`DELETE FROM ${projection}`);
|
|
384
399
|
db.exec(`INSERT INTO ${projection}(${projectionColumns})SELECT ${sourceId},${indexedColumns.join(', ')} FROM ${sourceTable}`);
|
|
385
400
|
}
|
|
386
|
-
if (!mapped ||
|
|
401
|
+
if (!mapped || stale) {
|
|
387
402
|
db.exec(`DELETE FROM ${mapping}`);
|
|
388
403
|
db.exec(`INSERT INTO ${mapping}(id,source_id)SELECT rowid,${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${projection}`);
|
|
389
404
|
}
|
|
@@ -438,7 +453,7 @@ export function ensureLocalBookkeepingSchema(db) {
|
|
|
438
453
|
createTable(BLOB_COMMIT_REFS, 'commit_id TEXT NOT NULL,blob_id TEXT NOT NULL,PRIMARY KEY(commit_id,blob_id)');
|
|
439
454
|
db.exec(`CREATE INDEX IF NOT EXISTS ${BLOB_COMMIT_REFS}_body ON ${BLOB_COMMIT_REFS}(blob_id)`);
|
|
440
455
|
db.exec(`INSERT OR IGNORE INTO ${META_TABLE}(key,value)VALUES('localRevision','0')`);
|
|
441
|
-
const outboxExisted = db
|
|
456
|
+
const outboxExisted = tableExists(db, OUTBOX_TABLE);
|
|
442
457
|
createTable(OUTBOX_TABLE, 'seq INTEGER PRIMARY KEY AUTOINCREMENT,client_commit_id TEXT NOT NULL UNIQUE,created_at_ms INTEGER NOT NULL,operations TEXT NOT NULL');
|
|
443
458
|
// §7.1: `outboxId` names the outbox sequence. A new outbox table starts
|
|
444
459
|
// a new sequence, so it replaces the id in the same transaction.
|
|
@@ -463,6 +478,13 @@ export function ensureLocalBookkeepingSchema(db) {
|
|
|
463
478
|
}
|
|
464
479
|
db.exec(`CREATE INDEX IF NOT EXISTS ${OUTCOMES_TABLE}_resolution_seq ON ${OUTCOMES_TABLE}(resolution,seq)`);
|
|
465
480
|
createTable(UNITS_TABLE, "id TEXT PRIMARY KEY,scopes TEXT NOT NULL,explicit INTEGER NOT NULL DEFAULT 0,status TEXT NOT NULL DEFAULT 'active',reason_code TEXT,cursor INTEGER DEFAULT -1,bootstrap_state TEXT,seeded TEXT NOT NULL DEFAULT '[]',echo TEXT");
|
|
481
|
+
// §5.6 applied-segment record, added after the table shipped.
|
|
482
|
+
try {
|
|
483
|
+
db.exec(`ALTER TABLE ${UNITS_TABLE} ADD COLUMN applied TEXT`);
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
// column already exists
|
|
487
|
+
}
|
|
466
488
|
// §4.8 window registry: base + value → the unit it holds, the
|
|
467
489
|
// completeness oracle and one of a unit's references (§4.1).
|
|
468
490
|
createTable(WINDOWS_TABLE, 'base TEXT NOT NULL,unit TEXT NOT NULL,sub_id TEXT NOT NULL,PRIMARY KEY(base,unit)');
|
|
@@ -474,7 +496,7 @@ export function ensureLocalBookkeepingSchema(db) {
|
|
|
474
496
|
// rows and pending evictions and registers its units fresh. Its
|
|
475
497
|
// subscriptions, cursors, echoes, resume tokens and window entries go.
|
|
476
498
|
const legacyEvictions = '_syncular_window_pending_evict';
|
|
477
|
-
if (db
|
|
499
|
+
if (tableExists(db, legacyEvictions)) {
|
|
478
500
|
if (db.query(`SELECT 1 FROM ${legacyEvictions} WHERE NOT json_valid(effective_scopes) OR json_type(effective_scopes)<>'object' OR EXISTS(SELECT 1 FROM json_each(effective_scopes) WHERE json_type(value)<>'array' OR EXISTS(SELECT 1 FROM json_each(value) v WHERE v.type<>'text'))`).length > 0)
|
|
479
501
|
throw new ClientSyncError('sync.local_corrupt', 'a version 3 pending eviction cannot be upgraded');
|
|
480
502
|
db.exec(`INSERT OR REPLACE INTO ${PENDING_EVICTIONS} SELECT sub_id,tbl,effective_scopes FROM ${legacyEvictions}`);
|
package/dist/state.d.ts
CHANGED
|
@@ -38,6 +38,9 @@ export interface UnitRecord {
|
|
|
38
38
|
readonly seeded: readonly string[];
|
|
39
39
|
/** The echo persisted at the last `SUB_END` (§4.4). */
|
|
40
40
|
readonly echo?: readonly FencedRecord[] | undefined;
|
|
41
|
+
/** §5.6 applied-segment record: segment id → rows applied at the unit's
|
|
42
|
+
* current cursor and token. */
|
|
43
|
+
readonly applied?: Readonly<Record<string, number>> | undefined;
|
|
41
44
|
}
|
|
42
45
|
/** §4.1: a catching-up unit holding cursor `-1` or a resume token. */
|
|
43
46
|
export declare function bootstrapping(unit: UnitRecord): boolean;
|
|
@@ -57,8 +60,12 @@ export declare function unitOwners(units: readonly UnitRecord[], table: Compiled
|
|
|
57
60
|
/** Every unit in registration order (§4.1). */
|
|
58
61
|
export declare function loadUnits(db: ClientDatabase): UnitRecord[];
|
|
59
62
|
export declare function getUnit(db: ClientDatabase, id: string): UnitRecord | undefined;
|
|
60
|
-
/** Insert or update in place: an update keeps the registration order.
|
|
63
|
+
/** Insert or update in place: an update keeps the registration order. The
|
|
64
|
+
* §5.6 applied-segment record ends when the cursor or the token changes. */
|
|
61
65
|
export declare function saveUnit(db: ClientDatabase, record: UnitRecord): void;
|
|
66
|
+
/** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
|
|
67
|
+
* in the transaction that commits them. */
|
|
68
|
+
export declare function recordAppliedRows(db: ClientDatabase, id: string, segmentId: string, rows: number): void;
|
|
62
69
|
export declare function deleteUnit(db: ClientDatabase, id: string): void;
|
|
63
70
|
/**
|
|
64
71
|
* §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
|
package/dist/state.js
CHANGED
|
@@ -77,6 +77,7 @@ function rowToRecord(row) {
|
|
|
77
77
|
const scopes = json(row.scopes);
|
|
78
78
|
const seeded = json(row.seeded);
|
|
79
79
|
const echo = row.echo === null ? undefined : json(row.echo);
|
|
80
|
+
const applied = row.applied === null ? undefined : json(row.applied);
|
|
80
81
|
const { id, status, cursor } = row;
|
|
81
82
|
if (typeof id !== 'string' ||
|
|
82
83
|
!isMap(scopes) ||
|
|
@@ -84,6 +85,7 @@ function rowToRecord(row) {
|
|
|
84
85
|
(status !== 'active' && status !== 'revoked' && status !== 'failed') ||
|
|
85
86
|
(cursor !== null && !Number.isInteger(cursor)) ||
|
|
86
87
|
!isStrings(seeded) ||
|
|
88
|
+
(applied !== undefined && !isMap(applied)) ||
|
|
87
89
|
(echo !== undefined &&
|
|
88
90
|
!(Array.isArray(echo) &&
|
|
89
91
|
echo.every((record) => isMap(record) &&
|
|
@@ -109,6 +111,8 @@ function rowToRecord(row) {
|
|
|
109
111
|
record.bootstrapState = bootstrapState;
|
|
110
112
|
if (echo !== undefined)
|
|
111
113
|
record.echo = echo;
|
|
114
|
+
if (applied !== undefined)
|
|
115
|
+
record.applied = applied;
|
|
112
116
|
return record;
|
|
113
117
|
}
|
|
114
118
|
/** Every unit in registration order (§4.1). */
|
|
@@ -121,9 +125,10 @@ export function getUnit(db, id) {
|
|
|
121
125
|
const row = db.query(`SELECT * FROM ${UNITS_TABLE} WHERE id=?`, [id])[0];
|
|
122
126
|
return row === undefined ? undefined : rowToRecord(row);
|
|
123
127
|
}
|
|
124
|
-
/** Insert or update in place: an update keeps the registration order.
|
|
128
|
+
/** Insert or update in place: an update keeps the registration order. The
|
|
129
|
+
* §5.6 applied-segment record ends when the cursor or the token changes. */
|
|
125
130
|
export function saveUnit(db, record) {
|
|
126
|
-
db.exec(`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo`, [
|
|
131
|
+
db.exec(`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo,applied=iif((cursor,bootstrap_state)IS(excluded.cursor,excluded.bootstrap_state),applied,NULL)`, [
|
|
127
132
|
record.id,
|
|
128
133
|
JSON.stringify(record.scopes),
|
|
129
134
|
record.explicit ? 1 : 0,
|
|
@@ -135,6 +140,11 @@ export function saveUnit(db, record) {
|
|
|
135
140
|
record.echo === undefined ? null : JSON.stringify(record.echo),
|
|
136
141
|
]);
|
|
137
142
|
}
|
|
143
|
+
/** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
|
|
144
|
+
* in the transaction that commits them. */
|
|
145
|
+
export function recordAppliedRows(db, id, segmentId, rows) {
|
|
146
|
+
db.exec(`UPDATE ${UNITS_TABLE} SET applied=json_patch(coalesce(applied,'{}'),?) WHERE id=?`, [JSON.stringify({ [segmentId]: rows }), id]);
|
|
147
|
+
}
|
|
138
148
|
export function deleteUnit(db, id) {
|
|
139
149
|
db.exec(`DELETE FROM ${UNITS_TABLE} WHERE id=?`, [id]);
|
|
140
150
|
}
|
|
@@ -146,7 +156,7 @@ export function deleteUnit(db, id) {
|
|
|
146
156
|
* kept unit is live. Caller owns the transaction.
|
|
147
157
|
*/
|
|
148
158
|
export function resetUnits(db, keep = []) {
|
|
149
|
-
db.exec(`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`, [...keep]);
|
|
159
|
+
db.exec(`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL,applied=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`, [...keep]);
|
|
150
160
|
db.exec(`DELETE FROM ${META_TABLE} WHERE key IN(?,?) OR key=? AND NOT EXISTS(SELECT 1 FROM ${UNITS_TABLE} WHERE cursor IS NULL)`, [ECHO_DIGEST_KEY, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY]);
|
|
151
161
|
}
|
|
152
162
|
/** §4.5: the replica cursor, `-1` while no unit is live. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.78.
|
|
3
|
+
"version": "0.78.3",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -121,11 +121,11 @@
|
|
|
121
121
|
},
|
|
122
122
|
"dependencies": {
|
|
123
123
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
124
|
-
"@syncular/core": "0.78.
|
|
124
|
+
"@syncular/core": "0.78.3"
|
|
125
125
|
},
|
|
126
126
|
"devDependencies": {
|
|
127
127
|
"@electric-sql/pglite": "^0.5.4",
|
|
128
|
-
"@syncular/server": "0.78.
|
|
128
|
+
"@syncular/server": "0.78.3",
|
|
129
129
|
"playwright": "1.63.0",
|
|
130
130
|
"vite": "8.3.4"
|
|
131
131
|
}
|
package/src/apply.ts
CHANGED
|
@@ -526,6 +526,10 @@ export async function applySqliteSegment(
|
|
|
526
526
|
* to the next (§5.3).
|
|
527
527
|
*/
|
|
528
528
|
readonly chunkRows?: Map<string, number>;
|
|
529
|
+
/** Rows of this image an earlier attempt committed (§5.6). */
|
|
530
|
+
readonly appliedRows?: number | undefined;
|
|
531
|
+
/** Runs inside each chunk's transaction with the rows applied so far. */
|
|
532
|
+
readonly onChunk?: (appliedRows: number) => void;
|
|
529
533
|
},
|
|
530
534
|
): Promise<number> {
|
|
531
535
|
const now = options.now ?? (() => performance.now());
|
|
@@ -619,8 +623,12 @@ export async function applySqliteSegment(
|
|
|
619
623
|
}
|
|
620
624
|
return [boundary, predicates.join(' AND ') || 'true', params] as const;
|
|
621
625
|
};
|
|
622
|
-
|
|
623
|
-
let
|
|
626
|
+
const resumed = options.appliedRows ?? 0;
|
|
627
|
+
let [boundary, where, params] = range(
|
|
628
|
+
resumed > 0 ? range(undefined, resumed)[0] : undefined,
|
|
629
|
+
1024,
|
|
630
|
+
);
|
|
631
|
+
let applied = resumed;
|
|
624
632
|
// §5.3: a table's first chunk holds 1,024 rows; a later range image
|
|
625
633
|
// starts at the size its previous range reached. A later chunk holds at least
|
|
626
634
|
// 1,024 rows, at most twice the rows of the one before, at most 65,536,
|
|
@@ -631,7 +639,7 @@ export async function applySqliteSegment(
|
|
|
631
639
|
(!options.clearFirst && options.chunkRows?.get(table.name)) || 1024;
|
|
632
640
|
;
|
|
633
641
|
) {
|
|
634
|
-
if (options.isCurrent?.() === false) return applied;
|
|
642
|
+
if (options.isCurrent?.() === false) return applied - resumed;
|
|
635
643
|
const started = now();
|
|
636
644
|
let chunk = 0;
|
|
637
645
|
const first = applied === 0;
|
|
@@ -672,7 +680,7 @@ export async function applySqliteSegment(
|
|
|
672
680
|
);
|
|
673
681
|
for (;;) {
|
|
674
682
|
db.exec(
|
|
675
|
-
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where}
|
|
683
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} AS image WHERE ${where} AND NOT EXISTS(SELECT 1 FROM ${quoteIdent(table.name)} AS held WHERE held.${primaryKey}=image.${primaryKey} AND held.${version}>=image._syncular_version) ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`,
|
|
676
684
|
params,
|
|
677
685
|
);
|
|
678
686
|
const copied = Number(
|
|
@@ -711,6 +719,7 @@ export async function applySqliteSegment(
|
|
|
711
719
|
Number(row._syncular_version),
|
|
712
720
|
);
|
|
713
721
|
}
|
|
722
|
+
options.onChunk?.(applied);
|
|
714
723
|
return rebaseScope(scope, kept, skipped);
|
|
715
724
|
});
|
|
716
725
|
const duration = now() - started;
|
|
@@ -727,7 +736,7 @@ export async function applySqliteSegment(
|
|
|
727
736
|
imageInvalid('image applied row count does not match descriptor');
|
|
728
737
|
// A file replica's WAL keeps its high-water size until truncated.
|
|
729
738
|
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
730
|
-
return applied;
|
|
739
|
+
return applied - resumed;
|
|
731
740
|
});
|
|
732
741
|
}
|
|
733
742
|
|
package/src/blob.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
|
10
10
|
import type { ClientDatabase } from './database';
|
|
11
11
|
import { ClientSyncError } from './errors';
|
|
12
12
|
import type { CompiledClientSchema } from './schema';
|
|
13
|
-
import { quoteIdent } from './schema';
|
|
13
|
+
import { quoteIdent, tableExists } from './schema';
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* A blob download result (§5.9.5). The authorized endpoint either serves the
|
|
@@ -121,10 +121,7 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
|
121
121
|
}
|
|
122
122
|
|
|
123
123
|
export function ensureBlobSchema(db: ClientDatabase): void {
|
|
124
|
-
|
|
125
|
-
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='_syncular_blobs'",
|
|
126
|
-
);
|
|
127
|
-
if (existing.length > 0) {
|
|
124
|
+
if (tableExists(db, '_syncular_blobs')) {
|
|
128
125
|
const columns = db
|
|
129
126
|
.query('PRAGMA table_info(_syncular_blobs)')
|
|
130
127
|
.map((row) => row.name)
|
package/src/client.ts
CHANGED
|
@@ -258,6 +258,7 @@ import {
|
|
|
258
258
|
owning,
|
|
259
259
|
pairScopes,
|
|
260
260
|
pruneUnknownUnits,
|
|
261
|
+
recordAppliedRows,
|
|
261
262
|
REGISTERED_DIGEST_KEY,
|
|
262
263
|
REPLICA_CURSOR_KEY,
|
|
263
264
|
replicaCursor,
|
|
@@ -476,7 +477,8 @@ export interface SyncClientConfig {
|
|
|
476
477
|
* `transport.timeout` (`causeKind` `heartbeat`).
|
|
477
478
|
*/
|
|
478
479
|
readonly realtimeHeartbeatMs?: number;
|
|
479
|
-
/** Timer seam for the heartbeat watchdog
|
|
480
|
+
/** Timer seam for the heartbeat watchdog and the `syncUntilIdle` round
|
|
481
|
+
* bound; defaults to `setTimeout`. */
|
|
480
482
|
readonly schedule?: (callback: () => void, delayMs: number) => () => void;
|
|
481
483
|
/** Stable per-device id (§1.5); defaults to a persisted random UUID. */
|
|
482
484
|
readonly clientId?: string;
|
|
@@ -748,10 +750,6 @@ const outcomeNotFound = () =>
|
|
|
748
750
|
'no durable outcome exists for the commit',
|
|
749
751
|
);
|
|
750
752
|
|
|
751
|
-
/** The position a unit presents in a pull (§5.6 applied-segment rule). */
|
|
752
|
-
const position = (unit: UnitRecord) =>
|
|
753
|
-
JSON.stringify([unit.id, unit.cursor, unit.bootstrapState]);
|
|
754
|
-
|
|
755
753
|
interface OpenSection {
|
|
756
754
|
readonly start: SubStartFrame;
|
|
757
755
|
readonly unit: UnitRecord;
|
|
@@ -986,13 +984,10 @@ export class SyncClient {
|
|
|
986
984
|
#pendingRound: PendingRound | undefined;
|
|
987
985
|
#needsPull = false;
|
|
988
986
|
readonly #progress = new ProgressEmitter();
|
|
989
|
-
/**
|
|
990
|
-
* §5.6 applied-segment rule: unit position → the segments applied in
|
|
991
|
-
* sections whose SUB_END did not persist.
|
|
992
|
-
*/
|
|
993
|
-
readonly #appliedSegments = new Map<string, Set<string>>();
|
|
994
987
|
/** §5.3: a table's next image chunk size, carried across its ranges. */
|
|
995
988
|
readonly #chunkRows = new Map<string, number>();
|
|
989
|
+
/** §7.8: restarts the running drive's round bound when rows apply. */
|
|
990
|
+
#roundProgressed: (() => void) | undefined;
|
|
996
991
|
#progressAttempt = 0;
|
|
997
992
|
/**
|
|
998
993
|
* True from the synchronous entry of `sync()` until its serialized round
|
|
@@ -4806,26 +4801,19 @@ export class SyncClient {
|
|
|
4806
4801
|
deferredCommits: 0,
|
|
4807
4802
|
schemaFloor: undefined as SchemaFloor | undefined,
|
|
4808
4803
|
};
|
|
4809
|
-
const result = (budgetExhausted: boolean): SyncIdleResult =>
|
|
4810
|
-
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
? { deferredCommits: aggregate.deferredCommits }
|
|
4823
|
-
: {}),
|
|
4824
|
-
...(aggregate.schemaFloor !== undefined
|
|
4825
|
-
? { schemaFloor: aggregate.schemaFloor }
|
|
4826
|
-
: {}),
|
|
4827
|
-
budgetExhausted,
|
|
4828
|
-
});
|
|
4804
|
+
const result = (budgetExhausted: boolean): SyncIdleResult => {
|
|
4805
|
+
const { deferredCommits, schemaFloor, ...summary } = aggregate;
|
|
4806
|
+
return {
|
|
4807
|
+
...summary,
|
|
4808
|
+
...(deferredCommits > 0 ? { deferredCommits } : {}),
|
|
4809
|
+
...(schemaFloor !== undefined ? { schemaFloor } : {}),
|
|
4810
|
+
budgetExhausted,
|
|
4811
|
+
};
|
|
4812
|
+
};
|
|
4813
|
+
const states = () =>
|
|
4814
|
+
JSON.stringify(
|
|
4815
|
+
loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]),
|
|
4816
|
+
);
|
|
4829
4817
|
const startedAtMs = this.#now();
|
|
4830
4818
|
let completed = false;
|
|
4831
4819
|
for (let round = 0; round < budget; round++) {
|
|
@@ -4833,13 +4821,11 @@ export class SyncClient {
|
|
|
4833
4821
|
// like an exhausted round budget; the next drive resumes.
|
|
4834
4822
|
if (completed && this.#now() - startedAtMs >= timeoutMs)
|
|
4835
4823
|
return result(true);
|
|
4836
|
-
const before =
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
// Each round is bounded by `timeoutMs` on its own. One signal reaches
|
|
4840
|
-
// every transport call of the round.
|
|
4824
|
+
const before = states();
|
|
4825
|
+
// Each round is bounded by `timeoutMs` from its start or its last
|
|
4826
|
+
// applied rows. One signal reaches every transport call of the round.
|
|
4841
4827
|
const controller = new AbortController();
|
|
4842
|
-
let
|
|
4828
|
+
let cancel: (() => void) | undefined;
|
|
4843
4829
|
const sync = this.#sync(controller.signal);
|
|
4844
4830
|
const last = await (
|
|
4845
4831
|
timeoutMs === Number.POSITIVE_INFINITY
|
|
@@ -4847,49 +4833,59 @@ export class SyncClient {
|
|
|
4847
4833
|
: Promise.race([
|
|
4848
4834
|
sync,
|
|
4849
4835
|
new Promise<never>((_, reject) => {
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4836
|
+
this.#roundProgressed = () => {
|
|
4837
|
+
cancel?.();
|
|
4838
|
+
cancel = this.#schedule(() => {
|
|
4839
|
+
const error = new ClientSyncError(
|
|
4840
|
+
'sync.idle_timeout',
|
|
4841
|
+
'syncUntilIdle exceeded its deadline',
|
|
4842
|
+
true,
|
|
4843
|
+
{ timeoutMs, rounds: round },
|
|
4844
|
+
);
|
|
4845
|
+
// The transport may never answer; release the round so the
|
|
4846
|
+
// client can run another one after the caller sees the error.
|
|
4847
|
+
controller.abort(error);
|
|
4848
|
+
this.#abortPendingRound(error);
|
|
4849
|
+
reject(error);
|
|
4850
|
+
}, timeoutMs);
|
|
4851
|
+
};
|
|
4852
|
+
this.#roundProgressed();
|
|
4863
4853
|
}),
|
|
4864
4854
|
])
|
|
4865
|
-
).finally(() =>
|
|
4855
|
+
).finally(() => {
|
|
4856
|
+
cancel?.();
|
|
4857
|
+
this.#roundProgressed = undefined;
|
|
4858
|
+
});
|
|
4866
4859
|
completed = true;
|
|
4867
4860
|
// Historical counters accumulate; current readiness (`bootstrapping`,
|
|
4868
4861
|
// `deferredCommits`, `schemaFloor`) describes the latest round.
|
|
4869
|
-
|
|
4870
|
-
|
|
4871
|
-
|
|
4872
|
-
|
|
4862
|
+
for (const key of [
|
|
4863
|
+
'pushed',
|
|
4864
|
+
'commitsApplied',
|
|
4865
|
+
'segmentRowsApplied',
|
|
4866
|
+
] as const)
|
|
4867
|
+
aggregate[key] += last[key];
|
|
4868
|
+
for (const key of [
|
|
4869
|
+
'applied',
|
|
4870
|
+
'rejected',
|
|
4871
|
+
'retryable',
|
|
4872
|
+
'resets',
|
|
4873
|
+
'revoked',
|
|
4874
|
+
'failed',
|
|
4875
|
+
] as const)
|
|
4876
|
+
aggregate[key].push(...last[key]);
|
|
4873
4877
|
aggregate.conflicts.push(...last.conflicts);
|
|
4874
|
-
aggregate.commitsApplied += last.commitsApplied;
|
|
4875
|
-
aggregate.segmentRowsApplied += last.segmentRowsApplied;
|
|
4876
4878
|
aggregate.bootstrapping = [...last.bootstrapping];
|
|
4877
|
-
aggregate.resets.push(...last.resets);
|
|
4878
|
-
aggregate.revoked.push(...last.revoked);
|
|
4879
|
-
aggregate.failed.push(...last.failed);
|
|
4880
4879
|
aggregate.deferredCommits = last.deferredCommits ?? 0;
|
|
4881
4880
|
aggregate.schemaFloor = last.schemaFloor;
|
|
4882
4881
|
// A disabled transport or a schema floor ends the loop, not the budget.
|
|
4883
|
-
if (!this.#transportEnabled ||
|
|
4882
|
+
if (!this.#transportEnabled || aggregate.schemaFloor)
|
|
4884
4883
|
return result(false);
|
|
4885
4884
|
if (
|
|
4886
4885
|
maxRounds === undefined &&
|
|
4887
4886
|
last.segmentRowsApplied > 0 &&
|
|
4888
4887
|
last.bootstrapping.length > 0 &&
|
|
4889
|
-
before !==
|
|
4890
|
-
JSON.stringify(
|
|
4891
|
-
loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]),
|
|
4892
|
-
)
|
|
4888
|
+
before !== states()
|
|
4893
4889
|
)
|
|
4894
4890
|
round = -1;
|
|
4895
4891
|
if (
|
|
@@ -4897,7 +4893,7 @@ export class SyncClient {
|
|
|
4897
4893
|
last.segmentRowsApplied === 0 &&
|
|
4898
4894
|
last.bootstrapping.length === 0 &&
|
|
4899
4895
|
last.resets.length === 0 &&
|
|
4900
|
-
|
|
4896
|
+
!aggregate.deferredCommits &&
|
|
4901
4897
|
!this.#needsPull
|
|
4902
4898
|
) {
|
|
4903
4899
|
return result(false);
|
|
@@ -5465,7 +5461,7 @@ export class SyncClient {
|
|
|
5465
5461
|
frame.type === 'SEGMENT_REF' &&
|
|
5466
5462
|
open !== undefined &&
|
|
5467
5463
|
this.#config.segments !== undefined &&
|
|
5468
|
-
|
|
5464
|
+
open.applied?.[frame.segmentId] !== frame.rowCount
|
|
5469
5465
|
) {
|
|
5470
5466
|
queued.push([frame, open.scopes]);
|
|
5471
5467
|
}
|
|
@@ -5597,16 +5593,16 @@ export class SyncClient {
|
|
|
5597
5593
|
throw invalidRequest(
|
|
5598
5594
|
'unadvertised or paged sqlite segment (§4.2, §5.3)',
|
|
5599
5595
|
);
|
|
5600
|
-
const applied =
|
|
5601
|
-
this.#appliedSegments.get(position(section.unit)) ??
|
|
5602
|
-
new Set<string>();
|
|
5603
|
-
if (frame.type === 'SEGMENT_REF' && applied.has(frame.segmentId)) {
|
|
5604
|
-
// §5.6: applied in an earlier attempt at this position.
|
|
5605
|
-
section.seeded.add(frame.table);
|
|
5606
|
-
break;
|
|
5607
|
-
}
|
|
5608
5596
|
let bytes: Uint8Array;
|
|
5597
|
+
let appliedRows: number | undefined;
|
|
5609
5598
|
if (frame.type === 'SEGMENT_REF') {
|
|
5599
|
+
// §5.6: rows an earlier attempt at this position applied. A
|
|
5600
|
+
// partly applied image already ran its table's first-page clear.
|
|
5601
|
+
appliedRows = section.unit.applied?.[frame.segmentId];
|
|
5602
|
+
if (appliedRows !== undefined) {
|
|
5603
|
+
section.seeded.add(frame.table);
|
|
5604
|
+
if (appliedRows === frame.rowCount) break;
|
|
5605
|
+
}
|
|
5610
5606
|
this.#progress.update({
|
|
5611
5607
|
...NO_PAYLOAD,
|
|
5612
5608
|
phase: 'download',
|
|
@@ -5685,7 +5681,17 @@ export class SyncClient {
|
|
|
5685
5681
|
table,
|
|
5686
5682
|
bytes,
|
|
5687
5683
|
image,
|
|
5688
|
-
|
|
5684
|
+
{
|
|
5685
|
+
...options,
|
|
5686
|
+
appliedRows,
|
|
5687
|
+
onChunk: (rows) =>
|
|
5688
|
+
recordAppliedRows(
|
|
5689
|
+
this.#db,
|
|
5690
|
+
unitId,
|
|
5691
|
+
image.segmentId,
|
|
5692
|
+
rows,
|
|
5693
|
+
),
|
|
5694
|
+
},
|
|
5689
5695
|
)
|
|
5690
5696
|
: applyRowsSegment(
|
|
5691
5697
|
this.#db,
|
|
@@ -5698,9 +5704,11 @@ export class SyncClient {
|
|
|
5698
5704
|
},
|
|
5699
5705
|
);
|
|
5700
5706
|
if (frame.type === 'SEGMENT_REF' && !section.skip)
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5707
|
+
recordAppliedRows(
|
|
5708
|
+
this.#db,
|
|
5709
|
+
unitId,
|
|
5710
|
+
frame.segmentId,
|
|
5711
|
+
frame.rowCount,
|
|
5704
5712
|
);
|
|
5705
5713
|
break;
|
|
5706
5714
|
}
|
|
@@ -5714,7 +5722,6 @@ export class SyncClient {
|
|
|
5714
5722
|
this.#localResetEpoch === resetEpoch
|
|
5715
5723
|
) {
|
|
5716
5724
|
this.#finishSection(section, frame, summary);
|
|
5717
|
-
this.#appliedSegments.delete(position(section.unit));
|
|
5718
5725
|
}
|
|
5719
5726
|
section = undefined;
|
|
5720
5727
|
// The counters stay: `complete` and `failed` report them.
|
|
@@ -6207,7 +6214,10 @@ export class SyncClient {
|
|
|
6207
6214
|
effective,
|
|
6208
6215
|
owners,
|
|
6209
6216
|
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
6210
|
-
onProgress: (rowsProcessed) =>
|
|
6217
|
+
onProgress: (rowsProcessed) => {
|
|
6218
|
+
this.#progress.update({ rowsProcessed });
|
|
6219
|
+
this.#roundProgressed?.();
|
|
6220
|
+
},
|
|
6211
6221
|
transaction: (fn) => {
|
|
6212
6222
|
const started = performance.now();
|
|
6213
6223
|
let written = started;
|
package/src/previous-version.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
localColumnType,
|
|
42
42
|
quoteIdent,
|
|
43
43
|
rowValueToJson,
|
|
44
|
+
tableExists,
|
|
44
45
|
} from './schema';
|
|
45
46
|
import { getMeta, setMeta, META_TABLE } from './state';
|
|
46
47
|
|
|
@@ -449,11 +450,7 @@ export function withPreviousVersionContainer<T>(
|
|
|
449
450
|
export function readPreviousVersionContainer(
|
|
450
451
|
db: ClientDatabase,
|
|
451
452
|
): PreviousVersionRecord | undefined {
|
|
452
|
-
|
|
453
|
-
"SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
|
|
454
|
-
[CONTAINER_META_TABLE],
|
|
455
|
-
);
|
|
456
|
-
if (meta.length === 0) return undefined;
|
|
453
|
+
if (!tableExists(db, CONTAINER_META_TABLE)) return undefined;
|
|
457
454
|
const raw = db.query(
|
|
458
455
|
`SELECT record FROM ${quoteIdent(CONTAINER_META_TABLE)} WHERE id=1`,
|
|
459
456
|
)[0]?.record;
|
package/src/schema.ts
CHANGED
|
@@ -356,6 +356,14 @@ export function quoteIdent(name: string): string {
|
|
|
356
356
|
return `"${name.replaceAll('"', '""')}"`;
|
|
357
357
|
}
|
|
358
358
|
|
|
359
|
+
export function tableExists(db: ClientDatabase, name: string): boolean {
|
|
360
|
+
return (
|
|
361
|
+
db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
|
|
362
|
+
name,
|
|
363
|
+
]).length > 0
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
|
|
359
367
|
/**
|
|
360
368
|
* §5.11: the app-side type of a column for local (plaintext) storage. For an
|
|
361
369
|
* encrypted column this is `declaredType` — the local mirror stays plaintext,
|
|
@@ -445,8 +453,11 @@ const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
|
|
|
445
453
|
* Limitation: because a BEFORE trigger runs before SQLite resolves the
|
|
446
454
|
* conflict, an `OR IGNORE` write whose row is dropped still executes these
|
|
447
455
|
* displacement DELETEs, transiently removing the surviving row's projection
|
|
448
|
-
* entry.
|
|
449
|
-
*
|
|
456
|
+
* entry. The same holds for an upsert whose `DO UPDATE ... WHERE` skips the
|
|
457
|
+
* row, so Syncular's mirror writes never offer a row that will not replace
|
|
458
|
+
* (§5.3); see the guard-block comment below for the full rationale. An open
|
|
459
|
+
* rebuilds a projection whose mapping holds a different row count than its
|
|
460
|
+
* source table.
|
|
450
461
|
*/
|
|
451
462
|
function createFtsProjection(
|
|
452
463
|
db: ClientDatabase,
|
|
@@ -456,11 +467,7 @@ function createFtsProjection(
|
|
|
456
467
|
const projection = quoteIdent(index.name);
|
|
457
468
|
const sourceTable = quoteIdent(table.name);
|
|
458
469
|
const primaryKey = quoteIdent(table.primaryKey);
|
|
459
|
-
const existed =
|
|
460
|
-
db.query(
|
|
461
|
-
"SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
|
|
462
|
-
[index.name],
|
|
463
|
-
).length > 0;
|
|
470
|
+
const existed = tableExists(db, index.name);
|
|
464
471
|
const indexedColumns = index.columns.map(quoteIdent);
|
|
465
472
|
const tokenizer = index.tokenize.replaceAll("'", "''");
|
|
466
473
|
db.exec(
|
|
@@ -468,10 +475,7 @@ function createFtsProjection(
|
|
|
468
475
|
);
|
|
469
476
|
|
|
470
477
|
const mapping = quoteIdent(`_syncular_fts_${index.name}`);
|
|
471
|
-
const mapped =
|
|
472
|
-
db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
|
|
473
|
-
`_syncular_fts_${index.name}`,
|
|
474
|
-
]).length > 0;
|
|
478
|
+
const mapped = tableExists(db, `_syncular_fts_${index.name}`);
|
|
475
479
|
db.exec(
|
|
476
480
|
`CREATE TABLE IF NOT EXISTS ${mapping}(id INTEGER PRIMARY KEY,source_id TEXT NOT NULL UNIQUE)`,
|
|
477
481
|
);
|
|
@@ -506,16 +510,19 @@ function createFtsProjection(
|
|
|
506
510
|
// `UPDATE OR REPLACE` needs no BEFORE guard: the new pk equals the displaced
|
|
507
511
|
// row's pk, so the AFTER UPDATE delete of the new source id already clears it.
|
|
508
512
|
//
|
|
509
|
-
// Known limitation
|
|
510
|
-
//
|
|
511
|
-
//
|
|
512
|
-
//
|
|
513
|
-
// SURVIVING row's projection entry (a
|
|
514
|
-
// rewritten). Moving cleanup to AFTER
|
|
515
|
-
// reintroduce the REPLACE-doesn't-fire-
|
|
516
|
-
// per write, so the guards stay
|
|
517
|
-
//
|
|
518
|
-
//
|
|
513
|
+
// Known limitation: a BEFORE trigger fires before SQLite resolves the
|
|
514
|
+
// conflict, and SQLite exposes no signal for the eventual resolution. So an
|
|
515
|
+
// `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is dropped on conflict,
|
|
516
|
+
// and an upsert whose `DO UPDATE ... WHERE` skips the row, still run these
|
|
517
|
+
// displacement DELETEs, removing the SURVIVING row's projection entry (a
|
|
518
|
+
// false negative until that row is next rewritten). Moving cleanup to AFTER
|
|
519
|
+
// triggers would dodge the no-op but reintroduce the REPLACE-doesn't-fire-
|
|
520
|
+
// DELETE gap and a full-projection scan per write, so the guards stay
|
|
521
|
+
// BEFORE. Syncular's own mirror writes use an unconditional
|
|
522
|
+
// `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts); the image copy filters
|
|
523
|
+
// held rows at the same or a newer version out of its SELECT instead of a
|
|
524
|
+
// `DO UPDATE ... WHERE`. Only hand-written `OR IGNORE` against a mirror
|
|
525
|
+
// table is exposed.
|
|
519
526
|
for (const suffix of ['bi', 'ai', 'ad', 'au', 'bu']) {
|
|
520
527
|
db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(`${index.name}_${suffix}`)}`);
|
|
521
528
|
}
|
|
@@ -568,12 +575,23 @@ function createFtsProjection(
|
|
|
568
575
|
createTrigger('bu', 'BEFORE UPDATE', updateGuardBody, updateGuardCondition);
|
|
569
576
|
}
|
|
570
577
|
|
|
571
|
-
|
|
578
|
+
// Every source row has exactly one mapping row, so a count mismatch marks a
|
|
579
|
+
// projection an earlier release damaged; the open rebuilds it.
|
|
580
|
+
const stale =
|
|
581
|
+
!existed ||
|
|
582
|
+
(mapped &&
|
|
583
|
+
Number(
|
|
584
|
+
db.query(
|
|
585
|
+
`SELECT (SELECT count(*) FROM ${mapping})=(SELECT count(*) FROM ${sourceTable}) AS whole`,
|
|
586
|
+
)[0]?.whole,
|
|
587
|
+
) !== 1);
|
|
588
|
+
if (stale) {
|
|
589
|
+
db.exec(`DELETE FROM ${projection}`);
|
|
572
590
|
db.exec(
|
|
573
591
|
`INSERT INTO ${projection}(${projectionColumns})SELECT ${sourceId},${indexedColumns.join(', ')} FROM ${sourceTable}`,
|
|
574
592
|
);
|
|
575
593
|
}
|
|
576
|
-
if (!mapped ||
|
|
594
|
+
if (!mapped || stale) {
|
|
577
595
|
db.exec(`DELETE FROM ${mapping}`);
|
|
578
596
|
db.exec(
|
|
579
597
|
`INSERT INTO ${mapping}(id,source_id)SELECT rowid,${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${projection}`,
|
|
@@ -649,10 +667,7 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
649
667
|
db.exec(
|
|
650
668
|
`INSERT OR IGNORE INTO ${META_TABLE}(key,value)VALUES('localRevision','0')`,
|
|
651
669
|
);
|
|
652
|
-
const outboxExisted =
|
|
653
|
-
db.query(
|
|
654
|
-
`SELECT 1 FROM sqlite_master WHERE type='table' AND name='${OUTBOX_TABLE}'`,
|
|
655
|
-
).length > 0;
|
|
670
|
+
const outboxExisted = tableExists(db, OUTBOX_TABLE);
|
|
656
671
|
createTable(
|
|
657
672
|
OUTBOX_TABLE,
|
|
658
673
|
'seq INTEGER PRIMARY KEY AUTOINCREMENT,client_commit_id TEXT NOT NULL UNIQUE,created_at_ms INTEGER NOT NULL,operations TEXT NOT NULL',
|
|
@@ -697,6 +712,12 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
697
712
|
UNITS_TABLE,
|
|
698
713
|
"id TEXT PRIMARY KEY,scopes TEXT NOT NULL,explicit INTEGER NOT NULL DEFAULT 0,status TEXT NOT NULL DEFAULT 'active',reason_code TEXT,cursor INTEGER DEFAULT -1,bootstrap_state TEXT,seeded TEXT NOT NULL DEFAULT '[]',echo TEXT",
|
|
699
714
|
);
|
|
715
|
+
// §5.6 applied-segment record, added after the table shipped.
|
|
716
|
+
try {
|
|
717
|
+
db.exec(`ALTER TABLE ${UNITS_TABLE} ADD COLUMN applied TEXT`);
|
|
718
|
+
} catch {
|
|
719
|
+
// column already exists
|
|
720
|
+
}
|
|
700
721
|
// §4.8 window registry: base + value → the unit it holds, the
|
|
701
722
|
// completeness oracle and one of a unit's references (§4.1).
|
|
702
723
|
createTable(
|
|
@@ -716,11 +737,7 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
716
737
|
// rows and pending evictions and registers its units fresh. Its
|
|
717
738
|
// subscriptions, cursors, echoes, resume tokens and window entries go.
|
|
718
739
|
const legacyEvictions = '_syncular_window_pending_evict';
|
|
719
|
-
if (
|
|
720
|
-
db.query(
|
|
721
|
-
`SELECT 1 FROM sqlite_master WHERE type='table' AND name='${legacyEvictions}'`,
|
|
722
|
-
).length > 0
|
|
723
|
-
) {
|
|
740
|
+
if (tableExists(db, legacyEvictions)) {
|
|
724
741
|
if (
|
|
725
742
|
db.query(
|
|
726
743
|
`SELECT 1 FROM ${legacyEvictions} WHERE NOT json_valid(effective_scopes) OR json_type(effective_scopes)<>'object' OR EXISTS(SELECT 1 FROM json_each(effective_scopes) WHERE json_type(value)<>'array' OR EXISTS(SELECT 1 FROM json_each(value) v WHERE v.type<>'text'))`,
|
package/src/state.ts
CHANGED
|
@@ -45,6 +45,9 @@ export interface UnitRecord {
|
|
|
45
45
|
readonly seeded: readonly string[];
|
|
46
46
|
/** The echo persisted at the last `SUB_END` (§4.4). */
|
|
47
47
|
readonly echo?: readonly FencedRecord[] | undefined;
|
|
48
|
+
/** §5.6 applied-segment record: segment id → rows applied at the unit's
|
|
49
|
+
* current cursor and token. */
|
|
50
|
+
readonly applied?: Readonly<Record<string, number>> | undefined;
|
|
48
51
|
}
|
|
49
52
|
|
|
50
53
|
/** §4.1: a catching-up unit holding cursor `-1` or a resume token. */
|
|
@@ -138,6 +141,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
|
|
|
138
141
|
const scopes = json(row.scopes);
|
|
139
142
|
const seeded = json(row.seeded);
|
|
140
143
|
const echo = row.echo === null ? undefined : json(row.echo);
|
|
144
|
+
const applied = row.applied === null ? undefined : json(row.applied);
|
|
141
145
|
const { id, status, cursor } = row;
|
|
142
146
|
if (
|
|
143
147
|
typeof id !== 'string' ||
|
|
@@ -146,6 +150,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
|
|
|
146
150
|
(status !== 'active' && status !== 'revoked' && status !== 'failed') ||
|
|
147
151
|
(cursor !== null && !Number.isInteger(cursor)) ||
|
|
148
152
|
!isStrings(seeded) ||
|
|
153
|
+
(applied !== undefined && !isMap(applied)) ||
|
|
149
154
|
(echo !== undefined &&
|
|
150
155
|
!(
|
|
151
156
|
Array.isArray(echo) &&
|
|
@@ -173,6 +178,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
|
|
|
173
178
|
const bootstrapState = optionalText(row.bootstrap_state);
|
|
174
179
|
if (bootstrapState !== undefined) record.bootstrapState = bootstrapState;
|
|
175
180
|
if (echo !== undefined) record.echo = echo as FencedRecord[];
|
|
181
|
+
if (applied !== undefined) record.applied = applied as Record<string, number>;
|
|
176
182
|
return record;
|
|
177
183
|
}
|
|
178
184
|
|
|
@@ -191,10 +197,11 @@ export function getUnit(
|
|
|
191
197
|
return row === undefined ? undefined : rowToRecord(row);
|
|
192
198
|
}
|
|
193
199
|
|
|
194
|
-
/** Insert or update in place: an update keeps the registration order.
|
|
200
|
+
/** Insert or update in place: an update keeps the registration order. The
|
|
201
|
+
* §5.6 applied-segment record ends when the cursor or the token changes. */
|
|
195
202
|
export function saveUnit(db: ClientDatabase, record: UnitRecord): void {
|
|
196
203
|
db.exec(
|
|
197
|
-
`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo`,
|
|
204
|
+
`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo,applied=iif((cursor,bootstrap_state)IS(excluded.cursor,excluded.bootstrap_state),applied,NULL)`,
|
|
198
205
|
[
|
|
199
206
|
record.id,
|
|
200
207
|
JSON.stringify(record.scopes),
|
|
@@ -209,6 +216,20 @@ export function saveUnit(db: ClientDatabase, record: UnitRecord): void {
|
|
|
209
216
|
);
|
|
210
217
|
}
|
|
211
218
|
|
|
219
|
+
/** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
|
|
220
|
+
* in the transaction that commits them. */
|
|
221
|
+
export function recordAppliedRows(
|
|
222
|
+
db: ClientDatabase,
|
|
223
|
+
id: string,
|
|
224
|
+
segmentId: string,
|
|
225
|
+
rows: number,
|
|
226
|
+
): void {
|
|
227
|
+
db.exec(
|
|
228
|
+
`UPDATE ${UNITS_TABLE} SET applied=json_patch(coalesce(applied,'{}'),?) WHERE id=?`,
|
|
229
|
+
[JSON.stringify({ [segmentId]: rows }), id],
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
|
|
212
233
|
export function deleteUnit(db: ClientDatabase, id: string): void {
|
|
213
234
|
db.exec(`DELETE FROM ${UNITS_TABLE} WHERE id=?`, [id]);
|
|
214
235
|
}
|
|
@@ -225,7 +246,7 @@ export function resetUnits(
|
|
|
225
246
|
keep: readonly string[] = [],
|
|
226
247
|
): void {
|
|
227
248
|
db.exec(
|
|
228
|
-
`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`,
|
|
249
|
+
`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL,applied=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`,
|
|
229
250
|
[...keep],
|
|
230
251
|
);
|
|
231
252
|
db.exec(
|