@syncular/client 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/apply.d.ts +14 -7
- package/dist/apply.js +75 -53
- package/dist/bun-database.d.ts +1 -1
- package/dist/bun-database.js +2 -2
- package/dist/client.d.ts +4 -4
- package/dist/client.js +64 -34
- package/dist/database.d.ts +3 -2
- package/dist/node-database.d.ts +1 -1
- package/dist/node-database.js +2 -2
- package/dist/reactive-store.js +49 -8
- package/dist/schema.js +15 -4
- package/dist/wasm-database.js +3 -3
- package/package.json +3 -3
- package/src/apply.ts +86 -60
- package/src/bun-database.ts +6 -2
- package/src/client.ts +81 -37
- package/src/database.ts +7 -2
- package/src/node-database.ts +6 -2
- package/src/reactive-store.ts +58 -8
- package/src/schema.ts +22 -4
- package/src/wasm-database.ts +7 -3
package/README.md
CHANGED
|
@@ -676,7 +676,8 @@ through the core, without intermediate download updates.
|
|
|
676
676
|
|
|
677
677
|
Import reports every 1,024 rows and its final count. `rowsProcessed` includes work
|
|
678
678
|
inside the current transaction; a failed import can roll those rows back. SQLite
|
|
679
|
-
images
|
|
679
|
+
images commit in chunks of at most 1,024 rows and yield after each commit.
|
|
680
|
+
Earlier committed chunks survive a later interruption. `complete` follows checkpoint persistence
|
|
680
681
|
and optimistic-state reconciliation for the round. More bootstrap pages can
|
|
681
682
|
remain; use subscription coverage to determine whether a query is complete.
|
|
682
683
|
Listener exceptions do not fail sync. Listeners should hand UI work off and avoid
|
|
@@ -723,3 +724,19 @@ request. Status and diagnostics use `COUNT(*)` without parsing pending bodies.
|
|
|
723
724
|
Optimistic replay still reads the remaining outbox after each response. The
|
|
724
725
|
100/1,000/10,000-commit workload and measured limits are recorded in
|
|
725
726
|
[the reliability RFC](../../docs/RFC-RELIABILITY-DX.md#9-implementation-evidence-2026-09-05).
|
|
727
|
+
|
|
728
|
+
## Reads during bootstrap and eviction
|
|
729
|
+
|
|
730
|
+
Reactive queries read cached snapshots while window registration runs. Coverage
|
|
731
|
+
continues to distinguish complete, pending, and missing windows; registration
|
|
732
|
+
errors remain observable. Retained owners reuse acknowledged units immediately.
|
|
733
|
+
|
|
734
|
+
Both segment formats yield between committed import chunks automatically.
|
|
735
|
+
Window eviction deletes at most 1,024 unpinned rows per chunk and resumes from a
|
|
736
|
+
durable pending record after interruption. Security purges remain atomic.
|
|
737
|
+
See [windowing](https://syncular.dev/concepts-windowing/) and
|
|
738
|
+
[bootstrap](https://syncular.dev/concepts-bootstrap/) for coverage and recovery.
|
|
739
|
+
|
|
740
|
+
Custom database adapters implementing `withSqliteImage` must return a promise and
|
|
741
|
+
await their callback before detaching the image. The callback can await between
|
|
742
|
+
committed import chunks; no write transaction spans that await.
|
package/dist/apply.d.ts
CHANGED
|
@@ -43,12 +43,15 @@ export declare function deleteScopedRows(db: ClientDatabase, table: CompiledClie
|
|
|
43
43
|
* scopes (same local-scope-column rule and fail-closed clause as
|
|
44
44
|
* {@link deleteScopedRows}) EXCEPT rows whose primary key is in
|
|
45
45
|
* `pinnedRowIds` (E1 — pinned by a still-pending outbox commit). Returns
|
|
46
|
-
*
|
|
47
|
-
*
|
|
46
|
+
* whether unpinned work remains and whether any scoped rows remain. Each
|
|
47
|
+
* call removes at most 1,024 rows; the caller commits before yielding. Also removes the
|
|
48
48
|
* evicted rows' `server_version` with them (E2 — no residual version
|
|
49
49
|
* cache), which is automatic since the version column is per-row.
|
|
50
50
|
*/
|
|
51
|
-
export declare function evictScopedRows(db: ClientDatabase, table: CompiledClientTable, effective: ScopeMap, pinnedRowIds: ReadonlySet<string>):
|
|
51
|
+
export declare function evictScopedRows(db: ClientDatabase, table: CompiledClientTable, effective: ScopeMap, pinnedRowIds: ReadonlySet<string>): {
|
|
52
|
+
remaining: boolean;
|
|
53
|
+
deferred: boolean;
|
|
54
|
+
};
|
|
52
55
|
/** Descriptor fields a sqlite image is validated against (§5.3). */
|
|
53
56
|
export interface SqliteSegmentDescriptor {
|
|
54
57
|
readonly table: string;
|
|
@@ -57,11 +60,11 @@ export interface SqliteSegmentDescriptor {
|
|
|
57
60
|
readonly scopeDigest: string;
|
|
58
61
|
}
|
|
59
62
|
/**
|
|
60
|
-
* Apply a §5.3 sqlite
|
|
63
|
+
* Apply a §5.3 sqlite image in committed chunks of at most 1,024 rows: validate
|
|
61
64
|
* the in-file metadata against the descriptor, validate the data table's
|
|
62
65
|
* column names/order against the generated schema, run the §5.6
|
|
63
|
-
* first-page clear when fresh, then copy
|
|
64
|
-
*
|
|
66
|
+
* first-page clear when fresh, then bulk-copy each chunk through a
|
|
67
|
+
* primary-key upsert — `_syncular_version` lands in
|
|
65
68
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
66
69
|
*/
|
|
67
70
|
export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledClientSchema, table: CompiledClientTable, bytes: Uint8Array, descriptor: SqliteSegmentDescriptor, options: {
|
|
@@ -69,7 +72,8 @@ export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledC
|
|
|
69
72
|
readonly effective: ScopeMap;
|
|
70
73
|
readonly transaction?: ApplyTransaction;
|
|
71
74
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
72
|
-
|
|
75
|
+
readonly isCurrent?: () => boolean;
|
|
76
|
+
}): Promise<number>;
|
|
73
77
|
/**
|
|
74
78
|
* Apply a decoded rows segment: each block in one local transaction
|
|
75
79
|
* (§5.2/§1.4); `clearFirst` implements the §5.6 fresh-bootstrap first-page
|
|
@@ -83,4 +87,7 @@ export declare function applyRowsSegment(db: ClientDatabase, schema: CompiledCli
|
|
|
83
87
|
readonly effective: ScopeMap;
|
|
84
88
|
readonly transaction?: ApplyTransaction;
|
|
85
89
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
90
|
+
readonly isCurrent?: () => boolean;
|
|
86
91
|
}, encryption?: EncryptionConfig): Promise<number>;
|
|
92
|
+
/** Let worker messages run only after the current SQLite transaction commits. */
|
|
93
|
+
export declare function yieldToHost(): Promise<void>;
|
package/dist/apply.js
CHANGED
|
@@ -138,15 +138,15 @@ export function deleteScopedRows(db, table, effective) {
|
|
|
138
138
|
* scopes (same local-scope-column rule and fail-closed clause as
|
|
139
139
|
* {@link deleteScopedRows}) EXCEPT rows whose primary key is in
|
|
140
140
|
* `pinnedRowIds` (E1 — pinned by a still-pending outbox commit). Returns
|
|
141
|
-
*
|
|
142
|
-
*
|
|
141
|
+
* whether unpinned work remains and whether any scoped rows remain. Each
|
|
142
|
+
* call removes at most 1,024 rows; the caller commits before yielding. Also removes the
|
|
143
143
|
* evicted rows' `server_version` with them (E2 — no residual version
|
|
144
144
|
* cache), which is automatic since the version column is per-row.
|
|
145
145
|
*/
|
|
146
146
|
export function evictScopedRows(db, table, effective, pinnedRowIds) {
|
|
147
147
|
const entries = Object.entries(effective);
|
|
148
148
|
if (entries.length === 0)
|
|
149
|
-
return false;
|
|
149
|
+
return { remaining: false, deferred: false };
|
|
150
150
|
const clauses = [];
|
|
151
151
|
const params = [];
|
|
152
152
|
for (const [variable, values] of entries) {
|
|
@@ -155,7 +155,7 @@ export function evictScopedRows(db, table, effective, pinnedRowIds) {
|
|
|
155
155
|
throw new ClientSyncError('sync.scope_revoked', `table ${JSON.stringify(table.name)} has no local scope-column mapping for ${JSON.stringify(variable)} (§4.8/§3.3 fail-closed)`);
|
|
156
156
|
}
|
|
157
157
|
if (values.length === 0)
|
|
158
|
-
return false;
|
|
158
|
+
return { remaining: false, deferred: false };
|
|
159
159
|
clauses.push(`${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`);
|
|
160
160
|
params.push(...values);
|
|
161
161
|
}
|
|
@@ -166,31 +166,28 @@ export function evictScopedRows(db, table, effective, pinnedRowIds) {
|
|
|
166
166
|
pinnedClause = ` AND ${pk} NOT IN (${ids.map(() => '?').join(', ')})`;
|
|
167
167
|
params.push(...ids);
|
|
168
168
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
return true;
|
|
178
|
-
}
|
|
179
|
-
return false;
|
|
169
|
+
const target = quoteIdent(table.name);
|
|
170
|
+
db.exec(`DELETE FROM ${target} WHERE ${pk} IN (
|
|
171
|
+
SELECT ${pk} FROM ${target} WHERE ${clauses.join(' AND ')}${pinnedClause} LIMIT 1024
|
|
172
|
+
)`, params);
|
|
173
|
+
const remaining = db.query(`SELECT 1 FROM ${target} WHERE ${clauses.join(' AND ')}${pinnedClause} LIMIT 1`, params).length > 0;
|
|
174
|
+
const deferred = remaining ||
|
|
175
|
+
db.query(`SELECT 1 FROM ${target} WHERE ${clauses.join(' AND ')} LIMIT 1`, params.slice(0, params.length - pinnedRowIds.size)).length > 0;
|
|
176
|
+
return { remaining, deferred };
|
|
180
177
|
}
|
|
181
178
|
const IMAGE_ALIAS = 'syncular_image';
|
|
182
179
|
function imageInvalid(detail) {
|
|
183
180
|
throw new ClientSyncError('sync.invalid_request', `sqlite segment rejected: ${detail} (§5.3)`);
|
|
184
181
|
}
|
|
185
182
|
/**
|
|
186
|
-
* Apply a §5.3 sqlite
|
|
183
|
+
* Apply a §5.3 sqlite image in committed chunks of at most 1,024 rows: validate
|
|
187
184
|
* the in-file metadata against the descriptor, validate the data table's
|
|
188
185
|
* column names/order against the generated schema, run the §5.6
|
|
189
|
-
* first-page clear when fresh, then copy
|
|
190
|
-
*
|
|
186
|
+
* first-page clear when fresh, then bulk-copy each chunk through a
|
|
187
|
+
* primary-key upsert — `_syncular_version` lands in
|
|
191
188
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
192
189
|
*/
|
|
193
|
-
export function applySqliteSegment(db, schema, table, bytes, descriptor, options) {
|
|
190
|
+
export async function applySqliteSegment(db, schema, table, bytes, descriptor, options) {
|
|
194
191
|
const withImage = db.withSqliteImage?.bind(db);
|
|
195
192
|
if (withImage === undefined) {
|
|
196
193
|
throw new ClientSyncError('sync.invalid_request', 'received a sqlite segment but the database backend cannot import images (§4.2: do not advertise accept bit 2)');
|
|
@@ -198,7 +195,7 @@ export function applySqliteSegment(db, schema, table, bytes, descriptor, options
|
|
|
198
195
|
if (descriptor.table !== table.name) {
|
|
199
196
|
imageInvalid(`descriptor table ${JSON.stringify(descriptor.table)}`);
|
|
200
197
|
}
|
|
201
|
-
return withImage(bytes, IMAGE_ALIAS, () => {
|
|
198
|
+
return withImage(bytes, IMAGE_ALIAS, async () => {
|
|
202
199
|
// 1. Metadata vs descriptor (§5.3 rule 2). A file that is not a
|
|
203
200
|
// SQLite database or lacks the metadata table fails right here.
|
|
204
201
|
let meta;
|
|
@@ -244,51 +241,58 @@ export function applySqliteSegment(db, schema, table, bytes, descriptor, options
|
|
|
244
241
|
expected.some((name, index) => actual[index] !== name)) {
|
|
245
242
|
throw new ClientSyncError('sync.schema_mismatch', `sqlite segment for ${JSON.stringify(table.name)} does not match the generated schema: columns [${actual.join(', ')}] (§5.3)`);
|
|
246
243
|
}
|
|
247
|
-
|
|
244
|
+
const primaryKeys = info.filter((column) => Number(column.pk) > 0);
|
|
245
|
+
if (primaryKeys.length !== 1 || primaryKeys[0]?.name !== table.primaryKey)
|
|
246
|
+
imageInvalid('image primary key does not match generated schema');
|
|
247
|
+
const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
|
|
248
|
+
const count = Number(db.query(`SELECT count(*) AS n FROM ${source}`)[0]?.n);
|
|
249
|
+
if (count !== descriptor.rowCount)
|
|
250
|
+
imageInvalid('image row count does not match descriptor');
|
|
248
251
|
const names = table.columns.map((column) => quoteIdent(column.name));
|
|
249
252
|
const insertNames = [...names, quoteIdent(SYNC_VERSION_COLUMN)];
|
|
253
|
+
const primaryKey = quoteIdent(table.primaryKey);
|
|
250
254
|
const updates = insertNames
|
|
251
|
-
.filter((name) => name !==
|
|
255
|
+
.filter((name) => name !== primaryKey)
|
|
252
256
|
.map((name) => `${name}=excluded.${name}`)
|
|
253
257
|
.join(', ');
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
258
|
+
let after;
|
|
259
|
+
let applied = 0;
|
|
260
|
+
for (;;) {
|
|
261
|
+
if (options.isCurrent?.() === false)
|
|
262
|
+
return applied;
|
|
263
|
+
const boundary = db.query(`SELECT ${primaryKey} AS boundary FROM ${source}
|
|
264
|
+
${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
|
|
265
|
+
ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, after === undefined ? [] : [after])[0]?.boundary;
|
|
266
|
+
if (boundary === null)
|
|
267
|
+
imageInvalid('image primary key must not be null');
|
|
268
|
+
const predicates = [];
|
|
269
|
+
const params = [];
|
|
270
|
+
if (after !== undefined) {
|
|
271
|
+
predicates.push(`${primaryKey} > ?`);
|
|
272
|
+
params.push(after);
|
|
273
|
+
}
|
|
274
|
+
if (boundary !== undefined) {
|
|
275
|
+
predicates.push(`${primaryKey} <= ?`);
|
|
276
|
+
params.push(boundary);
|
|
257
277
|
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
let applied = 0;
|
|
262
|
-
for (;;) {
|
|
263
|
-
const boundary = db.query(`SELECT ${primaryKey} AS boundary FROM ${source}
|
|
264
|
-
${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
|
|
265
|
-
ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, after === undefined ? [] : [after])[0]?.boundary;
|
|
266
|
-
const predicates = [];
|
|
267
|
-
const params = [];
|
|
268
|
-
if (after !== undefined) {
|
|
269
|
-
predicates.push(`${primaryKey} > ?`);
|
|
270
|
-
params.push(after);
|
|
271
|
-
}
|
|
272
|
-
if (boundary !== undefined) {
|
|
273
|
-
predicates.push(`${primaryKey} <= ?`);
|
|
274
|
-
params.push(boundary);
|
|
275
|
-
}
|
|
278
|
+
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
279
|
+
if (after === undefined && options.clearFirst)
|
|
280
|
+
deleteScopedRows(db, table, options.effective);
|
|
276
281
|
db.exec(`INSERT INTO ${quoteIdent(table.name)} (${insertNames.join(', ')})
|
|
277
282
|
SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
|
|
278
283
|
FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}
|
|
279
284
|
ON CONFLICT (${primaryKey}) DO UPDATE SET ${updates}`, params);
|
|
280
285
|
applied += Number(db.query('SELECT changes() AS n')[0]?.n ?? 0);
|
|
281
|
-
|
|
282
|
-
break;
|
|
283
|
-
options.onProgress?.(applied);
|
|
284
|
-
after = boundary;
|
|
285
|
-
}
|
|
286
|
-
if (applied !== descriptor.rowCount) {
|
|
287
|
-
imageInvalid(`image holds ${applied} rows, descriptor says ${descriptor.rowCount}`);
|
|
288
|
-
}
|
|
286
|
+
});
|
|
289
287
|
options.onProgress?.(applied);
|
|
290
|
-
|
|
291
|
-
|
|
288
|
+
await yieldToHost();
|
|
289
|
+
if (boundary === undefined || applied === count)
|
|
290
|
+
break;
|
|
291
|
+
after = boundary;
|
|
292
|
+
}
|
|
293
|
+
if (applied !== count)
|
|
294
|
+
imageInvalid('image applied row count does not match descriptor');
|
|
295
|
+
return applied;
|
|
292
296
|
});
|
|
293
297
|
}
|
|
294
298
|
/**
|
|
@@ -305,6 +309,8 @@ export async function applyRowsSegment(db, schema, table, segment, options, encr
|
|
|
305
309
|
let first = true;
|
|
306
310
|
const blocks = segment.blocks.length > 0 ? segment.blocks : [[]];
|
|
307
311
|
for (const block of blocks) {
|
|
312
|
+
if (options.isCurrent?.() === false)
|
|
313
|
+
break;
|
|
308
314
|
// §5.11: decrypt this block's rows before opening the sync transaction
|
|
309
315
|
// (WebCrypto is async; the SQLite transaction is not). Decrypt failure
|
|
310
316
|
// aborts before any write in this block.
|
|
@@ -318,6 +324,8 @@ export async function applyRowsSegment(db, schema, table, segment, options, encr
|
|
|
318
324
|
: row.values;
|
|
319
325
|
rows.push({ values, serverVersion: row.serverVersion });
|
|
320
326
|
}
|
|
327
|
+
if (options.isCurrent?.() === false)
|
|
328
|
+
break;
|
|
321
329
|
const clearThisBlock = first && options.clearFirst;
|
|
322
330
|
first = false;
|
|
323
331
|
if (!clearThisBlock && rows.length === 0)
|
|
@@ -333,7 +341,21 @@ export async function applyRowsSegment(db, schema, table, segment, options, encr
|
|
|
333
341
|
options.onProgress?.(applied);
|
|
334
342
|
}
|
|
335
343
|
});
|
|
344
|
+
options.onProgress?.(applied);
|
|
345
|
+
await yieldToHost();
|
|
336
346
|
}
|
|
337
347
|
options.onProgress?.(applied);
|
|
338
348
|
return applied;
|
|
339
349
|
}
|
|
350
|
+
/** Let worker messages run only after the current SQLite transaction commits. */
|
|
351
|
+
export function yieldToHost() {
|
|
352
|
+
return new Promise((resolve) => {
|
|
353
|
+
const channel = new MessageChannel();
|
|
354
|
+
channel.port1.onmessage = () => {
|
|
355
|
+
channel.port1.close();
|
|
356
|
+
channel.port2.close();
|
|
357
|
+
resolve();
|
|
358
|
+
};
|
|
359
|
+
channel.port2.postMessage(null);
|
|
360
|
+
});
|
|
361
|
+
}
|
package/dist/bun-database.d.ts
CHANGED
|
@@ -21,7 +21,7 @@ export declare class BunClientDatabase implements ClientDatabase {
|
|
|
21
21
|
* §5.3 image import: bun:sqlite attaches files, not buffers, so the
|
|
22
22
|
* image lands in a private temp file for the duration of the ATTACH.
|
|
23
23
|
*/
|
|
24
|
-
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T
|
|
24
|
+
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T | Promise<T>): Promise<T>;
|
|
25
25
|
close(): void;
|
|
26
26
|
}
|
|
27
27
|
export declare function openBunDatabase(path?: string): ClientDatabase;
|
package/dist/bun-database.js
CHANGED
|
@@ -50,7 +50,7 @@ export class BunClientDatabase {
|
|
|
50
50
|
* §5.3 image import: bun:sqlite attaches files, not buffers, so the
|
|
51
51
|
* image lands in a private temp file for the duration of the ATTACH.
|
|
52
52
|
*/
|
|
53
|
-
withSqliteImage(bytes, alias, fn) {
|
|
53
|
+
async withSqliteImage(bytes, alias, fn) {
|
|
54
54
|
assertImageAlias(alias);
|
|
55
55
|
const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
|
|
56
56
|
const path = join(dir, 'segment.db');
|
|
@@ -58,7 +58,7 @@ export class BunClientDatabase {
|
|
|
58
58
|
writeFileSync(path, bytes);
|
|
59
59
|
this.db.run(`ATTACH DATABASE ? AS ${alias}`, [path]);
|
|
60
60
|
try {
|
|
61
|
-
return fn();
|
|
61
|
+
return await fn();
|
|
62
62
|
}
|
|
63
63
|
finally {
|
|
64
64
|
this.db.run(`DETACH DATABASE ${alias}`);
|
package/dist/client.d.ts
CHANGED
|
@@ -91,8 +91,8 @@ export interface SyncClientLimits {
|
|
|
91
91
|
readonly maxSnapshotPages?: number;
|
|
92
92
|
/**
|
|
93
93
|
* §4.2 accept bitmask; defaults to inline + external rows (0b0011)
|
|
94
|
-
* plus
|
|
95
|
-
*
|
|
94
|
+
* plus SQLite images and signed URLs when supported. Both image chunks
|
|
95
|
+
* and rows blocks commit before yielding to local reads (§7.7).
|
|
96
96
|
*/
|
|
97
97
|
readonly accept?: number;
|
|
98
98
|
/**
|
|
@@ -383,8 +383,8 @@ export declare class SyncClient {
|
|
|
383
383
|
* Set the live window units for a base (§4.8): a value-sharded family of
|
|
384
384
|
* subscriptions, one per unit. Computes the diff against the registry —
|
|
385
385
|
* added units get fresh subscriptions (image-lane bootstrap on the next
|
|
386
|
-
* sync); removed units are unsubscribed
|
|
387
|
-
* transaction (E1–E4). Idempotent: calling with the same units is a
|
|
386
|
+
* sync); removed units are unsubscribed with their first eviction chunk
|
|
387
|
+
* in one local transaction (E1–E4). Idempotent: calling with the same units is a
|
|
388
388
|
* no-op. Re-entry (a unit removed then re-added) cancels any deferred
|
|
389
389
|
* eviction and fresh-bootstraps.
|
|
390
390
|
*
|
package/dist/client.js
CHANGED
|
@@ -9,7 +9,7 @@ import { ProgressEmitter, } from './progress.js';
|
|
|
9
9
|
* `sync()` round (§7.2); local reads go straight to the database.
|
|
10
10
|
*/
|
|
11
11
|
import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
12
|
-
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
|
|
12
|
+
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, yieldToHost, upsertLocalRow, } from './apply.js';
|
|
13
13
|
import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
|
|
14
14
|
import { registerDevtools } from './devtools.js';
|
|
15
15
|
import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
|
|
@@ -236,6 +236,7 @@ export class SyncClient {
|
|
|
236
236
|
// an application-issued sync() call.
|
|
237
237
|
const startupWork = this.#schemaFloor === undefined &&
|
|
238
238
|
(countOutbox(this.#db) > 0 ||
|
|
239
|
+
loadPendingEvictions(this.#db).length > 0 ||
|
|
239
240
|
subscriptions.some((sub) => sub.status === 'active'));
|
|
240
241
|
if (startupWork && this.#securityLifecycle === 'active') {
|
|
241
242
|
this.#needsPull = true;
|
|
@@ -1126,22 +1127,17 @@ export class SyncClient {
|
|
|
1126
1127
|
}
|
|
1127
1128
|
socket.send(encodePresencePublish(scopeKey, doc));
|
|
1128
1129
|
}
|
|
1129
|
-
/**
|
|
1130
|
-
* §4.2 accept mask: the configured override, or the rows baseline plus
|
|
1131
|
-
* bit 2 when the backend can import sqlite images (§5.3) and a segment
|
|
1132
|
-
* downloader exists (sqlite segments are never inline, §5.7), plus
|
|
1133
|
-
* bit 3 when the downloader exposes a direct URL fetch (§5.4
|
|
1134
|
-
* capability negotiation).
|
|
1135
|
-
*/
|
|
1130
|
+
/** Advertise supported segment formats; all imports yield between commits. */
|
|
1136
1131
|
#acceptMask() {
|
|
1137
1132
|
const configured = this.#config.limits?.accept;
|
|
1138
1133
|
if (configured !== undefined)
|
|
1139
1134
|
return configured;
|
|
1140
1135
|
const segments = this.#config.segments;
|
|
1141
|
-
const sqliteCapable = typeof this.#db.withSqliteImage === 'function' && segments !== undefined;
|
|
1142
1136
|
const urlCapable = typeof segments?.fetchUrl === 'function';
|
|
1143
1137
|
return (ACCEPT_ROWS_BASELINE |
|
|
1144
|
-
(
|
|
1138
|
+
(typeof this.#db.withSqliteImage === 'function' && segments !== undefined
|
|
1139
|
+
? ACCEPT_SQLITE
|
|
1140
|
+
: 0) |
|
|
1145
1141
|
(urlCapable ? ACCEPT_SIGNED_URLS : 0));
|
|
1146
1142
|
}
|
|
1147
1143
|
subscriptions() {
|
|
@@ -1199,8 +1195,8 @@ export class SyncClient {
|
|
|
1199
1195
|
* Set the live window units for a base (§4.8): a value-sharded family of
|
|
1200
1196
|
* subscriptions, one per unit. Computes the diff against the registry —
|
|
1201
1197
|
* added units get fresh subscriptions (image-lane bootstrap on the next
|
|
1202
|
-
* sync); removed units are unsubscribed
|
|
1203
|
-
* transaction (E1–E4). Idempotent: calling with the same units is a
|
|
1198
|
+
* sync); removed units are unsubscribed with their first eviction chunk
|
|
1199
|
+
* in one local transaction (E1–E4). Idempotent: calling with the same units is a
|
|
1204
1200
|
* no-op. Re-entry (a unit removed then re-added) cancels any deferred
|
|
1205
1201
|
* eviction and fresh-bootstraps.
|
|
1206
1202
|
*
|
|
@@ -1258,7 +1254,7 @@ export class SyncClient {
|
|
|
1258
1254
|
for (const { unit, subId } of live) {
|
|
1259
1255
|
if (wanted.has(unit))
|
|
1260
1256
|
continue;
|
|
1261
|
-
this.#evictUnit(baseKey, base, unit, subId);
|
|
1257
|
+
await this.#evictUnit(baseKey, base, unit, subId);
|
|
1262
1258
|
changed = true;
|
|
1263
1259
|
}
|
|
1264
1260
|
});
|
|
@@ -1297,8 +1293,8 @@ export class SyncClient {
|
|
|
1297
1293
|
return { units: live.map((u) => u.unit), pending };
|
|
1298
1294
|
}
|
|
1299
1295
|
/**
|
|
1300
|
-
* §4.8 E1–E4: evict one departing unit
|
|
1301
|
-
*
|
|
1296
|
+
* §4.8 E1–E4: evict one departing unit in committed chunks, fusing
|
|
1297
|
+
* its first chunk with unsubscription. Deletes the unit's rows EXCEPT those pinned by a
|
|
1302
1298
|
* pending outbox commit (E1); if any pin remains, records a deferred
|
|
1303
1299
|
* eviction retried on the next outbox drain. Discards the subscription's
|
|
1304
1300
|
* cursor/resume/effective-echo (E3) and its version state with the rows
|
|
@@ -1306,19 +1302,21 @@ export class SyncClient {
|
|
|
1306
1302
|
* with no local scope-column mapping, surfaces a configuration error and
|
|
1307
1303
|
* evicts nothing (§4.8/§3.3).
|
|
1308
1304
|
*/
|
|
1309
|
-
#evictUnit(baseKey, base, unit, subId) {
|
|
1305
|
+
async #evictUnit(baseKey, base, unit, subId) {
|
|
1310
1306
|
const table = this.#table(base.table);
|
|
1311
1307
|
const sub = getSubscription(this.#db, subId);
|
|
1312
1308
|
// The rows a unit holds live under its LAST effective scopes if it ever
|
|
1313
1309
|
// synced; before first sync, the requested unit scopes are the match.
|
|
1314
1310
|
const effective = sub?.effectiveScopes ?? unitScopes(base, unit);
|
|
1315
1311
|
const pinned = this.#pinnedRowIds(base.table);
|
|
1312
|
+
let remaining = false;
|
|
1316
1313
|
this.#applyBatch((batch) => {
|
|
1317
1314
|
this.#db.transaction(() => {
|
|
1318
|
-
const
|
|
1315
|
+
const result = evictScopedRows(this.#db, table, effective, pinned);
|
|
1316
|
+
remaining = result.remaining;
|
|
1319
1317
|
deleteWindowUnit(this.#db, baseKey, unit);
|
|
1320
1318
|
deleteSubscription(this.#db, subId);
|
|
1321
|
-
if (deferred) {
|
|
1319
|
+
if (result.deferred) {
|
|
1322
1320
|
savePendingEviction(this.#db, subId, base.table, effective);
|
|
1323
1321
|
}
|
|
1324
1322
|
else {
|
|
@@ -1330,13 +1328,17 @@ export class SyncClient {
|
|
|
1330
1328
|
this.#needsPull = true;
|
|
1331
1329
|
batch.status();
|
|
1332
1330
|
});
|
|
1331
|
+
if (remaining) {
|
|
1332
|
+
await yieldToHost();
|
|
1333
|
+
await this.#drainPendingEvictions();
|
|
1334
|
+
}
|
|
1333
1335
|
}
|
|
1334
1336
|
/**
|
|
1335
1337
|
* §4.8 E1: retry deferred evictions after the outbox drains. A pinned
|
|
1336
1338
|
* unit's rows are removed once no pending commit references them; a unit
|
|
1337
1339
|
* that re-entered the window in the meantime has no pending record left.
|
|
1338
1340
|
*/
|
|
1339
|
-
#drainPendingEvictions() {
|
|
1341
|
+
async #drainPendingEvictions() {
|
|
1340
1342
|
const pending = loadPendingEvictions(this.#db);
|
|
1341
1343
|
if (pending.length === 0)
|
|
1342
1344
|
return;
|
|
@@ -1346,16 +1348,23 @@ export class SyncClient {
|
|
|
1346
1348
|
deletePendingEviction(this.#db, entry.subId);
|
|
1347
1349
|
continue;
|
|
1348
1350
|
}
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1351
|
+
let remaining;
|
|
1352
|
+
do {
|
|
1353
|
+
if (!loadPendingEvictions(this.#db).some((item) => item.subId === entry.subId))
|
|
1354
|
+
break;
|
|
1355
|
+
// Local writes may add pins while the worker yields between chunks.
|
|
1356
|
+
const pinned = this.#pinnedRowIds(entry.table);
|
|
1357
|
+
remaining = false;
|
|
1358
|
+
this.#applyBatch((batch) => {
|
|
1359
|
+
const result = evictScopedRows(this.#db, table, entry.effective, pinned);
|
|
1360
|
+
remaining = result.remaining;
|
|
1361
|
+
if (!result.deferred)
|
|
1355
1362
|
deletePendingEviction(this.#db, entry.subId);
|
|
1363
|
+
batch.scopeMap(table, entry.effective);
|
|
1356
1364
|
});
|
|
1357
|
-
|
|
1358
|
-
|
|
1365
|
+
if (remaining)
|
|
1366
|
+
await yieldToHost();
|
|
1367
|
+
} while (remaining);
|
|
1359
1368
|
}
|
|
1360
1369
|
}
|
|
1361
1370
|
/**
|
|
@@ -1920,6 +1929,7 @@ export class SyncClient {
|
|
|
1920
1929
|
// survive it — the reference server keeps no replay buffer (§8.2).
|
|
1921
1930
|
this.#setSyncNeeded(false);
|
|
1922
1931
|
try {
|
|
1932
|
+
await this.#drainPendingEvictions();
|
|
1923
1933
|
const logEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
|
|
1924
1934
|
// §5.9.7 B4: upload pending blobs BEFORE pushing rows that reference
|
|
1925
1935
|
// them, so the server-side existence check (§6.6) passes.
|
|
@@ -1981,7 +1991,7 @@ export class SyncClient {
|
|
|
1981
1991
|
const summary = await this.#processResponse(message, outbox, subs, 'pull', resetEpoch);
|
|
1982
1992
|
// §4.8 E1: the push half may have drained commits that pinned rows of
|
|
1983
1993
|
// a shrunk window unit — retry any deferred evictions now.
|
|
1984
|
-
this.#drainPendingEvictions();
|
|
1994
|
+
await this.#drainPendingEvictions();
|
|
1985
1995
|
this.#progress.update(summary.failed.length > 0
|
|
1986
1996
|
? { state: 'failed', errorCode: 'sync.scope_revoked' }
|
|
1987
1997
|
: { state: 'complete' });
|
|
@@ -2028,10 +2038,21 @@ export class SyncClient {
|
|
|
2028
2038
|
* Pull repeatedly until quiescent: no commits delivered, no bootstrap
|
|
2029
2039
|
* pages pending, no resets to recover (§4.5 "pull again" SHOULD).
|
|
2030
2040
|
*/
|
|
2031
|
-
async syncUntilIdle(maxRounds
|
|
2041
|
+
async syncUntilIdle(maxRounds) {
|
|
2042
|
+
const budget = maxRounds ?? 20;
|
|
2032
2043
|
let last;
|
|
2033
|
-
for (let round = 0; round <
|
|
2044
|
+
for (let round = 0; round < budget; round++) {
|
|
2045
|
+
const before = JSON.stringify(loadSubscriptions(this.#db).map((sub) => [sub.id, sub.bootstrapState]));
|
|
2034
2046
|
last = await this.sync();
|
|
2047
|
+
if (maxRounds === undefined &&
|
|
2048
|
+
last.segmentRowsApplied > 0 &&
|
|
2049
|
+
last.bootstrapping.length > 0 &&
|
|
2050
|
+
before !==
|
|
2051
|
+
JSON.stringify(loadSubscriptions(this.#db).map((sub) => [
|
|
2052
|
+
sub.id,
|
|
2053
|
+
sub.bootstrapState,
|
|
2054
|
+
])))
|
|
2055
|
+
round = -1;
|
|
2035
2056
|
if (last.schemaFloor !== undefined)
|
|
2036
2057
|
return last;
|
|
2037
2058
|
if (last.commitsApplied === 0 &&
|
|
@@ -2043,7 +2064,7 @@ export class SyncClient {
|
|
|
2043
2064
|
return last;
|
|
2044
2065
|
}
|
|
2045
2066
|
}
|
|
2046
|
-
throw new ClientSyncError('sync.invalid_request',
|
|
2067
|
+
throw new ClientSyncError('sync.invalid_request', 'sync did not reach idle within the round budget');
|
|
2047
2068
|
}
|
|
2048
2069
|
/**
|
|
2049
2070
|
* One request/response round trip (§8.7): over the socket whenever it
|
|
@@ -2469,6 +2490,7 @@ export class SyncClient {
|
|
|
2469
2490
|
});
|
|
2470
2491
|
await this.#applySegmentOrFail(section, summary, (table, clearFirst, effective) => applyRowsSegment(this.#db, this.#schema, table, segment, {
|
|
2471
2492
|
clearFirst,
|
|
2493
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2472
2494
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2473
2495
|
effective,
|
|
2474
2496
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
@@ -2477,7 +2499,9 @@ export class SyncClient {
|
|
|
2477
2499
|
this.#scopedRowsExist(table, effective))) {
|
|
2478
2500
|
batch.table(table.name);
|
|
2479
2501
|
}
|
|
2480
|
-
|
|
2502
|
+
const result = fn();
|
|
2503
|
+
this.#replayOutbox();
|
|
2504
|
+
return result;
|
|
2481
2505
|
}),
|
|
2482
2506
|
}, this.#encryption), section.fresh && !section.cleared);
|
|
2483
2507
|
break;
|
|
@@ -2510,6 +2534,7 @@ export class SyncClient {
|
|
|
2510
2534
|
scopeDigest: frame.scopeDigest,
|
|
2511
2535
|
}, {
|
|
2512
2536
|
clearFirst,
|
|
2537
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2513
2538
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2514
2539
|
effective,
|
|
2515
2540
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
@@ -2518,7 +2543,9 @@ export class SyncClient {
|
|
|
2518
2543
|
this.#scopedRowsExist(table, effective))) {
|
|
2519
2544
|
batch.table(table.name);
|
|
2520
2545
|
}
|
|
2521
|
-
|
|
2546
|
+
const result = fn();
|
|
2547
|
+
this.#replayOutbox();
|
|
2548
|
+
return result;
|
|
2522
2549
|
}),
|
|
2523
2550
|
}), section.fresh && !section.cleared);
|
|
2524
2551
|
}
|
|
@@ -2526,6 +2553,7 @@ export class SyncClient {
|
|
|
2526
2553
|
const segment = decodeRowsSegment(bytes);
|
|
2527
2554
|
await this.#applySegmentOrFail(section, summary, (table, clearFirst, effective) => applyRowsSegment(this.#db, this.#schema, table, segment, {
|
|
2528
2555
|
clearFirst,
|
|
2556
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2529
2557
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2530
2558
|
effective,
|
|
2531
2559
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
@@ -2534,7 +2562,9 @@ export class SyncClient {
|
|
|
2534
2562
|
this.#scopedRowsExist(table, effective))) {
|
|
2535
2563
|
batch.table(table.name);
|
|
2536
2564
|
}
|
|
2537
|
-
|
|
2565
|
+
const result = fn();
|
|
2566
|
+
this.#replayOutbox();
|
|
2567
|
+
return result;
|
|
2538
2568
|
}),
|
|
2539
2569
|
}, this.#encryption), section.fresh &&
|
|
2540
2570
|
!section.cleared &&
|
package/dist/database.d.ts
CHANGED
|
@@ -20,11 +20,12 @@ export interface ClientDatabase {
|
|
|
20
20
|
/**
|
|
21
21
|
* Optional sqlite-image capability (SPEC §5.3): expose `bytes` as an
|
|
22
22
|
* attached read-only database schema named `alias` for the duration of
|
|
23
|
-
* `fn`,
|
|
23
|
+
* `fn`, awaiting its result before detaching. No transaction spans its awaits.
|
|
24
|
+
* Presence of this method is what makes the client
|
|
24
25
|
* advertise `accept` bit 2 (§4.2). MUST be called outside any open
|
|
25
26
|
* transaction (SQLite cannot ATTACH inside one); `fn` may open its own.
|
|
26
27
|
*/
|
|
27
|
-
withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T): T
|
|
28
|
+
withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T | Promise<T>): Promise<T>;
|
|
28
29
|
close(): void;
|
|
29
30
|
}
|
|
30
31
|
/** Attach-schema aliases are code-chosen, never user input — enforce it. */
|
package/dist/node-database.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export declare class NodeClientDatabase implements ClientDatabase {
|
|
|
12
12
|
query(sql: string, params?: readonly SqlValue[]): SqlRow[];
|
|
13
13
|
transaction<T>(fn: () => T): T;
|
|
14
14
|
/** §5.3 image import through a private file attached for one callback. */
|
|
15
|
-
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T
|
|
15
|
+
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T | Promise<T>): Promise<T>;
|
|
16
16
|
close(): void;
|
|
17
17
|
}
|
|
18
18
|
export declare function openNodeDatabase(path?: string): ClientDatabase;
|
package/dist/node-database.js
CHANGED
|
@@ -52,7 +52,7 @@ export class NodeClientDatabase {
|
|
|
52
52
|
return runTransaction(this.#tx, (sql) => this.db.exec(sql), fn);
|
|
53
53
|
}
|
|
54
54
|
/** §5.3 image import through a private file attached for one callback. */
|
|
55
|
-
withSqliteImage(bytes, alias, fn) {
|
|
55
|
+
async withSqliteImage(bytes, alias, fn) {
|
|
56
56
|
assertImageAlias(alias);
|
|
57
57
|
const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
|
|
58
58
|
const path = join(dir, 'segment.db');
|
|
@@ -60,7 +60,7 @@ export class NodeClientDatabase {
|
|
|
60
60
|
writeFileSync(path, bytes);
|
|
61
61
|
this.db.prepare(`ATTACH DATABASE ? AS ${alias}`).run(path);
|
|
62
62
|
try {
|
|
63
|
-
return fn();
|
|
63
|
+
return await fn();
|
|
64
64
|
}
|
|
65
65
|
finally {
|
|
66
66
|
this.db.prepare(`DETACH DATABASE ${alias}`).run();
|