@syncular/client 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/apply.d.ts +1 -1
- package/dist/apply.js +25 -26
- package/dist/client.d.ts +19 -9
- package/dist/client.js +62 -45
- package/dist/encryption.js +3 -9
- package/dist/failed-overlay.d.ts +8 -2
- package/dist/failed-overlay.js +31 -20
- package/dist/native-bridge.d.ts +3 -2
- package/dist/native-bridge.js +5 -1
- package/dist/outcomes.d.ts +2 -1
- package/dist/outcomes.js +2 -4
- package/dist/progress.d.ts +4 -0
- package/dist/progress.js +14 -2
- package/dist/realtime-supervisor.d.ts +4 -1
- package/dist/realtime-supervisor.js +2 -2
- package/dist/sync-scheduler.d.ts +8 -2
- package/dist/sync-scheduler.js +11 -3
- package/dist/worker-core.js +7 -2
- package/dist/worker-host.d.ts +5 -0
- package/dist/worker-host.js +6 -0
- package/dist/worker-protocol.d.ts +2 -0
- package/package.json +3 -3
- package/src/apply.ts +34 -47
- package/src/client.ts +85 -53
- package/src/encryption.ts +8 -15
- package/src/failed-overlay.ts +37 -23
- package/src/native-bridge.ts +5 -1
- package/src/outcomes.ts +2 -6
- package/src/progress.ts +18 -2
- package/src/realtime-supervisor.ts +7 -3
- package/src/sync-scheduler.ts +17 -5
- package/src/worker-core.ts +5 -2
- package/src/worker-host.ts +13 -0
- package/src/worker-protocol.ts +2 -0
package/README.md
CHANGED
|
@@ -676,12 +676,13 @@ directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
|
|
|
676
676
|
|
|
677
677
|
### Size cap
|
|
678
678
|
|
|
679
|
-
|
|
679
|
+
Set `limits.blobCacheMaxBytes` to cap the on-device cache. The worker handle
|
|
680
|
+
forwards `limits` to its core, and the native core reads the same key. When the sum of cached body
|
|
680
681
|
sizes exceeds the cap, the client evicts bodies with no visible row or pending
|
|
681
682
|
commit by creation timestamp, then blob ID, until back under the cap:
|
|
682
683
|
|
|
683
684
|
```ts
|
|
684
|
-
new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
|
|
685
|
+
new SyncClient({ /* … */, limits: { blobCacheMaxBytes: 256 * 1024 * 1024 } }); // 256 MiB
|
|
685
686
|
```
|
|
686
687
|
|
|
687
688
|
- A body **referenced by a live row** is **never** evicted. It
|
|
@@ -691,7 +692,7 @@ new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
|
|
|
691
692
|
- Evicting an unreferenced body only costs a future re-download, never correctness:
|
|
692
693
|
any surviving `blob_ref` value re-enables the fetch (§5.9.7 B3). If every
|
|
693
694
|
over-cap body is referenced or pinned, the cache stays over the cap
|
|
694
|
-
(correctness beats the cap). Absent `blobCacheMaxBytes` ⇒ retain until storage
|
|
695
|
+
(correctness beats the cap). Absent `limits.blobCacheMaxBytes` ⇒ retain until storage
|
|
695
696
|
pressure (the default).
|
|
696
697
|
|
|
697
698
|
### Very large media — the escape hatch
|
|
@@ -846,6 +847,12 @@ under a new `epoch`. `phase` is `request`,
|
|
|
846
847
|
`download`, or `import`; `state` is `running`, `complete`, or `failed`. Unknown
|
|
847
848
|
totals remain absent. A new attempt clears prior errors and counters.
|
|
848
849
|
|
|
850
|
+
`unitRowsTotal` and `unitRowsProcessed` cover the bootstrap of the unit in
|
|
851
|
+
`unitId` as a whole: the rows the server counted when the bootstrap started and
|
|
852
|
+
the rows imported since, across segments and rounds. The count only rises and
|
|
853
|
+
ends equal to the total; both are absent when the server states no count
|
|
854
|
+
(SPEC §7.7).
|
|
855
|
+
|
|
849
856
|
HTTP downloads report decoded body bytes at 64 KiB intervals or larger network
|
|
850
857
|
chunks. Custom segment downloaders receive `request.onProgress`; signed-URL
|
|
851
858
|
implementations receive an optional second callback argument. Call it with the
|
package/dist/apply.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { EncryptionConfig, SealedRow } from './encryption.js';
|
|
|
10
10
|
import { type CompiledClientSchema, type CompiledClientTable, type JsonRowValue } from './schema.js';
|
|
11
11
|
/** Lets the client wrap the physical write in its revision transaction. */
|
|
12
12
|
export type ApplyTransaction = (fn: () => OverlayScope) => unknown;
|
|
13
|
-
export declare function upsertLocalRow(db: ClientDatabase, table: CompiledClientTable, values: readonly RowValue[], syncVersion: number,
|
|
13
|
+
export declare function upsertLocalRow(db: ClientDatabase, table: CompiledClientTable, values: readonly RowValue[], syncVersion: number, sealed?: SealedRow): void;
|
|
14
14
|
export declare function deleteLocalRow(db: ClientDatabase, table: CompiledClientTable, rowId: string, serverWrite?: boolean): void;
|
|
15
15
|
/**
|
|
16
16
|
* Apply one `COMMIT` frame in one local transaction (§1.4 rule 4) under the
|
package/dist/apply.js
CHANGED
|
@@ -9,7 +9,7 @@ import { retainedBaseWrite, restoreFailedBases } from './failed-overlay.js';
|
|
|
9
9
|
*/
|
|
10
10
|
import { decodeRow, } from '@syncular/core';
|
|
11
11
|
import { ClientSyncError, invalidRequest } from './errors.js';
|
|
12
|
-
import { quoteIdent, matchesLocalScopes, pkSqlValue, SEALED_COLUMN, SEALED_KEY_COLUMN, SYNC_VERSION_COLUMN, toSqlValue,
|
|
12
|
+
import { quoteIdent, matchesLocalScopes, pkSqlValue, SEALED_COLUMN, SEALED_KEY_COLUMN, SYNC_VERSION_COLUMN, toSqlValue, } from './schema.js';
|
|
13
13
|
function upsertSql(table) {
|
|
14
14
|
const names = [
|
|
15
15
|
...table.columns.map((column) => quoteIdent(column.name)),
|
|
@@ -25,7 +25,7 @@ function upsertSql(table) {
|
|
|
25
25
|
.join(', ');
|
|
26
26
|
return `INSERT INTO ${quoteIdent(table.name)}(${names.join(', ')})VALUES(${placeholders})ON CONFLICT(${quoteIdent(table.primaryKey)})DO UPDATE SET ${updates}`;
|
|
27
27
|
}
|
|
28
|
-
export function upsertLocalRow(db, table, values, syncVersion,
|
|
28
|
+
export function upsertLocalRow(db, table, values, syncVersion, sealed) {
|
|
29
29
|
db.exec(upsertSql(table), [
|
|
30
30
|
...values.map(toSqlValue),
|
|
31
31
|
syncVersion,
|
|
@@ -33,16 +33,11 @@ export function upsertLocalRow(db, table, values, syncVersion, serverWrite = tru
|
|
|
33
33
|
? [sealed?.keyId ?? null, sealed?.envelopes ?? null]
|
|
34
34
|
: []),
|
|
35
35
|
]);
|
|
36
|
-
if (serverWrite)
|
|
37
|
-
retainedBaseWrite(db, table, String(values[table.primaryKeyIndex]), Object.fromEntries(table.columns.map((column, index) => [
|
|
38
|
-
column.name,
|
|
39
|
-
rowValueToJson(values[index] ?? null),
|
|
40
|
-
])), syncVersion);
|
|
41
36
|
}
|
|
42
37
|
export function deleteLocalRow(db, table, rowId, serverWrite = true) {
|
|
43
|
-
if (serverWrite)
|
|
44
|
-
retainedBaseWrite(db, table, rowId);
|
|
45
38
|
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`, [pkSqlValue(table, rowId)]);
|
|
39
|
+
if (serverWrite)
|
|
40
|
+
retainedBaseWrite(db, table, new Map([[rowId, undefined]]));
|
|
46
41
|
}
|
|
47
42
|
/**
|
|
48
43
|
* §4.5 version guard for one server row: whether a write at `version` lands.
|
|
@@ -139,8 +134,10 @@ export async function applyCommitFrame(db, schema, frame, encryption, transactio
|
|
|
139
134
|
continue;
|
|
140
135
|
if (change.op === 'delete')
|
|
141
136
|
deleteLocalRow(db, change.table, rowId);
|
|
142
|
-
else
|
|
143
|
-
upsertLocalRow(db, change.table, change.values, rowVersion,
|
|
137
|
+
else {
|
|
138
|
+
upsertLocalRow(db, change.table, change.values, rowVersion, change.sealed);
|
|
139
|
+
retainedBaseWrite(db, change.table, new Map([[rowId, [change.values, rowVersion]]]));
|
|
140
|
+
}
|
|
144
141
|
}
|
|
145
142
|
return rebaseScope(scope, kept, skipped);
|
|
146
143
|
});
|
|
@@ -240,11 +237,9 @@ export function deleteScopedRows(db, table, effective, owners) {
|
|
|
240
237
|
if (!scope)
|
|
241
238
|
return;
|
|
242
239
|
const { sql, params } = scope;
|
|
243
|
-
|
|
244
|
-
for (const row of scopedRows(db, table, scope)) {
|
|
245
|
-
retainedBaseWrite(db, table, String(row.id));
|
|
246
|
-
}
|
|
240
|
+
const retained = retainsRows(db, table) ? scopedRows(db, table, scope) : [];
|
|
247
241
|
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${sql}`, params);
|
|
242
|
+
retainedBaseWrite(db, table, new Map(retained.map((row) => [String(row.id), undefined])));
|
|
248
243
|
}
|
|
249
244
|
/**
|
|
250
245
|
* §4.8 window eviction: delete rows matching a departing unit's effective
|
|
@@ -396,7 +391,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
396
391
|
: []),
|
|
397
392
|
...rows.map((row) => JSON.stringify([table.name, String(row[table.primaryKey])])),
|
|
398
393
|
];
|
|
399
|
-
|
|
394
|
+
restoreFailedBases(db, schema, scope);
|
|
400
395
|
if (first && options.clearFirst)
|
|
401
396
|
deleteScopedRows(db, table, options.effective, options.owners());
|
|
402
397
|
const kept = new Set();
|
|
@@ -415,13 +410,13 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
415
410
|
break;
|
|
416
411
|
[boundary, where, params] = range(boundary, Math.min(1024, size - chunk));
|
|
417
412
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
413
|
+
retainedBaseWrite(db, table, new Map(landing.map((row) => [
|
|
414
|
+
String(row[table.primaryKey]),
|
|
415
|
+
[
|
|
416
|
+
table.columns.map((column) => row[column.name]),
|
|
417
|
+
Number(row._syncular_version),
|
|
418
|
+
],
|
|
419
|
+
])));
|
|
425
420
|
options.onChunk?.(applied);
|
|
426
421
|
return rebaseScope(scope, kept, skipped);
|
|
427
422
|
});
|
|
@@ -489,20 +484,24 @@ export async function applyRowsSegment(db, schema, table, segment, options, encr
|
|
|
489
484
|
String(row.values[table.primaryKeyIndex]),
|
|
490
485
|
])),
|
|
491
486
|
];
|
|
492
|
-
|
|
487
|
+
restoreFailedBases(db, schema, scope);
|
|
493
488
|
if (clearThisBlock) {
|
|
494
489
|
deleteScopedRows(db, table, options.effective, options.owners());
|
|
495
490
|
}
|
|
496
491
|
const kept = new Set();
|
|
497
492
|
const skipped = new Set();
|
|
493
|
+
const written = new Map();
|
|
498
494
|
for (const row of rows) {
|
|
499
495
|
const rowId = String(row.values[table.primaryKeyIndex]);
|
|
500
|
-
if (lands(db, table, rowId, row.serverVersion, kept, skipped))
|
|
501
|
-
upsertLocalRow(db, table, row.values, row.serverVersion,
|
|
496
|
+
if (lands(db, table, rowId, row.serverVersion, kept, skipped)) {
|
|
497
|
+
upsertLocalRow(db, table, row.values, row.serverVersion, row.sealed);
|
|
498
|
+
written.set(rowId, [row.values, row.serverVersion]);
|
|
499
|
+
}
|
|
502
500
|
applied += 1;
|
|
503
501
|
if (applied % 1024 === 0)
|
|
504
502
|
options.onProgress?.(applied);
|
|
505
503
|
}
|
|
504
|
+
retainedBaseWrite(db, table, written);
|
|
506
505
|
return rebaseScope(scope, kept, skipped);
|
|
507
506
|
});
|
|
508
507
|
options.onProgress?.(applied);
|
package/dist/client.d.ts
CHANGED
|
@@ -99,7 +99,7 @@ export interface SyncIdleResult extends SyncSummary {
|
|
|
99
99
|
}
|
|
100
100
|
export interface SyncIdleOptions {
|
|
101
101
|
/** §7.8: a strict positive round count. Default: 20. */
|
|
102
|
-
readonly maxRounds?: number;
|
|
102
|
+
readonly maxRounds?: number | undefined;
|
|
103
103
|
/**
|
|
104
104
|
* Total budget for one drive, across every round, on the client clock. A
|
|
105
105
|
* drive past it after a completed round resolves with `budgetExhausted`.
|
|
@@ -108,6 +108,12 @@ export interface SyncIdleOptions {
|
|
|
108
108
|
* `Infinity` disables the bound.
|
|
109
109
|
*/
|
|
110
110
|
readonly timeoutMs?: number;
|
|
111
|
+
/**
|
|
112
|
+
* §8.4 host policy: end the drive without a round when the replica has no
|
|
113
|
+
* unit, no outbox commit, and no unit list to clear on the server. The
|
|
114
|
+
* shipped host loops pass it; the native bridge ignores it.
|
|
115
|
+
*/
|
|
116
|
+
readonly skipEmpty?: boolean;
|
|
111
117
|
}
|
|
112
118
|
export type CurrentBaseProof = {
|
|
113
119
|
readonly proven: true;
|
|
@@ -149,6 +155,18 @@ export interface SyncClientLimits {
|
|
|
149
155
|
* raises `client.push_request_too_large` and stays queued.
|
|
150
156
|
*/
|
|
151
157
|
readonly maxPushRequestBytes?: number;
|
|
158
|
+
/**
|
|
159
|
+
* §5.4: rows segment downloads one pull response runs at once. Default 4.
|
|
160
|
+
* SQLite images keep their own bound of two (§5.3).
|
|
161
|
+
*/
|
|
162
|
+
readonly segmentPrefetch?: number;
|
|
163
|
+
/**
|
|
164
|
+
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
165
|
+
* exceeds it, unreferenced bodies without pending work are evicted oldest
|
|
166
|
+
* first after each cache write. Referenced bodies stay cached even past the
|
|
167
|
+
* cap. Absent: retain until storage pressure.
|
|
168
|
+
*/
|
|
169
|
+
readonly blobCacheMaxBytes?: number;
|
|
152
170
|
}
|
|
153
171
|
export interface SyncClientConfig {
|
|
154
172
|
/** Explicit development console registry; absent means no registration. */
|
|
@@ -162,14 +180,6 @@ export interface SyncClientConfig {
|
|
|
162
180
|
readonly segments?: SegmentDownloader;
|
|
163
181
|
/** Blob upload/download (§5.9). Required to use `uploadBlob`/`fetchBlob`. */
|
|
164
182
|
readonly blobs?: BlobTransport;
|
|
165
|
-
/**
|
|
166
|
-
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
167
|
-
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
168
|
-
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
169
|
-
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
170
|
-
* default; a referenced body always stays resolvable without a re-download).
|
|
171
|
-
*/
|
|
172
|
-
readonly blobCacheMaxBytes?: number;
|
|
173
183
|
readonly realtime?: RealtimeConnector;
|
|
174
184
|
/**
|
|
175
185
|
* SPEC §8.8: how sync rounds treat the realtime binding. `optional`
|
package/dist/client.js
CHANGED
|
@@ -141,6 +141,14 @@ function realtimeErrorCode(error) {
|
|
|
141
141
|
const code = error?.code;
|
|
142
142
|
return typeof code === 'string' ? code : undefined;
|
|
143
143
|
}
|
|
144
|
+
/** Encodes a request in the current wire version. */
|
|
145
|
+
function encodeRequestFrames(frames) {
|
|
146
|
+
return encodeRequest({
|
|
147
|
+
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
148
|
+
msgKind: 'request',
|
|
149
|
+
frames,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
144
152
|
/** §7.7: progress fields of a payload; no payload is current. */
|
|
145
153
|
const NO_PAYLOAD = {
|
|
146
154
|
table: undefined,
|
|
@@ -292,10 +300,11 @@ export class SyncClient {
|
|
|
292
300
|
config.limits?.maxPushOperationsPerRequest,
|
|
293
301
|
],
|
|
294
302
|
['maxPushRequestBytes', config.limits?.maxPushRequestBytes],
|
|
303
|
+
['segmentPrefetch', config.limits?.segmentPrefetch],
|
|
295
304
|
]) {
|
|
296
305
|
if (value !== undefined &&
|
|
297
306
|
(!Number.isInteger(value) || value < 1 || value > 0xffffffff)) {
|
|
298
|
-
throw new ClientSyncError('sync.invalid_request', '
|
|
307
|
+
throw new ClientSyncError('sync.invalid_request', 'limit must be an integer in 1..=4294967295', false, { limit: name });
|
|
299
308
|
}
|
|
300
309
|
}
|
|
301
310
|
if (config.securityPreflight === true && config.encryption !== undefined) {
|
|
@@ -1360,8 +1369,7 @@ export class SyncClient {
|
|
|
1360
1369
|
const blobBytes = this.#hasBlobs
|
|
1361
1370
|
? Number(this.#db.query('SELECT COALESCE(SUM(byte_length),0)AS bytes FROM _syncular_blobs')[0]?.bytes ?? 0)
|
|
1362
1371
|
: 0;
|
|
1363
|
-
const pressure = this.#config.blobCacheMaxBytes
|
|
1364
|
-
blobBytes > this.#config.blobCacheMaxBytes;
|
|
1372
|
+
const pressure = blobBytes > (this.#config.limits?.blobCacheMaxBytes ?? Infinity);
|
|
1365
1373
|
return {
|
|
1366
1374
|
status: pressure ? 'pressure' : 'healthy',
|
|
1367
1375
|
databaseBytesApprox: Math.max(0, pageCount * pageSize),
|
|
@@ -1609,7 +1617,7 @@ export class SyncClient {
|
|
|
1609
1617
|
}
|
|
1610
1618
|
/** §5.9.7 B1: trim the blob cache to the configured cap (no-op if unset). */
|
|
1611
1619
|
#enforceBlobCacheCap() {
|
|
1612
|
-
const cap = this.#config.blobCacheMaxBytes;
|
|
1620
|
+
const cap = this.#config.limits?.blobCacheMaxBytes;
|
|
1613
1621
|
if (cap === undefined)
|
|
1614
1622
|
return;
|
|
1615
1623
|
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
@@ -2756,13 +2764,7 @@ export class SyncClient {
|
|
|
2756
2764
|
// A request must start with REQ_HEADER and carry PULL_HEADER; those two
|
|
2757
2765
|
// frames make a valid probe whose encoded size cancels out of the
|
|
2758
2766
|
// per-commit delta measured against the canonical encoder.
|
|
2759
|
-
const baseBytes = byteCap === undefined
|
|
2760
|
-
? 0
|
|
2761
|
-
: encodeRequest({
|
|
2762
|
-
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
2763
|
-
msgKind: 'request',
|
|
2764
|
-
frames: [header, pull],
|
|
2765
|
-
}).length;
|
|
2767
|
+
const baseBytes = byteCap === undefined ? 0 : encodeRequestFrames([header, pull]).length;
|
|
2766
2768
|
const pushFrames = [];
|
|
2767
2769
|
const outbox = [];
|
|
2768
2770
|
const frameBytes = [];
|
|
@@ -2830,11 +2832,7 @@ export class SyncClient {
|
|
|
2830
2832
|
}
|
|
2831
2833
|
const frame = byteCap === undefined
|
|
2832
2834
|
? 0
|
|
2833
|
-
:
|
|
2834
|
-
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
2835
|
-
msgKind: 'request',
|
|
2836
|
-
frames: [header, pushFrame, pull],
|
|
2837
|
-
}).length - baseBytes;
|
|
2835
|
+
: encodeRequestFrames([header, pushFrame, pull]).length - baseBytes;
|
|
2838
2836
|
// The head is always encoded so the caller can report the full
|
|
2839
2837
|
// projected request size with the final header and subscription bytes;
|
|
2840
2838
|
// a later commit defers once it no longer fits.
|
|
@@ -3192,11 +3190,7 @@ export class SyncClient {
|
|
|
3192
3190
|
// The fixed frames encode as envelope + fixed frames + END; each
|
|
3193
3191
|
// admitted push frame adds its own (5 + payload). Trimming the tail
|
|
3194
3192
|
// re-fits the request to the final registration state.
|
|
3195
|
-
const fixedBytes =
|
|
3196
|
-
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
3197
|
-
msgKind: 'request',
|
|
3198
|
-
frames: fixedFrames,
|
|
3199
|
-
}).length;
|
|
3193
|
+
const fixedBytes = encodeRequestFrames(fixedFrames).length;
|
|
3200
3194
|
if (fixedBytes > byteCap)
|
|
3201
3195
|
throw this.#pushCapacityError('bytes', byteCap, fixedBytes);
|
|
3202
3196
|
let admittedBytes = 0;
|
|
@@ -3218,11 +3212,7 @@ export class SyncClient {
|
|
|
3218
3212
|
...pushFrames,
|
|
3219
3213
|
...fixedFrames.slice(lead.length),
|
|
3220
3214
|
];
|
|
3221
|
-
const requestBytes =
|
|
3222
|
-
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
3223
|
-
msgKind: 'request',
|
|
3224
|
-
frames,
|
|
3225
|
-
});
|
|
3215
|
+
const requestBytes = encodeRequestFrames(frames);
|
|
3226
3216
|
const responseBytes = await this.#roundTrip(requestBytes, signal);
|
|
3227
3217
|
const message = decodeResponse(responseBytes);
|
|
3228
3218
|
// §1.5: a complete, decodable response ends the report obligation.
|
|
@@ -3339,19 +3329,20 @@ export class SyncClient {
|
|
|
3339
3329
|
if (timeoutMs !== Number.POSITIVE_INFINITY &&
|
|
3340
3330
|
!(Number.isFinite(timeoutMs) && timeoutMs > 0))
|
|
3341
3331
|
throw invalidRequest('timeoutMs must be a positive number or Infinity');
|
|
3342
|
-
const
|
|
3332
|
+
const skipEmpty = typeof options === 'object' && options.skipEmpty === true;
|
|
3333
|
+
const key = `${maxRounds ?? 'default'}:${timeoutMs}:${skipEmpty}`;
|
|
3343
3334
|
const pending = this.#syncIdlePending.get(key);
|
|
3344
3335
|
if (pending !== undefined)
|
|
3345
3336
|
return pending;
|
|
3346
3337
|
const previous = this.#syncIdleTail;
|
|
3347
3338
|
let drive;
|
|
3348
3339
|
if (previous === undefined) {
|
|
3349
|
-
drive = this.#syncIdleRounds(maxRounds, timeoutMs);
|
|
3340
|
+
drive = this.#syncIdleRounds(maxRounds, timeoutMs, skipEmpty);
|
|
3350
3341
|
}
|
|
3351
3342
|
else {
|
|
3352
3343
|
drive = previous.then(() => {
|
|
3353
3344
|
this.#syncIdlePending.delete(key);
|
|
3354
|
-
return this.#syncIdleRounds(maxRounds, timeoutMs);
|
|
3345
|
+
return this.#syncIdleRounds(maxRounds, timeoutMs, skipEmpty);
|
|
3355
3346
|
});
|
|
3356
3347
|
this.#syncIdlePending.set(key, drive);
|
|
3357
3348
|
}
|
|
@@ -3364,7 +3355,7 @@ export class SyncClient {
|
|
|
3364
3355
|
return drive;
|
|
3365
3356
|
}
|
|
3366
3357
|
/** The rounds loop of one idle drive; `syncUntilIdle` owns coalescing. */
|
|
3367
|
-
async #syncIdleRounds(maxRounds, timeoutMs) {
|
|
3358
|
+
async #syncIdleRounds(maxRounds, timeoutMs, skipEmpty) {
|
|
3368
3359
|
const budget = maxRounds ?? 20;
|
|
3369
3360
|
const aggregate = {
|
|
3370
3361
|
...emptySummary(0),
|
|
@@ -3381,6 +3372,19 @@ export class SyncClient {
|
|
|
3381
3372
|
budgetExhausted,
|
|
3382
3373
|
};
|
|
3383
3374
|
};
|
|
3375
|
+
// §8.4: a replica without units, outbox work, or a server-side unit list
|
|
3376
|
+
// to clear has nothing to exchange; a host drive ends without a round.
|
|
3377
|
+
this.#requireActive();
|
|
3378
|
+
if (skipEmpty &&
|
|
3379
|
+
this.#schemaFloor === undefined &&
|
|
3380
|
+
!this.#hasStartupWork() &&
|
|
3381
|
+
loadUnits(this.#db).length === 0 &&
|
|
3382
|
+
(getMeta(this.#db, LOG_EPOCH_META_KEY) === undefined ||
|
|
3383
|
+
getMeta(this.#db, REGISTERED_DIGEST_KEY) ===
|
|
3384
|
+
bytesToHex(registrationDigest([])))) {
|
|
3385
|
+
this.#setSyncNeeded(false);
|
|
3386
|
+
return result(false);
|
|
3387
|
+
}
|
|
3384
3388
|
const states = () => JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]));
|
|
3385
3389
|
const startedAtMs = this.#now();
|
|
3386
3390
|
let completed = false;
|
|
@@ -3904,11 +3908,12 @@ export class SyncClient {
|
|
|
3904
3908
|
let receivers = [];
|
|
3905
3909
|
// §5.4/§5.5: queue every segment download of this response, so a paged
|
|
3906
3910
|
// bootstrap overlaps its page fetches instead of paying one blocking round
|
|
3907
|
-
// trip per page.
|
|
3908
|
-
// transport's pool size: enough to overlap
|
|
3909
|
-
// holds
|
|
3910
|
-
//
|
|
3911
|
-
// images (§5.3 client memory)
|
|
3911
|
+
// trip per page. `limits.segmentPrefetch` downloads run at once, by
|
|
3912
|
+
// default four, the Rust native transport's pool size: enough to overlap
|
|
3913
|
+
// round trips, while the client holds that many bodies, the one being
|
|
3914
|
+
// applied and the rest ahead. An image starts only beside at most one
|
|
3915
|
+
// other body, so at most two are images (§5.3 client memory) whatever the
|
|
3916
|
+
// limit; later descriptors wait in order. The
|
|
3912
3917
|
// loop takes each body out of the map before applying it and tops the
|
|
3913
3918
|
// window up at the next descriptor. Eligibility mirrors the loop: a
|
|
3914
3919
|
// descriptor outside an applied section or from an unconfigured
|
|
@@ -3918,7 +3923,7 @@ export class SyncClient {
|
|
|
3918
3923
|
const queued = [];
|
|
3919
3924
|
const prefetched = new Map();
|
|
3920
3925
|
const prefetch = () => {
|
|
3921
|
-
for (let next; prefetched.size < 4 &&
|
|
3926
|
+
for (let next; prefetched.size < (this.#config.limits?.segmentPrefetch ?? 4) &&
|
|
3922
3927
|
!(queued[0]?.[0].mediaType === 'sqlite' && prefetched.size > 1) &&
|
|
3923
3928
|
(next = queued.shift());) {
|
|
3924
3929
|
const [frame, scopes] = next;
|
|
@@ -4031,11 +4036,16 @@ export class SyncClient {
|
|
|
4031
4036
|
skip: false,
|
|
4032
4037
|
}
|
|
4033
4038
|
: undefined;
|
|
4034
|
-
// §7.7: a section boundary ends the previous payload
|
|
4039
|
+
// §7.7: a section boundary ends the previous payload; the same
|
|
4040
|
+
// snapshot takes the section's extent (§4.4).
|
|
4041
|
+
const extent = message.frames[index + 1];
|
|
4042
|
+
const stated = section !== undefined && extent?.type === 'UNIT_EXTENT';
|
|
4035
4043
|
this.#progress.update({
|
|
4036
4044
|
...NO_PAYLOAD,
|
|
4037
4045
|
phase: 'import',
|
|
4038
4046
|
unitId: section?.unit.id,
|
|
4047
|
+
unitRowsTotal: stated ? extent.rowsTotal : undefined,
|
|
4048
|
+
unitRowsProcessed: stated ? extent.rowsDone : undefined,
|
|
4039
4049
|
});
|
|
4040
4050
|
break;
|
|
4041
4051
|
}
|
|
@@ -4059,20 +4069,23 @@ export class SyncClient {
|
|
|
4059
4069
|
// §5.6: rows an earlier attempt at this position applied. A
|
|
4060
4070
|
// partly applied image already ran its table's first-page clear.
|
|
4061
4071
|
appliedRows = section.unit.applied?.[frame.segmentId];
|
|
4062
|
-
if (appliedRows !== undefined)
|
|
4072
|
+
if (appliedRows !== undefined)
|
|
4063
4073
|
section.seeded.add(frame.table);
|
|
4064
|
-
|
|
4065
|
-
break;
|
|
4066
|
-
}
|
|
4074
|
+
const imported = appliedRows === frame.rowCount;
|
|
4067
4075
|
this.#progress.update({
|
|
4068
4076
|
...NO_PAYLOAD,
|
|
4069
|
-
phase: 'download',
|
|
4077
|
+
phase: imported ? 'import' : 'download',
|
|
4070
4078
|
unitId,
|
|
4071
4079
|
table: frame.table,
|
|
4072
4080
|
segmentId: frame.segmentId,
|
|
4073
|
-
bytesTotal: frame.byteLength,
|
|
4081
|
+
bytesTotal: imported ? undefined : frame.byteLength,
|
|
4074
4082
|
rowsTotal: frame.rowCount,
|
|
4075
4083
|
});
|
|
4084
|
+
if (imported) {
|
|
4085
|
+
// §7.7: an earlier attempt imported this payload.
|
|
4086
|
+
this.#progress.update({ rowsProcessed: frame.rowCount });
|
|
4087
|
+
break;
|
|
4088
|
+
}
|
|
4076
4089
|
prefetch();
|
|
4077
4090
|
const pending = prefetched.get(frame);
|
|
4078
4091
|
prefetched.delete(frame);
|
|
@@ -4135,6 +4148,10 @@ export class SyncClient {
|
|
|
4135
4148
|
!section.skip &&
|
|
4136
4149
|
this.#localResetEpoch === resetEpoch) {
|
|
4137
4150
|
this.#finishSection(section, frame, summary);
|
|
4151
|
+
// §7.7: a completed bootstrap ends at what it imported.
|
|
4152
|
+
const processed = this.#progress.snapshot()?.unitRowsProcessed;
|
|
4153
|
+
if (frame.bootstrapState === undefined && processed !== undefined)
|
|
4154
|
+
this.#progress.update({ unitRowsTotal: processed });
|
|
4138
4155
|
}
|
|
4139
4156
|
section = undefined;
|
|
4140
4157
|
// The counters stay: `complete` and `failed` report them.
|
|
@@ -4873,7 +4890,7 @@ export class SyncClient {
|
|
|
4873
4890
|
throw new ClientSyncError('sync.local_corrupt', `rollback image for ${operation.table}/${operation.rowId} is incomplete`);
|
|
4874
4891
|
}
|
|
4875
4892
|
const values = table.columns.map((column) => jsonToRowValue(image.values?.[column.name] ?? null));
|
|
4876
|
-
upsertLocalRow(this.#db, table, values, image.syncVersion
|
|
4893
|
+
upsertLocalRow(this.#db, table, values, image.syncVersion);
|
|
4877
4894
|
}
|
|
4878
4895
|
#legacyUndoOptimisticRows(commit, batch) {
|
|
4879
4896
|
for (const operation of commit.operations) {
|
|
@@ -5021,7 +5038,7 @@ export class SyncClient {
|
|
|
5021
5038
|
values[index] ?? null,
|
|
5022
5039
|
]))).length)
|
|
5023
5040
|
continue;
|
|
5024
|
-
upsertLocalRow(this.#db, table, values, version
|
|
5041
|
+
upsertLocalRow(this.#db, table, values, version);
|
|
5025
5042
|
}
|
|
5026
5043
|
}
|
|
5027
5044
|
/** Re-apply every pending outbox commit on top of server state (§7.1). */
|
package/dist/encryption.js
CHANGED
|
@@ -46,16 +46,10 @@ function encryptionKeyId(config, table, rowId, values) {
|
|
|
46
46
|
const selectorColumn = config.keyIdColumns?.[table.name];
|
|
47
47
|
if (selectorColumn === undefined)
|
|
48
48
|
return table.name;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
throw new EncryptError(`encryption key-id column ${JSON.stringify(selectorColumn)} is not present on table ${JSON.stringify(table.name)}`);
|
|
52
|
-
}
|
|
53
|
-
if (table.columns[index]?.encrypted === true) {
|
|
54
|
-
throw new EncryptError(`encryption key-id column ${JSON.stringify(selectorColumn)} on table ${JSON.stringify(table.name)} must not be encrypted`);
|
|
55
|
-
}
|
|
56
|
-
const selected = values[index];
|
|
49
|
+
// The column must exist, stay plaintext, and hold a non-empty string.
|
|
50
|
+
const selected = values[table.columns.findIndex((column) => column.name === selectorColumn && column.encrypted !== true)];
|
|
57
51
|
if (typeof selected !== 'string' || selected.length === 0) {
|
|
58
|
-
throw new EncryptError(`encryption key-id column ${JSON.stringify(selectorColumn)} on table ${JSON.stringify(table.name)} must
|
|
52
|
+
throw new EncryptError(`encryption key-id column ${JSON.stringify(selectorColumn)} on table ${JSON.stringify(table.name)} must be a plaintext column holding a non-empty string`);
|
|
59
53
|
}
|
|
60
54
|
return selected;
|
|
61
55
|
}
|
package/dist/failed-overlay.d.ts
CHANGED
|
@@ -5,12 +5,18 @@ import type { RowValue } from '@syncular/core';
|
|
|
5
5
|
import type { RetainedCommitRow } from './outcomes.js';
|
|
6
6
|
import type { OutboxCommit, OutboxBeforeImage } from './outbox.js';
|
|
7
7
|
export declare function ensureFailedOverlaySchema(db: ClientDatabase): void;
|
|
8
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Server writes, after they are stored: each `written` row replaces the base
|
|
10
|
+
* of its key's retained rows (`undefined`: deleted), and the table's retained
|
|
11
|
+
* intents re-read their unique conflicts with the written keys. A written row
|
|
12
|
+
* without a retained row costs no statement.
|
|
13
|
+
*/
|
|
14
|
+
export declare function retainedBaseWrite(db: ClientDatabase, table: CompiledClientTable, written: ReadonlyMap<string, readonly [readonly RowValue[], number] | undefined>): void;
|
|
9
15
|
export declare function retainFailedRows(db: ClientDatabase, commit: OutboxCommit, images: readonly OutboxBeforeImage[], commitSeq?: number): void;
|
|
10
16
|
/** Match declared keys through SQLite, preserving index affinity and NULL semantics. */
|
|
11
17
|
export declare function uniqueConflicts(db: ClientDatabase, table: CompiledClientTable, values: Readonly<Record<string, RowValue>>): NonNullable<RetainedCommitRow['uniqueConflicts']>;
|
|
12
18
|
/** Exact row keys are JSON [table,id]; table scopes include unique-key peers. */
|
|
13
19
|
export type OverlayScope = readonly string[];
|
|
14
|
-
export declare function restoreFailedBases(db: ClientDatabase, schema: CompiledClientSchema, scope?: OverlayScope):
|
|
20
|
+
export declare function restoreFailedBases(db: ClientDatabase, schema: CompiledClientSchema, scope?: OverlayScope): void;
|
|
15
21
|
export declare function failedOverlayCommits(db: ClientDatabase, acknowledged?: boolean, scope?: OverlayScope, schema?: CompiledClientSchema): OutboxCommit[];
|
|
16
22
|
export declare function dropFailedRows(db: ClientDatabase, schema: CompiledClientSchema, matches: (table: string, values: Readonly<Record<string, JsonRowValue>>) => boolean): OutboxCommit[];
|
package/dist/failed-overlay.js
CHANGED
|
@@ -6,7 +6,7 @@ import { ClientSyncError } from './errors.js';
|
|
|
6
6
|
import { mapRowValues, fromSqlValue, jsonToRowValue, pkSqlValue, quoteIdent, rowValueToJson, SYNC_VERSION_COLUMN, toSqlValue, } from './schema.js';
|
|
7
7
|
import { deleteLocalRow, upsertLocalRow } from './apply.js';
|
|
8
8
|
/** Protected failure state. The server base advances independently of intent. */
|
|
9
|
-
const SET_UNIQUE_CONFLICTS = `UPDATE ${RETAINED_ROWS} SET unique_conflicts=? WHERE
|
|
9
|
+
const SET_UNIQUE_CONFLICTS = `UPDATE ${RETAINED_ROWS} SET unique_conflicts=? WHERE rowid=?`;
|
|
10
10
|
export function ensureFailedOverlaySchema(db) {
|
|
11
11
|
db.exec(`CREATE TABLE IF NOT EXISTS ${RETAINED_ROWS}(commit_id TEXT NOT NULL,idx INTEGER NOT NULL,tbl TEXT NOT NULL,id TEXT NOT NULL,at INTEGER NOT NULL,op TEXT NOT NULL,intent TEXT,base TEXT,version INTEGER,unique_conflicts TEXT,commit_seq INTEGER,PRIMARY KEY(commit_id,idx))`);
|
|
12
12
|
const columns = db.query(`PRAGMA table_info(${RETAINED_ROWS})`);
|
|
@@ -15,24 +15,38 @@ export function ensureFailedOverlaySchema(db) {
|
|
|
15
15
|
db.exec(`ALTER TABLE ${RETAINED_ROWS} ADD COLUMN ${column}`);
|
|
16
16
|
db.exec(`CREATE INDEX IF NOT EXISTS ${RETAINED_ROWS}_row ON ${RETAINED_ROWS}(tbl,id)`);
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (!
|
|
18
|
+
/**
|
|
19
|
+
* Server writes, after they are stored: each `written` row replaces the base
|
|
20
|
+
* of its key's retained rows (`undefined`: deleted), and the table's retained
|
|
21
|
+
* intents re-read their unique conflicts with the written keys. A written row
|
|
22
|
+
* without a retained row costs no statement.
|
|
23
|
+
*/
|
|
24
|
+
export function retainedBaseWrite(db, table, written) {
|
|
25
|
+
if (!written.size)
|
|
26
26
|
return;
|
|
27
|
-
for (const retained of db.query(`SELECT
|
|
27
|
+
for (const retained of db.query(`SELECT rowid AS r,* FROM ${RETAINED_ROWS} WHERE tbl=?`, [table.name])) {
|
|
28
|
+
const base = written.get(retained.id);
|
|
29
|
+
if (written.has(retained.id))
|
|
30
|
+
db.exec(`UPDATE ${RETAINED_ROWS} SET base=?,version=? WHERE rowid=?`, [
|
|
31
|
+
base
|
|
32
|
+
? JSON.stringify(Object.fromEntries(table.columns.map((column, index) => [
|
|
33
|
+
column.name,
|
|
34
|
+
rowValueToJson(base[0][index] ?? null),
|
|
35
|
+
])))
|
|
36
|
+
: null,
|
|
37
|
+
base?.[1] ?? null,
|
|
38
|
+
retained.r,
|
|
39
|
+
]);
|
|
40
|
+
if (!hasUniqueIndex(table) || retained.commit_seq !== null)
|
|
41
|
+
continue;
|
|
28
42
|
const prior = retained.unique_conflicts === null
|
|
29
43
|
? []
|
|
30
44
|
: JSON.parse(retained.unique_conflicts);
|
|
31
|
-
const conflicts = prior.filter((conflict) => conflict.rowId
|
|
32
|
-
if (
|
|
45
|
+
const conflicts = prior.filter((conflict) => !written.has(conflict.rowId));
|
|
46
|
+
if (retained.intent !== null) {
|
|
33
47
|
const intent = JSON.parse(retained.intent);
|
|
34
48
|
conflicts.push(...uniqueConflicts(db, table, mapRowValues(intent, jsonToRowValue))
|
|
35
|
-
.filter((conflict) => conflict.rowId
|
|
49
|
+
.filter((conflict) => written.has(conflict.rowId))
|
|
36
50
|
.map((conflict) => ({
|
|
37
51
|
...conflict,
|
|
38
52
|
serverRow: mapRowValues(conflict.serverRow, rowValueToJson),
|
|
@@ -43,8 +57,7 @@ export function retainedBaseWrite(db, table, rowId, values, version) {
|
|
|
43
57
|
? JSON.stringify(conflicts.sort((a, b) => table.indexes.findIndex((index) => index.name === a.index) -
|
|
44
58
|
table.indexes.findIndex((index) => index.name === b.index)))
|
|
45
59
|
: null,
|
|
46
|
-
retained.
|
|
47
|
-
Number(retained.idx),
|
|
60
|
+
retained.r,
|
|
48
61
|
]);
|
|
49
62
|
}
|
|
50
63
|
}
|
|
@@ -95,7 +108,7 @@ export function uniqueConflicts(db, table, values) {
|
|
|
95
108
|
function retainedRows(db, scope, acknowledged) {
|
|
96
109
|
// A scope joins through the (tbl, id) index; a scan per applied commit
|
|
97
110
|
// would make a drained batch quadratic.
|
|
98
|
-
return db.query(`SELECT
|
|
111
|
+
return db.query(`SELECT rowid AS r,* FROM ${RETAINED_ROWS} WHERE (?1 IS NULL OR rowid IN(SELECT r.rowid FROM json_each(?1) k JOIN ${RETAINED_ROWS} r ON r.tbl=json_extract(k.value,'$[0]') AND r.id=json_extract(k.value,'$[1]') UNION SELECT r.rowid FROM json_each(?1) k JOIN ${RETAINED_ROWS} r ON json_array_length(k.value)=1 AND r.tbl=json_extract(k.value,'$[0]')))${acknowledged === undefined ? '' : ` AND commit_seq IS${acknowledged ? ' NOT' : ''} NULL`} ORDER BY commit_seq,at,commit_id,idx`, [scope ? JSON.stringify(scope) : null]);
|
|
99
112
|
}
|
|
100
113
|
export function restoreFailedBases(db, schema, scope) {
|
|
101
114
|
const seen = new Set();
|
|
@@ -114,10 +127,9 @@ export function restoreFailedBases(db, schema, scope) {
|
|
|
114
127
|
deleteLocalRow(db, table, row.id, false);
|
|
115
128
|
else {
|
|
116
129
|
const values = JSON.parse(String(row.base));
|
|
117
|
-
upsertLocalRow(db, table, table.columns.map((column) => jsonToRowValue(values[column.name] ?? null)), (row.version ?? -1)
|
|
130
|
+
upsertLocalRow(db, table, table.columns.map((column) => jsonToRowValue(values[column.name] ?? null)), (row.version ?? -1));
|
|
118
131
|
}
|
|
119
132
|
}
|
|
120
|
-
return seen.size > 0;
|
|
121
133
|
}
|
|
122
134
|
export function failedOverlayCommits(db, acknowledged = false, scope, schema) {
|
|
123
135
|
const commits = new Map();
|
|
@@ -149,8 +161,7 @@ export function failedOverlayCommits(db, acknowledged = false, scope, schema) {
|
|
|
149
161
|
...conflict,
|
|
150
162
|
serverRow: mapRowValues(conflict.serverRow, rowValueToJson),
|
|
151
163
|
}))),
|
|
152
|
-
row.
|
|
153
|
-
row.idx,
|
|
164
|
+
row.r,
|
|
154
165
|
]);
|
|
155
166
|
}
|
|
156
167
|
}
|
package/dist/native-bridge.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* `{$bigint: decimal}`, the convention of the Rust command router. The browser
|
|
12
12
|
* client and worker entries never import this module.
|
|
13
13
|
*/
|
|
14
|
-
import { type ClientSnapshotMethods, type CurrentBaseProof, type MutationInput, type PresencePeer, type PromiseMethods, type QueryReadSpec, type QuerySnapshot, type SecurityLifecycle, type SnapshotRead, type SnapshotReadRequest, type SyncClient, type SyncClientConfig, type UnitCatchup, type WindowState } from './client.js';
|
|
14
|
+
import { type ClientSnapshotMethods, type CurrentBaseProof, type MutationInput, type PresencePeer, type PromiseMethods, type QueryReadSpec, type QuerySnapshot, type SecurityLifecycle, type SnapshotRead, type SnapshotReadRequest, type SyncClient, type SyncClientConfig, type SyncIdleOptions, type UnitCatchup, type WindowState } from './client.js';
|
|
15
15
|
import type { SqlRow, SqlValue } from './database.js';
|
|
16
16
|
import type { PresenceErrorCode, UnitScopes } from '@syncular/core';
|
|
17
17
|
export { unitId, type UnitScopes } from '@syncular/core';
|
|
@@ -216,7 +216,8 @@ export declare abstract class NativeBridgeClient implements PromiseMethods<Clien
|
|
|
216
216
|
setWindow(base: WindowBase, units: readonly string[]): Promise<void>;
|
|
217
217
|
windowState(base: WindowBase): Promise<WindowState>;
|
|
218
218
|
sync(): Promise<unknown>;
|
|
219
|
-
|
|
219
|
+
/** The Rust core has no §8.4 empty-replica gate: `skipEmpty` is not sent. */
|
|
220
|
+
syncUntilIdle(options?: number | SyncIdleOptions, timeoutMs?: number): Promise<unknown>;
|
|
220
221
|
proveCurrentBase(outcome: Pick<CommitOutcome, 'clientCommitId' | 'sequence'>, maxRounds?: number): Promise<CurrentBaseProof>;
|
|
221
222
|
conflicts(): Promise<readonly ConflictRecord[]>;
|
|
222
223
|
rejections(): Promise<readonly RejectionRecord[]>;
|