@syncular/client 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -34
- package/dist/apply.d.ts +24 -14
- package/dist/apply.js +119 -70
- package/dist/authority.js +21 -16
- package/dist/blob.d.ts +17 -9
- package/dist/blob.js +5 -6
- package/dist/bun-database.d.ts +1 -0
- package/dist/bun-database.js +15 -0
- package/dist/client.d.ts +83 -75
- package/dist/client.js +982 -910
- package/dist/database.d.ts +5 -0
- package/dist/devtools.d.ts +2 -2
- package/dist/devtools.js +3 -3
- package/dist/diagnostics.d.ts +17 -28
- package/dist/diagnostics.js +1 -2
- package/dist/encryption.d.ts +2 -2
- package/dist/encryption.js +8 -2
- package/dist/errors.d.ts +3 -1
- package/dist/errors.js +5 -1
- package/dist/failed-overlay.d.ts +0 -1
- package/dist/failed-overlay.js +6 -6
- package/dist/http.d.ts +4 -1
- package/dist/http.js +67 -39
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/invalidation.d.ts +14 -14
- package/dist/invalidation.js +24 -48
- package/dist/local-purge.js +9 -12
- package/dist/local-rebootstrap.js +2 -2
- package/dist/multi-tab.d.ts +89 -1
- package/dist/multi-tab.js +70 -31
- package/dist/native-bridge.d.ts +226 -0
- package/dist/native-bridge.js +764 -0
- package/dist/node-database.d.ts +1 -0
- package/dist/node-database.js +21 -1
- package/dist/outbox.d.ts +9 -12
- package/dist/outbox.js +20 -36
- package/dist/outcomes.js +3 -3
- package/dist/previous-version.js +96 -114
- package/dist/progress.d.ts +2 -2
- package/dist/query-guard.d.ts +34 -24
- package/dist/query-guard.js +61 -186
- package/dist/reactive-store.js +55 -13
- package/dist/realtime-supervisor.d.ts +9 -2
- package/dist/realtime-supervisor.js +31 -7
- package/dist/remote.js +19 -23
- package/dist/schema.d.ts +3 -2
- package/dist/schema.js +43 -22
- package/dist/state.d.ts +69 -36
- package/dist/state.js +178 -89
- package/dist/sync-scheduler.js +28 -2
- package/dist/transport.d.ts +19 -4
- package/dist/wasm-database.js +58 -21
- package/dist/window.d.ts +30 -52
- package/dist/window.js +33 -77
- package/dist/worker-core.d.ts +5 -0
- package/dist/worker-core.js +83 -75
- package/dist/worker-host.d.ts +19 -13
- package/dist/worker-host.js +75 -18
- package/dist/worker-local-authority.js +2 -2
- package/dist/worker-protocol.d.ts +15 -10
- package/package.json +11 -3
- package/src/apply.ts +176 -86
- package/src/authority.ts +31 -17
- package/src/blob.ts +28 -15
- package/src/bun-database.ts +24 -0
- package/src/client.ts +1282 -1217
- package/src/database.ts +5 -0
- package/src/devtools.ts +5 -5
- package/src/diagnostics.ts +17 -37
- package/src/encryption.ts +10 -5
- package/src/errors.ts +9 -1
- package/src/failed-overlay.ts +8 -7
- package/src/http.ts +95 -79
- package/src/index.ts +1 -1
- package/src/invalidation.ts +26 -61
- package/src/local-purge.ts +11 -13
- package/src/local-rebootstrap.ts +2 -3
- package/src/multi-tab.ts +88 -30
- package/src/native-bridge.ts +1153 -0
- package/src/node-database.ts +32 -1
- package/src/outbox.ts +20 -41
- package/src/outcomes.ts +3 -5
- package/src/previous-version.ts +103 -114
- package/src/progress.ts +2 -2
- package/src/query-guard.ts +77 -177
- package/src/reactive-store.ts +52 -12
- package/src/realtime-supervisor.ts +37 -12
- package/src/remote.ts +23 -31
- package/src/schema.ts +67 -25
- package/src/state.ts +246 -125
- package/src/sync-scheduler.ts +31 -1
- package/src/transport.ts +21 -2
- package/src/wasm-database.ts +76 -27
- package/src/window.ts +55 -134
- package/src/worker-core.ts +97 -77
- package/src/worker-host.ts +76 -32
- package/src/worker-local-authority.ts +2 -5
- package/src/worker-protocol.ts +14 -10
package/src/apply.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { hasUniqueIndex } from './schema';
|
|
2
|
-
import {
|
|
2
|
+
import { OUTBOX_IMAGES, countOutbox } from './outbox';
|
|
3
3
|
import { RETAINED_ROWS, type OverlayScope } from './failed-overlay';
|
|
4
4
|
import { retainedBaseWrite, restoreFailedBases } from './failed-overlay';
|
|
5
5
|
/**
|
|
@@ -83,9 +83,43 @@ export function deleteLocalRow(
|
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
86
|
+
* §4.5 version guard for one server row: whether a write at `version` lands.
|
|
87
|
+
* A skipped row keeps its overlay, so its key leaves the rebase scope unless a
|
|
88
|
+
* retained base was restored for it; `kept` and `skipped` collect the keys.
|
|
89
|
+
*/
|
|
90
|
+
function lands(
|
|
91
|
+
db: ClientDatabase,
|
|
92
|
+
table: CompiledClientTable,
|
|
93
|
+
rowId: string,
|
|
94
|
+
version: number,
|
|
95
|
+
kept: Set<string>,
|
|
96
|
+
skipped: Set<string>,
|
|
97
|
+
): boolean {
|
|
98
|
+
const key = JSON.stringify([table.name, rowId]);
|
|
99
|
+
const local = db.query(
|
|
100
|
+
`SELECT ${quoteIdent(SYNC_VERSION_COLUMN)} AS v,EXISTS(SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? AND id=?) AS retained FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`,
|
|
101
|
+
[table.name, rowId, pkSqlValue(table, rowId)],
|
|
102
|
+
)[0];
|
|
103
|
+
const land = local === undefined || Number(local.v) < version;
|
|
104
|
+
if (land || local.retained) kept.add(key);
|
|
105
|
+
else skipped.add(key);
|
|
106
|
+
return land;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The scope a guarded write leaves for the overlay replay. */
|
|
110
|
+
function rebaseScope(
|
|
111
|
+
scope: OverlayScope,
|
|
112
|
+
kept: Set<string>,
|
|
113
|
+
skipped: Set<string>,
|
|
114
|
+
): OverlayScope {
|
|
115
|
+
return scope.filter((key) => kept.has(key) || !skipped.has(key));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Apply one `COMMIT` frame in one local transaction (§1.4 rule 4) under the
|
|
120
|
+
* §4.5 version guard: a change lands only when its `rowVersion` exceeds the
|
|
121
|
+
* local row's `_sync_version`. Upserts land with `_sync_version = rowVersion`;
|
|
122
|
+
* deletes remove the row. Re-application changes nothing.
|
|
89
123
|
*
|
|
90
124
|
* §5.11: encrypted columns are decrypted here — the apply seam. Because
|
|
91
125
|
* WebCrypto is async and the local transaction is synchronous, every row is
|
|
@@ -102,13 +136,8 @@ export async function applyCommitFrame(
|
|
|
102
136
|
transaction: (fn: () => OverlayScope) => unknown = (fn) => db.transaction(fn),
|
|
103
137
|
): Promise<void> {
|
|
104
138
|
type Resolved =
|
|
105
|
-
| { op: 'delete'; table: CompiledClientTable
|
|
106
|
-
| {
|
|
107
|
-
op: 'upsert';
|
|
108
|
-
table: CompiledClientTable;
|
|
109
|
-
values: readonly RowValue[];
|
|
110
|
-
rowVersion: number;
|
|
111
|
-
};
|
|
139
|
+
| { op: 'delete'; table: CompiledClientTable }
|
|
140
|
+
| { op: 'upsert'; table: CompiledClientTable; values: readonly RowValue[] };
|
|
112
141
|
const resolved: Resolved[] = [];
|
|
113
142
|
for (const change of frame.changes) {
|
|
114
143
|
const tableName = frame.tables[change.tableIndex];
|
|
@@ -125,50 +154,47 @@ export async function applyCommitFrame(
|
|
|
125
154
|
);
|
|
126
155
|
}
|
|
127
156
|
if (change.op === 'delete') {
|
|
128
|
-
resolved.push({ op: 'delete', table
|
|
157
|
+
resolved.push({ op: 'delete', table });
|
|
129
158
|
continue;
|
|
130
159
|
}
|
|
131
|
-
if (change.row === undefined
|
|
160
|
+
if (change.row === undefined) {
|
|
132
161
|
throw invalidRequest('upsert change without row payload');
|
|
133
162
|
}
|
|
134
163
|
let values = decodeRow(table.columns, change.row);
|
|
135
|
-
if (
|
|
164
|
+
if (table.hasEncryptedColumns) {
|
|
136
165
|
const { decryptRowValues } = await import('./encryption');
|
|
137
166
|
values = await decryptRowValues(encryption, table, values);
|
|
138
167
|
}
|
|
139
|
-
resolved.push({
|
|
140
|
-
op: 'upsert',
|
|
141
|
-
table,
|
|
142
|
-
values,
|
|
143
|
-
rowVersion: change.rowVersion,
|
|
144
|
-
});
|
|
168
|
+
resolved.push({ op: 'upsert', table, values });
|
|
145
169
|
}
|
|
146
170
|
transaction(() => {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
171
|
+
// Exact keys name the rows this frame rebases; unique-key peers widen the
|
|
172
|
+
// replay to the table.
|
|
173
|
+
const scope = resolved.flatMap((change, index) => [
|
|
174
|
+
...(hasUniqueIndex(change.table)
|
|
175
|
+
? [JSON.stringify([change.table.name])]
|
|
176
|
+
: []),
|
|
177
|
+
JSON.stringify([change.table.name, frame.changes[index]!.rowId]),
|
|
178
|
+
]);
|
|
154
179
|
restoreFailedBases(db, schema, scope);
|
|
180
|
+
const kept = new Set<string>();
|
|
181
|
+
const skipped = new Set<string>();
|
|
155
182
|
for (const [index, change] of resolved.entries()) {
|
|
156
|
-
const rowId = frame.changes[index]
|
|
183
|
+
const { rowId, rowVersion } = frame.changes[index]!;
|
|
184
|
+
const land = lands(db, change.table, rowId, rowVersion, kept, skipped);
|
|
157
185
|
db.exec(
|
|
158
|
-
`UPDATE ${OUTBOX_IMAGES} SET delivery_seq=MAX(COALESCE(delivery_seq,0),?) WHERE
|
|
186
|
+
`UPDATE ${OUTBOX_IMAGES} SET delivery_seq=MAX(COALESCE(delivery_seq,0),?) WHERE tbl=? AND row_id=?`,
|
|
159
187
|
[frame.commitSeq, change.table.name, rowId],
|
|
160
188
|
);
|
|
161
189
|
db.exec(
|
|
162
190
|
`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq<=?`,
|
|
163
191
|
[change.table.name, rowId, frame.commitSeq],
|
|
164
192
|
);
|
|
165
|
-
if (
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
upsertLocalRow(db, change.table, change.values, change.rowVersion);
|
|
169
|
-
}
|
|
193
|
+
if (!land) continue;
|
|
194
|
+
if (change.op === 'delete') deleteLocalRow(db, change.table, rowId);
|
|
195
|
+
else upsertLocalRow(db, change.table, change.values, rowVersion);
|
|
170
196
|
}
|
|
171
|
-
return scope;
|
|
197
|
+
return rebaseScope(scope, kept, skipped);
|
|
172
198
|
});
|
|
173
199
|
}
|
|
174
200
|
|
|
@@ -212,15 +238,19 @@ export function validateSegmentColumns(
|
|
|
212
238
|
}
|
|
213
239
|
|
|
214
240
|
/**
|
|
215
|
-
* Scope-matched local delete (§3.3 purge / §5.6 first-page
|
|
216
|
-
* rows whose generated local scope columns match `effective`
|
|
217
|
-
* column value must be in the key's value list
|
|
218
|
-
*
|
|
219
|
-
*
|
|
241
|
+
* Scope-matched local delete (§3.3 purge / §4.8 eviction / §5.6 first-page
|
|
242
|
+
* rule): delete rows whose generated local scope columns match `effective`
|
|
243
|
+
* (every key's column value must be in the key's value list) and that no
|
|
244
|
+
* entry of `owners` matches. `owners` holds the effective scopes of every
|
|
245
|
+
* other active subscription on the table that holds rows; a row one of them
|
|
246
|
+
* still covers is never removed. Fails closed (`sync.scope_revoked`) when
|
|
247
|
+
* the table has no local mapping for a key: precision or nothing, never
|
|
248
|
+
* clear-the-table.
|
|
220
249
|
*/
|
|
221
250
|
export function localScopePredicate(
|
|
222
251
|
table: CompiledClientTable,
|
|
223
252
|
effective: ScopeMap,
|
|
253
|
+
owners: readonly ScopeMap[],
|
|
224
254
|
): { sql: string; params: string[] } | undefined {
|
|
225
255
|
const entries = Object.entries(effective);
|
|
226
256
|
if (!entries.length || entries.some(([, values]) => !values.length)) return;
|
|
@@ -237,7 +267,26 @@ export function localScopePredicate(
|
|
|
237
267
|
return `${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`;
|
|
238
268
|
})
|
|
239
269
|
.join(' AND ');
|
|
240
|
-
|
|
270
|
+
const kept = owners.flatMap((owner) => {
|
|
271
|
+
const scope = localScopePredicate(table, owner, []);
|
|
272
|
+
if (scope === undefined) return [];
|
|
273
|
+
params.push(...scope.params);
|
|
274
|
+
return [` AND NOT (${scope.sql})`];
|
|
275
|
+
});
|
|
276
|
+
return { sql: sql + kept.join(''), params };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** {@link localScopePredicate} over decoded row values (retained rows). */
|
|
280
|
+
export function ownsLocalRow(
|
|
281
|
+
table: CompiledClientTable,
|
|
282
|
+
effective: ScopeMap,
|
|
283
|
+
owners: readonly ScopeMap[],
|
|
284
|
+
values: Readonly<Record<string, JsonRowValue>>,
|
|
285
|
+
): boolean {
|
|
286
|
+
return (
|
|
287
|
+
matchesLocalScopes(table, effective, values) &&
|
|
288
|
+
!owners.some((owner) => matchesLocalScopes(table, owner, values))
|
|
289
|
+
);
|
|
241
290
|
}
|
|
242
291
|
|
|
243
292
|
function scopedRows(
|
|
@@ -257,8 +306,13 @@ function bootstrapScope(
|
|
|
257
306
|
db: ClientDatabase,
|
|
258
307
|
table: CompiledClientTable,
|
|
259
308
|
effective: ScopeMap,
|
|
309
|
+
owners: readonly ScopeMap[],
|
|
260
310
|
): string[] {
|
|
261
|
-
const rows = scopedRows(
|
|
311
|
+
const rows = scopedRows(
|
|
312
|
+
db,
|
|
313
|
+
table,
|
|
314
|
+
localScopePredicate(table, effective, owners),
|
|
315
|
+
);
|
|
262
316
|
rows.push(
|
|
263
317
|
...db
|
|
264
318
|
.query(`SELECT id,base,intent FROM ${RETAINED_ROWS} WHERE tbl=?`, [
|
|
@@ -268,7 +322,7 @@ function bootstrapScope(
|
|
|
268
322
|
[row.base, row.intent].some(
|
|
269
323
|
(json) =>
|
|
270
324
|
json !== null &&
|
|
271
|
-
|
|
325
|
+
ownsLocalRow(table, effective, owners, JSON.parse(json as string)),
|
|
272
326
|
),
|
|
273
327
|
),
|
|
274
328
|
);
|
|
@@ -279,8 +333,9 @@ export function deleteScopedRows(
|
|
|
279
333
|
db: ClientDatabase,
|
|
280
334
|
table: CompiledClientTable,
|
|
281
335
|
effective: ScopeMap,
|
|
336
|
+
owners: readonly ScopeMap[],
|
|
282
337
|
): void {
|
|
283
|
-
const scope = localScopePredicate(table, effective);
|
|
338
|
+
const scope = localScopePredicate(table, effective, owners);
|
|
284
339
|
if (!scope) return;
|
|
285
340
|
const { sql, params } = scope;
|
|
286
341
|
for (const row of scopedRows(db, table, scope)) {
|
|
@@ -291,8 +346,8 @@ export function deleteScopedRows(
|
|
|
291
346
|
|
|
292
347
|
/**
|
|
293
348
|
* §4.8 window eviction: delete rows matching a departing unit's effective
|
|
294
|
-
* scopes (same local-scope-column rule and fail-closed
|
|
295
|
-
* {@link deleteScopedRows}) EXCEPT rows whose primary key is in
|
|
349
|
+
* scopes (same local-scope-column rule, owner exclusion and fail-closed
|
|
350
|
+
* clause as {@link deleteScopedRows}) EXCEPT rows whose primary key is in
|
|
296
351
|
* `pinnedRowIds` (E1 — pinned by a still-pending outbox commit). Returns
|
|
297
352
|
* whether unpinned work remains and whether any scoped rows remain. Each
|
|
298
353
|
* call removes at most 1,024 rows; the caller commits before yielding. Also removes the
|
|
@@ -303,9 +358,10 @@ export function evictScopedRows(
|
|
|
303
358
|
db: ClientDatabase,
|
|
304
359
|
table: CompiledClientTable,
|
|
305
360
|
effective: ScopeMap,
|
|
361
|
+
owners: readonly ScopeMap[],
|
|
306
362
|
pinnedRowIds: ReadonlySet<string>,
|
|
307
363
|
): { remaining: boolean; deferred: boolean } {
|
|
308
|
-
const scope = localScopePredicate(table, effective);
|
|
364
|
+
const scope = localScopePredicate(table, effective, owners);
|
|
309
365
|
if (!scope) return { remaining: false, deferred: false };
|
|
310
366
|
const { sql, params } = scope;
|
|
311
367
|
const pk = quoteIdent(table.primaryKey);
|
|
@@ -325,9 +381,10 @@ export function evictScopedRows(
|
|
|
325
381
|
[row.intent, row.base].some(
|
|
326
382
|
(json) =>
|
|
327
383
|
json !== null &&
|
|
328
|
-
|
|
384
|
+
ownsLocalRow(
|
|
329
385
|
table,
|
|
330
386
|
effective,
|
|
387
|
+
owners,
|
|
331
388
|
JSON.parse(json as string) as Record<string, JsonRowValue>,
|
|
332
389
|
),
|
|
333
390
|
)
|
|
@@ -386,6 +443,8 @@ export async function applySqliteSegment(
|
|
|
386
443
|
options: {
|
|
387
444
|
readonly clearFirst: boolean;
|
|
388
445
|
readonly effective: ScopeMap;
|
|
446
|
+
/** Pair scopes of the units that own rows of the table (§3.3). */
|
|
447
|
+
readonly owners: readonly ScopeMap[];
|
|
389
448
|
readonly transaction?: ApplyTransaction;
|
|
390
449
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
391
450
|
readonly isCurrent?: () => boolean;
|
|
@@ -400,6 +459,9 @@ export async function applySqliteSegment(
|
|
|
400
459
|
if (descriptor.table !== table.name) {
|
|
401
460
|
imageInvalid(`descriptor table ${JSON.stringify(descriptor.table)}`);
|
|
402
461
|
}
|
|
462
|
+
// §5.11: an image bypasses the decrypt seam, so a server cannot serve one
|
|
463
|
+
// for a table with encrypted columns.
|
|
464
|
+
if (table.hasEncryptedColumns) imageInvalid('table has encrypted columns');
|
|
403
465
|
return withImage(bytes, IMAGE_ALIAS, async () => {
|
|
404
466
|
// 1. Metadata vs descriptor (§5.3 rule 2). A file that is not a
|
|
405
467
|
// SQLite database or lacks the metadata table fails right here.
|
|
@@ -466,6 +528,7 @@ export async function applySqliteSegment(
|
|
|
466
528
|
.filter((name) => name !== primaryKey)
|
|
467
529
|
.map((name) => `${name}=excluded.${name}`)
|
|
468
530
|
.join(', ');
|
|
531
|
+
const version = quoteIdent(SYNC_VERSION_COLUMN);
|
|
469
532
|
let after: SqlValue | undefined;
|
|
470
533
|
let applied = 0;
|
|
471
534
|
for (;;) {
|
|
@@ -499,26 +562,43 @@ export async function applySqliteSegment(
|
|
|
499
562
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
500
563
|
const scope = !replay
|
|
501
564
|
? []
|
|
502
|
-
:
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
];
|
|
565
|
+
: [
|
|
566
|
+
...(hasUniqueIndex(table) ? [JSON.stringify([table.name])] : []),
|
|
567
|
+
...(after === undefined && options.clearFirst
|
|
568
|
+
? bootstrapScope(db, table, options.effective, options.owners)
|
|
569
|
+
: []),
|
|
570
|
+
...rows.map((row) =>
|
|
571
|
+
JSON.stringify([table.name, String(row[table.primaryKey])]),
|
|
572
|
+
),
|
|
573
|
+
];
|
|
512
574
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
513
575
|
if (after === undefined && options.clearFirst)
|
|
514
|
-
deleteScopedRows(db, table, options.effective);
|
|
576
|
+
deleteScopedRows(db, table, options.effective, options.owners);
|
|
577
|
+
const kept = new Set<string>();
|
|
578
|
+
const skipped = new Set<string>();
|
|
579
|
+
const landing = rows.filter((row) =>
|
|
580
|
+
lands(
|
|
581
|
+
db,
|
|
582
|
+
table,
|
|
583
|
+
String(row[table.primaryKey]),
|
|
584
|
+
Number(row._syncular_version),
|
|
585
|
+
kept,
|
|
586
|
+
skipped,
|
|
587
|
+
),
|
|
588
|
+
);
|
|
589
|
+
const where = predicates.join(' AND ') || 'true';
|
|
515
590
|
db.exec(
|
|
516
|
-
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${
|
|
591
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
|
|
517
592
|
params,
|
|
518
593
|
);
|
|
519
|
-
applied += Number(
|
|
594
|
+
applied += Number(
|
|
595
|
+
db.query(
|
|
596
|
+
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
597
|
+
params,
|
|
598
|
+
)[0]?.n,
|
|
599
|
+
);
|
|
520
600
|
if (protectedRows)
|
|
521
|
-
for (const row of
|
|
601
|
+
for (const row of landing) {
|
|
522
602
|
retainedBaseWrite(
|
|
523
603
|
db,
|
|
524
604
|
table,
|
|
@@ -532,7 +612,7 @@ export async function applySqliteSegment(
|
|
|
532
612
|
Number(row._syncular_version),
|
|
533
613
|
);
|
|
534
614
|
}
|
|
535
|
-
return scope;
|
|
615
|
+
return rebaseScope(scope, kept, skipped);
|
|
536
616
|
});
|
|
537
617
|
options.onProgress?.(applied);
|
|
538
618
|
await yieldToHost();
|
|
@@ -561,6 +641,8 @@ export async function applyRowsSegment(
|
|
|
561
641
|
options: {
|
|
562
642
|
readonly clearFirst: boolean;
|
|
563
643
|
readonly effective: ScopeMap;
|
|
644
|
+
/** Pair scopes of the units that own rows of the table (§3.3). */
|
|
645
|
+
readonly owners: readonly ScopeMap[];
|
|
564
646
|
readonly transaction?: ApplyTransaction;
|
|
565
647
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
566
648
|
readonly isCurrent?: () => boolean;
|
|
@@ -579,13 +661,12 @@ export async function applyRowsSegment(
|
|
|
579
661
|
// aborts before any write in this block.
|
|
580
662
|
const rows: { values: readonly RowValue[]; serverVersion: number }[] = [];
|
|
581
663
|
for (const row of block) {
|
|
582
|
-
const values =
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
: row.values;
|
|
664
|
+
const values = table.hasEncryptedColumns
|
|
665
|
+
? await (async () => {
|
|
666
|
+
const { decryptRowValues } = await import('./encryption');
|
|
667
|
+
return decryptRowValues(encryption, table, row.values);
|
|
668
|
+
})()
|
|
669
|
+
: row.values;
|
|
589
670
|
rows.push({ values, serverVersion: row.serverVersion });
|
|
590
671
|
}
|
|
591
672
|
if (options.isCurrent?.() === false) break;
|
|
@@ -593,29 +674,38 @@ export async function applyRowsSegment(
|
|
|
593
674
|
first = false;
|
|
594
675
|
if (!clearThisBlock && rows.length === 0) continue;
|
|
595
676
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
596
|
-
const scope: OverlayScope =
|
|
597
|
-
? [JSON.stringify([table.name])]
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
];
|
|
677
|
+
const scope: OverlayScope = [
|
|
678
|
+
...(hasUniqueIndex(table) ? [JSON.stringify([table.name])] : []),
|
|
679
|
+
...(clearThisBlock
|
|
680
|
+
? bootstrapScope(db, table, options.effective, options.owners)
|
|
681
|
+
: []),
|
|
682
|
+
...rows.map((row) =>
|
|
683
|
+
JSON.stringify([
|
|
684
|
+
table.name,
|
|
685
|
+
String(row.values[table.primaryKeyIndex]),
|
|
686
|
+
]),
|
|
687
|
+
),
|
|
688
|
+
];
|
|
609
689
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
610
690
|
if (clearThisBlock) {
|
|
611
|
-
deleteScopedRows(db, table, options.effective);
|
|
691
|
+
deleteScopedRows(db, table, options.effective, options.owners);
|
|
612
692
|
}
|
|
693
|
+
const kept = new Set<string>();
|
|
694
|
+
const skipped = new Set<string>();
|
|
613
695
|
for (const row of rows) {
|
|
614
|
-
|
|
696
|
+
const rowId = String(row.values[table.primaryKeyIndex]);
|
|
697
|
+
if (lands(db, table, rowId, row.serverVersion, kept, skipped))
|
|
698
|
+
upsertLocalRow(
|
|
699
|
+
db,
|
|
700
|
+
table,
|
|
701
|
+
row.values,
|
|
702
|
+
row.serverVersion,
|
|
703
|
+
protectedRows,
|
|
704
|
+
);
|
|
615
705
|
applied += 1;
|
|
616
706
|
if (applied % 1024 === 0) options.onProgress?.(applied);
|
|
617
707
|
}
|
|
618
|
-
return scope;
|
|
708
|
+
return rebaseScope(scope, kept, skipped);
|
|
619
709
|
});
|
|
620
710
|
options.onProgress?.(applied);
|
|
621
711
|
await yieldToHost();
|
package/src/authority.ts
CHANGED
|
@@ -2,7 +2,15 @@ import type { ClientDatabase, SqlRow, SqlValue } from './database';
|
|
|
2
2
|
import { ClientSyncError } from './errors';
|
|
3
3
|
import { listOutbox, listOutboxBeforeImages } from './outbox';
|
|
4
4
|
import { RETAINED_ROWS } from './failed-overlay';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
bootstrapping,
|
|
7
|
+
covers,
|
|
8
|
+
getLocalRevision,
|
|
9
|
+
getMeta,
|
|
10
|
+
loadUnits,
|
|
11
|
+
pairScopes,
|
|
12
|
+
replicaCursor,
|
|
13
|
+
} from './state';
|
|
6
14
|
import { fromSqlValue, quoteIdent, SYNC_VERSION_COLUMN } from './schema';
|
|
7
15
|
import type { CompiledClientSchema, JsonRowValue } from './schema';
|
|
8
16
|
|
|
@@ -127,7 +135,8 @@ function readAuthoritySnapshot(
|
|
|
127
135
|
'authority revision is missing or invalid',
|
|
128
136
|
);
|
|
129
137
|
const revision = getLocalRevision(db);
|
|
130
|
-
const
|
|
138
|
+
const units = loadUnits(db);
|
|
139
|
+
const cursor = replicaCursor(db);
|
|
131
140
|
const tables = declarations.map((read) => {
|
|
132
141
|
const table = schema.tables.get(read.table)!;
|
|
133
142
|
const columns = [
|
|
@@ -259,23 +268,23 @@ function readAuthoritySnapshot(
|
|
|
259
268
|
},
|
|
260
269
|
];
|
|
261
270
|
});
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
.
|
|
265
|
-
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
s.cursor >= 0 &&
|
|
271
|
-
s.bootstrapState === undefined &&
|
|
272
|
-
s.params === undefined;
|
|
271
|
+
// Every unit covering the table, as its (unit, table) pair (§4.4).
|
|
272
|
+
const persisted = units
|
|
273
|
+
.filter((unit) => covers(table, unit.scopes))
|
|
274
|
+
.map((unit) => {
|
|
275
|
+
const effective =
|
|
276
|
+
unit.echo === undefined
|
|
277
|
+
? undefined
|
|
278
|
+
: pairScopes(unit.scopes, read.table, unit.echo);
|
|
273
279
|
return {
|
|
274
280
|
requestedScopes: Object.fromEntries(
|
|
275
|
-
Object.keys(read.scopes).map((v) => [
|
|
281
|
+
Object.keys(read.scopes).map((v) => [
|
|
282
|
+
v,
|
|
283
|
+
unit.scopes[v] === undefined ? [] : [unit.scopes[v]],
|
|
284
|
+
]),
|
|
276
285
|
),
|
|
277
|
-
status:
|
|
278
|
-
cursor:
|
|
286
|
+
status: unit.status,
|
|
287
|
+
cursor: unit.cursor ?? cursor,
|
|
279
288
|
effectiveScopes:
|
|
280
289
|
effective === undefined
|
|
281
290
|
? null
|
|
@@ -285,7 +294,12 @@ function readAuthoritySnapshot(
|
|
|
285
294
|
effective[v] ?? [],
|
|
286
295
|
]),
|
|
287
296
|
),
|
|
288
|
-
complete
|
|
297
|
+
complete:
|
|
298
|
+
effective !== undefined &&
|
|
299
|
+
Object.keys(effective).length ===
|
|
300
|
+
Object.keys(read.scopes).length &&
|
|
301
|
+
unit.status === 'active' &&
|
|
302
|
+
!bootstrapping(unit),
|
|
289
303
|
};
|
|
290
304
|
});
|
|
291
305
|
const selectors = Object.entries(read.scopes);
|
package/src/blob.ts
CHANGED
|
@@ -42,22 +42,33 @@ export type BlobUploadGrant =
|
|
|
42
42
|
| { readonly kind: 'present' }
|
|
43
43
|
| { readonly kind: 'none' };
|
|
44
44
|
|
|
45
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* The transport seam for blob upload/download (§5.9.3/§5.9.5). The upload
|
|
47
|
+
* calls receive the round's `signal`, which aborts when the `syncUntilIdle`
|
|
48
|
+
* deadline elapses (§7.8).
|
|
49
|
+
*/
|
|
46
50
|
export interface BlobTransport {
|
|
47
51
|
/** `PUT <mount>/blobs/{blobId}` — host-authenticated direct upload (§5.9.3). */
|
|
48
|
-
upload(
|
|
52
|
+
upload(
|
|
53
|
+
blobId: string,
|
|
54
|
+
bytes: Uint8Array,
|
|
55
|
+
mediaType?: string,
|
|
56
|
+
signal?: AbortSignal,
|
|
57
|
+
): Promise<void>;
|
|
49
58
|
/**
|
|
50
59
|
* `GET <mount>/blobs/{blobId}` — re-authorized (§5.9.5). Returns inline
|
|
51
60
|
* bytes, or a presigned `url` the client core fetches via `fetchUrl`.
|
|
61
|
+
* `byteLength` is the BlobRef's declared size when the caller passed one;
|
|
62
|
+
* reading inline bytes stops once the body exceeds it.
|
|
52
63
|
*/
|
|
53
|
-
download(blobId: string): Promise<BlobDownloadResponse>;
|
|
64
|
+
download(blobId: string, byteLength?: number): Promise<BlobDownloadResponse>;
|
|
54
65
|
/**
|
|
55
66
|
* §5.9.5 presigned-download fetch: a bare GET of the signed `url`. Present
|
|
56
67
|
* iff the transport can consume URLs. MUST attach NO host authentication —
|
|
57
68
|
* the URL is the entire grant (§5.4). Only called when `download` returned
|
|
58
|
-
* a `url` arm.
|
|
69
|
+
* a `url` arm. Reading stops once the body exceeds `byteLength`.
|
|
59
70
|
*/
|
|
60
|
-
fetchUrl?(url: string): Promise<Uint8Array>;
|
|
71
|
+
fetchUrl?(url: string, byteLength?: number): Promise<Uint8Array>;
|
|
61
72
|
/**
|
|
62
73
|
* §5.9.3 presigned-upload grant: `POST /blobs/{blobId}/upload-grant` with
|
|
63
74
|
* the declared size. Present iff the transport supports the grant flow;
|
|
@@ -68,16 +79,21 @@ export interface BlobTransport {
|
|
|
68
79
|
blobId: string,
|
|
69
80
|
byteLength: number,
|
|
70
81
|
mediaType?: string,
|
|
82
|
+
signal?: AbortSignal,
|
|
71
83
|
): Promise<BlobUploadGrant>;
|
|
72
84
|
/**
|
|
73
85
|
* §5.9.3 direct-to-storage PUT of the granted `url`. MUST attach NO host
|
|
74
|
-
* authentication — the presigned URL is the entire grant (§5.4)
|
|
75
|
-
*
|
|
86
|
+
* authentication — the presigned URL is the entire grant (§5.4) — and MUST
|
|
87
|
+
* send `x-amz-checksum-sha256` (base64 of the digest `blobId` names), which
|
|
88
|
+
* the grant signs beside `content-length`. Only called when `uploadGrant`
|
|
89
|
+
* returned a `url` arm.
|
|
76
90
|
*/
|
|
77
91
|
uploadToUrl?(
|
|
92
|
+
blobId: string,
|
|
78
93
|
url: string,
|
|
79
94
|
bytes: Uint8Array,
|
|
80
95
|
mediaType?: string,
|
|
96
|
+
signal?: AbortSignal,
|
|
81
97
|
): Promise<void>;
|
|
82
98
|
}
|
|
83
99
|
|
|
@@ -188,7 +204,7 @@ export function enforceBlobCacheCap(
|
|
|
188
204
|
if (total <= maxBytes) return [];
|
|
189
205
|
// Oldest eligible body first, with blob ID as a stable tie.
|
|
190
206
|
const candidates = db.query(
|
|
191
|
-
`SELECT blob_id,byte_length
|
|
207
|
+
`SELECT blob_id,byte_length ${unreferencedBlobsSql(schema)}ORDER BY created_at_ms ASC,blob_id ASC`,
|
|
192
208
|
);
|
|
193
209
|
const evicted: string[] = [];
|
|
194
210
|
db.transaction(() => {
|
|
@@ -266,7 +282,8 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
|
|
|
266
282
|
return false;
|
|
267
283
|
}
|
|
268
284
|
|
|
269
|
-
|
|
285
|
+
/** The cached bodies no upload, pending commit, or visible row references. */
|
|
286
|
+
function unreferencedBlobsSql(schema: CompiledClientSchema): string {
|
|
270
287
|
const selects: string[] = [];
|
|
271
288
|
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
272
289
|
for (const column of columns) {
|
|
@@ -276,18 +293,14 @@ function visibleBlobIdsSql(schema: CompiledClientSchema): string {
|
|
|
276
293
|
);
|
|
277
294
|
}
|
|
278
295
|
}
|
|
279
|
-
return selects.length === 0
|
|
280
|
-
? 'SELECT NULL AS blob_id WHERE 0'
|
|
281
|
-
: selects.join(' UNION ');
|
|
296
|
+
return `FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM ${BLOB_COMMIT_REFS})AND blob_id NOT IN(${selects.length === 0 ? 'SELECT NULL AS blob_id WHERE 0' : selects.join(' UNION ')})`;
|
|
282
297
|
}
|
|
283
298
|
|
|
284
299
|
export function deleteUnreferencedCachedBlobs(
|
|
285
300
|
db: ClientDatabase,
|
|
286
301
|
schema: CompiledClientSchema,
|
|
287
302
|
): void {
|
|
288
|
-
db.exec(
|
|
289
|
-
`DELETE FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM ${BLOB_COMMIT_REFS})AND blob_id NOT IN(${visibleBlobIdsSql(schema)})`,
|
|
290
|
-
);
|
|
303
|
+
db.exec(`DELETE ${unreferencedBlobsSql(schema)}`);
|
|
291
304
|
}
|
|
292
305
|
|
|
293
306
|
export type { BlobRef };
|
package/src/bun-database.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
type SqlRow,
|
|
16
16
|
type SqlValue,
|
|
17
17
|
} from './database';
|
|
18
|
+
import { prepareReadOnly, runReadOnly } from './query-guard';
|
|
18
19
|
|
|
19
20
|
declare module 'bun:sqlite' {
|
|
20
21
|
interface Database {
|
|
@@ -34,6 +35,7 @@ function coerceParams(params: readonly SqlValue[]): BunParam[] {
|
|
|
34
35
|
export class BunClientDatabase implements ClientDatabase {
|
|
35
36
|
readonly db: Database;
|
|
36
37
|
readonly #path: string;
|
|
38
|
+
readonly #proven = new Set<string>();
|
|
37
39
|
#tx = { depth: 0 };
|
|
38
40
|
|
|
39
41
|
constructor(path = ':memory:') {
|
|
@@ -64,6 +66,28 @@ export class BunClientDatabase implements ClientDatabase {
|
|
|
64
66
|
return this.db.query(sql).all(...coerceParams(params)) as SqlRow[];
|
|
65
67
|
}
|
|
66
68
|
|
|
69
|
+
queryReadOnly(sql: string, params: readonly SqlValue[] = []): SqlRow[] {
|
|
70
|
+
const statement = prepareReadOnly(sql, () => {
|
|
71
|
+
const prepared = this.db.query(sql);
|
|
72
|
+
// bun:sqlite exposes no `sqlite3_sql`, so the first statement ends at
|
|
73
|
+
// the first `;` outside literals, identifiers and comments.
|
|
74
|
+
return [
|
|
75
|
+
prepared,
|
|
76
|
+
/^(?:'[^']*'|"[^"]*"|`[^`]*`|\[[^\]]*\]|--[^\n]*|\/\*(?:[^*]|\*(?!\/))*(?:\*\/|$)|[^;])*/.exec(
|
|
77
|
+
sql,
|
|
78
|
+
)![0],
|
|
79
|
+
prepared.columnNames.length > 0,
|
|
80
|
+
] as const;
|
|
81
|
+
});
|
|
82
|
+
return runReadOnly(
|
|
83
|
+
this.#proven,
|
|
84
|
+
sql,
|
|
85
|
+
(pragma) => this.db.run(pragma),
|
|
86
|
+
(error) => (error as { code?: unknown }).code === 'SQLITE_READONLY',
|
|
87
|
+
() => statement.all(...coerceParams(params)) as SqlRow[],
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
67
91
|
/** RFC 0005: a sibling file beside the replica, e.g. `<path>.prev-context`. */
|
|
68
92
|
openSibling(name: string): SiblingDatabase {
|
|
69
93
|
const siblingPath =
|