@syncular/client 0.59.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -27
- package/dist/apply.d.ts +6 -5
- package/dist/apply.js +52 -25
- package/dist/authority.js +21 -16
- package/dist/blob.d.ts +4 -4
- package/dist/bun-database.js +3 -16
- package/dist/client.d.ts +57 -66
- package/dist/client.js +701 -796
- package/dist/devtools.d.ts +2 -2
- package/dist/devtools.js +3 -3
- package/dist/diagnostics.d.ts +17 -28
- package/dist/diagnostics.js +1 -2
- package/dist/errors.d.ts +1 -1
- package/dist/errors.js +1 -1
- package/dist/failed-overlay.js +4 -1
- package/dist/http.js +10 -4
- package/dist/invalidation.d.ts +3 -4
- package/dist/invalidation.js +9 -22
- package/dist/native-bridge.d.ts +7 -16
- package/dist/native-bridge.js +16 -41
- package/dist/node-database.js +16 -25
- package/dist/outbox.js +1 -0
- package/dist/progress.d.ts +2 -2
- package/dist/query-guard.d.ts +15 -6
- package/dist/query-guard.js +38 -6
- package/dist/reactive-store.js +2 -2
- package/dist/remote.js +9 -9
- package/dist/schema.d.ts +3 -2
- package/dist/schema.js +28 -12
- package/dist/state.d.ts +69 -36
- package/dist/state.js +178 -89
- package/dist/sync-scheduler.js +2 -2
- package/dist/transport.d.ts +2 -2
- package/dist/window.d.ts +30 -52
- package/dist/window.js +33 -77
- package/dist/worker-core.js +6 -6
- package/dist/worker-host.d.ts +9 -8
- package/dist/worker-host.js +11 -11
- package/dist/worker-protocol.d.ts +9 -8
- package/package.json +3 -3
- package/src/apply.ts +87 -33
- package/src/authority.ts +31 -17
- package/src/blob.ts +4 -3
- package/src/bun-database.ts +9 -14
- package/src/client.ts +886 -1037
- package/src/devtools.ts +5 -5
- package/src/diagnostics.ts +17 -37
- package/src/errors.ts +1 -1
- package/src/failed-overlay.ts +6 -1
- package/src/http.ts +13 -8
- package/src/invalidation.ts +9 -29
- package/src/native-bridge.ts +22 -58
- package/src/node-database.ts +26 -27
- package/src/outbox.ts +1 -0
- package/src/progress.ts +2 -2
- package/src/query-guard.ts +45 -8
- package/src/reactive-store.ts +2 -2
- package/src/remote.ts +12 -16
- package/src/schema.ts +48 -13
- package/src/state.ts +246 -125
- package/src/sync-scheduler.ts +2 -2
- package/src/transport.ts +2 -2
- package/src/window.ts +55 -134
- package/src/worker-core.ts +6 -6
- package/src/worker-host.ts +13 -16
- package/src/worker-protocol.ts +8 -8
package/src/apply.ts
CHANGED
|
@@ -83,9 +83,43 @@ export function deleteLocalRow(
|
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
86
|
+
* §4.5 version guard for one server row: whether a write at `version` lands.
|
|
87
|
+
* A skipped row keeps its overlay, so its key leaves the rebase scope unless a
|
|
88
|
+
* retained base was restored for it; `kept` and `skipped` collect the keys.
|
|
89
|
+
*/
|
|
90
|
+
function lands(
|
|
91
|
+
db: ClientDatabase,
|
|
92
|
+
table: CompiledClientTable,
|
|
93
|
+
rowId: string,
|
|
94
|
+
version: number,
|
|
95
|
+
kept: Set<string>,
|
|
96
|
+
skipped: Set<string>,
|
|
97
|
+
): boolean {
|
|
98
|
+
const key = JSON.stringify([table.name, rowId]);
|
|
99
|
+
const local = db.query(
|
|
100
|
+
`SELECT ${quoteIdent(SYNC_VERSION_COLUMN)} AS v,EXISTS(SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? AND id=?) AS retained FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`,
|
|
101
|
+
[table.name, rowId, pkSqlValue(table, rowId)],
|
|
102
|
+
)[0];
|
|
103
|
+
const land = local === undefined || Number(local.v) < version;
|
|
104
|
+
if (land || local.retained) kept.add(key);
|
|
105
|
+
else skipped.add(key);
|
|
106
|
+
return land;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The scope a guarded write leaves for the overlay replay. */
|
|
110
|
+
function rebaseScope(
|
|
111
|
+
scope: OverlayScope,
|
|
112
|
+
kept: Set<string>,
|
|
113
|
+
skipped: Set<string>,
|
|
114
|
+
): OverlayScope {
|
|
115
|
+
return scope.filter((key) => kept.has(key) || !skipped.has(key));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Apply one `COMMIT` frame in one local transaction (§1.4 rule 4) under the
|
|
120
|
+
* §4.5 version guard: a change lands only when its `rowVersion` exceeds the
|
|
121
|
+
* local row's `_sync_version`. Upserts land with `_sync_version = rowVersion`;
|
|
122
|
+
* deletes remove the row. Re-application changes nothing.
|
|
89
123
|
*
|
|
90
124
|
* §5.11: encrypted columns are decrypted here — the apply seam. Because
|
|
91
125
|
* WebCrypto is async and the local transaction is synchronous, every row is
|
|
@@ -102,13 +136,8 @@ export async function applyCommitFrame(
|
|
|
102
136
|
transaction: (fn: () => OverlayScope) => unknown = (fn) => db.transaction(fn),
|
|
103
137
|
): Promise<void> {
|
|
104
138
|
type Resolved =
|
|
105
|
-
| { op: 'delete'; table: CompiledClientTable
|
|
106
|
-
| {
|
|
107
|
-
op: 'upsert';
|
|
108
|
-
table: CompiledClientTable;
|
|
109
|
-
values: readonly RowValue[];
|
|
110
|
-
rowVersion: number;
|
|
111
|
-
};
|
|
139
|
+
| { op: 'delete'; table: CompiledClientTable }
|
|
140
|
+
| { op: 'upsert'; table: CompiledClientTable; values: readonly RowValue[] };
|
|
112
141
|
const resolved: Resolved[] = [];
|
|
113
142
|
for (const change of frame.changes) {
|
|
114
143
|
const tableName = frame.tables[change.tableIndex];
|
|
@@ -125,10 +154,10 @@ export async function applyCommitFrame(
|
|
|
125
154
|
);
|
|
126
155
|
}
|
|
127
156
|
if (change.op === 'delete') {
|
|
128
|
-
resolved.push({ op: 'delete', table
|
|
157
|
+
resolved.push({ op: 'delete', table });
|
|
129
158
|
continue;
|
|
130
159
|
}
|
|
131
|
-
if (change.row === undefined
|
|
160
|
+
if (change.row === undefined) {
|
|
132
161
|
throw invalidRequest('upsert change without row payload');
|
|
133
162
|
}
|
|
134
163
|
let values = decodeRow(table.columns, change.row);
|
|
@@ -136,12 +165,7 @@ export async function applyCommitFrame(
|
|
|
136
165
|
const { decryptRowValues } = await import('./encryption');
|
|
137
166
|
values = await decryptRowValues(encryption, table, values);
|
|
138
167
|
}
|
|
139
|
-
resolved.push({
|
|
140
|
-
op: 'upsert',
|
|
141
|
-
table,
|
|
142
|
-
values,
|
|
143
|
-
rowVersion: change.rowVersion,
|
|
144
|
-
});
|
|
168
|
+
resolved.push({ op: 'upsert', table, values });
|
|
145
169
|
}
|
|
146
170
|
transaction(() => {
|
|
147
171
|
// Exact keys name the rows this frame rebases; unique-key peers widen the
|
|
@@ -153,8 +177,11 @@ export async function applyCommitFrame(
|
|
|
153
177
|
JSON.stringify([change.table.name, frame.changes[index]!.rowId]),
|
|
154
178
|
]);
|
|
155
179
|
restoreFailedBases(db, schema, scope);
|
|
180
|
+
const kept = new Set<string>();
|
|
181
|
+
const skipped = new Set<string>();
|
|
156
182
|
for (const [index, change] of resolved.entries()) {
|
|
157
|
-
const rowId = frame.changes[index]
|
|
183
|
+
const { rowId, rowVersion } = frame.changes[index]!;
|
|
184
|
+
const land = lands(db, change.table, rowId, rowVersion, kept, skipped);
|
|
158
185
|
db.exec(
|
|
159
186
|
`UPDATE ${OUTBOX_IMAGES} SET delivery_seq=MAX(COALESCE(delivery_seq,0),?) WHERE tbl=? AND row_id=?`,
|
|
160
187
|
[frame.commitSeq, change.table.name, rowId],
|
|
@@ -163,13 +190,11 @@ export async function applyCommitFrame(
|
|
|
163
190
|
`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq<=?`,
|
|
164
191
|
[change.table.name, rowId, frame.commitSeq],
|
|
165
192
|
);
|
|
166
|
-
if (
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
upsertLocalRow(db, change.table, change.values, change.rowVersion);
|
|
170
|
-
}
|
|
193
|
+
if (!land) continue;
|
|
194
|
+
if (change.op === 'delete') deleteLocalRow(db, change.table, rowId);
|
|
195
|
+
else upsertLocalRow(db, change.table, change.values, rowVersion);
|
|
171
196
|
}
|
|
172
|
-
return scope;
|
|
197
|
+
return rebaseScope(scope, kept, skipped);
|
|
173
198
|
});
|
|
174
199
|
}
|
|
175
200
|
|
|
@@ -418,7 +443,7 @@ export async function applySqliteSegment(
|
|
|
418
443
|
options: {
|
|
419
444
|
readonly clearFirst: boolean;
|
|
420
445
|
readonly effective: ScopeMap;
|
|
421
|
-
/**
|
|
446
|
+
/** Pair scopes of the units that own rows of the table (§3.3). */
|
|
422
447
|
readonly owners: readonly ScopeMap[];
|
|
423
448
|
readonly transaction?: ApplyTransaction;
|
|
424
449
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
@@ -503,6 +528,7 @@ export async function applySqliteSegment(
|
|
|
503
528
|
.filter((name) => name !== primaryKey)
|
|
504
529
|
.map((name) => `${name}=excluded.${name}`)
|
|
505
530
|
.join(', ');
|
|
531
|
+
const version = quoteIdent(SYNC_VERSION_COLUMN);
|
|
506
532
|
let after: SqlValue | undefined;
|
|
507
533
|
let applied = 0;
|
|
508
534
|
for (;;) {
|
|
@@ -548,13 +574,31 @@ export async function applySqliteSegment(
|
|
|
548
574
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
549
575
|
if (after === undefined && options.clearFirst)
|
|
550
576
|
deleteScopedRows(db, table, options.effective, options.owners);
|
|
577
|
+
const kept = new Set<string>();
|
|
578
|
+
const skipped = new Set<string>();
|
|
579
|
+
const landing = rows.filter((row) =>
|
|
580
|
+
lands(
|
|
581
|
+
db,
|
|
582
|
+
table,
|
|
583
|
+
String(row[table.primaryKey]),
|
|
584
|
+
Number(row._syncular_version),
|
|
585
|
+
kept,
|
|
586
|
+
skipped,
|
|
587
|
+
),
|
|
588
|
+
);
|
|
589
|
+
const where = predicates.join(' AND ') || 'true';
|
|
551
590
|
db.exec(
|
|
552
|
-
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${
|
|
591
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
|
|
553
592
|
params,
|
|
554
593
|
);
|
|
555
|
-
applied += Number(
|
|
594
|
+
applied += Number(
|
|
595
|
+
db.query(
|
|
596
|
+
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
597
|
+
params,
|
|
598
|
+
)[0]?.n,
|
|
599
|
+
);
|
|
556
600
|
if (protectedRows)
|
|
557
|
-
for (const row of
|
|
601
|
+
for (const row of landing) {
|
|
558
602
|
retainedBaseWrite(
|
|
559
603
|
db,
|
|
560
604
|
table,
|
|
@@ -568,7 +612,7 @@ export async function applySqliteSegment(
|
|
|
568
612
|
Number(row._syncular_version),
|
|
569
613
|
);
|
|
570
614
|
}
|
|
571
|
-
return scope;
|
|
615
|
+
return rebaseScope(scope, kept, skipped);
|
|
572
616
|
});
|
|
573
617
|
options.onProgress?.(applied);
|
|
574
618
|
await yieldToHost();
|
|
@@ -597,7 +641,7 @@ export async function applyRowsSegment(
|
|
|
597
641
|
options: {
|
|
598
642
|
readonly clearFirst: boolean;
|
|
599
643
|
readonly effective: ScopeMap;
|
|
600
|
-
/**
|
|
644
|
+
/** Pair scopes of the units that own rows of the table (§3.3). */
|
|
601
645
|
readonly owners: readonly ScopeMap[];
|
|
602
646
|
readonly transaction?: ApplyTransaction;
|
|
603
647
|
readonly onProgress?: (rowsProcessed: number) => void;
|
|
@@ -646,12 +690,22 @@ export async function applyRowsSegment(
|
|
|
646
690
|
if (clearThisBlock) {
|
|
647
691
|
deleteScopedRows(db, table, options.effective, options.owners);
|
|
648
692
|
}
|
|
693
|
+
const kept = new Set<string>();
|
|
694
|
+
const skipped = new Set<string>();
|
|
649
695
|
for (const row of rows) {
|
|
650
|
-
|
|
696
|
+
const rowId = String(row.values[table.primaryKeyIndex]);
|
|
697
|
+
if (lands(db, table, rowId, row.serverVersion, kept, skipped))
|
|
698
|
+
upsertLocalRow(
|
|
699
|
+
db,
|
|
700
|
+
table,
|
|
701
|
+
row.values,
|
|
702
|
+
row.serverVersion,
|
|
703
|
+
protectedRows,
|
|
704
|
+
);
|
|
651
705
|
applied += 1;
|
|
652
706
|
if (applied % 1024 === 0) options.onProgress?.(applied);
|
|
653
707
|
}
|
|
654
|
-
return scope;
|
|
708
|
+
return rebaseScope(scope, kept, skipped);
|
|
655
709
|
});
|
|
656
710
|
options.onProgress?.(applied);
|
|
657
711
|
await yieldToHost();
|
package/src/authority.ts
CHANGED
|
@@ -2,7 +2,15 @@ import type { ClientDatabase, SqlRow, SqlValue } from './database';
|
|
|
2
2
|
import { ClientSyncError } from './errors';
|
|
3
3
|
import { listOutbox, listOutboxBeforeImages } from './outbox';
|
|
4
4
|
import { RETAINED_ROWS } from './failed-overlay';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
bootstrapping,
|
|
7
|
+
covers,
|
|
8
|
+
getLocalRevision,
|
|
9
|
+
getMeta,
|
|
10
|
+
loadUnits,
|
|
11
|
+
pairScopes,
|
|
12
|
+
replicaCursor,
|
|
13
|
+
} from './state';
|
|
6
14
|
import { fromSqlValue, quoteIdent, SYNC_VERSION_COLUMN } from './schema';
|
|
7
15
|
import type { CompiledClientSchema, JsonRowValue } from './schema';
|
|
8
16
|
|
|
@@ -127,7 +135,8 @@ function readAuthoritySnapshot(
|
|
|
127
135
|
'authority revision is missing or invalid',
|
|
128
136
|
);
|
|
129
137
|
const revision = getLocalRevision(db);
|
|
130
|
-
const
|
|
138
|
+
const units = loadUnits(db);
|
|
139
|
+
const cursor = replicaCursor(db);
|
|
131
140
|
const tables = declarations.map((read) => {
|
|
132
141
|
const table = schema.tables.get(read.table)!;
|
|
133
142
|
const columns = [
|
|
@@ -259,23 +268,23 @@ function readAuthoritySnapshot(
|
|
|
259
268
|
},
|
|
260
269
|
];
|
|
261
270
|
});
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
.
|
|
265
|
-
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
s.cursor >= 0 &&
|
|
271
|
-
s.bootstrapState === undefined &&
|
|
272
|
-
s.params === undefined;
|
|
271
|
+
// Every unit covering the table, as its (unit, table) pair (§4.4).
|
|
272
|
+
const persisted = units
|
|
273
|
+
.filter((unit) => covers(table, unit.scopes))
|
|
274
|
+
.map((unit) => {
|
|
275
|
+
const effective =
|
|
276
|
+
unit.echo === undefined
|
|
277
|
+
? undefined
|
|
278
|
+
: pairScopes(unit.scopes, read.table, unit.echo);
|
|
273
279
|
return {
|
|
274
280
|
requestedScopes: Object.fromEntries(
|
|
275
|
-
Object.keys(read.scopes).map((v) => [
|
|
281
|
+
Object.keys(read.scopes).map((v) => [
|
|
282
|
+
v,
|
|
283
|
+
unit.scopes[v] === undefined ? [] : [unit.scopes[v]],
|
|
284
|
+
]),
|
|
276
285
|
),
|
|
277
|
-
status:
|
|
278
|
-
cursor:
|
|
286
|
+
status: unit.status,
|
|
287
|
+
cursor: unit.cursor ?? cursor,
|
|
279
288
|
effectiveScopes:
|
|
280
289
|
effective === undefined
|
|
281
290
|
? null
|
|
@@ -285,7 +294,12 @@ function readAuthoritySnapshot(
|
|
|
285
294
|
effective[v] ?? [],
|
|
286
295
|
]),
|
|
287
296
|
),
|
|
288
|
-
complete
|
|
297
|
+
complete:
|
|
298
|
+
effective !== undefined &&
|
|
299
|
+
Object.keys(effective).length ===
|
|
300
|
+
Object.keys(read.scopes).length &&
|
|
301
|
+
unit.status === 'active' &&
|
|
302
|
+
!bootstrapping(unit),
|
|
289
303
|
};
|
|
290
304
|
});
|
|
291
305
|
const selectors = Object.entries(read.scopes);
|
package/src/blob.ts
CHANGED
|
@@ -45,7 +45,7 @@ export type BlobUploadGrant =
|
|
|
45
45
|
/**
|
|
46
46
|
* The transport seam for blob upload/download (§5.9.3/§5.9.5). The upload
|
|
47
47
|
* calls receive the round's `signal`, which aborts when the `syncUntilIdle`
|
|
48
|
-
* deadline elapses (§7.
|
|
48
|
+
* deadline elapses (§7.8).
|
|
49
49
|
*/
|
|
50
50
|
export interface BlobTransport {
|
|
51
51
|
/** `PUT <mount>/blobs/{blobId}` — host-authenticated direct upload (§5.9.3). */
|
|
@@ -84,11 +84,12 @@ export interface BlobTransport {
|
|
|
84
84
|
/**
|
|
85
85
|
* §5.9.3 direct-to-storage PUT of the granted `url`. MUST attach NO host
|
|
86
86
|
* authentication — the presigned URL is the entire grant (§5.4) — and MUST
|
|
87
|
-
* send `x-amz-checksum-sha256` (base64 of the
|
|
88
|
-
* grant signs beside `content-length`. Only called when `uploadGrant`
|
|
87
|
+
* send `x-amz-checksum-sha256` (base64 of the digest `blobId` names), which
|
|
88
|
+
* the grant signs beside `content-length`. Only called when `uploadGrant`
|
|
89
89
|
* returned a `url` arm.
|
|
90
90
|
*/
|
|
91
91
|
uploadToUrl?(
|
|
92
|
+
blobId: string,
|
|
92
93
|
url: string,
|
|
93
94
|
bytes: Uint8Array,
|
|
94
95
|
mediaType?: string,
|
package/src/bun-database.ts
CHANGED
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
type SqlRow,
|
|
16
16
|
type SqlValue,
|
|
17
17
|
} from './database';
|
|
18
|
-
import { prepareReadOnly,
|
|
18
|
+
import { prepareReadOnly, runReadOnly } from './query-guard';
|
|
19
19
|
|
|
20
20
|
declare module 'bun:sqlite' {
|
|
21
21
|
interface Database {
|
|
@@ -35,6 +35,7 @@ function coerceParams(params: readonly SqlValue[]): BunParam[] {
|
|
|
35
35
|
export class BunClientDatabase implements ClientDatabase {
|
|
36
36
|
readonly db: Database;
|
|
37
37
|
readonly #path: string;
|
|
38
|
+
readonly #proven = new Set<string>();
|
|
38
39
|
#tx = { depth: 0 };
|
|
39
40
|
|
|
40
41
|
constructor(path = ':memory:') {
|
|
@@ -78,19 +79,13 @@ export class BunClientDatabase implements ClientDatabase {
|
|
|
78
79
|
prepared.columnNames.length > 0,
|
|
79
80
|
] as const;
|
|
80
81
|
});
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
if ((error as { code?: unknown }).code === 'SQLITE_READONLY')
|
|
89
|
-
refuseRawSql();
|
|
90
|
-
throw error;
|
|
91
|
-
} finally {
|
|
92
|
-
this.db.run('PRAGMA query_only = OFF');
|
|
93
|
-
}
|
|
82
|
+
return runReadOnly(
|
|
83
|
+
this.#proven,
|
|
84
|
+
sql,
|
|
85
|
+
(pragma) => this.db.run(pragma),
|
|
86
|
+
(error) => (error as { code?: unknown }).code === 'SQLITE_READONLY',
|
|
87
|
+
() => statement.all(...coerceParams(params)) as SqlRow[],
|
|
88
|
+
);
|
|
94
89
|
}
|
|
95
90
|
|
|
96
91
|
/** RFC 0005: a sibling file beside the replica, e.g. `<path>.prev-context`. */
|