@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/dist/pull.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pull:
|
|
3
|
-
* (SPEC.md §4, §5).
|
|
2
|
+
* Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
|
|
4
3
|
*/
|
|
5
|
-
import { encodeRowsSegment, } from '@syncular/core';
|
|
4
|
+
import { encodeRowsSegment, unitId, } from '@syncular/core';
|
|
6
5
|
import { clockOf, limitsOf } from './context.js';
|
|
6
|
+
import { syncError } from './errors.js';
|
|
7
7
|
import { compileSchema, } from './schema.js';
|
|
8
|
-
import { scopeDigest } from './scopes.js';
|
|
8
|
+
import { computeEffective, scopeDigest } from './scopes.js';
|
|
9
9
|
import { issueSegmentUrl } from './signed-url.js';
|
|
10
10
|
import { projectRowPayload, projectRowValues } from './schema-window.js';
|
|
11
11
|
// One artifact build per owning storage pair and complete immutable identity.
|
|
@@ -20,7 +20,7 @@ function clamp(value, min, max, dflt) {
|
|
|
20
20
|
return dflt;
|
|
21
21
|
return Math.min(max, Math.max(min, value));
|
|
22
22
|
}
|
|
23
|
-
/** §4.2 defaults and silent clamps
|
|
23
|
+
/** §4.2 defaults and silent clamps. */
|
|
24
24
|
export function clampPullLimits(header) {
|
|
25
25
|
return {
|
|
26
26
|
limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
|
|
@@ -29,16 +29,122 @@ export function clampPullLimits(header) {
|
|
|
29
29
|
accept: header.accept,
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
|
-
|
|
32
|
+
/**
|
|
33
|
+
* §4.1 unit validation against one schema version: the covered tables, or
|
|
34
|
+
* `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
|
|
35
|
+
* variable no table declares, or no covered table.
|
|
36
|
+
*/
|
|
37
|
+
export function coveredTables(schema, scopes) {
|
|
38
|
+
const keys = Object.keys(scopes);
|
|
39
|
+
const reason = keys.length === 0
|
|
40
|
+
? 'empty_scope_map'
|
|
41
|
+
: keys.some((key) => scopes[key] === '*')
|
|
42
|
+
? 'wildcard_value'
|
|
43
|
+
: keys.some((key) => !schema.declaredVariables.has(key))
|
|
44
|
+
? 'undeclared_variable'
|
|
45
|
+
: undefined;
|
|
46
|
+
const tables = reason === undefined
|
|
47
|
+
? [...schema.tables.values()].filter((table) => keys.every((key) => table.declaredVariables.has(key)))
|
|
48
|
+
: [];
|
|
49
|
+
if (tables.length === 0) {
|
|
50
|
+
throw syncError('sync.invalid_subscription', 'the server cannot serve this unit (§4.1)', JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }));
|
|
51
|
+
}
|
|
52
|
+
return tables;
|
|
53
|
+
}
|
|
54
|
+
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
55
|
+
export function planUnit(scopes, tables, resolved) {
|
|
56
|
+
const requested = {};
|
|
57
|
+
for (const [key, value] of Object.entries(scopes))
|
|
58
|
+
requested[key] = [value];
|
|
59
|
+
const pairs = new Map();
|
|
60
|
+
const fenced = [];
|
|
61
|
+
for (const table of tables) {
|
|
62
|
+
const outcome = computeEffective(table.declaredVariables, requested, resolved);
|
|
63
|
+
if (outcome.status === 'revoked') {
|
|
64
|
+
fenced.push({ table: table.name, status: 'revoked', scopes: {} });
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
pairs.set(table.name, outcome.effective);
|
|
68
|
+
const added = Object.entries(outcome.effective).filter(([key]) => !Object.hasOwn(scopes, key));
|
|
69
|
+
if (added.length > 0) {
|
|
70
|
+
fenced.push({
|
|
71
|
+
table: table.name,
|
|
72
|
+
status: 'active',
|
|
73
|
+
scopes: Object.fromEntries(added),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const active = pairs.size > 0;
|
|
78
|
+
return {
|
|
79
|
+
id: unitId(scopes),
|
|
80
|
+
scopes,
|
|
81
|
+
tables,
|
|
82
|
+
pairs,
|
|
83
|
+
status: active ? 'active' : 'revoked',
|
|
84
|
+
fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The `COMMIT` frame of one live-section commit (§4.5): its frame-local
|
|
89
|
+
* table dictionary and every change, projected onto the client's codec.
|
|
90
|
+
*/
|
|
91
|
+
export function commitFrame(commit, current, target) {
|
|
92
|
+
const tables = [];
|
|
93
|
+
const changes = commit.changes.map((change) => {
|
|
94
|
+
// §4.5: apply is version-guarded, so every change carries its version:
|
|
95
|
+
// an upsert the row's version after the write, a delete the version its
|
|
96
|
+
// tombstone records. A change without one cannot be served.
|
|
97
|
+
if (change.rowVersion === undefined) {
|
|
98
|
+
throw new Error('a change record carries no row version (§4.5)');
|
|
99
|
+
}
|
|
100
|
+
let tableIndex = tables.indexOf(change.table);
|
|
101
|
+
if (tableIndex === -1)
|
|
102
|
+
tableIndex = tables.push(change.table) - 1;
|
|
103
|
+
return {
|
|
104
|
+
tableIndex,
|
|
105
|
+
rowId: change.rowId,
|
|
106
|
+
op: change.op,
|
|
107
|
+
rowVersion: change.rowVersion,
|
|
108
|
+
scopes: change.scopes,
|
|
109
|
+
...(change.payload !== undefined
|
|
110
|
+
? {
|
|
111
|
+
row: projectRowPayload(current.tables.get(change.table), target.tables.get(change.table), change.payload),
|
|
112
|
+
}
|
|
113
|
+
: {}),
|
|
114
|
+
};
|
|
115
|
+
});
|
|
116
|
+
return {
|
|
117
|
+
type: 'COMMIT',
|
|
118
|
+
commitSeq: commit.commitSeq,
|
|
119
|
+
createdAtMs: commit.createdAtMs,
|
|
120
|
+
actorId: commit.actorId,
|
|
121
|
+
tables,
|
|
122
|
+
changes,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* §4.7: a usable resume token, or `undefined`. A token whose pin is not a
|
|
127
|
+
* commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
|
|
128
|
+
* differ from the unit's covered tables in bootstrap order, or whose position
|
|
129
|
+
* names none of them is unusable.
|
|
130
|
+
*/
|
|
131
|
+
function parseBootstrapToken(raw, tables, maxSeq) {
|
|
33
132
|
if (raw === undefined)
|
|
34
133
|
return undefined;
|
|
35
134
|
try {
|
|
36
135
|
const parsed = JSON.parse(raw);
|
|
37
|
-
if (
|
|
136
|
+
if (!Number.isSafeInteger(parsed.asOfCommitSeq) ||
|
|
137
|
+
parsed.asOfCommitSeq === undefined ||
|
|
138
|
+
parsed.asOfCommitSeq < 0 ||
|
|
139
|
+
parsed.asOfCommitSeq > maxSeq ||
|
|
38
140
|
!Array.isArray(parsed.tables) ||
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
parsed.
|
|
141
|
+
parsed.tables.length !== tables.length ||
|
|
142
|
+
parsed.tables.some((name, i) => name !== tables[i]?.name) ||
|
|
143
|
+
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
144
|
+
parsed.tableIndex === undefined ||
|
|
145
|
+
parsed.tableIndex < 0 ||
|
|
146
|
+
parsed.tableIndex >= tables.length ||
|
|
147
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
|
|
42
148
|
return undefined;
|
|
43
149
|
}
|
|
44
150
|
return {
|
|
@@ -52,28 +158,6 @@ function parseBootstrapToken(raw, table) {
|
|
|
52
158
|
return undefined;
|
|
53
159
|
}
|
|
54
160
|
}
|
|
55
|
-
function commitFrame(table, commit, current, target) {
|
|
56
|
-
const changes = commit.changes.map((change) => ({
|
|
57
|
-
tableIndex: 0,
|
|
58
|
-
rowId: change.rowId,
|
|
59
|
-
op: change.op,
|
|
60
|
-
...(change.rowVersion !== undefined
|
|
61
|
-
? { rowVersion: change.rowVersion }
|
|
62
|
-
: {}),
|
|
63
|
-
scopes: change.scopes,
|
|
64
|
-
...(change.payload !== undefined
|
|
65
|
-
? { row: projectRowPayload(current, target, change.payload) }
|
|
66
|
-
: {}),
|
|
67
|
-
}));
|
|
68
|
-
return {
|
|
69
|
-
type: 'COMMIT',
|
|
70
|
-
commitSeq: commit.commitSeq,
|
|
71
|
-
createdAtMs: commit.createdAtMs,
|
|
72
|
-
actorId: commit.actorId,
|
|
73
|
-
tables: [table],
|
|
74
|
-
changes,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
161
|
function chunkRows(rows, size) {
|
|
78
162
|
if (rows.length === 0)
|
|
79
163
|
return [];
|
|
@@ -103,10 +187,10 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
103
187
|
byteLength: record.byteLength,
|
|
104
188
|
rowCount: record.rowCount,
|
|
105
189
|
asOfCommitSeq: record.asOfCommitSeq,
|
|
106
|
-
// The frame reports the digest THIS
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
//
|
|
190
|
+
// The frame reports the digest THIS pair is authorized under. A stored
|
|
191
|
+
// entry can carry several digests when identical content was published
|
|
192
|
+
// under different scopes (§5.1), and the record's primary digest is not
|
|
193
|
+
// necessarily the caller's (§5.5).
|
|
110
194
|
scopeDigest,
|
|
111
195
|
...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
|
|
112
196
|
...(record.nextRowCursor !== null
|
|
@@ -126,20 +210,57 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
126
210
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
127
211
|
* lane (which decrypts per row on the client).
|
|
128
212
|
*/
|
|
129
|
-
function sqliteLaneEligible(limits,
|
|
213
|
+
function sqliteLaneEligible(limits, table, startRowCursor) {
|
|
130
214
|
return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
131
215
|
startRowCursor === null &&
|
|
132
|
-
|
|
216
|
+
table.encryptedColumnIndices.length === 0);
|
|
217
|
+
}
|
|
218
|
+
async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, maxSeq, horizonSeq, logEpoch) {
|
|
219
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
220
|
+
table: pair.table.name,
|
|
221
|
+
scopeFilter: pair.effective,
|
|
222
|
+
afterRowId: startRowCursor,
|
|
223
|
+
limit: limits.limitSnapshotRows + 1,
|
|
224
|
+
});
|
|
225
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
226
|
+
return { firstPage: await scan };
|
|
227
|
+
}
|
|
228
|
+
// §4.7: a sqlite-lane table pins at the newest change carrying the unit's
|
|
229
|
+
// index key (never below the horizon, never above maxCommitSeq), so a
|
|
230
|
+
// stored image stays the reuse key's match until this scope changes
|
|
231
|
+
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
232
|
+
const [firstPage, latest] = await Promise.all([
|
|
233
|
+
scan,
|
|
234
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
235
|
+
table: pair.table.name,
|
|
236
|
+
scopes: unit.scopes,
|
|
237
|
+
}),
|
|
238
|
+
]);
|
|
239
|
+
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
240
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than one
|
|
241
|
+
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
242
|
+
// on a small table would be a wasted round trip.
|
|
243
|
+
const stored = firstPage.length <= limits.limitSnapshotRows
|
|
244
|
+
? undefined
|
|
245
|
+
: await ctx.segments.find({
|
|
246
|
+
partition: ctx.partition,
|
|
247
|
+
logEpoch,
|
|
248
|
+
table: pair.table.name,
|
|
249
|
+
schemaVersion: schema.version,
|
|
250
|
+
mediaType: 'sqlite',
|
|
251
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
252
|
+
asOfCommitSeq: scopePin,
|
|
253
|
+
}, clockOf(ctx)());
|
|
254
|
+
return { firstPage, image: { scopePin, stored } };
|
|
133
255
|
}
|
|
134
256
|
/**
|
|
135
|
-
* §5.3 sqlite-image lane: only
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
* (the rows lane takes over).
|
|
257
|
+
* §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
|
|
258
|
+
* Reuses an unexpired stored image for the same (partition, table,
|
|
259
|
+
* schemaVersion, scope digest, scope pin) instead of rebuilding — the
|
|
260
|
+
* bootstrap-storm rule. Returns false when no image is served (the rows lane
|
|
261
|
+
* takes over).
|
|
141
262
|
*/
|
|
142
|
-
async function* sqliteImageSegment(ctx, schema, limits,
|
|
263
|
+
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
|
|
143
264
|
const { storage, segments, partition } = ctx;
|
|
144
265
|
const now = clockOf(ctx)();
|
|
145
266
|
if (existing !== undefined) {
|
|
@@ -153,9 +274,6 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
153
274
|
yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
|
|
154
275
|
return true;
|
|
155
276
|
}
|
|
156
|
-
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
157
|
-
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
158
|
-
// The probe is the rows lane's first page, read by `prepareSections`.
|
|
159
277
|
if (probe.length <= limits.limitSnapshotRows)
|
|
160
278
|
return false;
|
|
161
279
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
@@ -176,12 +294,13 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
176
294
|
const identity = {
|
|
177
295
|
partition,
|
|
178
296
|
logEpoch,
|
|
179
|
-
table:
|
|
297
|
+
table: pair.table.name,
|
|
180
298
|
schemaVersion: schema.version,
|
|
181
299
|
mediaType: 'sqlite',
|
|
182
300
|
scopeDigest: digest,
|
|
183
|
-
asOfCommitSeq:
|
|
301
|
+
asOfCommitSeq: scopePin,
|
|
184
302
|
};
|
|
303
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
185
304
|
const key = JSON.stringify(identity);
|
|
186
305
|
let building = builds.get(key);
|
|
187
306
|
let builtHere = false;
|
|
@@ -194,28 +313,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
194
313
|
builtHere = true;
|
|
195
314
|
let rowCount = 0;
|
|
196
315
|
const bytes = await buildImage({
|
|
197
|
-
table:
|
|
316
|
+
table: pair.table,
|
|
198
317
|
schemaVersion: schema.version,
|
|
199
|
-
asOfCommitSeq:
|
|
318
|
+
asOfCommitSeq: scopePin,
|
|
200
319
|
scopeDigest: digest,
|
|
201
320
|
rowBatches: (async function* () {
|
|
202
321
|
rowCount += probe.length;
|
|
203
322
|
yield probe.map((row) => ({
|
|
204
323
|
...row,
|
|
205
|
-
payload: projectRowPayload(
|
|
324
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
206
325
|
}));
|
|
207
326
|
let afterRowId = probe[probe.length - 1].rowId;
|
|
208
327
|
for (;;) {
|
|
209
328
|
const rows = await storage.scanRows(partition, {
|
|
210
|
-
table:
|
|
211
|
-
scopeFilter:
|
|
329
|
+
table: pair.table.name,
|
|
330
|
+
scopeFilter: pair.effective,
|
|
212
331
|
afterRowId,
|
|
213
332
|
limit: 5_000,
|
|
214
333
|
});
|
|
215
334
|
rowCount += rows.length;
|
|
216
335
|
yield rows.map((row) => ({
|
|
217
336
|
...row,
|
|
218
|
-
payload: projectRowPayload(
|
|
337
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
219
338
|
}));
|
|
220
339
|
const last = rows[rows.length - 1];
|
|
221
340
|
if (rows.length < 5_000 || last === undefined)
|
|
@@ -246,23 +365,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
246
365
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
|
|
247
366
|
return true;
|
|
248
367
|
}
|
|
249
|
-
|
|
368
|
+
/**
|
|
369
|
+
* One table's segments within the page budget. Returns the pages it used
|
|
370
|
+
* and the row cursor to resume at, `null` once the table is exhausted.
|
|
371
|
+
*/
|
|
372
|
+
async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
|
|
250
373
|
const { storage, segments, partition } = ctx;
|
|
251
374
|
const serverLimits = limitsOf(ctx);
|
|
252
|
-
const digest = await scopeDigest(
|
|
375
|
+
const digest = await scopeDigest(pair.effective);
|
|
253
376
|
const now = clockOf(ctx)();
|
|
254
|
-
if (
|
|
255
|
-
const imaged = yield* sqliteImageSegment(ctx, schema, limits,
|
|
377
|
+
if (start.image !== undefined) {
|
|
378
|
+
const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
|
|
256
379
|
if (imaged)
|
|
257
|
-
return {
|
|
380
|
+
return { pages: 1, rowCursor: null };
|
|
258
381
|
}
|
|
382
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
259
383
|
let rowCursor = startRowCursor;
|
|
260
|
-
for (let page = 0; page <
|
|
384
|
+
for (let page = 0; page < pageBudget; page++) {
|
|
261
385
|
const scanned = page === 0
|
|
262
|
-
? firstPage
|
|
386
|
+
? start.firstPage
|
|
263
387
|
: await storage.scanRows(partition, {
|
|
264
|
-
table:
|
|
265
|
-
scopeFilter:
|
|
388
|
+
table: pair.table.name,
|
|
389
|
+
scopeFilter: pair.effective,
|
|
266
390
|
afterRowId: rowCursor,
|
|
267
391
|
limit: limits.limitSnapshotRows + 1,
|
|
268
392
|
});
|
|
@@ -271,12 +395,12 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
271
395
|
// §5.2: every row record carries the row's current server_version.
|
|
272
396
|
const decoded = pageRows.map((row) => ({
|
|
273
397
|
serverVersion: row.serverVersion,
|
|
274
|
-
values: projectRowValues(
|
|
398
|
+
values: projectRowValues(current, pair.table, row.payload),
|
|
275
399
|
}));
|
|
276
400
|
const bytes = encodeRowsSegment({
|
|
277
|
-
table:
|
|
401
|
+
table: pair.table.name,
|
|
278
402
|
schemaVersion: schema.version,
|
|
279
|
-
columns:
|
|
403
|
+
columns: pair.table.columns,
|
|
280
404
|
blocks: chunkRows(decoded, 1000),
|
|
281
405
|
});
|
|
282
406
|
const lastRow = pageRows[pageRows.length - 1];
|
|
@@ -299,11 +423,11 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
299
423
|
const record = await segments.put({
|
|
300
424
|
partition,
|
|
301
425
|
logEpoch,
|
|
302
|
-
table:
|
|
426
|
+
table: pair.table.name,
|
|
303
427
|
schemaVersion: schema.version,
|
|
304
428
|
mediaType: 'rows',
|
|
305
429
|
scopeDigest: digest,
|
|
306
|
-
asOfCommitSeq:
|
|
430
|
+
asOfCommitSeq: pin,
|
|
307
431
|
rowCount: pageRows.length,
|
|
308
432
|
rowCursor,
|
|
309
433
|
nextRowCursor,
|
|
@@ -318,232 +442,84 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
318
442
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
|
|
319
443
|
}
|
|
320
444
|
if (!hasMore)
|
|
321
|
-
return {
|
|
445
|
+
return { pages: page + 1, rowCursor: null };
|
|
322
446
|
rowCursor = nextRowCursor;
|
|
323
447
|
}
|
|
324
|
-
return {
|
|
325
|
-
}
|
|
326
|
-
/** Run the pre-frame storage reads of every subscription concurrently. */
|
|
327
|
-
export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizonSeq, logEpoch) {
|
|
328
|
-
const reads = plans.map((plan) => {
|
|
329
|
-
const sub = plan.frame;
|
|
330
|
-
if (plan.status === 'revoked')
|
|
331
|
-
return {};
|
|
332
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
333
|
-
if (token === undefined &&
|
|
334
|
-
sub.cursor >= 0 &&
|
|
335
|
-
sub.cursor >= horizonSeq &&
|
|
336
|
-
sub.cursor <= maxSeq) {
|
|
337
|
-
return {
|
|
338
|
-
commits: ctx.storage.readCommitWindow(ctx.partition, {
|
|
339
|
-
table: sub.table,
|
|
340
|
-
scopeFilter: plan.effective,
|
|
341
|
-
afterSeq: sub.cursor,
|
|
342
|
-
throughSeq: maxSeq,
|
|
343
|
-
limitChanges: limits.limitCommits + 1,
|
|
344
|
-
}),
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
// §4.6 reset: no reads.
|
|
348
|
-
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
349
|
-
return {};
|
|
350
|
-
}
|
|
351
|
-
// Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
|
|
352
|
-
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
353
|
-
? token
|
|
354
|
-
: undefined;
|
|
355
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
356
|
-
const firstPage = ctx.storage.scanRows(ctx.partition, {
|
|
357
|
-
table: plan.table.name,
|
|
358
|
-
scopeFilter: plan.effective,
|
|
359
|
-
afterRowId: startCursor,
|
|
360
|
-
limit: limits.limitSnapshotRows + 1,
|
|
361
|
-
});
|
|
362
|
-
if (!sqliteLaneEligible(limits, plan, startCursor))
|
|
363
|
-
return { firstPage };
|
|
364
|
-
// §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
|
|
365
|
-
// scope (never below the horizon, never above maxCommitSeq), so a stored
|
|
366
|
-
// image stays the reuse key's match until this scope changes (§5.3).
|
|
367
|
-
const pinned = resume !== undefined
|
|
368
|
-
? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
|
|
369
|
-
: ctx.storage
|
|
370
|
-
.latestScopedChangeSeq(ctx.partition, {
|
|
371
|
-
table: plan.table.name,
|
|
372
|
-
scopeFilter: plan.effective,
|
|
373
|
-
})
|
|
374
|
-
.then((latest) => {
|
|
375
|
-
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
376
|
-
return { asOf: pin, pin };
|
|
377
|
-
});
|
|
378
|
-
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
379
|
-
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
380
|
-
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
381
|
-
// `find` on a small table would be a wasted round trip.
|
|
382
|
-
const probe = await firstPage;
|
|
383
|
-
if (probe.length <= limits.limitSnapshotRows) {
|
|
384
|
-
return { pin, image: undefined };
|
|
385
|
-
}
|
|
386
|
-
return {
|
|
387
|
-
pin,
|
|
388
|
-
image: await ctx.segments.find({
|
|
389
|
-
partition: ctx.partition,
|
|
390
|
-
logEpoch,
|
|
391
|
-
table: plan.table.name,
|
|
392
|
-
schemaVersion: schema.version,
|
|
393
|
-
mediaType: 'sqlite',
|
|
394
|
-
scopeDigest: await scopeDigest(plan.effective),
|
|
395
|
-
asOfCommitSeq: asOf,
|
|
396
|
-
}, clockOf(ctx)()),
|
|
397
|
-
};
|
|
398
|
-
});
|
|
399
|
-
return { firstPage, lane };
|
|
400
|
-
});
|
|
401
|
-
const settled = await Promise.all(reads.map(async (read) => ({
|
|
402
|
-
commits: read.commits === undefined ? undefined : await read.commits,
|
|
403
|
-
firstPage: await read.firstPage,
|
|
404
|
-
lane: await read.lane,
|
|
405
|
-
})));
|
|
406
|
-
// Validate continuity before committing to an active section. A prune
|
|
407
|
-
// during a paged read can otherwise make an incomplete window look empty.
|
|
408
|
-
const windowHorizon = settled.some((read) => read.commits !== undefined)
|
|
409
|
-
? Math.max(horizonSeq, await ctx.storage.getHorizonSeq(ctx.partition))
|
|
410
|
-
: horizonSeq;
|
|
411
|
-
return settled.map((read) => ({
|
|
412
|
-
commits: read.commits ?? [],
|
|
413
|
-
horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
|
|
414
|
-
firstPage: read.firstPage,
|
|
415
|
-
pin: read.lane?.pin,
|
|
416
|
-
image: read.lane?.image,
|
|
417
|
-
}));
|
|
448
|
+
return { pages: pageBudget, rowCursor };
|
|
418
449
|
}
|
|
419
450
|
/**
|
|
420
|
-
*
|
|
421
|
-
*
|
|
451
|
+
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
452
|
+
* in this section: each visited table costs at least one page, so at most
|
|
453
|
+
* `maxSnapshotPages` tables. The reads of every planned unit start before any
|
|
454
|
+
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
455
|
+
* D1) answers them in one statement per table.
|
|
422
456
|
*/
|
|
423
|
-
export
|
|
424
|
-
|
|
425
|
-
|
|
457
|
+
export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
|
|
458
|
+
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
459
|
+
// fell behind the horizon; then restart from a fresh pin.
|
|
460
|
+
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
461
|
+
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
462
|
+
? token
|
|
463
|
+
: undefined;
|
|
464
|
+
const tableIndex = resume?.tableIndex ?? 0;
|
|
465
|
+
const rowCursor = resume?.rowCursor ?? null;
|
|
466
|
+
const starts = new Map();
|
|
467
|
+
for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
|
|
468
|
+
const table = unit.tables[index];
|
|
469
|
+
const effective = unit.pairs.get(table.name);
|
|
470
|
+
if (effective === undefined)
|
|
471
|
+
continue;
|
|
472
|
+
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
|
|
473
|
+
// A section that stops early never awaits the later starts.
|
|
474
|
+
start.catch(() => undefined);
|
|
475
|
+
starts.set(index, start);
|
|
426
476
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
477
|
+
return {
|
|
478
|
+
unit,
|
|
479
|
+
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
480
|
+
tableIndex,
|
|
481
|
+
rowCursor,
|
|
482
|
+
starts,
|
|
483
|
+
};
|
|
431
484
|
}
|
|
432
|
-
/**
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
448
|
-
return { nextCursor: sub.cursor, active: false };
|
|
449
|
-
}
|
|
450
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
451
|
-
const { commits, horizonSeq } = prepared;
|
|
452
|
-
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
453
|
-
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
454
|
-
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
455
|
-
yield {
|
|
456
|
-
type: 'SUB_START',
|
|
457
|
-
id: sub.id,
|
|
458
|
-
status: 'reset',
|
|
459
|
-
reasonCode: 'sync.cursor_expired',
|
|
460
|
-
effectiveScopes: {},
|
|
461
|
-
bootstrap: false,
|
|
462
|
-
};
|
|
463
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
464
|
-
return { nextCursor: sub.cursor, active: false };
|
|
465
|
-
}
|
|
466
|
-
const bootstrapping = token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
|
|
467
|
-
if (bootstrapping) {
|
|
468
|
-
// §4.7: resume at the pinned point unless the pin fell behind the
|
|
469
|
-
// horizon (or the token is unusable) — then restart from a fresh pin.
|
|
470
|
-
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
471
|
-
? token
|
|
472
|
-
: undefined;
|
|
473
|
-
const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
|
|
474
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
475
|
-
if (prepared.firstPage === undefined) {
|
|
476
|
-
throw new Error('prepareSections did not read the bootstrap page');
|
|
477
|
-
}
|
|
478
|
-
yield {
|
|
479
|
-
type: 'SUB_START',
|
|
480
|
-
id: sub.id,
|
|
481
|
-
status: 'active',
|
|
482
|
-
reasonCode: '',
|
|
483
|
-
effectiveScopes: plan.effective,
|
|
484
|
-
bootstrap: true,
|
|
485
|
-
};
|
|
486
|
-
const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
|
|
487
|
-
if (outcome.complete) {
|
|
488
|
-
yield { type: 'SUB_END', nextCursor: asOf };
|
|
485
|
+
/**
|
|
486
|
+
* The segments of one bootstrap section (§4.7): tables in bootstrap order,
|
|
487
|
+
* each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
|
|
488
|
+
* the resume token, or `undefined` once every table is exhausted. A pair the
|
|
489
|
+
* wildcard fence revoked receives no segment (§3.3).
|
|
490
|
+
*/
|
|
491
|
+
export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
|
|
492
|
+
const { unit, pin } = plan;
|
|
493
|
+
let { tableIndex, rowCursor } = plan;
|
|
494
|
+
let pages = 0;
|
|
495
|
+
for (;;) {
|
|
496
|
+
while (tableIndex < unit.tables.length &&
|
|
497
|
+
!unit.pairs.has(unit.tables[tableIndex].name)) {
|
|
498
|
+
tableIndex += 1;
|
|
499
|
+
rowCursor = null;
|
|
489
500
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
type: 'SUB_END',
|
|
499
|
-
nextCursor: asOf,
|
|
500
|
-
bootstrapState: JSON.stringify(nextToken),
|
|
501
|
+
if (tableIndex === unit.tables.length)
|
|
502
|
+
return undefined;
|
|
503
|
+
if (pages === limits.maxSnapshotPages) {
|
|
504
|
+
const token = {
|
|
505
|
+
asOfCommitSeq: pin,
|
|
506
|
+
tables: unit.tables.map((table) => table.name),
|
|
507
|
+
tableIndex,
|
|
508
|
+
rowCursor,
|
|
501
509
|
};
|
|
510
|
+
return JSON.stringify(token);
|
|
502
511
|
}
|
|
503
|
-
|
|
512
|
+
const table = unit.tables[tableIndex];
|
|
513
|
+
const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
|
|
514
|
+
pages += outcome.pages;
|
|
515
|
+
rowCursor = outcome.rowCursor;
|
|
516
|
+
if (rowCursor === null)
|
|
517
|
+
tableIndex += 1;
|
|
504
518
|
}
|
|
505
|
-
// Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
|
|
506
|
-
// cut off at limitCommits total changes, never splitting a commit.
|
|
507
|
-
yield {
|
|
508
|
-
type: 'SUB_START',
|
|
509
|
-
id: sub.id,
|
|
510
|
-
status: 'active',
|
|
511
|
-
reasonCode: '',
|
|
512
|
-
effectiveScopes: plan.effective,
|
|
513
|
-
bootstrap: false,
|
|
514
|
-
};
|
|
515
|
-
let delivered = 0;
|
|
516
|
-
let deliveredCommits = 0;
|
|
517
|
-
let lastDeliveredSeq = sub.cursor;
|
|
518
|
-
for (const commit of commits) {
|
|
519
|
-
if (delivered > 0 &&
|
|
520
|
-
delivered + commit.changes.length > limits.limitCommits) {
|
|
521
|
-
break;
|
|
522
|
-
}
|
|
523
|
-
yield commitFrame(sub.table, commit, compileSchema(ctx.schema).tables.get(sub.table), plan.table);
|
|
524
|
-
delivered += commit.changes.length;
|
|
525
|
-
deliveredCommits += 1;
|
|
526
|
-
lastDeliveredSeq = commit.commitSeq;
|
|
527
|
-
if (delivered >= limits.limitCommits)
|
|
528
|
-
break;
|
|
529
|
-
}
|
|
530
|
-
const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
|
|
531
|
-
// The window is proven exhausted only when the storage lookahead came
|
|
532
|
-
// back under budget (it scanned to `throughSeq`) and every returned
|
|
533
|
-
// commit was delivered.
|
|
534
|
-
const exhausted = totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
|
|
535
|
-
// §4.5: the cursor advances even when no matching changes exist; when
|
|
536
|
-
// the change limit truncated the window it stops at the last fully
|
|
537
|
-
// delivered commit.
|
|
538
|
-
const nextCursor = exhausted
|
|
539
|
-
? Math.max(sub.cursor, maxSeq)
|
|
540
|
-
: Math.max(sub.cursor, lastDeliveredSeq);
|
|
541
|
-
yield { type: 'SUB_END', nextCursor };
|
|
542
|
-
return { nextCursor, active: true };
|
|
543
519
|
}
|
|
544
520
|
/**
|
|
545
|
-
* Publish the §5.3 sqlite image that a pull
|
|
546
|
-
*
|
|
521
|
+
* Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
|
|
522
|
+
* from a host with a SQLite engine (a Bun process with the production
|
|
547
523
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
548
524
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
549
525
|
* serving host without a builder finds. The image stays current until a
|
|
@@ -556,9 +532,16 @@ export async function publishSqliteImage(input) {
|
|
|
556
532
|
throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
|
|
557
533
|
}
|
|
558
534
|
const schema = compileSchema(config.schema);
|
|
559
|
-
const table = schema.
|
|
535
|
+
const table = coveredTables(schema, input.scopes).find((candidate) => candidate.name === input.table);
|
|
560
536
|
if (table === undefined) {
|
|
561
|
-
throw new Error('publishSqliteImage: the
|
|
537
|
+
throw new Error('publishSqliteImage: the unit does not cover the table');
|
|
538
|
+
}
|
|
539
|
+
const allowed = input.allowed ??
|
|
540
|
+
Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
|
|
541
|
+
const unit = planUnit(input.scopes, [table], { ok: true, allowed });
|
|
542
|
+
const effective = unit.pairs.get(table.name);
|
|
543
|
+
if (effective === undefined) {
|
|
544
|
+
throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
|
|
562
545
|
}
|
|
563
546
|
await config.storage.ensureSchema(schema, config.checkpoints);
|
|
564
547
|
const logEpoch = await config.storage.getPartitionLogEpoch(partition);
|
|
@@ -570,18 +553,6 @@ export async function publishSqliteImage(input) {
|
|
|
570
553
|
partition,
|
|
571
554
|
actorId: 'syncular.image-publisher',
|
|
572
555
|
};
|
|
573
|
-
const plan = {
|
|
574
|
-
frame: {
|
|
575
|
-
type: 'SUBSCRIPTION',
|
|
576
|
-
id: 'image',
|
|
577
|
-
table: input.table,
|
|
578
|
-
scopes: input.scopes,
|
|
579
|
-
cursor: -1,
|
|
580
|
-
},
|
|
581
|
-
table,
|
|
582
|
-
status: 'active',
|
|
583
|
-
effective: input.scopes,
|
|
584
|
-
};
|
|
585
556
|
// One-row pages make every table above one row image-eligible (§5.3).
|
|
586
557
|
const limits = {
|
|
587
558
|
limitCommits: 1,
|
|
@@ -592,11 +563,9 @@ export async function publishSqliteImage(input) {
|
|
|
592
563
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
593
564
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
594
565
|
const trace = { segments: [] };
|
|
595
|
-
const
|
|
596
|
-
if (prepared === undefined)
|
|
597
|
-
throw new Error('prepareSections lost a plan');
|
|
566
|
+
const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
|
|
598
567
|
let ref;
|
|
599
|
-
for await (const frame of
|
|
568
|
+
for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
|
|
600
569
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
601
570
|
ref = frame;
|
|
602
571
|
}
|