@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/dist/pull.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pull:
|
|
3
|
-
* (SPEC.md §4, §5).
|
|
2
|
+
* Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
|
|
4
3
|
*/
|
|
5
|
-
import { encodeRowsSegment, } from '@syncular/core';
|
|
4
|
+
import { encodeRowsSegment, unitId, } from '@syncular/core';
|
|
6
5
|
import { clockOf, limitsOf } from './context.js';
|
|
6
|
+
import { syncError } from './errors.js';
|
|
7
7
|
import { compileSchema, } from './schema.js';
|
|
8
|
-
import { scopeDigest } from './scopes.js';
|
|
8
|
+
import { computeEffective, scopeDigest } from './scopes.js';
|
|
9
9
|
import { issueSegmentUrl } from './signed-url.js';
|
|
10
10
|
import { projectRowPayload, projectRowValues } from './schema-window.js';
|
|
11
11
|
// One artifact build per owning storage pair and complete immutable identity.
|
|
@@ -20,7 +20,7 @@ function clamp(value, min, max, dflt) {
|
|
|
20
20
|
return dflt;
|
|
21
21
|
return Math.min(max, Math.max(min, value));
|
|
22
22
|
}
|
|
23
|
-
/** §4.2 defaults and silent clamps
|
|
23
|
+
/** §4.2 defaults and silent clamps. */
|
|
24
24
|
export function clampPullLimits(header) {
|
|
25
25
|
return {
|
|
26
26
|
limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
|
|
@@ -30,11 +30,105 @@ export function clampPullLimits(header) {
|
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
32
|
/**
|
|
33
|
-
* §4.
|
|
34
|
-
*
|
|
35
|
-
*
|
|
33
|
+
* §4.1 unit validation against one schema version: the covered tables, or
|
|
34
|
+
* `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
|
|
35
|
+
* variable no table declares, or no covered table.
|
|
36
36
|
*/
|
|
37
|
-
function
|
|
37
|
+
export function coveredTables(schema, scopes) {
|
|
38
|
+
const keys = Object.keys(scopes);
|
|
39
|
+
const reason = keys.length === 0
|
|
40
|
+
? 'empty_scope_map'
|
|
41
|
+
: keys.some((key) => scopes[key] === '*')
|
|
42
|
+
? 'wildcard_value'
|
|
43
|
+
: keys.some((key) => !schema.declaredVariables.has(key))
|
|
44
|
+
? 'undeclared_variable'
|
|
45
|
+
: undefined;
|
|
46
|
+
const tables = reason === undefined
|
|
47
|
+
? [...schema.tables.values()].filter((table) => keys.every((key) => table.declaredVariables.has(key)))
|
|
48
|
+
: [];
|
|
49
|
+
if (tables.length === 0) {
|
|
50
|
+
throw syncError('sync.invalid_subscription', 'the server cannot serve this unit (§4.1)', JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }));
|
|
51
|
+
}
|
|
52
|
+
return tables;
|
|
53
|
+
}
|
|
54
|
+
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
55
|
+
export function planUnit(scopes, tables, resolved) {
|
|
56
|
+
const requested = {};
|
|
57
|
+
for (const [key, value] of Object.entries(scopes))
|
|
58
|
+
requested[key] = [value];
|
|
59
|
+
const pairs = new Map();
|
|
60
|
+
const fenced = [];
|
|
61
|
+
for (const table of tables) {
|
|
62
|
+
const outcome = computeEffective(table.declaredVariables, requested, resolved);
|
|
63
|
+
if (outcome.status === 'revoked') {
|
|
64
|
+
fenced.push({ table: table.name, status: 'revoked', scopes: {} });
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
pairs.set(table.name, outcome.effective);
|
|
68
|
+
const added = Object.entries(outcome.effective).filter(([key]) => !Object.hasOwn(scopes, key));
|
|
69
|
+
if (added.length > 0) {
|
|
70
|
+
fenced.push({
|
|
71
|
+
table: table.name,
|
|
72
|
+
status: 'active',
|
|
73
|
+
scopes: Object.fromEntries(added),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const active = pairs.size > 0;
|
|
78
|
+
return {
|
|
79
|
+
id: unitId(scopes),
|
|
80
|
+
scopes,
|
|
81
|
+
tables,
|
|
82
|
+
pairs,
|
|
83
|
+
status: active ? 'active' : 'revoked',
|
|
84
|
+
fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The `COMMIT` frame of one live-section commit (§4.5): its frame-local
|
|
89
|
+
* table dictionary and every change, projected onto the client's codec.
|
|
90
|
+
*/
|
|
91
|
+
export function commitFrame(commit, current, target) {
|
|
92
|
+
const tables = [];
|
|
93
|
+
const changes = commit.changes.map((change) => {
|
|
94
|
+
// §4.5: apply is version-guarded, so every change carries its version:
|
|
95
|
+
// an upsert the row's version after the write, a delete the version its
|
|
96
|
+
// tombstone records. A change without one cannot be served.
|
|
97
|
+
if (change.rowVersion === undefined) {
|
|
98
|
+
throw new Error('a change record carries no row version (§4.5)');
|
|
99
|
+
}
|
|
100
|
+
let tableIndex = tables.indexOf(change.table);
|
|
101
|
+
if (tableIndex === -1)
|
|
102
|
+
tableIndex = tables.push(change.table) - 1;
|
|
103
|
+
return {
|
|
104
|
+
tableIndex,
|
|
105
|
+
rowId: change.rowId,
|
|
106
|
+
op: change.op,
|
|
107
|
+
rowVersion: change.rowVersion,
|
|
108
|
+
scopes: change.scopes,
|
|
109
|
+
...(change.payload !== undefined
|
|
110
|
+
? {
|
|
111
|
+
row: projectRowPayload(current.tables.get(change.table), target.tables.get(change.table), change.payload),
|
|
112
|
+
}
|
|
113
|
+
: {}),
|
|
114
|
+
};
|
|
115
|
+
});
|
|
116
|
+
return {
|
|
117
|
+
type: 'COMMIT',
|
|
118
|
+
commitSeq: commit.commitSeq,
|
|
119
|
+
createdAtMs: commit.createdAtMs,
|
|
120
|
+
actorId: commit.actorId,
|
|
121
|
+
tables,
|
|
122
|
+
changes,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* §4.7: a usable resume token, or `undefined`. A token whose pin is not a
|
|
127
|
+
* commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
|
|
128
|
+
* differ from the unit's covered tables in bootstrap order, or whose position
|
|
129
|
+
* names none of them is unusable.
|
|
130
|
+
*/
|
|
131
|
+
function parseBootstrapToken(raw, tables, maxSeq) {
|
|
38
132
|
if (raw === undefined)
|
|
39
133
|
return undefined;
|
|
40
134
|
try {
|
|
@@ -44,11 +138,12 @@ function parseBootstrapToken(raw, table, maxSeq) {
|
|
|
44
138
|
parsed.asOfCommitSeq < 0 ||
|
|
45
139
|
parsed.asOfCommitSeq > maxSeq ||
|
|
46
140
|
!Array.isArray(parsed.tables) ||
|
|
47
|
-
parsed.tables
|
|
141
|
+
parsed.tables.length !== tables.length ||
|
|
142
|
+
parsed.tables.some((name, i) => name !== tables[i]?.name) ||
|
|
48
143
|
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
49
144
|
parsed.tableIndex === undefined ||
|
|
50
145
|
parsed.tableIndex < 0 ||
|
|
51
|
-
parsed.tableIndex >=
|
|
146
|
+
parsed.tableIndex >= tables.length ||
|
|
52
147
|
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
|
|
53
148
|
return undefined;
|
|
54
149
|
}
|
|
@@ -63,28 +158,6 @@ function parseBootstrapToken(raw, table, maxSeq) {
|
|
|
63
158
|
return undefined;
|
|
64
159
|
}
|
|
65
160
|
}
|
|
66
|
-
function commitFrame(table, commit, current, target) {
|
|
67
|
-
const changes = commit.changes.map((change) => ({
|
|
68
|
-
tableIndex: 0,
|
|
69
|
-
rowId: change.rowId,
|
|
70
|
-
op: change.op,
|
|
71
|
-
...(change.rowVersion !== undefined
|
|
72
|
-
? { rowVersion: change.rowVersion }
|
|
73
|
-
: {}),
|
|
74
|
-
scopes: change.scopes,
|
|
75
|
-
...(change.payload !== undefined
|
|
76
|
-
? { row: projectRowPayload(current, target, change.payload) }
|
|
77
|
-
: {}),
|
|
78
|
-
}));
|
|
79
|
-
return {
|
|
80
|
-
type: 'COMMIT',
|
|
81
|
-
commitSeq: commit.commitSeq,
|
|
82
|
-
createdAtMs: commit.createdAtMs,
|
|
83
|
-
actorId: commit.actorId,
|
|
84
|
-
tables: [table],
|
|
85
|
-
changes,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
161
|
function chunkRows(rows, size) {
|
|
89
162
|
if (rows.length === 0)
|
|
90
163
|
return [];
|
|
@@ -114,10 +187,10 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
114
187
|
byteLength: record.byteLength,
|
|
115
188
|
rowCount: record.rowCount,
|
|
116
189
|
asOfCommitSeq: record.asOfCommitSeq,
|
|
117
|
-
// The frame reports the digest THIS
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
190
|
+
// The frame reports the digest THIS pair is authorized under. A stored
|
|
191
|
+
// entry can carry several digests when identical content was published
|
|
192
|
+
// under different scopes (§5.1), and the record's primary digest is not
|
|
193
|
+
// necessarily the caller's (§5.5).
|
|
121
194
|
scopeDigest,
|
|
122
195
|
...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
|
|
123
196
|
...(record.nextRowCursor !== null
|
|
@@ -137,20 +210,57 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
137
210
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
138
211
|
* lane (which decrypts per row on the client).
|
|
139
212
|
*/
|
|
140
|
-
function sqliteLaneEligible(limits,
|
|
213
|
+
function sqliteLaneEligible(limits, table, startRowCursor) {
|
|
141
214
|
return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
142
215
|
startRowCursor === null &&
|
|
143
|
-
|
|
216
|
+
table.encryptedColumnIndices.length === 0);
|
|
217
|
+
}
|
|
218
|
+
async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, maxSeq, horizonSeq, logEpoch) {
|
|
219
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
220
|
+
table: pair.table.name,
|
|
221
|
+
scopeFilter: pair.effective,
|
|
222
|
+
afterRowId: startRowCursor,
|
|
223
|
+
limit: limits.limitSnapshotRows + 1,
|
|
224
|
+
});
|
|
225
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
226
|
+
return { firstPage: await scan };
|
|
227
|
+
}
|
|
228
|
+
// §4.7: a sqlite-lane table pins at the newest change carrying the unit's
|
|
229
|
+
// index key (never below the horizon, never above maxCommitSeq), so a
|
|
230
|
+
// stored image stays the reuse key's match until this scope changes
|
|
231
|
+
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
232
|
+
const [firstPage, latest] = await Promise.all([
|
|
233
|
+
scan,
|
|
234
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
235
|
+
table: pair.table.name,
|
|
236
|
+
scopes: unit.scopes,
|
|
237
|
+
}),
|
|
238
|
+
]);
|
|
239
|
+
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
240
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than one
|
|
241
|
+
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
242
|
+
// on a small table would be a wasted round trip.
|
|
243
|
+
const stored = firstPage.length <= limits.limitSnapshotRows
|
|
244
|
+
? undefined
|
|
245
|
+
: await ctx.segments.find({
|
|
246
|
+
partition: ctx.partition,
|
|
247
|
+
logEpoch,
|
|
248
|
+
table: pair.table.name,
|
|
249
|
+
schemaVersion: schema.version,
|
|
250
|
+
mediaType: 'sqlite',
|
|
251
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
252
|
+
asOfCommitSeq: scopePin,
|
|
253
|
+
}, clockOf(ctx)());
|
|
254
|
+
return { firstPage, image: { scopePin, stored } };
|
|
144
255
|
}
|
|
145
256
|
/**
|
|
146
|
-
* §5.3 sqlite-image lane: only
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
* (the rows lane takes over).
|
|
257
|
+
* §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
|
|
258
|
+
* Reuses an unexpired stored image for the same (partition, table,
|
|
259
|
+
* schemaVersion, scope digest, scope pin) instead of rebuilding — the
|
|
260
|
+
* bootstrap-storm rule. Returns false when no image is served (the rows lane
|
|
261
|
+
* takes over).
|
|
152
262
|
*/
|
|
153
|
-
async function* sqliteImageSegment(ctx, schema, limits,
|
|
263
|
+
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
|
|
154
264
|
const { storage, segments, partition } = ctx;
|
|
155
265
|
const now = clockOf(ctx)();
|
|
156
266
|
if (existing !== undefined) {
|
|
@@ -164,9 +274,6 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
164
274
|
yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
|
|
165
275
|
return true;
|
|
166
276
|
}
|
|
167
|
-
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
168
|
-
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
169
|
-
// The probe is the rows lane's first page, read by `prepareSections`.
|
|
170
277
|
if (probe.length <= limits.limitSnapshotRows)
|
|
171
278
|
return false;
|
|
172
279
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
@@ -187,12 +294,13 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
187
294
|
const identity = {
|
|
188
295
|
partition,
|
|
189
296
|
logEpoch,
|
|
190
|
-
table:
|
|
297
|
+
table: pair.table.name,
|
|
191
298
|
schemaVersion: schema.version,
|
|
192
299
|
mediaType: 'sqlite',
|
|
193
300
|
scopeDigest: digest,
|
|
194
|
-
asOfCommitSeq:
|
|
301
|
+
asOfCommitSeq: scopePin,
|
|
195
302
|
};
|
|
303
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
196
304
|
const key = JSON.stringify(identity);
|
|
197
305
|
let building = builds.get(key);
|
|
198
306
|
let builtHere = false;
|
|
@@ -205,28 +313,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
205
313
|
builtHere = true;
|
|
206
314
|
let rowCount = 0;
|
|
207
315
|
const bytes = await buildImage({
|
|
208
|
-
table:
|
|
316
|
+
table: pair.table,
|
|
209
317
|
schemaVersion: schema.version,
|
|
210
|
-
asOfCommitSeq:
|
|
318
|
+
asOfCommitSeq: scopePin,
|
|
211
319
|
scopeDigest: digest,
|
|
212
320
|
rowBatches: (async function* () {
|
|
213
321
|
rowCount += probe.length;
|
|
214
322
|
yield probe.map((row) => ({
|
|
215
323
|
...row,
|
|
216
|
-
payload: projectRowPayload(
|
|
324
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
217
325
|
}));
|
|
218
326
|
let afterRowId = probe[probe.length - 1].rowId;
|
|
219
327
|
for (;;) {
|
|
220
328
|
const rows = await storage.scanRows(partition, {
|
|
221
|
-
table:
|
|
222
|
-
scopeFilter:
|
|
329
|
+
table: pair.table.name,
|
|
330
|
+
scopeFilter: pair.effective,
|
|
223
331
|
afterRowId,
|
|
224
332
|
limit: 5_000,
|
|
225
333
|
});
|
|
226
334
|
rowCount += rows.length;
|
|
227
335
|
yield rows.map((row) => ({
|
|
228
336
|
...row,
|
|
229
|
-
payload: projectRowPayload(
|
|
337
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
230
338
|
}));
|
|
231
339
|
const last = rows[rows.length - 1];
|
|
232
340
|
if (rows.length < 5_000 || last === undefined)
|
|
@@ -257,23 +365,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
|
|
|
257
365
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
|
|
258
366
|
return true;
|
|
259
367
|
}
|
|
260
|
-
|
|
368
|
+
/**
|
|
369
|
+
* One table's segments within the page budget. Returns the pages it used
|
|
370
|
+
* and the row cursor to resume at, `null` once the table is exhausted.
|
|
371
|
+
*/
|
|
372
|
+
async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
|
|
261
373
|
const { storage, segments, partition } = ctx;
|
|
262
374
|
const serverLimits = limitsOf(ctx);
|
|
263
|
-
const digest = await scopeDigest(
|
|
375
|
+
const digest = await scopeDigest(pair.effective);
|
|
264
376
|
const now = clockOf(ctx)();
|
|
265
|
-
if (
|
|
266
|
-
const imaged = yield* sqliteImageSegment(ctx, schema, limits,
|
|
377
|
+
if (start.image !== undefined) {
|
|
378
|
+
const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
|
|
267
379
|
if (imaged)
|
|
268
|
-
return {
|
|
380
|
+
return { pages: 1, rowCursor: null };
|
|
269
381
|
}
|
|
382
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
270
383
|
let rowCursor = startRowCursor;
|
|
271
|
-
for (let page = 0; page <
|
|
384
|
+
for (let page = 0; page < pageBudget; page++) {
|
|
272
385
|
const scanned = page === 0
|
|
273
|
-
? firstPage
|
|
386
|
+
? start.firstPage
|
|
274
387
|
: await storage.scanRows(partition, {
|
|
275
|
-
table:
|
|
276
|
-
scopeFilter:
|
|
388
|
+
table: pair.table.name,
|
|
389
|
+
scopeFilter: pair.effective,
|
|
277
390
|
afterRowId: rowCursor,
|
|
278
391
|
limit: limits.limitSnapshotRows + 1,
|
|
279
392
|
});
|
|
@@ -282,12 +395,12 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
282
395
|
// §5.2: every row record carries the row's current server_version.
|
|
283
396
|
const decoded = pageRows.map((row) => ({
|
|
284
397
|
serverVersion: row.serverVersion,
|
|
285
|
-
values: projectRowValues(
|
|
398
|
+
values: projectRowValues(current, pair.table, row.payload),
|
|
286
399
|
}));
|
|
287
400
|
const bytes = encodeRowsSegment({
|
|
288
|
-
table:
|
|
401
|
+
table: pair.table.name,
|
|
289
402
|
schemaVersion: schema.version,
|
|
290
|
-
columns:
|
|
403
|
+
columns: pair.table.columns,
|
|
291
404
|
blocks: chunkRows(decoded, 1000),
|
|
292
405
|
});
|
|
293
406
|
const lastRow = pageRows[pageRows.length - 1];
|
|
@@ -310,11 +423,11 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
310
423
|
const record = await segments.put({
|
|
311
424
|
partition,
|
|
312
425
|
logEpoch,
|
|
313
|
-
table:
|
|
426
|
+
table: pair.table.name,
|
|
314
427
|
schemaVersion: schema.version,
|
|
315
428
|
mediaType: 'rows',
|
|
316
429
|
scopeDigest: digest,
|
|
317
|
-
asOfCommitSeq:
|
|
430
|
+
asOfCommitSeq: pin,
|
|
318
431
|
rowCount: pageRows.length,
|
|
319
432
|
rowCursor,
|
|
320
433
|
nextRowCursor,
|
|
@@ -329,219 +442,84 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
329
442
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
|
|
330
443
|
}
|
|
331
444
|
if (!hasMore)
|
|
332
|
-
return {
|
|
445
|
+
return { pages: page + 1, rowCursor: null };
|
|
333
446
|
rowCursor = nextRowCursor;
|
|
334
447
|
}
|
|
335
|
-
return {
|
|
448
|
+
return { pages: pageBudget, rowCursor };
|
|
336
449
|
}
|
|
337
450
|
/**
|
|
338
|
-
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
451
|
+
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
452
|
+
* in this section: each visited table costs at least one page, so at most
|
|
453
|
+
* `maxSnapshotPages` tables. The reads of every planned unit start before any
|
|
454
|
+
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
455
|
+
* D1) answers them in one statement per table.
|
|
342
456
|
*/
|
|
343
|
-
export
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
limitChanges: limits.limitCommits + 1,
|
|
363
|
-
}),
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
// §4.7: resume at the token's pin unless the token is unusable or the
|
|
367
|
-
// pin fell behind the horizon; then restart from a fresh pin.
|
|
368
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
|
|
369
|
-
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
370
|
-
? token
|
|
371
|
-
: undefined;
|
|
372
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
373
|
-
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
374
|
-
table: plan.table.name,
|
|
375
|
-
scopeFilter: plan.effective,
|
|
376
|
-
afterRowId: startCursor,
|
|
377
|
-
limit: limits.limitSnapshotRows + 1,
|
|
378
|
-
});
|
|
379
|
-
if (!sqliteLaneEligible(limits, plan, startCursor)) {
|
|
380
|
-
return {
|
|
381
|
-
kind: 'bootstrap',
|
|
382
|
-
asOf: resume?.asOfCommitSeq ?? maxSeq,
|
|
383
|
-
startCursor,
|
|
384
|
-
firstPage: await scan,
|
|
385
|
-
image: undefined,
|
|
386
|
-
};
|
|
387
|
-
}
|
|
388
|
-
// §4.7: a sqlite-lane bootstrap starts at the table start and pins at
|
|
389
|
-
// the newest change in its scope (never below the horizon, never above
|
|
390
|
-
// maxCommitSeq), so a stored image stays the reuse key's match until
|
|
391
|
-
// this scope changes (§5.3). The server computes the pin; a client
|
|
392
|
-
// token never supplies the reuse key.
|
|
393
|
-
const [firstPage, latest] = await Promise.all([
|
|
394
|
-
scan,
|
|
395
|
-
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
396
|
-
table: plan.table.name,
|
|
397
|
-
scopeFilter: plan.effective,
|
|
398
|
-
}),
|
|
399
|
-
]);
|
|
400
|
-
const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
401
|
-
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
402
|
-
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
403
|
-
// `find` on a small table would be a wasted round trip.
|
|
404
|
-
const image = firstPage.length <= limits.limitSnapshotRows
|
|
405
|
-
? undefined
|
|
406
|
-
: await ctx.segments.find({
|
|
407
|
-
partition: ctx.partition,
|
|
408
|
-
logEpoch,
|
|
409
|
-
table: plan.table.name,
|
|
410
|
-
schemaVersion: schema.version,
|
|
411
|
-
mediaType: 'sqlite',
|
|
412
|
-
scopeDigest: await scopeDigest(plan.effective),
|
|
413
|
-
asOfCommitSeq: asOf,
|
|
414
|
-
}, clockOf(ctx)());
|
|
415
|
-
return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
|
|
416
|
-
}));
|
|
417
|
-
// Validate continuity before committing to an active section. A prune
|
|
418
|
-
// during a paged read can otherwise make an incomplete window look empty.
|
|
419
|
-
if (!prepared.some((section) => section.kind === 'incremental')) {
|
|
420
|
-
return prepared;
|
|
457
|
+
export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
|
|
458
|
+
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
459
|
+
// fell behind the horizon; then restart from a fresh pin.
|
|
460
|
+
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
461
|
+
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
462
|
+
? token
|
|
463
|
+
: undefined;
|
|
464
|
+
const tableIndex = resume?.tableIndex ?? 0;
|
|
465
|
+
const rowCursor = resume?.rowCursor ?? null;
|
|
466
|
+
const starts = new Map();
|
|
467
|
+
for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
|
|
468
|
+
const table = unit.tables[index];
|
|
469
|
+
const effective = unit.pairs.get(table.name);
|
|
470
|
+
if (effective === undefined)
|
|
471
|
+
continue;
|
|
472
|
+
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
|
|
473
|
+
// A section that stops early never awaits the later starts.
|
|
474
|
+
start.catch(() => undefined);
|
|
475
|
+
starts.set(index, start);
|
|
421
476
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
477
|
+
return {
|
|
478
|
+
unit,
|
|
479
|
+
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
480
|
+
tableIndex,
|
|
481
|
+
rowCursor,
|
|
482
|
+
starts,
|
|
483
|
+
};
|
|
427
484
|
}
|
|
428
485
|
/**
|
|
429
|
-
*
|
|
430
|
-
*
|
|
486
|
+
* The segments of one bootstrap section (§4.7): tables in bootstrap order,
|
|
487
|
+
* each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
|
|
488
|
+
* the resume token, or `undefined` once every table is exhausted. A pair the
|
|
489
|
+
* wildcard fence revoked receives no segment (§3.3).
|
|
431
490
|
*/
|
|
432
|
-
export async function*
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
/** The section of one subscription whose reads `prepareSections` ran. */
|
|
442
|
-
export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch) {
|
|
443
|
-
const sub = plan.frame;
|
|
444
|
-
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
445
|
-
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
446
|
-
}
|
|
447
|
-
if (prepared.kind === 'revoked') {
|
|
448
|
-
yield {
|
|
449
|
-
type: 'SUB_START',
|
|
450
|
-
id: sub.id,
|
|
451
|
-
status: 'revoked',
|
|
452
|
-
reasonCode: 'sync.scope_revoked',
|
|
453
|
-
effectiveScopes: {},
|
|
454
|
-
bootstrap: false,
|
|
455
|
-
};
|
|
456
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
457
|
-
return { nextCursor: sub.cursor, active: false };
|
|
458
|
-
}
|
|
459
|
-
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
460
|
-
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
461
|
-
if (prepared.kind === 'reset') {
|
|
462
|
-
yield {
|
|
463
|
-
type: 'SUB_START',
|
|
464
|
-
id: sub.id,
|
|
465
|
-
status: 'reset',
|
|
466
|
-
reasonCode: 'sync.cursor_expired',
|
|
467
|
-
effectiveScopes: {},
|
|
468
|
-
bootstrap: false,
|
|
469
|
-
};
|
|
470
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
471
|
-
return { nextCursor: sub.cursor, active: false };
|
|
472
|
-
}
|
|
473
|
-
if (prepared.kind === 'bootstrap') {
|
|
474
|
-
const { asOf } = prepared;
|
|
475
|
-
yield {
|
|
476
|
-
type: 'SUB_START',
|
|
477
|
-
id: sub.id,
|
|
478
|
-
status: 'active',
|
|
479
|
-
reasonCode: '',
|
|
480
|
-
effectiveScopes: plan.effective,
|
|
481
|
-
bootstrap: true,
|
|
482
|
-
};
|
|
483
|
-
const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, prepared.startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
|
|
484
|
-
if (outcome.complete) {
|
|
485
|
-
yield { type: 'SUB_END', nextCursor: asOf };
|
|
491
|
+
export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
|
|
492
|
+
const { unit, pin } = plan;
|
|
493
|
+
let { tableIndex, rowCursor } = plan;
|
|
494
|
+
let pages = 0;
|
|
495
|
+
for (;;) {
|
|
496
|
+
while (tableIndex < unit.tables.length &&
|
|
497
|
+
!unit.pairs.has(unit.tables[tableIndex].name)) {
|
|
498
|
+
tableIndex += 1;
|
|
499
|
+
rowCursor = null;
|
|
486
500
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
type: 'SUB_END',
|
|
496
|
-
nextCursor: asOf,
|
|
497
|
-
bootstrapState: JSON.stringify(nextToken),
|
|
501
|
+
if (tableIndex === unit.tables.length)
|
|
502
|
+
return undefined;
|
|
503
|
+
if (pages === limits.maxSnapshotPages) {
|
|
504
|
+
const token = {
|
|
505
|
+
asOfCommitSeq: pin,
|
|
506
|
+
tables: unit.tables.map((table) => table.name),
|
|
507
|
+
tableIndex,
|
|
508
|
+
rowCursor,
|
|
498
509
|
};
|
|
510
|
+
return JSON.stringify(token);
|
|
499
511
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
id: sub.id,
|
|
507
|
-
status: 'active',
|
|
508
|
-
reasonCode: '',
|
|
509
|
-
effectiveScopes: plan.effective,
|
|
510
|
-
bootstrap: false,
|
|
511
|
-
};
|
|
512
|
-
let delivered = 0;
|
|
513
|
-
let deliveredCommits = 0;
|
|
514
|
-
let lastDeliveredSeq = sub.cursor;
|
|
515
|
-
const { commits } = prepared;
|
|
516
|
-
for (const commit of commits) {
|
|
517
|
-
if (delivered > 0 &&
|
|
518
|
-
delivered + commit.changes.length > limits.limitCommits) {
|
|
519
|
-
break;
|
|
520
|
-
}
|
|
521
|
-
yield commitFrame(sub.table, commit, compileSchema(ctx.schema).tables.get(sub.table), plan.table);
|
|
522
|
-
delivered += commit.changes.length;
|
|
523
|
-
deliveredCommits += 1;
|
|
524
|
-
lastDeliveredSeq = commit.commitSeq;
|
|
525
|
-
if (delivered >= limits.limitCommits)
|
|
526
|
-
break;
|
|
512
|
+
const table = unit.tables[tableIndex];
|
|
513
|
+
const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
|
|
514
|
+
pages += outcome.pages;
|
|
515
|
+
rowCursor = outcome.rowCursor;
|
|
516
|
+
if (rowCursor === null)
|
|
517
|
+
tableIndex += 1;
|
|
527
518
|
}
|
|
528
|
-
const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
|
|
529
|
-
// The window is proven exhausted only when the storage lookahead came
|
|
530
|
-
// back under budget (it scanned to `throughSeq`) and every returned
|
|
531
|
-
// commit was delivered.
|
|
532
|
-
const exhausted = totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
|
|
533
|
-
// §4.5: the cursor advances even when no matching changes exist; when
|
|
534
|
-
// the change limit truncated the window it stops at the last fully
|
|
535
|
-
// delivered commit.
|
|
536
|
-
const nextCursor = exhausted
|
|
537
|
-
? Math.max(sub.cursor, maxSeq)
|
|
538
|
-
: Math.max(sub.cursor, lastDeliveredSeq);
|
|
539
|
-
yield { type: 'SUB_END', nextCursor };
|
|
540
|
-
return { nextCursor, active: true };
|
|
541
519
|
}
|
|
542
520
|
/**
|
|
543
|
-
* Publish the §5.3 sqlite image that a pull
|
|
544
|
-
*
|
|
521
|
+
* Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
|
|
522
|
+
* from a host with a SQLite engine (a Bun process with the production
|
|
545
523
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
546
524
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
547
525
|
* serving host without a builder finds. The image stays current until a
|
|
@@ -554,9 +532,16 @@ export async function publishSqliteImage(input) {
|
|
|
554
532
|
throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
|
|
555
533
|
}
|
|
556
534
|
const schema = compileSchema(config.schema);
|
|
557
|
-
const table = schema.
|
|
535
|
+
const table = coveredTables(schema, input.scopes).find((candidate) => candidate.name === input.table);
|
|
558
536
|
if (table === undefined) {
|
|
559
|
-
throw new Error('publishSqliteImage: the
|
|
537
|
+
throw new Error('publishSqliteImage: the unit does not cover the table');
|
|
538
|
+
}
|
|
539
|
+
const allowed = input.allowed ??
|
|
540
|
+
Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
|
|
541
|
+
const unit = planUnit(input.scopes, [table], { ok: true, allowed });
|
|
542
|
+
const effective = unit.pairs.get(table.name);
|
|
543
|
+
if (effective === undefined) {
|
|
544
|
+
throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
|
|
560
545
|
}
|
|
561
546
|
await config.storage.ensureSchema(schema, config.checkpoints);
|
|
562
547
|
const logEpoch = await config.storage.getPartitionLogEpoch(partition);
|
|
@@ -568,18 +553,6 @@ export async function publishSqliteImage(input) {
|
|
|
568
553
|
partition,
|
|
569
554
|
actorId: 'syncular.image-publisher',
|
|
570
555
|
};
|
|
571
|
-
const plan = {
|
|
572
|
-
frame: {
|
|
573
|
-
type: 'SUBSCRIPTION',
|
|
574
|
-
id: 'image',
|
|
575
|
-
table: input.table,
|
|
576
|
-
scopes: input.scopes,
|
|
577
|
-
cursor: -1,
|
|
578
|
-
},
|
|
579
|
-
table,
|
|
580
|
-
status: 'active',
|
|
581
|
-
effective: input.scopes,
|
|
582
|
-
};
|
|
583
556
|
// One-row pages make every table above one row image-eligible (§5.3).
|
|
584
557
|
const limits = {
|
|
585
558
|
limitCommits: 1,
|
|
@@ -590,11 +563,9 @@ export async function publishSqliteImage(input) {
|
|
|
590
563
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
591
564
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
592
565
|
const trace = { segments: [] };
|
|
593
|
-
const
|
|
594
|
-
if (prepared === undefined)
|
|
595
|
-
throw new Error('prepareSections lost a plan');
|
|
566
|
+
const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
|
|
596
567
|
let ref;
|
|
597
|
-
for await (const frame of
|
|
568
|
+
for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
|
|
598
569
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
599
570
|
ref = frame;
|
|
600
571
|
}
|