@syncular/server 0.59.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +0 -97
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +0 -105
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/pull.ts
CHANGED
|
@@ -1,26 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pull:
|
|
3
|
-
* (SPEC.md §4, §5).
|
|
2
|
+
* Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
|
|
4
3
|
*/
|
|
5
4
|
import {
|
|
6
5
|
type CommitChange,
|
|
6
|
+
type CommitFrame,
|
|
7
7
|
encodeRowsSegment,
|
|
8
|
+
type FencedRecord,
|
|
8
9
|
type PullHeaderFrame,
|
|
9
10
|
type ResponseFrame,
|
|
10
11
|
type ScopeMap,
|
|
11
12
|
type SegmentRefFrame,
|
|
12
13
|
type SegmentRow,
|
|
13
|
-
type
|
|
14
|
+
type UnitScopes,
|
|
15
|
+
unitId,
|
|
14
16
|
} from '@syncular/core';
|
|
15
17
|
import type { SyncRequestContext, SyncServerConfig } from './context';
|
|
16
18
|
import { clockOf, limitsOf } from './context';
|
|
19
|
+
import { syncError } from './errors';
|
|
17
20
|
import type { PullSegmentSummary } from './events';
|
|
18
21
|
import {
|
|
19
22
|
type CompiledSchema,
|
|
20
23
|
type CompiledTable,
|
|
21
24
|
compileSchema,
|
|
22
25
|
} from './schema';
|
|
23
|
-
import { scopeDigest } from './scopes';
|
|
26
|
+
import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
|
|
24
27
|
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
25
28
|
import { issueSegmentUrl } from './signed-url';
|
|
26
29
|
import type { ServerStorage, StoredCommit, StoredRow } from './storage';
|
|
@@ -50,7 +53,7 @@ function clamp(value: number, min: number, max: number, dflt: number): number {
|
|
|
50
53
|
return Math.min(max, Math.max(min, value));
|
|
51
54
|
}
|
|
52
55
|
|
|
53
|
-
/** §4.2 defaults and silent clamps
|
|
56
|
+
/** §4.2 defaults and silent clamps. */
|
|
54
57
|
export function clampPullLimits(header: PullHeaderFrame): PullLimits {
|
|
55
58
|
return {
|
|
56
59
|
limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
|
|
@@ -60,16 +63,140 @@ export function clampPullLimits(header: PullHeaderFrame): PullLimits {
|
|
|
60
63
|
};
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
readonly
|
|
66
|
+
/** One unit of the effective list, resolved against the request (§3.2, §4.1). */
|
|
67
|
+
export interface UnitPlan {
|
|
68
|
+
readonly id: string;
|
|
69
|
+
readonly scopes: UnitScopes;
|
|
70
|
+
/** Covered tables in bootstrap order (§4.1, §4.7). */
|
|
71
|
+
readonly tables: readonly CompiledTable[];
|
|
72
|
+
/** Active (unit, table) pairs: table → effective scopes (§3.2). */
|
|
73
|
+
readonly pairs: ReadonlyMap<string, ScopeMap>;
|
|
66
74
|
readonly status: 'active' | 'revoked';
|
|
67
|
-
|
|
75
|
+
/** The echo (§4.4): ascending table order, empty when revoked. */
|
|
76
|
+
readonly fenced: readonly FencedRecord[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* §4.1 unit validation against one schema version: the covered tables, or
|
|
81
|
+
* `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
|
|
82
|
+
* variable no table declares, or no covered table.
|
|
83
|
+
*/
|
|
84
|
+
export function coveredTables(
|
|
85
|
+
schema: CompiledSchema,
|
|
86
|
+
scopes: UnitScopes,
|
|
87
|
+
): CompiledTable[] {
|
|
88
|
+
const keys = Object.keys(scopes);
|
|
89
|
+
const reason =
|
|
90
|
+
keys.length === 0
|
|
91
|
+
? 'empty_scope_map'
|
|
92
|
+
: keys.some((key) => scopes[key] === '*')
|
|
93
|
+
? 'wildcard_value'
|
|
94
|
+
: keys.some((key) => !schema.declaredVariables.has(key))
|
|
95
|
+
? 'undeclared_variable'
|
|
96
|
+
: undefined;
|
|
97
|
+
const tables =
|
|
98
|
+
reason === undefined
|
|
99
|
+
? [...schema.tables.values()].filter((table) =>
|
|
100
|
+
keys.every((key) => table.declaredVariables.has(key)),
|
|
101
|
+
)
|
|
102
|
+
: [];
|
|
103
|
+
if (tables.length === 0) {
|
|
104
|
+
throw syncError(
|
|
105
|
+
'sync.invalid_subscription',
|
|
106
|
+
'the server cannot serve this unit (§4.1)',
|
|
107
|
+
JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }),
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return tables;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
114
|
+
export function planUnit(
|
|
115
|
+
scopes: UnitScopes,
|
|
116
|
+
tables: readonly CompiledTable[],
|
|
117
|
+
resolved: ResolvedScopes,
|
|
118
|
+
): UnitPlan {
|
|
119
|
+
const requested: ScopeMap = {};
|
|
120
|
+
for (const [key, value] of Object.entries(scopes)) requested[key] = [value];
|
|
121
|
+
const pairs = new Map<string, ScopeMap>();
|
|
122
|
+
const fenced: FencedRecord[] = [];
|
|
123
|
+
for (const table of tables) {
|
|
124
|
+
const outcome = computeEffective(
|
|
125
|
+
table.declaredVariables,
|
|
126
|
+
requested,
|
|
127
|
+
resolved,
|
|
128
|
+
);
|
|
129
|
+
if (outcome.status === 'revoked') {
|
|
130
|
+
fenced.push({ table: table.name, status: 'revoked', scopes: {} });
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
pairs.set(table.name, outcome.effective);
|
|
134
|
+
const added = Object.entries(outcome.effective).filter(
|
|
135
|
+
([key]) => !Object.hasOwn(scopes, key),
|
|
136
|
+
);
|
|
137
|
+
if (added.length > 0) {
|
|
138
|
+
fenced.push({
|
|
139
|
+
table: table.name,
|
|
140
|
+
status: 'active',
|
|
141
|
+
scopes: Object.fromEntries(added),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const active = pairs.size > 0;
|
|
146
|
+
return {
|
|
147
|
+
id: unitId(scopes),
|
|
148
|
+
scopes,
|
|
149
|
+
tables,
|
|
150
|
+
pairs,
|
|
151
|
+
status: active ? 'active' : 'revoked',
|
|
152
|
+
fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
|
|
153
|
+
};
|
|
68
154
|
}
|
|
69
155
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
156
|
+
/**
|
|
157
|
+
* The `COMMIT` frame of one live-section commit (§4.5): its frame-local
|
|
158
|
+
* table dictionary and every change, projected onto the client's codec.
|
|
159
|
+
*/
|
|
160
|
+
export function commitFrame(
|
|
161
|
+
commit: StoredCommit,
|
|
162
|
+
current: CompiledSchema,
|
|
163
|
+
target: CompiledSchema,
|
|
164
|
+
): CommitFrame {
|
|
165
|
+
const tables: string[] = [];
|
|
166
|
+
const changes: CommitChange[] = commit.changes.map((change) => {
|
|
167
|
+
// §4.5: apply is version-guarded, so every change carries its version:
|
|
168
|
+
// an upsert the row's version after the write, a delete the version its
|
|
169
|
+
// tombstone records. A change without one cannot be served.
|
|
170
|
+
if (change.rowVersion === undefined) {
|
|
171
|
+
throw new Error('a change record carries no row version (§4.5)');
|
|
172
|
+
}
|
|
173
|
+
let tableIndex = tables.indexOf(change.table);
|
|
174
|
+
if (tableIndex === -1) tableIndex = tables.push(change.table) - 1;
|
|
175
|
+
return {
|
|
176
|
+
tableIndex,
|
|
177
|
+
rowId: change.rowId,
|
|
178
|
+
op: change.op,
|
|
179
|
+
rowVersion: change.rowVersion,
|
|
180
|
+
scopes: change.scopes,
|
|
181
|
+
...(change.payload !== undefined
|
|
182
|
+
? {
|
|
183
|
+
row: projectRowPayload(
|
|
184
|
+
current.tables.get(change.table)!,
|
|
185
|
+
target.tables.get(change.table)!,
|
|
186
|
+
change.payload,
|
|
187
|
+
),
|
|
188
|
+
}
|
|
189
|
+
: {}),
|
|
190
|
+
};
|
|
191
|
+
});
|
|
192
|
+
return {
|
|
193
|
+
type: 'COMMIT',
|
|
194
|
+
commitSeq: commit.commitSeq,
|
|
195
|
+
createdAtMs: commit.createdAtMs,
|
|
196
|
+
actorId: commit.actorId,
|
|
197
|
+
tables,
|
|
198
|
+
changes,
|
|
199
|
+
};
|
|
73
200
|
}
|
|
74
201
|
|
|
75
202
|
/**
|
|
@@ -89,13 +216,14 @@ interface BootstrapToken {
|
|
|
89
216
|
}
|
|
90
217
|
|
|
91
218
|
/**
|
|
92
|
-
* §4.7: a usable resume token
|
|
93
|
-
*
|
|
94
|
-
*
|
|
219
|
+
* §4.7: a usable resume token, or `undefined`. A token whose pin is not a
|
|
220
|
+
* commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
|
|
221
|
+
* differ from the unit's covered tables in bootstrap order, or whose position
|
|
222
|
+
* names none of them is unusable.
|
|
95
223
|
*/
|
|
96
224
|
function parseBootstrapToken(
|
|
97
225
|
raw: string | undefined,
|
|
98
|
-
|
|
226
|
+
tables: readonly CompiledTable[],
|
|
99
227
|
maxSeq: number,
|
|
100
228
|
): BootstrapToken | undefined {
|
|
101
229
|
if (raw === undefined) return undefined;
|
|
@@ -107,11 +235,12 @@ function parseBootstrapToken(
|
|
|
107
235
|
parsed.asOfCommitSeq < 0 ||
|
|
108
236
|
parsed.asOfCommitSeq > maxSeq ||
|
|
109
237
|
!Array.isArray(parsed.tables) ||
|
|
110
|
-
parsed.tables
|
|
238
|
+
parsed.tables.length !== tables.length ||
|
|
239
|
+
parsed.tables.some((name, i) => name !== tables[i]?.name) ||
|
|
111
240
|
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
112
241
|
parsed.tableIndex === undefined ||
|
|
113
242
|
parsed.tableIndex < 0 ||
|
|
114
|
-
parsed.tableIndex >=
|
|
243
|
+
parsed.tableIndex >= tables.length ||
|
|
115
244
|
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
116
245
|
) {
|
|
117
246
|
return undefined;
|
|
@@ -127,34 +256,6 @@ function parseBootstrapToken(
|
|
|
127
256
|
}
|
|
128
257
|
}
|
|
129
258
|
|
|
130
|
-
function commitFrame(
|
|
131
|
-
table: string,
|
|
132
|
-
commit: StoredCommit,
|
|
133
|
-
current: CompiledTable,
|
|
134
|
-
target: CompiledTable,
|
|
135
|
-
): ResponseFrame {
|
|
136
|
-
const changes: CommitChange[] = commit.changes.map((change) => ({
|
|
137
|
-
tableIndex: 0,
|
|
138
|
-
rowId: change.rowId,
|
|
139
|
-
op: change.op,
|
|
140
|
-
...(change.rowVersion !== undefined
|
|
141
|
-
? { rowVersion: change.rowVersion }
|
|
142
|
-
: {}),
|
|
143
|
-
scopes: change.scopes,
|
|
144
|
-
...(change.payload !== undefined
|
|
145
|
-
? { row: projectRowPayload(current, target, change.payload) }
|
|
146
|
-
: {}),
|
|
147
|
-
}));
|
|
148
|
-
return {
|
|
149
|
-
type: 'COMMIT',
|
|
150
|
-
commitSeq: commit.commitSeq,
|
|
151
|
-
createdAtMs: commit.createdAtMs,
|
|
152
|
-
actorId: commit.actorId,
|
|
153
|
-
tables: [table],
|
|
154
|
-
changes,
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
|
|
158
259
|
function chunkRows(rows: SegmentRow[], size: number): SegmentRow[][] {
|
|
159
260
|
if (rows.length === 0) return [];
|
|
160
261
|
const blocks: SegmentRow[][] = [];
|
|
@@ -194,10 +295,10 @@ function segmentRefFrame(
|
|
|
194
295
|
byteLength: record.byteLength,
|
|
195
296
|
rowCount: record.rowCount,
|
|
196
297
|
asOfCommitSeq: record.asOfCommitSeq,
|
|
197
|
-
// The frame reports the digest THIS
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
298
|
+
// The frame reports the digest THIS pair is authorized under. A stored
|
|
299
|
+
// entry can carry several digests when identical content was published
|
|
300
|
+
// under different scopes (§5.1), and the record's primary digest is not
|
|
301
|
+
// necessarily the caller's (§5.5).
|
|
201
302
|
scopeDigest,
|
|
202
303
|
...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
|
|
203
304
|
...(record.nextRowCursor !== null
|
|
@@ -210,6 +311,12 @@ function segmentRefFrame(
|
|
|
210
311
|
};
|
|
211
312
|
}
|
|
212
313
|
|
|
314
|
+
/** One active (unit, table) pair of a bootstrap (§3.2). */
|
|
315
|
+
interface BootstrapPair {
|
|
316
|
+
readonly table: CompiledTable;
|
|
317
|
+
readonly effective: ScopeMap;
|
|
318
|
+
}
|
|
319
|
+
|
|
213
320
|
/**
|
|
214
321
|
* §5.3: the sqlite-image lane — whole-table, chosen only at the start of a
|
|
215
322
|
* table (a mid-table resume stays on the rows lane, never switching lanes),
|
|
@@ -220,30 +327,96 @@ function segmentRefFrame(
|
|
|
220
327
|
*/
|
|
221
328
|
function sqliteLaneEligible(
|
|
222
329
|
limits: PullLimits,
|
|
223
|
-
|
|
330
|
+
table: CompiledTable,
|
|
224
331
|
startRowCursor: string | null,
|
|
225
332
|
): boolean {
|
|
226
333
|
return (
|
|
227
334
|
(limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
228
335
|
startRowCursor === null &&
|
|
229
|
-
|
|
336
|
+
table.encryptedColumnIndices.length === 0
|
|
230
337
|
);
|
|
231
338
|
}
|
|
232
339
|
|
|
340
|
+
/** The reads a table's bootstrap needs before its first frame (§4.7, §5.3). */
|
|
341
|
+
interface TableStart {
|
|
342
|
+
/**
|
|
343
|
+
* `limitSnapshotRows + 1` rows from the start cursor: the first page and
|
|
344
|
+
* the §5.3 image eligibility probe.
|
|
345
|
+
*/
|
|
346
|
+
readonly firstPage: StoredRow[];
|
|
347
|
+
/** On the sqlite lane: the §4.7 scope pin and a stored image under it. */
|
|
348
|
+
readonly image?: {
|
|
349
|
+
readonly scopePin: number;
|
|
350
|
+
readonly stored: SegmentRecord | undefined;
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
async function readTableStart(
|
|
355
|
+
ctx: SyncRequestContext,
|
|
356
|
+
schema: CompiledSchema,
|
|
357
|
+
limits: PullLimits,
|
|
358
|
+
unit: UnitPlan,
|
|
359
|
+
pair: BootstrapPair,
|
|
360
|
+
startRowCursor: string | null,
|
|
361
|
+
maxSeq: number,
|
|
362
|
+
horizonSeq: number,
|
|
363
|
+
logEpoch: string,
|
|
364
|
+
): Promise<TableStart> {
|
|
365
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
366
|
+
table: pair.table.name,
|
|
367
|
+
scopeFilter: pair.effective,
|
|
368
|
+
afterRowId: startRowCursor,
|
|
369
|
+
limit: limits.limitSnapshotRows + 1,
|
|
370
|
+
});
|
|
371
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
372
|
+
return { firstPage: await scan };
|
|
373
|
+
}
|
|
374
|
+
// §4.7: a sqlite-lane table pins at the newest change carrying the unit's
|
|
375
|
+
// index key (never below the horizon, never above maxCommitSeq), so a
|
|
376
|
+
// stored image stays the reuse key's match until this scope changes
|
|
377
|
+
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
378
|
+
const [firstPage, latest] = await Promise.all([
|
|
379
|
+
scan,
|
|
380
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
381
|
+
table: pair.table.name,
|
|
382
|
+
scopes: unit.scopes,
|
|
383
|
+
}),
|
|
384
|
+
]);
|
|
385
|
+
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
386
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than one
|
|
387
|
+
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
388
|
+
// on a small table would be a wasted round trip.
|
|
389
|
+
const stored =
|
|
390
|
+
firstPage.length <= limits.limitSnapshotRows
|
|
391
|
+
? undefined
|
|
392
|
+
: await ctx.segments.find(
|
|
393
|
+
{
|
|
394
|
+
partition: ctx.partition,
|
|
395
|
+
logEpoch,
|
|
396
|
+
table: pair.table.name,
|
|
397
|
+
schemaVersion: schema.version,
|
|
398
|
+
mediaType: 'sqlite',
|
|
399
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
400
|
+
asOfCommitSeq: scopePin,
|
|
401
|
+
},
|
|
402
|
+
clockOf(ctx)(),
|
|
403
|
+
);
|
|
404
|
+
return { firstPage, image: { scopePin, stored } };
|
|
405
|
+
}
|
|
406
|
+
|
|
233
407
|
/**
|
|
234
|
-
* §5.3 sqlite-image lane: only
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
* (the rows lane takes over).
|
|
408
|
+
* §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
|
|
409
|
+
* Reuses an unexpired stored image for the same (partition, table,
|
|
410
|
+
* schemaVersion, scope digest, scope pin) instead of rebuilding — the
|
|
411
|
+
* bootstrap-storm rule. Returns false when no image is served (the rows lane
|
|
412
|
+
* takes over).
|
|
240
413
|
*/
|
|
241
414
|
async function* sqliteImageSegment(
|
|
242
415
|
ctx: SyncRequestContext,
|
|
243
416
|
schema: CompiledSchema,
|
|
244
417
|
limits: PullLimits,
|
|
245
|
-
|
|
246
|
-
|
|
418
|
+
pair: BootstrapPair,
|
|
419
|
+
scopePin: number,
|
|
247
420
|
digest: string,
|
|
248
421
|
trace: PullSectionTrace | undefined,
|
|
249
422
|
logEpoch: string,
|
|
@@ -267,9 +440,6 @@ async function* sqliteImageSegment(
|
|
|
267
440
|
);
|
|
268
441
|
return true;
|
|
269
442
|
}
|
|
270
|
-
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
271
|
-
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
272
|
-
// The probe is the rows lane's first page, read by `prepareSections`.
|
|
273
443
|
if (probe.length <= limits.limitSnapshotRows) return false;
|
|
274
444
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
275
445
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
@@ -288,12 +458,13 @@ async function* sqliteImageSegment(
|
|
|
288
458
|
const identity = {
|
|
289
459
|
partition,
|
|
290
460
|
logEpoch,
|
|
291
|
-
table:
|
|
461
|
+
table: pair.table.name,
|
|
292
462
|
schemaVersion: schema.version,
|
|
293
463
|
mediaType: 'sqlite' as const,
|
|
294
464
|
scopeDigest: digest,
|
|
295
|
-
asOfCommitSeq:
|
|
465
|
+
asOfCommitSeq: scopePin,
|
|
296
466
|
};
|
|
467
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
297
468
|
const key = JSON.stringify(identity);
|
|
298
469
|
let building = builds.get(key);
|
|
299
470
|
let builtHere = false;
|
|
@@ -305,36 +476,28 @@ async function* sqliteImageSegment(
|
|
|
305
476
|
builtHere = true;
|
|
306
477
|
let rowCount = 0;
|
|
307
478
|
const bytes = await buildImage({
|
|
308
|
-
table:
|
|
479
|
+
table: pair.table,
|
|
309
480
|
schemaVersion: schema.version,
|
|
310
|
-
asOfCommitSeq:
|
|
481
|
+
asOfCommitSeq: scopePin,
|
|
311
482
|
scopeDigest: digest,
|
|
312
483
|
rowBatches: (async function* () {
|
|
313
484
|
rowCount += probe.length;
|
|
314
485
|
yield probe.map((row) => ({
|
|
315
486
|
...row,
|
|
316
|
-
payload: projectRowPayload(
|
|
317
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
318
|
-
plan.table,
|
|
319
|
-
row.payload,
|
|
320
|
-
),
|
|
487
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
321
488
|
}));
|
|
322
489
|
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
323
490
|
for (;;) {
|
|
324
491
|
const rows = await storage.scanRows(partition, {
|
|
325
|
-
table:
|
|
326
|
-
scopeFilter:
|
|
492
|
+
table: pair.table.name,
|
|
493
|
+
scopeFilter: pair.effective,
|
|
327
494
|
afterRowId,
|
|
328
495
|
limit: 5_000,
|
|
329
496
|
});
|
|
330
497
|
rowCount += rows.length;
|
|
331
498
|
yield rows.map((row) => ({
|
|
332
499
|
...row,
|
|
333
|
-
payload: projectRowPayload(
|
|
334
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
335
|
-
plan.table,
|
|
336
|
-
row.payload,
|
|
337
|
-
),
|
|
500
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
338
501
|
}));
|
|
339
502
|
const last = rows[rows.length - 1];
|
|
340
503
|
if (rows.length < 5_000 || last === undefined) break;
|
|
@@ -377,48 +540,50 @@ async function* sqliteImageSegment(
|
|
|
377
540
|
return true;
|
|
378
541
|
}
|
|
379
542
|
|
|
380
|
-
|
|
543
|
+
/**
|
|
544
|
+
* One table's segments within the page budget. Returns the pages it used
|
|
545
|
+
* and the row cursor to resume at, `null` once the table is exhausted.
|
|
546
|
+
*/
|
|
547
|
+
async function* tableSegments(
|
|
381
548
|
ctx: SyncRequestContext,
|
|
382
549
|
schema: CompiledSchema,
|
|
383
550
|
limits: PullLimits,
|
|
384
|
-
|
|
385
|
-
|
|
551
|
+
pair: BootstrapPair,
|
|
552
|
+
pin: number,
|
|
386
553
|
startRowCursor: string | null,
|
|
554
|
+
start: TableStart,
|
|
555
|
+
pageBudget: number,
|
|
387
556
|
trace: PullSectionTrace | undefined,
|
|
388
557
|
logEpoch: string,
|
|
389
|
-
|
|
390
|
-
image: SegmentRecord | undefined,
|
|
391
|
-
): AsyncGenerator<
|
|
392
|
-
ResponseFrame,
|
|
393
|
-
{ complete: boolean; rowCursor: string | null }
|
|
394
|
-
> {
|
|
558
|
+
): AsyncGenerator<ResponseFrame, { pages: number; rowCursor: string | null }> {
|
|
395
559
|
const { storage, segments, partition } = ctx;
|
|
396
560
|
const serverLimits = limitsOf(ctx);
|
|
397
|
-
const digest = await scopeDigest(
|
|
561
|
+
const digest = await scopeDigest(pair.effective);
|
|
398
562
|
const now = clockOf(ctx)();
|
|
399
|
-
if (
|
|
563
|
+
if (start.image !== undefined) {
|
|
400
564
|
const imaged = yield* sqliteImageSegment(
|
|
401
565
|
ctx,
|
|
402
566
|
schema,
|
|
403
567
|
limits,
|
|
404
|
-
|
|
405
|
-
|
|
568
|
+
pair,
|
|
569
|
+
start.image.scopePin,
|
|
406
570
|
digest,
|
|
407
571
|
trace,
|
|
408
572
|
logEpoch,
|
|
409
|
-
image,
|
|
410
|
-
firstPage,
|
|
573
|
+
start.image.stored,
|
|
574
|
+
start.firstPage,
|
|
411
575
|
);
|
|
412
|
-
if (imaged) return {
|
|
576
|
+
if (imaged) return { pages: 1, rowCursor: null };
|
|
413
577
|
}
|
|
578
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
414
579
|
let rowCursor = startRowCursor;
|
|
415
|
-
for (let page = 0; page <
|
|
580
|
+
for (let page = 0; page < pageBudget; page++) {
|
|
416
581
|
const scanned =
|
|
417
582
|
page === 0
|
|
418
|
-
? firstPage
|
|
583
|
+
? start.firstPage
|
|
419
584
|
: await storage.scanRows(partition, {
|
|
420
|
-
table:
|
|
421
|
-
scopeFilter:
|
|
585
|
+
table: pair.table.name,
|
|
586
|
+
scopeFilter: pair.effective,
|
|
422
587
|
afterRowId: rowCursor,
|
|
423
588
|
limit: limits.limitSnapshotRows + 1,
|
|
424
589
|
});
|
|
@@ -427,16 +592,12 @@ async function* bootstrapSegments(
|
|
|
427
592
|
// §5.2: every row record carries the row's current server_version.
|
|
428
593
|
const decoded = pageRows.map((row: StoredRow) => ({
|
|
429
594
|
serverVersion: row.serverVersion,
|
|
430
|
-
values: projectRowValues(
|
|
431
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
432
|
-
plan.table,
|
|
433
|
-
row.payload,
|
|
434
|
-
),
|
|
595
|
+
values: projectRowValues(current, pair.table, row.payload),
|
|
435
596
|
}));
|
|
436
597
|
const bytes = encodeRowsSegment({
|
|
437
|
-
table:
|
|
598
|
+
table: pair.table.name,
|
|
438
599
|
schemaVersion: schema.version,
|
|
439
|
-
columns:
|
|
600
|
+
columns: pair.table.columns,
|
|
440
601
|
blocks: chunkRows(decoded, 1000),
|
|
441
602
|
});
|
|
442
603
|
const lastRow = pageRows[pageRows.length - 1];
|
|
@@ -462,11 +623,11 @@ async function* bootstrapSegments(
|
|
|
462
623
|
{
|
|
463
624
|
partition,
|
|
464
625
|
logEpoch,
|
|
465
|
-
table:
|
|
626
|
+
table: pair.table.name,
|
|
466
627
|
schemaVersion: schema.version,
|
|
467
628
|
mediaType: 'rows',
|
|
468
629
|
scopeDigest: digest,
|
|
469
|
-
asOfCommitSeq:
|
|
630
|
+
asOfCommitSeq: pin,
|
|
470
631
|
rowCount: pageRows.length,
|
|
471
632
|
rowCursor,
|
|
472
633
|
nextRowCursor,
|
|
@@ -487,314 +648,134 @@ async function* bootstrapSegments(
|
|
|
487
648
|
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
488
649
|
);
|
|
489
650
|
}
|
|
490
|
-
if (!hasMore) return {
|
|
651
|
+
if (!hasMore) return { pages: page + 1, rowCursor: null };
|
|
491
652
|
rowCursor = nextRowCursor;
|
|
492
653
|
}
|
|
493
|
-
return {
|
|
654
|
+
return { pages: pageBudget, rowCursor };
|
|
494
655
|
}
|
|
495
656
|
|
|
496
|
-
/**
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
*/
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
| {
|
|
507
|
-
readonly kind: 'bootstrap';
|
|
508
|
-
/** §4.7 pin: a usable resume token's, the sqlite lane's scope pin, or `maxCommitSeq`. */
|
|
509
|
-
readonly asOf: number;
|
|
510
|
-
readonly startCursor: string | null;
|
|
511
|
-
/**
|
|
512
|
-
* The first snapshot page (`limitSnapshotRows + 1` rows from
|
|
513
|
-
* `startCursor`), which is also the §5.3 image eligibility probe.
|
|
514
|
-
*/
|
|
515
|
-
readonly firstPage: StoredRow[];
|
|
516
|
-
/** §5.3 stored image under the reuse key, read only for an eligible probe. */
|
|
517
|
-
readonly image: SegmentRecord | undefined;
|
|
518
|
-
};
|
|
657
|
+
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
658
|
+
export interface BootstrapPlan {
|
|
659
|
+
readonly unit: UnitPlan;
|
|
660
|
+
/** The unit's pin: a usable resume token's, or `maxCommitSeq`. */
|
|
661
|
+
readonly pin: number;
|
|
662
|
+
readonly tableIndex: number;
|
|
663
|
+
readonly rowCursor: string | null;
|
|
664
|
+
/** Table index → its start reads. */
|
|
665
|
+
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
666
|
+
}
|
|
519
667
|
|
|
520
668
|
/**
|
|
521
|
-
*
|
|
522
|
-
*
|
|
523
|
-
*
|
|
524
|
-
*
|
|
669
|
+
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
670
|
+
* in this section: each visited table costs at least one page, so at most
|
|
671
|
+
* `maxSnapshotPages` tables. The reads of every planned unit start before any
|
|
672
|
+
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
673
|
+
* D1) answers them in one statement per table.
|
|
525
674
|
*/
|
|
526
|
-
export
|
|
675
|
+
export function planBootstrap(
|
|
527
676
|
ctx: SyncRequestContext,
|
|
528
677
|
schema: CompiledSchema,
|
|
529
678
|
limits: PullLimits,
|
|
530
|
-
|
|
679
|
+
unit: UnitPlan,
|
|
680
|
+
bootstrapState: string | undefined,
|
|
531
681
|
maxSeq: number,
|
|
532
682
|
horizonSeq: number,
|
|
533
683
|
logEpoch: string,
|
|
534
|
-
):
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
table: plan.table.name,
|
|
568
|
-
scopeFilter: plan.effective,
|
|
569
|
-
afterRowId: startCursor,
|
|
570
|
-
limit: limits.limitSnapshotRows + 1,
|
|
571
|
-
});
|
|
572
|
-
if (!sqliteLaneEligible(limits, plan, startCursor)) {
|
|
573
|
-
return {
|
|
574
|
-
kind: 'bootstrap',
|
|
575
|
-
asOf: resume?.asOfCommitSeq ?? maxSeq,
|
|
576
|
-
startCursor,
|
|
577
|
-
firstPage: await scan,
|
|
578
|
-
image: undefined,
|
|
579
|
-
};
|
|
580
|
-
}
|
|
581
|
-
// §4.7: a sqlite-lane bootstrap starts at the table start and pins at
|
|
582
|
-
// the newest change in its scope (never below the horizon, never above
|
|
583
|
-
// maxCommitSeq), so a stored image stays the reuse key's match until
|
|
584
|
-
// this scope changes (§5.3). The server computes the pin; a client
|
|
585
|
-
// token never supplies the reuse key.
|
|
586
|
-
const [firstPage, latest] = await Promise.all([
|
|
587
|
-
scan,
|
|
588
|
-
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
589
|
-
table: plan.table.name,
|
|
590
|
-
scopeFilter: plan.effective,
|
|
591
|
-
}),
|
|
592
|
-
]);
|
|
593
|
-
const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
594
|
-
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
595
|
-
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
596
|
-
// `find` on a small table would be a wasted round trip.
|
|
597
|
-
const image =
|
|
598
|
-
firstPage.length <= limits.limitSnapshotRows
|
|
599
|
-
? undefined
|
|
600
|
-
: await ctx.segments.find(
|
|
601
|
-
{
|
|
602
|
-
partition: ctx.partition,
|
|
603
|
-
logEpoch,
|
|
604
|
-
table: plan.table.name,
|
|
605
|
-
schemaVersion: schema.version,
|
|
606
|
-
mediaType: 'sqlite',
|
|
607
|
-
scopeDigest: await scopeDigest(plan.effective),
|
|
608
|
-
asOfCommitSeq: asOf,
|
|
609
|
-
},
|
|
610
|
-
clockOf(ctx)(),
|
|
611
|
-
);
|
|
612
|
-
return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
|
|
613
|
-
}),
|
|
614
|
-
);
|
|
615
|
-
// Validate continuity before committing to an active section. A prune
|
|
616
|
-
// during a paged read can otherwise make an incomplete window look empty.
|
|
617
|
-
if (!prepared.some((section) => section.kind === 'incremental')) {
|
|
618
|
-
return prepared;
|
|
684
|
+
): BootstrapPlan {
|
|
685
|
+
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
686
|
+
// fell behind the horizon; then restart from a fresh pin.
|
|
687
|
+
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
688
|
+
const resume =
|
|
689
|
+
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
690
|
+
? token
|
|
691
|
+
: undefined;
|
|
692
|
+
const tableIndex = resume?.tableIndex ?? 0;
|
|
693
|
+
const rowCursor = resume?.rowCursor ?? null;
|
|
694
|
+
const starts = new Map<number, Promise<TableStart>>();
|
|
695
|
+
for (
|
|
696
|
+
let index = tableIndex;
|
|
697
|
+
index < unit.tables.length && starts.size < limits.maxSnapshotPages;
|
|
698
|
+
index++
|
|
699
|
+
) {
|
|
700
|
+
const table = unit.tables[index]!;
|
|
701
|
+
const effective = unit.pairs.get(table.name);
|
|
702
|
+
if (effective === undefined) continue;
|
|
703
|
+
const start = readTableStart(
|
|
704
|
+
ctx,
|
|
705
|
+
schema,
|
|
706
|
+
limits,
|
|
707
|
+
unit,
|
|
708
|
+
{ table, effective },
|
|
709
|
+
index === tableIndex ? rowCursor : null,
|
|
710
|
+
maxSeq,
|
|
711
|
+
horizonSeq,
|
|
712
|
+
logEpoch,
|
|
713
|
+
);
|
|
714
|
+
// A section that stops early never awaits the later starts.
|
|
715
|
+
start.catch(() => undefined);
|
|
716
|
+
starts.set(index, start);
|
|
619
717
|
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
718
|
+
return {
|
|
719
|
+
unit,
|
|
720
|
+
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
721
|
+
tableIndex,
|
|
722
|
+
rowCursor,
|
|
723
|
+
starts,
|
|
724
|
+
};
|
|
627
725
|
}
|
|
628
726
|
|
|
629
727
|
/**
|
|
630
|
-
*
|
|
631
|
-
*
|
|
728
|
+
* The segments of one bootstrap section (§4.7): tables in bootstrap order,
|
|
729
|
+
* each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
|
|
730
|
+
* the resume token, or `undefined` once every table is exhausted. A pair the
|
|
731
|
+
* wildcard fence revoked receives no segment (§3.3).
|
|
632
732
|
*/
|
|
633
|
-
export async function*
|
|
733
|
+
export async function* bootstrapSegments(
|
|
634
734
|
ctx: SyncRequestContext,
|
|
635
735
|
schema: CompiledSchema,
|
|
636
736
|
limits: PullLimits,
|
|
637
|
-
plan:
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
logEpoch,
|
|
665
|
-
);
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
/** The section of one subscription whose reads `prepareSections` ran. */
|
|
669
|
-
export async function* preparedSection(
|
|
670
|
-
ctx: SyncRequestContext,
|
|
671
|
-
schema: CompiledSchema,
|
|
672
|
-
limits: PullLimits,
|
|
673
|
-
plan: SubscriptionPlan,
|
|
674
|
-
maxSeq: number,
|
|
675
|
-
prepared: PreparedSection,
|
|
676
|
-
trace?: PullSectionTrace,
|
|
677
|
-
logEpoch?: string,
|
|
678
|
-
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
679
|
-
const sub = plan.frame;
|
|
680
|
-
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
681
|
-
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
if (prepared.kind === 'revoked') {
|
|
685
|
-
yield {
|
|
686
|
-
type: 'SUB_START',
|
|
687
|
-
id: sub.id,
|
|
688
|
-
status: 'revoked',
|
|
689
|
-
reasonCode: 'sync.scope_revoked',
|
|
690
|
-
effectiveScopes: {},
|
|
691
|
-
bootstrap: false,
|
|
692
|
-
};
|
|
693
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
694
|
-
return { nextCursor: sub.cursor, active: false };
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
698
|
-
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
699
|
-
if (prepared.kind === 'reset') {
|
|
700
|
-
yield {
|
|
701
|
-
type: 'SUB_START',
|
|
702
|
-
id: sub.id,
|
|
703
|
-
status: 'reset',
|
|
704
|
-
reasonCode: 'sync.cursor_expired',
|
|
705
|
-
effectiveScopes: {},
|
|
706
|
-
bootstrap: false,
|
|
707
|
-
};
|
|
708
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
709
|
-
return { nextCursor: sub.cursor, active: false };
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
if (prepared.kind === 'bootstrap') {
|
|
713
|
-
const { asOf } = prepared;
|
|
714
|
-
yield {
|
|
715
|
-
type: 'SUB_START',
|
|
716
|
-
id: sub.id,
|
|
717
|
-
status: 'active',
|
|
718
|
-
reasonCode: '',
|
|
719
|
-
effectiveScopes: plan.effective,
|
|
720
|
-
bootstrap: true,
|
|
721
|
-
};
|
|
722
|
-
const outcome = yield* bootstrapSegments(
|
|
737
|
+
plan: BootstrapPlan,
|
|
738
|
+
trace: PullSectionTrace | undefined,
|
|
739
|
+
logEpoch: string,
|
|
740
|
+
): AsyncGenerator<ResponseFrame, string | undefined> {
|
|
741
|
+
const { unit, pin } = plan;
|
|
742
|
+
let { tableIndex, rowCursor } = plan;
|
|
743
|
+
let pages = 0;
|
|
744
|
+
for (;;) {
|
|
745
|
+
while (
|
|
746
|
+
tableIndex < unit.tables.length &&
|
|
747
|
+
!unit.pairs.has(unit.tables[tableIndex]!.name)
|
|
748
|
+
) {
|
|
749
|
+
tableIndex += 1;
|
|
750
|
+
rowCursor = null;
|
|
751
|
+
}
|
|
752
|
+
if (tableIndex === unit.tables.length) return undefined;
|
|
753
|
+
if (pages === limits.maxSnapshotPages) {
|
|
754
|
+
const token: BootstrapToken = {
|
|
755
|
+
asOfCommitSeq: pin,
|
|
756
|
+
tables: unit.tables.map((table) => table.name),
|
|
757
|
+
tableIndex,
|
|
758
|
+
rowCursor,
|
|
759
|
+
};
|
|
760
|
+
return JSON.stringify(token);
|
|
761
|
+
}
|
|
762
|
+
const table = unit.tables[tableIndex]!;
|
|
763
|
+
const outcome = yield* tableSegments(
|
|
723
764
|
ctx,
|
|
724
765
|
schema,
|
|
725
766
|
limits,
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
767
|
+
{ table, effective: unit.pairs.get(table.name)! },
|
|
768
|
+
pin,
|
|
769
|
+
rowCursor,
|
|
770
|
+
await plan.starts.get(tableIndex)!,
|
|
771
|
+
limits.maxSnapshotPages - pages,
|
|
729
772
|
trace,
|
|
730
773
|
logEpoch,
|
|
731
|
-
prepared.firstPage,
|
|
732
|
-
prepared.image,
|
|
733
774
|
);
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
const nextToken: BootstrapToken = {
|
|
738
|
-
asOfCommitSeq: asOf,
|
|
739
|
-
tables: [sub.table],
|
|
740
|
-
tableIndex: 0,
|
|
741
|
-
rowCursor: outcome.rowCursor,
|
|
742
|
-
};
|
|
743
|
-
yield {
|
|
744
|
-
type: 'SUB_END',
|
|
745
|
-
nextCursor: asOf,
|
|
746
|
-
bootstrapState: JSON.stringify(nextToken),
|
|
747
|
-
};
|
|
748
|
-
}
|
|
749
|
-
return { nextCursor: asOf, active: true };
|
|
775
|
+
pages += outcome.pages;
|
|
776
|
+
rowCursor = outcome.rowCursor;
|
|
777
|
+
if (rowCursor === null) tableIndex += 1;
|
|
750
778
|
}
|
|
751
|
-
|
|
752
|
-
// Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
|
|
753
|
-
// cut off at limitCommits total changes, never splitting a commit.
|
|
754
|
-
yield {
|
|
755
|
-
type: 'SUB_START',
|
|
756
|
-
id: sub.id,
|
|
757
|
-
status: 'active',
|
|
758
|
-
reasonCode: '',
|
|
759
|
-
effectiveScopes: plan.effective,
|
|
760
|
-
bootstrap: false,
|
|
761
|
-
};
|
|
762
|
-
let delivered = 0;
|
|
763
|
-
let deliveredCommits = 0;
|
|
764
|
-
let lastDeliveredSeq = sub.cursor;
|
|
765
|
-
const { commits } = prepared;
|
|
766
|
-
for (const commit of commits) {
|
|
767
|
-
if (
|
|
768
|
-
delivered > 0 &&
|
|
769
|
-
delivered + commit.changes.length > limits.limitCommits
|
|
770
|
-
) {
|
|
771
|
-
break;
|
|
772
|
-
}
|
|
773
|
-
yield commitFrame(
|
|
774
|
-
sub.table,
|
|
775
|
-
commit,
|
|
776
|
-
compileSchema(ctx.schema).tables.get(sub.table)!,
|
|
777
|
-
plan.table,
|
|
778
|
-
);
|
|
779
|
-
delivered += commit.changes.length;
|
|
780
|
-
deliveredCommits += 1;
|
|
781
|
-
lastDeliveredSeq = commit.commitSeq;
|
|
782
|
-
if (delivered >= limits.limitCommits) break;
|
|
783
|
-
}
|
|
784
|
-
const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
|
|
785
|
-
// The window is proven exhausted only when the storage lookahead came
|
|
786
|
-
// back under budget (it scanned to `throughSeq`) and every returned
|
|
787
|
-
// commit was delivered.
|
|
788
|
-
const exhausted =
|
|
789
|
-
totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
|
|
790
|
-
// §4.5: the cursor advances even when no matching changes exist; when
|
|
791
|
-
// the change limit truncated the window it stops at the last fully
|
|
792
|
-
// delivered commit.
|
|
793
|
-
const nextCursor = exhausted
|
|
794
|
-
? Math.max(sub.cursor, maxSeq)
|
|
795
|
-
: Math.max(sub.cursor, lastDeliveredSeq);
|
|
796
|
-
yield { type: 'SUB_END', nextCursor };
|
|
797
|
-
return { nextCursor, active: true };
|
|
798
779
|
}
|
|
799
780
|
|
|
800
781
|
export interface PublishSqliteImageInput {
|
|
@@ -802,12 +783,15 @@ export interface PublishSqliteImageInput {
|
|
|
802
783
|
readonly config: SyncServerConfig;
|
|
803
784
|
readonly partition: string;
|
|
804
785
|
readonly table: string;
|
|
786
|
+
/** The unit whose clients bootstrap the table (§4.1). */
|
|
787
|
+
readonly scopes: UnitScopes;
|
|
805
788
|
/**
|
|
806
|
-
* The
|
|
807
|
-
*
|
|
808
|
-
*
|
|
789
|
+
* The allowed scopes of the actors that register the unit (§3.2). The
|
|
790
|
+
* reuse key carries the digest of the pair's effective scopes, so a
|
|
791
|
+
* wildcard fence on the table changes the image. Default: `'*'` for every
|
|
792
|
+
* declared variable, which leaves the unit's own scope map.
|
|
809
793
|
*/
|
|
810
|
-
readonly
|
|
794
|
+
readonly allowed?: ScopeMap;
|
|
811
795
|
}
|
|
812
796
|
|
|
813
797
|
export interface PublishedSqliteImage {
|
|
@@ -822,8 +806,8 @@ export interface PublishedSqliteImage {
|
|
|
822
806
|
}
|
|
823
807
|
|
|
824
808
|
/**
|
|
825
|
-
* Publish the §5.3 sqlite image that a pull
|
|
826
|
-
*
|
|
809
|
+
* Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
|
|
810
|
+
* from a host with a SQLite engine (a Bun process with the production
|
|
827
811
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
828
812
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
829
813
|
* serving host without a builder finds. The image stays current until a
|
|
@@ -838,9 +822,19 @@ export async function publishSqliteImage(
|
|
|
838
822
|
throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
|
|
839
823
|
}
|
|
840
824
|
const schema = compileSchema(config.schema);
|
|
841
|
-
const table = schema.
|
|
825
|
+
const table = coveredTables(schema, input.scopes).find(
|
|
826
|
+
(candidate) => candidate.name === input.table,
|
|
827
|
+
);
|
|
842
828
|
if (table === undefined) {
|
|
843
|
-
throw new Error('publishSqliteImage: the
|
|
829
|
+
throw new Error('publishSqliteImage: the unit does not cover the table');
|
|
830
|
+
}
|
|
831
|
+
const allowed =
|
|
832
|
+
input.allowed ??
|
|
833
|
+
Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
|
|
834
|
+
const unit = planUnit(input.scopes, [table], { ok: true, allowed });
|
|
835
|
+
const effective = unit.pairs.get(table.name);
|
|
836
|
+
if (effective === undefined) {
|
|
837
|
+
throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
|
|
844
838
|
}
|
|
845
839
|
await config.storage.ensureSchema(schema, config.checkpoints);
|
|
846
840
|
const logEpoch = await config.storage.getPartitionLogEpoch(partition);
|
|
@@ -852,18 +846,6 @@ export async function publishSqliteImage(
|
|
|
852
846
|
partition,
|
|
853
847
|
actorId: 'syncular.image-publisher',
|
|
854
848
|
};
|
|
855
|
-
const plan: SubscriptionPlan = {
|
|
856
|
-
frame: {
|
|
857
|
-
type: 'SUBSCRIPTION',
|
|
858
|
-
id: 'image',
|
|
859
|
-
table: input.table,
|
|
860
|
-
scopes: input.scopes,
|
|
861
|
-
cursor: -1,
|
|
862
|
-
},
|
|
863
|
-
table,
|
|
864
|
-
status: 'active',
|
|
865
|
-
effective: input.scopes,
|
|
866
|
-
};
|
|
867
849
|
// One-row pages make every table above one row image-eligible (§5.3).
|
|
868
850
|
const limits: PullLimits = {
|
|
869
851
|
limitCommits: 1,
|
|
@@ -874,24 +856,22 @@ export async function publishSqliteImage(
|
|
|
874
856
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
875
857
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
876
858
|
const trace: PullSectionTrace = { segments: [] };
|
|
877
|
-
const
|
|
859
|
+
const plan = planBootstrap(
|
|
878
860
|
ctx,
|
|
879
861
|
schema,
|
|
880
862
|
limits,
|
|
881
|
-
|
|
863
|
+
unit,
|
|
864
|
+
undefined,
|
|
882
865
|
maxSeq,
|
|
883
866
|
horizonSeq,
|
|
884
867
|
logEpoch,
|
|
885
868
|
);
|
|
886
|
-
if (prepared === undefined) throw new Error('prepareSections lost a plan');
|
|
887
869
|
let ref: SegmentRefFrame | undefined;
|
|
888
|
-
for await (const frame of
|
|
870
|
+
for await (const frame of bootstrapSegments(
|
|
889
871
|
ctx,
|
|
890
872
|
schema,
|
|
891
873
|
limits,
|
|
892
874
|
plan,
|
|
893
|
-
maxSeq,
|
|
894
|
-
prepared,
|
|
895
875
|
trace,
|
|
896
876
|
logEpoch,
|
|
897
877
|
)) {
|