@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/pull.ts
CHANGED
|
@@ -1,26 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Pull:
|
|
3
|
-
* (SPEC.md §4, §5).
|
|
2
|
+
* Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
|
|
4
3
|
*/
|
|
5
4
|
import {
|
|
6
5
|
type CommitChange,
|
|
6
|
+
type CommitFrame,
|
|
7
7
|
encodeRowsSegment,
|
|
8
|
+
type FencedRecord,
|
|
8
9
|
type PullHeaderFrame,
|
|
9
10
|
type ResponseFrame,
|
|
10
11
|
type ScopeMap,
|
|
11
12
|
type SegmentRefFrame,
|
|
12
13
|
type SegmentRow,
|
|
13
|
-
type
|
|
14
|
+
type UnitScopes,
|
|
15
|
+
unitId,
|
|
14
16
|
} from '@syncular/core';
|
|
15
17
|
import type { SyncRequestContext, SyncServerConfig } from './context';
|
|
16
18
|
import { clockOf, limitsOf } from './context';
|
|
19
|
+
import { syncError } from './errors';
|
|
17
20
|
import type { PullSegmentSummary } from './events';
|
|
18
21
|
import {
|
|
19
22
|
type CompiledSchema,
|
|
20
23
|
type CompiledTable,
|
|
21
24
|
compileSchema,
|
|
22
25
|
} from './schema';
|
|
23
|
-
import { scopeDigest } from './scopes';
|
|
26
|
+
import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
|
|
24
27
|
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
25
28
|
import { issueSegmentUrl } from './signed-url';
|
|
26
29
|
import type { ServerStorage, StoredCommit, StoredRow } from './storage';
|
|
@@ -50,7 +53,7 @@ function clamp(value: number, min: number, max: number, dflt: number): number {
|
|
|
50
53
|
return Math.min(max, Math.max(min, value));
|
|
51
54
|
}
|
|
52
55
|
|
|
53
|
-
/** §4.2 defaults and silent clamps
|
|
56
|
+
/** §4.2 defaults and silent clamps. */
|
|
54
57
|
export function clampPullLimits(header: PullHeaderFrame): PullLimits {
|
|
55
58
|
return {
|
|
56
59
|
limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
|
|
@@ -60,16 +63,140 @@ export function clampPullLimits(header: PullHeaderFrame): PullLimits {
|
|
|
60
63
|
};
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
readonly
|
|
66
|
+
/** One unit of the effective list, resolved against the request (§3.2, §4.1). */
|
|
67
|
+
export interface UnitPlan {
|
|
68
|
+
readonly id: string;
|
|
69
|
+
readonly scopes: UnitScopes;
|
|
70
|
+
/** Covered tables in bootstrap order (§4.1, §4.7). */
|
|
71
|
+
readonly tables: readonly CompiledTable[];
|
|
72
|
+
/** Active (unit, table) pairs: table → effective scopes (§3.2). */
|
|
73
|
+
readonly pairs: ReadonlyMap<string, ScopeMap>;
|
|
66
74
|
readonly status: 'active' | 'revoked';
|
|
67
|
-
|
|
75
|
+
/** The echo (§4.4): ascending table order, empty when revoked. */
|
|
76
|
+
readonly fenced: readonly FencedRecord[];
|
|
68
77
|
}
|
|
69
78
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
79
|
+
/**
|
|
80
|
+
* §4.1 unit validation against one schema version: the covered tables, or
|
|
81
|
+
* `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
|
|
82
|
+
* variable no table declares, or no covered table.
|
|
83
|
+
*/
|
|
84
|
+
export function coveredTables(
|
|
85
|
+
schema: CompiledSchema,
|
|
86
|
+
scopes: UnitScopes,
|
|
87
|
+
): CompiledTable[] {
|
|
88
|
+
const keys = Object.keys(scopes);
|
|
89
|
+
const reason =
|
|
90
|
+
keys.length === 0
|
|
91
|
+
? 'empty_scope_map'
|
|
92
|
+
: keys.some((key) => scopes[key] === '*')
|
|
93
|
+
? 'wildcard_value'
|
|
94
|
+
: keys.some((key) => !schema.declaredVariables.has(key))
|
|
95
|
+
? 'undeclared_variable'
|
|
96
|
+
: undefined;
|
|
97
|
+
const tables =
|
|
98
|
+
reason === undefined
|
|
99
|
+
? [...schema.tables.values()].filter((table) =>
|
|
100
|
+
keys.every((key) => table.declaredVariables.has(key)),
|
|
101
|
+
)
|
|
102
|
+
: [];
|
|
103
|
+
if (tables.length === 0) {
|
|
104
|
+
throw syncError(
|
|
105
|
+
'sync.invalid_subscription',
|
|
106
|
+
'the server cannot serve this unit (§4.1)',
|
|
107
|
+
JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }),
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return tables;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** §3.2 per covered table: the unit's active pairs, status and echo. */
|
|
114
|
+
export function planUnit(
|
|
115
|
+
scopes: UnitScopes,
|
|
116
|
+
tables: readonly CompiledTable[],
|
|
117
|
+
resolved: ResolvedScopes,
|
|
118
|
+
): UnitPlan {
|
|
119
|
+
const requested: ScopeMap = {};
|
|
120
|
+
for (const [key, value] of Object.entries(scopes)) requested[key] = [value];
|
|
121
|
+
const pairs = new Map<string, ScopeMap>();
|
|
122
|
+
const fenced: FencedRecord[] = [];
|
|
123
|
+
for (const table of tables) {
|
|
124
|
+
const outcome = computeEffective(
|
|
125
|
+
table.declaredVariables,
|
|
126
|
+
requested,
|
|
127
|
+
resolved,
|
|
128
|
+
);
|
|
129
|
+
if (outcome.status === 'revoked') {
|
|
130
|
+
fenced.push({ table: table.name, status: 'revoked', scopes: {} });
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
pairs.set(table.name, outcome.effective);
|
|
134
|
+
const added = Object.entries(outcome.effective).filter(
|
|
135
|
+
([key]) => !Object.hasOwn(scopes, key),
|
|
136
|
+
);
|
|
137
|
+
if (added.length > 0) {
|
|
138
|
+
fenced.push({
|
|
139
|
+
table: table.name,
|
|
140
|
+
status: 'active',
|
|
141
|
+
scopes: Object.fromEntries(added),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const active = pairs.size > 0;
|
|
146
|
+
return {
|
|
147
|
+
id: unitId(scopes),
|
|
148
|
+
scopes,
|
|
149
|
+
tables,
|
|
150
|
+
pairs,
|
|
151
|
+
status: active ? 'active' : 'revoked',
|
|
152
|
+
fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* The `COMMIT` frame of one live-section commit (§4.5): its frame-local
|
|
158
|
+
* table dictionary and every change, projected onto the client's codec.
|
|
159
|
+
*/
|
|
160
|
+
export function commitFrame(
|
|
161
|
+
commit: StoredCommit,
|
|
162
|
+
current: CompiledSchema,
|
|
163
|
+
target: CompiledSchema,
|
|
164
|
+
): CommitFrame {
|
|
165
|
+
const tables: string[] = [];
|
|
166
|
+
const changes: CommitChange[] = commit.changes.map((change) => {
|
|
167
|
+
// §4.5: apply is version-guarded, so every change carries its version:
|
|
168
|
+
// an upsert the row's version after the write, a delete the version its
|
|
169
|
+
// tombstone records. A change without one cannot be served.
|
|
170
|
+
if (change.rowVersion === undefined) {
|
|
171
|
+
throw new Error('a change record carries no row version (§4.5)');
|
|
172
|
+
}
|
|
173
|
+
let tableIndex = tables.indexOf(change.table);
|
|
174
|
+
if (tableIndex === -1) tableIndex = tables.push(change.table) - 1;
|
|
175
|
+
return {
|
|
176
|
+
tableIndex,
|
|
177
|
+
rowId: change.rowId,
|
|
178
|
+
op: change.op,
|
|
179
|
+
rowVersion: change.rowVersion,
|
|
180
|
+
scopes: change.scopes,
|
|
181
|
+
...(change.payload !== undefined
|
|
182
|
+
? {
|
|
183
|
+
row: projectRowPayload(
|
|
184
|
+
current.tables.get(change.table)!,
|
|
185
|
+
target.tables.get(change.table)!,
|
|
186
|
+
change.payload,
|
|
187
|
+
),
|
|
188
|
+
}
|
|
189
|
+
: {}),
|
|
190
|
+
};
|
|
191
|
+
});
|
|
192
|
+
return {
|
|
193
|
+
type: 'COMMIT',
|
|
194
|
+
commitSeq: commit.commitSeq,
|
|
195
|
+
createdAtMs: commit.createdAtMs,
|
|
196
|
+
actorId: commit.actorId,
|
|
197
|
+
tables,
|
|
198
|
+
changes,
|
|
199
|
+
};
|
|
73
200
|
}
|
|
74
201
|
|
|
75
202
|
/**
|
|
@@ -88,19 +215,33 @@ interface BootstrapToken {
|
|
|
88
215
|
rowCursor: string | null;
|
|
89
216
|
}
|
|
90
217
|
|
|
218
|
+
/**
|
|
219
|
+
* §4.7: a usable resume token, or `undefined`. A token whose pin is not a
|
|
220
|
+
* commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
|
|
221
|
+
* differ from the unit's covered tables in bootstrap order, or whose position
|
|
222
|
+
* names none of them is unusable.
|
|
223
|
+
*/
|
|
91
224
|
function parseBootstrapToken(
|
|
92
225
|
raw: string | undefined,
|
|
93
|
-
|
|
226
|
+
tables: readonly CompiledTable[],
|
|
227
|
+
maxSeq: number,
|
|
94
228
|
): BootstrapToken | undefined {
|
|
95
229
|
if (raw === undefined) return undefined;
|
|
96
230
|
try {
|
|
97
231
|
const parsed = JSON.parse(raw) as Partial<BootstrapToken>;
|
|
98
232
|
if (
|
|
99
|
-
|
|
233
|
+
!Number.isSafeInteger(parsed.asOfCommitSeq) ||
|
|
234
|
+
parsed.asOfCommitSeq === undefined ||
|
|
235
|
+
parsed.asOfCommitSeq < 0 ||
|
|
236
|
+
parsed.asOfCommitSeq > maxSeq ||
|
|
100
237
|
!Array.isArray(parsed.tables) ||
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
parsed.
|
|
238
|
+
parsed.tables.length !== tables.length ||
|
|
239
|
+
parsed.tables.some((name, i) => name !== tables[i]?.name) ||
|
|
240
|
+
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
241
|
+
parsed.tableIndex === undefined ||
|
|
242
|
+
parsed.tableIndex < 0 ||
|
|
243
|
+
parsed.tableIndex >= tables.length ||
|
|
244
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
104
245
|
) {
|
|
105
246
|
return undefined;
|
|
106
247
|
}
|
|
@@ -115,34 +256,6 @@ function parseBootstrapToken(
|
|
|
115
256
|
}
|
|
116
257
|
}
|
|
117
258
|
|
|
118
|
-
function commitFrame(
|
|
119
|
-
table: string,
|
|
120
|
-
commit: StoredCommit,
|
|
121
|
-
current: CompiledTable,
|
|
122
|
-
target: CompiledTable,
|
|
123
|
-
): ResponseFrame {
|
|
124
|
-
const changes: CommitChange[] = commit.changes.map((change) => ({
|
|
125
|
-
tableIndex: 0,
|
|
126
|
-
rowId: change.rowId,
|
|
127
|
-
op: change.op,
|
|
128
|
-
...(change.rowVersion !== undefined
|
|
129
|
-
? { rowVersion: change.rowVersion }
|
|
130
|
-
: {}),
|
|
131
|
-
scopes: change.scopes,
|
|
132
|
-
...(change.payload !== undefined
|
|
133
|
-
? { row: projectRowPayload(current, target, change.payload) }
|
|
134
|
-
: {}),
|
|
135
|
-
}));
|
|
136
|
-
return {
|
|
137
|
-
type: 'COMMIT',
|
|
138
|
-
commitSeq: commit.commitSeq,
|
|
139
|
-
createdAtMs: commit.createdAtMs,
|
|
140
|
-
actorId: commit.actorId,
|
|
141
|
-
tables: [table],
|
|
142
|
-
changes,
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
|
|
146
259
|
function chunkRows(rows: SegmentRow[], size: number): SegmentRow[][] {
|
|
147
260
|
if (rows.length === 0) return [];
|
|
148
261
|
const blocks: SegmentRow[][] = [];
|
|
@@ -182,10 +295,10 @@ function segmentRefFrame(
|
|
|
182
295
|
byteLength: record.byteLength,
|
|
183
296
|
rowCount: record.rowCount,
|
|
184
297
|
asOfCommitSeq: record.asOfCommitSeq,
|
|
185
|
-
// The frame reports the digest THIS
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
//
|
|
298
|
+
// The frame reports the digest THIS pair is authorized under. A stored
|
|
299
|
+
// entry can carry several digests when identical content was published
|
|
300
|
+
// under different scopes (§5.1), and the record's primary digest is not
|
|
301
|
+
// necessarily the caller's (§5.5).
|
|
189
302
|
scopeDigest,
|
|
190
303
|
...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
|
|
191
304
|
...(record.nextRowCursor !== null
|
|
@@ -198,6 +311,12 @@ function segmentRefFrame(
|
|
|
198
311
|
};
|
|
199
312
|
}
|
|
200
313
|
|
|
314
|
+
/** One active (unit, table) pair of a bootstrap (§3.2). */
|
|
315
|
+
interface BootstrapPair {
|
|
316
|
+
readonly table: CompiledTable;
|
|
317
|
+
readonly effective: ScopeMap;
|
|
318
|
+
}
|
|
319
|
+
|
|
201
320
|
/**
|
|
202
321
|
* §5.3: the sqlite-image lane — whole-table, chosen only at the start of a
|
|
203
322
|
* table (a mid-table resume stays on the rows lane, never switching lanes),
|
|
@@ -208,30 +327,96 @@ function segmentRefFrame(
|
|
|
208
327
|
*/
|
|
209
328
|
function sqliteLaneEligible(
|
|
210
329
|
limits: PullLimits,
|
|
211
|
-
|
|
330
|
+
table: CompiledTable,
|
|
212
331
|
startRowCursor: string | null,
|
|
213
332
|
): boolean {
|
|
214
333
|
return (
|
|
215
334
|
(limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
216
335
|
startRowCursor === null &&
|
|
217
|
-
|
|
336
|
+
table.encryptedColumnIndices.length === 0
|
|
218
337
|
);
|
|
219
338
|
}
|
|
220
339
|
|
|
340
|
+
/** The reads a table's bootstrap needs before its first frame (§4.7, §5.3). */
|
|
341
|
+
interface TableStart {
|
|
342
|
+
/**
|
|
343
|
+
* `limitSnapshotRows + 1` rows from the start cursor: the first page and
|
|
344
|
+
* the §5.3 image eligibility probe.
|
|
345
|
+
*/
|
|
346
|
+
readonly firstPage: StoredRow[];
|
|
347
|
+
/** On the sqlite lane: the §4.7 scope pin and a stored image under it. */
|
|
348
|
+
readonly image?: {
|
|
349
|
+
readonly scopePin: number;
|
|
350
|
+
readonly stored: SegmentRecord | undefined;
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
async function readTableStart(
|
|
355
|
+
ctx: SyncRequestContext,
|
|
356
|
+
schema: CompiledSchema,
|
|
357
|
+
limits: PullLimits,
|
|
358
|
+
unit: UnitPlan,
|
|
359
|
+
pair: BootstrapPair,
|
|
360
|
+
startRowCursor: string | null,
|
|
361
|
+
maxSeq: number,
|
|
362
|
+
horizonSeq: number,
|
|
363
|
+
logEpoch: string,
|
|
364
|
+
): Promise<TableStart> {
|
|
365
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
366
|
+
table: pair.table.name,
|
|
367
|
+
scopeFilter: pair.effective,
|
|
368
|
+
afterRowId: startRowCursor,
|
|
369
|
+
limit: limits.limitSnapshotRows + 1,
|
|
370
|
+
});
|
|
371
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
372
|
+
return { firstPage: await scan };
|
|
373
|
+
}
|
|
374
|
+
// §4.7: a sqlite-lane table pins at the newest change carrying the unit's
|
|
375
|
+
// index key (never below the horizon, never above maxCommitSeq), so a
|
|
376
|
+
// stored image stays the reuse key's match until this scope changes
|
|
377
|
+
// (§5.3). The server computes the pin; a client token never supplies it.
|
|
378
|
+
const [firstPage, latest] = await Promise.all([
|
|
379
|
+
scan,
|
|
380
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
381
|
+
table: pair.table.name,
|
|
382
|
+
scopes: unit.scopes,
|
|
383
|
+
}),
|
|
384
|
+
]);
|
|
385
|
+
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
386
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than one
|
|
387
|
+
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
388
|
+
// on a small table would be a wasted round trip.
|
|
389
|
+
const stored =
|
|
390
|
+
firstPage.length <= limits.limitSnapshotRows
|
|
391
|
+
? undefined
|
|
392
|
+
: await ctx.segments.find(
|
|
393
|
+
{
|
|
394
|
+
partition: ctx.partition,
|
|
395
|
+
logEpoch,
|
|
396
|
+
table: pair.table.name,
|
|
397
|
+
schemaVersion: schema.version,
|
|
398
|
+
mediaType: 'sqlite',
|
|
399
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
400
|
+
asOfCommitSeq: scopePin,
|
|
401
|
+
},
|
|
402
|
+
clockOf(ctx)(),
|
|
403
|
+
);
|
|
404
|
+
return { firstPage, image: { scopePin, stored } };
|
|
405
|
+
}
|
|
406
|
+
|
|
221
407
|
/**
|
|
222
|
-
* §5.3 sqlite-image lane: only
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
* (the rows lane takes over).
|
|
408
|
+
* §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
|
|
409
|
+
* Reuses an unexpired stored image for the same (partition, table,
|
|
410
|
+
* schemaVersion, scope digest, scope pin) instead of rebuilding — the
|
|
411
|
+
* bootstrap-storm rule. Returns false when no image is served (the rows lane
|
|
412
|
+
* takes over).
|
|
228
413
|
*/
|
|
229
414
|
async function* sqliteImageSegment(
|
|
230
415
|
ctx: SyncRequestContext,
|
|
231
416
|
schema: CompiledSchema,
|
|
232
417
|
limits: PullLimits,
|
|
233
|
-
|
|
234
|
-
|
|
418
|
+
pair: BootstrapPair,
|
|
419
|
+
scopePin: number,
|
|
235
420
|
digest: string,
|
|
236
421
|
trace: PullSectionTrace | undefined,
|
|
237
422
|
logEpoch: string,
|
|
@@ -255,9 +440,6 @@ async function* sqliteImageSegment(
|
|
|
255
440
|
);
|
|
256
441
|
return true;
|
|
257
442
|
}
|
|
258
|
-
// Eligibility probe (§5.3): image only when the snapshot exceeds one
|
|
259
|
-
// rows page — smaller tables stay on the (typically inline) rows lane.
|
|
260
|
-
// The probe is the rows lane's first page, read by `prepareSections`.
|
|
261
443
|
if (probe.length <= limits.limitSnapshotRows) return false;
|
|
262
444
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
263
445
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
@@ -276,12 +458,13 @@ async function* sqliteImageSegment(
|
|
|
276
458
|
const identity = {
|
|
277
459
|
partition,
|
|
278
460
|
logEpoch,
|
|
279
|
-
table:
|
|
461
|
+
table: pair.table.name,
|
|
280
462
|
schemaVersion: schema.version,
|
|
281
463
|
mediaType: 'sqlite' as const,
|
|
282
464
|
scopeDigest: digest,
|
|
283
|
-
asOfCommitSeq:
|
|
465
|
+
asOfCommitSeq: scopePin,
|
|
284
466
|
};
|
|
467
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
285
468
|
const key = JSON.stringify(identity);
|
|
286
469
|
let building = builds.get(key);
|
|
287
470
|
let builtHere = false;
|
|
@@ -293,36 +476,28 @@ async function* sqliteImageSegment(
|
|
|
293
476
|
builtHere = true;
|
|
294
477
|
let rowCount = 0;
|
|
295
478
|
const bytes = await buildImage({
|
|
296
|
-
table:
|
|
479
|
+
table: pair.table,
|
|
297
480
|
schemaVersion: schema.version,
|
|
298
|
-
asOfCommitSeq:
|
|
481
|
+
asOfCommitSeq: scopePin,
|
|
299
482
|
scopeDigest: digest,
|
|
300
483
|
rowBatches: (async function* () {
|
|
301
484
|
rowCount += probe.length;
|
|
302
485
|
yield probe.map((row) => ({
|
|
303
486
|
...row,
|
|
304
|
-
payload: projectRowPayload(
|
|
305
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
306
|
-
plan.table,
|
|
307
|
-
row.payload,
|
|
308
|
-
),
|
|
487
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
309
488
|
}));
|
|
310
489
|
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
311
490
|
for (;;) {
|
|
312
491
|
const rows = await storage.scanRows(partition, {
|
|
313
|
-
table:
|
|
314
|
-
scopeFilter:
|
|
492
|
+
table: pair.table.name,
|
|
493
|
+
scopeFilter: pair.effective,
|
|
315
494
|
afterRowId,
|
|
316
495
|
limit: 5_000,
|
|
317
496
|
});
|
|
318
497
|
rowCount += rows.length;
|
|
319
498
|
yield rows.map((row) => ({
|
|
320
499
|
...row,
|
|
321
|
-
payload: projectRowPayload(
|
|
322
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
323
|
-
plan.table,
|
|
324
|
-
row.payload,
|
|
325
|
-
),
|
|
500
|
+
payload: projectRowPayload(current, pair.table, row.payload),
|
|
326
501
|
}));
|
|
327
502
|
const last = rows[rows.length - 1];
|
|
328
503
|
if (rows.length < 5_000 || last === undefined) break;
|
|
@@ -365,48 +540,50 @@ async function* sqliteImageSegment(
|
|
|
365
540
|
return true;
|
|
366
541
|
}
|
|
367
542
|
|
|
368
|
-
|
|
543
|
+
/**
|
|
544
|
+
* One table's segments within the page budget. Returns the pages it used
|
|
545
|
+
* and the row cursor to resume at, `null` once the table is exhausted.
|
|
546
|
+
*/
|
|
547
|
+
async function* tableSegments(
|
|
369
548
|
ctx: SyncRequestContext,
|
|
370
549
|
schema: CompiledSchema,
|
|
371
550
|
limits: PullLimits,
|
|
372
|
-
|
|
373
|
-
|
|
551
|
+
pair: BootstrapPair,
|
|
552
|
+
pin: number,
|
|
374
553
|
startRowCursor: string | null,
|
|
554
|
+
start: TableStart,
|
|
555
|
+
pageBudget: number,
|
|
375
556
|
trace: PullSectionTrace | undefined,
|
|
376
557
|
logEpoch: string,
|
|
377
|
-
|
|
378
|
-
image: SegmentRecord | undefined,
|
|
379
|
-
): AsyncGenerator<
|
|
380
|
-
ResponseFrame,
|
|
381
|
-
{ complete: boolean; rowCursor: string | null }
|
|
382
|
-
> {
|
|
558
|
+
): AsyncGenerator<ResponseFrame, { pages: number; rowCursor: string | null }> {
|
|
383
559
|
const { storage, segments, partition } = ctx;
|
|
384
560
|
const serverLimits = limitsOf(ctx);
|
|
385
|
-
const digest = await scopeDigest(
|
|
561
|
+
const digest = await scopeDigest(pair.effective);
|
|
386
562
|
const now = clockOf(ctx)();
|
|
387
|
-
if (
|
|
563
|
+
if (start.image !== undefined) {
|
|
388
564
|
const imaged = yield* sqliteImageSegment(
|
|
389
565
|
ctx,
|
|
390
566
|
schema,
|
|
391
567
|
limits,
|
|
392
|
-
|
|
393
|
-
|
|
568
|
+
pair,
|
|
569
|
+
start.image.scopePin,
|
|
394
570
|
digest,
|
|
395
571
|
trace,
|
|
396
572
|
logEpoch,
|
|
397
|
-
image,
|
|
398
|
-
firstPage,
|
|
573
|
+
start.image.stored,
|
|
574
|
+
start.firstPage,
|
|
399
575
|
);
|
|
400
|
-
if (imaged) return {
|
|
576
|
+
if (imaged) return { pages: 1, rowCursor: null };
|
|
401
577
|
}
|
|
578
|
+
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
402
579
|
let rowCursor = startRowCursor;
|
|
403
|
-
for (let page = 0; page <
|
|
580
|
+
for (let page = 0; page < pageBudget; page++) {
|
|
404
581
|
const scanned =
|
|
405
582
|
page === 0
|
|
406
|
-
? firstPage
|
|
583
|
+
? start.firstPage
|
|
407
584
|
: await storage.scanRows(partition, {
|
|
408
|
-
table:
|
|
409
|
-
scopeFilter:
|
|
585
|
+
table: pair.table.name,
|
|
586
|
+
scopeFilter: pair.effective,
|
|
410
587
|
afterRowId: rowCursor,
|
|
411
588
|
limit: limits.limitSnapshotRows + 1,
|
|
412
589
|
});
|
|
@@ -415,16 +592,12 @@ async function* bootstrapSegments(
|
|
|
415
592
|
// §5.2: every row record carries the row's current server_version.
|
|
416
593
|
const decoded = pageRows.map((row: StoredRow) => ({
|
|
417
594
|
serverVersion: row.serverVersion,
|
|
418
|
-
values: projectRowValues(
|
|
419
|
-
compileSchema(ctx.schema).tables.get(plan.table.name)!,
|
|
420
|
-
plan.table,
|
|
421
|
-
row.payload,
|
|
422
|
-
),
|
|
595
|
+
values: projectRowValues(current, pair.table, row.payload),
|
|
423
596
|
}));
|
|
424
597
|
const bytes = encodeRowsSegment({
|
|
425
|
-
table:
|
|
598
|
+
table: pair.table.name,
|
|
426
599
|
schemaVersion: schema.version,
|
|
427
|
-
columns:
|
|
600
|
+
columns: pair.table.columns,
|
|
428
601
|
blocks: chunkRows(decoded, 1000),
|
|
429
602
|
});
|
|
430
603
|
const lastRow = pageRows[pageRows.length - 1];
|
|
@@ -450,11 +623,11 @@ async function* bootstrapSegments(
|
|
|
450
623
|
{
|
|
451
624
|
partition,
|
|
452
625
|
logEpoch,
|
|
453
|
-
table:
|
|
626
|
+
table: pair.table.name,
|
|
454
627
|
schemaVersion: schema.version,
|
|
455
628
|
mediaType: 'rows',
|
|
456
629
|
scopeDigest: digest,
|
|
457
|
-
asOfCommitSeq:
|
|
630
|
+
asOfCommitSeq: pin,
|
|
458
631
|
rowCount: pageRows.length,
|
|
459
632
|
rowCursor,
|
|
460
633
|
nextRowCursor,
|
|
@@ -475,340 +648,134 @@ async function* bootstrapSegments(
|
|
|
475
648
|
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
476
649
|
);
|
|
477
650
|
}
|
|
478
|
-
if (!hasMore) return {
|
|
651
|
+
if (!hasMore) return { pages: page + 1, rowCursor: null };
|
|
479
652
|
rowCursor = nextRowCursor;
|
|
480
653
|
}
|
|
481
|
-
return {
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
/**
|
|
485
|
-
* The storage reads a section needs before its first frame. A pull starts
|
|
486
|
-
* them for every subscription before awaiting any, so a storage that batches
|
|
487
|
-
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
488
|
-
* subscription in one statement, and the horizon re-check runs once per pull.
|
|
489
|
-
*/
|
|
490
|
-
export interface PreparedSection {
|
|
491
|
-
/** §4.5 window read for an incremental cursor; empty when none was read. */
|
|
492
|
-
readonly commits: StoredCommit[];
|
|
493
|
-
/** Horizon to judge the cursor against, re-read after any window read. */
|
|
494
|
-
readonly horizonSeq: number;
|
|
495
|
-
/**
|
|
496
|
-
* A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
|
|
497
|
-
* resume cursor), which is also the §5.3 image eligibility probe.
|
|
498
|
-
*/
|
|
499
|
-
readonly firstPage: StoredRow[] | undefined;
|
|
500
|
-
/** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
|
|
501
|
-
readonly pin: number | undefined;
|
|
502
|
-
/**
|
|
503
|
-
* §5.3 stored image for a sqlite-lane bootstrap's reuse key. Read only when
|
|
504
|
-
* the eligibility probe shows a snapshot larger than one rows page, since a
|
|
505
|
-
* smaller table can never match a stored image.
|
|
506
|
-
*/
|
|
507
|
-
readonly image: SegmentRecord | undefined;
|
|
654
|
+
return { pages: pageBudget, rowCursor };
|
|
508
655
|
}
|
|
509
656
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
readonly
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
/** Run the pre-frame storage reads of every subscription concurrently. */
|
|
520
|
-
export async function prepareSections(
|
|
521
|
-
ctx: SyncRequestContext,
|
|
522
|
-
schema: CompiledSchema,
|
|
523
|
-
limits: PullLimits,
|
|
524
|
-
plans: readonly SubscriptionPlan[],
|
|
525
|
-
maxSeq: number,
|
|
526
|
-
horizonSeq: number,
|
|
527
|
-
logEpoch: string,
|
|
528
|
-
): Promise<PreparedSection[]> {
|
|
529
|
-
const reads = plans.map((plan): PendingSection => {
|
|
530
|
-
const sub = plan.frame;
|
|
531
|
-
if (plan.status === 'revoked') return {};
|
|
532
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
533
|
-
if (
|
|
534
|
-
token === undefined &&
|
|
535
|
-
sub.cursor >= 0 &&
|
|
536
|
-
sub.cursor >= horizonSeq &&
|
|
537
|
-
sub.cursor <= maxSeq
|
|
538
|
-
) {
|
|
539
|
-
return {
|
|
540
|
-
commits: ctx.storage.readCommitWindow(ctx.partition, {
|
|
541
|
-
table: sub.table,
|
|
542
|
-
scopeFilter: plan.effective,
|
|
543
|
-
afterSeq: sub.cursor,
|
|
544
|
-
throughSeq: maxSeq,
|
|
545
|
-
limitChanges: limits.limitCommits + 1,
|
|
546
|
-
}),
|
|
547
|
-
};
|
|
548
|
-
}
|
|
549
|
-
// §4.6 reset: no reads.
|
|
550
|
-
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
551
|
-
return {};
|
|
552
|
-
}
|
|
553
|
-
// Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
|
|
554
|
-
const resume =
|
|
555
|
-
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
556
|
-
? token
|
|
557
|
-
: undefined;
|
|
558
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
559
|
-
const firstPage = ctx.storage.scanRows(ctx.partition, {
|
|
560
|
-
table: plan.table.name,
|
|
561
|
-
scopeFilter: plan.effective,
|
|
562
|
-
afterRowId: startCursor,
|
|
563
|
-
limit: limits.limitSnapshotRows + 1,
|
|
564
|
-
});
|
|
565
|
-
if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
|
|
566
|
-
// §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
|
|
567
|
-
// scope (never below the horizon, never above maxCommitSeq), so a stored
|
|
568
|
-
// image stays the reuse key's match until this scope changes (§5.3).
|
|
569
|
-
const pinned =
|
|
570
|
-
resume !== undefined
|
|
571
|
-
? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
|
|
572
|
-
: ctx.storage
|
|
573
|
-
.latestScopedChangeSeq(ctx.partition, {
|
|
574
|
-
table: plan.table.name,
|
|
575
|
-
scopeFilter: plan.effective,
|
|
576
|
-
})
|
|
577
|
-
.then((latest) => {
|
|
578
|
-
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
579
|
-
return { asOf: pin, pin };
|
|
580
|
-
});
|
|
581
|
-
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
582
|
-
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
583
|
-
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
584
|
-
// `find` on a small table would be a wasted round trip.
|
|
585
|
-
const probe = await firstPage;
|
|
586
|
-
if (probe.length <= limits.limitSnapshotRows) {
|
|
587
|
-
return { pin, image: undefined };
|
|
588
|
-
}
|
|
589
|
-
return {
|
|
590
|
-
pin,
|
|
591
|
-
image: await ctx.segments.find(
|
|
592
|
-
{
|
|
593
|
-
partition: ctx.partition,
|
|
594
|
-
logEpoch,
|
|
595
|
-
table: plan.table.name,
|
|
596
|
-
schemaVersion: schema.version,
|
|
597
|
-
mediaType: 'sqlite',
|
|
598
|
-
scopeDigest: await scopeDigest(plan.effective),
|
|
599
|
-
asOfCommitSeq: asOf,
|
|
600
|
-
},
|
|
601
|
-
clockOf(ctx)(),
|
|
602
|
-
),
|
|
603
|
-
};
|
|
604
|
-
});
|
|
605
|
-
return { firstPage, lane };
|
|
606
|
-
});
|
|
607
|
-
const settled = await Promise.all(
|
|
608
|
-
reads.map(async (read) => ({
|
|
609
|
-
commits: read.commits === undefined ? undefined : await read.commits,
|
|
610
|
-
firstPage: await read.firstPage,
|
|
611
|
-
lane: await read.lane,
|
|
612
|
-
})),
|
|
613
|
-
);
|
|
614
|
-
// Validate continuity before committing to an active section. A prune
|
|
615
|
-
// during a paged read can otherwise make an incomplete window look empty.
|
|
616
|
-
const windowHorizon = settled.some((read) => read.commits !== undefined)
|
|
617
|
-
? Math.max(horizonSeq, await ctx.storage.getHorizonSeq(ctx.partition))
|
|
618
|
-
: horizonSeq;
|
|
619
|
-
return settled.map((read) => ({
|
|
620
|
-
commits: read.commits ?? [],
|
|
621
|
-
horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
|
|
622
|
-
firstPage: read.firstPage,
|
|
623
|
-
pin: read.lane?.pin,
|
|
624
|
-
image: read.lane?.image,
|
|
625
|
-
}));
|
|
657
|
+
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
658
|
+
export interface BootstrapPlan {
|
|
659
|
+
readonly unit: UnitPlan;
|
|
660
|
+
/** The unit's pin: a usable resume token's, or `maxCommitSeq`. */
|
|
661
|
+
readonly pin: number;
|
|
662
|
+
readonly tableIndex: number;
|
|
663
|
+
readonly rowCursor: string | null;
|
|
664
|
+
/** Table index → its start reads. */
|
|
665
|
+
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
626
666
|
}
|
|
627
667
|
|
|
628
668
|
/**
|
|
629
|
-
*
|
|
630
|
-
*
|
|
669
|
+
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
670
|
+
* in this section: each visited table costs at least one page, so at most
|
|
671
|
+
* `maxSnapshotPages` tables. The reads of every planned unit start before any
|
|
672
|
+
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
673
|
+
* D1) answers them in one statement per table.
|
|
631
674
|
*/
|
|
632
|
-
export
|
|
675
|
+
export function planBootstrap(
|
|
633
676
|
ctx: SyncRequestContext,
|
|
634
677
|
schema: CompiledSchema,
|
|
635
678
|
limits: PullLimits,
|
|
636
|
-
|
|
679
|
+
unit: UnitPlan,
|
|
680
|
+
bootstrapState: string | undefined,
|
|
637
681
|
maxSeq: number,
|
|
638
682
|
horizonSeq: number,
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
683
|
+
logEpoch: string,
|
|
684
|
+
): BootstrapPlan {
|
|
685
|
+
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
686
|
+
// fell behind the horizon; then restart from a fresh pin.
|
|
687
|
+
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
688
|
+
const resume =
|
|
689
|
+
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
690
|
+
? token
|
|
691
|
+
: undefined;
|
|
692
|
+
const tableIndex = resume?.tableIndex ?? 0;
|
|
693
|
+
const rowCursor = resume?.rowCursor ?? null;
|
|
694
|
+
const starts = new Map<number, Promise<TableStart>>();
|
|
695
|
+
for (
|
|
696
|
+
let index = tableIndex;
|
|
697
|
+
index < unit.tables.length && starts.size < limits.maxSnapshotPages;
|
|
698
|
+
index++
|
|
699
|
+
) {
|
|
700
|
+
const table = unit.tables[index]!;
|
|
701
|
+
const effective = unit.pairs.get(table.name);
|
|
702
|
+
if (effective === undefined) continue;
|
|
703
|
+
const start = readTableStart(
|
|
704
|
+
ctx,
|
|
705
|
+
schema,
|
|
706
|
+
limits,
|
|
707
|
+
unit,
|
|
708
|
+
{ table, effective },
|
|
709
|
+
index === tableIndex ? rowCursor : null,
|
|
710
|
+
maxSeq,
|
|
711
|
+
horizonSeq,
|
|
712
|
+
logEpoch,
|
|
713
|
+
);
|
|
714
|
+
// A section that stops early never awaits the later starts.
|
|
715
|
+
start.catch(() => undefined);
|
|
716
|
+
starts.set(index, start);
|
|
644
717
|
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
logEpoch,
|
|
653
|
-
);
|
|
654
|
-
if (prepared === undefined) throw new Error('prepareSections lost a plan');
|
|
655
|
-
return yield* preparedSection(
|
|
656
|
-
ctx,
|
|
657
|
-
schema,
|
|
658
|
-
limits,
|
|
659
|
-
plan,
|
|
660
|
-
maxSeq,
|
|
661
|
-
prepared,
|
|
662
|
-
trace,
|
|
663
|
-
logEpoch,
|
|
664
|
-
);
|
|
718
|
+
return {
|
|
719
|
+
unit,
|
|
720
|
+
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
721
|
+
tableIndex,
|
|
722
|
+
rowCursor,
|
|
723
|
+
starts,
|
|
724
|
+
};
|
|
665
725
|
}
|
|
666
726
|
|
|
667
|
-
/**
|
|
668
|
-
|
|
727
|
+
/**
|
|
728
|
+
* The segments of one bootstrap section (§4.7): tables in bootstrap order,
|
|
729
|
+
* each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
|
|
730
|
+
* the resume token, or `undefined` once every table is exhausted. A pair the
|
|
731
|
+
* wildcard fence revoked receives no segment (§3.3).
|
|
732
|
+
*/
|
|
733
|
+
export async function* bootstrapSegments(
|
|
669
734
|
ctx: SyncRequestContext,
|
|
670
735
|
schema: CompiledSchema,
|
|
671
736
|
limits: PullLimits,
|
|
672
|
-
plan:
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
id: sub.id,
|
|
687
|
-
status: 'revoked',
|
|
688
|
-
reasonCode: 'sync.scope_revoked',
|
|
689
|
-
effectiveScopes: {},
|
|
690
|
-
bootstrap: false,
|
|
691
|
-
};
|
|
692
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
693
|
-
return { nextCursor: sub.cursor, active: false };
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
697
|
-
const { commits, horizonSeq } = prepared;
|
|
698
|
-
|
|
699
|
-
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
700
|
-
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
701
|
-
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
702
|
-
yield {
|
|
703
|
-
type: 'SUB_START',
|
|
704
|
-
id: sub.id,
|
|
705
|
-
status: 'reset',
|
|
706
|
-
reasonCode: 'sync.cursor_expired',
|
|
707
|
-
effectiveScopes: {},
|
|
708
|
-
bootstrap: false,
|
|
709
|
-
};
|
|
710
|
-
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
711
|
-
return { nextCursor: sub.cursor, active: false };
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
const bootstrapping =
|
|
715
|
-
token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
|
|
716
|
-
|
|
717
|
-
if (bootstrapping) {
|
|
718
|
-
// §4.7: resume at the pinned point unless the pin fell behind the
|
|
719
|
-
// horizon (or the token is unusable) — then restart from a fresh pin.
|
|
720
|
-
const resume =
|
|
721
|
-
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
722
|
-
? token
|
|
723
|
-
: undefined;
|
|
724
|
-
const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
|
|
725
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
726
|
-
if (prepared.firstPage === undefined) {
|
|
727
|
-
throw new Error('prepareSections did not read the bootstrap page');
|
|
737
|
+
plan: BootstrapPlan,
|
|
738
|
+
trace: PullSectionTrace | undefined,
|
|
739
|
+
logEpoch: string,
|
|
740
|
+
): AsyncGenerator<ResponseFrame, string | undefined> {
|
|
741
|
+
const { unit, pin } = plan;
|
|
742
|
+
let { tableIndex, rowCursor } = plan;
|
|
743
|
+
let pages = 0;
|
|
744
|
+
for (;;) {
|
|
745
|
+
while (
|
|
746
|
+
tableIndex < unit.tables.length &&
|
|
747
|
+
!unit.pairs.has(unit.tables[tableIndex]!.name)
|
|
748
|
+
) {
|
|
749
|
+
tableIndex += 1;
|
|
750
|
+
rowCursor = null;
|
|
728
751
|
}
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
752
|
+
if (tableIndex === unit.tables.length) return undefined;
|
|
753
|
+
if (pages === limits.maxSnapshotPages) {
|
|
754
|
+
const token: BootstrapToken = {
|
|
755
|
+
asOfCommitSeq: pin,
|
|
756
|
+
tables: unit.tables.map((table) => table.name),
|
|
757
|
+
tableIndex,
|
|
758
|
+
rowCursor,
|
|
759
|
+
};
|
|
760
|
+
return JSON.stringify(token);
|
|
761
|
+
}
|
|
762
|
+
const table = unit.tables[tableIndex]!;
|
|
763
|
+
const outcome = yield* tableSegments(
|
|
738
764
|
ctx,
|
|
739
765
|
schema,
|
|
740
766
|
limits,
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
767
|
+
{ table, effective: unit.pairs.get(table.name)! },
|
|
768
|
+
pin,
|
|
769
|
+
rowCursor,
|
|
770
|
+
await plan.starts.get(tableIndex)!,
|
|
771
|
+
limits.maxSnapshotPages - pages,
|
|
744
772
|
trace,
|
|
745
773
|
logEpoch,
|
|
746
|
-
prepared.firstPage,
|
|
747
|
-
prepared.image,
|
|
748
774
|
);
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
const nextToken: BootstrapToken = {
|
|
753
|
-
asOfCommitSeq: asOf,
|
|
754
|
-
tables: [sub.table],
|
|
755
|
-
tableIndex: 0,
|
|
756
|
-
rowCursor: outcome.rowCursor,
|
|
757
|
-
};
|
|
758
|
-
yield {
|
|
759
|
-
type: 'SUB_END',
|
|
760
|
-
nextCursor: asOf,
|
|
761
|
-
bootstrapState: JSON.stringify(nextToken),
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
return { nextCursor: asOf, active: true };
|
|
775
|
+
pages += outcome.pages;
|
|
776
|
+
rowCursor = outcome.rowCursor;
|
|
777
|
+
if (rowCursor === null) tableIndex += 1;
|
|
765
778
|
}
|
|
766
|
-
|
|
767
|
-
// Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
|
|
768
|
-
// cut off at limitCommits total changes, never splitting a commit.
|
|
769
|
-
yield {
|
|
770
|
-
type: 'SUB_START',
|
|
771
|
-
id: sub.id,
|
|
772
|
-
status: 'active',
|
|
773
|
-
reasonCode: '',
|
|
774
|
-
effectiveScopes: plan.effective,
|
|
775
|
-
bootstrap: false,
|
|
776
|
-
};
|
|
777
|
-
let delivered = 0;
|
|
778
|
-
let deliveredCommits = 0;
|
|
779
|
-
let lastDeliveredSeq = sub.cursor;
|
|
780
|
-
for (const commit of commits) {
|
|
781
|
-
if (
|
|
782
|
-
delivered > 0 &&
|
|
783
|
-
delivered + commit.changes.length > limits.limitCommits
|
|
784
|
-
) {
|
|
785
|
-
break;
|
|
786
|
-
}
|
|
787
|
-
yield commitFrame(
|
|
788
|
-
sub.table,
|
|
789
|
-
commit,
|
|
790
|
-
compileSchema(ctx.schema).tables.get(sub.table)!,
|
|
791
|
-
plan.table,
|
|
792
|
-
);
|
|
793
|
-
delivered += commit.changes.length;
|
|
794
|
-
deliveredCommits += 1;
|
|
795
|
-
lastDeliveredSeq = commit.commitSeq;
|
|
796
|
-
if (delivered >= limits.limitCommits) break;
|
|
797
|
-
}
|
|
798
|
-
const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
|
|
799
|
-
// The window is proven exhausted only when the storage lookahead came
|
|
800
|
-
// back under budget (it scanned to `throughSeq`) and every returned
|
|
801
|
-
// commit was delivered.
|
|
802
|
-
const exhausted =
|
|
803
|
-
totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
|
|
804
|
-
// §4.5: the cursor advances even when no matching changes exist; when
|
|
805
|
-
// the change limit truncated the window it stops at the last fully
|
|
806
|
-
// delivered commit.
|
|
807
|
-
const nextCursor = exhausted
|
|
808
|
-
? Math.max(sub.cursor, maxSeq)
|
|
809
|
-
: Math.max(sub.cursor, lastDeliveredSeq);
|
|
810
|
-
yield { type: 'SUB_END', nextCursor };
|
|
811
|
-
return { nextCursor, active: true };
|
|
812
779
|
}
|
|
813
780
|
|
|
814
781
|
export interface PublishSqliteImageInput {
|
|
@@ -816,12 +783,15 @@ export interface PublishSqliteImageInput {
|
|
|
816
783
|
readonly config: SyncServerConfig;
|
|
817
784
|
readonly partition: string;
|
|
818
785
|
readonly table: string;
|
|
786
|
+
/** The unit whose clients bootstrap the table (§4.1). */
|
|
787
|
+
readonly scopes: UnitScopes;
|
|
819
788
|
/**
|
|
820
|
-
* The
|
|
821
|
-
*
|
|
822
|
-
*
|
|
789
|
+
* The allowed scopes of the actors that register the unit (§3.2). The
|
|
790
|
+
* reuse key carries the digest of the pair's effective scopes, so a
|
|
791
|
+
* wildcard fence on the table changes the image. Default: `'*'` for every
|
|
792
|
+
* declared variable, which leaves the unit's own scope map.
|
|
823
793
|
*/
|
|
824
|
-
readonly
|
|
794
|
+
readonly allowed?: ScopeMap;
|
|
825
795
|
}
|
|
826
796
|
|
|
827
797
|
export interface PublishedSqliteImage {
|
|
@@ -836,8 +806,8 @@ export interface PublishedSqliteImage {
|
|
|
836
806
|
}
|
|
837
807
|
|
|
838
808
|
/**
|
|
839
|
-
* Publish the §5.3 sqlite image that a pull
|
|
840
|
-
*
|
|
809
|
+
* Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
|
|
810
|
+
* from a host with a SQLite engine (a Bun process with the production
|
|
841
811
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
842
812
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
843
813
|
* serving host without a builder finds. The image stays current until a
|
|
@@ -852,9 +822,19 @@ export async function publishSqliteImage(
|
|
|
852
822
|
throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
|
|
853
823
|
}
|
|
854
824
|
const schema = compileSchema(config.schema);
|
|
855
|
-
const table = schema.
|
|
825
|
+
const table = coveredTables(schema, input.scopes).find(
|
|
826
|
+
(candidate) => candidate.name === input.table,
|
|
827
|
+
);
|
|
856
828
|
if (table === undefined) {
|
|
857
|
-
throw new Error('publishSqliteImage: the
|
|
829
|
+
throw new Error('publishSqliteImage: the unit does not cover the table');
|
|
830
|
+
}
|
|
831
|
+
const allowed =
|
|
832
|
+
input.allowed ??
|
|
833
|
+
Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
|
|
834
|
+
const unit = planUnit(input.scopes, [table], { ok: true, allowed });
|
|
835
|
+
const effective = unit.pairs.get(table.name);
|
|
836
|
+
if (effective === undefined) {
|
|
837
|
+
throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
|
|
858
838
|
}
|
|
859
839
|
await config.storage.ensureSchema(schema, config.checkpoints);
|
|
860
840
|
const logEpoch = await config.storage.getPartitionLogEpoch(partition);
|
|
@@ -866,18 +846,6 @@ export async function publishSqliteImage(
|
|
|
866
846
|
partition,
|
|
867
847
|
actorId: 'syncular.image-publisher',
|
|
868
848
|
};
|
|
869
|
-
const plan: SubscriptionPlan = {
|
|
870
|
-
frame: {
|
|
871
|
-
type: 'SUBSCRIPTION',
|
|
872
|
-
id: 'image',
|
|
873
|
-
table: input.table,
|
|
874
|
-
scopes: input.scopes,
|
|
875
|
-
cursor: -1,
|
|
876
|
-
},
|
|
877
|
-
table,
|
|
878
|
-
status: 'active',
|
|
879
|
-
effective: input.scopes,
|
|
880
|
-
};
|
|
881
849
|
// One-row pages make every table above one row image-eligible (§5.3).
|
|
882
850
|
const limits: PullLimits = {
|
|
883
851
|
limitCommits: 1,
|
|
@@ -888,24 +856,22 @@ export async function publishSqliteImage(
|
|
|
888
856
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
889
857
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
890
858
|
const trace: PullSectionTrace = { segments: [] };
|
|
891
|
-
const
|
|
859
|
+
const plan = planBootstrap(
|
|
892
860
|
ctx,
|
|
893
861
|
schema,
|
|
894
862
|
limits,
|
|
895
|
-
|
|
863
|
+
unit,
|
|
864
|
+
undefined,
|
|
896
865
|
maxSeq,
|
|
897
866
|
horizonSeq,
|
|
898
867
|
logEpoch,
|
|
899
868
|
);
|
|
900
|
-
if (prepared === undefined) throw new Error('prepareSections lost a plan');
|
|
901
869
|
let ref: SegmentRefFrame | undefined;
|
|
902
|
-
for await (const frame of
|
|
870
|
+
for await (const frame of bootstrapSegments(
|
|
903
871
|
ctx,
|
|
904
872
|
schema,
|
|
905
873
|
limits,
|
|
906
874
|
plan,
|
|
907
|
-
maxSeq,
|
|
908
|
-
prepared,
|
|
909
875
|
trace,
|
|
910
876
|
logEpoch,
|
|
911
877
|
)) {
|