@syncular/server 0.75.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +6 -0
- package/dist/context.js +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +128 -76
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +8 -0
- package/src/index.ts +1 -0
- package/src/pull.ts +199 -94
- package/src/segment-store.ts +10 -6
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
|
@@ -23,10 +23,11 @@
|
|
|
23
23
|
* objects.
|
|
24
24
|
*
|
|
25
25
|
* {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* Image reuse pointer (§5.3): written for every sqlite image and every
|
|
27
|
+
* first rows page, body = the record JSON. The reuse key is the
|
|
28
28
|
* canonical JSON array `[partition, logEpoch, table, schemaVersion,
|
|
29
|
-
* mediaType, scopeDigest, asOfCommitSeq]
|
|
29
|
+
* mediaType, scopeDigest, asOfCommitSeq]` plus the range start when it is
|
|
30
|
+
* not null, so `find` is one GET (plus a
|
|
30
31
|
* HEAD to confirm the segment object itself still exists — lifecycle GC
|
|
31
32
|
* may remove objects independently of pointers).
|
|
32
33
|
*
|
|
@@ -214,6 +214,8 @@ export class BucketSegmentStore {
|
|
|
214
214
|
key.mediaType,
|
|
215
215
|
key.scopeDigest,
|
|
216
216
|
key.asOfCommitSeq,
|
|
217
|
+
// A first range keeps the pointer key it had before range images.
|
|
218
|
+
...(key.rowCursor === null ? [] : [key.rowCursor]),
|
|
217
219
|
]);
|
|
218
220
|
return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
|
|
219
221
|
}
|
|
@@ -300,7 +302,7 @@ export class BucketSegmentStore {
|
|
|
300
302
|
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
301
303
|
});
|
|
302
304
|
}
|
|
303
|
-
if (metadata.rowCursor === null) {
|
|
305
|
+
if (metadata.rowCursor === null || metadata.mediaType === 'sqlite') {
|
|
304
306
|
await this.#bucket.put(await this.#findKeyFor(metadata), {
|
|
305
307
|
body: new TextEncoder().encode(recordJson),
|
|
306
308
|
contentType: 'application/json',
|
|
@@ -415,7 +417,7 @@ export class BucketSegmentStore {
|
|
|
415
417
|
mediaType: key.mediaType,
|
|
416
418
|
scopeDigest: key.scopeDigest,
|
|
417
419
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
418
|
-
rowCursor:
|
|
420
|
+
rowCursor: key.rowCursor,
|
|
419
421
|
nowMs,
|
|
420
422
|
});
|
|
421
423
|
if (publication === undefined)
|
package/dist/context.d.ts
CHANGED
|
@@ -86,6 +86,12 @@ export interface ServerLimits {
|
|
|
86
86
|
* pass it alone resets the units it reaches, which bootstrap instead.
|
|
87
87
|
*/
|
|
88
88
|
readonly maxLiveSectionBytes: number;
|
|
89
|
+
/**
|
|
90
|
+
* §5.3 bound on one sqlite image's file size: the server cuts a table into
|
|
91
|
+
* range images in row order, each within this bound. A row larger than the
|
|
92
|
+
* bound gets an image of its own.
|
|
93
|
+
*/
|
|
94
|
+
readonly maxImageSegmentBytes: number;
|
|
89
95
|
}
|
|
90
96
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
91
97
|
/** Receives applied commits for realtime fanout (§8.2). */
|
package/dist/context.js
CHANGED
|
@@ -23,6 +23,8 @@ export const DEFAULT_LIMITS = {
|
|
|
23
23
|
maxWatchesPerSession: 100,
|
|
24
24
|
// The live section is buffered whole before the first unit section.
|
|
25
25
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
26
|
+
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
27
|
+
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
26
28
|
};
|
|
27
29
|
/** §1.1: `authenticate` returned a context longer than 256 UTF-8 bytes. */
|
|
28
30
|
export class AuthContextTooLargeError extends Error {
|
package/dist/index.d.ts
CHANGED
|
@@ -50,7 +50,7 @@ export * from './segment-store.js';
|
|
|
50
50
|
export * from './signed-url.js';
|
|
51
51
|
export * from './sqlite-dialect.js';
|
|
52
52
|
export * from './sqlite-driver.js';
|
|
53
|
-
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImageBuilder, type SqliteImageInput, writeSqliteImage, } from './sqlite-image.js';
|
|
53
|
+
export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImage, type SqliteImageBuilder, type SqliteImageInput, writeSqliteImage, } from './sqlite-image.js';
|
|
54
54
|
export * from './sqlite-blob-store.js';
|
|
55
55
|
export * from './sqlite-lease-store.js';
|
|
56
56
|
export * from './sqlite-segment-store.js';
|
package/dist/pull.d.ts
CHANGED
|
@@ -74,6 +74,8 @@ export interface BootstrapPlan {
|
|
|
74
74
|
readonly pin: number;
|
|
75
75
|
readonly tableIndex: number;
|
|
76
76
|
readonly rowCursor: string | null;
|
|
77
|
+
/** Whether the position continues the table's range images (§5.3). */
|
|
78
|
+
readonly image: boolean;
|
|
77
79
|
/** Table index → its start reads. */
|
|
78
80
|
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
79
81
|
}
|
|
@@ -108,17 +110,20 @@ export interface PublishSqliteImageInput {
|
|
|
108
110
|
readonly allowed?: ScopeMap;
|
|
109
111
|
}
|
|
110
112
|
export interface PublishedSqliteImage {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
/** The table's range images in row order (§5.3). */
|
|
114
|
+
readonly segmentIds: readonly string[];
|
|
115
|
+
/** The §4.7 scope pin the images are stored under. */
|
|
113
116
|
readonly asOfCommitSeq: number;
|
|
114
117
|
readonly scopeDigest: string;
|
|
118
|
+
/** Rows across every range image. */
|
|
115
119
|
readonly rowCount: number;
|
|
120
|
+
/** Bytes across every range image. */
|
|
116
121
|
readonly byteLength: number;
|
|
117
|
-
/** `reused` when an unexpired image for
|
|
122
|
+
/** `reused` when an unexpired image existed for every range's reuse key. */
|
|
118
123
|
readonly origin: 'built' | 'reused';
|
|
119
124
|
}
|
|
120
125
|
/**
|
|
121
|
-
* Publish the §5.3
|
|
126
|
+
* Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
|
|
122
127
|
* from a host with a SQLite engine (a Bun process with the production
|
|
123
128
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
124
129
|
* reuse key, the build), so the stored identity is exactly the one a
|
package/dist/pull.js
CHANGED
|
@@ -144,7 +144,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
|
|
|
144
144
|
parsed.tableIndex === undefined ||
|
|
145
145
|
parsed.tableIndex < 0 ||
|
|
146
146
|
parsed.tableIndex >= tables.length ||
|
|
147
|
-
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
147
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
148
|
+
(parsed.image !== undefined &&
|
|
149
|
+
(parsed.image !== true || parsed.rowCursor === null))) {
|
|
148
150
|
return undefined;
|
|
149
151
|
}
|
|
150
152
|
return {
|
|
@@ -152,6 +154,7 @@ function parseBootstrapToken(raw, tables, maxSeq) {
|
|
|
152
154
|
tables: parsed.tables,
|
|
153
155
|
tableIndex: parsed.tableIndex,
|
|
154
156
|
rowCursor: parsed.rowCursor ?? null,
|
|
157
|
+
...(parsed.image === true ? { image: true } : {}),
|
|
155
158
|
};
|
|
156
159
|
}
|
|
157
160
|
catch {
|
|
@@ -193,8 +196,12 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
193
196
|
// under different scopes (§5.1), and the record's primary digest is not
|
|
194
197
|
// necessarily the caller's (§5.5).
|
|
195
198
|
scopeDigest,
|
|
196
|
-
|
|
197
|
-
|
|
199
|
+
// §5.3: an image descriptor carries no cursors; the resume token holds
|
|
200
|
+
// the position of a table's next range.
|
|
201
|
+
...(record.mediaType === 'rows' && record.rowCursor !== null
|
|
202
|
+
? { rowCursor: record.rowCursor }
|
|
203
|
+
: {}),
|
|
204
|
+
...(record.mediaType === 'rows' && record.nextRowCursor !== null
|
|
198
205
|
? { nextRowCursor: record.nextRowCursor }
|
|
199
206
|
: {}),
|
|
200
207
|
...(extra.url !== undefined ? { url: extra.url } : {}),
|
|
@@ -204,26 +211,51 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
204
211
|
};
|
|
205
212
|
}
|
|
206
213
|
/**
|
|
207
|
-
* §5.3: the sqlite-image lane
|
|
208
|
-
*
|
|
214
|
+
* §5.3: the sqlite-image lane, chosen at the start of a table or continued
|
|
215
|
+
* after a range image (a table on the rows lane never switches to images),
|
|
209
216
|
* and only when the client advertised bit 2. §5.11: a table with any
|
|
210
217
|
* encrypted column is image-INELIGIBLE — an image copies ciphertext
|
|
211
218
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
212
219
|
* lane (which decrypts per row on the client).
|
|
213
220
|
*/
|
|
214
|
-
function sqliteLaneEligible(limits, table, startRowCursor) {
|
|
221
|
+
function sqliteLaneEligible(limits, table, startRowCursor, imageLane) {
|
|
215
222
|
return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
216
|
-
startRowCursor === null &&
|
|
223
|
+
(startRowCursor === null || imageLane) &&
|
|
217
224
|
table.encryptedColumnIndices.length === 0);
|
|
218
225
|
}
|
|
219
|
-
|
|
226
|
+
/**
|
|
227
|
+
* §5.3 reuse lookup for the range image that starts after `rowCursor`. Only
|
|
228
|
+
* a position with more than one rows page left can match a stored image, so
|
|
229
|
+
* a smaller remainder skips the lookup. A stored image past the bound (built
|
|
230
|
+
* under a larger one) does not match unless it holds a single row.
|
|
231
|
+
*/
|
|
232
|
+
async function findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch) {
|
|
233
|
+
if (probe.length <= limits.limitSnapshotRows)
|
|
234
|
+
return undefined;
|
|
235
|
+
const stored = await ctx.segments.find({
|
|
236
|
+
partition: ctx.partition,
|
|
237
|
+
logEpoch,
|
|
238
|
+
table: pair.table.name,
|
|
239
|
+
schemaVersion: schema.version,
|
|
240
|
+
mediaType: 'sqlite',
|
|
241
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
242
|
+
asOfCommitSeq: scopePin,
|
|
243
|
+
rowCursor,
|
|
244
|
+
}, clockOf(ctx)());
|
|
245
|
+
return stored !== undefined &&
|
|
246
|
+
(stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
|
|
247
|
+
stored.rowCount === 1)
|
|
248
|
+
? stored
|
|
249
|
+
: undefined;
|
|
250
|
+
}
|
|
251
|
+
async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, imageLane, maxSeq, horizonSeq, logEpoch) {
|
|
220
252
|
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
221
253
|
table: pair.table.name,
|
|
222
254
|
scopeFilter: pair.effective,
|
|
223
255
|
afterRowId: startRowCursor,
|
|
224
256
|
limit: limits.limitSnapshotRows + 1,
|
|
225
257
|
});
|
|
226
|
-
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
258
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor, imageLane)) {
|
|
227
259
|
return { firstPage: await scan };
|
|
228
260
|
}
|
|
229
261
|
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
@@ -238,30 +270,17 @@ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, m
|
|
|
238
270
|
}),
|
|
239
271
|
]);
|
|
240
272
|
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
241
|
-
|
|
242
|
-
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
243
|
-
// on a small table would be a wasted round trip.
|
|
244
|
-
const stored = firstPage.length <= limits.limitSnapshotRows
|
|
245
|
-
? undefined
|
|
246
|
-
: await ctx.segments.find({
|
|
247
|
-
partition: ctx.partition,
|
|
248
|
-
logEpoch,
|
|
249
|
-
table: pair.table.name,
|
|
250
|
-
schemaVersion: schema.version,
|
|
251
|
-
mediaType: 'sqlite',
|
|
252
|
-
scopeDigest: await scopeDigest(pair.effective),
|
|
253
|
-
asOfCommitSeq: scopePin,
|
|
254
|
-
}, clockOf(ctx)());
|
|
273
|
+
const stored = await findImage(ctx, schema, limits, pair, scopePin, startRowCursor, firstPage, logEpoch);
|
|
255
274
|
return { firstPage, image: { scopePin, stored } };
|
|
256
275
|
}
|
|
257
276
|
/**
|
|
258
|
-
* §5.3 sqlite-image lane:
|
|
259
|
-
*
|
|
260
|
-
* schemaVersion, scope digest, scope
|
|
261
|
-
*
|
|
262
|
-
* takes over).
|
|
277
|
+
* §5.3 sqlite-image lane: one range image starting after `rowCursor`, only
|
|
278
|
+
* when more than one rows page remains there. Reuses an unexpired stored
|
|
279
|
+
* image for the same (partition, table, schemaVersion, scope digest, scope
|
|
280
|
+
* pin, range start) instead of rebuilding — the bootstrap-storm rule.
|
|
281
|
+
* Returns undefined when no image is served (the rows lane takes over).
|
|
263
282
|
*/
|
|
264
|
-
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
|
|
283
|
+
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, rowCursor, digest, trace, logEpoch, existing, probe) {
|
|
265
284
|
const { storage, segments, partition } = ctx;
|
|
266
285
|
const now = clockOf(ctx)();
|
|
267
286
|
if (existing !== undefined) {
|
|
@@ -273,15 +292,15 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
273
292
|
rows: existing.rowCount,
|
|
274
293
|
});
|
|
275
294
|
yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
|
|
276
|
-
return
|
|
295
|
+
return existing;
|
|
277
296
|
}
|
|
278
297
|
if (probe.length <= limits.limitSnapshotRows)
|
|
279
|
-
return
|
|
298
|
+
return undefined;
|
|
280
299
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
281
300
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
282
301
|
const buildImage = ctx.sqliteImageBuilder;
|
|
283
302
|
if (buildImage === undefined)
|
|
284
|
-
return
|
|
303
|
+
return undefined;
|
|
285
304
|
let stores = imageBuilds.get(storage);
|
|
286
305
|
if (stores === undefined) {
|
|
287
306
|
stores = new WeakMap();
|
|
@@ -300,6 +319,7 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
300
319
|
mediaType: 'sqlite',
|
|
301
320
|
scopeDigest: digest,
|
|
302
321
|
asOfCommitSeq: scopePin,
|
|
322
|
+
rowCursor,
|
|
303
323
|
};
|
|
304
324
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
305
325
|
const key = JSON.stringify(identity);
|
|
@@ -308,43 +328,55 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
308
328
|
if (building === undefined) {
|
|
309
329
|
building = (async () => {
|
|
310
330
|
// Another request can finish while this one's eligibility probe awaits.
|
|
311
|
-
const cached = await
|
|
331
|
+
const cached = await findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch);
|
|
312
332
|
if (cached !== undefined)
|
|
313
333
|
return cached;
|
|
314
334
|
builtHere = true;
|
|
315
|
-
|
|
316
|
-
|
|
335
|
+
// The builder stops at the size bound inside the latest batch; these
|
|
336
|
+
// locate the last row it took and whether rows follow it.
|
|
337
|
+
let yielded = 0;
|
|
338
|
+
let batch = [];
|
|
339
|
+
let beforeBatch = rowCursor;
|
|
340
|
+
let exhausted = false;
|
|
341
|
+
const image = await buildImage({
|
|
317
342
|
table: pair.table,
|
|
318
343
|
schemaVersion: schema.version,
|
|
319
344
|
asOfCommitSeq: scopePin,
|
|
320
345
|
scopeDigest: digest,
|
|
346
|
+
maxBytes: limitsOf(ctx).maxImageSegmentBytes,
|
|
321
347
|
rowBatches: (async function* () {
|
|
322
|
-
|
|
323
|
-
yield probe.map((row) => ({
|
|
324
|
-
...row,
|
|
325
|
-
payload: projectRowPayload(current, pair.table, row.payload),
|
|
326
|
-
}));
|
|
327
|
-
let afterRowId = probe[probe.length - 1].rowId;
|
|
348
|
+
let rows = probe;
|
|
328
349
|
for (;;) {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
afterRowId,
|
|
333
|
-
limit: 5_000,
|
|
334
|
-
});
|
|
335
|
-
rowCount += rows.length;
|
|
350
|
+
beforeBatch = batch[batch.length - 1]?.rowId ?? beforeBatch;
|
|
351
|
+
batch = rows;
|
|
352
|
+
yielded += rows.length;
|
|
336
353
|
yield rows.map((row) => ({
|
|
337
354
|
...row,
|
|
338
355
|
payload: projectRowPayload(current, pair.table, row.payload),
|
|
339
356
|
}));
|
|
340
357
|
const last = rows[rows.length - 1];
|
|
341
|
-
if (rows.length < 5_000
|
|
358
|
+
if (last === undefined || (rows !== probe && rows.length < 5_000))
|
|
342
359
|
break;
|
|
343
|
-
|
|
360
|
+
rows = await storage.scanRows(partition, {
|
|
361
|
+
table: pair.table.name,
|
|
362
|
+
scopeFilter: pair.effective,
|
|
363
|
+
afterRowId: last.rowId,
|
|
364
|
+
limit: 5_000,
|
|
365
|
+
});
|
|
344
366
|
}
|
|
367
|
+
exhausted = true;
|
|
345
368
|
})(),
|
|
346
369
|
});
|
|
347
|
-
|
|
370
|
+
const index = image.rowCount - (yielded - batch.length) - 1;
|
|
371
|
+
if (image.rowCount < 1 || image.rowCount > yielded || index < -1) {
|
|
372
|
+
throw new Error('sqliteImageBuilder consumed rows out of order');
|
|
373
|
+
}
|
|
374
|
+
const last = index < 0 ? beforeBatch : batch[index].rowId;
|
|
375
|
+
return segments.put({
|
|
376
|
+
...identity,
|
|
377
|
+
rowCount: image.rowCount,
|
|
378
|
+
nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
|
|
379
|
+
}, image.bytes, clockOf(ctx)());
|
|
348
380
|
})();
|
|
349
381
|
builds.set(key, building);
|
|
350
382
|
}
|
|
@@ -364,24 +396,22 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
364
396
|
rows: record.rowCount,
|
|
365
397
|
});
|
|
366
398
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
|
|
367
|
-
return
|
|
399
|
+
return record;
|
|
368
400
|
}
|
|
369
401
|
/**
|
|
370
|
-
* One table's segments within the page budget
|
|
371
|
-
*
|
|
402
|
+
* One table's segments within the page budget: range images while the table
|
|
403
|
+
* is on the image lane, then rows pages. Returns the pages it used, the row
|
|
404
|
+
* cursor to resume at (`null` once the table is exhausted) and whether that
|
|
405
|
+
* position is on the image lane.
|
|
372
406
|
*/
|
|
373
407
|
async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
|
|
374
408
|
const { storage, segments, partition } = ctx;
|
|
375
409
|
const serverLimits = limitsOf(ctx);
|
|
376
410
|
const digest = await scopeDigest(pair.effective);
|
|
377
411
|
const now = clockOf(ctx)();
|
|
378
|
-
if (start.image !== undefined) {
|
|
379
|
-
const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
|
|
380
|
-
if (imaged)
|
|
381
|
-
return { pages: 1, rowCursor: null };
|
|
382
|
-
}
|
|
383
412
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
384
413
|
let rowCursor = startRowCursor;
|
|
414
|
+
let image = start.image;
|
|
385
415
|
for (let page = 0; page < pageBudget; page++) {
|
|
386
416
|
const scanned = page === 0
|
|
387
417
|
? start.firstPage
|
|
@@ -391,6 +421,20 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
391
421
|
afterRowId: rowCursor,
|
|
392
422
|
limit: limits.limitSnapshotRows + 1,
|
|
393
423
|
});
|
|
424
|
+
if (image !== undefined) {
|
|
425
|
+
// §5.3: every range of the table shares the scope pin computed at this
|
|
426
|
+
// section's table start.
|
|
427
|
+
const record = yield* sqliteImageSegment(ctx, schema, limits, pair, image.scopePin, rowCursor, digest, trace, logEpoch, page === 0
|
|
428
|
+
? image.stored
|
|
429
|
+
: await findImage(ctx, schema, limits, pair, image.scopePin, rowCursor, scanned, logEpoch), scanned);
|
|
430
|
+
if (record !== undefined) {
|
|
431
|
+
if (record.nextRowCursor === null)
|
|
432
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
433
|
+
rowCursor = record.nextRowCursor;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
image = undefined;
|
|
437
|
+
}
|
|
394
438
|
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
395
439
|
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
396
440
|
// §5.2: every row record carries the row's current server_version.
|
|
@@ -443,10 +487,10 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
443
487
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
|
|
444
488
|
}
|
|
445
489
|
if (!hasMore)
|
|
446
|
-
return { pages: page + 1, rowCursor: null };
|
|
490
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
447
491
|
rowCursor = nextRowCursor;
|
|
448
492
|
}
|
|
449
|
-
return { pages: pageBudget, rowCursor };
|
|
493
|
+
return { pages: pageBudget, rowCursor, image: image !== undefined };
|
|
450
494
|
}
|
|
451
495
|
/**
|
|
452
496
|
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
@@ -464,13 +508,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
464
508
|
: undefined;
|
|
465
509
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
466
510
|
const rowCursor = resume?.rowCursor ?? null;
|
|
511
|
+
const image = resume?.image === true;
|
|
467
512
|
const starts = new Map();
|
|
468
513
|
for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
|
|
469
514
|
const table = unit.tables[index];
|
|
470
515
|
const effective = unit.pairs.get(table.name);
|
|
471
516
|
if (effective === undefined)
|
|
472
517
|
continue;
|
|
473
|
-
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
|
|
518
|
+
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, index === tableIndex && image, maxSeq, horizonSeq, logEpoch);
|
|
474
519
|
// A section that stops early never awaits the later starts.
|
|
475
520
|
start.catch(() => undefined);
|
|
476
521
|
starts.set(index, start);
|
|
@@ -480,6 +525,7 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
480
525
|
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
481
526
|
tableIndex,
|
|
482
527
|
rowCursor,
|
|
528
|
+
image,
|
|
483
529
|
starts,
|
|
484
530
|
};
|
|
485
531
|
}
|
|
@@ -491,13 +537,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
491
537
|
*/
|
|
492
538
|
export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
|
|
493
539
|
const { unit, pin } = plan;
|
|
494
|
-
let { tableIndex, rowCursor } = plan;
|
|
540
|
+
let { tableIndex, rowCursor, image } = plan;
|
|
495
541
|
let pages = 0;
|
|
496
542
|
for (;;) {
|
|
497
543
|
while (tableIndex < unit.tables.length &&
|
|
498
544
|
!unit.pairs.has(unit.tables[tableIndex].name)) {
|
|
499
545
|
tableIndex += 1;
|
|
500
546
|
rowCursor = null;
|
|
547
|
+
image = false;
|
|
501
548
|
}
|
|
502
549
|
if (tableIndex === unit.tables.length)
|
|
503
550
|
return undefined;
|
|
@@ -507,6 +554,7 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
|
|
|
507
554
|
tables: unit.tables.map((table) => table.name),
|
|
508
555
|
tableIndex,
|
|
509
556
|
rowCursor,
|
|
557
|
+
...(image ? { image: true } : {}),
|
|
510
558
|
};
|
|
511
559
|
return JSON.stringify(token);
|
|
512
560
|
}
|
|
@@ -514,12 +562,13 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
|
|
|
514
562
|
const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
|
|
515
563
|
pages += outcome.pages;
|
|
516
564
|
rowCursor = outcome.rowCursor;
|
|
565
|
+
image = outcome.image;
|
|
517
566
|
if (rowCursor === null)
|
|
518
567
|
tableIndex += 1;
|
|
519
568
|
}
|
|
520
569
|
}
|
|
521
570
|
/**
|
|
522
|
-
* Publish the §5.3
|
|
571
|
+
* Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
|
|
523
572
|
* from a host with a SQLite engine (a Bun process with the production
|
|
524
573
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
525
574
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
@@ -554,33 +603,36 @@ export async function publishSqliteImage(input) {
|
|
|
554
603
|
partition,
|
|
555
604
|
actorId: 'syncular.image-publisher',
|
|
556
605
|
};
|
|
557
|
-
// One-row pages make every
|
|
606
|
+
// One-row pages make every range above one row image-eligible (§5.3), and
|
|
607
|
+
// an unbounded page budget stores every range under one scope pin.
|
|
558
608
|
const limits = {
|
|
559
609
|
limitCommits: 1,
|
|
560
610
|
limitSnapshotRows: 1,
|
|
561
|
-
maxSnapshotPages:
|
|
611
|
+
maxSnapshotPages: Number.MAX_SAFE_INTEGER,
|
|
562
612
|
accept: ACCEPT_SQLITE,
|
|
563
613
|
};
|
|
564
614
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
565
615
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
566
616
|
const trace = { segments: [] };
|
|
567
617
|
const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
|
|
568
|
-
|
|
618
|
+
const refs = [];
|
|
569
619
|
for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
|
|
570
620
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
571
|
-
|
|
621
|
+
refs.push(frame);
|
|
572
622
|
}
|
|
573
623
|
}
|
|
574
|
-
const
|
|
575
|
-
if (
|
|
624
|
+
const first = refs[0];
|
|
625
|
+
if (first === undefined) {
|
|
576
626
|
throw new Error('publishSqliteImage: the scope holds at most one row, which pulls serve inline');
|
|
577
627
|
}
|
|
578
628
|
return {
|
|
579
|
-
|
|
580
|
-
asOfCommitSeq:
|
|
581
|
-
scopeDigest:
|
|
582
|
-
rowCount: ref.rowCount,
|
|
583
|
-
byteLength: ref.byteLength,
|
|
584
|
-
origin:
|
|
629
|
+
segmentIds: refs.map((ref) => ref.segmentId),
|
|
630
|
+
asOfCommitSeq: first.asOfCommitSeq,
|
|
631
|
+
scopeDigest: first.scopeDigest,
|
|
632
|
+
rowCount: refs.reduce((sum, ref) => sum + ref.rowCount, 0),
|
|
633
|
+
byteLength: refs.reduce((sum, ref) => sum + ref.byteLength, 0),
|
|
634
|
+
origin: trace.segments.every((segment) => segment.mediaType !== 'sqlite' || segment.origin === 'reused')
|
|
635
|
+
? 'reused'
|
|
636
|
+
: 'built',
|
|
585
637
|
};
|
|
586
638
|
}
|
package/dist/segment-store.d.ts
CHANGED
|
@@ -99,7 +99,8 @@ export declare function mergeSegmentRecord(existing: SegmentRecord | undefined,
|
|
|
99
99
|
/**
|
|
100
100
|
* The §5.3 reuse key: sqlite images are not byte-deterministic, so
|
|
101
101
|
* cross-client dedup is a metadata lookup, not hash convergence — one
|
|
102
|
-
* stored image per (partition, table, schemaVersion, scope digest, pin
|
|
102
|
+
* stored image per (partition, table, schemaVersion, scope digest, pin, range
|
|
103
|
+
* start).
|
|
103
104
|
*/
|
|
104
105
|
export interface SegmentFindKey {
|
|
105
106
|
readonly partition: string;
|
|
@@ -109,6 +110,8 @@ export interface SegmentFindKey {
|
|
|
109
110
|
readonly mediaType: 'rows' | 'sqlite';
|
|
110
111
|
readonly scopeDigest: string;
|
|
111
112
|
readonly asOfCommitSeq: number;
|
|
113
|
+
/** The range start: the row id the image follows, `null` for the first. */
|
|
114
|
+
readonly rowCursor: string | null;
|
|
112
115
|
}
|
|
113
116
|
/**
|
|
114
117
|
* Coarse store-level counters for the admin/console read surface. Optional:
|
|
@@ -146,9 +149,8 @@ export interface SegmentStore {
|
|
|
146
149
|
bytes: Uint8Array;
|
|
147
150
|
} | undefined>;
|
|
148
151
|
/**
|
|
149
|
-
* Unexpired record for the §5.3 reuse key
|
|
150
|
-
*
|
|
151
|
-
* rebuilding sqlite images while one exists (§5.3).
|
|
152
|
+
* Unexpired record for the §5.3 reuse key, or undefined. Servers MUST
|
|
153
|
+
* reuse instead of rebuilding sqlite images while one exists (§5.3).
|
|
152
154
|
*
|
|
153
155
|
* The digest matches when it is one of the record's `scopeDigests`
|
|
154
156
|
* (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
|
package/dist/segment-store.js
CHANGED
|
@@ -188,7 +188,9 @@ export class MemorySegmentStore {
|
|
|
188
188
|
};
|
|
189
189
|
}
|
|
190
190
|
async find(key, nowMs) {
|
|
191
|
-
|
|
191
|
+
// The newest publication wins: a rebuild supersedes an image stored
|
|
192
|
+
// under the same key that the server no longer serves (§5.3 bound).
|
|
193
|
+
for (const { record } of [...this.#entries.values()].reverse()) {
|
|
192
194
|
const publication = selectSegmentPublication(record, {
|
|
193
195
|
partition: key.partition,
|
|
194
196
|
logEpoch: key.logEpoch,
|
|
@@ -197,7 +199,7 @@ export class MemorySegmentStore {
|
|
|
197
199
|
mediaType: key.mediaType,
|
|
198
200
|
scopeDigest: key.scopeDigest,
|
|
199
201
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
200
|
-
rowCursor:
|
|
202
|
+
rowCursor: key.rowCursor,
|
|
201
203
|
nowMs,
|
|
202
204
|
});
|
|
203
205
|
if (publication !== undefined) {
|
package/dist/sqlite-bun.js
CHANGED
|
@@ -30,8 +30,8 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
30
30
|
export const buildSqliteImage = async (input) => {
|
|
31
31
|
const db = new BunSqliteDatabase();
|
|
32
32
|
try {
|
|
33
|
-
await writeSqliteImage(db, input);
|
|
34
|
-
return db.serialize();
|
|
33
|
+
const rowCount = await writeSqliteImage(db, input);
|
|
34
|
+
return { bytes: db.serialize(), rowCount };
|
|
35
35
|
}
|
|
36
36
|
finally {
|
|
37
37
|
db.close();
|
package/dist/sqlite-image.d.ts
CHANGED
|
@@ -10,6 +10,13 @@ export interface SqliteImageInput {
|
|
|
10
10
|
readonly schemaVersion: number;
|
|
11
11
|
readonly asOfCommitSeq: number;
|
|
12
12
|
readonly scopeDigest: string;
|
|
13
|
+
/**
|
|
14
|
+
* The image's file-size bound (`ServerLimits.maxImageSegmentBytes`). The
|
|
15
|
+
* builder stops before the row that could pass it; the first row always
|
|
16
|
+
* goes in.
|
|
17
|
+
*/
|
|
18
|
+
readonly maxBytes: number;
|
|
19
|
+
/** The range's rows in row order, from its start to the table's end. */
|
|
13
20
|
readonly rowBatches: AsyncIterable<readonly StoredRow[]> | Iterable<readonly StoredRow[]>;
|
|
14
21
|
}
|
|
15
22
|
/**
|
|
@@ -20,6 +27,14 @@ export interface SqliteImageInput {
|
|
|
20
27
|
* on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
|
|
21
28
|
* host omits it and serves matching stored images or the rows lane.
|
|
22
29
|
*/
|
|
23
|
-
export type SqliteImageBuilder = (input: SqliteImageInput) => Promise<
|
|
24
|
-
/**
|
|
25
|
-
export
|
|
30
|
+
export type SqliteImageBuilder = (input: SqliteImageInput) => Promise<SqliteImage>;
|
|
31
|
+
/** A built image and the number of leading input rows it holds. */
|
|
32
|
+
export interface SqliteImage {
|
|
33
|
+
readonly bytes: Uint8Array;
|
|
34
|
+
readonly rowCount: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Populate a §5.3 image database with the leading input rows that fit
|
|
38
|
+
* `maxBytes`. Returns how many rows it consumed.
|
|
39
|
+
*/
|
|
40
|
+
export declare function writeSqliteImage(db: SqliteDatabase, input: SqliteImageInput): Promise<number>;
|