@syncular/server 0.75.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +6 -0
- package/dist/context.js +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +128 -76
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +8 -0
- package/src/index.ts +1 -0
- package/src/pull.ts +199 -94
- package/src/segment-store.ts +10 -6
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
package/src/sqlite-image.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { rollbackAfterError } from './storage-errors';
|
|
2
2
|
/**
|
|
3
3
|
* SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
|
|
4
|
-
* database file carrying one table's
|
|
5
|
-
* the
|
|
4
|
+
* database file carrying one range of a table's effective-scope snapshot at
|
|
5
|
+
* the table's scope pin, plus the single-row `_syncular_segment` metadata
|
|
6
6
|
* table. Runtime entries provide the concrete SQLite database.
|
|
7
7
|
*
|
|
8
8
|
* Images are NOT byte-deterministic (§5.3): the content address pins the
|
|
@@ -58,6 +58,13 @@ export interface SqliteImageInput {
|
|
|
58
58
|
readonly schemaVersion: number;
|
|
59
59
|
readonly asOfCommitSeq: number;
|
|
60
60
|
readonly scopeDigest: string;
|
|
61
|
+
/**
|
|
62
|
+
* The image's file-size bound (`ServerLimits.maxImageSegmentBytes`). The
|
|
63
|
+
* builder stops before the row that could pass it; the first row always
|
|
64
|
+
* goes in.
|
|
65
|
+
*/
|
|
66
|
+
readonly maxBytes: number;
|
|
67
|
+
/** The range's rows in row order, from its start to the table's end. */
|
|
61
68
|
readonly rowBatches:
|
|
62
69
|
| AsyncIterable<readonly StoredRow[]>
|
|
63
70
|
| Iterable<readonly StoredRow[]>;
|
|
@@ -73,13 +80,22 @@ export interface SqliteImageInput {
|
|
|
73
80
|
*/
|
|
74
81
|
export type SqliteImageBuilder = (
|
|
75
82
|
input: SqliteImageInput,
|
|
76
|
-
) => Promise<
|
|
83
|
+
) => Promise<SqliteImage>;
|
|
77
84
|
|
|
78
|
-
/**
|
|
85
|
+
/** A built image and the number of leading input rows it holds. */
|
|
86
|
+
export interface SqliteImage {
|
|
87
|
+
readonly bytes: Uint8Array;
|
|
88
|
+
readonly rowCount: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Populate a §5.3 image database with the leading input rows that fit
|
|
93
|
+
* `maxBytes`. Returns how many rows it consumed.
|
|
94
|
+
*/
|
|
79
95
|
export async function writeSqliteImage(
|
|
80
96
|
db: SqliteDatabase,
|
|
81
97
|
input: SqliteImageInput,
|
|
82
|
-
): Promise<
|
|
98
|
+
): Promise<number> {
|
|
83
99
|
const { table, rowBatches } = input;
|
|
84
100
|
const primaryKey = table.columns[table.primaryKeyIndex]?.name;
|
|
85
101
|
const columnDefs = table.columns.map((column) => {
|
|
@@ -103,15 +119,33 @@ export async function writeSqliteImage(
|
|
|
103
119
|
`INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
|
|
104
120
|
VALUES (${names.map(() => '?').join(', ')})`,
|
|
105
121
|
);
|
|
122
|
+
const pageSize = db
|
|
123
|
+
.query<{ page_size: number }>('PRAGMA page_size')
|
|
124
|
+
.get()!.page_size;
|
|
125
|
+
const pageCount = db.query<{ page_count: number }>('PRAGMA page_count');
|
|
106
126
|
db.exec('BEGIN');
|
|
107
127
|
try {
|
|
108
128
|
let rowCount = 0;
|
|
109
|
-
|
|
129
|
+
// An upper bound of the file size, re-measured when it nears the bound.
|
|
130
|
+
let size = pageCount.get()!.page_count * pageSize;
|
|
131
|
+
fill: for await (const rows of rowBatches) {
|
|
110
132
|
for (const row of rows) {
|
|
133
|
+
// An appended row grows the table and its primary-key index by at
|
|
134
|
+
// most its cell (bounded by the payload plus a 9-byte varint per
|
|
135
|
+
// column, the version and the rowid), overflow rounding and a split
|
|
136
|
+
// per tree level.
|
|
137
|
+
const growth =
|
|
138
|
+
2 * (row.payload.length + 9 * (table.columns.length + 2)) +
|
|
139
|
+
16 * pageSize;
|
|
140
|
+
if (rowCount > 0 && size + growth > input.maxBytes) {
|
|
141
|
+
size = pageCount.get()!.page_count * pageSize;
|
|
142
|
+
if (size + growth > input.maxBytes) break fill;
|
|
143
|
+
}
|
|
111
144
|
const values = decodeRow(table.columns, row.payload);
|
|
112
145
|
insert.run(...values.map(toSql), row.serverVersion);
|
|
146
|
+
size += growth;
|
|
147
|
+
rowCount += 1;
|
|
113
148
|
}
|
|
114
|
-
rowCount += rows.length;
|
|
115
149
|
}
|
|
116
150
|
db.query(
|
|
117
151
|
`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
|
|
@@ -124,6 +158,7 @@ export async function writeSqliteImage(
|
|
|
124
158
|
);
|
|
125
159
|
|
|
126
160
|
db.exec('COMMIT');
|
|
161
|
+
return rowCount;
|
|
127
162
|
} catch (error) {
|
|
128
163
|
rollbackAfterError(() => {
|
|
129
164
|
db.exec('ROLLBACK');
|
package/src/sqlite-node.ts
CHANGED
|
@@ -48,12 +48,13 @@ export const buildSqliteImage: SqliteImageBuilder = async (input) => {
|
|
|
48
48
|
const path = join(directory, 'segment.db');
|
|
49
49
|
const db = new NodeSqliteDatabase(path);
|
|
50
50
|
try {
|
|
51
|
+
let rowCount: number;
|
|
51
52
|
try {
|
|
52
|
-
await writeSqliteImage(db, input);
|
|
53
|
+
rowCount = await writeSqliteImage(db, input);
|
|
53
54
|
} finally {
|
|
54
55
|
db.close();
|
|
55
56
|
}
|
|
56
|
-
return new Uint8Array(readFileSync(path));
|
|
57
|
+
return { bytes: new Uint8Array(readFileSync(path)), rowCount };
|
|
57
58
|
} finally {
|
|
58
59
|
rmSync(directory, { recursive: true, force: true });
|
|
59
60
|
}
|
|
@@ -313,13 +313,23 @@ export class SqliteSegmentStore implements SegmentStore {
|
|
|
313
313
|
const publication = this.db
|
|
314
314
|
.query<
|
|
315
315
|
{ segment_id: string },
|
|
316
|
-
[
|
|
316
|
+
[
|
|
317
|
+
string,
|
|
318
|
+
string,
|
|
319
|
+
string,
|
|
320
|
+
number,
|
|
321
|
+
string,
|
|
322
|
+
string,
|
|
323
|
+
number,
|
|
324
|
+
string | null,
|
|
325
|
+
number,
|
|
326
|
+
]
|
|
317
327
|
>(
|
|
318
328
|
`SELECT segment_id FROM sync_segment_publications
|
|
319
329
|
WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
|
|
320
|
-
AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS
|
|
330
|
+
AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS ?
|
|
321
331
|
AND expires_at_ms > ?
|
|
322
|
-
LIMIT 1`,
|
|
332
|
+
ORDER BY created_at_ms DESC LIMIT 1`,
|
|
323
333
|
)
|
|
324
334
|
.get(
|
|
325
335
|
key.partition,
|
|
@@ -329,20 +339,31 @@ export class SqliteSegmentStore implements SegmentStore {
|
|
|
329
339
|
key.mediaType,
|
|
330
340
|
key.scopeDigest,
|
|
331
341
|
key.asOfCommitSeq,
|
|
342
|
+
key.rowCursor,
|
|
332
343
|
nowMs,
|
|
333
344
|
);
|
|
334
345
|
const legacy = this.db
|
|
335
346
|
.query<
|
|
336
347
|
{ segment_id: string },
|
|
337
|
-
[
|
|
348
|
+
[
|
|
349
|
+
string,
|
|
350
|
+
string,
|
|
351
|
+
string,
|
|
352
|
+
number,
|
|
353
|
+
string,
|
|
354
|
+
string,
|
|
355
|
+
number,
|
|
356
|
+
string | null,
|
|
357
|
+
number,
|
|
358
|
+
]
|
|
338
359
|
>(
|
|
339
360
|
`SELECT s.segment_id
|
|
340
361
|
FROM sync_segments s
|
|
341
362
|
JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
|
|
342
363
|
WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
|
|
343
|
-
AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS
|
|
364
|
+
AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS ?
|
|
344
365
|
AND s.expires_at_ms > ?
|
|
345
|
-
LIMIT 1`,
|
|
366
|
+
ORDER BY s.created_at_ms DESC LIMIT 1`,
|
|
346
367
|
)
|
|
347
368
|
.get(
|
|
348
369
|
key.partition,
|
|
@@ -352,6 +373,7 @@ export class SqliteSegmentStore implements SegmentStore {
|
|
|
352
373
|
key.mediaType,
|
|
353
374
|
key.scopeDigest,
|
|
354
375
|
key.asOfCommitSeq,
|
|
376
|
+
key.rowCursor,
|
|
355
377
|
nowMs,
|
|
356
378
|
);
|
|
357
379
|
const segmentId = publication?.segment_id ?? legacy?.segment_id;
|
|
@@ -366,7 +388,7 @@ export class SqliteSegmentStore implements SegmentStore {
|
|
|
366
388
|
mediaType: key.mediaType,
|
|
367
389
|
scopeDigest: key.scopeDigest,
|
|
368
390
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
369
|
-
rowCursor:
|
|
391
|
+
rowCursor: key.rowCursor,
|
|
370
392
|
nowMs,
|
|
371
393
|
});
|
|
372
394
|
if (match === undefined) return undefined;
|