@syncular/server 0.75.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +6 -0
- package/dist/context.js +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +128 -76
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +8 -0
- package/src/index.ts +1 -0
- package/src/pull.ts +199 -94
- package/src/segment-store.ts +10 -6
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
package/dist/sqlite-image.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { rollbackAfterError } from './storage-errors.js';
|
|
2
2
|
/**
|
|
3
3
|
* SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
|
|
4
|
-
* database file carrying one table's
|
|
5
|
-
* the
|
|
4
|
+
* database file carrying one range of a table's effective-scope snapshot at
|
|
5
|
+
* the table's scope pin, plus the single-row `_syncular_segment` metadata
|
|
6
6
|
* table. Runtime entries provide the concrete SQLite database.
|
|
7
7
|
*
|
|
8
8
|
* Images are NOT byte-deterministic (§5.3): the content address pins the
|
|
@@ -45,7 +45,10 @@ function toSql(value) {
|
|
|
45
45
|
return value ? 1 : 0;
|
|
46
46
|
return value;
|
|
47
47
|
}
|
|
48
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Populate a §5.3 image database with the leading input rows that fit
|
|
50
|
+
* `maxBytes`. Returns how many rows it consumed.
|
|
51
|
+
*/
|
|
49
52
|
export async function writeSqliteImage(db, input) {
|
|
50
53
|
const { table, rowBatches } = input;
|
|
51
54
|
const primaryKey = table.columns[table.primaryKeyIndex]?.name;
|
|
@@ -66,18 +69,37 @@ export async function writeSqliteImage(db, input) {
|
|
|
66
69
|
];
|
|
67
70
|
const insert = db.query(`INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
|
|
68
71
|
VALUES (${names.map(() => '?').join(', ')})`);
|
|
72
|
+
const pageSize = db
|
|
73
|
+
.query('PRAGMA page_size')
|
|
74
|
+
.get().page_size;
|
|
75
|
+
const pageCount = db.query('PRAGMA page_count');
|
|
69
76
|
db.exec('BEGIN');
|
|
70
77
|
try {
|
|
71
78
|
let rowCount = 0;
|
|
72
|
-
|
|
79
|
+
// An upper bound of the file size, re-measured when it nears the bound.
|
|
80
|
+
let size = pageCount.get().page_count * pageSize;
|
|
81
|
+
fill: for await (const rows of rowBatches) {
|
|
73
82
|
for (const row of rows) {
|
|
83
|
+
// An appended row grows the table and its primary-key index by at
|
|
84
|
+
// most its cell (bounded by the payload plus a 9-byte varint per
|
|
85
|
+
// column, the version and the rowid), overflow rounding and a split
|
|
86
|
+
// per tree level.
|
|
87
|
+
const growth = 2 * (row.payload.length + 9 * (table.columns.length + 2)) +
|
|
88
|
+
16 * pageSize;
|
|
89
|
+
if (rowCount > 0 && size + growth > input.maxBytes) {
|
|
90
|
+
size = pageCount.get().page_count * pageSize;
|
|
91
|
+
if (size + growth > input.maxBytes)
|
|
92
|
+
break fill;
|
|
93
|
+
}
|
|
74
94
|
const values = decodeRow(table.columns, row.payload);
|
|
75
95
|
insert.run(...values.map(toSql), row.serverVersion);
|
|
96
|
+
size += growth;
|
|
97
|
+
rowCount += 1;
|
|
76
98
|
}
|
|
77
|
-
rowCount += rows.length;
|
|
78
99
|
}
|
|
79
100
|
db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(table.name, input.schemaVersion, input.asOfCommitSeq, input.scopeDigest, rowCount);
|
|
80
101
|
db.exec('COMMIT');
|
|
102
|
+
return rowCount;
|
|
81
103
|
}
|
|
82
104
|
catch (error) {
|
|
83
105
|
rollbackAfterError(() => {
|
package/dist/sqlite-node.js
CHANGED
|
@@ -35,13 +35,14 @@ export const buildSqliteImage = async (input) => {
|
|
|
35
35
|
const path = join(directory, 'segment.db');
|
|
36
36
|
const db = new NodeSqliteDatabase(path);
|
|
37
37
|
try {
|
|
38
|
+
let rowCount;
|
|
38
39
|
try {
|
|
39
|
-
await writeSqliteImage(db, input);
|
|
40
|
+
rowCount = await writeSqliteImage(db, input);
|
|
40
41
|
}
|
|
41
42
|
finally {
|
|
42
43
|
db.close();
|
|
43
44
|
}
|
|
44
|
-
return new Uint8Array(readFileSync(path));
|
|
45
|
+
return { bytes: new Uint8Array(readFileSync(path)), rowCount };
|
|
45
46
|
}
|
|
46
47
|
finally {
|
|
47
48
|
rmSync(directory, { recursive: true, force: true });
|
|
@@ -183,19 +183,19 @@ export class SqliteSegmentStore {
|
|
|
183
183
|
const publication = this.db
|
|
184
184
|
.query(`SELECT segment_id FROM sync_segment_publications
|
|
185
185
|
WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
|
|
186
|
-
AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS
|
|
186
|
+
AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS ?
|
|
187
187
|
AND expires_at_ms > ?
|
|
188
|
-
LIMIT 1`)
|
|
189
|
-
.get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
|
|
188
|
+
ORDER BY created_at_ms DESC LIMIT 1`)
|
|
189
|
+
.get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, key.rowCursor, nowMs);
|
|
190
190
|
const legacy = this.db
|
|
191
191
|
.query(`SELECT s.segment_id
|
|
192
192
|
FROM sync_segments s
|
|
193
193
|
JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
|
|
194
194
|
WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
|
|
195
|
-
AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS
|
|
195
|
+
AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS ?
|
|
196
196
|
AND s.expires_at_ms > ?
|
|
197
|
-
LIMIT 1`)
|
|
198
|
-
.get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
|
|
197
|
+
ORDER BY s.created_at_ms DESC LIMIT 1`)
|
|
198
|
+
.get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, key.rowCursor, nowMs);
|
|
199
199
|
const segmentId = publication?.segment_id ?? legacy?.segment_id;
|
|
200
200
|
if (segmentId === undefined)
|
|
201
201
|
return undefined;
|
|
@@ -210,7 +210,7 @@ export class SqliteSegmentStore {
|
|
|
210
210
|
mediaType: key.mediaType,
|
|
211
211
|
scopeDigest: key.scopeDigest,
|
|
212
212
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
213
|
-
rowCursor:
|
|
213
|
+
rowCursor: key.rowCursor,
|
|
214
214
|
nowMs,
|
|
215
215
|
});
|
|
216
216
|
if (match === undefined)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.77.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.77.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
|
@@ -23,10 +23,11 @@
|
|
|
23
23
|
* objects.
|
|
24
24
|
*
|
|
25
25
|
* {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* Image reuse pointer (§5.3): written for every sqlite image and every
|
|
27
|
+
* first rows page, body = the record JSON. The reuse key is the
|
|
28
28
|
* canonical JSON array `[partition, logEpoch, table, schemaVersion,
|
|
29
|
-
* mediaType, scopeDigest, asOfCommitSeq]
|
|
29
|
+
* mediaType, scopeDigest, asOfCommitSeq]` plus the range start when it is
|
|
30
|
+
* not null, so `find` is one GET (plus a
|
|
30
31
|
* HEAD to confirm the segment object itself still exists — lifecycle GC
|
|
31
32
|
* may remove objects independently of pointers).
|
|
32
33
|
*
|
|
@@ -319,6 +320,8 @@ export class BucketSegmentStore implements SegmentStore {
|
|
|
319
320
|
key.mediaType,
|
|
320
321
|
key.scopeDigest,
|
|
321
322
|
key.asOfCommitSeq,
|
|
323
|
+
// A first range keeps the pointer key it had before range images.
|
|
324
|
+
...(key.rowCursor === null ? [] : [key.rowCursor]),
|
|
322
325
|
]);
|
|
323
326
|
return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
|
|
324
327
|
}
|
|
@@ -425,7 +428,7 @@ export class BucketSegmentStore implements SegmentStore {
|
|
|
425
428
|
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
426
429
|
});
|
|
427
430
|
}
|
|
428
|
-
if (metadata.rowCursor === null) {
|
|
431
|
+
if (metadata.rowCursor === null || metadata.mediaType === 'sqlite') {
|
|
429
432
|
await this.#bucket.put(await this.#findKeyFor(metadata), {
|
|
430
433
|
body: new TextEncoder().encode(recordJson),
|
|
431
434
|
contentType: 'application/json',
|
|
@@ -556,7 +559,7 @@ export class BucketSegmentStore implements SegmentStore {
|
|
|
556
559
|
mediaType: key.mediaType,
|
|
557
560
|
scopeDigest: key.scopeDigest,
|
|
558
561
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
559
|
-
rowCursor:
|
|
562
|
+
rowCursor: key.rowCursor,
|
|
560
563
|
nowMs,
|
|
561
564
|
});
|
|
562
565
|
if (publication === undefined) return undefined;
|
package/src/context.ts
CHANGED
|
@@ -104,6 +104,12 @@ export interface ServerLimits {
|
|
|
104
104
|
* pass it alone resets the units it reaches, which bootstrap instead.
|
|
105
105
|
*/
|
|
106
106
|
readonly maxLiveSectionBytes: number;
|
|
107
|
+
/**
|
|
108
|
+
* §5.3 bound on one sqlite image's file size: the server cuts a table into
|
|
109
|
+
* range images in row order, each within this bound. A row larger than the
|
|
110
|
+
* bound gets an image of its own.
|
|
111
|
+
*/
|
|
112
|
+
readonly maxImageSegmentBytes: number;
|
|
107
113
|
}
|
|
108
114
|
|
|
109
115
|
export const DEFAULT_LIMITS: ServerLimits = {
|
|
@@ -118,6 +124,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
118
124
|
maxWatchesPerSession: 100,
|
|
119
125
|
// The live section is buffered whole before the first unit section.
|
|
120
126
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
127
|
+
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
128
|
+
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
121
129
|
};
|
|
122
130
|
|
|
123
131
|
/** Receives applied commits for realtime fanout (§8.2). */
|