@syncular/server 0.75.0 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  import { rollbackAfterError } from './storage-errors.js';
2
2
  /**
3
3
  * SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
4
- * database file carrying one table's whole effective-scope snapshot at
5
- * the bootstrap pin, plus the single-row `_syncular_segment` metadata
4
+ * database file carrying one range of a table's effective-scope snapshot at
5
+ * the table's scope pin, plus the single-row `_syncular_segment` metadata
6
6
  * table. Runtime entries provide the concrete SQLite database.
7
7
  *
8
8
  * Images are NOT byte-deterministic (§5.3): the content address pins the
@@ -45,7 +45,10 @@ function toSql(value) {
45
45
  return value ? 1 : 0;
46
46
  return value;
47
47
  }
48
- /** Populate a §5.3 image database for a whole-table snapshot. */
48
+ /**
49
+ * Populate a §5.3 image database with the leading input rows that fit
50
+ * `maxBytes`. Returns how many rows it consumed.
51
+ */
49
52
  export async function writeSqliteImage(db, input) {
50
53
  const { table, rowBatches } = input;
51
54
  const primaryKey = table.columns[table.primaryKeyIndex]?.name;
@@ -66,18 +69,37 @@ export async function writeSqliteImage(db, input) {
66
69
  ];
67
70
  const insert = db.query(`INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
68
71
  VALUES (${names.map(() => '?').join(', ')})`);
72
+ const pageSize = db
73
+ .query('PRAGMA page_size')
74
+ .get().page_size;
75
+ const pageCount = db.query('PRAGMA page_count');
69
76
  db.exec('BEGIN');
70
77
  try {
71
78
  let rowCount = 0;
72
- for await (const rows of rowBatches) {
79
+ // An upper bound of the file size, re-measured when it nears the bound.
80
+ let size = pageCount.get().page_count * pageSize;
81
+ fill: for await (const rows of rowBatches) {
73
82
  for (const row of rows) {
83
+ // An appended row grows the table and its primary-key index by at
84
+ // most its cell (bounded by the payload plus a 9-byte varint per
85
+ // column, the version and the rowid), overflow rounding and a split
86
+ // per tree level.
87
+ const growth = 2 * (row.payload.length + 9 * (table.columns.length + 2)) +
88
+ 16 * pageSize;
89
+ if (rowCount > 0 && size + growth > input.maxBytes) {
90
+ size = pageCount.get().page_count * pageSize;
91
+ if (size + growth > input.maxBytes)
92
+ break fill;
93
+ }
74
94
  const values = decodeRow(table.columns, row.payload);
75
95
  insert.run(...values.map(toSql), row.serverVersion);
96
+ size += growth;
97
+ rowCount += 1;
76
98
  }
77
- rowCount += rows.length;
78
99
  }
79
100
  db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(table.name, input.schemaVersion, input.asOfCommitSeq, input.scopeDigest, rowCount);
80
101
  db.exec('COMMIT');
102
+ return rowCount;
81
103
  }
82
104
  catch (error) {
83
105
  rollbackAfterError(() => {
@@ -35,13 +35,14 @@ export const buildSqliteImage = async (input) => {
35
35
  const path = join(directory, 'segment.db');
36
36
  const db = new NodeSqliteDatabase(path);
37
37
  try {
38
+ let rowCount;
38
39
  try {
39
- await writeSqliteImage(db, input);
40
+ rowCount = await writeSqliteImage(db, input);
40
41
  }
41
42
  finally {
42
43
  db.close();
43
44
  }
44
- return new Uint8Array(readFileSync(path));
45
+ return { bytes: new Uint8Array(readFileSync(path)), rowCount };
45
46
  }
46
47
  finally {
47
48
  rmSync(directory, { recursive: true, force: true });
@@ -183,19 +183,19 @@ export class SqliteSegmentStore {
183
183
  const publication = this.db
184
184
  .query(`SELECT segment_id FROM sync_segment_publications
185
185
  WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
186
- AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
186
+ AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS ?
187
187
  AND expires_at_ms > ?
188
- LIMIT 1`)
189
- .get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
188
+ ORDER BY created_at_ms DESC LIMIT 1`)
189
+ .get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, key.rowCursor, nowMs);
190
190
  const legacy = this.db
191
191
  .query(`SELECT s.segment_id
192
192
  FROM sync_segments s
193
193
  JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
194
194
  WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
195
- AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS NULL
195
+ AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS ?
196
196
  AND s.expires_at_ms > ?
197
- LIMIT 1`)
198
- .get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
197
+ ORDER BY s.created_at_ms DESC LIMIT 1`)
198
+ .get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, key.rowCursor, nowMs);
199
199
  const segmentId = publication?.segment_id ?? legacy?.segment_id;
200
200
  if (segmentId === undefined)
201
201
  return undefined;
@@ -210,7 +210,7 @@ export class SqliteSegmentStore {
210
210
  mediaType: key.mediaType,
211
211
  scopeDigest: key.scopeDigest,
212
212
  asOfCommitSeq: key.asOfCommitSeq,
213
- rowCursor: null,
213
+ rowCursor: key.rowCursor,
214
214
  nowMs,
215
215
  });
216
216
  if (match === undefined)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.75.0",
3
+ "version": "0.76.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.75.0"
71
+ "@syncular/core": "0.76.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
@@ -23,10 +23,11 @@
23
23
  * objects.
24
24
  *
25
25
  * {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
26
- * Whole-table reuse pointer (§5.3): written only when
27
- * `rowCursor === null`, body = the record JSON. The reuse key is the
26
+ * Image reuse pointer (§5.3): written for every sqlite image and every
27
+ * first rows page, body = the record JSON. The reuse key is the
28
28
  * canonical JSON array `[partition, logEpoch, table, schemaVersion,
29
- * mediaType, scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a
29
+ * mediaType, scopeDigest, asOfCommitSeq]` plus the range start when it is
30
+ * not null, so `find` is one GET (plus a
30
31
  * HEAD to confirm the segment object itself still exists — lifecycle GC
31
32
  * may remove objects independently of pointers).
32
33
  *
@@ -319,6 +320,8 @@ export class BucketSegmentStore implements SegmentStore {
319
320
  key.mediaType,
320
321
  key.scopeDigest,
321
322
  key.asOfCommitSeq,
323
+ // A first range keeps the pointer key it had before range images.
324
+ ...(key.rowCursor === null ? [] : [key.rowCursor]),
322
325
  ]);
323
326
  return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
324
327
  }
@@ -425,7 +428,7 @@ export class BucketSegmentStore implements SegmentStore {
425
428
  sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
426
429
  });
427
430
  }
428
- if (metadata.rowCursor === null) {
431
+ if (metadata.rowCursor === null || metadata.mediaType === 'sqlite') {
429
432
  await this.#bucket.put(await this.#findKeyFor(metadata), {
430
433
  body: new TextEncoder().encode(recordJson),
431
434
  contentType: 'application/json',
@@ -556,7 +559,7 @@ export class BucketSegmentStore implements SegmentStore {
556
559
  mediaType: key.mediaType,
557
560
  scopeDigest: key.scopeDigest,
558
561
  asOfCommitSeq: key.asOfCommitSeq,
559
- rowCursor: null,
562
+ rowCursor: key.rowCursor,
560
563
  nowMs,
561
564
  });
562
565
  if (publication === undefined) return undefined;
package/src/context.ts CHANGED
@@ -104,6 +104,12 @@ export interface ServerLimits {
104
104
  * pass it alone resets the units it reaches, which bootstrap instead.
105
105
  */
106
106
  readonly maxLiveSectionBytes: number;
107
+ /**
108
+ * §5.3 bound on one sqlite image's file size: the server cuts a table into
109
+ * range images in row order, each within this bound. A row larger than the
110
+ * bound gets an image of its own.
111
+ */
112
+ readonly maxImageSegmentBytes: number;
107
113
  }
108
114
 
109
115
  export const DEFAULT_LIMITS: ServerLimits = {
@@ -118,6 +124,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
118
124
  maxWatchesPerSession: 100,
119
125
  // The live section is buffered whole before the first unit section.
120
126
  maxLiveSectionBytes: 16 * 1024 * 1024,
127
+ // §5.3 DECISION (2026-11-17): a client holds about three images at once.
128
+ maxImageSegmentBytes: 32 * 1024 * 1024,
121
129
  };
122
130
 
123
131
  /** Receives applied commits for realtime fanout (§8.2). */
package/src/index.ts CHANGED
@@ -85,6 +85,7 @@ export * from './sqlite-driver';
85
85
  export {
86
86
  IMAGE_METADATA_TABLE,
87
87
  IMAGE_VERSION_COLUMN,
88
+ type SqliteImage,
88
89
  type SqliteImageBuilder,
89
90
  type SqliteImageInput,
90
91
  writeSqliteImage,