@syncular/server 0.75.0 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  import { rollbackAfterError } from './storage-errors';
2
2
  /**
3
3
  * SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
4
- * database file carrying one table's whole effective-scope snapshot at
5
- * the bootstrap pin, plus the single-row `_syncular_segment` metadata
4
+ * database file carrying one range of a table's effective-scope snapshot at
5
+ * the table's scope pin, plus the single-row `_syncular_segment` metadata
6
6
  * table. Runtime entries provide the concrete SQLite database.
7
7
  *
8
8
  * Images are NOT byte-deterministic (§5.3): the content address pins the
@@ -58,6 +58,13 @@ export interface SqliteImageInput {
58
58
  readonly schemaVersion: number;
59
59
  readonly asOfCommitSeq: number;
60
60
  readonly scopeDigest: string;
61
+ /**
62
+ * The image's file-size bound (`ServerLimits.maxImageSegmentBytes`). The
63
+ * builder stops before the row that could pass it; the first row always
64
+ * goes in.
65
+ */
66
+ readonly maxBytes: number;
67
+ /** The range's rows in row order, from its start to the table's end. */
61
68
  readonly rowBatches:
62
69
  | AsyncIterable<readonly StoredRow[]>
63
70
  | Iterable<readonly StoredRow[]>;
@@ -73,13 +80,22 @@ export interface SqliteImageInput {
73
80
  */
74
81
  export type SqliteImageBuilder = (
75
82
  input: SqliteImageInput,
76
- ) => Promise<Uint8Array>;
83
+ ) => Promise<SqliteImage>;
77
84
 
78
- /** Populate a §5.3 image database for a whole-table snapshot. */
85
+ /** A built image and the number of leading input rows it holds. */
86
+ export interface SqliteImage {
87
+ readonly bytes: Uint8Array;
88
+ readonly rowCount: number;
89
+ }
90
+
91
+ /**
92
+ * Populate a §5.3 image database with the leading input rows that fit
93
+ * `maxBytes`. Returns how many rows it consumed.
94
+ */
79
95
  export async function writeSqliteImage(
80
96
  db: SqliteDatabase,
81
97
  input: SqliteImageInput,
82
- ): Promise<void> {
98
+ ): Promise<number> {
83
99
  const { table, rowBatches } = input;
84
100
  const primaryKey = table.columns[table.primaryKeyIndex]?.name;
85
101
  const columnDefs = table.columns.map((column) => {
@@ -103,15 +119,33 @@ export async function writeSqliteImage(
103
119
  `INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
104
120
  VALUES (${names.map(() => '?').join(', ')})`,
105
121
  );
122
+ const pageSize = db
123
+ .query<{ page_size: number }>('PRAGMA page_size')
124
+ .get()!.page_size;
125
+ const pageCount = db.query<{ page_count: number }>('PRAGMA page_count');
106
126
  db.exec('BEGIN');
107
127
  try {
108
128
  let rowCount = 0;
109
- for await (const rows of rowBatches) {
129
+ // An upper bound of the file size, re-measured when it nears the bound.
130
+ let size = pageCount.get()!.page_count * pageSize;
131
+ fill: for await (const rows of rowBatches) {
110
132
  for (const row of rows) {
133
+ // An appended row grows the table and its primary-key index by at
134
+ // most its cell (bounded by the payload plus a 9-byte varint per
135
+ // column, the version and the rowid), overflow rounding and a split
136
+ // per tree level.
137
+ const growth =
138
+ 2 * (row.payload.length + 9 * (table.columns.length + 2)) +
139
+ 16 * pageSize;
140
+ if (rowCount > 0 && size + growth > input.maxBytes) {
141
+ size = pageCount.get()!.page_count * pageSize;
142
+ if (size + growth > input.maxBytes) break fill;
143
+ }
111
144
  const values = decodeRow(table.columns, row.payload);
112
145
  insert.run(...values.map(toSql), row.serverVersion);
146
+ size += growth;
147
+ rowCount += 1;
113
148
  }
114
- rowCount += rows.length;
115
149
  }
116
150
  db.query(
117
151
  `INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
@@ -124,6 +158,7 @@ export async function writeSqliteImage(
124
158
  );
125
159
 
126
160
  db.exec('COMMIT');
161
+ return rowCount;
127
162
  } catch (error) {
128
163
  rollbackAfterError(() => {
129
164
  db.exec('ROLLBACK');
@@ -48,12 +48,13 @@ export const buildSqliteImage: SqliteImageBuilder = async (input) => {
48
48
  const path = join(directory, 'segment.db');
49
49
  const db = new NodeSqliteDatabase(path);
50
50
  try {
51
+ let rowCount: number;
51
52
  try {
52
- await writeSqliteImage(db, input);
53
+ rowCount = await writeSqliteImage(db, input);
53
54
  } finally {
54
55
  db.close();
55
56
  }
56
- return new Uint8Array(readFileSync(path));
57
+ return { bytes: new Uint8Array(readFileSync(path)), rowCount };
57
58
  } finally {
58
59
  rmSync(directory, { recursive: true, force: true });
59
60
  }
@@ -313,13 +313,23 @@ export class SqliteSegmentStore implements SegmentStore {
313
313
  const publication = this.db
314
314
  .query<
315
315
  { segment_id: string },
316
- [string, string, string, number, string, string, number, number]
316
+ [
317
+ string,
318
+ string,
319
+ string,
320
+ number,
321
+ string,
322
+ string,
323
+ number,
324
+ string | null,
325
+ number,
326
+ ]
317
327
  >(
318
328
  `SELECT segment_id FROM sync_segment_publications
319
329
  WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
320
- AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
330
+ AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS ?
321
331
  AND expires_at_ms > ?
322
- LIMIT 1`,
332
+ ORDER BY created_at_ms DESC LIMIT 1`,
323
333
  )
324
334
  .get(
325
335
  key.partition,
@@ -329,20 +339,31 @@ export class SqliteSegmentStore implements SegmentStore {
329
339
  key.mediaType,
330
340
  key.scopeDigest,
331
341
  key.asOfCommitSeq,
342
+ key.rowCursor,
332
343
  nowMs,
333
344
  );
334
345
  const legacy = this.db
335
346
  .query<
336
347
  { segment_id: string },
337
- [string, string, string, number, string, string, number, number]
348
+ [
349
+ string,
350
+ string,
351
+ string,
352
+ number,
353
+ string,
354
+ string,
355
+ number,
356
+ string | null,
357
+ number,
358
+ ]
338
359
  >(
339
360
  `SELECT s.segment_id
340
361
  FROM sync_segments s
341
362
  JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
342
363
  WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
343
- AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS NULL
364
+ AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS ?
344
365
  AND s.expires_at_ms > ?
345
- LIMIT 1`,
366
+ ORDER BY s.created_at_ms DESC LIMIT 1`,
346
367
  )
347
368
  .get(
348
369
  key.partition,
@@ -352,6 +373,7 @@ export class SqliteSegmentStore implements SegmentStore {
352
373
  key.mediaType,
353
374
  key.scopeDigest,
354
375
  key.asOfCommitSeq,
376
+ key.rowCursor,
355
377
  nowMs,
356
378
  );
357
379
  const segmentId = publication?.segment_id ?? legacy?.segment_id;
@@ -366,7 +388,7 @@ export class SqliteSegmentStore implements SegmentStore {
366
388
  mediaType: key.mediaType,
367
389
  scopeDigest: key.scopeDigest,
368
390
  asOfCommitSeq: key.asOfCommitSeq,
369
- rowCursor: null,
391
+ rowCursor: key.rowCursor,
370
392
  nowMs,
371
393
  });
372
394
  if (match === undefined) return undefined;