@syncular/client 0.72.0 → 0.72.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/apply.d.ts CHANGED
@@ -71,9 +71,9 @@ export interface SqliteSegmentDescriptor {
71
71
  readonly scopeDigest: string;
72
72
  }
73
73
  /**
74
- * Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
75
- * rows: validate the in-file metadata against the descriptor, validate the
76
- * data table's column names/order against the generated schema, run the
74
+ * Apply a §5.3 sqlite image in committed chunks of at most 65,536 rows,
75
+ * each sized from the time the chunk before took: validate the in-file
76
+ * metadata against the descriptor, validate the data table's column names/order against the generated schema, run the
77
77
  * §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
78
78
  * primary-key upsert — `_syncular_version` lands in
79
79
  * `_sync_version` exactly like a rows segment's per-row `serverVersion`.
@@ -87,6 +87,8 @@ export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledC
87
87
  readonly transaction?: ApplyTransaction;
88
88
  readonly onProgress?: (rowsProcessed: number) => void;
89
89
  readonly isCurrent?: () => boolean;
90
+ /** Monotonic milliseconds that time each chunk. */
91
+ readonly now?: () => number;
90
92
  }): Promise<number>;
91
93
  /**
92
94
  * Apply a decoded rows segment: each block in one local transaction
package/dist/apply.js CHANGED
@@ -266,14 +266,15 @@ function imageInvalid(detail) {
266
266
  throw invalidRequest(`sqlite segment rejected: ${detail} (§5.3)`);
267
267
  }
268
268
  /**
269
- * Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
270
- * rows: validate the in-file metadata against the descriptor, validate the
271
- * data table's column names/order against the generated schema, run the
269
+ * Apply a §5.3 sqlite image in committed chunks of at most 65,536 rows,
270
+ * each sized from the time the chunk before took: validate the in-file
271
+ * metadata against the descriptor, validate the data table's column names/order against the generated schema, run the
272
272
  * §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
273
273
  * primary-key upsert — `_syncular_version` lands in
274
274
  * `_sync_version` exactly like a rows segment's per-row `serverVersion`.
275
275
  */
276
276
  export async function applySqliteSegment(db, schema, table, bytes, descriptor, options) {
277
+ const now = options.now ?? (() => performance.now());
277
278
  const withImage = db.withSqliteImage?.bind(db);
278
279
  if (withImage === undefined) {
279
280
  throw invalidRequest('database cannot import images; disable accept bit 2 (§4.2)');
@@ -333,12 +334,12 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
333
334
  .map((name) => `${name}=excluded.${name}`)
334
335
  .join(', ');
335
336
  const version = quoteIdent(SYNC_VERSION_COLUMN);
336
- // A statement copies the 1,024 rows after `from`; NULL keys cannot hide
337
- // between statements.
338
- const range = (from) => {
337
+ // A statement copies the `rows` rows after `from`; NULL keys cannot
338
+ // hide between statements.
339
+ const range = (from, rows) => {
339
340
  const params = from === undefined ? [] : [from];
340
341
  const predicates = from === undefined ? [] : [`${primaryKey}>?`];
341
- const boundary = db.query(`SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, params)[0]?.b;
342
+ const boundary = db.query(`SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET ${rows - 1}`, params)[0]?.b;
342
343
  if (boundary === null)
343
344
  imageInvalid('image primary key must not be null');
344
345
  if (boundary !== undefined) {
@@ -347,13 +348,17 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
347
348
  }
348
349
  return [boundary, predicates.join(' AND ') || 'true', params];
349
350
  };
350
- let [boundary, where, params] = range();
351
+ let [boundary, where, params] = range(undefined, 1024);
351
352
  let applied = 0;
352
- // §5.3: a transaction holds one statement at first and doubles after each
353
- // commit up to 64; a replaying transaction holds one.
354
- for (let statements = 1;; statements = Math.min(statements * 2, 64)) {
353
+ // §5.3: the first chunk holds 1,024 rows. A later chunk holds at most
354
+ // twice the rows of the one before, at most 65,536, and at most the rows
355
+ // the one before applied per 100 ms from its start to its COMMIT; a
356
+ // replaying chunk holds one statement.
357
+ for (let size = 1024;;) {
355
358
  if (options.isCurrent?.() === false)
356
359
  return applied;
360
+ const started = now();
361
+ let chunk = 0;
357
362
  const first = applied === 0;
358
363
  const replay = countOutbox(db) ||
359
364
  db.query(`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? LIMIT 1`, [
@@ -378,16 +383,18 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
378
383
  const kept = new Set();
379
384
  const skipped = new Set();
380
385
  const landing = rows.filter((row) => lands(db, table, String(row[table.primaryKey]), Number(row._syncular_version), kept, skipped));
381
- for (let statement = 1;; statement++) {
386
+ for (;;) {
382
387
  db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`, params);
383
- applied += Number(db.query(`SELECT count(*)AS n FROM ${source} WHERE ${where}`, params)[0]?.n);
388
+ const copied = Number(db.query(`SELECT count(*)AS n FROM ${source} WHERE ${where}`, params)[0]?.n);
389
+ applied += copied;
390
+ chunk += copied;
384
391
  options.onProgress?.(applied);
385
392
  if (replay ||
386
- statement === statements ||
393
+ chunk >= size ||
387
394
  boundary === undefined ||
388
395
  applied === count)
389
396
  break;
390
- [boundary, where, params] = range(boundary);
397
+ [boundary, where, params] = range(boundary, Math.min(1024, size - chunk));
391
398
  }
392
399
  if (protectedRows)
393
400
  for (const row of landing) {
@@ -398,10 +405,12 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
398
405
  }
399
406
  return rebaseScope(scope, kept, skipped);
400
407
  });
408
+ const duration = now() - started;
401
409
  await yieldToHost();
402
410
  if (boundary === undefined || applied === count)
403
411
  break;
404
- [boundary, where, params] = range(boundary);
412
+ size = Math.max(1, Math.min(2 * chunk, 65_536, Math.floor((chunk * 100) / duration)));
413
+ [boundary, where, params] = range(boundary, Math.min(1024, size));
405
414
  }
406
415
  if (applied !== count)
407
416
  imageInvalid('image applied row count does not match descriptor');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/client",
3
- "version": "0.72.0",
3
+ "version": "0.72.1",
4
4
  "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -121,11 +121,11 @@
121
121
  },
122
122
  "dependencies": {
123
123
  "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
124
- "@syncular/core": "0.72.0"
124
+ "@syncular/core": "0.72.1"
125
125
  },
126
126
  "devDependencies": {
127
127
  "@electric-sql/pglite": "^0.5.4",
128
- "@syncular/server": "0.72.0",
128
+ "@syncular/server": "0.72.1",
129
129
  "playwright": "1.63.0",
130
130
  "vite": "8.3.4"
131
131
  }
package/src/apply.ts CHANGED
@@ -469,9 +469,9 @@ function imageInvalid(detail: string): never {
469
469
  }
470
470
 
471
471
  /**
472
- * Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
473
- * rows: validate the in-file metadata against the descriptor, validate the
474
- * data table's column names/order against the generated schema, run the
472
+ * Apply a §5.3 sqlite image in committed chunks of at most 65,536 rows,
473
+ * each sized from the time the chunk before took: validate the in-file
474
+ * metadata against the descriptor, validate the data table's column names/order against the generated schema, run the
475
475
  * §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
476
476
  * primary-key upsert — `_syncular_version` lands in
477
477
  * `_sync_version` exactly like a rows segment's per-row `serverVersion`.
@@ -491,8 +491,11 @@ export async function applySqliteSegment(
491
491
  readonly transaction?: ApplyTransaction;
492
492
  readonly onProgress?: (rowsProcessed: number) => void;
493
493
  readonly isCurrent?: () => boolean;
494
+ /** Monotonic milliseconds that time each chunk. */
495
+ readonly now?: () => number;
494
496
  },
495
497
  ): Promise<number> {
498
+ const now = options.now ?? (() => performance.now());
496
499
  const withImage = db.withSqliteImage?.bind(db);
497
500
  if (withImage === undefined) {
498
501
  throw invalidRequest(
@@ -567,13 +570,13 @@ export async function applySqliteSegment(
567
570
  .map((name) => `${name}=excluded.${name}`)
568
571
  .join(', ');
569
572
  const version = quoteIdent(SYNC_VERSION_COLUMN);
570
- // A statement copies the 1,024 rows after `from`; NULL keys cannot hide
571
- // between statements.
572
- const range = (from?: SqlValue) => {
573
+ // A statement copies the `rows` rows after `from`; NULL keys cannot
574
+ // hide between statements.
575
+ const range = (from: SqlValue | undefined, rows: number) => {
573
576
  const params = from === undefined ? [] : [from];
574
577
  const predicates = from === undefined ? [] : [`${primaryKey}>?`];
575
578
  const boundary = db.query(
576
- `SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
579
+ `SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET ${rows - 1}`,
577
580
  params,
578
581
  )[0]?.b;
579
582
  if (boundary === null) imageInvalid('image primary key must not be null');
@@ -583,12 +586,16 @@ export async function applySqliteSegment(
583
586
  }
584
587
  return [boundary, predicates.join(' AND ') || 'true', params] as const;
585
588
  };
586
- let [boundary, where, params] = range();
589
+ let [boundary, where, params] = range(undefined, 1024);
587
590
  let applied = 0;
588
- // §5.3: a transaction holds one statement at first and doubles after each
589
- // commit up to 64; a replaying transaction holds one.
590
- for (let statements = 1; ; statements = Math.min(statements * 2, 64)) {
591
+ // §5.3: the first chunk holds 1,024 rows. A later chunk holds at most
592
+ // twice the rows of the one before, at most 65,536, and at most the rows
593
+ // the one before applied per 100 ms from its start to its COMMIT; a
594
+ // replaying chunk holds one statement.
595
+ for (let size = 1024; ;) {
591
596
  if (options.isCurrent?.() === false) return applied;
597
+ const started = now();
598
+ let chunk = 0;
592
599
  const first = applied === 0;
593
600
  const replay =
594
601
  countOutbox(db) ||
@@ -625,26 +632,31 @@ export async function applySqliteSegment(
625
632
  skipped,
626
633
  ),
627
634
  );
628
- for (let statement = 1; ; statement++) {
635
+ for (;;) {
629
636
  db.exec(
630
637
  `INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
631
638
  params,
632
639
  );
633
- applied += Number(
640
+ const copied = Number(
634
641
  db.query(
635
642
  `SELECT count(*)AS n FROM ${source} WHERE ${where}`,
636
643
  params,
637
644
  )[0]?.n,
638
645
  );
646
+ applied += copied;
647
+ chunk += copied;
639
648
  options.onProgress?.(applied);
640
649
  if (
641
650
  replay ||
642
- statement === statements ||
651
+ chunk >= size ||
643
652
  boundary === undefined ||
644
653
  applied === count
645
654
  )
646
655
  break;
647
- [boundary, where, params] = range(boundary);
656
+ [boundary, where, params] = range(
657
+ boundary,
658
+ Math.min(1024, size - chunk),
659
+ );
648
660
  }
649
661
  if (protectedRows)
650
662
  for (const row of landing) {
@@ -663,9 +675,14 @@ export async function applySqliteSegment(
663
675
  }
664
676
  return rebaseScope(scope, kept, skipped);
665
677
  });
678
+ const duration = now() - started;
666
679
  await yieldToHost();
667
680
  if (boundary === undefined || applied === count) break;
668
- [boundary, where, params] = range(boundary);
681
+ size = Math.max(
682
+ 1,
683
+ Math.min(2 * chunk, 65_536, Math.floor((chunk * 100) / duration)),
684
+ );
685
+ [boundary, where, params] = range(boundary, Math.min(1024, size));
669
686
  }
670
687
  if (applied !== count)
671
688
  imageInvalid('image applied row count does not match descriptor');