@syncular/server 0.15.40 → 0.15.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -217,6 +217,16 @@ unknown command outcome, reuse the original idempotency key because changing it
217
217
  can execute the operation twice. The full inspection and recovery recipe is in
218
218
  the public [server guide](https://syncular.dev/guide-server/#seeding-data).
219
219
 
220
+ One extra rule applies when the seed actor changes. A Syncular `clientId` is
221
+ bound to its first actor inside a partition. Keep the client ID stable for seed
222
+ revisions under that actor, but allocate both a new purpose-specific `clientId`
223
+ and a new `commitId` when moving a seed to another actor. Changing only the
224
+ commit ID correctly fails with `sync.invalid_client_id`; do not erase the
225
+ database to bypass that identity evidence. For example, move
226
+ `catalog-import/seed-user/catalog-v1` to
227
+ `catalog-import/server-authority/catalog-v2`, not merely to
228
+ `catalog-import/seed-user/catalog-v2`.
229
+
220
230
  The task-oriented [concurrency and conflict-correction guide](https://syncular.dev/guide-concurrency-correction/)
221
231
  shows version projection, aggregate rollback, corrected replacement commits,
222
232
  explicit acknowledgement, and restart-safe recovery UI together.
@@ -64,7 +64,17 @@ class D1Transaction {
64
64
  /** Live snapshot of `max_commit_seq`, advanced within this transaction. */
65
65
  #maxCommitSeq;
66
66
  #pushApplyCheckpoint;
67
- #lastApplicationOpIndex;
67
+ /**
68
+ * Distinct opIndexes of buffered application upserts. A constraint that
69
+ * only fires at `db.batch(...)` commit time (e.g. NOT NULL/CHECK the
70
+ * `#assertNoUniqueCollision` pre-check cannot see) is reported by D1 for
71
+ * the batch as a whole, and the batch API offers no way to re-run
72
+ * statements individually without applying them. So attribution at commit
73
+ * time is exact when the commit buffered exactly one application opIndex
74
+ * and conservatively omitted otherwise (the push layer then records its
75
+ * first-op default).
76
+ */
77
+ #applicationOpIndexes = new Set();
68
78
  /**
69
79
  * Read-your-own-writes overlay (§6.2 needs `getRow` to see buffered writes
70
80
  * of the same commit — e.g. two ops touching the same row): keyed
@@ -99,13 +109,26 @@ class D1Transaction {
99
109
  .first();
100
110
  return record === null ? undefined : toStoredRow(record);
101
111
  }
112
+ async getPushResult(clientId, clientCommitId) {
113
+ this.#assertOpen();
114
+ // D1 reads run in autocommit; the push layer's duplicate re-check happens
115
+ // before this transaction buffers any write, so a direct read is exact.
116
+ const record = await this.#db
117
+ .prepare('SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?')
118
+ .bind(this.#partition, clientId, clientCommitId)
119
+ .first();
120
+ if (record === null)
121
+ return undefined;
122
+ try {
123
+ return deserializePushResult(record.result);
124
+ }
125
+ catch {
126
+ throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
127
+ }
128
+ }
102
129
  async scanRows(query) {
103
130
  this.#assertOpen();
104
- assertScopeIndexedScan(query);
105
- const variables = Object.keys(query.scopeFilter).sort();
106
- const firstVariable = variables[0];
107
- if (firstVariable === undefined)
108
- return [];
131
+ const firstVariable = assertScopeIndexedScan(query);
109
132
  const firstValues = query.scopeFilter[firstVariable] ?? [];
110
133
  if (firstValues.length === 0)
111
134
  return [];
@@ -206,6 +229,7 @@ class D1Transaction {
206
229
  this.#buffer.length = checkpoint;
207
230
  this.#pending.clear();
208
231
  this.#maxCommitSeq = undefined;
232
+ this.#applicationOpIndexes.clear();
209
233
  await this.putPushResult(clientId, clientCommitId, result);
210
234
  await this.commit();
211
235
  }
@@ -272,7 +296,9 @@ class D1Transaction {
272
296
  this.#assertOpen();
273
297
  const compiled = this.#resolveTable(table);
274
298
  await this.#assertNoUniqueCollision(compiled, row, context?.opIndex);
275
- this.#lastApplicationOpIndex = context?.opIndex;
299
+ if (context?.opIndex !== undefined) {
300
+ this.#applicationOpIndexes.add(context.opIndex);
301
+ }
276
302
  this.#pending.set(_a.#key(table, row.rowId), {
277
303
  kind: 'row',
278
304
  row,
@@ -367,7 +393,9 @@ class D1Transaction {
367
393
  // D1 batches are atomic. Keep this logical transaction open so the
368
394
  // push layer can discard its buffered candidates and persist the
369
395
  // terminal rejection while the external partition queue is retained.
370
- throw new StorageConstraintError(error, this.#lastApplicationOpIndex);
396
+ // See `#applicationOpIndexes` for the attribution contract.
397
+ const opIndexes = [...this.#applicationOpIndexes];
398
+ throw new StorageConstraintError(error, opIndexes.length === 1 ? opIndexes[0] : undefined);
371
399
  }
372
400
  throw error;
373
401
  }
@@ -620,11 +648,7 @@ export class D1ServerStorage {
620
648
  return commits;
621
649
  }
622
650
  async scanRows(partition, query) {
623
- assertScopeIndexedScan(query);
624
- const variables = Object.keys(query.scopeFilter).sort();
625
- const firstVariable = variables[0];
626
- if (firstVariable === undefined)
627
- return [];
651
+ const firstVariable = assertScopeIndexedScan(query);
628
652
  const firstValues = query.scopeFilter[firstVariable] ?? [];
629
653
  if (firstValues.length === 0)
630
654
  return [];
@@ -8,9 +8,11 @@
8
8
  * `PgExecutor` interface (see the server README).
9
9
  *
10
10
  * pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
11
- * `ROLLBACK` on the one connection and relies on the caller not interleaving
12
- * unrelated queries during a transaction scope (the storage layer holds one
13
- * transaction at a time per push).
11
+ * `ROLLBACK` on the one connection. Overlapping `transaction` calls are
12
+ * serialized through a promise chain (mirroring `SqliteServerStorage`'s
13
+ * begin() FIFO): a nested BEGIN on Postgres is a warning-level no-op, so
14
+ * interleaved scopes would silently collapse into one SQL transaction and
15
+ * break the push layer's serialization guarantee on this dev driver.
14
16
  */
15
17
  import type { PGlite } from '@electric-sql/pglite';
16
18
  import type { PgExecutor } from './pg-executor.js';
@@ -12,18 +12,32 @@ function queryable(db) {
12
12
  export function pgliteExecutor(db) {
13
13
  const like = db;
14
14
  const q = queryable(like);
15
+ // One BEGIN…COMMIT/ROLLBACK scope at a time on the single connection (see
16
+ // the file header).
17
+ let transactionTail = Promise.resolve();
15
18
  return {
16
19
  query: q.query,
17
20
  async transaction(fn) {
18
- await like.exec('BEGIN');
21
+ const previous = transactionTail;
22
+ let release;
23
+ transactionTail = new Promise((resolve) => {
24
+ release = resolve;
25
+ });
26
+ await previous;
19
27
  try {
20
- const result = await fn(q);
21
- await like.exec('COMMIT');
22
- return result;
28
+ await like.exec('BEGIN');
29
+ try {
30
+ const result = await fn(q);
31
+ await like.exec('COMMIT');
32
+ return result;
33
+ }
34
+ catch (error) {
35
+ await like.exec('ROLLBACK');
36
+ throw error;
37
+ }
23
38
  }
24
- catch (error) {
25
- await like.exec('ROLLBACK');
26
- throw error;
39
+ finally {
40
+ release();
27
41
  }
28
42
  },
29
43
  async close() {
@@ -240,6 +240,17 @@ async function writeRowOn(q, compiled, partition, row) {
240
240
  VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`, [partition, compiled.name, variable, value, row.rowId]);
241
241
  }
242
242
  }
243
+ async function getPushResultOn(q, partition, clientId, clientCommitId) {
244
+ const { rows } = await q.query('SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3', [partition, clientId, clientCommitId]);
245
+ if (rows[0] === undefined)
246
+ return undefined;
247
+ try {
248
+ return deserializePushResult(asJson(rows[0].result));
249
+ }
250
+ catch {
251
+ throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
252
+ }
253
+ }
243
254
  class PostgresTransaction {
244
255
  #client;
245
256
  #partition;
@@ -264,13 +275,16 @@ class PostgresTransaction {
264
275
  this.#assertOpen();
265
276
  return getRowOn(this.#client, this.#resolveTable(table), this.#partition, rowId);
266
277
  }
278
+ getPushResult(clientId, clientCommitId) {
279
+ this.#assertOpen();
280
+ // Runs on this transaction's pinned client: the push layer's duplicate
281
+ // re-check happens while the partition lock is held, and a pool-level
282
+ // read there would wait for a second connection.
283
+ return getPushResultOn(this.#client, this.#partition, clientId, clientCommitId);
284
+ }
267
285
  async scanRows(query) {
268
286
  this.#assertOpen();
269
- assertScopeIndexedScan(query);
270
- const variables = Object.keys(query.scopeFilter).sort();
271
- const firstVariable = variables[0];
272
- if (firstVariable === undefined)
273
- return [];
287
+ const firstVariable = assertScopeIndexedScan(query);
274
288
  const firstValues = query.scopeFilter[firstVariable] ?? [];
275
289
  if (firstValues.length === 0)
276
290
  return [];
@@ -498,6 +512,11 @@ export class PostgresServerStorage {
498
512
  WHERE table_schema = current_schema() AND table_name = $1`, [table.name]);
499
513
  if (rows.length > 0) {
500
514
  existing.set(table.name, new Set(rows.map((r) => r.column_name)));
515
+ // Parity with the SQLite/D1 `origin === 'c'` filter: only
516
+ // free-standing indexes enter the rebuild set. Constraint-owned
517
+ // indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
518
+ // through their constraint, so DROP INDEX on them would abort
519
+ // the migration transaction.
501
520
  const indexes = await client.query(`SELECT index_class.relname AS index_name
502
521
  FROM pg_catalog.pg_class AS table_class
503
522
  JOIN pg_catalog.pg_namespace AS namespace
@@ -508,7 +527,12 @@ export class PostgresServerStorage {
508
527
  ON index_class.oid = index_meta.indexrelid
509
528
  WHERE namespace.nspname = current_schema()
510
529
  AND table_class.relname = $1
511
- AND NOT index_meta.indisprimary`, [table.name]);
530
+ AND NOT index_meta.indisprimary
531
+ AND NOT EXISTS (
532
+ SELECT 1
533
+ FROM pg_catalog.pg_constraint AS owning_constraint
534
+ WHERE owning_constraint.conindid = index_meta.indexrelid
535
+ )`, [table.name]);
512
536
  existingIndexes.set(table.name, new Set(indexes.rows.map((index) => index.index_name)));
513
537
  }
514
538
  }
@@ -597,16 +621,8 @@ export class PostgresServerStorage {
597
621
  getRow(partition, table, rowId) {
598
622
  return getRowOn(this.#exec, this.table(table), partition, rowId);
599
623
  }
600
- async getPushResult(partition, clientId, clientCommitId) {
601
- const { rows } = await this.#exec.query('SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3', [partition, clientId, clientCommitId]);
602
- if (rows[0] === undefined)
603
- return undefined;
604
- try {
605
- return deserializePushResult(asJson(rows[0].result));
606
- }
607
- catch {
608
- throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
609
- }
624
+ getPushResult(partition, clientId, clientCommitId) {
625
+ return getPushResultOn(this.#exec, partition, clientId, clientCommitId);
610
626
  }
611
627
  async readCommitWindow(partition, query) {
612
628
  const variables = Object.keys(query.scopeFilter).sort();
@@ -697,11 +713,7 @@ export class PostgresServerStorage {
697
713
  return commits;
698
714
  }
699
715
  async scanRows(partition, query) {
700
- assertScopeIndexedScan(query);
701
- const variables = Object.keys(query.scopeFilter).sort();
702
- const firstVariable = variables[0];
703
- if (firstVariable === undefined)
704
- return [];
716
+ const firstVariable = assertScopeIndexedScan(query);
705
717
  const firstValues = query.scopeFilter[firstVariable] ?? [];
706
718
  if (firstValues.length === 0)
707
719
  return [];
package/dist/push.js CHANGED
@@ -551,9 +551,16 @@ export async function processPushCommitWithTrace(ctx, schema, resolved, clientId
551
551
  await lockPartitionForPush();
552
552
  // The optimistic lookup above may have raced another delivery. Re-check
553
553
  // only after acquiring partition serialization and before any operation
554
- // read, validation, merge, or staged write.
554
+ // read, validation, merge, or staged write. The re-check runs on the
555
+ // transaction's own connection when the backend provides one: a pooled
556
+ // Postgres client holding the partition lock would otherwise wait for a
557
+ // second pool slot and deadlock against pushes waiting on the lock. A
558
+ // racing duplicate's result commits before the lock releases, so the
559
+ // transaction-scoped read observes it.
555
560
  try {
556
- const serializedPersisted = await storage.getPushResult(partition, clientId, frame.clientCommitId);
561
+ const serializedPersisted = tx.getPushResult !== undefined
562
+ ? await tx.getPushResult(clientId, frame.clientCommitId)
563
+ : await storage.getPushResult(partition, clientId, frame.clientCommitId);
557
564
  if (serializedPersisted !== undefined) {
558
565
  await tx.rollback();
559
566
  return processedPushCommit(frame.clientCommitId, serializedPersisted, true);
@@ -652,12 +659,28 @@ export async function processPushCommitWithTrace(ctx, schema, resolved, clientId
652
659
  await tx.rollback();
653
660
  throw new Error('storage transaction lost atomic push rejection finalization support');
654
661
  }
655
- await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
656
- const canonical = await storage.getPushResult(partition, clientId, frame.clientCommitId);
657
- if (canonical === undefined) {
658
- throw new Error('push rejection finalization did not persist an outcome');
662
+ try {
663
+ await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
664
+ const canonical = await storage.getPushResult(partition, clientId, frame.clientCommitId);
665
+ if (canonical === undefined) {
666
+ throw new Error('push rejection finalization did not persist an outcome');
667
+ }
668
+ return processedPushCommit(frame.clientCommitId, canonical, canonical.cacheIdentity !== stored.cacheIdentity);
669
+ }
670
+ catch (finalizationError) {
671
+ // A failed finalization must still release the transaction: on
672
+ // SQLite an unreleased BEGIN wedges the storage's global begin()
673
+ // queue, on Postgres it leaks the pinned pool client. rollback() is
674
+ // a no-op when the finalization already committed.
675
+ try {
676
+ await tx.rollback();
677
+ }
678
+ catch {
679
+ // Surface the finalization failure; a rollback failure would
680
+ // otherwise mask it.
681
+ }
682
+ throw finalizationError;
659
683
  }
660
- return processedPushCommit(frame.clientCommitId, canonical, canonical.cacheIdentity !== stored.cacheIdentity);
661
684
  }
662
685
  await tx.rollback();
663
686
  throw error;
@@ -69,11 +69,28 @@ export declare function createTableDdl(table: CompiledTable, dialect: Relational
69
69
  * `existingColumns` (introspected). Added columns are nullable (header note).
70
70
  */
71
71
  export declare function addColumnDdl(table: CompiledTable, existingColumns: ReadonlySet<string>, dialect: RelationalDialect): string[];
72
+ /**
73
+ * Ownership marker for server-created projection indexes. A version bump
74
+ * rebuilds declared indexes by dropping every index Syncular owns and
75
+ * re-creating the declared set; this prefix is what marks an index as
76
+ * Syncular-owned, so operator-added tuning indexes survive bumps. Portable
77
+ * identifier validation reserves the `sync_` prefix for the server storage
78
+ * namespace, which keeps user and operator index names out of it.
79
+ */
80
+ export declare const SYNC_INDEX_PREFIX = "sync_ix_";
81
+ /**
82
+ * Physical name of one declared index: the ownership prefix plus the
83
+ * declared name. When that would exceed the 63-byte Postgres identifier
84
+ * limit, the declared name is replaced by its FNV-1a hash so the physical
85
+ * name stays deterministic, unique, and within the limit.
86
+ */
87
+ export declare function physicalIndexName(declaredName: string): string;
72
88
  /**
73
89
  * CREATE INDEX IF NOT EXISTS for the table's user-declared indexes
74
- * (DESIGN "user indexes" — the same names/columns the client materializes;
75
- * cross-table index-name uniqueness is the user's schema concern, exactly
76
- * as it is client-side).
90
+ * (DESIGN "user indexes" — the same declared names/columns the client
91
+ * materializes; cross-table index-name uniqueness is the user's schema
92
+ * concern, exactly as it is client-side). Server-side the physical name
93
+ * carries the {@link SYNC_INDEX_PREFIX} ownership marker.
77
94
  */
78
95
  export declare function createIndexDdl(table: CompiledTable): string[];
79
96
  /** Idempotent removal of one Syncular-owned relational projection index. */
@@ -246,5 +263,11 @@ export declare function rewritePlan(table: CompiledTable, oldLayout: readonly St
246
263
  * missing columns, and rebuild declared secondary indexes. Rebuilding during
247
264
  * a version bump supports DROP INDEX and same-name index replacement without
248
265
  * requiring historical index definitions in the stored column-layout marker.
266
+ *
267
+ * The drop set is limited to Syncular-owned indexes: physical names carrying
268
+ * {@link SYNC_INDEX_PREFIX}, plus bare declared names (databases whose
269
+ * projection indexes predate the ownership prefix migrate onto the prefixed
270
+ * scheme through this rule). Operator-added tuning indexes keep their own
271
+ * names and survive every bump.
249
272
  */
250
273
  export declare function schemaDdl(schema: CompiledSchema, existingColumnsByTable: ReadonlyMap<string, ReadonlySet<string>>, dialect: RelationalDialect, existingIndexesByTable?: ReadonlyMap<string, ReadonlySet<string>>): string[];
@@ -144,11 +144,43 @@ export function addColumnDdl(table, existingColumns, dialect) {
144
144
  }
145
145
  return out;
146
146
  }
147
+ /**
148
+ * Ownership marker for server-created projection indexes. A version bump
149
+ * rebuilds declared indexes by dropping every index Syncular owns and
150
+ * re-creating the declared set; this prefix is what marks an index as
151
+ * Syncular-owned, so operator-added tuning indexes survive bumps. Portable
152
+ * identifier validation reserves the `sync_` prefix for the server storage
153
+ * namespace, which keeps user and operator index names out of it.
154
+ */
155
+ export const SYNC_INDEX_PREFIX = 'sync_ix_';
156
+ /** Deterministic FNV-1a 64-bit hash, hex-encoded. Dependency-free so it runs
157
+ * identically on every server runtime. */
158
+ function fnv1a64Hex(value) {
159
+ let hash = 0xcbf29ce484222325n;
160
+ for (const byte of new TextEncoder().encode(value)) {
161
+ hash ^= BigInt(byte);
162
+ hash = (hash * 0x100000001b3n) & 0xffffffffffffffffn;
163
+ }
164
+ return hash.toString(16).padStart(16, '0');
165
+ }
166
+ /**
167
+ * Physical name of one declared index: the ownership prefix plus the
168
+ * declared name. When that would exceed the 63-byte Postgres identifier
169
+ * limit, the declared name is replaced by its FNV-1a hash so the physical
170
+ * name stays deterministic, unique, and within the limit.
171
+ */
172
+ export function physicalIndexName(declaredName) {
173
+ const prefixed = `${SYNC_INDEX_PREFIX}${declaredName}`;
174
+ if (new TextEncoder().encode(prefixed).length <= 63)
175
+ return prefixed;
176
+ return `${SYNC_INDEX_PREFIX}${fnv1a64Hex(declaredName)}`;
177
+ }
147
178
  /**
148
179
  * CREATE INDEX IF NOT EXISTS for the table's user-declared indexes
149
- * (DESIGN "user indexes" — the same names/columns the client materializes;
150
- * cross-table index-name uniqueness is the user's schema concern, exactly
151
- * as it is client-side).
180
+ * (DESIGN "user indexes" — the same declared names/columns the client
181
+ * materializes; cross-table index-name uniqueness is the user's schema
182
+ * concern, exactly as it is client-side). Server-side the physical name
183
+ * carries the {@link SYNC_INDEX_PREFIX} ownership marker.
152
184
  */
153
185
  export function createIndexDdl(table) {
154
186
  // User indexes name app columns — nothing to index without the projection.
@@ -159,7 +191,7 @@ export function createIndexDdl(table) {
159
191
  const columns = index.columns
160
192
  .map((column) => quoteIdent(column))
161
193
  .join(', ');
162
- return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(index.name)} ON ${quoteIdent(table.name)} (${columns})`;
194
+ return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
163
195
  });
164
196
  }
165
197
  /** Idempotent removal of one Syncular-owned relational projection index. */
@@ -541,21 +573,35 @@ export function rewritePlan(table, oldLayout, physicalColumnsBefore) {
541
573
  * missing columns, and rebuild declared secondary indexes. Rebuilding during
542
574
  * a version bump supports DROP INDEX and same-name index replacement without
543
575
  * requiring historical index definitions in the stored column-layout marker.
576
+ *
577
+ * The drop set is limited to Syncular-owned indexes: physical names carrying
578
+ * {@link SYNC_INDEX_PREFIX}, plus bare declared names (databases whose
579
+ * projection indexes predate the ownership prefix migrate onto the prefixed
580
+ * scheme through this rule). Operator-added tuning indexes keep their own
581
+ * names and survive every bump.
544
582
  */
545
583
  export function schemaDdl(schema, existingColumnsByTable, dialect, existingIndexesByTable = new Map()) {
546
- const out = [];
584
+ const drops = [];
585
+ const creates = [];
547
586
  for (const table of schema.tables.values()) {
548
587
  const existing = existingColumnsByTable.get(table.name);
549
588
  if (existing === undefined) {
550
- out.push(createTableDdl(table, dialect));
589
+ creates.push(createTableDdl(table, dialect));
551
590
  }
552
591
  else {
592
+ const declared = new Set(table.indexes.map((index) => index.name));
553
593
  for (const indexName of existingIndexesByTable.get(table.name) ?? []) {
554
- out.push(dropIndexDdl(indexName));
594
+ if (indexName.startsWith(SYNC_INDEX_PREFIX) ||
595
+ declared.has(indexName)) {
596
+ drops.push(dropIndexDdl(indexName));
597
+ }
555
598
  }
556
- out.push(...addColumnDdl(table, existing, dialect));
599
+ creates.push(...addColumnDdl(table, existing, dialect));
557
600
  }
558
- out.push(...createIndexDdl(table));
601
+ creates.push(...createIndexDdl(table));
559
602
  }
560
- return out;
603
+ // Every drop precedes every create: an index name moving between tables in
604
+ // one bump must release the global (SQLite) index namespace before the
605
+ // receiving table re-creates it.
606
+ return [...drops, ...creates];
561
607
  }
package/dist/schema.js CHANGED
@@ -8,11 +8,8 @@
8
8
  import { validatePortableRelationalIdentifier, } from '@syncular/core';
9
9
  const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
10
10
  /**
11
- * Identifier rules for relational server storage (DESIGN "current-row
12
- * tables"): app tables live in the same namespace as the sync
13
- * infrastructure tables (`sync_*`) and carry `_sync_*` meta columns, so
14
- * both prefixes are reserved; identifiers over 63 bytes would be silently
15
- * truncated by Postgres.
11
+ * Compile one `prefix:{variable}` scope pattern against the table's column
12
+ * list, resolving the scope column's positional index for row extraction.
16
13
  */
17
14
  function compilePattern(table, spec, columnIndex) {
18
15
  const pattern = typeof spec === 'string' ? spec : spec.pattern;
@@ -40,6 +37,11 @@ export function compileSchema(schema) {
40
37
  if (tables.has(table.name)) {
41
38
  throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
42
39
  }
40
+ // Identifier rules (DESIGN "current-row tables") live in core's
41
+ // validatePortableRelationalIdentifier, shared with typegen: app tables
42
+ // share a namespace with the sync infrastructure tables (`sync_*`) and
43
+ // carry `_sync_*` meta columns, so both prefixes are reserved, and
44
+ // Postgres silently truncates identifiers over 63 bytes.
43
45
  validatePortableRelationalIdentifier('table', table.name);
44
46
  const columnIndex = new Map();
45
47
  table.columns.forEach((column, index) => {
@@ -33,6 +33,12 @@ class SqliteTransaction {
33
33
  this.#assertOpen();
34
34
  return this.#storage.getRow(this.#partition, table, rowId);
35
35
  }
36
+ getPushResult(clientId, clientCommitId) {
37
+ this.#assertOpen();
38
+ // One shared bun:sqlite connection — this read runs inside this
39
+ // transaction's BEGIN IMMEDIATE.
40
+ return this.#storage.getPushResult(this.#partition, clientId, clientCommitId);
41
+ }
36
42
  scanRows(query) {
37
43
  this.#assertOpen();
38
44
  return this.#storage.scanRows(this.#partition, query);
@@ -114,11 +120,25 @@ class SqliteTransaction {
114
120
  }
115
121
  async commit() {
116
122
  this.#assertOpen();
117
- this.#open = false;
118
123
  try {
119
124
  this.#storage.db.exec('COMMIT');
120
125
  }
126
+ catch (error) {
127
+ // A failed COMMIT (SQLITE_BUSY from an external writer, I/O error)
128
+ // leaves the connection inside BEGIN IMMEDIATE. Roll back before the
129
+ // FIFO releases, so the next queued transaction starts on a clean
130
+ // connection.
131
+ try {
132
+ this.#storage.db.exec('ROLLBACK');
133
+ }
134
+ catch {
135
+ // Surface the COMMIT failure; a rollback failure would otherwise
136
+ // mask it.
137
+ }
138
+ throw error;
139
+ }
121
140
  finally {
141
+ this.#open = false;
122
142
  this.#release();
123
143
  }
124
144
  }
@@ -251,6 +271,10 @@ export class SqliteServerStorage {
251
271
  }
252
272
  }
253
273
  async begin(partition) {
274
+ // Deliberately global across partitions: SQLite is single-writer per
275
+ // database file, so one FIFO over the shared connection is the correct
276
+ // serialization unit — a per-partition queue would still contend on the
277
+ // same BEGIN IMMEDIATE writer lock.
254
278
  const previous = this.#transactionTail;
255
279
  let release;
256
280
  this.#transactionTail = new Promise((resolve) => {
@@ -370,11 +394,7 @@ export class SqliteServerStorage {
370
394
  return commits;
371
395
  }
372
396
  async scanRows(partition, query) {
373
- assertScopeIndexedScan(query);
374
- const variables = Object.keys(query.scopeFilter).sort();
375
- const firstVariable = variables[0];
376
- if (firstVariable === undefined)
377
- return [];
397
+ const firstVariable = assertScopeIndexedScan(query);
378
398
  const firstValues = query.scopeFilter[firstVariable] ?? [];
379
399
  if (firstValues.length === 0)
380
400
  return [];
@@ -1,6 +1,10 @@
1
1
  import type { CompiledTable, IndexSchema } from './schema.js';
2
2
  import type { IndexRowScanQuery, RowScanQuery } from './storage.js';
3
- /** Fail loudly instead of making an unsupported unscoped scan look empty. */
4
- export declare function assertScopeIndexedScan(query: RowScanQuery): void;
3
+ /**
4
+ * Fail loudly instead of making an unsupported unscoped scan look empty.
5
+ * Returns the first scope variable in sorted order — the one every adapter
6
+ * drives its inverted-index candidate selection with.
7
+ */
8
+ export declare function assertScopeIndexedScan(query: RowScanQuery): string;
5
9
  /** Validate and resolve one exact trusted-host relational index lookup. */
6
10
  export declare function resolveIndexRowScan(table: CompiledTable, query: IndexRowScanQuery): IndexSchema;
@@ -1,12 +1,18 @@
1
1
  import { StorageQueryError } from './storage-errors.js';
2
- /** Fail loudly instead of making an unsupported unscoped scan look empty. */
2
+ /**
3
+ * Fail loudly instead of making an unsupported unscoped scan look empty.
4
+ * Returns the first scope variable in sorted order — the one every adapter
5
+ * drives its inverted-index candidate selection with.
6
+ */
3
7
  export function assertScopeIndexedScan(query) {
4
8
  const scopeFilter = query.scopeFilter;
5
- if (scopeFilter === undefined ||
6
- scopeFilter === null ||
7
- Object.keys(scopeFilter).length === 0) {
9
+ const firstVariable = scopeFilter === undefined || scopeFilter === null
10
+ ? undefined
11
+ : Object.keys(scopeFilter).sort()[0];
12
+ if (firstVariable === undefined) {
8
13
  throw new StorageQueryError('sync.storage.scan_requires_scope');
9
14
  }
15
+ return firstVariable;
10
16
  }
11
17
  /** Validate and resolve one exact trusted-host relational index lookup. */
12
18
  export function resolveIndexRowScan(table, query) {
package/dist/storage.d.ts CHANGED
@@ -173,6 +173,17 @@ export interface ScopeActivityQuery {
173
173
  */
174
174
  export interface StorageTransaction {
175
175
  getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
176
+ /**
177
+ * Optional transaction-scoped idempotency lookup (§2.3) with the same
178
+ * semantics as `ServerStorage.getPushResult`, including the
179
+ * `sync.idempotency_cache_miss` throw. The push layer prefers it for the
180
+ * post-serialization duplicate re-check so the read runs on the
181
+ * transaction's own connection — a pooled Postgres client that holds the
182
+ * partition lock must never wait for a second pool slot mid-push. In-tree
183
+ * SQLite/Postgres/D1 adapters implement it; a custom adapter that omits it
184
+ * keeps the pool-level read.
185
+ */
186
+ getPushResult?(clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
176
187
  /**
177
188
  * Optional candidate-state scan used only by whole-commit validation.
178
189
  * In-tree SQLite/Postgres/D1 backends implement it with read-your-own-writes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.15.40",
3
+ "version": "0.15.43",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -53,7 +53,7 @@
53
53
  "!dist/**/*.test.d.ts"
54
54
  ],
55
55
  "dependencies": {
56
- "@syncular/core": "0.15.40"
56
+ "@syncular/core": "0.15.43"
57
57
  },
58
58
  "devDependencies": {
59
59
  "@electric-sql/pglite": "^0.5.4"
package/src/d1-storage.ts CHANGED
@@ -149,7 +149,17 @@ class D1Transaction implements StorageTransaction {
149
149
  /** Live snapshot of `max_commit_seq`, advanced within this transaction. */
150
150
  #maxCommitSeq: number | undefined;
151
151
  #pushApplyCheckpoint: number | undefined;
152
- #lastApplicationOpIndex: number | undefined;
152
+ /**
153
+ * Distinct opIndexes of buffered application upserts. A constraint that
154
+ * only fires at `db.batch(...)` commit time (e.g. NOT NULL/CHECK the
155
+ * `#assertNoUniqueCollision` pre-check cannot see) is reported by D1 for
156
+ * the batch as a whole, and the batch API offers no way to re-run
157
+ * statements individually without applying them. So attribution at commit
158
+ * time is exact when the commit buffered exactly one application opIndex
159
+ * and conservatively omitted otherwise (the push layer then records its
160
+ * first-op default).
161
+ */
162
+ readonly #applicationOpIndexes = new Set<number>();
153
163
  /**
154
164
  * Read-your-own-writes overlay (§6.2 needs `getRow` to see buffered writes
155
165
  * of the same commit — e.g. two ops touching the same row): keyed
@@ -194,12 +204,33 @@ class D1Transaction implements StorageTransaction {
194
204
  return record === null ? undefined : toStoredRow(record);
195
205
  }
196
206
 
207
+ async getPushResult(
208
+ clientId: string,
209
+ clientCommitId: string,
210
+ ): Promise<StoredPushResult | undefined> {
211
+ this.#assertOpen();
212
+ // D1 reads run in autocommit; the push layer's duplicate re-check happens
213
+ // before this transaction buffers any write, so a direct read is exact.
214
+ const record = await this.#db
215
+ .prepare(
216
+ 'SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?',
217
+ )
218
+ .bind(this.#partition, clientId, clientCommitId)
219
+ .first<{ result: string }>();
220
+ if (record === null) return undefined;
221
+ try {
222
+ return deserializePushResult(record.result);
223
+ } catch {
224
+ throw syncError(
225
+ 'sync.idempotency_cache_miss',
226
+ 'persisted push result unreadable (§6.3)',
227
+ );
228
+ }
229
+ }
230
+
197
231
  async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
198
232
  this.#assertOpen();
199
- assertScopeIndexedScan(query);
200
- const variables = Object.keys(query.scopeFilter).sort();
201
- const firstVariable = variables[0];
202
- if (firstVariable === undefined) return [];
233
+ const firstVariable = assertScopeIndexedScan(query);
203
234
  const firstValues = query.scopeFilter[firstVariable] ?? [];
204
235
  if (firstValues.length === 0) return [];
205
236
 
@@ -337,6 +368,7 @@ class D1Transaction implements StorageTransaction {
337
368
  this.#buffer.length = checkpoint;
338
369
  this.#pending.clear();
339
370
  this.#maxCommitSeq = undefined;
371
+ this.#applicationOpIndexes.clear();
340
372
  await this.putPushResult(clientId, clientCommitId, result);
341
373
  await this.commit();
342
374
  }
@@ -427,7 +459,9 @@ class D1Transaction implements StorageTransaction {
427
459
  this.#assertOpen();
428
460
  const compiled = this.#resolveTable(table);
429
461
  await this.#assertNoUniqueCollision(compiled, row, context?.opIndex);
430
- this.#lastApplicationOpIndex = context?.opIndex;
462
+ if (context?.opIndex !== undefined) {
463
+ this.#applicationOpIndexes.add(context.opIndex);
464
+ }
431
465
  this.#pending.set(D1Transaction.#key(table, row.rowId), {
432
466
  kind: 'row',
433
467
  row,
@@ -572,7 +606,12 @@ class D1Transaction implements StorageTransaction {
572
606
  // D1 batches are atomic. Keep this logical transaction open so the
573
607
  // push layer can discard its buffered candidates and persist the
574
608
  // terminal rejection while the external partition queue is retained.
575
- throw new StorageConstraintError(error, this.#lastApplicationOpIndex);
609
+ // See `#applicationOpIndexes` for the attribution contract.
610
+ const opIndexes = [...this.#applicationOpIndexes];
611
+ throw new StorageConstraintError(
612
+ error,
613
+ opIndexes.length === 1 ? opIndexes[0] : undefined,
614
+ );
576
615
  }
577
616
  throw error;
578
617
  }
@@ -928,10 +967,7 @@ export class D1ServerStorage implements ServerStorage {
928
967
  }
929
968
 
930
969
  async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
931
- assertScopeIndexedScan(query);
932
- const variables = Object.keys(query.scopeFilter).sort();
933
- const firstVariable = variables[0];
934
- if (firstVariable === undefined) return [];
970
+ const firstVariable = assertScopeIndexedScan(query);
935
971
  const firstValues = query.scopeFilter[firstVariable] ?? [];
936
972
  if (firstValues.length === 0) return [];
937
973
  // Candidates via the inverted index LEFT JOINed to the row table — one
@@ -8,9 +8,11 @@
8
8
  * `PgExecutor` interface (see the server README).
9
9
  *
10
10
  * pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
11
- * `ROLLBACK` on the one connection and relies on the caller not interleaving
12
- * unrelated queries during a transaction scope (the storage layer holds one
13
- * transaction at a time per push).
11
+ * `ROLLBACK` on the one connection. Overlapping `transaction` calls are
12
+ * serialized through a promise chain (mirroring `SqliteServerStorage`'s
13
+ * begin() FIFO): a nested BEGIN on Postgres is a warning-level no-op, so
14
+ * interleaved scopes would silently collapse into one SQL transaction and
15
+ * break the push layer's serialization guarantee on this dev driver.
14
16
  */
15
17
  import type { PGlite } from '@electric-sql/pglite';
16
18
  import type { PgExecutor, PgQueryable, PgRow } from './pg-executor';
@@ -43,17 +45,30 @@ function queryable(db: PgliteLike): PgQueryable {
43
45
  export function pgliteExecutor(db: PGlite | PgliteLike): PgExecutor {
44
46
  const like = db as PgliteLike;
45
47
  const q = queryable(like);
48
+ // One BEGIN…COMMIT/ROLLBACK scope at a time on the single connection (see
49
+ // the file header).
50
+ let transactionTail: Promise<void> = Promise.resolve();
46
51
  return {
47
52
  query: q.query,
48
53
  async transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T> {
49
- await like.exec('BEGIN');
54
+ const previous = transactionTail;
55
+ let release!: () => void;
56
+ transactionTail = new Promise<void>((resolve) => {
57
+ release = resolve;
58
+ });
59
+ await previous;
50
60
  try {
51
- const result = await fn(q);
52
- await like.exec('COMMIT');
53
- return result;
54
- } catch (error) {
55
- await like.exec('ROLLBACK');
56
- throw error;
61
+ await like.exec('BEGIN');
62
+ try {
63
+ const result = await fn(q);
64
+ await like.exec('COMMIT');
65
+ return result;
66
+ } catch (error) {
67
+ await like.exec('ROLLBACK');
68
+ throw error;
69
+ }
70
+ } finally {
71
+ release();
57
72
  }
58
73
  },
59
74
  async close() {
@@ -434,6 +434,27 @@ async function writeRowOn(
434
434
  }
435
435
  }
436
436
 
437
+ async function getPushResultOn(
438
+ q: PgQueryable,
439
+ partition: string,
440
+ clientId: string,
441
+ clientCommitId: string,
442
+ ): Promise<StoredPushResult | undefined> {
443
+ const { rows } = await q.query<{ result: unknown }>(
444
+ 'SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3',
445
+ [partition, clientId, clientCommitId],
446
+ );
447
+ if (rows[0] === undefined) return undefined;
448
+ try {
449
+ return deserializePushResult(asJson(rows[0].result));
450
+ } catch {
451
+ throw syncError(
452
+ 'sync.idempotency_cache_miss',
453
+ 'persisted push result unreadable (§6.3)',
454
+ );
455
+ }
456
+ }
457
+
437
458
  class PostgresTransaction implements StorageTransaction {
438
459
  #client: PgQueryable;
439
460
  #partition: string;
@@ -472,12 +493,25 @@ class PostgresTransaction implements StorageTransaction {
472
493
  );
473
494
  }
474
495
 
496
+ getPushResult(
497
+ clientId: string,
498
+ clientCommitId: string,
499
+ ): Promise<StoredPushResult | undefined> {
500
+ this.#assertOpen();
501
+ // Runs on this transaction's pinned client: the push layer's duplicate
502
+ // re-check happens while the partition lock is held, and a pool-level
503
+ // read there would wait for a second connection.
504
+ return getPushResultOn(
505
+ this.#client,
506
+ this.#partition,
507
+ clientId,
508
+ clientCommitId,
509
+ );
510
+ }
511
+
475
512
  async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
476
513
  this.#assertOpen();
477
- assertScopeIndexedScan(query);
478
- const variables = Object.keys(query.scopeFilter).sort();
479
- const firstVariable = variables[0];
480
- if (firstVariable === undefined) return [];
514
+ const firstVariable = assertScopeIndexedScan(query);
481
515
  const firstValues = query.scopeFilter[firstVariable] ?? [];
482
516
  if (firstValues.length === 0) return [];
483
517
  const sql = scanRowPageSql(
@@ -791,6 +825,11 @@ export class PostgresServerStorage implements ServerStorage {
791
825
  );
792
826
  if (rows.length > 0) {
793
827
  existing.set(table.name, new Set(rows.map((r) => r.column_name)));
828
+ // Parity with the SQLite/D1 `origin === 'c'` filter: only
829
+ // free-standing indexes enter the rebuild set. Constraint-owned
830
+ // indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
831
+ // through their constraint, so DROP INDEX on them would abort
832
+ // the migration transaction.
794
833
  const indexes = await client.query<{ index_name: string }>(
795
834
  `SELECT index_class.relname AS index_name
796
835
  FROM pg_catalog.pg_class AS table_class
@@ -802,7 +841,12 @@ export class PostgresServerStorage implements ServerStorage {
802
841
  ON index_class.oid = index_meta.indexrelid
803
842
  WHERE namespace.nspname = current_schema()
804
843
  AND table_class.relname = $1
805
- AND NOT index_meta.indisprimary`,
844
+ AND NOT index_meta.indisprimary
845
+ AND NOT EXISTS (
846
+ SELECT 1
847
+ FROM pg_catalog.pg_constraint AS owning_constraint
848
+ WHERE owning_constraint.conindid = index_meta.indexrelid
849
+ )`,
806
850
  [table.name],
807
851
  );
808
852
  existingIndexes.set(
@@ -948,24 +992,12 @@ export class PostgresServerStorage implements ServerStorage {
948
992
  return getRowOn(this.#exec, this.table(table), partition, rowId);
949
993
  }
950
994
 
951
- async getPushResult(
995
+ getPushResult(
952
996
  partition: string,
953
997
  clientId: string,
954
998
  clientCommitId: string,
955
999
  ): Promise<StoredPushResult | undefined> {
956
- const { rows } = await this.#exec.query<{ result: unknown }>(
957
- 'SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3',
958
- [partition, clientId, clientCommitId],
959
- );
960
- if (rows[0] === undefined) return undefined;
961
- try {
962
- return deserializePushResult(asJson(rows[0].result));
963
- } catch {
964
- throw syncError(
965
- 'sync.idempotency_cache_miss',
966
- 'persisted push result unreadable (§6.3)',
967
- );
968
- }
1000
+ return getPushResultOn(this.#exec, partition, clientId, clientCommitId);
969
1001
  }
970
1002
 
971
1003
  async readCommitWindow(
@@ -1054,10 +1086,7 @@ export class PostgresServerStorage implements ServerStorage {
1054
1086
  }
1055
1087
 
1056
1088
  async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
1057
- assertScopeIndexedScan(query);
1058
- const variables = Object.keys(query.scopeFilter).sort();
1059
- const firstVariable = variables[0];
1060
- if (firstVariable === undefined) return [];
1089
+ const firstVariable = assertScopeIndexedScan(query);
1061
1090
  const firstValues = query.scopeFilter[firstVariable] ?? [];
1062
1091
  if (firstValues.length === 0) return [];
1063
1092
  // Candidates via the inverted index (ordered + LIMITed at the covering
package/src/push.ts CHANGED
@@ -886,13 +886,21 @@ export async function processPushCommitWithTrace(
886
886
  await lockPartitionForPush();
887
887
  // The optimistic lookup above may have raced another delivery. Re-check
888
888
  // only after acquiring partition serialization and before any operation
889
- // read, validation, merge, or staged write.
889
+ // read, validation, merge, or staged write. The re-check runs on the
890
+ // transaction's own connection when the backend provides one: a pooled
891
+ // Postgres client holding the partition lock would otherwise wait for a
892
+ // second pool slot and deadlock against pushes waiting on the lock. A
893
+ // racing duplicate's result commits before the lock releases, so the
894
+ // transaction-scoped read observes it.
890
895
  try {
891
- const serializedPersisted = await storage.getPushResult(
892
- partition,
893
- clientId,
894
- frame.clientCommitId,
895
- );
896
+ const serializedPersisted =
897
+ tx.getPushResult !== undefined
898
+ ? await tx.getPushResult(clientId, frame.clientCommitId)
899
+ : await storage.getPushResult(
900
+ partition,
901
+ clientId,
902
+ frame.clientCommitId,
903
+ );
896
904
  if (serializedPersisted !== undefined) {
897
905
  await tx.rollback();
898
906
  return processedPushCommit(
@@ -1028,22 +1036,36 @@ export async function processPushCommitWithTrace(
1028
1036
  'storage transaction lost atomic push rejection finalization support',
1029
1037
  );
1030
1038
  }
1031
- await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
1032
- const canonical = await storage.getPushResult(
1033
- partition,
1034
- clientId,
1035
- frame.clientCommitId,
1036
- );
1037
- if (canonical === undefined) {
1038
- throw new Error(
1039
- 'push rejection finalization did not persist an outcome',
1039
+ try {
1040
+ await commitRejectedPushResult(clientId, frame.clientCommitId, stored);
1041
+ const canonical = await storage.getPushResult(
1042
+ partition,
1043
+ clientId,
1044
+ frame.clientCommitId,
1045
+ );
1046
+ if (canonical === undefined) {
1047
+ throw new Error(
1048
+ 'push rejection finalization did not persist an outcome',
1049
+ );
1050
+ }
1051
+ return processedPushCommit(
1052
+ frame.clientCommitId,
1053
+ canonical,
1054
+ canonical.cacheIdentity !== stored.cacheIdentity,
1040
1055
  );
1056
+ } catch (finalizationError) {
1057
+ // A failed finalization must still release the transaction: on
1058
+ // SQLite an unreleased BEGIN wedges the storage's global begin()
1059
+ // queue, on Postgres it leaks the pinned pool client. rollback() is
1060
+ // a no-op when the finalization already committed.
1061
+ try {
1062
+ await tx.rollback();
1063
+ } catch {
1064
+ // Surface the finalization failure; a rollback failure would
1065
+ // otherwise mask it.
1066
+ }
1067
+ throw finalizationError;
1041
1068
  }
1042
- return processedPushCommit(
1043
- frame.clientCommitId,
1044
- canonical,
1045
- canonical.cacheIdentity !== stored.cacheIdentity,
1046
- );
1047
1069
  }
1048
1070
  await tx.rollback();
1049
1071
  throw error;
@@ -170,11 +170,45 @@ export function addColumnDdl(
170
170
  return out;
171
171
  }
172
172
 
173
+ /**
174
+ * Ownership marker for server-created projection indexes. A version bump
175
+ * rebuilds declared indexes by dropping every index Syncular owns and
176
+ * re-creating the declared set; this prefix is what marks an index as
177
+ * Syncular-owned, so operator-added tuning indexes survive bumps. Portable
178
+ * identifier validation reserves the `sync_` prefix for the server storage
179
+ * namespace, which keeps user and operator index names out of it.
180
+ */
181
+ export const SYNC_INDEX_PREFIX = 'sync_ix_';
182
+
183
+ /** Deterministic FNV-1a 64-bit hash, hex-encoded. Dependency-free so it runs
184
+ * identically on every server runtime. */
185
+ function fnv1a64Hex(value: string): string {
186
+ let hash = 0xcbf29ce484222325n;
187
+ for (const byte of new TextEncoder().encode(value)) {
188
+ hash ^= BigInt(byte);
189
+ hash = (hash * 0x100000001b3n) & 0xffffffffffffffffn;
190
+ }
191
+ return hash.toString(16).padStart(16, '0');
192
+ }
193
+
194
+ /**
195
+ * Physical name of one declared index: the ownership prefix plus the
196
+ * declared name. When that would exceed the 63-byte Postgres identifier
197
+ * limit, the declared name is replaced by its FNV-1a hash so the physical
198
+ * name stays deterministic, unique, and within the limit.
199
+ */
200
+ export function physicalIndexName(declaredName: string): string {
201
+ const prefixed = `${SYNC_INDEX_PREFIX}${declaredName}`;
202
+ if (new TextEncoder().encode(prefixed).length <= 63) return prefixed;
203
+ return `${SYNC_INDEX_PREFIX}${fnv1a64Hex(declaredName)}`;
204
+ }
205
+
173
206
  /**
174
207
  * CREATE INDEX IF NOT EXISTS for the table's user-declared indexes
175
- * (DESIGN "user indexes" — the same names/columns the client materializes;
176
- * cross-table index-name uniqueness is the user's schema concern, exactly
177
- * as it is client-side).
208
+ * (DESIGN "user indexes" — the same declared names/columns the client
209
+ * materializes; cross-table index-name uniqueness is the user's schema
210
+ * concern, exactly as it is client-side). Server-side the physical name
211
+ * carries the {@link SYNC_INDEX_PREFIX} ownership marker.
178
212
  */
179
213
  export function createIndexDdl(table: CompiledTable): string[] {
180
214
  // User indexes name app columns — nothing to index without the projection.
@@ -184,7 +218,7 @@ export function createIndexDdl(table: CompiledTable): string[] {
184
218
  const columns = index.columns
185
219
  .map((column) => quoteIdent(column))
186
220
  .join(', ');
187
- return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(index.name)} ON ${quoteIdent(table.name)} (${columns})`;
221
+ return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
188
222
  });
189
223
  }
190
224
 
@@ -678,6 +712,12 @@ export function rewritePlan(
678
712
  * missing columns, and rebuild declared secondary indexes. Rebuilding during
679
713
  * a version bump supports DROP INDEX and same-name index replacement without
680
714
  * requiring historical index definitions in the stored column-layout marker.
715
+ *
716
+ * The drop set is limited to Syncular-owned indexes: physical names carrying
717
+ * {@link SYNC_INDEX_PREFIX}, plus bare declared names (databases whose
718
+ * projection indexes predate the ownership prefix migrate onto the prefixed
719
+ * scheme through this rule). Operator-added tuning indexes keep their own
720
+ * names and survive every bump.
681
721
  */
682
722
  export function schemaDdl(
683
723
  schema: CompiledSchema,
@@ -685,18 +725,28 @@ export function schemaDdl(
685
725
  dialect: RelationalDialect,
686
726
  existingIndexesByTable: ReadonlyMap<string, ReadonlySet<string>> = new Map(),
687
727
  ): string[] {
688
- const out: string[] = [];
728
+ const drops: string[] = [];
729
+ const creates: string[] = [];
689
730
  for (const table of schema.tables.values()) {
690
731
  const existing = existingColumnsByTable.get(table.name);
691
732
  if (existing === undefined) {
692
- out.push(createTableDdl(table, dialect));
733
+ creates.push(createTableDdl(table, dialect));
693
734
  } else {
735
+ const declared = new Set(table.indexes.map((index) => index.name));
694
736
  for (const indexName of existingIndexesByTable.get(table.name) ?? []) {
695
- out.push(dropIndexDdl(indexName));
737
+ if (
738
+ indexName.startsWith(SYNC_INDEX_PREFIX) ||
739
+ declared.has(indexName)
740
+ ) {
741
+ drops.push(dropIndexDdl(indexName));
742
+ }
696
743
  }
697
- out.push(...addColumnDdl(table, existing, dialect));
744
+ creates.push(...addColumnDdl(table, existing, dialect));
698
745
  }
699
- out.push(...createIndexDdl(table));
746
+ creates.push(...createIndexDdl(table));
700
747
  }
701
- return out;
748
+ // Every drop precedes every create: an index name moving between tables in
749
+ // one bump must release the global (SQLite) index namespace before the
750
+ // receiving table re-creates it.
751
+ return [...drops, ...creates];
702
752
  }
package/src/schema.ts CHANGED
@@ -104,11 +104,8 @@ export interface CompiledSchema {
104
104
  const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
105
105
 
106
106
  /**
107
- * Identifier rules for relational server storage (DESIGN "current-row
108
- * tables"): app tables live in the same namespace as the sync
109
- * infrastructure tables (`sync_*`) and carry `_sync_*` meta columns, so
110
- * both prefixes are reserved; identifiers over 63 bytes would be silently
111
- * truncated by Postgres.
107
+ * Compile one `prefix:{variable}` scope pattern against the table's column
108
+ * list, resolving the scope column's positional index for row extraction.
112
109
  */
113
110
  function compilePattern(
114
111
  table: TableSchema,
@@ -145,6 +142,11 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
145
142
  if (tables.has(table.name)) {
146
143
  throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
147
144
  }
145
+ // Identifier rules (DESIGN "current-row tables") live in core's
146
+ // validatePortableRelationalIdentifier, shared with typegen: app tables
147
+ // share a namespace with the sync infrastructure tables (`sync_*`) and
148
+ // carry `_sync_*` meta columns, so both prefixes are reserved, and
149
+ // Postgres silently truncates identifiers over 63 bytes.
148
150
  validatePortableRelationalIdentifier('table', table.name);
149
151
  const columnIndex = new Map<string, number>();
150
152
  table.columns.forEach((column, index) => {
@@ -92,6 +92,20 @@ class SqliteTransaction implements StorageTransaction {
92
92
  return this.#storage.getRow(this.#partition, table, rowId);
93
93
  }
94
94
 
95
+ getPushResult(
96
+ clientId: string,
97
+ clientCommitId: string,
98
+ ): Promise<StoredPushResult | undefined> {
99
+ this.#assertOpen();
100
+ // One shared bun:sqlite connection — this read runs inside this
101
+ // transaction's BEGIN IMMEDIATE.
102
+ return this.#storage.getPushResult(
103
+ this.#partition,
104
+ clientId,
105
+ clientCommitId,
106
+ );
107
+ }
108
+
95
109
  scanRows(query: RowScanQuery): Promise<StoredRow[]> {
96
110
  this.#assertOpen();
97
111
  return this.#storage.scanRows(this.#partition, query);
@@ -244,10 +258,22 @@ class SqliteTransaction implements StorageTransaction {
244
258
 
245
259
  async commit(): Promise<void> {
246
260
  this.#assertOpen();
247
- this.#open = false;
248
261
  try {
249
262
  this.#storage.db.exec('COMMIT');
263
+ } catch (error) {
264
+ // A failed COMMIT (SQLITE_BUSY from an external writer, I/O error)
265
+ // leaves the connection inside BEGIN IMMEDIATE. Roll back before the
266
+ // FIFO releases, so the next queued transaction starts on a clean
267
+ // connection.
268
+ try {
269
+ this.#storage.db.exec('ROLLBACK');
270
+ } catch {
271
+ // Surface the COMMIT failure; a rollback failure would otherwise
272
+ // mask it.
273
+ }
274
+ throw error;
250
275
  } finally {
276
+ this.#open = false;
251
277
  this.#release();
252
278
  }
253
279
  }
@@ -415,6 +441,10 @@ export class SqliteServerStorage implements ServerStorage {
415
441
  }
416
442
 
417
443
  async begin(partition: string): Promise<StorageTransaction> {
444
+ // Deliberately global across partitions: SQLite is single-writer per
445
+ // database file, so one FIFO over the shared connection is the correct
446
+ // serialization unit — a per-partition queue would still contend on the
447
+ // same BEGIN IMMEDIATE writer lock.
418
448
  const previous = this.#transactionTail;
419
449
  let release!: () => void;
420
450
  this.#transactionTail = new Promise<void>((resolve) => {
@@ -593,10 +623,7 @@ export class SqliteServerStorage implements ServerStorage {
593
623
  }
594
624
 
595
625
  async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
596
- assertScopeIndexedScan(query);
597
- const variables = Object.keys(query.scopeFilter).sort();
598
- const firstVariable = variables[0];
599
- if (firstVariable === undefined) return [];
626
+ const firstVariable = assertScopeIndexedScan(query);
600
627
  const firstValues = query.scopeFilter[firstVariable] ?? [];
601
628
  if (firstValues.length === 0) return [];
602
629
  // Candidates via the inverted index LEFT JOINed to the row table — one
@@ -3,18 +3,23 @@ import type { CompiledTable, IndexSchema } from './schema';
3
3
  import type { IndexRowScanQuery, RowScanQuery } from './storage';
4
4
  import { StorageQueryError } from './storage-errors';
5
5
 
6
- /** Fail loudly instead of making an unsupported unscoped scan look empty. */
7
- export function assertScopeIndexedScan(query: RowScanQuery): void {
6
+ /**
7
+ * Fail loudly instead of making an unsupported unscoped scan look empty.
8
+ * Returns the first scope variable in sorted order — the one every adapter
9
+ * drives its inverted-index candidate selection with.
10
+ */
11
+ export function assertScopeIndexedScan(query: RowScanQuery): string {
8
12
  const scopeFilter = (
9
13
  query as RowScanQuery & { readonly scopeFilter?: ScopeMap | null }
10
14
  ).scopeFilter;
11
- if (
12
- scopeFilter === undefined ||
13
- scopeFilter === null ||
14
- Object.keys(scopeFilter).length === 0
15
- ) {
15
+ const firstVariable =
16
+ scopeFilter === undefined || scopeFilter === null
17
+ ? undefined
18
+ : Object.keys(scopeFilter).sort()[0];
19
+ if (firstVariable === undefined) {
16
20
  throw new StorageQueryError('sync.storage.scan_requires_scope');
17
21
  }
22
+ return firstVariable;
18
23
  }
19
24
 
20
25
  /** Validate and resolve one exact trusted-host relational index lookup. */
package/src/storage.ts CHANGED
@@ -190,6 +190,20 @@ export interface ScopeActivityQuery {
190
190
  */
191
191
  export interface StorageTransaction {
192
192
  getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
193
+ /**
194
+ * Optional transaction-scoped idempotency lookup (§2.3) with the same
195
+ * semantics as `ServerStorage.getPushResult`, including the
196
+ * `sync.idempotency_cache_miss` throw. The push layer prefers it for the
197
+ * post-serialization duplicate re-check so the read runs on the
198
+ * transaction's own connection — a pooled Postgres client that holds the
199
+ * partition lock must never wait for a second pool slot mid-push. In-tree
200
+ * SQLite/Postgres/D1 adapters implement it; a custom adapter that omits it
201
+ * keeps the pool-level read.
202
+ */
203
+ getPushResult?(
204
+ clientId: string,
205
+ clientCommitId: string,
206
+ ): Promise<StoredPushResult | undefined>;
193
207
  /**
194
208
  * Optional candidate-state scan used only by whole-commit validation.
195
209
  * In-tree SQLite/Postgres/D1 backends implement it with read-your-own-writes