@syncular/server 0.30.2 → 0.30.4

package/dist/postgres-storage.js

@@ -686,34 +686,41 @@ class PostgresTransaction {
         const commitSeq = asNumber(rows[0]?.commit_seq);
         if (commit.changes.length === 0)
             return commitSeq;
-        // One statement for the whole commit: its changes, their inverted scope
-        // entries, and the §5 delete tombstones ride one writable-CTE chain.
+        // One writable-CTE chain per chunk of changes: the changes, their
+        // inverted scope entries, and the §5 delete tombstones. A statement binds
+        // at most a 16-bit count of parameters, so a chunk carries
+        // PG_CHANGES_PER_STATEMENT changes; every chunk runs in the
+        // commit's transaction under the same commit_seq with its global idx.
         // Serialized scopes bind as text before parsing JSONB; drivers that
         // encode JSONB parameters would otherwise store the string as a scalar.
-        // A commit can delete one row twice, so the tombstone insert keeps one
-        // row per key (every candidate carries the same commit_seq).
-        const params = [p, commitSeq];
-        const values = commit.changes.map((change, idx) => {
-            const base = params.length;
-            params.push(idx, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
-            return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
-        });
-        await q.query(`WITH inserted AS (
-         INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
-         VALUES ${values.join(',')}
-         RETURNING partition, tbl, row_id, op, commit_seq, scopes
-       ),
-       scoped AS (
-         INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
-         SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
-         FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
-         ON CONFLICT DO NOTHING
-       )
-       INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
-       SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
-         FROM inserted WHERE op = 2
-       ON CONFLICT (partition, tbl, row_id)
-       DO UPDATE SET commit_seq = excluded.commit_seq`, params);
+        // A commit can delete one row twice, so a chunk's tombstone insert keeps
+        // one row per key and a later chunk updates it to the same commit_seq.
+        for (let first = 0; first < commit.changes.length; first += PG_CHANGES_PER_STATEMENT) {
+            const params = [p, commitSeq];
+            const values = commit.changes
+                .slice(first, first + PG_CHANGES_PER_STATEMENT)
+                .map((change, offset) => {
+                const base = params.length;
+                params.push(first + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
+                return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
+            });
+            await q.query(`WITH inserted AS (
+           INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
+           VALUES ${values.join(',')}
+           RETURNING partition, tbl, row_id, op, commit_seq, scopes
+         ),
+         scoped AS (
+           INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
+           SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
+           FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
+           ON CONFLICT DO NOTHING
+         )
+         INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
+         SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
+           FROM inserted WHERE op = 2
+         ON CONFLICT (partition, tbl, row_id)
+         DO UPDATE SET commit_seq = excluded.commit_seq`, params);
+        }
         return commitSeq;
     }
     async putPushResult(clientId, clientCommitId, result) {
@@ -776,6 +783,10 @@ class RollbackSignal extends Error {
         this.name = 'RollbackSignal';
     }
 }
+/** Changes per sync_changes statement: 2 + 7 × 4,096 = 28,674 bound
+ * parameters. The Bind message counts parameters in 16 bits; PostgreSQL reads
+ * up to 65,535, and PGlite 0.5 corrupts its state above 32,767. */
+const PG_CHANGES_PER_STATEMENT = 4096;
 export class PostgresServerStorage {
     #exec;
     /** Set by `ensureSchema`: app-table lookup for the relational row store. */
@@ -999,6 +1010,17 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
                 await client.query(`INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, $1, $2)
            ON CONFLICT (id) DO UPDATE
              SET schema_version=EXCLUDED.schema_version, layouts=EXCLUDED.layouts`, [schema.version, layoutsOf(schema)]);
+                // A writer of the previous schema still running against this
+                // database would append payloads in the previous layout. Its push
+                // is refused by the serve gate, but a host's server-side
+                // `begin()`/`appendCommit` reads no gate, so the bump raises the
+                // writer fence of every partition to the new version.
+                await client.query(`INSERT INTO sync_writer_fence(partition, required_writer_version)
+           SELECT partition, $1 FROM sync_partitions
+           ON CONFLICT (partition) DO UPDATE SET
+             required_writer_version=GREATEST(
+               sync_writer_fence.required_writer_version,
+               EXCLUDED.required_writer_version)`, [schema.version]);
                 // The host's declared checkpoints install in the schema-bump
                 // transaction, so no observer sees the bumped marker without the
                 // fence, or the fence without its declaration row.

package/dist/sqlite-storage.js

@@ -412,6 +412,17 @@ END`);
                     this.db
                         .query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
                         .run(schema.version, layoutsOf(schema));
+                    // A writer of the previous schema would append payloads in the
+                    // previous layout; the bump fences every partition at the new
+                    // version (see PostgresServerStorage.ensureSchema).
+                    this.db
+                        .query(`INSERT INTO sync_writer_fence(partition, required_writer_version)
+               SELECT partition, ? FROM sync_partitions WHERE true
+               ON CONFLICT(partition) DO UPDATE SET
+                 required_writer_version=max(
+                   sync_writer_fence.required_writer_version,
+                   excluded.required_writer_version)`)
+                        .run(schema.version);
                     // The host's declared checkpoints install in the schema-bump
                     // transaction, so no observer sees the bumped marker without the
                     // fence, or the fence without its declaration row.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.30.2",
+  "version": "0.30.4",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.30.2"
+    "@syncular/core": "0.30.4"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/postgres-storage.ts

@@ -1117,45 +1117,56 @@ class PostgresTransaction implements StorageTransaction {
     );
     const commitSeq = asNumber(rows[0]?.commit_seq);
     if (commit.changes.length === 0) return commitSeq;
-    // One statement for the whole commit: its changes, their inverted scope
-    // entries, and the §5 delete tombstones ride one writable-CTE chain.
+    // One writable-CTE chain per chunk of changes: the changes, their
+    // inverted scope entries, and the §5 delete tombstones. A statement binds
+    // at most a 16-bit count of parameters, so a chunk carries
+    // PG_CHANGES_PER_STATEMENT changes; every chunk runs in the
+    // commit's transaction under the same commit_seq with its global idx.
     // Serialized scopes bind as text before parsing JSONB; drivers that
     // encode JSONB parameters would otherwise store the string as a scalar.
-    // A commit can delete one row twice, so the tombstone insert keeps one
-    // row per key (every candidate carries the same commit_seq).
-    const params: unknown[] = [p, commitSeq];
-    const values = commit.changes.map((change, idx) => {
-      const base = params.length;
-      params.push(
-        idx,
-        change.table,
-        change.rowId,
-        change.op === 'upsert' ? 1 : 2,
-        change.rowVersion ?? null,
-        JSON.stringify(change.scopes),
-        change.payload ?? null,
+    // A commit can delete one row twice, so a chunk's tombstone insert keeps
+    // one row per key and a later chunk updates it to the same commit_seq.
+    for (
+      let first = 0;
+      first < commit.changes.length;
+      first += PG_CHANGES_PER_STATEMENT
+    ) {
+      const params: unknown[] = [p, commitSeq];
+      const values = commit.changes
+        .slice(first, first + PG_CHANGES_PER_STATEMENT)
+        .map((change, offset) => {
+          const base = params.length;
+          params.push(
+            first + offset,
+            change.table,
+            change.rowId,
+            change.op === 'upsert' ? 1 : 2,
+            change.rowVersion ?? null,
+            JSON.stringify(change.scopes),
+            change.payload ?? null,
+          );
+          return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
+        });
+      await q.query(
+        `WITH inserted AS (
+           INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
+           VALUES ${values.join(',')}
+           RETURNING partition, tbl, row_id, op, commit_seq, scopes
+         ),
+         scoped AS (
+           INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
+           SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
+           FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
+           ON CONFLICT DO NOTHING
+         )
+         INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
+         SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
+           FROM inserted WHERE op = 2
+         ON CONFLICT (partition, tbl, row_id)
+         DO UPDATE SET commit_seq = excluded.commit_seq`,
+        params,
       );
-      return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
-    });
-    await q.query(
-      `WITH inserted AS (
-         INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
-         VALUES ${values.join(',')}
-         RETURNING partition, tbl, row_id, op, commit_seq, scopes
-       ),
-       scoped AS (
-         INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
-         SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
-         FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
-         ON CONFLICT DO NOTHING
-       )
-       INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
-       SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
-         FROM inserted WHERE op = 2
-       ON CONFLICT (partition, tbl, row_id)
-       DO UPDATE SET commit_seq = excluded.commit_seq`,
-      params,
-    );
+    }
     return commitSeq;
   }
 
@@ -1234,6 +1245,11 @@ class RollbackSignal extends Error {
   }
 }
 
+/** Changes per sync_changes statement: 2 + 7 × 4,096 = 28,674 bound
+ * parameters. The Bind message counts parameters in 16 bits; PostgreSQL reads
+ * up to 65,535, and PGlite 0.5 corrupts its state above 32,767. */
+const PG_CHANGES_PER_STATEMENT = 4096;
+
 /** One page read waiting in `PostgresServerStorage#batchedPage`. */
 interface PendingPage {
   readonly spec: Record<string, unknown>;
@@ -1542,6 +1558,20 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
              SET schema_version=EXCLUDED.schema_version, layouts=EXCLUDED.layouts`,
           [schema.version, layoutsOf(schema)],
         );
+        // A writer of the previous schema still running against this
+        // database would append payloads in the previous layout. Its push
+        // is refused by the serve gate, but a host's server-side
+        // `begin()`/`appendCommit` reads no gate, so the bump raises the
+        // writer fence of every partition to the new version.
+        await client.query(
+          `INSERT INTO sync_writer_fence(partition, required_writer_version)
+           SELECT partition, $1 FROM sync_partitions
+           ON CONFLICT (partition) DO UPDATE SET
+             required_writer_version=GREATEST(
+               sync_writer_fence.required_writer_version,
+               EXCLUDED.required_writer_version)`,
+          [schema.version],
+        );
         // The host's declared checkpoints install in the schema-bump
         // transaction, so no observer sees the bumped marker without the
         // fence, or the fence without its declaration row.

package/src/sqlite-storage.ts

@@ -698,6 +698,19 @@ END`);
               'INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts',
             )
             .run(schema.version, layoutsOf(schema));
+          // A writer of the previous schema would append payloads in the
+          // previous layout; the bump fences every partition at the new
+          // version (see PostgresServerStorage.ensureSchema).
+          this.db
+            .query(
+              `INSERT INTO sync_writer_fence(partition, required_writer_version)
+               SELECT partition, ? FROM sync_partitions WHERE true
+               ON CONFLICT(partition) DO UPDATE SET
+                 required_writer_version=max(
+                   sync_writer_fence.required_writer_version,
+                   excluded.required_writer_version)`,
+            )
+            .run(schema.version);
           // The host's declared checkpoints install in the schema-bump
           // transaction, so no observer sees the bumped marker without the
           // fence, or the fence without its declaration row.