@syncular/server 0.38.0 → 0.39.0

package/dist/relational-rows.js

@@ -464,24 +464,22 @@ export function upsertValues(table, partition, row, dialect) {
         row.columnVersions ?? null,
     ];
 }
-function placeholderList(count, dialect) {
-    if (dialect === 'sqlite')
-        return new Array(count).fill('?').join(',');
-    const out = [];
-    for (let i = 1; i <= count; i++)
-        out.push(`$${i}`);
-    return out.join(',');
-}
-/** Upsert statement text (bind with `upsertValues`). */
-export function upsertSql(table, dialect) {
+/**
+ * Upsert statement text (bind with `upsertValues`). PostgreSQL takes `rows`
+ * rows in one statement, bound as consecutive `upsertValues` lists; their
+ * row ids must be distinct.
+ */
+export function upsertSql(table, dialect, rows = 1) {
     const names = tableColumnNames(table);
     const quoted = names.map((name) => quoteIdent(name)).join(', ');
-    const values = placeholderList(names.length, dialect);
+    const values = dialect === 'sqlite'
+        ? `(${new Array(names.length).fill('?').join(',')})`
+        : Array.from({ length: rows }, (_, row) => `(${names.map((_, column) => `$${row * names.length + column + 1}`).join(',')})`).join(',');
     const updates = names
         .slice(2) // partition + row id are the conflict key
         .map((name) => `${quoteIdent(name)}=excluded.${quoteIdent(name)}`)
         .join(', ');
-    return `INSERT INTO ${quoteIdent(table.name)} (${quoted}) VALUES (${values})
+    return `INSERT INTO ${quoteIdent(table.name)} (${quoted}) VALUES ${values}
      ON CONFLICT (${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)}) DO UPDATE SET ${updates}`;
 }
 /**
@@ -611,26 +609,40 @@ export function commitWindowPageSql(valueCount, dialect) {
 }
 /**
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
- * page of every subscription scanning `table` in a pull. `$2` is a JSON array
- * of `{i, tbl, var, vals, after, lim}` records; each record runs the same
- * index-first candidate subquery through a LATERAL join, so every
- * subscription keeps its own keyset cursor and LIMIT. Rows arrive ordered
- * by (i, row_id).
+ * page of every subscription scanning any table in a pull. `$2` is a JSON
+ * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
+ * same index-first candidate subquery through a LATERAL join, so every
+ * subscription keeps its own keyset cursor and LIMIT. The candidate side is
+ * table-agnostic (a spec names its table), so the statement joins the row
+ * table once per table present in the batch through a UNION ALL branch that
+ * keeps only its own candidates. Rows arrive ordered by (i, row_id).
  *
  * Params: [partition, specs (JSON text)]. The specs bind as text before
  * the JSONB cast, so a driver that JSON-encodes a string bound to a `jsonb`
  * parameter (Bun.sql) cannot turn the array into a JSON string.
  */
-export function scanRowBatchSql(table) {
-    return `SELECT s.i AS sub, c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
-     FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
-     CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
-       WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
-         AND row_id>s.after
-       ORDER BY row_id LIMIT s.lim) c
-     LEFT JOIN ${quoteIdent(table.name)} r
-       ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
-     ORDER BY s.i, c.row_id`;
+export function scanRowBatchSql(tables) {
+    // One branch per table, each keeping only its own candidates and selecting
+    // the same meta columns so the caller decodes one row shape.
+    const projection = tables
+        .map((table) => {
+        const quoted = quoteIdent(table.name);
+        return `SELECT c.sub AS sub, c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
+       FROM cand c
+       LEFT JOIN ${quoted} r
+         ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
+      WHERE c.tbl='${table.name.replaceAll("'", "''")}'`;
+    })
+        .join('\n     UNION ALL ');
+    return `WITH cand AS (
+       SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
+         FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
+         CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
+           WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
+             AND row_id>s.after
+           ORDER BY row_id LIMIT s.lim) c)
+     ${projection}
+     ORDER BY sub, row_id`;
 }
 /**
  * PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
@@ -863,23 +875,17 @@ export function selectRowsForRewriteSql(table, dialect) {
      LIMIT ${p[2]}`;
 }
 /**
- * Rewrite one row during a migration: refresh the projection columns (when
+ * SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
  * materialized) and the payload. Bind with `rewriteValues`.
  */
-export function rewriteRowSql(table, dialect) {
-    const sets = [];
-    let i = 1;
-    if (table.materialize) {
-        for (const column of table.columns) {
-            sets.push(`${quoteIdent(column.name)}=${dialect === 'sqlite' ? '?' : `$${i}`}`);
-            i++;
-        }
-    }
-    sets.push(`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=${dialect === 'sqlite' ? '?' : `$${i}`}`);
-    i++;
-    const wherePartition = dialect === 'sqlite' ? '?' : `$${i}`;
-    const whereRowId = dialect === 'sqlite' ? '?' : `$${i + 1}`;
-    return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${wherePartition} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${whereRowId}`;
+export function rewriteRowSql(table) {
+    const sets = [
+        ...(table.materialize
+            ? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
+            : []),
+        `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
+    ];
+    return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
 }
 /** Bind values for `rewriteRowSql` from the (already migrated) payload. */
 export function rewriteValues(table, partition, rowId, payload, dialect) {

package/dist/sqlite-storage.js

@@ -364,8 +364,6 @@ END`);
             // rewrite stored rows (payload re-encode for layout changes, and/or
             // projection backfill for flipped-on materialization). One
             // transaction: a failed bump leaves no half-state.
-            const layouts = parseLayouts(marker?.layouts);
-            const retiredTables = retiredTableNames(schema, layouts);
             const existing = new Map();
             const existingIndexes = new Map();
             for (const table of schema.tables.values()) {
@@ -388,6 +386,21 @@ END`);
             await this.#serializeWrite(() => {
                 this.db.exec('BEGIN IMMEDIATE');
                 try {
+                    // Another process sharing the database may have bumped between the
+                    // marker read above and the writer lock; the locked marker decides
+                    // (SPEC "Schema upgrade writer fence").
+                    const locked = this.db
+                        .query('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1')
+                        .get();
+                    if (locked !== null && locked.schema_version > schema.version) {
+                        throw new Error(`stored schema version ${locked.schema_version} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`);
+                    }
+                    if (locked !== null && locked.schema_version === schema.version) {
+                        this.db.exec('COMMIT');
+                        return;
+                    }
+                    const layouts = parseLayouts(locked?.layouts);
+                    const retiredTables = retiredTableNames(schema, layouts);
                     for (const tableName of retiredTables) {
                         this.db
                             .query('DELETE FROM sync_row_scopes WHERE tbl=?')
@@ -521,7 +534,7 @@ END`);
      */
     #rewriteRows(table, oldLayout) {
         const select = selectRowsForRewriteSql(table, 'sqlite');
-        const update = this.db.query(rewriteRowSql(table, 'sqlite'));
+        const update = this.db.query(rewriteRowSql(table));
         const BATCH = 500;
         let afterPartition = '';
         let afterRowId = '';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.38.0",
+  "version": "0.39.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.38.0"
+    "@syncular/core": "0.39.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/content-encoding.ts

@@ -1,24 +1,29 @@
 /**
- * §5.8 shipped compression posture for the direct segment endpoint:
- * negotiate `Content-Encoding` from the request's `Accept-Encoding`
- * (zstd preferred, gzip fallback, identity when the client offers
- * neither), feature-detecting the runtime's codecs — zero dependencies.
+ * §1.3 transport compression for every buffered HTTP body the adapters
+ * serve — `<mount>/sync`, `<mount>/operations`, blob GETs and segment
+ * downloads: negotiate `Content-Encoding` from the request's
+ * `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
+ * offers neither), feature-detecting the runtime's codecs, zero
+ * dependencies. Bun supplies zstd + gzip natively; every other runtime
+ * (workerd, Node, Deno) gets gzip through the standard `CompressionStream`.
  *
- * Segment bytes are stored and content-addressed UNCOMPRESSED (§5.1);
- * this is a per-response serving concern only. It is never applied to
- * stored objects or the signed-URL path (§5.8: presigned GETs serve the
- * addressed bytes verbatim; edge compression is a deployment concern).
+ * Body bytes are stored and content-addressed UNCOMPRESSED (§5.1); this is a
+ * per-response serving concern only. It is never applied to stored objects or
+ * the signed-URL path (§5.8: presigned GETs serve the addressed bytes
+ * verbatim; edge compression is a deployment concern). Clients MAY gzip a
+ * request body (§1.3) — `decodeRequestBody` is the accepting half.
  *
  * Decision data (2026-07-03, 100k-row bench table, Bun 1.3): rows
  * segments 6.2× in 8 ms (zstd) / 7.7× in 42 ms (gzip); sqlite images
  * 3.4× in 14 ms (zstd) / 51 ms (gzip); client decompression ≤ 8 ms.
  */
+import { syncError } from './errors';
 
-export type SegmentContentEncoding = 'zstd' | 'gzip';
+export type ContentEncoding = 'zstd' | 'gzip';
 
 interface Codec {
-  readonly encoding: SegmentContentEncoding;
-  readonly compress: (bytes: Uint8Array) => Uint8Array;
+  readonly encoding: ContentEncoding;
+  readonly compress: (bytes: Uint8Array) => Uint8Array | Promise<Uint8Array>;
 }
 
 interface BunCompressors {
@@ -26,8 +31,22 @@ interface BunCompressors {
   readonly gzipSync?: (bytes: Uint8Array) => Uint8Array;
 }
 
-/** Preference order is zstd, gzip (§5.8). Non-Bun runtimes without these
- * codecs serve identity — negotiation, never a hard requirement here. */
+/** Bodies below this size are served identity (headers outweigh gains). */
+const MIN_COMPRESS_BYTES = 1024;
+
+async function gzipWithCompressionStream(
+  bytes: Uint8Array,
+): Promise<Uint8Array> {
+  const compressed = await new Response(
+    new Blob([bytes.slice().buffer as ArrayBuffer])
+      .stream()
+      .pipeThrough(new CompressionStream('gzip')),
+  ).arrayBuffer();
+  return new Uint8Array(compressed);
+}
+
+/** Preference order is zstd, gzip (§1.3). A runtime with neither codec
+ * serves identity: negotiation, never a hard requirement here. */
 function detectCodecs(): readonly Codec[] {
   const bun = (globalThis as { Bun?: BunCompressors }).Bun;
   const codecs: Codec[] = [];
@@ -38,17 +57,19 @@ function detectCodecs(): readonly Codec[] {
   const gzip = bun?.gzipSync;
   if (typeof gzip === 'function') {
     codecs.push({ encoding: 'gzip', compress: (bytes) => gzip(bytes) });
+  } else if (typeof CompressionStream === 'function') {
+    codecs.push({ encoding: 'gzip', compress: gzipWithCompressionStream });
   }
   return codecs;
 }
 
 const CODECS = detectCodecs();
 
-/** Bodies below this size are served identity (headers outweigh gains). */
-const MIN_COMPRESS_BYTES = 1024;
-
 /** Does the `Accept-Encoding` header value accept `encoding` (q > 0)? */
-function accepts(header: string, encoding: SegmentContentEncoding): boolean {
+export function acceptsEncoding(
+  header: string,
+  encoding: ContentEncoding,
+): boolean {
   for (const entry of header.split(',')) {
     const [rawName, ...params] = entry.trim().split(';');
     const name = rawName?.trim().toLowerCase();
@@ -64,28 +85,41 @@ function accepts(header: string, encoding: SegmentContentEncoding): boolean {
   return false;
 }
 
-export interface EncodedSegmentBody {
+/**
+ * Add `Accept-Encoding` to a response's `Vary` header. Every response whose
+ * body was negotiated carries it, so a cache never serves one encoding to a
+ * client that asked for another.
+ */
+export function varyOnAcceptEncoding(headers: Headers): void {
+  const vary = headers.get('vary');
+  if (vary === null) headers.set('vary', 'Accept-Encoding');
+  else if (!/\baccept-encoding\b/i.test(vary)) {
+    headers.set('vary', `${vary}, Accept-Encoding`);
+  }
+}
+
+export interface EncodedBody {
   readonly bytes: Uint8Array;
   /** Absent ⇒ identity (no `Content-Encoding` header). */
-  readonly contentEncoding?: SegmentContentEncoding;
+  readonly contentEncoding?: ContentEncoding;
 }
 
 /**
- * Compress a buffered segment body per §5.8. Returns the original bytes
+ * Compress a buffered response body per §1.3. Returns the original bytes
  * (identity) when the client offered no supported encoding, the body is
  * tiny, or the runtime has no codec.
  */
-export function encodeSegmentBody(
+export async function encodeBody(
   bytes: Uint8Array,
   acceptEncoding: string | undefined,
-): EncodedSegmentBody {
+): Promise<EncodedBody> {
   if (acceptEncoding === undefined || bytes.length < MIN_COMPRESS_BYTES) {
     return { bytes };
   }
   for (const codec of CODECS) {
-    if (accepts(acceptEncoding, codec.encoding)) {
+    if (acceptsEncoding(acceptEncoding, codec.encoding)) {
       return {
-        bytes: codec.compress(bytes),
+        bytes: await codec.compress(bytes),
         contentEncoding: codec.encoding,
       };
     }
@@ -100,24 +134,24 @@ export function encodeSegmentBody(
  */
 export const SEGMENT_STREAM_THRESHOLD_BYTES = 16 * 1024 * 1024;
 
-export interface EncodedSegmentStream {
+export interface EncodedStreamBody {
   readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
   /** Absent ⇒ identity (no `Content-Encoding` header). */
   readonly contentEncoding?: 'gzip';
 }
 
 /**
- * Compress a streamed segment body per §5.8 with the runtime's
- * `CompressionStream`: gzip when the client accepts it, identity otherwise
- * (zstd has no standard streaming codec).
+ * Compress a streamed body per §1.3 with the runtime's `CompressionStream`:
+ * gzip when the client accepts it, identity otherwise (zstd has no standard
+ * streaming codec).
  */
-export function encodeSegmentStream(
+export function encodeStream(
   body: ReadableStream<Uint8Array<ArrayBuffer>>,
   acceptEncoding: string | undefined,
-): EncodedSegmentStream {
+): EncodedStreamBody {
   if (
     acceptEncoding === undefined ||
-    !accepts(acceptEncoding, 'gzip') ||
+    !acceptsEncoding(acceptEncoding, 'gzip') ||
     typeof CompressionStream !== 'function'
   ) {
     return { body };
@@ -127,3 +161,37 @@ export function encodeSegmentStream(
     contentEncoding: 'gzip',
   };
 }
+
+/**
+ * The bytes of a request body, decompressed when the client sent
+ * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
+ * servers MUST accept it). Returns `undefined` for any other declared
+ * encoding — the adapter answers 415 rather than parse bytes it cannot read.
+ * A gzip body that does not decompress is a `sync.invalid_request`.
+ */
+export async function decodeRequestBody(
+  request: Request,
+): Promise<Uint8Array | undefined> {
+  const encoding = request.headers
+    .get('content-encoding')
+    ?.trim()
+    .toLowerCase();
+  if (encoding === undefined || encoding === '' || encoding === 'identity') {
+    return new Uint8Array(await request.arrayBuffer());
+  }
+  if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
+    return undefined;
+  }
+  if (request.body === null) {
+    throw syncError('sync.invalid_request', 'malformed gzip request body');
+  }
+  try {
+    return new Uint8Array(
+      await new Response(
+        request.body.pipeThrough(new DecompressionStream('gzip')),
+      ).arrayBuffer(),
+    );
+  } catch {
+    throw syncError('sync.invalid_request', 'malformed gzip request body');
+  }
+}

package/src/d1-storage.ts

@@ -1425,7 +1425,7 @@ export class D1ServerStorage implements ServerStorage {
               ? migratePayload(plan.oldLayout, table, bytes)
               : bytes;
           return this.#db
-            .prepare(rewriteRowSql(table, 'sqlite'))
+            .prepare(rewriteRowSql(table))
             .bind(
               ...rewriteValues(
                 table,