@syncular/server 0.46.3 → 0.48.0

package/README.md

@@ -256,7 +256,8 @@ const config: SyncServerConfig = {
 ```
 
 The callback also receives `read.getRow()` and its batch form
-`read.getRows(table, rowIds)`, bounded `read.scanRows()`, and
+`read.getRows(table, rowIds)`, bounded `read.scanRows()`,
+`read.scanRowsByIndex()` for an exact declared-index lookup, and
 `read.queryAuthoritative()` APIs. Those reads see the final candidate state, including staged sibling upserts and
 deletes. Syncular serializes the partition before any operation read so two
 aggregate validators cannot both accept mutually invalid candidates. It also
@@ -359,7 +360,9 @@ SQLite, PostgreSQL, and D1.
 
 An authoritative command that needs an exact alternate lookup can instead use
 the optional `storage.scanRowsByIndex(partition, query)` or transactional
-`tx.scanRowsByIndex(query)` capability. The query names one declared
+`tx.scanRowsByIndex(query)` capability, or `command.scanRowsByIndex()` on its
+push transaction. The whole-commit validator reaches the same transaction
+lookup as `read.scanRowsByIndex()`; both readers default `limit` to 100. The query names one declared
 `TableSchema.indexes` entry, supplies one exact value per index column, uses an
 exclusive `afterRowId`, and has a required limit from 1 through 1,000. All
 shipped adapters implement it; transaction reads see staged writes and deletes.

package/dist/errors.js

@@ -139,6 +139,15 @@ export const ERROR_CATALOG = {
         recommendedAction: 'fixRequest',
         httpStatus: 400,
     },
+    // §6.8: the whole-commit validator threw an exception that is not a
+    // deliberate rejection. A host fault; the thrown message rides the §6.3.1
+    // details and the original reaches the host's `onError`.
+    'sync.validator_error': {
+        category: 'internal',
+        retryable: false,
+        recommendedAction: 'inspectServer',
+        httpStatus: 500,
+    },
     'sync.missing_scopes': {
         category: 'internal',
         retryable: false,

package/dist/operations.d.ts

@@ -2,7 +2,7 @@ import { type AuthoritativeRelationPlan } from './authoritative-query.js';
 import { type RemoteOperationRequest, type RemoteOperationResponse, type RowValue } from '@syncular/core';
 import type { ResolveScopes, SyncRequestContext } from './context.js';
 import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, AuthoritativeQueryValue } from './storage.js';
-import { type ValidateRow } from './validate.js';
+import { type IndexScanInput, type ValidateRow } from './validate.js';
 export interface RemoteQueryDependency {
     readonly table: string;
     readonly scopeKeys?: readonly string[];
@@ -76,6 +76,15 @@ export interface RemoteCommandContext {
      * D1 issue one lookup per id.
      */
     getRows(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, ValidateRow>>;
+    /**
+     * Exact lookup through one declared relational index on the command's push
+     * transaction. The values form the complete, order-sensitive index tuple
+     * and `afterRowId` is an exclusive keyset cursor. Unlike `getRow` and
+     * `getRows`, the rows are not filtered by the actor's scopes: `authorize`
+     * gates the command, and this is the same trusted server-host lookup
+     * `ServerStorage.scanRowsByIndex` performs.
+     */
+    scanRowsByIndex(input: IndexScanInput): Promise<ValidateRow[]>;
     /**
      * Run a generated registered query on the command's push transaction
      * (§6.7). The storage binds every relation to the partition; unlike

package/dist/operations.js

@@ -314,6 +314,23 @@ function commandContext(ctx, tx, resolved, schema, clientId, operationId, reques
             }
             return rows;
         },
+        scanRowsByIndex: async ({ table: tableName, index, values, afterRowId = null, limit = 100, }) => {
+            const table = schema.tables.get(tableName);
+            if (table === undefined) {
+                throw syncError('operation.invalid_request', 'command read targets an unknown table');
+            }
+            if (tx.scanRowsByIndex === undefined) {
+                throw new Error('storage transaction does not support command index scans');
+            }
+            const rows = await tx.scanRowsByIndex({
+                table: tableName,
+                index,
+                values,
+                afterRowId,
+                limit,
+            });
+            return rows.map((stored) => toValidateRow(table.columns, decodeRow(table.columns, stored.payload)));
+        },
         queryAuthoritative: (query) => transactionQuery(tx, query),
     };
 }

package/dist/push.js

@@ -20,7 +20,7 @@ import { retainRollbackFailure } from './storage-errors.js';
  */
 import { decodeRow, decodeSparseRow, encodeRow, parseBlobRef, } from '@syncular/core';
 import { clockOf, limitsOf } from './context.js';
-import { SyncError } from './errors.js';
+import { reportError, SyncError } from './errors.js';
 import { emitEvent } from './events.js';
 import { prepareReactions, toNewReactions, } from './reactions.js';
 import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes.js';
@@ -696,10 +696,27 @@ function commitValidationReader(tx, schema) {
                 serverVersion: stored.serverVersion,
             }));
         },
+        scanRowsByIndex: async ({ table: tableName, index, values, afterRowId = null, limit = 100, }) => {
+            const table = tableFor(tableName);
+            if (tx.scanRowsByIndex === undefined) {
+                throw new Error('storage transaction does not support commit-validator index scans');
+            }
+            const rows = await tx.scanRowsByIndex({
+                table: tableName,
+                index,
+                values,
+                afterRowId,
+                limit,
+            });
+            return rows.map((stored) => ({
+                row: toValidateRow(table.columns, decodeRow(table.columns, stored.payload)),
+                serverVersion: stored.serverVersion,
+            }));
+        },
         queryAuthoritative: (query) => transactionQuery(tx, query),
     };
 }
-async function runCommitValidator(validator, tx, schema, clientId, clientCommitId, actorId, partition, operations) {
+async function runCommitValidator(validator, onError, tx, schema, clientId, clientCommitId, actorId, partition, operations) {
     if (validator === undefined)
         return undefined;
     try {
@@ -722,7 +739,30 @@ async function runCommitValidator(validator, tx, schema, clientId, clientCommitI
         if (error instanceof ValidationRejection) {
             return errorRecord(operations[0]?.opIndex ?? 0, error.code, error.message, false, error.details);
         }
-        return errorRecord(operations[0]?.opIndex ?? 0, 'sync.constraint_violation', 'whole-commit validator failed');
+        // §6.8: a throw that is not a deliberate rejection is a host fault, not a
+        // rule refusal. The host's observation hook receives the original (its
+        // stack included); the client gets a distinct code and, in the §6.3.1
+        // details, the bounded thrown message. Stacks and thrown values stay off
+        // the wire.
+        reportError(error, onError, 'sync');
+        let text = '';
+        let bytes = 0;
+        const encoder = new TextEncoder();
+        const thrown = error instanceof Error ? error.message : String(error);
+        for (const char of thrown.replace(/\p{Cc}+/gu, ' ').trim()) {
+            const size = encoder.encode(char).length;
+            if (bytes + size > 256)
+                break;
+            bytes += size;
+            text += char;
+        }
+        text = text.trimEnd();
+        return errorRecord(operations[0]?.opIndex ?? 0, 'sync.validator_error', 'whole-commit validator failed', false, {
+            reason: 'validator_threw',
+            ...(text === ''
+                ? {}
+                : { references: { validator: 'commit', message: text } }),
+        });
     }
     return undefined;
 }
@@ -768,6 +808,32 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
             last: write,
         });
     }
+    // One candidate-state read per parent table, not one per reference: a
+    // commit that points several rows at the same parent (a fight's
+    // combatants, its chronicle) reads that parent once. Postgres binds the
+    // ids as one array per table; SQLite and D1 answer with one lookup per id.
+    const parentIds = new Map();
+    for (const { last: write } of finalByRow.values()) {
+        if (write.op !== 'upsert' || write.values === undefined)
+            continue;
+        for (const reference of write.table.references) {
+            const value = write.values[reference.columnIndex];
+            if (value === null || value === undefined)
+                continue;
+            const parentRowId = renderScopeValue(value);
+            if (parentRowId === undefined)
+                continue;
+            const ids = parentIds.get(reference.parentTable);
+            if (ids === undefined)
+                parentIds.set(reference.parentTable, new Set([parentRowId]));
+            else
+                ids.add(parentRowId);
+        }
+    }
+    const parents = new Map();
+    for (const [parentTable, ids] of parentIds) {
+        parents.set(parentTable, await readCandidateRows(tx, parentTable, [...ids]));
+    }
     for (const { last: write } of finalByRow.values()) {
         if (write.op !== 'upsert' || write.values === undefined)
             continue;
@@ -778,7 +844,7 @@ async function enforceReferences(tx, schema, staged, clientOpCount, cascadeLimit
             const parentRowId = renderScopeValue(value);
             if (parentRowId === undefined)
                 continue;
-            if ((await tx.getRow(reference.parentTable, parentRowId)) !== undefined) {
+            if (parents.get(reference.parentTable)?.has(parentRowId) === true) {
                 continue;
             }
             return {
@@ -1129,7 +1195,7 @@ export async function processPushOperationsWithTrace(ctx, schema, resolved, clie
             }
         }
         if (terminated === undefined) {
-            const commitReject = await runCommitValidator(commitValidator, tx, schema, clientId, clientCommitId, ctx.actorId, partition, validatedOperations);
+            const commitReject = await runCommitValidator(commitValidator, ctx.onError, tx, schema, clientId, clientCommitId, ctx.actorId, partition, validatedOperations);
             if (commitReject?.kind === 'terminate') {
                 // An appended §6.11 cascade operation carries a synthetic opIndex;
                 // surface the originating client operation instead.

package/dist/validate.d.ts

@@ -74,6 +74,21 @@ export interface CommitValidationScanInput {
     /** Bounded per call by the server to 1..1,000; defaults to 100. */
     readonly limit?: number;
 }
+/**
+ * One exact declared-index lookup a push-transaction reader runs (§6.8): the
+ * whole-commit reader and a registered command share this shape.
+ */
+export interface IndexScanInput {
+    readonly table: string;
+    /** `TableSchema.indexes[].name`; never exposed as a client subscription. */
+    readonly index: string;
+    /** Exact values in the index declaration's column order. */
+    readonly values: readonly RowValue[];
+    /** Resume after this rowId (exclusive); `null` = start of the match set. */
+    readonly afterRowId?: string | null;
+    /** Bounded per call by the server to 1..1,000; defaults to 100. */
+    readonly limit?: number;
+}
 /**
  * Candidate-state reads bound to the same storage transaction as the commit.
  * Reads observe every staged sibling operation and no uncommitted competing
@@ -91,6 +106,16 @@ export interface CommitValidationReader {
      */
     getRows(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, CommitValidationRow>>;
     scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
+    /**
+     * Candidate-state exact lookup through one declared relational index, with
+     * the same read-your-own-writes guarantee as `scanRows`: staged sibling
+     * upserts are present and staged deletes are absent. The values form the
+     * complete, order-sensitive index tuple and `afterRowId` is an exclusive
+     * keyset cursor, so a caller pages without a scope declaration. There is no
+     * scope filter and no actor scope authorization: the caller already holds
+     * partition authority, exactly as with `scanRows`.
+     */
+    scanRowsByIndex(input: IndexScanInput): Promise<CommitValidationRow[]>;
     /**
      * Run a generated registered query on the commit transaction (§6.8). The
      * request has the shape `ServerStorage.queryAuthoritative` takes; the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.46.3",
+  "version": "0.48.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.46.3"
+    "@syncular/core": "0.48.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/errors.ts

@@ -147,6 +147,15 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
     recommendedAction: 'fixRequest',
     httpStatus: 400,
   },
+  // §6.8: the whole-commit validator threw an exception that is not a
+  // deliberate rejection. A host fault; the thrown message rides the §6.3.1
+  // details and the original reaches the host's `onError`.
+  'sync.validator_error': {
+    category: 'internal',
+    retryable: false,
+    recommendedAction: 'inspectServer',
+    httpStatus: 500,
+  },
   'sync.missing_scopes': {
     category: 'internal',
     retryable: false,

package/src/operations.ts

@@ -37,6 +37,7 @@ import { transactionQuery } from './storage-query';
 import {
   toValidateRow,
   ValidationRejection,
+  type IndexScanInput,
   type ValidateRow,
 } from './validate';
 
@@ -138,6 +139,15 @@ export interface RemoteCommandContext {
     table: string,
     rowIds: readonly string[],
   ): Promise<ReadonlyMap<string, ValidateRow>>;
+  /**
+   * Exact lookup through one declared relational index on the command's push
+   * transaction. The values form the complete, order-sensitive index tuple
+   * and `afterRowId` is an exclusive keyset cursor. Unlike `getRow` and
+   * `getRows`, the rows are not filtered by the actor's scopes: `authorize`
+   * gates the command, and this is the same trusted server-host lookup
+   * `ServerStorage.scanRowsByIndex` performs.
+   */
+  scanRowsByIndex(input: IndexScanInput): Promise<ValidateRow[]>;
   /**
    * Run a generated registered query on the command's push transaction
    * (§6.7). The storage binds every relation to the partition; unlike
@@ -639,6 +649,36 @@ function commandContext(
       }
       return rows;
     },
+    scanRowsByIndex: async ({
+      table: tableName,
+      index,
+      values,
+      afterRowId = null,
+      limit = 100,
+    }) => {
+      const table = schema.tables.get(tableName);
+      if (table === undefined) {
+        throw syncError(
+          'operation.invalid_request',
+          'command read targets an unknown table',
+        );
+      }
+      if (tx.scanRowsByIndex === undefined) {
+        throw new Error(
+          'storage transaction does not support command index scans',
+        );
+      }
+      const rows = await tx.scanRowsByIndex({
+        table: tableName,
+        index,
+        values,
+        afterRowId,
+        limit,
+      });
+      return rows.map((stored) =>
+        toValidateRow(table.columns, decodeRow(table.columns, stored.payload)),
+      );
+    },
     queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }

package/src/push.ts

@@ -35,7 +35,7 @@ import type { BlobStore } from './blob-store';
 import type { SyncRequestContext } from './context';
 import { clockOf, limitsOf } from './context';
 import type { CrdtMergerRegistry } from './crdt-merger';
-import { SyncError } from './errors';
+import { reportError, SyncError, type SyncularErrorHandler } from './errors';
 import { emitEvent } from './events';
 import {
   prepareReactions,
@@ -1110,12 +1110,41 @@ function commitValidationReader(
         serverVersion: stored.serverVersion,
       }));
     },
+    scanRowsByIndex: async ({
+      table: tableName,
+      index,
+      values,
+      afterRowId = null,
+      limit = 100,
+    }) => {
+      const table = tableFor(tableName);
+      if (tx.scanRowsByIndex === undefined) {
+        throw new Error(
+          'storage transaction does not support commit-validator index scans',
+        );
+      }
+      const rows = await tx.scanRowsByIndex({
+        table: tableName,
+        index,
+        values,
+        afterRowId,
+        limit,
+      });
+      return rows.map((stored) => ({
+        row: toValidateRow(
+          table.columns,
+          decodeRow(table.columns, stored.payload),
+        ),
+        serverVersion: stored.serverVersion,
+      }));
+    },
     queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }
 
 async function runCommitValidator(
   validator: CommitValidator | undefined,
+  onError: SyncularErrorHandler | undefined,
   tx: StorageTransaction,
   schema: CompiledSchema,
   clientId: string,
@@ -1160,10 +1189,34 @@ async function runCommitValidator(
         error.details,
       );
     }
+    // §6.8: a throw that is not a deliberate rejection is a host fault, not a
+    // rule refusal. The host's observation hook receives the original (its
+    // stack included); the client gets a distinct code and, in the §6.3.1
+    // details, the bounded thrown message. Stacks and thrown values stay off
+    // the wire.
+    reportError(error, onError, 'sync');
+    let text = '';
+    let bytes = 0;
+    const encoder = new TextEncoder();
+    const thrown = error instanceof Error ? error.message : String(error);
+    for (const char of thrown.replace(/\p{Cc}+/gu, ' ').trim()) {
+      const size = encoder.encode(char).length;
+      if (bytes + size > 256) break;
+      bytes += size;
+      text += char;
+    }
+    text = text.trimEnd();
     return errorRecord(
       operations[0]?.opIndex ?? 0,
-      'sync.constraint_violation',
+      'sync.validator_error',
       'whole-commit validator failed',
+      false,
+      {
+        reason: 'validator_threw',
+        ...(text === ''
+          ? {}
+          : { references: { validator: 'commit', message: text } }),
+      },
     );
   }
   return undefined;
@@ -1243,6 +1296,32 @@ async function enforceReferences(
     });
   }
 
+  // One candidate-state read per parent table, not one per reference: a
+  // commit that points several rows at the same parent (a fight's
+  // combatants, its chronicle) reads that parent once. Postgres binds the
+  // ids as one array per table; SQLite and D1 answer with one lookup per id.
+  const parentIds = new Map<string, Set<string>>();
+  for (const { last: write } of finalByRow.values()) {
+    if (write.op !== 'upsert' || write.values === undefined) continue;
+    for (const reference of write.table.references) {
+      const value = write.values[reference.columnIndex];
+      if (value === null || value === undefined) continue;
+      const parentRowId = renderScopeValue(value);
+      if (parentRowId === undefined) continue;
+      const ids = parentIds.get(reference.parentTable);
+      if (ids === undefined)
+        parentIds.set(reference.parentTable, new Set([parentRowId]));
+      else ids.add(parentRowId);
+    }
+  }
+  const parents = new Map<string, ReadonlyMap<string, StoredRow>>();
+  for (const [parentTable, ids] of parentIds) {
+    parents.set(
+      parentTable,
+      await readCandidateRows(tx, parentTable, [...ids]),
+    );
+  }
+
   for (const { last: write } of finalByRow.values()) {
     if (write.op !== 'upsert' || write.values === undefined) continue;
     for (const reference of write.table.references) {
@@ -1250,7 +1329,7 @@ async function enforceReferences(
       if (value === null || value === undefined) continue;
       const parentRowId = renderScopeValue(value);
       if (parentRowId === undefined) continue;
-      if ((await tx.getRow(reference.parentTable, parentRowId)) !== undefined) {
+      if (parents.get(reference.parentTable)?.has(parentRowId) === true) {
         continue;
       }
       return {
@@ -1769,6 +1848,7 @@ export async function processPushOperationsWithTrace(
     if (terminated === undefined) {
       const commitReject = await runCommitValidator(
         commitValidator,
+        ctx.onError,
         tx,
         schema,
         clientId,

package/src/validate.ts

@@ -97,6 +97,22 @@ export interface CommitValidationScanInput {
   readonly limit?: number;
 }
 
+/**
+ * One exact declared-index lookup a push-transaction reader runs (§6.8): the
+ * whole-commit reader and a registered command share this shape.
+ */
+export interface IndexScanInput {
+  readonly table: string;
+  /** `TableSchema.indexes[].name`; never exposed as a client subscription. */
+  readonly index: string;
+  /** Exact values in the index declaration's column order. */
+  readonly values: readonly RowValue[];
+  /** Resume after this rowId (exclusive); `null` = start of the match set. */
+  readonly afterRowId?: string | null;
+  /** Bounded per call by the server to 1..1,000; defaults to 100. */
+  readonly limit?: number;
+}
+
 /**
  * Candidate-state reads bound to the same storage transaction as the commit.
  * Reads observe every staged sibling operation and no uncommitted competing
@@ -120,6 +136,16 @@ export interface CommitValidationReader {
     rowIds: readonly string[],
   ): Promise<ReadonlyMap<string, CommitValidationRow>>;
   scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
+  /**
+   * Candidate-state exact lookup through one declared relational index, with
+   * the same read-your-own-writes guarantee as `scanRows`: staged sibling
+   * upserts are present and staged deletes are absent. The values form the
+   * complete, order-sensitive index tuple and `afterRowId` is an exclusive
+   * keyset cursor, so a caller pages without a scope declaration. There is no
+   * scope filter and no actor scope authorization: the caller already holds
+   * partition authority, exactly as with `scanRows`.
+   */
+  scanRowsByIndex(input: IndexScanInput): Promise<CommitValidationRow[]>;
   /**
    * Run a generated registered query on the commit transaction (§6.8). The
    * request has the shape `ServerStorage.queryAuthoritative` takes; the