@syncular/server 0.33.0 → 0.35.0

package/src/operations.ts

@@ -8,11 +8,12 @@ import {
   encodeRemoteOperationResponse,
   encodeSparseRow,
   type PushOperation,
+  type RemoteOperationRequest,
   type RemoteOperationResponse,
   type RowValue,
   type ScopeMap,
 } from '@syncular/core';
-import type { SyncRequestContext } from './context';
+import type { ResolveScopes, SyncRequestContext } from './context';
 import {
   REMOTE_COMMAND_CLIENT_ID_PREFIX,
   RESOLVER_OUTAGE,
@@ -33,7 +34,11 @@ import type {
 } from './storage';
 import type { StorageTransaction } from './storage';
 import { transactionQuery } from './storage-query';
-import { toValidateRow, type ValidateRow } from './validate';
+import {
+  toValidateRow,
+  ValidationRejection,
+  type ValidateRow,
+} from './validate';
 
 export interface RemoteQueryDependency {
   readonly table: string;
@@ -683,10 +688,41 @@ export class RemoteOperationRegistry {
   }
 }
 
+/**
+ * The registry and identity one remote operation request runs with
+ * (REMOTE.md §5.1). `actorId` is the commit's actor and defaults to the
+ * authenticated actor; `resolveScopes` replaces the configured resolver for
+ * this request.
+ */
+export interface RemoteOperationBinding {
+  readonly registry: RemoteOperationRegistry;
+  readonly actorId?: string;
+  readonly resolveScopes?: ResolveScopes;
+}
+
+/**
+ * Chooses the binding of one decoded request (REMOTE.md §5.1). `ctx` carries
+ * the authenticated actor; the resolver runs after the client-id binding
+ * check and may throw a `SyncError` or a `ValidationRejection`.
+ */
+export type RemoteOperationResolver = (
+  request: RemoteOperationRequest,
+  ctx: SyncRequestContext,
+) => RemoteOperationBinding | Promise<RemoteOperationBinding>;
+
 function encodeRemoteOperationError(
   error: unknown,
   fallbackCode: 'operation.invalid_request' | 'operation.execution_failed',
 ): Uint8Array {
+  if (error instanceof ValidationRejection) {
+    return encodeRemoteOperationResponse({
+      revision: 1,
+      kind: 'error',
+      code: error.code,
+      message: error.message,
+      retryable: false,
+    });
+  }
   const sync =
     error instanceof SyncError
       ? error
@@ -707,8 +743,8 @@ function encodeRemoteOperationError(
 
 export async function handleRemoteOperation(
   bytes: Uint8Array,
-  ctx: SyncRequestContext,
-  registry: RemoteOperationRegistry,
+  authenticated: SyncRequestContext,
+  operations: RemoteOperationRegistry | RemoteOperationResolver,
 ): Promise<Uint8Array> {
   let request;
   try {
@@ -728,6 +764,7 @@ export async function handleRemoteOperation(
   } catch (error) {
     return encodeRemoteOperationError(error, 'operation.invalid_request');
   }
+  let ctx = authenticated;
   try {
     await touchAuthenticatedPartition(ctx);
     if (request.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
@@ -752,6 +789,16 @@ export async function handleRemoteOperation(
     ) {
       throw syncError('operation.invalid_request');
     }
+    let registry = operations;
+    if (typeof registry === 'function') {
+      const binding = await registry(request, ctx);
+      registry = binding.registry;
+      ctx = {
+        ...ctx,
+        actorId: binding.actorId ?? ctx.actorId,
+        resolveScopes: binding.resolveScopes ?? ctx.resolveScopes,
+      };
+    }
     const operation = registry.get(request.operationId);
     if (operation === undefined) {
       throw syncError('operation.unknown');
@@ -772,7 +819,7 @@ export async function handleRemoteOperation(
       ),
     );
   } catch (error) {
-    if (!(error instanceof SyncError)) {
+    if (!(error instanceof SyncError || error instanceof ValidationRejection)) {
       reportError(error, ctx.onError, 'operations');
     }
     return encodeRemoteOperationError(error, 'operation.execution_failed');

package/src/postgres-storage.ts

@@ -121,6 +121,8 @@ import type {
   RowScanQuery,
   ScopeActivityQuery,
   ScopeCommitActivity,
+  ScopeValueHistoryQuery,
+  ScopeValueQuery,
   ServerStorage,
   ServeGate,
   StorageTransaction,
@@ -2567,6 +2569,63 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
     return rows.map((r) => r.blob_id);
   }
 
+  async listScopeValueRows(
+    partition: string,
+    query: ScopeValueQuery,
+  ): Promise<string[]> {
+    const { rows } = await this.#exec.query<{ row_id: string }>(
+      `SELECT row_id FROM sync_row_scopes
+        WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=$4
+        ORDER BY row_id LIMIT $5`,
+      [partition, query.table, query.variable, query.value, query.limit],
+    );
+    return rows.map((row) => row.row_id);
+  }
+
+  async eraseScopeValueHistory(
+    partition: string,
+    query: ScopeValueHistoryQuery,
+  ): Promise<number> {
+    const params = [
+      partition,
+      query.table,
+      query.variable,
+      query.value,
+      query.throughSeq,
+      query.limit,
+    ];
+    return this.#exec.transaction(async (client) => {
+      await lockPartitionOn(client, partition);
+      // Upsert change records of the binding, then the change scope entries
+      // whose commit no longer holds a change of the binding.
+      const changes = await client.query(
+        `DELETE FROM sync_changes c USING (
+           SELECT c2.commit_seq, c2.idx FROM sync_change_scopes s
+             JOIN sync_changes c2 ON c2.partition=s.partition
+              AND c2.commit_seq=s.commit_seq AND c2.tbl=s.tbl
+            WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
+              AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
+            LIMIT $6) d
+         WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx`,
+        params,
+      );
+      const scopes = await client.query(
+        `DELETE FROM sync_change_scopes cs USING (
+           SELECT s.commit_seq FROM sync_change_scopes s
+            WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
+              AND s.commit_seq<=$5 AND NOT EXISTS (
+                SELECT 1 FROM sync_changes c
+                 WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
+                   AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
+            LIMIT $6) d
+         WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
+           AND cs.commit_seq=d.commit_seq`,
+        params,
+      );
+      return changes.rowCount + scopes.rowCount;
+    });
+  }
+
   // -- admin/console read surface --------------------------------------------
 
   async listClientRecords(partition: string): Promise<ClientRecord[]> {

package/src/push.ts

@@ -75,7 +75,7 @@ import {
  * columns (§5.9.4), skipping NULLs. Malformed BlobRefs already failed at
  * row-codec decode (§5.9.1), so `parseBlobRef` here is total.
  */
-function blobIdsInRow(
+export function blobIdsInRow(
   table: CompiledTable,
   values: readonly RowValue[],
 ): string[] {

package/src/relational-rows.ts

@@ -709,20 +709,17 @@ export function prefetchRowsSql(table: CompiledTable): string {
  * SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
  * one variable's values, as one index-only max per value (a `(partition,
  * tbl, var, value)` prefix), so the cost is independent of the table's
- * history. Bind order: [partition, tbl, var, value] per value.
+ * history. The values arrive as one JSON array, so the statement has no
+ * compound SELECT and four bound parameters for any value count (D1 caps a
+ * compound SELECT at 5 terms and a statement at 100 parameters).
+ * Params: [partition, tbl, var, values as JSON array text].
  */
-export function latestScopedChangeSeqSql(valueCount: number): string {
-  const perValue: string[] = [];
-  for (let i = 0; i < valueCount; i++) {
-    perValue.push(
-      'SELECT (SELECT max(commit_seq) FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?) AS seq',
-    );
-  }
-  return `SELECT max(seq) AS seq FROM (${perValue.join(' UNION ALL ')})`;
-}
+export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
+           WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
+     FROM json_each(?) AS v`;
 
 /**
- * PostgreSQL set-based form of `latestScopedChangeSeqSql` for the
+ * PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
  * subscriptions of one table in a pull. `$2` is a JSON array of
  * `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
  */

package/src/sqlite-blob-store.ts

@@ -33,7 +33,17 @@ export class SqliteBlobStore implements BlobStore {
     mediaType?: string,
   ): Promise<BlobRecord> {
     const existing = await this.get(partition, blobId);
-    if (existing !== undefined) return existing.record;
+    if (existing !== undefined) {
+      if (existing.record.mediaType !== undefined || mediaType === undefined)
+        return existing.record;
+      // A re-upload names the media type an earlier upload left out.
+      this.db
+        .query(
+          'UPDATE sync_blobs SET media_type=? WHERE partition=? AND blob_id=? AND media_type IS NULL',
+        )
+        .run(mediaType, partition, blobId);
+      return { ...existing.record, mediaType };
+    }
     const record: BlobRecord = {
       blobId,
       partition,
@@ -51,6 +61,12 @@ export class SqliteBlobStore implements BlobStore {
     return record;
   }
 
+  async delete(partition: string, blobId: string): Promise<void> {
+    this.db
+      .query('DELETE FROM sync_blobs WHERE partition=? AND blob_id=?')
+      .run(partition, blobId);
+  }
+
   async has(partition: string, blobId: string): Promise<boolean> {
     const row = this.db
       .query<{ n: number }, [string, string]>(

package/src/sqlite-dialect.ts

@@ -18,6 +18,7 @@
 import type { PushOperationResult, ScopeMap } from '@syncular/core';
 import { matchesEffective } from './scopes';
 import type {
+  ScopeValueHistoryQuery,
   StoredChange,
   StoredCommit,
   StoredPushResult,
@@ -151,6 +152,48 @@ export function sqliteDdlStatements(): string[] {
     .filter((statement) => statement.length > 0);
 }
 
+/** §4.6.1: rowIds of one table bound to one scope value, budgeted. */
+export const LIST_SCOPE_VALUE_ROWS_SQLITE_SQL =
+  'SELECT row_id FROM sync_row_scopes WHERE partition=? AND tbl=? AND var=? AND value=? ORDER BY row_id LIMIT ?';
+
+/**
+ * §4.6.1 history scrub, bound by `eraseScopeValueHistoryParams`: upsert change
+ * records of the binding first, then the change scope entries whose commit
+ * no longer holds a change of the binding. Each returns one row per removed
+ * record (D1 batch results carry no reliable change count).
+ */
+export const ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL = [
+  `DELETE FROM sync_changes WHERE rowid IN (
+     SELECT c.rowid FROM sync_change_scopes s JOIN sync_changes c
+       ON c.partition=s.partition AND c.commit_seq=s.commit_seq AND c.tbl=s.tbl
+     WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
+       AND s.commit_seq<=?5 AND c.op=1 AND json_extract(c.scopes, ?6)=s.value
+     LIMIT ?7) RETURNING 1 AS removed`,
+  `DELETE FROM sync_change_scopes WHERE rowid IN (
+     SELECT s.rowid FROM sync_change_scopes s
+     WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
+       AND s.commit_seq<=?5 AND NOT EXISTS (
+         SELECT 1 FROM sync_changes c
+         WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
+           AND c.tbl=s.tbl AND json_extract(c.scopes, ?6)=s.value)
+     LIMIT ?7) RETURNING 1 AS removed`,
+] as const;
+
+export function eraseScopeValueHistoryParams(
+  partition: string,
+  query: ScopeValueHistoryQuery,
+): [string, string, string, string, number, string, number] {
+  return [
+    partition,
+    query.table,
+    query.variable,
+    query.value,
+    query.throughSeq,
+    `$.${JSON.stringify(query.variable)}`,
+    query.limit,
+  ];
+}
+
 /** `?,?,…` for an `IN (…)` clause of `count` positional parameters. */
 export function placeholders(count: number): string {
   return new Array(count).fill('?').join(',');

package/src/sqlite-storage.ts

@@ -22,7 +22,7 @@ import {
   assertPhysicalColumns,
   assertStoredLayouts,
   commitWindowPageSql,
-  latestScopedChangeSeqSql,
+  LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
   deleteRowSql,
   deleteSqliteRowScopesSql,
   dropTableDdl,
@@ -56,6 +56,9 @@ import { matchesEffective } from './scopes';
 import {
   collectCommitWindowPage,
   deserializePushResult,
+  ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL,
+  eraseScopeValueHistoryParams,
+  LIST_SCOPE_VALUE_ROWS_SQLITE_SQL,
   placeholders,
   SQLITE_DDL,
   type SqliteChangeRecord,
@@ -100,6 +103,8 @@ import type {
   RowScanQuery,
   ScopeActivityQuery,
   ScopeCommitActivity,
+  ScopeValueHistoryQuery,
+  ScopeValueQuery,
   ServerStorage,
   ServeGate,
   StorageTransaction,
@@ -1639,16 +1644,9 @@ END`);
     if (firstVariable === undefined || values.length === 0) return 0;
     const row = this.db
       .query<{ seq: number | null }, string[]>(
-        latestScopedChangeSeqSql(values.length),
+        LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
       )
-      .get(
-        ...values.flatMap((value) => [
-          partition,
-          query.table,
-          firstVariable,
-          value,
-        ]),
-      );
+      .get(partition, query.table, firstVariable, JSON.stringify(values));
     return row?.seq ?? 0;
   }
 
@@ -1932,6 +1930,40 @@ END`);
     return rows.map((r) => r.blob_id);
   }
 
+  async listScopeValueRows(
+    partition: string,
+    query: ScopeValueQuery,
+  ): Promise<string[]> {
+    return this.db
+      .query<{ row_id: string }, [string, string, string, string, number]>(
+        LIST_SCOPE_VALUE_ROWS_SQLITE_SQL,
+      )
+      .all(partition, query.table, query.variable, query.value, query.limit)
+      .map((row) => row.row_id);
+  }
+
+  async eraseScopeValueHistory(
+    partition: string,
+    query: ScopeValueHistoryQuery,
+  ): Promise<number> {
+    const params = eraseScopeValueHistoryParams(partition, query);
+    return this.#serializeWrite(() => {
+      this.db.exec('BEGIN IMMEDIATE');
+      try {
+        let removed = 0;
+        for (const sql of ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL) {
+          removed += this.db.query(sql).all(...params).length;
+        }
+        this.db.exec('COMMIT');
+        return removed;
+      } catch (error) {
+        rollbackAfterError(() => {
+          this.db.exec('ROLLBACK');
+        }, error);
+      }
+    });
+  }
+
   // -- admin/console read surface --------------------------------------------
 
   async listClientRecords(partition: string): Promise<ClientRecord[]> {

package/src/storage-errors.ts

@@ -34,7 +34,9 @@ export type StorageQueryErrorCode =
   | 'sync.storage.stored_layout_mismatch'
   | 'sync.storage.physical_layout_mismatch'
   | 'sync.storage.transaction_query_unsupported'
-  | 'sync.storage.query_over_staged_writes';
+  | 'sync.storage.query_over_staged_writes'
+  | 'sync.storage.erase_unsupported'
+  | 'sync.storage.invalid_erase_query';
 
 const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
   {
@@ -78,6 +80,10 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
       'trusted row lookup requires one exact value per index column',
     'sync.storage.invalid_limit':
       'trusted row lookup limit must be an integer from 1 through 1,000',
+    'sync.storage.erase_unsupported':
+      'this storage backend cannot erase a scope value',
+    'sync.storage.invalid_erase_query':
+      'erasure requires a declared scope variable, a non-empty value, and a row budget within the operation cap',
   };
 
 /**

package/src/storage.ts

@@ -957,6 +957,29 @@ export interface ServerStorage {
   ): Promise<BlobReferencingRow[]>;
   listReferencedBlobIds?(partition: string): Promise<string[]>;
 
+  /**
+   * §4.6.1 scope-value erasure — ADDITIVE, optional; `eraseScopeValue`
+   * requires both.
+   *
+   * `listScopeValueRows`: up to `limit` rowIds of `table` whose stored scope
+   * binds `variable` to `value`, ordered by rowId (the row scope index).
+   *
+   * `eraseScopeValueHistory`: under the partition write lock, removes up to
+   * `limit` upsert change records of `table` at or below `throughSeq` whose
+   * scopes bind `variable` to `value`, then up to `limit` change scope index
+   * entries of that binding whose commit holds no such change any more.
+   * Delete change records stay: clients behind the erasure still need them.
+   * Returns the number of records removed.
+   */
+  listScopeValueRows?(
+    partition: string,
+    query: ScopeValueQuery,
+  ): Promise<string[]>;
+  eraseScopeValueHistory?(
+    partition: string,
+    query: ScopeValueHistoryQuery,
+  ): Promise<number>;
+
   /**
    * Admin/console read surface (`SyncularAdmin`): ADDITIVE,
    * optional. All read-only, partition-scoped, and JSON-able. A backend
@@ -1008,6 +1031,18 @@ export interface ServerStorage {
   listPartitions(): Promise<string[]>;
 }
 
+/** One scope binding of one table, budgeted (§4.6.1). */
+export interface ScopeValueQuery {
+  readonly table: string;
+  readonly variable: string;
+  readonly value: string;
+  readonly limit: number;
+}
+
+export interface ScopeValueHistoryQuery extends ScopeValueQuery {
+  readonly throughSeq: number;
+}
+
 /** A row referencing a blob, with the scopes needed to authorize download. */
 export interface BlobReferencingRow {
   readonly table: string;