@syncular/server 0.0.6-90 → 0.0.6-93

package/src/handlers/create-handler.ts

@@ -31,6 +31,7 @@ import type {
   UpdateQueryBuilder,
   UpdateResult,
 } from 'kysely';
+import { sql } from 'kysely';
 import type { SyncCoreDb } from '../schema';
 import type {
   ApplyOperationResult,
@@ -67,6 +68,14 @@ function isConstraintViolationError(message: string): boolean {
   );
 }
 
+function isMissingColumnReferenceError(message: string): boolean {
+  const normalized = message.toLowerCase();
+  return (
+    normalized.includes('no such column') ||
+    (normalized.includes('column') && normalized.includes('does not exist'))
+  );
+}
+
 /**
  * Scope definition for a column - maps scope variable to column name.
  */
@@ -505,96 +514,19 @@ export function createServerHandler<
         : {};
     const payload = applyInboundTransform(payloadRecord, ctx.schemaVersion);
 
-    // Check whether the row exists and fetch only version metadata for hot path.
-    const existingRow = await (
-      trx.selectFrom(table) as SelectQueryBuilder<
-        ServerDB,
-        keyof ServerDB & string,
-        Record<string, unknown>
-      >
-    )
-      .select(ref<string>(versionColumn))
-      .where(ref<string>(primaryKey), '=', op.row_id)
-      .executeTakeFirst();
-
-    const hasExistingRow = existingRow !== undefined;
-    const existingVersion =
-      (existingRow?.[versionColumn] as number | undefined) ?? 0;
-
-    // Check version conflict
-    if (
-      hasExistingRow &&
-      op.base_version != null &&
-      existingVersion !== op.base_version
-    ) {
-      const conflictRow = await (
-        trx.selectFrom(table).selectAll() as SelectQueryBuilder<
-          ServerDB,
-          keyof ServerDB & string,
-          Record<string, unknown>
-        >
-      )
-        .where(ref<string>(primaryKey), '=', op.row_id)
-        .executeTakeFirst();
-
-      if (!conflictRow) {
-        return {
-          result: {
-            opIndex,
-            status: 'error',
-            error: 'ROW_NOT_FOUND_FOR_BASE_VERSION',
-            code: 'ROW_MISSING',
-            retriable: false,
-          },
-          emittedChanges: [],
-        };
-      }
-
-      return {
-        result: {
-          opIndex,
-          status: 'conflict',
-          message: `Version conflict: server=${existingVersion}, base=${op.base_version}`,
-          server_version: existingVersion,
-          server_row: applyOutboundTransform(
-            conflictRow as Selectable<ServerDB[TableName]>
-          ),
-        },
-        emittedChanges: [],
-      };
-    }
-
-    // If the client provided a base version, they expected this row to exist.
-    // A missing row usually indicates stale local state after a server reset.
-    if (!hasExistingRow && op.base_version != null) {
-      return {
-        result: {
-          opIndex,
-          status: 'error',
-          error: 'ROW_NOT_FOUND_FOR_BASE_VERSION',
-          code: 'ROW_MISSING',
-          retriable: false,
-        },
-        emittedChanges: [],
-      };
-    }
-
-    const nextVersion = existingVersion + 1;
-
     let updated: Record<string, unknown> | undefined;
     let constraintError: { message: string; code: string } | null = null;
 
     try {
-      if (hasExistingRow) {
-        // Update - merge payload with existing
-        const updateSet: Record<string, unknown> = {
+      if (op.base_version != null) {
+        const expectedVersion = op.base_version;
+        const conditionalUpdateSet: Record<string, unknown> = {
           ...payload,
-          [versionColumn]: nextVersion,
+          [versionColumn]: expectedVersion + 1,
         };
-        // Don't update primary key or scope columns
-        delete updateSet[primaryKey];
+        delete conditionalUpdateSet[primaryKey];
         for (const col of Object.values(scopeColumns)) {
-          delete updateSet[col];
+          delete conditionalUpdateSet[col];
         }
 
         updated = (await (
@@ -605,31 +537,144 @@ export function createServerHandler<
             UpdateResult
           >
         )
-          .set(updateSet as UpdateSetObject)
+          .set(conditionalUpdateSet as UpdateSetObject)
           .where(ref<string>(primaryKey), '=', op.row_id)
+          .where(ref<string>(versionColumn), '=', expectedVersion)
           .returningAll()
           .executeTakeFirst()) as Record<string, unknown> | undefined;
+
+        if (!updated) {
+          const conflictRow = await (
+            trx.selectFrom(table).selectAll() as SelectQueryBuilder<
+              ServerDB,
+              keyof ServerDB & string,
+              Record<string, unknown>
+            >
+          )
+            .where(ref<string>(primaryKey), '=', op.row_id)
+            .executeTakeFirst();
+
+          if (!conflictRow) {
+            return {
+              result: {
+                opIndex,
+                status: 'error',
+                error: 'ROW_NOT_FOUND_FOR_BASE_VERSION',
+                code: 'ROW_MISSING',
+                retriable: false,
+              },
+              emittedChanges: [],
+            };
+          }
+
+          const existingVersion =
+            (conflictRow[versionColumn] as number | undefined) ?? 0;
+          return {
+            result: {
+              opIndex,
+              status: 'conflict',
+              message: `Version conflict: server=${existingVersion}, base=${expectedVersion}`,
+              server_version: existingVersion,
+              server_row: applyOutboundTransform(
+                conflictRow as Selectable<ServerDB[TableName]>
+              ),
+            },
+            emittedChanges: [],
+          };
+        }
       } else {
-        // Insert
-        const insertValues: Record<string, unknown> = {
+        const updateSet: Record<string, unknown> = {
           ...payload,
-          [primaryKey]: op.row_id,
-          [versionColumn]: 1,
+          [versionColumn]: sql`${sql.ref(versionColumn)} + 1`,
         };
+        delete updateSet[primaryKey];
+        for (const col of Object.values(scopeColumns)) {
+          delete updateSet[col];
+        }
 
         updated = (await (
-          trx.insertInto(table) as InsertQueryBuilder<
+          trx.updateTable(table) as UpdateQueryBuilder<
             ServerDB,
             TableName,
-            InsertResult
+            TableName,
+            UpdateResult
           >
         )
-          .values(insertValues as Insertable<ServerDB[TableName]>)
+          .set(updateSet as UpdateSetObject)
+          .where(ref<string>(primaryKey), '=', op.row_id)
           .returningAll()
           .executeTakeFirst()) as Record<string, unknown> | undefined;
+
+        if (!updated) {
+          const insertValues: Record<string, unknown> = {
+            ...payload,
+            [primaryKey]: op.row_id,
+            [versionColumn]: 1,
+          };
+
+          try {
+            updated = (await (
+              trx.insertInto(table) as InsertQueryBuilder<
+                ServerDB,
+                TableName,
+                InsertResult
+              >
+            )
+              .values(insertValues as Insertable<ServerDB[TableName]>)
+              .returningAll()
+              .executeTakeFirst()) as Record<string, unknown> | undefined;
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (!isConstraintViolationError(message)) {
+              throw err;
+            }
+            updated = (await (
+              trx.updateTable(table) as UpdateQueryBuilder<
+                ServerDB,
+                TableName,
+                TableName,
+                UpdateResult
+              >
+            )
+              .set(updateSet as UpdateSetObject)
+              .where(ref<string>(primaryKey), '=', op.row_id)
+              .returningAll()
+              .executeTakeFirst()) as Record<string, unknown> | undefined;
+            if (!updated) {
+              constraintError = {
+                message,
+                code: classifyConstraintViolationCode(message),
+              };
+            }
+          }
+        }
       }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
+      if (op.base_version != null && isMissingColumnReferenceError(message)) {
+        const row = await (
+          trx.selectFrom(table).selectAll() as SelectQueryBuilder<
+            ServerDB,
+            keyof ServerDB & string,
+            Record<string, unknown>
+          >
+        )
+          .where(ref<string>(primaryKey), '=', op.row_id)
+          .executeTakeFirst();
+        if (!row) {
+          return {
+            result: {
+              opIndex,
+              status: 'error',
+              error: 'ROW_NOT_FOUND_FOR_BASE_VERSION',
+              code: 'ROW_MISSING',
+              retriable: false,
+            },
+            emittedChanges: [],
+          };
+        }
+      }
+
       if (!isConstraintViolationError(message)) {
         throw err;
       }

package/src/pull.ts

@@ -35,8 +35,14 @@ import {
   readSnapshotChunkRefByPageKey,
 } from './snapshot-chunks';
 import type { SnapshotChunkStorage } from './snapshot-chunks/types';
+import {
+  createMemoryScopeCache,
+  type ScopeCacheBackend,
+} from './subscriptions/cache';
 import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
 
+const defaultScopeCache = createMemoryScopeCache();
+
 function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
   if (chunks.length === 1) {
     return chunks[0] ?? new Uint8Array();
@@ -67,6 +73,20 @@ export interface PullResult {
   clientCursor: number;
 }
 
+interface PendingExternalChunkWrite {
+  snapshot: SyncSnapshot;
+  cacheLookup: {
+    partitionId: string;
+    scopeKey: string;
+    scope: string;
+    asOfCommitSeq: number;
+    rowCursor: string | null;
+    rowLimit: number;
+  };
+  rowFramePayload: Uint8Array;
+  expiresAt: string;
+}
+
 /**
  * Generate a stable cache key for snapshot chunks.
  */
@@ -261,6 +281,12 @@ export async function pull<
    * instead of inline in the database.
    */
   chunkStorage?: SnapshotChunkStorage;
+  /**
+   * Optional shared scope cache backend.
+   * Request-local memoization is always applied, even with custom backends.
+   * Defaults to process-local memory cache.
+   */
+  scopeCache?: ScopeCacheBackend;
 }): Promise<PullResult> {
   const { request, dialect } = args;
   const db = args.db;
@@ -296,6 +322,7 @@ export async function pull<
           50
         );
         const dedupeRows = request.dedupeRows === true;
+        const pendingExternalChunkWrites: PendingExternalChunkWrite[] = [];
 
         // Resolve effective scopes for each subscription
         const resolved = await resolveEffectiveScopesForSubscriptions({
@@ -303,6 +330,7 @@ export async function pull<
           auth: args.auth,
           subscriptions: request.subscriptions ?? [],
           handlers: args.handlers,
+          scopeCache: args.scopeCache ?? defaultScopeCache,
         });
 
         const result = await dialect.executeInTransaction(db, async (trx) => {
@@ -470,63 +498,51 @@ export async function pull<
                   const rowFramePayload = concatByteChunks(
                     bundle.rowFrameParts
                   );
-                  const sha256 = await sha256Hex(rowFramePayload);
                   const expiresAt = new Date(
                     Date.now() + Math.max(1000, bundle.ttlMs)
                   ).toISOString();
 
                   if (args.chunkStorage) {
-                    if (args.chunkStorage.storeChunkStream) {
-                      const { stream: bodyStream, byteLength } =
-                        await gzipBytesToStream(rowFramePayload);
-                      chunkRef = await args.chunkStorage.storeChunkStream({
+                    const snapshot: SyncSnapshot = {
+                      table: bundle.table,
+                      rows: [],
+                      chunks: [],
+                      isFirstPage: bundle.isFirstPage,
+                      isLastPage: bundle.isLastPage,
+                    };
+                    snapshots.push(snapshot);
+                    pendingExternalChunkWrites.push({
+                      snapshot,
+                      cacheLookup: {
                         partitionId,
                         scopeKey: cacheKey,
                         scope: bundle.table,
                         asOfCommitSeq: effectiveState.asOfCommitSeq,
                         rowCursor: bundle.startCursor,
                         rowLimit: bundleRowLimit,
-                        encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                        sha256,
-                        byteLength,
-                        bodyStream,
-                        expiresAt,
-                      });
-                    } else {
-                      const compressedBody = await gzipBytes(rowFramePayload);
-                      chunkRef = await args.chunkStorage.storeChunk({
-                        partitionId,
-                        scopeKey: cacheKey,
-                        scope: bundle.table,
-                        asOfCommitSeq: effectiveState.asOfCommitSeq,
-                        rowCursor: bundle.startCursor,
-                        rowLimit: bundleRowLimit,
-                        encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                        sha256,
-                        body: compressedBody,
-                        expiresAt,
-                      });
-                    }
-                  } else {
-                    const compressedBody = await gzipBytes(rowFramePayload);
-                    const chunkId = randomId();
-                    chunkRef = await insertSnapshotChunk(trx, {
-                      chunkId,
-                      partitionId,
-                      scopeKey: cacheKey,
-                      scope: bundle.table,
-                      asOfCommitSeq: effectiveState.asOfCommitSeq,
-                      rowCursor: bundle.startCursor,
-                      rowLimit: bundleRowLimit,
-                      encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                      compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                      sha256,
-                      body: compressedBody,
+                      },
+                      rowFramePayload,
                       expiresAt,
                     });
+                    return;
                   }
+                  const sha256 = await sha256Hex(rowFramePayload);
+                  const compressedBody = await gzipBytes(rowFramePayload);
+                  const chunkId = randomId();
+                  chunkRef = await insertSnapshotChunk(trx, {
+                    chunkId,
+                    partitionId,
+                    scopeKey: cacheKey,
+                    scope: bundle.table,
+                    asOfCommitSeq: effectiveState.asOfCommitSeq,
+                    rowCursor: bundle.startCursor,
+                    rowLimit: bundleRowLimit,
+                    encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
+                    compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                    sha256,
+                    body: compressedBody,
+                    expiresAt,
+                  });
                 }
 
                 snapshots.push({
@@ -671,7 +687,6 @@ export async function pull<
                   commitSeq: number;
                   createdAt: string;
                   actorId: string;
-                  changeId: number;
                   change: SyncChange;
                 }
               >();
@@ -694,11 +709,15 @@ export async function pull<
                   scopes: r.scopes,
                 };
 
+                // Move row keys to insertion tail so Map iteration yields
+                // "latest change wins" order without a full array sort.
+                if (latestByRowKey.has(rowKey)) {
+                  latestByRowKey.delete(rowKey);
+                }
                 latestByRowKey.set(rowKey, {
                   commitSeq: r.commit_seq,
                   createdAt: r.created_at,
                   actorId: r.actor_id,
-                  changeId: r.change_id,
                   change,
                 });
               }
@@ -718,12 +737,8 @@ export async function pull<
                 continue;
               }
 
-              const latest = Array.from(latestByRowKey.values()).sort(
-                (a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId
-              );
-
               const commits: SyncCommit[] = [];
-              for (const item of latest) {
+              for (const item of latestByRowKey.values()) {
                 const lastCommit = commits[commits.length - 1];
                 if (!lastCommit || lastCommit.commitSeq !== item.commitSeq) {
                   commits.push({
@@ -829,6 +844,61 @@ export async function pull<
           };
         });
 
+        const chunkStorage = args.chunkStorage;
+        if (chunkStorage && pendingExternalChunkWrites.length > 0) {
+          for (const pending of pendingExternalChunkWrites) {
+            let chunkRef = await readSnapshotChunkRefByPageKey(db, {
+              partitionId: pending.cacheLookup.partitionId,
+              scopeKey: pending.cacheLookup.scopeKey,
+              scope: pending.cacheLookup.scope,
+              asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+              rowCursor: pending.cacheLookup.rowCursor,
+              rowLimit: pending.cacheLookup.rowLimit,
+              encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
+              compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+            });
+
+            if (!chunkRef) {
+              const sha256 = await sha256Hex(pending.rowFramePayload);
+              if (chunkStorage.storeChunkStream) {
+                const { stream: bodyStream, byteLength } =
+                  await gzipBytesToStream(pending.rowFramePayload);
+                chunkRef = await chunkStorage.storeChunkStream({
+                  partitionId: pending.cacheLookup.partitionId,
+                  scopeKey: pending.cacheLookup.scopeKey,
+                  scope: pending.cacheLookup.scope,
+                  asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+                  rowCursor: pending.cacheLookup.rowCursor,
+                  rowLimit: pending.cacheLookup.rowLimit,
+                  encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
+                  compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                  sha256,
+                  byteLength,
+                  bodyStream,
+                  expiresAt: pending.expiresAt,
+                });
+              } else {
+                const compressedBody = await gzipBytes(pending.rowFramePayload);
+                chunkRef = await chunkStorage.storeChunk({
+                  partitionId: pending.cacheLookup.partitionId,
+                  scopeKey: pending.cacheLookup.scopeKey,
+                  scope: pending.cacheLookup.scope,
+                  asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+                  rowCursor: pending.cacheLookup.rowCursor,
+                  rowLimit: pending.cacheLookup.rowLimit,
+                  encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
+                  compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                  sha256,
+                  body: compressedBody,
+                  expiresAt: pending.expiresAt,
+                });
+              }
+            }
+
+            pending.snapshot.chunks = [chunkRef];
+          }
+        }
+
         const durationMs = Math.max(0, Date.now() - startedAtMs);
         const stats = summarizePullResponse(result.response);