@syncular/server 0.0.6-96 → 0.1.0

package/src/snapshot-chunks.ts

@@ -6,11 +6,13 @@
  */
 
 import {
+  isSyncSnapshotChunkEncoding,
+  type ScopeValues,
   SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-  SYNC_SNAPSHOT_CHUNK_ENCODING,
   type SyncSnapshotChunkCompression,
   type SyncSnapshotChunkEncoding,
   type SyncSnapshotChunkRef,
+  sha256Hex,
 } from '@syncular/core';
 import { type Kysely, sql } from 'kysely';
 import type { SyncCoreDb } from './schema';
@@ -26,6 +28,21 @@ export interface SnapshotChunkPageKey {
   compression: SyncSnapshotChunkCompression;
 }
 
+export interface SnapshotChunkScopeCacheKeyInput {
+  partitionId: string;
+  scopes: ScopeValues;
+  schemaVersion?: number | string | null;
+  encoding: SyncSnapshotChunkEncoding;
+  compression: SyncSnapshotChunkCompression;
+  gzipLevel: number;
+  features?: readonly string[];
+}
+
+export type SnapshotChunkRefWithContinuation = SyncSnapshotChunkRef & {
+  nextRowCursor: string | null;
+  isLastPage: boolean;
+};
+
 export interface SnapshotChunkRow {
   chunkId: string;
   partitionId: string;
@@ -34,6 +51,8 @@ export interface SnapshotChunkRow {
   asOfCommitSeq: number;
   rowCursor: string;
   rowLimit: number;
+  nextRowCursor: string | null;
+  isLastPage: boolean;
   encoding: SyncSnapshotChunkEncoding;
   compression: SyncSnapshotChunkCompression;
   sha256: string;
@@ -42,6 +61,81 @@ export interface SnapshotChunkRow {
   expiresAt: string;
 }
 
+type SnapshotChunkDbRow = {
+  chunk_id: string;
+  partition_id: string;
+  scope_key: string;
+  scope: string;
+  as_of_commit_seq: number;
+  row_cursor: string;
+  row_limit: number;
+  next_row_cursor: unknown;
+  is_last_page: unknown;
+  encoding: string;
+  compression: string;
+  sha256: string;
+  byte_length: number;
+  blob_hash: string;
+  body?: unknown;
+  expires_at: unknown;
+};
+
+function coerceOptionalString(value: unknown): string | null {
+  if (value == null) return null;
+  return String(value);
+}
+
+function coerceFlag(value: unknown): boolean {
+  return value === true || value === 1 || value === '1';
+}
+
+/**
+ * Generate a stable cache key for snapshot chunks from effective scopes.
+ */
+export async function scopesToSnapshotChunkScopeKey(
+  scopes: ScopeValues
+): Promise<string> {
+  const sorted = Object.entries(scopes)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([key, value]) => {
+      const values = Array.isArray(value) ? [...value].sort() : [value];
+      return `${key}:${values.join(',')}`;
+    })
+    .join('|');
+  return sha256Hex(sorted);
+}
+
+/**
+ * Generate the full server-side snapshot chunk cache key.
+ *
+ * The database page key also stores table, as-of commit, row cursor, row
+ * limit, encoding, and compression as indexed columns. This scope key carries
+ * the remaining semantic dimensions that otherwise invalidate all pages for a
+ * subscription scope.
+ */
+export async function createSnapshotChunkScopeCacheKey(
+  input: SnapshotChunkScopeCacheKeyInput
+): Promise<string> {
+  const scopeDigest = await scopesToSnapshotChunkScopeKey(input.scopes);
+  const features = [...(input.features ?? [])].sort();
+  const digest = await sha256Hex(
+    JSON.stringify({
+      version: 2,
+      partitionId: input.partitionId,
+      schemaVersion:
+        input.schemaVersion === null || input.schemaVersion === undefined
+          ? 'unversioned'
+          : String(input.schemaVersion),
+      encoding: input.encoding,
+      compression: input.compression,
+      gzipLevel: input.gzipLevel,
+      features,
+      scopeDigest,
+    })
+  );
+  return `snapshot-v2:${digest}:scope:${scopeDigest}`;
+}
+
 function coerceChunkRow(value: unknown): Uint8Array {
   // pg returns Buffer (subclass of Uint8Array); sqlite returns Uint8Array
   if (value instanceof Uint8Array) return value;
@@ -64,36 +158,47 @@ function coerceIsoString(value: unknown): string {
 export async function readSnapshotChunkRefByPageKey<DB extends SyncCoreDb>(
   db: Kysely<DB>,
   args: SnapshotChunkPageKey & { nowIso?: string }
-): Promise<SyncSnapshotChunkRef | null> {
+): Promise<SnapshotChunkRefWithContinuation | null> {
   const nowIso = args.nowIso ?? new Date().toISOString();
   const rowCursorKey = args.rowCursor ?? '';
 
-  const rowResult = await sql<{
+  type PageKeyRow = {
     chunk_id: string;
     sha256: string;
     byte_length: number;
+    next_row_cursor: unknown;
+    is_last_page: unknown;
     encoding: string;
     compression: string;
-  }>`
-    select chunk_id, sha256, byte_length, encoding, compression
-    from ${sql.table('sync_snapshot_chunks')}
-    where
-      partition_id = ${args.partitionId}
-      and scope_key = ${args.scopeKey}
-      and scope = ${args.scope}
-      and as_of_commit_seq = ${args.asOfCommitSeq}
-      and row_cursor = ${rowCursorKey}
-      and row_limit = ${args.rowLimit}
-      and encoding = ${args.encoding}
-      and compression = ${args.compression}
-      and expires_at > ${nowIso}
-    limit 1
-  `.execute(db);
+  };
+
+  const rowResult = await sql<PageKeyRow>`
+        select
+          chunk_id,
+          sha256,
+          byte_length,
+          next_row_cursor,
+          is_last_page,
+          encoding,
+          compression
+        from ${sql.table('sync_snapshot_chunks')}
+        where
+          partition_id = ${args.partitionId}
+          and scope_key = ${args.scopeKey}
+          and scope = ${args.scope}
+          and as_of_commit_seq = ${args.asOfCommitSeq}
+          and row_cursor = ${rowCursorKey}
+          and row_limit = ${args.rowLimit}
+          and encoding = ${args.encoding}
+          and compression = ${args.compression}
+          and expires_at > ${nowIso}
+        limit 1
+      `.execute(db);
   const row = rowResult.rows[0];
 
   if (!row) return null;
 
-  if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
+  if (row.encoding !== args.encoding) {
     throw new Error(
       `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
     );
@@ -104,13 +209,16 @@ export async function readSnapshotChunkRefByPageKey<DB extends SyncCoreDb>(
     );
   }
 
-  return {
+  const ref: SnapshotChunkRefWithContinuation = {
     id: row.chunk_id,
     sha256: row.sha256,
     byteLength: Number(row.byte_length ?? 0),
-    encoding: row.encoding,
+    nextRowCursor: coerceOptionalString(row.next_row_cursor),
+    isLastPage: coerceFlag(row.is_last_page),
+    encoding: args.encoding,
     compression: row.compression,
   };
+  return ref;
 }
 
 export async function insertSnapshotChunk<DB extends SyncCoreDb>(
@@ -123,17 +231,19 @@ export async function insertSnapshotChunk<DB extends SyncCoreDb>(
     asOfCommitSeq: number;
     rowCursor: string | null;
     rowLimit: number;
+    nextRowCursor?: string | null;
+    isLastPage?: boolean;
     encoding: SyncSnapshotChunkEncoding;
     compression: SyncSnapshotChunkCompression;
     sha256: string;
     body: Uint8Array;
     expiresAt: string;
   }
-): Promise<SyncSnapshotChunkRef> {
+): Promise<SnapshotChunkRefWithContinuation> {
   const now = new Date().toISOString();
   const rowCursorKey = args.rowCursor ?? '';
 
-  // Use content hash as blob_hash for legacy storage in DB
+  // Database-backed chunks use their content digest as the blob identifier.
   const blobHash = `sha256:${args.sha256}`;
 
   await sql`
@@ -145,6 +255,8 @@ export async function insertSnapshotChunk<DB extends SyncCoreDb>(
       as_of_commit_seq,
       row_cursor,
       row_limit,
+      next_row_cursor,
+      is_last_page,
       encoding,
       compression,
       sha256,
@@ -162,6 +274,8 @@ export async function insertSnapshotChunk<DB extends SyncCoreDb>(
       ${args.asOfCommitSeq},
       ${rowCursorKey},
       ${args.rowLimit},
+      ${args.nextRowCursor ?? null},
+      ${args.isLastPage ? 1 : 0},
       ${args.encoding},
       ${args.compression},
       ${args.sha256},
@@ -183,6 +297,8 @@ export async function insertSnapshotChunk<DB extends SyncCoreDb>(
     )
     do update set
       expires_at = ${args.expiresAt},
+      next_row_cursor = ${args.nextRowCursor ?? null},
+      is_last_page = ${args.isLastPage ? 1 : 0},
       blob_hash = ${blobHash}
   `.execute(db);
 
@@ -217,46 +333,56 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
     };
   }
 ): Promise<SnapshotChunkRow | null> {
-  const rowResult = await sql<{
-    chunk_id: string;
-    partition_id: string;
-    scope_key: string;
-    scope: string;
-    as_of_commit_seq: number;
-    row_cursor: string;
-    row_limit: number;
-    encoding: string;
-    compression: string;
-    sha256: string;
-    byte_length: number;
-    blob_hash: string;
-    body: unknown;
-    expires_at: unknown;
-  }>`
-    select
-      chunk_id,
-      partition_id,
-      scope_key,
-      scope,
-      as_of_commit_seq,
-      row_cursor,
-      row_limit,
-      encoding,
-      compression,
-      sha256,
-      byte_length,
-      blob_hash,
-      body,
-      expires_at
-    from ${sql.table('sync_snapshot_chunks')}
-    where chunk_id = ${chunkId}
-    limit 1
-  `.execute(db);
+  const includeBody = !options?.chunkStorage;
+  const rowResult = includeBody
+    ? await sql<SnapshotChunkDbRow>`
+        select
+          chunk_id,
+          partition_id,
+          scope_key,
+          scope,
+          as_of_commit_seq,
+          row_cursor,
+          row_limit,
+          next_row_cursor,
+          is_last_page,
+          encoding,
+          compression,
+          sha256,
+          byte_length,
+          blob_hash,
+          body,
+          expires_at
+        from ${sql.table('sync_snapshot_chunks')}
+        where chunk_id = ${chunkId}
+        limit 1
+      `.execute(db)
+    : await sql<SnapshotChunkDbRow>`
+        select
+          chunk_id,
+          partition_id,
+          scope_key,
+          scope,
+          as_of_commit_seq,
+          row_cursor,
+          row_limit,
+          next_row_cursor,
+          is_last_page,
+          encoding,
+          compression,
+          sha256,
+          byte_length,
+          blob_hash,
+          expires_at
+        from ${sql.table('sync_snapshot_chunks')}
+        where chunk_id = ${chunkId}
+        limit 1
+      `.execute(db);
   const row = rowResult.rows[0];
 
   if (!row) return null;
 
-  if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
+  if (!isSyncSnapshotChunkEncoding(row.encoding)) {
     throw new Error(
       `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
     );
@@ -267,7 +393,6 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
     );
   }
 
-  // Read body from external storage if available, otherwise use inline body
   let body: Uint8Array | ReadableStream<Uint8Array>;
   if (options?.chunkStorage) {
     if (options.chunkStorage.readChunkStream) {
@@ -277,25 +402,22 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
         body = externalBodyStream;
       } else {
         const externalBody = await options.chunkStorage.readChunk(chunkId);
-        if (externalBody) {
-          body = externalBody;
-        } else if (row.body) {
-          body = coerceChunkRow(row.body);
-        } else {
+        if (!externalBody) {
           throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
         }
+        body = externalBody;
       }
     } else {
       const externalBody = await options.chunkStorage.readChunk(chunkId);
-      if (externalBody) {
-        body = externalBody;
-      } else if (row.body) {
-        body = coerceChunkRow(row.body);
-      } else {
+      if (!externalBody) {
         throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
       }
+      body = externalBody;
     }
   } else {
+    if (!row.body) {
+      throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+    }
     body = coerceChunkRow(row.body);
   }
 
@@ -307,6 +429,8 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
     asOfCommitSeq: Number(row.as_of_commit_seq ?? 0),
     rowCursor: row.row_cursor,
     rowLimit: Number(row.row_limit ?? 0),
+    nextRowCursor: coerceOptionalString(row.next_row_cursor),
+    isLastPage: coerceFlag(row.is_last_page),
     encoding: row.encoding,
     compression: row.compression,
     sha256: row.sha256,