@syncular/server 0.0.6-159 → 0.0.6-167

package/src/snapshot-chunks/db-metadata.ts

@@ -19,6 +19,27 @@ import type { Kysely } from 'kysely';
 import type { SyncCoreDb } from '../schema';
 import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
 
+export interface SnapshotChunkCleanupTuning {
+  /** Cleanup select/delete batch size. Default: 250 */
+  batchSize?: number;
+  /** Max concurrent blob deletes during cleanup. Default: 8 */
+  deleteConcurrency?: number;
+}
+
+export const SNAPSHOT_CHUNK_CLEANUP_TUNING_PRESETS = {
+  server: {
+    batchSize: 500,
+    deleteConcurrency: 16,
+  },
+  edge: {
+    batchSize: 100,
+    deleteConcurrency: 4,
+  },
+} as const satisfies Record<
+  'server' | 'edge',
+  Required<SnapshotChunkCleanupTuning>
+>;
+
 export interface DbMetadataSnapshotChunkStorageOptions {
   /** Database instance */
   db: Kysely<SyncCoreDb>;
@@ -26,6 +47,8 @@ export interface DbMetadataSnapshotChunkStorageOptions {
   blobAdapter: BlobStorageAdapter;
   /** Optional prefix for chunk IDs */
   chunkIdPrefix?: string;
+  /** Optional cleanup throughput tuning knobs. */
+  cleanupTuning?: SnapshotChunkCleanupTuning;
 }
 
 /**
@@ -63,7 +86,23 @@ export function createDbMetadataChunkStorage(
   ) => Promise<SyncSnapshotChunkRef | null>;
   cleanupExpired: (beforeIso: string) => Promise<number>;
 } {
-  const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
+  const { db, blobAdapter, chunkIdPrefix = 'chunk_', cleanupTuning } = options;
+
+  function positiveIntOrDefault(value: number | undefined, fallback: number) {
+    if (value === undefined) return fallback;
+    if (!Number.isFinite(value)) return fallback;
+    const normalized = Math.trunc(value);
+    return normalized > 0 ? normalized : fallback;
+  }
+
+  const CLEANUP_BATCH_SIZE = positiveIntOrDefault(
+    cleanupTuning?.batchSize,
+    250
+  );
+  const CLEANUP_DELETE_CONCURRENCY = positiveIntOrDefault(
+    cleanupTuning?.deleteConcurrency,
+    8
+  );
 
   // Generate deterministic blob hash from chunk identity metadata.
   async function computeBlobHash(metadata: {
@@ -132,6 +171,29 @@ export function createDbMetadataChunkStorage(
     }
   }
 
+  async function runWithConcurrency<T>(
+    items: readonly T[],
+    concurrency: number,
+    worker: (item: T) => Promise<void>
+  ): Promise<void> {
+    if (items.length === 0) return;
+
+    const workerCount = Math.max(1, Math.min(concurrency, items.length));
+    let nextIndex = 0;
+
+    async function runWorker(): Promise<void> {
+      while (nextIndex < items.length) {
+        const index = nextIndex;
+        nextIndex += 1;
+        const item = items[index];
+        if (item === undefined) continue;
+        await worker(item);
+      }
+    }
+
+    await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
+  }
+
   // Generate unique chunk ID
   function generateChunkId(): string {
     return `${chunkIdPrefix}${randomId()}`;
@@ -444,35 +506,53 @@ export function createDbMetadataChunkStorage(
     },
 
     async cleanupExpired(beforeIso: string): Promise<number> {
-      // Find expired chunks
-      const expiredRows = await db
-        .selectFrom('sync_snapshot_chunks')
-        .select(['chunk_id', 'blob_hash'])
-        .where('expires_at', '<=', beforeIso)
-        .execute();
-
-      if (expiredRows.length === 0) return 0;
-
-      // Delete from blob storage (best effort)
-      for (const row of expiredRows) {
-        try {
-          await blobAdapter.delete(row.blob_hash);
-        } catch {
-          // Ignore deletion errors - blob may be shared or already deleted
-          // Log for observability but don't fail the cleanup
-          console.warn(
-            `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
-          );
+      let deleted = 0;
+
+      while (true) {
+        const expiredRows = await db
+          .selectFrom('sync_snapshot_chunks')
+          .select(['chunk_id', 'blob_hash'])
+          .where('expires_at', '<=', beforeIso)
+          .orderBy('expires_at', 'asc')
+          .limit(CLEANUP_BATCH_SIZE)
+          .execute();
+
+        if (expiredRows.length === 0) {
+          break;
         }
-      }
 
-      // Delete metadata from database
-      const result = await db
-        .deleteFrom('sync_snapshot_chunks')
-        .where('expires_at', '<=', beforeIso)
-        .executeTakeFirst();
+        // Delete from blob storage (best effort).
+        await runWithConcurrency(
+          expiredRows,
+          CLEANUP_DELETE_CONCURRENCY,
+          async (row) => {
+            try {
+              await blobAdapter.delete(row.blob_hash);
+            } catch {
+              // Ignore deletion errors - blob may be shared or already deleted
+              // Log for observability but don't fail the cleanup
+              console.warn(
+                `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
+              );
+            }
+          }
+        );
+
+        const chunkIds = expiredRows.map((row) => row.chunk_id);
+        if (chunkIds.length > 0) {
+          const result = await db
+            .deleteFrom('sync_snapshot_chunks')
+            .where('chunk_id', 'in', chunkIds)
+            .executeTakeFirst();
+          deleted += Number(result.numDeletedRows ?? 0);
+        }
+
+        if (expiredRows.length < CLEANUP_BATCH_SIZE) {
+          break;
+        }
+      }
 
-      return Number(result.numDeletedRows ?? 0);
+      return deleted;
     },
   };
 }

package/src/snapshot-chunks.ts

@@ -6,11 +6,13 @@
  */
 
 import {
+  type ScopeValues,
   SYNC_SNAPSHOT_CHUNK_COMPRESSION,
   SYNC_SNAPSHOT_CHUNK_ENCODING,
   type SyncSnapshotChunkCompression,
   type SyncSnapshotChunkEncoding,
   type SyncSnapshotChunkRef,
+  sha256Hex,
 } from '@syncular/core';
 import { type Kysely, sql } from 'kysely';
 import type { SyncCoreDb } from './schema';
@@ -42,6 +44,22 @@ export interface SnapshotChunkRow {
   expiresAt: string;
 }
 
+/**
+ * Generate a stable cache key for snapshot chunks from effective scopes.
+ */
+export async function scopesToSnapshotChunkScopeKey(
+  scopes: ScopeValues
+): Promise<string> {
+  const sorted = Object.entries(scopes)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([key, value]) => {
+      const values = Array.isArray(value) ? [...value].sort() : [value];
+      return `${key}:${values.join(',')}`;
+    })
+    .join('|');
+  return sha256Hex(sorted);
+}
+
 function coerceChunkRow(value: unknown): Uint8Array {
   // pg returns Buffer (subclass of Uint8Array); sqlite returns Uint8Array
   if (value instanceof Uint8Array) return value;

package/src/stats.ts

@@ -4,23 +4,12 @@
 
 import type { Kysely, SelectQueryBuilder, SqlBool } from 'kysely';
 import { sql } from 'kysely';
+import { coerceNumber } from './dialect/helpers';
 import type { SyncCoreDb } from './schema';
 
 // biome-ignore lint/complexity/noBannedTypes: Kysely uses `{}` as the initial "no selected columns yet" marker.
 type EmptySelection = {};
 
-function coerceNumber(value: unknown): number | null {
-  if (value === null || value === undefined) return null;
-  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
-  if (typeof value === 'bigint')
-    return Number.isFinite(Number(value)) ? Number(value) : null;
-  if (typeof value === 'string') {
-    const n = Number(value);
-    return Number.isFinite(n) ? n : null;
-  }
-  return null;
-}
-
 export interface SyncStats {
   commitCount: number;
   changeCount: number;

package/src/subscriptions/resolve.ts

@@ -1,13 +1,10 @@
 import {
-  extractScopeVars,
+  collectScopeVars,
   type ScopeValues,
   type SyncSubscriptionRequest,
 } from '@syncular/core';
 import type { Kysely } from 'kysely';
-import {
-  getServerHandler,
-  type ServerHandlerCollection,
-} from '../handlers/collection';
+import type { ServerHandlerCollection } from '../handlers/collection';
 import type { SyncServerAuth } from '../handlers/types';
 import type { SyncCoreDb } from '../schema';
 import { createDefaultScopeCacheKey, type ScopeCacheBackend } from './cache';
@@ -92,19 +89,6 @@ function scopesEmpty(scopes: ScopeValues): boolean {
   return true;
 }
 
-/**
- * Collect valid scope keys from handler scope patterns.
- */
-function collectScopeKeys(scopePatterns: readonly string[]): Set<string> {
-  const keys = new Set<string>();
-  for (const pattern of scopePatterns) {
-    for (const key of extractScopeVars(pattern)) {
-      keys.add(key);
-    }
-  }
-  return keys;
-}
-
 function validateScopeKeys(args: {
   scopeValues: ScopeValues;
   validScopeKeys: Set<string>;
@@ -166,14 +150,14 @@ export async function resolveEffectiveScopesForSubscriptions<
       );
     }
 
-    const handler = getServerHandler(args.handlers, sub.table);
+    const handler = args.handlers.byTable.get(sub.table);
     if (!handler) {
       throw new InvalidSubscriptionScopeError(
         `Unknown table: ${sub.table} for subscription ${sub.id}`
       );
     }
 
-    const validScopeKeys = collectScopeKeys(handler.scopePatterns);
+    const validScopeKeys = collectScopeVars(handler.scopePatterns);
     const requested = sub.scopes ?? {};
     validateScopeKeys({
       scopeValues: requested,

package/src/sync.ts

@@ -3,6 +3,7 @@ import type {
   ColumnCodecSource,
   ScopeDefinition,
 } from '@syncular/core';
+import { registerTableOrThrow } from '@syncular/core';
 import {
   type CreateServerHandlerOptions,
   createServerHandler,
@@ -85,11 +86,11 @@ export function defineServerSync<
         ScopeDefs
       >
     ) {
-      if (registeredTables.has(handlerOptions.table)) {
-        throw new Error(
-          `Server table handler already registered: ${handlerOptions.table}`
-        );
-      }
+      registerTableOrThrow(
+        registeredTables,
+        handlerOptions.table,
+        (table) => `Server table handler already registered: ${table}`
+      );
 
       handlers.push(
         createServerHandler({
@@ -98,7 +99,6 @@ export function defineServerSync<
           codecDialect: options.codecDialect,
         })
       );
-      registeredTables.add(handlerOptions.table);
       return sync;
     },
     addPushPlugin(plugin: SyncServerPushPlugin<ServerDB, Auth>) {