@syncular/server 0.0.6-96 → 0.1.1

package/src/proxy/collection.ts

@@ -1,3 +1,4 @@
+import { createTableLookup } from '@syncular/core';
 import type { ProxyTableHandler } from './types';
 
 export interface ProxyHandlerCollection {
@@ -8,32 +9,9 @@ export interface ProxyHandlerCollection {
 export function createProxyHandlerCollection(
   handlers: ProxyTableHandler[]
 ): ProxyHandlerCollection {
-  const byTable = new Map<string, ProxyTableHandler>();
-  for (const handler of handlers) {
-    if (byTable.has(handler.table)) {
-      throw new Error(
-        `Proxy table handler already registered: ${handler.table}`
-      );
-    }
-    byTable.set(handler.table, handler);
-  }
+  const byTable = createTableLookup(
+    handlers,
+    (table) => `Proxy table handler already registered: ${table}`
+  );
   return { handlers, byTable };
 }
-
-export function getProxyHandler(
-  collection: ProxyHandlerCollection,
-  tableName: string
-): ProxyTableHandler | undefined {
-  return collection.byTable.get(tableName);
-}
-
-export function getProxyHandlerOrThrow(
-  collection: ProxyHandlerCollection,
-  tableName: string
-): ProxyTableHandler {
-  const handler = collection.byTable.get(tableName);
-  if (!handler) {
-    throw new Error(`No proxy table handler for table: ${tableName}`);
-  }
-  return handler;
-}

package/src/proxy/handler.ts

@@ -8,7 +8,7 @@ import type { Kysely, RawBuilder } from 'kysely';
 import { sql } from 'kysely';
 import type { ServerSyncDialect } from '../dialect/types';
 import type { SyncCoreDb } from '../schema';
-import { getProxyHandler, type ProxyHandlerCollection } from './collection';
+import type { ProxyHandlerCollection } from './collection';
 import {
   appendReturning,
   detectMutation,
@@ -111,7 +111,7 @@ export async function executeProxyQuery<DB extends SyncCoreDb>(
   }
 
   // Check if this table has a registered handler
-  const handler = getProxyHandler(handlers, mutation.tableName);
+  const handler = handlers.byTable.get(mutation.tableName);
   if (!handler) {
     // No handler registered - execute without oplog
     // This allows proxy operations on non-synced tables

package/src/proxy/index.ts

@@ -8,8 +8,6 @@
 // Collections
 export {
   createProxyHandlerCollection,
-  getProxyHandler,
-  getProxyHandlerOrThrow,
   type ProxyHandlerCollection,
 } from './collection';
 // Query execution

package/src/proxy/oplog.ts

@@ -6,19 +6,13 @@
 
 import { randomId, type SyncOp } from '@syncular/core';
 import { type Kysely, sql } from 'kysely';
+import { finalizeCommitIntegrity } from '../commit-integrity';
+import { toDialectJsonValue } from '../dialect/helpers';
 import type { ServerSyncDialect } from '../dialect/types';
+import { createScopeCommitIndexEntries } from '../helpers/scope-commit-index';
 import type { SyncCoreDb } from '../schema';
 import type { ProxyTableHandler } from './types';
 
-function toDialectJsonValue(
-  dialect: ServerSyncDialect,
-  value: unknown
-): unknown {
-  if (value === null || value === undefined) return null;
-  if (dialect.family === 'sqlite') return JSON.stringify(value);
-  return value;
-}
-
 /**
  * Create oplog entries for affected rows.
  *
@@ -88,6 +82,7 @@ export async function createOplogEntries<DB extends SyncCoreDb>(args: {
       row_json:
         operation === 'delete' ? null : toDialectJsonValue(dialect, row),
       row_version: row[versionCol] != null ? Number(row[versionCol]) : null,
+      raw_scopes: scopes,
       scopes: dialect.scopesToDb(scopes),
     };
   });
@@ -121,6 +116,35 @@ export async function createOplogEntries<DB extends SyncCoreDb>(args: {
     )}
   `.execute(trx);
 
+  const scopeEntries = createScopeCommitIndexEntries(
+    changes.map((change) => ({
+      table: change.table,
+      scopes: change.raw_scopes,
+    }))
+  );
+  if (scopeEntries.length > 0) {
+    await sql`
+      insert into ${sql.table('sync_scope_commits')} (
+        partition_id,
+        "table",
+        scope_key,
+        commit_seq
+      )
+      values ${sql.join(
+        scopeEntries.map(
+          (entry) => sql`(
+            ${partitionId},
+            ${entry.table},
+            ${entry.scopeKey},
+            ${commitSeq}
+          )`
+        ),
+        sql`, `
+      )}
+      on conflict (partition_id, "table", scope_key, commit_seq) do nothing
+    `.execute(trx);
+  }
+
   // Update commit with affected tables
   const affectedTables = Array.from(affectedTablesSet);
   const sortedAffectedTables = affectedTables.sort();
@@ -143,8 +167,15 @@ export async function createOplogEntries<DB extends SyncCoreDb>(args: {
           sql`, `
         )}
 	      on conflict (partition_id, "table", commit_seq) do nothing
-	    `.execute(trx);
+    `.execute(trx);
   }
 
+  await finalizeCommitIntegrity({
+    db: trx,
+    dialect,
+    partitionId,
+    commitSeq,
+  });
+
   return { commitSeq, affectedTables };
 }

package/src/prune.ts

@@ -18,22 +18,11 @@ import type {
   SqlBool,
 } from 'kysely';
 import { sql } from 'kysely';
+import { coerceNumber } from './dialect/helpers';
 import type { SyncCoreDb } from './schema';
 
 type EmptySelection = Record<string, never>;
 
-function coerceNumber(value: unknown): number | null {
-  if (value === null || value === undefined) return null;
-  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
-  if (typeof value === 'bigint')
-    return Number.isFinite(Number(value)) ? Number(value) : null;
-  if (typeof value === 'string') {
-    const n = Number(value);
-    return Number.isFinite(n) ? n : null;
-  }
-  return null;
-}
-
 export interface PruneOptions {
   /** Clients with updated_at older than this are ignored for watermark. Default: 14 days. */
   activeWindowMs?: number;
@@ -49,9 +38,56 @@ export interface PruneOptions {
   keepNewestCommits?: number;
 }
 
+export interface PartitionPrunePreview {
+  partitionId: string;
+  watermarkCommitSeq: number;
+  commitsToDelete: number;
+}
+
+async function readPrunePartitionIds<DB extends SyncCoreDb>(
+  db: Kysely<DB>
+): Promise<string[]> {
+  const res = await sql<{ partition_id: string }>`
+    SELECT DISTINCT partition_id
+    FROM sync_commits
+    ORDER BY partition_id ASC
+  `.execute(db);
+
+  return res.rows
+    .map((row) => row.partition_id)
+    .filter((partitionId) => typeof partitionId === 'string' && partitionId);
+}
+
+async function computePruneUpToCommitSeq<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  partitionId: string,
+  watermarkCommitSeq: number,
+  keepNewestCommits: number
+): Promise<number> {
+  if (watermarkCommitSeq <= 0) return 0;
+
+  type SyncDb = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
+  const syncDb = db as SyncDb;
+  const commitsQ = syncDb.selectFrom('sync_commits') as SelectQueryBuilder<
+    SyncCoreDb,
+    'sync_commits',
+    EmptySelection
+  >;
+
+  const maxRow = await commitsQ
+    .select(({ fn }) => [fn.max('commit_seq').as('maxSeq')])
+    .where('partition_id', '=', partitionId)
+    .executeTakeFirst();
+
+  const maxSeq = coerceNumber(maxRow?.maxSeq) ?? 0;
+  const minKept = Math.max(0, maxSeq - keepNewestCommits);
+  return Math.min(watermarkCommitSeq, minKept);
+}
+
 export async function computePruneWatermarkCommitSeq<DB extends SyncCoreDb>(
   db: Kysely<DB>,
-  options: PruneOptions = {}
+  options: PruneOptions = {},
+  partitionId = 'default'
 ): Promise<number> {
   type SyncDb = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
   const syncDb = db as SyncDb;
@@ -64,7 +100,8 @@ export async function computePruneWatermarkCommitSeq<DB extends SyncCoreDb>(
   ) as SelectQueryBuilder<SyncCoreDb, 'sync_client_cursors', EmptySelection>;
 
   const row = await cursorsQ
-    .select(({ fn }) => fn.min('cursor').as('minCursor'))
+    .select(({ fn }) => [fn.min('cursor').as('minCursor')])
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
     .where(sql<SqlBool>`updated_at >= ${cutoffIso}`)
     .where(sql<SqlBool>`cursor >= ${0}`)
     .executeTakeFirst();
@@ -82,7 +119,8 @@ export async function computePruneWatermarkCommitSeq<DB extends SyncCoreDb>(
   >;
 
   const ageRow = await commitsQ
-    .select(({ fn }) => fn.max('commit_seq').as('maxSeq'))
+    .select(({ fn }) => [fn.max('commit_seq').as('maxSeq')])
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
     .where(sql<SqlBool>`created_at < ${ageCutoffIso}`)
     .executeTakeFirst();
 
@@ -92,34 +130,41 @@ export async function computePruneWatermarkCommitSeq<DB extends SyncCoreDb>(
 
 export async function pruneSync<DB extends SyncCoreDb>(
   db: Kysely<DB>,
-  args: { watermarkCommitSeq: number; keepNewestCommits?: number }
+  args: {
+    partitionId?: string;
+    watermarkCommitSeq: number;
+    keepNewestCommits?: number;
+  }
 ): Promise<number> {
   if (args.watermarkCommitSeq <= 0) return 0;
 
   type SyncDb = Pick<Kysely<SyncCoreDb>, 'deleteFrom' | 'selectFrom'>;
   const syncDb = db as SyncDb;
+  const partitionId = args.partitionId ?? 'default';
 
   const keepNewestCommits = args.keepNewestCommits ?? 1000;
-
-  // Don't delete the newest N commits (even if watermark is higher)
-  const commitsQ = syncDb.selectFrom('sync_commits') as SelectQueryBuilder<
-    SyncCoreDb,
-    'sync_commits',
-    EmptySelection
-  >;
-
-  const maxRow = await commitsQ
-    .select(({ fn }) => fn.max('commit_seq').as('maxSeq'))
-    .executeTakeFirst();
-
-  const maxSeq = coerceNumber(maxRow?.maxSeq) ?? 0;
-  const minKept = Math.max(0, maxSeq - keepNewestCommits);
-  const pruneUpTo = Math.min(args.watermarkCommitSeq, minKept);
+  const pruneUpTo = await computePruneUpToCommitSeq(
+    db,
+    partitionId,
+    args.watermarkCommitSeq,
+    keepNewestCommits
+  );
 
   if (pruneUpTo <= 0) return 0;
 
   // Delete dependent rows explicitly to be robust across dialects and older
   // schemas that may not have FK cascade enabled.
+  await (
+    syncDb.deleteFrom('sync_scope_commits') as DeleteQueryBuilder<
+      SyncCoreDb,
+      'sync_scope_commits',
+      DeleteResult
+    >
+  )
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
+    .where(sql<SqlBool>`commit_seq <= ${pruneUpTo}`)
+    .executeTakeFirst();
+
   await (
     syncDb.deleteFrom('sync_table_commits') as DeleteQueryBuilder<
       SyncCoreDb,
@@ -127,6 +172,7 @@ export async function pruneSync<DB extends SyncCoreDb>(
       DeleteResult
     >
   )
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
     .where(sql<SqlBool>`commit_seq <= ${pruneUpTo}`)
     .executeTakeFirst();
 
@@ -137,6 +183,7 @@ export async function pruneSync<DB extends SyncCoreDb>(
       DeleteResult
     >
   )
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
     .where(sql<SqlBool>`commit_seq <= ${pruneUpTo}`)
     .executeTakeFirst();
 
@@ -147,12 +194,57 @@ export async function pruneSync<DB extends SyncCoreDb>(
       DeleteResult
     >
   )
+    .where(sql<SqlBool>`partition_id = ${partitionId}`)
     .where(sql<SqlBool>`commit_seq <= ${pruneUpTo}`)
     .executeTakeFirst();
 
   return Number(res?.numDeletedRows ?? 0);
 }
 
+export async function previewPruneSync<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  options: PruneOptions = {}
+): Promise<PartitionPrunePreview[]> {
+  type SyncDb = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
+  const syncDb = db as SyncDb;
+  const partitionIds = await readPrunePartitionIds(db);
+  const previews: PartitionPrunePreview[] = [];
+
+  for (const partitionId of partitionIds) {
+    const watermarkCommitSeq = await computePruneWatermarkCommitSeq(
+      db,
+      options,
+      partitionId
+    );
+    const pruneUpToCommitSeq = await computePruneUpToCommitSeq(
+      db,
+      partitionId,
+      watermarkCommitSeq,
+      options.keepNewestCommits ?? 1000
+    );
+
+    const commitsQ = syncDb.selectFrom('sync_commits') as SelectQueryBuilder<
+      SyncCoreDb,
+      'sync_commits',
+      EmptySelection
+    >;
+
+    const countRow = await commitsQ
+      .select(({ fn }) => [fn.countAll().as('count')])
+      .where('partition_id', '=', partitionId)
+      .where('commit_seq', '<=', pruneUpToCommitSeq)
+      .executeTakeFirst();
+
+    previews.push({
+      partitionId,
+      watermarkCommitSeq,
+      commitsToDelete: coerceNumber(countRow?.count) ?? 0,
+    });
+  }
+
+  return previews;
+}
+
 interface PruneState {
   lastPruneAtMs: number;
   pruneInFlight: Promise<number> | null;
@@ -184,11 +276,20 @@ export async function maybePruneSync<DB extends SyncCoreDb>(
 
   state.pruneInFlight = (async () => {
     try {
-      const watermark = await computePruneWatermarkCommitSeq(db, args.options);
-      const deleted = await pruneSync(db, {
-        watermarkCommitSeq: watermark,
-        keepNewestCommits: args.options?.keepNewestCommits,
-      });
+      const partitionIds = await readPrunePartitionIds(db);
+      let deleted = 0;
+      for (const partitionId of partitionIds) {
+        const watermark = await computePruneWatermarkCommitSeq(
+          db,
+          args.options,
+          partitionId
+        );
+        deleted += await pruneSync(db, {
+          partitionId,
+          watermarkCommitSeq: watermark,
+          keepNewestCommits: args.options?.keepNewestCommits,
+        });
+      }
       state.lastPruneAtMs = Date.now();
       return deleted;
     } finally {