@syncular/server 0.0.6-96 → 0.1.0

package/src/sqlite3.ts

@@ -0,0 +1,137 @@
+/**
+ * @syncular/server/sqlite3 - node-sqlite3 dialect for sync
+ *
+ * Provides a Kysely dialect for the callback-based `sqlite3` npm package.
+ * SQLite-compatible — use with @syncular/server/sqlite.
+ */
+
+import type {
+  CompiledQuery,
+  DatabaseConnection,
+  Dialect,
+  QueryResult,
+} from 'kysely';
+import { BaseSqliteDialect, BaseSqliteDriver } from 'kysely-generic-sqlite';
+import sqlite3 from 'sqlite3';
+
+export interface Sqlite3PathOptions {
+  /** Path to SQLite database file, or ':memory:' for in-memory */
+  path: string;
+}
+
+export interface Sqlite3InstanceOptions {
+  /** An existing sqlite3.Database instance */
+  database: sqlite3.Database;
+}
+
+export type Sqlite3Options = Sqlite3PathOptions | Sqlite3InstanceOptions;
+
+/**
+ * Create the sqlite3 dialect directly.
+ */
+export function createSqlite3Dialect(options: Sqlite3Options): Dialect {
+  return new BaseSqliteDialect(() => new Sqlite3Driver(options));
+}
+
+class Sqlite3Driver extends BaseSqliteDriver {
+  #db: sqlite3.Database | undefined;
+
+  constructor(options: Sqlite3Options) {
+    super(async () => {
+      this.#db = await resolveSqlite3Database(options);
+      this.conn = new Sqlite3Connection(this.#db);
+    });
+  }
+
+  async destroy(): Promise<void> {
+    const db = this.#db;
+    this.#db = undefined;
+    if (!db) return;
+
+    await new Promise<void>((resolve, reject) => {
+      db.close((err) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+}
+
+async function resolveSqlite3Database(
+  options: Sqlite3Options
+): Promise<sqlite3.Database> {
+  if ('database' in options) {
+    return options.database;
+  }
+
+  return new Promise<sqlite3.Database>((resolve, reject) => {
+    const db = new sqlite3.Database(options.path, (err) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve(db);
+    });
+  });
+}
+
+class Sqlite3Connection implements DatabaseConnection {
+  readonly #db: sqlite3.Database;
+
+  constructor(db: sqlite3.Database) {
+    this.#db = db;
+  }
+
+  async executeQuery<R>(compiledQuery: CompiledQuery): Promise<QueryResult<R>> {
+    const { sql, parameters } = compiledQuery;
+    const params = [...parameters];
+
+    const hasReturning = /\breturning\b/i.test(sql);
+    const isSelectLike = /^\s*(select|pragma|explain|with)\b/i.test(sql);
+    const isInsert = /^\s*(insert|replace)\b/i.test(sql);
+
+    if (isSelectLike || hasReturning) {
+      return new Promise<QueryResult<R>>((resolve, reject) => {
+        this.#db.all(sql, params, (err: Error | null, rows: R[]) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+
+          const normalizedRows = rows ?? [];
+          resolve({
+            rows: normalizedRows,
+            ...(hasReturning
+              ? { numAffectedRows: BigInt(normalizedRows.length) }
+              : {}),
+          });
+        });
+      });
+    }
+
+    return new Promise<QueryResult<R>>((resolve, reject) => {
+      this.#db.run(sql, params, function onRun(err: Error | null) {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        resolve({
+          rows: [],
+          numAffectedRows: BigInt(this.changes),
+          ...(isInsert ? { insertId: BigInt(this.lastID) } : {}),
+        });
+      });
+    });
+  }
+
+  streamQuery<R>(
+    _compiledQuery: CompiledQuery,
+    _chunkSize?: number
+  ): AsyncIterableIterator<QueryResult<R>> {
+    throw new Error('sqlite3 driver does not support streaming');
+  }
+}

package/src/stats.ts

@@ -4,23 +4,12 @@
 
 import type { Kysely, SelectQueryBuilder, SqlBool } from 'kysely';
 import { sql } from 'kysely';
+import { coerceNumber } from './dialect/helpers';
 import type { SyncCoreDb } from './schema';
 
 // biome-ignore lint/complexity/noBannedTypes: Kysely uses `{}` as the initial "no selected columns yet" marker.
 type EmptySelection = {};
 
-function coerceNumber(value: unknown): number | null {
-  if (value === null || value === undefined) return null;
-  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
-  if (typeof value === 'bigint')
-    return Number.isFinite(Number(value)) ? Number(value) : null;
-  if (typeof value === 'string') {
-    const n = Number(value);
-    return Number.isFinite(n) ? n : null;
-  }
-  return null;
-}
-
 export interface SyncStats {
   commitCount: number;
   changeCount: number;
@@ -30,6 +19,35 @@ export interface SyncStats {
   activeClientCount: number;
   minActiveClientCursor: number | null;
   maxActiveClientCursor: number | null;
+  snapshotChunkCount: number;
+  snapshotChunkBytes: number;
+  expiredSnapshotChunkCount: number;
+  expiredSnapshotChunkBytes: number;
+  snapshotArtifactCount: number;
+  snapshotArtifactBytes: number;
+  expiredSnapshotArtifactCount: number;
+  expiredSnapshotArtifactBytes: number;
+}
+
+interface CachePressureRow {
+  total_count: unknown;
+  total_bytes: unknown;
+  expired_count: unknown;
+  expired_bytes: unknown;
+}
+
+function mapCachePressure(row: CachePressureRow | undefined): {
+  count: number;
+  bytes: number;
+  expiredCount: number;
+  expiredBytes: number;
+} {
+  return {
+    count: coerceNumber(row?.total_count) ?? 0,
+    bytes: coerceNumber(row?.total_bytes) ?? 0,
+    expiredCount: coerceNumber(row?.expired_count) ?? 0,
+    expiredBytes: coerceNumber(row?.expired_bytes) ?? 0,
+  };
 }
 
 export async function readSyncStats<DB extends SyncCoreDb>(
@@ -41,6 +59,7 @@ export async function readSyncStats<DB extends SyncCoreDb>(
 
   const activeWindowMs = options.activeWindowMs ?? 14 * 24 * 60 * 60 * 1000;
   const cutoffIso = new Date(Date.now() - activeWindowMs).toISOString();
+  const nowIso = new Date().toISOString();
   const partitionId = options.partitionId;
 
   let commitQuery = (
@@ -97,13 +116,49 @@ export async function readSyncStats<DB extends SyncCoreDb>(
     );
   }
 
-  const [commitRow, changeRow, clientRow, activeClientRow] = await Promise.all([
+  const partitionFilter = partitionId
+    ? sql`where partition_id = ${partitionId}`
+    : sql``;
+
+  const [
+    commitRow,
+    changeRow,
+    clientRow,
+    activeClientRow,
+    chunkPressureRow,
+    artifactPressureRow,
+  ] = await Promise.all([
     commitQuery.executeTakeFirst(),
     changeQuery.executeTakeFirst(),
     clientQuery.executeTakeFirst(),
     activeClientQuery.executeTakeFirst(),
+    sql<CachePressureRow>`
+      select
+        count(*) as total_count,
+        coalesce(sum(byte_length), 0) as total_bytes,
+        coalesce(sum(case when expires_at <= ${nowIso} then 1 else 0 end), 0) as expired_count,
+        coalesce(sum(case when expires_at <= ${nowIso} then byte_length else 0 end), 0) as expired_bytes
+      from ${sql.table('sync_snapshot_chunks')}
+      ${partitionFilter}
+    `
+      .execute(db)
+      .then((result) => result.rows[0]),
+    sql<CachePressureRow>`
+      select
+        count(*) as total_count,
+        coalesce(sum(byte_length), 0) as total_bytes,
+        coalesce(sum(case when expires_at <= ${nowIso} then 1 else 0 end), 0) as expired_count,
+        coalesce(sum(case when expires_at <= ${nowIso} then byte_length else 0 end), 0) as expired_bytes
+      from ${sql.table('sync_snapshot_artifacts')}
+      ${partitionFilter}
+    `
+      .execute(db)
+      .then((result) => result.rows[0]),
   ]);
 
+  const chunkPressure = mapCachePressure(chunkPressureRow);
+  const artifactPressure = mapCachePressure(artifactPressureRow);
+
   return {
     commitCount: coerceNumber(commitRow?.commitCount) ?? 0,
     changeCount: coerceNumber(changeRow?.changeCount) ?? 0,
@@ -113,5 +168,13 @@ export async function readSyncStats<DB extends SyncCoreDb>(
     activeClientCount: coerceNumber(activeClientRow?.activeClientCount) ?? 0,
     minActiveClientCursor: coerceNumber(activeClientRow?.minActiveClientCursor),
     maxActiveClientCursor: coerceNumber(activeClientRow?.maxActiveClientCursor),
+    snapshotChunkCount: chunkPressure.count,
+    snapshotChunkBytes: chunkPressure.bytes,
+    expiredSnapshotChunkCount: chunkPressure.expiredCount,
+    expiredSnapshotChunkBytes: chunkPressure.expiredBytes,
+    snapshotArtifactCount: artifactPressure.count,
+    snapshotArtifactBytes: artifactPressure.bytes,
+    expiredSnapshotArtifactCount: artifactPressure.expiredCount,
+    expiredSnapshotArtifactBytes: artifactPressure.expiredBytes,
   };
 }

package/src/subscriptions/cache.ts

@@ -37,6 +37,11 @@ export interface ScopeCacheBackend {
   get<DB extends SyncCoreDb, Auth extends SyncServerAuth>(
     args: ScopeCacheContext<DB, Auth>
   ): Promise<ScopeValues | null>;
+  getOrResolve?<DB extends SyncCoreDb, Auth extends SyncServerAuth>(
+    args: ScopeCacheContext<DB, Auth> & {
+      load: () => Promise<ScopeValues | null>;
+    }
+  ): Promise<ScopeValues | null>;
   set<DB extends SyncCoreDb, Auth extends SyncServerAuth>(
     args: ScopeCacheSetContext<DB, Auth>
   ): Promise<void>;
@@ -117,6 +122,10 @@ export function createMemoryScopeCache(
     options?.maxEntries ?? DEFAULT_MEMORY_SCOPE_CACHE_MAX_ENTRIES;
   const now = options?.now ?? Date.now;
   const buckets = new WeakMap<object, Map<string, MemoryScopeCacheEntry>>();
+  const inflightByDb = new WeakMap<
+    object,
+    Map<string, Promise<ScopeValues | null>>
+  >();
 
   function getBucket(db: object): Map<string, MemoryScopeCacheEntry> {
     const existing = buckets.get(db);
@@ -128,6 +137,18 @@ export function createMemoryScopeCache(
     return created;
   }
 
+  function getInflightBucket(
+    db: object
+  ): Map<string, Promise<ScopeValues | null>> {
+    const existing = inflightByDb.get(db);
+    if (existing) {
+      return existing;
+    }
+    const created = new Map<string, Promise<ScopeValues | null>>();
+    inflightByDb.set(db, created);
+    return created;
+  }
+
   function evictOldest(bucket: Map<string, MemoryScopeCacheEntry>): void {
     while (bucket.size > maxEntries) {
       const oldestKey = bucket.keys().next().value;
@@ -155,6 +176,43 @@ export function createMemoryScopeCache(
 
       return cloneScopeValues(entry.scopes);
     },
+    async getOrResolve(args) {
+      const cached = await this.get(args);
+      if (cached !== null) {
+        return cached;
+      }
+
+      const inflightBucket = getInflightBucket(args.db);
+      const inflight = inflightBucket.get(args.cacheKey);
+      if (inflight) {
+        const resolved = await inflight;
+        return resolved ? cloneScopeValues(resolved) : null;
+      }
+
+      const pending = args
+        .load()
+        .then(async (resolved) => {
+          if (resolved !== null && ttlMs > 0) {
+            const bucket = getBucket(args.db);
+            if (bucket.has(args.cacheKey)) {
+              bucket.delete(args.cacheKey);
+            }
+            bucket.set(args.cacheKey, {
+              scopes: cloneScopeValues(resolved),
+              expiresAt: now() + ttlMs,
+            });
+            evictOldest(bucket);
+          }
+          return resolved;
+        })
+        .finally(() => {
+          inflightBucket.delete(args.cacheKey);
+        });
+
+      inflightBucket.set(args.cacheKey, pending);
+      const resolved = await pending;
+      return resolved ? cloneScopeValues(resolved) : null;
+    },
     async set(args) {
       const bucket = getBucket(args.db);
       if (ttlMs <= 0) {

package/src/subscriptions/resolve.ts

@@ -1,13 +1,10 @@
 import {
-  extractScopeVars,
+  collectScopeVars,
   type ScopeValues,
   type SyncSubscriptionRequest,
 } from '@syncular/core';
 import type { Kysely } from 'kysely';
-import {
-  getServerHandler,
-  type ServerHandlerCollection,
-} from '../handlers/collection';
+import type { ServerHandlerCollection } from '../handlers/collection';
 import type { SyncServerAuth } from '../handlers/types';
 import type { SyncCoreDb } from '../schema';
 import { createDefaultScopeCacheKey, type ScopeCacheBackend } from './cache';
@@ -29,6 +26,8 @@ export interface ResolvedSubscription {
   params: Record<string, unknown> | undefined;
   cursor: number;
   bootstrapState?: SyncSubscriptionRequest['bootstrapState'];
+  verifiedRoot?: string;
+  crdtStateVectors: SyncSubscriptionRequest['crdtStateVectors'];
   status: 'active' | 'revoked';
 }
 
@@ -92,19 +91,6 @@ function scopesEmpty(scopes: ScopeValues): boolean {
   return true;
 }
 
-/**
- * Collect valid scope keys from handler scope patterns.
- */
-function collectScopeKeys(scopePatterns: readonly string[]): Set<string> {
-  const keys = new Set<string>();
-  for (const pattern of scopePatterns) {
-    for (const key of extractScopeVars(pattern)) {
-      keys.add(key);
-    }
-  }
-  return keys;
-}
-
 function validateScopeKeys(args: {
   scopeValues: ScopeValues;
   validScopeKeys: Set<string>;
@@ -166,14 +152,14 @@ export async function resolveEffectiveScopesForSubscriptions<
       );
     }
 
-    const handler = getServerHandler(args.handlers, sub.table);
+    const handler = args.handlers.byTable.get(sub.table);
     if (!handler) {
       throw new InvalidSubscriptionScopeError(
         `Unknown table: ${sub.table} for subscription ${sub.id}`
       );
     }
 
-    const validScopeKeys = collectScopeKeys(handler.scopePatterns);
+    const validScopeKeys = collectScopeVars(handler.scopePatterns);
     const requested = sub.scopes ?? {};
     validateScopeKeys({
       scopeValues: requested,
@@ -193,72 +179,101 @@ export async function resolveEffectiveScopesForSubscriptions<
     if (requestScopeCache.has(scopeCacheKey)) {
       allowed = requestScopeCache.get(scopeCacheKey) ?? null;
     } else {
-      allowed = null;
-      let sharedCacheHit = false;
+      let resolveFailed = false;
 
-      if (args.scopeCache) {
+      const loadAllowedScopes = async (): Promise<ScopeValues> => {
         try {
-          const cachedAllowed = await args.scopeCache.get({
+          return await handler.resolveScopes({
             db: args.db,
+            actorId: args.auth.actorId,
             auth: args.auth,
-            table: sub.table,
-            cacheKey: scopeCacheKey,
           });
-          if (cachedAllowed !== null) {
-            allowed = cachedAllowed;
-            sharedCacheHit = true;
-          }
-        } catch (cacheErr) {
+        } catch (resolveErr) {
+          resolveFailed = true;
           console.error(
-            `[scopeCache.get] Failed for table ${sub.table}, subscription ${sub.id}:`,
-            cacheErr
+            `[resolveScopes] Failed for table ${sub.table}, subscription ${sub.id}:`,
+            resolveErr
           );
+          throw resolveErr;
         }
-      }
+      };
 
-      if (!sharedCacheHit) {
+      if (args.scopeCache?.getOrResolve) {
         try {
-          allowed = await handler.resolveScopes({
+          allowed = await args.scopeCache.getOrResolve({
             db: args.db,
-            actorId: args.auth.actorId,
             auth: args.auth,
-          });
-        } catch (resolveErr) {
-          // Scope resolution failed - mark subscription as revoked
-          // rather than failing the entire pull
-          console.error(
-            `[resolveScopes] Failed for table ${sub.table}, subscription ${sub.id}:`,
-            resolveErr
-          );
-          requestScopeCache.set(scopeCacheKey, null);
-          out.push({
-            id: sub.id,
             table: sub.table,
-            scopes: {},
-            params: sub.params,
-            cursor: sub.cursor,
-            bootstrapState: sub.bootstrapState ?? null,
-            status: 'revoked',
+            cacheKey: scopeCacheKey,
+            load: loadAllowedScopes,
           });
-          continue;
+        } catch {
+          allowed = null;
         }
+      } else {
+        allowed = null;
+        let sharedCacheHit = false;
 
-        if (args.scopeCache && allowed !== null) {
+        if (args.scopeCache) {
           try {
-            await args.scopeCache.set({
+            const cachedAllowed = await args.scopeCache.get({
               db: args.db,
               auth: args.auth,
               table: sub.table,
               cacheKey: scopeCacheKey,
-              scopes: allowed,
             });
+            if (cachedAllowed !== null) {
+              allowed = cachedAllowed;
+              sharedCacheHit = true;
+            }
           } catch (cacheErr) {
             console.error(
-              `[scopeCache.set] Failed for table ${sub.table}, subscription ${sub.id}:`,
+              `[scopeCache.get] Failed for table ${sub.table}, subscription ${sub.id}:`,
               cacheErr
             );
           }
         }
+
+        if (!sharedCacheHit) {
+          try {
+            allowed = await loadAllowedScopes();
+          } catch {
+            allowed = null;
+          }
+
+          if (args.scopeCache && allowed !== null) {
+            try {
+              await args.scopeCache.set({
+                db: args.db,
+                auth: args.auth,
+                table: sub.table,
+                cacheKey: scopeCacheKey,
+                scopes: allowed,
+              });
+            } catch (cacheErr) {
+              console.error(
+                `[scopeCache.set] Failed for table ${sub.table}, subscription ${sub.id}:`,
+                cacheErr
+              );
+            }
+          }
+        }
+      }
+
+      if (resolveFailed) {
+        requestScopeCache.set(scopeCacheKey, null);
+        out.push({
+          id: sub.id,
+          table: sub.table,
+          scopes: {},
+          params: sub.params,
+          cursor: sub.cursor,
+          bootstrapState: sub.bootstrapState ?? null,
+          verifiedRoot: sub.verifiedRoot,
+          crdtStateVectors: sub.crdtStateVectors,
+          status: 'revoked',
+        });
+        continue;
       }
 
       requestScopeCache.set(scopeCacheKey, allowed);
@@ -272,6 +287,8 @@ export async function resolveEffectiveScopesForSubscriptions<
         params: sub.params,
         cursor: sub.cursor,
         bootstrapState: sub.bootstrapState ?? null,
+        verifiedRoot: sub.verifiedRoot,
+        crdtStateVectors: sub.crdtStateVectors,
         status: 'revoked',
       });
       continue;
@@ -288,6 +305,26 @@ export async function resolveEffectiveScopesForSubscriptions<
     // Intersect with requested scopes
     const effective = intersectScopes(requested, allowed);
 
+    // If a requested scope key lost all access after intersection,
+    // treat the subscription as revoked to avoid partially-scoped leaks.
+    const missingRequestedKey = Object.keys(requested).some(
+      (key) => !Object.hasOwn(effective, key)
+    );
+    if (missingRequestedKey) {
+      out.push({
+        id: sub.id,
+        table: sub.table,
+        scopes: {},
+        params: sub.params,
+        cursor: sub.cursor,
+        bootstrapState: sub.bootstrapState ?? null,
+        verifiedRoot: sub.verifiedRoot,
+        crdtStateVectors: sub.crdtStateVectors,
+        status: 'revoked',
+      });
+      continue;
+    }
+
     if (scopesEmpty(effective)) {
       out.push({
         id: sub.id,
@@ -296,6 +333,8 @@ export async function resolveEffectiveScopesForSubscriptions<
         params: sub.params,
         cursor: sub.cursor,
         bootstrapState: sub.bootstrapState ?? null,
+        verifiedRoot: sub.verifiedRoot,
+        crdtStateVectors: sub.crdtStateVectors,
         status: 'revoked',
       });
       continue;
@@ -308,6 +347,8 @@ export async function resolveEffectiveScopesForSubscriptions<
       params: sub.params,
       cursor: sub.cursor,
       bootstrapState: sub.bootstrapState ?? null,
+      verifiedRoot: sub.verifiedRoot,
+      crdtStateVectors: sub.crdtStateVectors,
       status: 'active',
     });
   }

package/src/sync.ts

@@ -3,11 +3,13 @@ import type {
   ColumnCodecSource,
   ScopeDefinition,
 } from '@syncular/core';
+import { registerTableOrThrow } from '@syncular/core';
 import {
   type CreateServerHandlerOptions,
   createServerHandler,
 } from './handlers';
 import type { ServerTableHandler, SyncServerAuth } from './handlers/types';
+import type { SyncServerPushPlugin } from './plugins/types';
 import type { SyncCoreDb } from './schema';
 
 type SharedTableName<ServerDB, ClientDB> = keyof ServerDB &
@@ -31,6 +33,7 @@ export interface ServerSyncConfig<
 > {
   authenticate: (request: Request) => Promise<Auth | null> | Auth | null;
   handlers: ServerTableHandler<ServerDB, Auth>[];
+  plugins?: SyncServerPushPlugin<ServerDB, Auth>[];
 }
 
 export interface DefineServerSyncOptions<Auth extends SyncServerAuth> {
@@ -54,6 +57,8 @@ export interface ServerSyncBuilder<
       ScopeDefs
     >
   ): this;
+
+  addPushPlugin(plugin: SyncServerPushPlugin<ServerDB, Auth>): this;
 }
 
 export function defineServerSync<
@@ -65,11 +70,13 @@ export function defineServerSync<
   options: DefineServerSyncOptions<Auth>
 ): ServerSyncBuilder<ServerDB, ClientDB, ScopeDefs, Auth> {
   const handlers: ServerTableHandler<ServerDB, Auth>[] = [];
+  const plugins: SyncServerPushPlugin<ServerDB, Auth>[] = [];
   const registeredTables = new Set<string>();
 
   const sync: ServerSyncBuilder<ServerDB, ClientDB, ScopeDefs, Auth> = {
     authenticate: options.authenticate,
     handlers,
+    plugins,
     addHandler<TableName extends SharedTableName<ServerDB, ClientDB>>(
       handlerOptions: ServerSyncHandlerOptionsForTable<
         ServerDB,
@@ -79,11 +86,11 @@ export function defineServerSync<
         ScopeDefs
       >
     ) {
-      if (registeredTables.has(handlerOptions.table)) {
-        throw new Error(
-          `Server table handler already registered: ${handlerOptions.table}`
-        );
-      }
+      registerTableOrThrow(
+        registeredTables,
+        handlerOptions.table,
+        (table) => `Server table handler already registered: ${table}`
+      );
 
       handlers.push(
         createServerHandler({
@@ -92,7 +99,10 @@ export function defineServerSync<
           codecDialect: options.codecDialect,
         })
       );
-      registeredTables.add(handlerOptions.table);
+      return sync;
+    },
+    addPushPlugin(plugin: SyncServerPushPlugin<ServerDB, Auth>) {
+      plugins.push(plugin);
       return sync;
     },
   };