@syncular/server 0.0.1-60

package/src/snapshot-chunks.ts

@@ -0,0 +1,301 @@
+/**
+ * @syncular/server - Encoded snapshot chunk cache (server-side)
+ *
+ * Used for efficiently serving large bootstrap snapshots (e.g. catalogs)
+ * without embedding huge JSON payloads into pull responses.
+ */
+
+import type { SyncSnapshotChunkRef } from '@syncular/core';
+import { type Kysely, sql } from 'kysely';
+import type { SyncCoreDb } from './schema';
+
+export interface SnapshotChunkPageKey {
+  partitionId: string;
+  scopeKey: string;
+  scope: string;
+  asOfCommitSeq: number;
+  rowCursor: string | null;
+  rowLimit: number;
+  encoding: 'ndjson';
+  compression: 'gzip';
+}
+
+export interface SnapshotChunkRow {
+  chunkId: string;
+  partitionId: string;
+  scopeKey: string;
+  scope: string;
+  asOfCommitSeq: number;
+  rowCursor: string;
+  rowLimit: number;
+  encoding: 'ndjson';
+  compression: 'gzip';
+  sha256: string;
+  byteLength: number;
+  body: Uint8Array;
+  expiresAt: string;
+}
+
+function coerceChunkRow(value: unknown): Uint8Array {
+  // pg returns Buffer (subclass of Uint8Array); sqlite returns Uint8Array
+  if (value instanceof Uint8Array) return value;
+  if (typeof Buffer !== 'undefined' && value instanceof Buffer) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (Array.isArray(value) && value.every((v) => typeof v === 'number')) {
+    return new Uint8Array(value);
+  }
+  throw new Error(
+    `Unexpected snapshot chunk body type: ${Object.prototype.toString.call(value)}`
+  );
+}
+
+function coerceIsoString(value: unknown): string {
+  if (typeof value === 'string') return value;
+  if (value instanceof Date) return value.toISOString();
+  return String(value);
+}
+
+export async function readSnapshotChunkRefByPageKey<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  args: SnapshotChunkPageKey & { nowIso?: string }
+): Promise<SyncSnapshotChunkRef | null> {
+  const nowIso = args.nowIso ?? new Date().toISOString();
+  const rowCursorKey = args.rowCursor ?? '';
+
+  const rowResult = await sql<{
+    chunk_id: string;
+    sha256: string;
+    byte_length: number;
+    encoding: string;
+    compression: string;
+  }>`
+    select chunk_id, sha256, byte_length, encoding, compression
+    from ${sql.table('sync_snapshot_chunks')}
+    where
+      partition_id = ${args.partitionId}
+      and scope_key = ${args.scopeKey}
+      and scope = ${args.scope}
+      and as_of_commit_seq = ${args.asOfCommitSeq}
+      and row_cursor = ${rowCursorKey}
+      and row_limit = ${args.rowLimit}
+      and encoding = ${args.encoding}
+      and compression = ${args.compression}
+      and expires_at > ${nowIso}
+    limit 1
+  `.execute(db);
+  const row = rowResult.rows[0];
+
+  if (!row) return null;
+
+  if (row.encoding !== 'ndjson') {
+    throw new Error(
+      `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
+    );
+  }
+  if (row.compression !== 'gzip') {
+    throw new Error(
+      `Unexpected snapshot chunk compression: ${String(row.compression)}`
+    );
+  }
+
+  return {
+    id: row.chunk_id,
+    sha256: row.sha256,
+    byteLength: Number(row.byte_length ?? 0),
+    encoding: row.encoding,
+    compression: row.compression,
+  };
+}
+
+export async function insertSnapshotChunk<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  args: {
+    chunkId: string;
+    partitionId: string;
+    scopeKey: string;
+    scope: string;
+    asOfCommitSeq: number;
+    rowCursor: string | null;
+    rowLimit: number;
+    encoding: 'ndjson';
+    compression: 'gzip';
+    sha256: string;
+    body: Uint8Array;
+    expiresAt: string;
+  }
+): Promise<SyncSnapshotChunkRef> {
+  const now = new Date().toISOString();
+  const rowCursorKey = args.rowCursor ?? '';
+
+  // Use content hash as blob_hash for legacy storage in DB
+  const blobHash = `sha256:${args.sha256}`;
+
+  await sql`
+    insert into ${sql.table('sync_snapshot_chunks')} (
+      chunk_id,
+      partition_id,
+      scope_key,
+      scope,
+      as_of_commit_seq,
+      row_cursor,
+      row_limit,
+      encoding,
+      compression,
+      sha256,
+      byte_length,
+      blob_hash,
+      body,
+      created_at,
+      expires_at
+    )
+    values (
+      ${args.chunkId},
+      ${args.partitionId},
+      ${args.scopeKey},
+      ${args.scope},
+      ${args.asOfCommitSeq},
+      ${rowCursorKey},
+      ${args.rowLimit},
+      ${args.encoding},
+      ${args.compression},
+      ${args.sha256},
+      ${args.body.length},
+      ${blobHash},
+      ${args.body},
+      ${now},
+      ${args.expiresAt}
+    )
+    on conflict (
+      partition_id,
+      scope_key,
+      scope,
+      as_of_commit_seq,
+      row_cursor,
+      row_limit,
+      encoding,
+      compression
+    )
+    do update set
+      expires_at = ${args.expiresAt},
+      blob_hash = ${blobHash}
+  `.execute(db);
+
+  const ref = await readSnapshotChunkRefByPageKey(db, {
+    partitionId: args.partitionId,
+    scopeKey: args.scopeKey,
+    scope: args.scope,
+    asOfCommitSeq: args.asOfCommitSeq,
+    rowCursor: args.rowCursor,
+    rowLimit: args.rowLimit,
+    encoding: args.encoding,
+    compression: args.compression,
+  });
+
+  if (!ref) {
+    throw new Error('Failed to read inserted snapshot chunk');
+  }
+
+  return ref;
+}
+
+export async function readSnapshotChunk<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  chunkId: string,
+  options?: {
+    /** External chunk storage for reading from S3/R2/etc */
+    chunkStorage?: { readChunk(chunkId: string): Promise<Uint8Array | null> };
+  }
+): Promise<SnapshotChunkRow | null> {
+  const rowResult = await sql<{
+    chunk_id: string;
+    partition_id: string;
+    scope_key: string;
+    scope: string;
+    as_of_commit_seq: number;
+    row_cursor: string;
+    row_limit: number;
+    encoding: string;
+    compression: string;
+    sha256: string;
+    byte_length: number;
+    blob_hash: string;
+    body: unknown;
+    expires_at: unknown;
+  }>`
+    select
+      chunk_id,
+      partition_id,
+      scope_key,
+      scope,
+      as_of_commit_seq,
+      row_cursor,
+      row_limit,
+      encoding,
+      compression,
+      sha256,
+      byte_length,
+      blob_hash,
+      body,
+      expires_at
+    from ${sql.table('sync_snapshot_chunks')}
+    where chunk_id = ${chunkId}
+    limit 1
+  `.execute(db);
+  const row = rowResult.rows[0];
+
+  if (!row) return null;
+
+  if (row.encoding !== 'ndjson') {
+    throw new Error(
+      `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
+    );
+  }
+  if (row.compression !== 'gzip') {
+    throw new Error(
+      `Unexpected snapshot chunk compression: ${String(row.compression)}`
+    );
+  }
+
+  // Read body from external storage if available, otherwise use inline body
+  let body: Uint8Array;
+  if (options?.chunkStorage) {
+    const externalBody = await options.chunkStorage.readChunk(chunkId);
+    if (externalBody) {
+      body = externalBody;
+    } else if (row.body) {
+      body = coerceChunkRow(row.body);
+    } else {
+      throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+    }
+  } else {
+    body = coerceChunkRow(row.body);
+  }
+
+  return {
+    chunkId: row.chunk_id,
+    partitionId: row.partition_id,
+    scopeKey: row.scope_key,
+    scope: row.scope,
+    asOfCommitSeq: Number(row.as_of_commit_seq ?? 0),
+    rowCursor: row.row_cursor,
+    rowLimit: Number(row.row_limit ?? 0),
+    encoding: row.encoding,
+    compression: row.compression,
+    sha256: row.sha256,
+    byteLength: Number(row.byte_length ?? 0),
+    body,
+    expiresAt: coerceIsoString(row.expires_at),
+  };
+}
+
+export async function deleteExpiredSnapshotChunks<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  nowIso = new Date().toISOString()
+): Promise<number> {
+  const res = await sql`
+    delete from ${sql.table('sync_snapshot_chunks')}
+    where expires_at <= ${nowIso}
+  `.execute(db);
+
+  return Number(res.numAffectedRows ?? 0);
+}

package/src/stats.ts

@@ -0,0 +1,104 @@
+/**
+ * @syncular/server - Observability helpers
+ */
+
+import type { Kysely, SelectQueryBuilder, SqlBool } from 'kysely';
+import { sql } from 'kysely';
+import type { SyncCoreDb } from './schema';
+
+// biome-ignore lint/complexity/noBannedTypes: Kysely uses `{}` as the initial "no selected columns yet" marker.
+type EmptySelection = {};
+
+function coerceNumber(value: unknown): number | null {
+  if (value === null || value === undefined) return null;
+  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
+  if (typeof value === 'bigint')
+    return Number.isFinite(Number(value)) ? Number(value) : null;
+  if (typeof value === 'string') {
+    const n = Number(value);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+export interface SyncStats {
+  commitCount: number;
+  changeCount: number;
+  minCommitSeq: number;
+  maxCommitSeq: number;
+  clientCount: number;
+  activeClientCount: number;
+  minActiveClientCursor: number | null;
+  maxActiveClientCursor: number | null;
+}
+
+export async function readSyncStats<DB extends SyncCoreDb>(
+  db: Kysely<DB>,
+  options: { activeWindowMs?: number } = {}
+): Promise<SyncStats> {
+  type SyncDb = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
+  const syncDb = db as SyncDb;
+
+  const activeWindowMs = options.activeWindowMs ?? 14 * 24 * 60 * 60 * 1000;
+  const cutoffIso = new Date(Date.now() - activeWindowMs).toISOString();
+
+  const [commitRow, changeRow, clientRow, activeClientRow] = await Promise.all([
+    (
+      syncDb.selectFrom('sync_commits') as SelectQueryBuilder<
+        SyncCoreDb,
+        'sync_commits',
+        EmptySelection
+      >
+    )
+      .select(({ fn }) => [
+        fn.countAll().as('commitCount'),
+        fn.min('commit_seq').as('minCommitSeq'),
+        fn.max('commit_seq').as('maxCommitSeq'),
+      ])
+      .executeTakeFirst(),
+    (
+      syncDb.selectFrom('sync_changes') as SelectQueryBuilder<
+        SyncCoreDb,
+        'sync_changes',
+        EmptySelection
+      >
+    )
+      .select(({ fn }) => [fn.countAll().as('changeCount')])
+      .executeTakeFirst(),
+    (
+      syncDb.selectFrom('sync_client_cursors') as SelectQueryBuilder<
+        SyncCoreDb,
+        'sync_client_cursors',
+        EmptySelection
+      >
+    )
+      .select(({ fn }) => [fn.countAll().as('clientCount')])
+      .executeTakeFirst(),
+    (
+      syncDb.selectFrom('sync_client_cursors') as SelectQueryBuilder<
+        SyncCoreDb,
+        'sync_client_cursors',
+        EmptySelection
+      >
+    )
+      .where(sql<SqlBool>`updated_at >= ${cutoffIso}`)
+      .where(sql<SqlBool>`cursor >= ${0}`)
+      .select(({ fn }) => [
+        fn.countAll().as('activeClientCount'),
+        fn.min('cursor').as('minActiveClientCursor'),
+        fn.max('cursor').as('maxActiveClientCursor'),
+      ])
+      .executeTakeFirst(),
+  ]);
+
+  return {
+    commitCount: coerceNumber(commitRow?.commitCount) ?? 0,
+    changeCount: coerceNumber(changeRow?.changeCount) ?? 0,
+    minCommitSeq: coerceNumber(commitRow?.minCommitSeq) ?? 0,
+    maxCommitSeq: coerceNumber(commitRow?.maxCommitSeq) ?? 0,
+    clientCount: coerceNumber(clientRow?.clientCount) ?? 0,
+    activeClientCount: coerceNumber(activeClientRow?.activeClientCount) ?? 0,
+    minActiveClientCursor: coerceNumber(activeClientRow?.minActiveClientCursor),
+    maxActiveClientCursor: coerceNumber(activeClientRow?.maxActiveClientCursor),
+  };
+}

package/src/subscriptions/index.ts

@@ -0,0 +1 @@
+export * from './resolve';

package/src/subscriptions/resolve.ts

@@ -0,0 +1,185 @@
+import type { ScopeValues, SyncSubscriptionRequest } from '@syncular/core';
+import type { Kysely } from 'kysely';
+import type { SyncCoreDb } from '../schema';
+import type { TableRegistry } from '../shapes/registry';
+
+export class InvalidSubscriptionScopeError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'InvalidSubscriptionScopeError';
+  }
+}
+
+/**
+ * Resolved subscription with effective scopes.
+ */
+export interface ResolvedSubscription {
+  id: string;
+  shape: string;
+  scopes: ScopeValues;
+  params: Record<string, unknown> | undefined;
+  cursor: number;
+  bootstrapState?: SyncSubscriptionRequest['bootstrapState'];
+  status: 'active' | 'revoked';
+}
+
+/**
+ * Intersect requested scopes with allowed scopes.
+ *
+ * For each key in requested:
+ * - If allowed has the same key, intersect the values
+ * - If allowed doesn't have the key, exclude it (no access)
+ *
+ * Returns only keys where there's intersection.
+ */
+function intersectScopes(
+  requested: ScopeValues,
+  allowed: ScopeValues
+): ScopeValues {
+  const result: ScopeValues = {};
+
+  for (const [key, reqValues] of Object.entries(requested)) {
+    const allowedValues = allowed[key];
+    if (allowedValues === undefined) {
+      // No access to this scope key
+      continue;
+    }
+
+    const reqArray = Array.isArray(reqValues) ? reqValues : [reqValues];
+    const allowedArray = Array.isArray(allowedValues)
+      ? allowedValues
+      : [allowedValues];
+
+    // Wildcard: allowed '*' means "allow any requested values for this key".
+    if (allowedArray.includes('*')) {
+      result[key] = reqValues;
+      continue;
+    }
+    const allowedSet = new Set(allowedArray);
+
+    // Intersect
+    const intersection = reqArray.filter((v) => allowedSet.has(v));
+
+    if (intersection.length > 0) {
+      // Keep as array if original was array, otherwise single value
+      result[key] =
+        intersection.length === 1 && !Array.isArray(reqValues)
+          ? intersection[0]!
+          : intersection;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Check if scopes are empty (no effective scope values).
+ */
+function scopesEmpty(scopes: ScopeValues): boolean {
+  for (const value of Object.values(scopes)) {
+    const arr = Array.isArray(value) ? value : [value];
+    if (arr.length > 0) return false;
+  }
+  return true;
+}
+
+/**
+ * Resolve effective scopes for subscriptions.
+ *
+ * For each subscription:
+ * 1. Look up the shape by subscription.shape
+ * 2. Call shape.resolveScopes() to get allowed scopes for this actor
+ * 3. Intersect requested scopes with allowed scopes
+ * 4. Mark as revoked if no effective scopes
+ */
+export async function resolveEffectiveScopesForSubscriptions<
+  DB extends SyncCoreDb,
+>(args: {
+  db: Kysely<DB>;
+  actorId: string;
+  subscriptions: SyncSubscriptionRequest[];
+  shapes: TableRegistry<DB>;
+}): Promise<ResolvedSubscription[]> {
+  const out: ResolvedSubscription[] = [];
+  const seenIds = new Set<string>();
+
+  for (const sub of args.subscriptions) {
+    if (!sub.id || typeof sub.id !== 'string') {
+      throw new InvalidSubscriptionScopeError('Subscription id is required');
+    }
+    if (seenIds.has(sub.id)) {
+      throw new InvalidSubscriptionScopeError(
+        `Duplicate subscription id: ${sub.id}`
+      );
+    }
+    seenIds.add(sub.id);
+
+    if (!sub.shape || typeof sub.shape !== 'string') {
+      throw new InvalidSubscriptionScopeError(
+        `Subscription ${sub.id} requires a shape (table name)`
+      );
+    }
+
+    const shape = args.shapes.get(sub.shape);
+    if (!shape) {
+      throw new InvalidSubscriptionScopeError(
+        `Unknown shape: ${sub.shape} for subscription ${sub.id}`
+      );
+    }
+
+    // Get allowed scopes from the shape
+    let allowed: ScopeValues;
+    try {
+      allowed = await shape.resolveScopes({
+        db: args.db,
+        actorId: args.actorId,
+      });
+    } catch (resolveErr) {
+      // Scope resolution failed - mark subscription as revoked
+      // rather than failing the entire pull
+      console.error(
+        `[resolveScopes] Failed for shape ${sub.shape}, subscription ${sub.id}:`,
+        resolveErr
+      );
+      out.push({
+        id: sub.id,
+        shape: sub.shape,
+        scopes: {},
+        params: sub.params,
+        cursor: sub.cursor,
+        bootstrapState: sub.bootstrapState ?? null,
+        status: 'revoked',
+      });
+      continue;
+    }
+
+    // Intersect with requested scopes
+    const requested = sub.scopes ?? {};
+    const effective = intersectScopes(requested, allowed);
+
+    if (scopesEmpty(effective)) {
+      out.push({
+        id: sub.id,
+        shape: sub.shape,
+        scopes: {},
+        params: sub.params,
+        cursor: sub.cursor,
+        bootstrapState: sub.bootstrapState ?? null,
+        status: 'revoked',
+      });
+      continue;
+    }
+
+    out.push({
+      id: sub.id,
+      shape: sub.shape,
+      scopes: effective,
+      params: sub.params,
+      cursor: sub.cursor,
+      bootstrapState: sub.bootstrapState ?? null,
+      status: 'active',
+    });
+  }
+
+  return out;
+}