@syncular/server 0.1.3 → 0.2.1

package/src/snapshot-chunks/db-metadata.ts

@@ -1,537 +0,0 @@
-/**
- * @syncular/server - Database-backed metadata store for snapshot chunks
- *
- * Stores chunk metadata in sync_snapshot_chunks_metadata table,
- * body content in blob storage adapter.
- */
-
-import {
-  type BlobStorageAdapter,
-  bytesToReadableStream,
-  randomId,
-  readAllBytesFromStream,
-  SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-  type SyncSnapshotChunkCompression,
-  type SyncSnapshotChunkEncoding,
-  type SyncSnapshotChunkRef,
-  sha256Hex,
-} from '@syncular/core';
-import type { Kysely } from 'kysely';
-import type { SyncCoreDb } from '../schema';
-import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
-
-export interface SnapshotChunkCleanupTuning {
-  /** Cleanup select/delete batch size. Default: 250 */
-  batchSize?: number;
-  /** Max concurrent blob deletes during cleanup. Default: 8 */
-  deleteConcurrency?: number;
-}
-
-export const SNAPSHOT_CHUNK_CLEANUP_TUNING_PRESETS = {
-  server: {
-    batchSize: 500,
-    deleteConcurrency: 16,
-  },
-  edge: {
-    batchSize: 100,
-    deleteConcurrency: 4,
-  },
-} as const satisfies Record<
-  'server' | 'edge',
-  Required<SnapshotChunkCleanupTuning>
->;
-
-export interface DbMetadataSnapshotChunkStorageOptions {
-  /** Database instance */
-  db: Kysely<SyncCoreDb>;
-  /** Blob storage adapter for body content */
-  blobAdapter: BlobStorageAdapter;
-  /** Optional prefix for chunk IDs */
-  chunkIdPrefix?: string;
-  /** Optional cleanup throughput tuning knobs. */
-  cleanupTuning?: SnapshotChunkCleanupTuning;
-}
-
-/**
- * Create a snapshot chunk storage that uses:
- * - Database for metadata (scope, commit seq, etc.)
- * - Blob adapter for body content
- */
-export function createDbMetadataChunkStorage(
-  options: DbMetadataSnapshotChunkStorageOptions
-): {
-  name: string;
-  storeChunk: (
-    metadata: Omit<
-      SnapshotChunkMetadata,
-      'chunkId' | 'byteLength' | 'blobHash'
-    > & {
-      body: Uint8Array;
-    }
-  ) => Promise<SyncSnapshotChunkRef>;
-  storeChunkStream: (
-    metadata: Omit<
-      SnapshotChunkMetadata,
-      'chunkId' | 'byteLength' | 'blobHash'
-    > & {
-      bodyStream: ReadableStream<Uint8Array>;
-      byteLength?: number;
-    }
-  ) => Promise<SyncSnapshotChunkRef>;
-  readChunk: (chunkId: string) => Promise<Uint8Array | null>;
-  readChunkStream: (
-    chunkId: string
-  ) => Promise<ReadableStream<Uint8Array> | null>;
-  findChunk: (
-    pageKey: SnapshotChunkPageKey
-  ) => Promise<SyncSnapshotChunkRef | null>;
-  cleanupExpired: (beforeIso: string) => Promise<number>;
-} {
-  const { db, blobAdapter, chunkIdPrefix = 'chunk_', cleanupTuning } = options;
-
-  function positiveIntOrDefault(value: number | undefined, fallback: number) {
-    if (value === undefined) return fallback;
-    if (!Number.isFinite(value)) return fallback;
-    const normalized = Math.trunc(value);
-    return normalized > 0 ? normalized : fallback;
-  }
-
-  const CLEANUP_BATCH_SIZE = positiveIntOrDefault(
-    cleanupTuning?.batchSize,
-    250
-  );
-  const CLEANUP_DELETE_CONCURRENCY = positiveIntOrDefault(
-    cleanupTuning?.deleteConcurrency,
-    8
-  );
-
-  // Generate deterministic blob hash from chunk identity metadata.
-  async function computeBlobHash(metadata: {
-    encoding: SyncSnapshotChunkEncoding;
-    compression: SyncSnapshotChunkCompression;
-    sha256: string;
-  }): Promise<string> {
-    const digest = await sha256Hex(
-      `${metadata.encoding}:${metadata.compression}:${metadata.sha256}`
-    );
-    return `sha256:${digest}`;
-  }
-
-  async function streamByteLength(
-    stream: ReadableStream<Uint8Array>
-  ): Promise<number> {
-    const reader = stream.getReader();
-    try {
-      let total = 0;
-      while (true) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        if (!value) continue;
-        total += value.length;
-      }
-      return total;
-    } finally {
-      reader.releaseLock();
-    }
-  }
-
-  async function runWithConcurrency<T>(
-    items: readonly T[],
-    concurrency: number,
-    worker: (item: T) => Promise<void>
-  ): Promise<void> {
-    if (items.length === 0) return;
-
-    const workerCount = Math.max(1, Math.min(concurrency, items.length));
-    let nextIndex = 0;
-
-    async function runWorker(): Promise<void> {
-      while (nextIndex < items.length) {
-        const index = nextIndex;
-        nextIndex += 1;
-        const item = items[index];
-        if (item === undefined) continue;
-        await worker(item);
-      }
-    }
-
-    await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
-  }
-
-  // Generate unique chunk ID
-  function generateChunkId(): string {
-    return `${chunkIdPrefix}${randomId()}`;
-  }
-
-  async function readStoredRef(args: {
-    partitionId: string;
-    scopeKey: string;
-    scope: string;
-    asOfCommitSeq: number;
-    rowCursor: string | null;
-    rowLimit: number;
-    encoding: SyncSnapshotChunkEncoding;
-    compression: SyncSnapshotChunkCompression;
-    nowIso?: string;
-    includeExpired?: boolean;
-  }): Promise<SyncSnapshotChunkRef | null> {
-    const nowIso = args.nowIso ?? new Date().toISOString();
-    const rowCursorKey = args.rowCursor ?? '';
-    const baseQuery = db
-      .selectFrom('sync_snapshot_chunks')
-      .select([
-        'chunk_id',
-        'sha256',
-        'byte_length',
-        'next_row_cursor',
-        'is_last_page',
-        'encoding',
-        'compression',
-      ])
-      .where('partition_id', '=', args.partitionId)
-      .where('scope_key', '=', args.scopeKey)
-      .where('scope', '=', args.scope)
-      .where('as_of_commit_seq', '=', args.asOfCommitSeq)
-      .where('row_cursor', '=', rowCursorKey)
-      .where('row_limit', '=', args.rowLimit)
-      .where('encoding', '=', args.encoding)
-      .where('compression', '=', args.compression);
-
-    const row = await (args.includeExpired
-      ? baseQuery.executeTakeFirst()
-      : baseQuery.where('expires_at', '>', nowIso).executeTakeFirst());
-
-    if (!row) return null;
-
-    if (row.encoding !== args.encoding) {
-      throw new Error(
-        `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
-      );
-    }
-    if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
-      throw new Error(
-        `Unexpected snapshot chunk compression: ${String(row.compression)}`
-      );
-    }
-
-    return {
-      id: row.chunk_id,
-      sha256: row.sha256,
-      byteLength: Number(row.byte_length ?? 0),
-      nextRowCursor:
-        row.next_row_cursor == null ? null : String(row.next_row_cursor),
-      isLastPage: Number(row.is_last_page ?? 0) === 1,
-      encoding: args.encoding,
-      compression: row.compression,
-    } as SyncSnapshotChunkRef;
-  }
-
-  async function readBlobHash(chunkId: string): Promise<string | null> {
-    const row = await db
-      .selectFrom('sync_snapshot_chunks')
-      .select(['blob_hash'])
-      .where('chunk_id', '=', chunkId)
-      .executeTakeFirst();
-    return row?.blob_hash ?? null;
-  }
-
-  async function upsertChunkMetadata(
-    metadata: Omit<
-      SnapshotChunkMetadata,
-      'chunkId' | 'byteLength' | 'blobHash'
-    >,
-    args: { blobHash: string; byteLength: number }
-  ): Promise<void> {
-    const chunkId = generateChunkId();
-    const now = new Date().toISOString();
-
-    await db
-      .insertInto('sync_snapshot_chunks')
-      .values({
-        chunk_id: chunkId,
-        partition_id: metadata.partitionId,
-        scope_key: metadata.scopeKey,
-        scope: metadata.scope,
-        as_of_commit_seq: metadata.asOfCommitSeq,
-        row_cursor: metadata.rowCursor ?? '',
-        row_limit: metadata.rowLimit,
-        next_row_cursor: metadata.nextRowCursor ?? null,
-        is_last_page: metadata.isLastPage ? 1 : 0,
-        encoding: metadata.encoding,
-        compression: metadata.compression,
-        sha256: metadata.sha256,
-        byte_length: args.byteLength,
-        blob_hash: args.blobHash,
-        expires_at: metadata.expiresAt,
-        created_at: now,
-      })
-      .onConflict((oc) =>
-        oc
-          .columns([
-            'partition_id',
-            'scope_key',
-            'scope',
-            'as_of_commit_seq',
-            'row_cursor',
-            'row_limit',
-            'encoding',
-            'compression',
-          ])
-          .doUpdateSet({
-            expires_at: metadata.expiresAt,
-            blob_hash: args.blobHash,
-            sha256: metadata.sha256,
-            byte_length: args.byteLength,
-            next_row_cursor: metadata.nextRowCursor ?? null,
-            is_last_page: metadata.isLastPage ? 1 : 0,
-            row_cursor: metadata.rowCursor ?? '',
-          })
-      )
-      .execute();
-  }
-
-  async function readChunkStreamById(
-    chunkId: string
-  ): Promise<ReadableStream<Uint8Array> | null> {
-    const blobHash = await readBlobHash(chunkId);
-    if (!blobHash) return null;
-
-    if (blobAdapter.getStream) {
-      return blobAdapter.getStream(blobHash);
-    }
-
-    if (blobAdapter.get) {
-      const bytes = await blobAdapter.get(blobHash);
-      return bytes ? bytesToReadableStream(bytes) : null;
-    }
-
-    throw new Error(
-      `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
-    );
-  }
-
-  return {
-    name: `db-metadata+${blobAdapter.name}`,
-
-    async storeChunk(
-      metadata: Omit<
-        SnapshotChunkMetadata,
-        'chunkId' | 'byteLength' | 'blobHash'
-      > & {
-        body: Uint8Array;
-      }
-    ): Promise<SyncSnapshotChunkRef> {
-      const { body, ...metaWithoutBody } = metadata;
-      const blobHash = await computeBlobHash(metaWithoutBody);
-
-      // Check if blob already exists (content-addressed dedup)
-      const blobExists = await blobAdapter.exists(blobHash);
-
-      if (!blobExists) {
-        // Store body in blob adapter
-        if (blobAdapter.put) {
-          await blobAdapter.put(blobHash, body, {
-            disableChecksum: true,
-            byteLength: body.length,
-            contentLength: body.length,
-          });
-        } else if (blobAdapter.putStream) {
-          await blobAdapter.putStream(blobHash, bytesToReadableStream(body), {
-            disableChecksum: true,
-            byteLength: body.length,
-            contentLength: body.length,
-          });
-        } else {
-          throw new Error(
-            `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
-          );
-        }
-      }
-
-      await upsertChunkMetadata(metaWithoutBody, {
-        blobHash,
-        byteLength: body.length,
-      });
-
-      const storedRef = await readStoredRef({
-        partitionId: metaWithoutBody.partitionId,
-        scopeKey: metaWithoutBody.scopeKey,
-        scope: metaWithoutBody.scope,
-        asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
-        rowCursor: metaWithoutBody.rowCursor,
-        rowLimit: metaWithoutBody.rowLimit,
-        encoding: metaWithoutBody.encoding,
-        compression: metaWithoutBody.compression,
-        includeExpired: true,
-      });
-
-      if (!storedRef) {
-        throw new Error('Failed to read stored snapshot chunk reference');
-      }
-
-      return storedRef;
-    },
-
-    async storeChunkStream(
-      metadata: Omit<
-        SnapshotChunkMetadata,
-        'chunkId' | 'byteLength' | 'blobHash'
-      > & {
-        bodyStream: ReadableStream<Uint8Array>;
-        byteLength?: number;
-      }
-    ): Promise<SyncSnapshotChunkRef> {
-      const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
-      const blobHash = await computeBlobHash(metaWithoutBody);
-
-      const blobExists = await blobAdapter.exists(blobHash);
-      let observedByteLength: number;
-
-      if (!blobExists) {
-        if (blobAdapter.putStream) {
-          const [uploadStream, countStream] = bodyStream.tee();
-          const uploadPromise =
-            typeof byteLength === 'number'
-              ? blobAdapter.putStream(blobHash, uploadStream, {
-                  disableChecksum: true,
-                  byteLength,
-                  contentLength: byteLength,
-                })
-              : blobAdapter.putStream(blobHash, uploadStream, {
-                  disableChecksum: true,
-                });
-          const countPromise = streamByteLength(countStream);
-
-          const [, countedByteLength] = await Promise.all([
-            uploadPromise,
-            countPromise,
-          ]);
-          observedByteLength = countedByteLength;
-        } else if (blobAdapter.put) {
-          const body = await readAllBytesFromStream(bodyStream);
-          await blobAdapter.put(blobHash, body);
-          observedByteLength = body.length;
-        } else {
-          throw new Error(
-            `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
-          );
-        }
-      } else if (typeof byteLength === 'number') {
-        observedByteLength = byteLength;
-        await bodyStream.cancel();
-      } else if (blobAdapter.getMetadata) {
-        const metadata = await blobAdapter.getMetadata(blobHash);
-        if (!metadata) {
-          throw new Error(
-            `Blob metadata missing for existing chunk ${blobHash}`
-          );
-        }
-        observedByteLength = metadata.size;
-        await bodyStream.cancel();
-      } else {
-        observedByteLength = await streamByteLength(bodyStream);
-      }
-
-      if (
-        typeof byteLength === 'number' &&
-        Number.isFinite(byteLength) &&
-        observedByteLength !== byteLength
-      ) {
-        throw new Error(
-          `Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`
-        );
-      }
-
-      await upsertChunkMetadata(metaWithoutBody, {
-        blobHash,
-        byteLength: observedByteLength,
-      });
-
-      const storedRef = await readStoredRef({
-        partitionId: metaWithoutBody.partitionId,
-        scopeKey: metaWithoutBody.scopeKey,
-        scope: metaWithoutBody.scope,
-        asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
-        rowCursor: metaWithoutBody.rowCursor,
-        rowLimit: metaWithoutBody.rowLimit,
-        encoding: metaWithoutBody.encoding,
-        compression: metaWithoutBody.compression,
-        includeExpired: true,
-      });
-
-      if (!storedRef) {
-        throw new Error('Failed to read stored snapshot chunk reference');
-      }
-
-      return storedRef;
-    },
-
-    async readChunk(chunkId: string): Promise<Uint8Array | null> {
-      const stream = await readChunkStreamById(chunkId);
-      if (!stream) return null;
-      return readAllBytesFromStream(stream);
-    },
-
-    async readChunkStream(
-      chunkId: string
-    ): Promise<ReadableStream<Uint8Array> | null> {
-      return readChunkStreamById(chunkId);
-    },
-
-    async findChunk(
-      pageKey: SnapshotChunkPageKey
-    ): Promise<SyncSnapshotChunkRef | null> {
-      return readStoredRef(pageKey);
-    },
-
-    async cleanupExpired(beforeIso: string): Promise<number> {
-      let deleted = 0;
-
-      while (true) {
-        const expiredRows = await db
-          .selectFrom('sync_snapshot_chunks')
-          .select(['chunk_id', 'blob_hash'])
-          .where('expires_at', '<=', beforeIso)
-          .orderBy('expires_at', 'asc')
-          .limit(CLEANUP_BATCH_SIZE)
-          .execute();
-
-        if (expiredRows.length === 0) {
-          break;
-        }
-
-        // Delete from blob storage (best effort).
-        await runWithConcurrency(
-          expiredRows,
-          CLEANUP_DELETE_CONCURRENCY,
-          async (row) => {
-            try {
-              await blobAdapter.delete(row.blob_hash);
-            } catch {
-              // Ignore deletion errors - blob may be shared or already deleted
-              // Log for observability but don't fail the cleanup
-              console.warn(
-                `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
-              );
-            }
-          }
-        );
-
-        const chunkIds = expiredRows.map((row) => row.chunk_id);
-        if (chunkIds.length > 0) {
-          const result = await db
-            .deleteFrom('sync_snapshot_chunks')
-            .where('chunk_id', 'in', chunkIds)
-            .executeTakeFirst();
-          deleted += Number(result.numDeletedRows ?? 0);
-        }
-
-        if (expiredRows.length < CLEANUP_BATCH_SIZE) {
-          break;
-        }
-      }
-
-      return deleted;
-    },
-  };
-}

package/src/snapshot-chunks/index.ts

@@ -1,8 +0,0 @@
-/**
- * @syncular/server - Snapshot chunk storage
- *
- * Separates chunk metadata (database) from body content (blob storage).
- */
-
-export * from './db-metadata';
-export * from './types';

package/src/snapshot-chunks/types.ts

@@ -1,105 +0,0 @@
-/**
- * @syncular/server - Snapshot chunk storage types
- *
- * Separates chunk metadata (in database) from chunk body (in blob storage).
- * Enables flexible storage backends (database, S3, R2, etc.)
- */
-
-import type {
-  SyncSnapshotChunkCompression,
-  SyncSnapshotChunkEncoding,
-  SyncSnapshotChunkRef,
-} from '@syncular/core';
-
-/**
- * Page key for identifying a specific chunk
- */
-export interface SnapshotChunkPageKey {
-  partitionId: string;
-  scopeKey: string;
-  scope: string;
-  asOfCommitSeq: number;
-  rowCursor: string | null;
-  rowLimit: number;
-  encoding: SyncSnapshotChunkEncoding;
-  compression: SyncSnapshotChunkCompression;
-}
-
-/**
- * Metadata stored in the database for each chunk
- */
-export interface SnapshotChunkMetadata {
-  chunkId: string;
-  partitionId: string;
-  scopeKey: string;
-  scope: string;
-  asOfCommitSeq: number;
-  rowCursor: string | null;
-  rowLimit: number;
-  nextRowCursor?: string | null;
-  isLastPage?: boolean;
-  encoding: SyncSnapshotChunkEncoding;
-  compression: SyncSnapshotChunkCompression;
-  sha256: string;
-  byteLength: number;
-  blobHash: string; // Reference to blob storage
-  expiresAt: string;
-}
-
-/**
- * Storage interface for snapshot chunks
- */
-export interface SnapshotChunkStorage {
-  /** Storage adapter name */
-  readonly name: string;
-
-  /**
-   * Store a chunk. Returns chunk reference.
-   * If chunk with same content already exists (by hash), returns existing reference.
-   */
-  storeChunk(
-    metadata: Omit<
-      SnapshotChunkMetadata,
-      'chunkId' | 'byteLength' | 'blobHash'
-    > & {
-      body: Uint8Array;
-    }
-  ): Promise<SyncSnapshotChunkRef>;
-
-  /**
-   * Store a chunk from a stream.
-   * Preferred for large payloads to avoid full buffering in memory.
-   */
-  storeChunkStream?(
-    metadata: Omit<
-      SnapshotChunkMetadata,
-      'chunkId' | 'byteLength' | 'blobHash'
-    > & {
-      bodyStream: ReadableStream<Uint8Array>;
-      byteLength?: number;
-    }
-  ): Promise<SyncSnapshotChunkRef>;
-
-  /**
-   * Read chunk body by chunk ID
-   */
-  readChunk(chunkId: string): Promise<Uint8Array | null>;
-
-  /**
-   * Read chunk body as a stream.
-   * Preferred for large payloads to avoid full buffering in memory.
-   */
-  readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
-
-  /**
-   * Find existing chunk by page key
-   */
-  findChunk(
-    pageKey: SnapshotChunkPageKey
-  ): Promise<SyncSnapshotChunkRef | null>;
-
-  /**
-   * Delete expired chunks. Returns number deleted.
-   */
-  cleanupExpired(beforeIso: string): Promise<number>;
-}