@syncular/server-hono 0.0.1 → 0.0.2-127

package/src/__tests__/pull-chunk-storage.test.ts

@@ -1,9 +1,14 @@
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import { gunzipSync } from 'node:zlib';
 import {
+  configureSyncTelemetry,
+  decodeSnapshotRows,
+  getSyncTelemetry,
+  SyncCombinedResponseSchema,
   type SyncPullResponse,
-  SyncPullResponseSchema,
   type SyncSnapshotChunkRef,
+  type SyncSpan,
+  type SyncTelemetry,
 } from '@syncular/core';
 import {
   createServerHandler,
@@ -32,6 +37,35 @@ interface ClientDb {
   tasks: TasksTable;
 }
 
+function createExceptionCaptureTelemetry(calls: {
+  exceptions: Array<{
+    error: unknown;
+    context: Record<string, unknown> | undefined;
+  }>;
+}): SyncTelemetry {
+  return {
+    log() {},
+    tracer: {
+      startSpan(_options, callback) {
+        const span: SyncSpan = {
+          setAttribute() {},
+          setAttributes() {},
+          setStatus() {},
+        };
+        return callback(span);
+      },
+    },
+    metrics: {
+      count() {},
+      gauge() {},
+      distribution() {},
+    },
+    captureException(error, context) {
+      calls.exceptions.push({ error, context });
+    },
+  };
+}
+
 function mustGetFirstChunkId(payload: SyncPullResponse): string {
   const chunkId = payload.subscriptions[0]?.snapshots?.[0]?.chunks?.[0]?.id;
   if (!chunkId) {
@@ -40,6 +74,33 @@ function mustGetFirstChunkId(payload: SyncPullResponse): string {
   return chunkId;
 }
 
+async function streamToBytes(
+  stream: ReadableStream<Uint8Array>
+): Promise<Uint8Array> {
+  const reader = stream.getReader();
+  try {
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      chunks.push(value);
+      total += value.length;
+    }
+
+    const out = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+      out.set(chunk, offset);
+      offset += chunk.length;
+    }
+    return out;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
 describe('createSyncRoutes chunkStorage wiring', () => {
   let db: Kysely<ServerDb>;
   const dialect = createSqliteServerDialect();
@@ -123,7 +184,7 @@ describe('createSyncRoutes chunkStorage wiring', () => {
     app.route('/sync', routes);
 
     const pullResponse = await app.request(
-      new Request('http://localhost/sync/pull', {
+      new Request('http://localhost/sync', {
         method: 'POST',
         headers: {
           'content-type': 'application/json',
@@ -131,23 +192,28 @@ describe('createSyncRoutes chunkStorage wiring', () => {
         },
         body: JSON.stringify({
           clientId: 'client-1',
-          limitCommits: 10,
-          limitSnapshotRows: 100,
-          maxSnapshotPages: 1,
-          subscriptions: [
-            {
-              id: 'sub-1',
-              shape: 'tasks',
-              scopes: { user_id: 'u1' },
-              cursor: -1,
-            },
-          ],
+          pull: {
+            limitCommits: 10,
+            limitSnapshotRows: 100,
+            maxSnapshotPages: 1,
+            subscriptions: [
+              {
+                id: 'sub-1',
+                table: 'tasks',
+                scopes: { user_id: 'u1' },
+                cursor: -1,
+              },
+            ],
+          },
         }),
       })
     );
 
     expect(pullResponse.status).toBe(200);
-    const parsed = SyncPullResponseSchema.parse(await pullResponse.json());
+    const combined = SyncCombinedResponseSchema.parse(
+      await pullResponse.json()
+    );
+    const parsed = combined.pull!;
     const chunkId = mustGetFirstChunkId(parsed);
     expect(storeChunkCalls).toBe(1);
     expect(externalChunkBodies.has(chunkId)).toBe(true);
@@ -157,19 +223,7 @@ describe('createSyncRoutes chunkStorage wiring', () => {
       throw new Error('Expected external chunk body to be stored.');
     }
 
-    const decoded = new TextDecoder().decode(gunzipSync(storedExternal));
-    const rows = decoded
-      .split('\n')
-      .filter((line) => line.length > 0)
-      .map(
-        (line) =>
-          JSON.parse(line) as {
-            id: string;
-            user_id: string;
-            title: string;
-            server_version: number;
-          }
-      );
+    const rows = decodeSnapshotRows(gunzipSync(storedExternal));
 
     const snapshotChunkCountRow = await db
       .selectFrom('sync_snapshot_chunks')
@@ -181,4 +235,338 @@ describe('createSyncRoutes chunkStorage wiring', () => {
       { id: 't1', user_id: 'u1', title: 'Task 1', server_version: 1 },
     ]);
   }, 10_000);
+
+  it('uses storeChunkStream when the adapter provides it', async () => {
+    await db
+      .insertInto('tasks')
+      .values([
+        { id: 't1', user_id: 'u1', title: 'Task 1', server_version: 1 },
+        { id: 't2', user_id: 'u1', title: 'Task 2', server_version: 2 },
+      ])
+      .execute();
+
+    const tasksHandler = createServerHandler<ServerDb, ClientDb, 'tasks'>({
+      table: 'tasks',
+      scopes: ['user:{user_id}'],
+      resolveScopes: async (ctx) => ({ user_id: [ctx.actorId] }),
+    });
+
+    const externalChunkBodies = new Map<string, Uint8Array>();
+    let storeChunkCalls = 0;
+    let storeChunkStreamCalls = 0;
+    const chunkStorage: SnapshotChunkStorage = {
+      name: 'test-external-stream',
+      async storeChunk(metadata) {
+        storeChunkCalls += 1;
+        const ref: SyncSnapshotChunkRef = {
+          id: `chunk-${storeChunkCalls}`,
+          sha256: metadata.sha256,
+          byteLength: metadata.body.length,
+          encoding: metadata.encoding,
+          compression: metadata.compression,
+        };
+        externalChunkBodies.set(ref.id, new Uint8Array(metadata.body));
+        return ref;
+      },
+      async storeChunkStream(metadata) {
+        storeChunkStreamCalls += 1;
+        const body = await streamToBytes(metadata.bodyStream);
+        const ref: SyncSnapshotChunkRef = {
+          id: `chunk-stream-${storeChunkStreamCalls}`,
+          sha256: metadata.sha256,
+          byteLength: body.length,
+          encoding: metadata.encoding,
+          compression: metadata.compression,
+        };
+        externalChunkBodies.set(ref.id, body);
+        return ref;
+      },
+      async readChunk(chunkId: string) {
+        const body = externalChunkBodies.get(chunkId);
+        return body ? new Uint8Array(body) : null;
+      },
+      async findChunk() {
+        return null;
+      },
+      async cleanupExpired() {
+        return 0;
+      },
+    };
+
+    const routes = createSyncRoutes({
+      db,
+      dialect,
+      handlers: [tasksHandler],
+      authenticate: async (c) => {
+        const actorId = c.req.header('x-user-id');
+        return actorId ? { actorId } : null;
+      },
+      chunkStorage,
+    });
+
+    const app = new Hono();
+    app.route('/sync', routes);
+
+    const pullResponse = await app.request(
+      new Request('http://localhost/sync', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-user-id': 'u1',
+        },
+        body: JSON.stringify({
+          clientId: 'client-1',
+          pull: {
+            limitCommits: 10,
+            limitSnapshotRows: 1,
+            maxSnapshotPages: 2,
+            subscriptions: [
+              {
+                id: 'sub-1',
+                table: 'tasks',
+                scopes: { user_id: 'u1' },
+                cursor: -1,
+              },
+            ],
+          },
+        }),
+      })
+    );
+
+    expect(pullResponse.status).toBe(200);
+    const combined = SyncCombinedResponseSchema.parse(
+      await pullResponse.json()
+    );
+    const parsed = combined.pull!;
+    const chunkId = mustGetFirstChunkId(parsed);
+
+    expect(storeChunkStreamCalls).toBe(1);
+    expect(storeChunkCalls).toBe(0);
+    expect(externalChunkBodies.has(chunkId)).toBe(true);
+
+    const storedExternal = externalChunkBodies.get(chunkId);
+    if (!storedExternal) {
+      throw new Error('Expected external chunk body to be stored.');
+    }
+
+    const rows = decodeSnapshotRows(gunzipSync(storedExternal));
+    expect(rows).toEqual([
+      { id: 't1', user_id: 'u1', title: 'Task 1', server_version: 1 },
+      { id: 't2', user_id: 'u1', title: 'Task 2', server_version: 2 },
+    ]);
+  }, 10_000);
+
+  it('bundles multiple snapshot pages into one stored chunk', async () => {
+    await db
+      .insertInto('tasks')
+      .values([
+        { id: 't1', user_id: 'u1', title: 'Task 1', server_version: 1 },
+        { id: 't2', user_id: 'u1', title: 'Task 2', server_version: 2 },
+        { id: 't3', user_id: 'u1', title: 'Task 3', server_version: 3 },
+      ])
+      .execute();
+
+    const tasksHandler = createServerHandler<ServerDb, ClientDb, 'tasks'>({
+      table: 'tasks',
+      scopes: ['user:{user_id}'],
+      resolveScopes: async (ctx) => ({ user_id: [ctx.actorId] }),
+    });
+
+    const externalChunkBodies = new Map<string, Uint8Array>();
+    let storeChunkCalls = 0;
+    const chunkStorage: SnapshotChunkStorage = {
+      name: 'test-external',
+      async storeChunk(metadata) {
+        storeChunkCalls += 1;
+        const ref: SyncSnapshotChunkRef = {
+          id: `chunk-${storeChunkCalls}`,
+          sha256: metadata.sha256,
+          byteLength: metadata.body.length,
+          encoding: metadata.encoding,
+          compression: metadata.compression,
+        };
+        externalChunkBodies.set(ref.id, new Uint8Array(metadata.body));
+        return ref;
+      },
+      async readChunk(chunkId: string) {
+        const body = externalChunkBodies.get(chunkId);
+        return body ? new Uint8Array(body) : null;
+      },
+      async findChunk() {
+        return null;
+      },
+      async cleanupExpired() {
+        return 0;
+      },
+    };
+
+    const routes = createSyncRoutes({
+      db,
+      dialect,
+      handlers: [tasksHandler],
+      authenticate: async (c) => {
+        const actorId = c.req.header('x-user-id');
+        return actorId ? { actorId } : null;
+      },
+      chunkStorage,
+    });
+
+    const app = new Hono();
+    app.route('/sync', routes);
+
+    const pullResponse = await app.request(
+      new Request('http://localhost/sync', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-user-id': 'u1',
+        },
+        body: JSON.stringify({
+          clientId: 'client-1',
+          pull: {
+            limitCommits: 10,
+            limitSnapshotRows: 1,
+            maxSnapshotPages: 3,
+            subscriptions: [
+              {
+                id: 'sub-1',
+                table: 'tasks',
+                scopes: { user_id: 'u1' },
+                cursor: -1,
+              },
+            ],
+          },
+        }),
+      })
+    );
+
+    expect(pullResponse.status).toBe(200);
+    const combined = SyncCombinedResponseSchema.parse(
+      await pullResponse.json()
+    );
+    const parsed = combined.pull!;
+    const chunkId = mustGetFirstChunkId(parsed);
+
+    expect(parsed.subscriptions[0]?.snapshots?.length).toBe(1);
+    expect(storeChunkCalls).toBe(1);
+
+    const storedExternal = externalChunkBodies.get(chunkId);
+    if (!storedExternal) {
+      throw new Error('Expected external chunk body to be stored.');
+    }
+
+    const rows = decodeSnapshotRows(gunzipSync(storedExternal));
+
+    expect(rows).toEqual([
+      { id: 't1', user_id: 'u1', title: 'Task 1', server_version: 1 },
+      { id: 't2', user_id: 'u1', title: 'Task 2', server_version: 2 },
+      { id: 't3', user_id: 'u1', title: 'Task 3', server_version: 3 },
+    ]);
+  }, 10_000);
+
+  it('captures unhandled pull exceptions via telemetry', async () => {
+    await db
+      .insertInto('tasks')
+      .values({
+        id: 't1',
+        user_id: 'u1',
+        title: 'Task 1',
+        server_version: 1,
+      })
+      .execute();
+
+    const tasksHandler = createServerHandler<ServerDb, ClientDb, 'tasks'>({
+      table: 'tasks',
+      scopes: ['user:{user_id}'],
+      resolveScopes: async (ctx) => ({ user_id: [ctx.actorId] }),
+    });
+
+    const chunkStorageError = new Error('chunk storage failed');
+    const chunkStorage: SnapshotChunkStorage = {
+      name: 'failing-external',
+      async storeChunk() {
+        throw chunkStorageError;
+      },
+      async storeChunkStream() {
+        throw chunkStorageError;
+      },
+      async readChunk() {
+        return null;
+      },
+      async findChunk() {
+        return null;
+      },
+      async cleanupExpired() {
+        return 0;
+      },
+    };
+
+    const captured = {
+      exceptions: [] as Array<{
+        error: unknown;
+        context: Record<string, unknown> | undefined;
+      }>,
+    };
+    const previousTelemetry = getSyncTelemetry();
+    configureSyncTelemetry(createExceptionCaptureTelemetry(captured));
+
+    try {
+      const routes = createSyncRoutes({
+        db,
+        dialect,
+        handlers: [tasksHandler],
+        authenticate: async (c) => {
+          const actorId = c.req.header('x-user-id');
+          return actorId ? { actorId } : null;
+        },
+        chunkStorage,
+      });
+
+      const app = new Hono();
+      app.route('/sync', routes);
+
+      const response = await app.request(
+        new Request('http://localhost/sync', {
+          method: 'POST',
+          headers: {
+            'content-type': 'application/json',
+            'x-user-id': 'u1',
+          },
+          body: JSON.stringify({
+            clientId: 'client-1',
+            pull: {
+              limitCommits: 10,
+              limitSnapshotRows: 100,
+              maxSnapshotPages: 1,
+              subscriptions: [
+                {
+                  id: 'sub-1',
+                  table: 'tasks',
+                  scopes: { user_id: 'u1' },
+                  cursor: -1,
+                },
+              ],
+            },
+          }),
+        })
+      );
+      expect(response.status).toBe(500);
+
+      const capturedUnhandledException = captured.exceptions.find(
+        (entry) => entry.context?.event === 'sync.route.unhandled'
+      );
+      expect(capturedUnhandledException).toBeDefined();
+      if (!capturedUnhandledException) {
+        throw new Error('Expected unhandled exception telemetry entry');
+      }
+      expect(capturedUnhandledException.context).toEqual({
+        event: 'sync.route.unhandled',
+        method: 'POST',
+        path: '/sync',
+      });
+      expect(capturedUnhandledException.error).toBe(chunkStorageError);
+    } finally {
+      configureSyncTelemetry(previousTelemetry);
+    }
+  });
 });

package/src/__tests__/realtime-bridge.test.ts

@@ -22,6 +22,7 @@ describe('realtime broadcaster bridge', () => {
     const commit = await db
       .insertInto('sync_commits')
       .values({
+        partition_id: 'default',
         actor_id: 'u1',
         client_id: 'client-1',
         client_commit_id: 'c1',
@@ -37,6 +38,7 @@ describe('realtime broadcaster bridge', () => {
       .insertInto('sync_changes')
       .values({
         commit_seq: commitSeq,
+        partition_id: 'default',
         table: 'tasks',
         row_id: 't1',
         op: 'upsert',
@@ -97,7 +99,7 @@ describe('realtime broadcaster bridge', () => {
         sendError: mock(() => {}),
         close: mock(() => {}),
       },
-      ['user:u1']
+      ['default::user:u1']
     );
 
     // Publish without scopeKeys to exercise DB lookup on the receiving instance.

package/src/__tests__/sync-rate-limit-routing.test.ts

@@ -0,0 +1,181 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import {
+  createServerHandler,
+  ensureSyncSchema,
+  type SyncCoreDb,
+} from '@syncular/server';
+import { Hono } from 'hono';
+import type { Kysely } from 'kysely';
+import { createBunSqliteDb } from '../../../dialect-bun-sqlite/src';
+import { createSqliteServerDialect } from '../../../server-dialect-sqlite/src';
+import { resetRateLimitStore } from '../rate-limit';
+import { createSyncRoutes } from '../routes';
+
+interface TasksTable {
+  id: string;
+  user_id: string;
+  title: string;
+  server_version: number;
+}
+
+interface ServerDb extends SyncCoreDb {
+  tasks: TasksTable;
+}
+
+interface ClientDb {
+  tasks: TasksTable;
+}
+
+describe('createSyncRoutes rate limit routing', () => {
+  let db: Kysely<ServerDb>;
+  const dialect = createSqliteServerDialect();
+
+  beforeEach(async () => {
+    db = createBunSqliteDb<ServerDb>({ path: ':memory:' });
+    await ensureSyncSchema(db, dialect);
+
+    await db.schema
+      .createTable('tasks')
+      .addColumn('id', 'text', (col) => col.primaryKey())
+      .addColumn('user_id', 'text', (col) => col.notNull())
+      .addColumn('title', 'text', (col) => col.notNull())
+      .addColumn('server_version', 'integer', (col) =>
+        col.notNull().defaultTo(0)
+      )
+      .execute();
+  });
+
+  afterEach(async () => {
+    resetRateLimitStore();
+    await db.destroy();
+  });
+
+  const tasksHandler = createServerHandler<ServerDb, ClientDb, 'tasks'>({
+    table: 'tasks',
+    scopes: ['user:{user_id}'],
+    resolveScopes: async (ctx) => ({ user_id: [ctx.actorId] }),
+  });
+
+  const pullPayload = {
+    limitCommits: 10,
+    subscriptions: [
+      {
+        id: 'sub-1',
+        table: 'tasks',
+        scopes: { user_id: 'u1' },
+        cursor: -1,
+      },
+    ],
+  };
+
+  function pushPayload(clientCommitId: string, rowId: string) {
+    return {
+      clientCommitId,
+      schemaVersion: 1,
+      operations: [
+        {
+          table: 'tasks',
+          row_id: rowId,
+          op: 'upsert' as const,
+          base_version: null,
+          payload: {
+            id: rowId,
+            user_id: 'u1',
+            title: `Task ${rowId}`,
+            server_version: 0,
+          },
+        },
+      ],
+    };
+  }
+
+  function createJsonRequest(body: object): Request {
+    return new Request('http://localhost/sync', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+  }
+
+  it('does not consume push quota for pull-only requests', async () => {
+    const routes = createSyncRoutes({
+      db,
+      dialect,
+      handlers: [tasksHandler],
+      authenticate: async () => ({ actorId: 'u1' }),
+      sync: {
+        rateLimit: {
+          pull: { maxRequests: 1, windowMs: 60_000 },
+          push: { maxRequests: 1, windowMs: 60_000 },
+        },
+      },
+    });
+
+    const app = new Hono();
+    app.route('/sync', routes);
+
+    const pullFirst = await app.request(
+      createJsonRequest({
+        clientId: 'client-1',
+        pull: pullPayload,
+      })
+    );
+    const pushFirst = await app.request(
+      createJsonRequest({
+        clientId: 'client-1',
+        push: pushPayload('commit-1', 't1'),
+      })
+    );
+    const pullSecond = await app.request(
+      createJsonRequest({
+        clientId: 'client-1',
+        pull: pullPayload,
+      })
+    );
+    const pushSecond = await app.request(
+      createJsonRequest({
+        clientId: 'client-1',
+        push: pushPayload('commit-2', 't2'),
+      })
+    );
+
+    expect(pullFirst.status).toBe(200);
+    expect(pushFirst.status).toBe(200);
+    expect(pullSecond.status).toBe(429);
+    expect(pushSecond.status).toBe(429);
+  });
+
+  it('authenticates once per combined sync request with rate limiting enabled', async () => {
+    let authCalls = 0;
+
+    const routes = createSyncRoutes({
+      db,
+      dialect,
+      handlers: [tasksHandler],
+      authenticate: async () => {
+        authCalls += 1;
+        return { actorId: 'u1' };
+      },
+      sync: {
+        rateLimit: {
+          pull: { maxRequests: 10, windowMs: 60_000 },
+          push: { maxRequests: 10, windowMs: 60_000 },
+        },
+      },
+    });
+
+    const app = new Hono();
+    app.route('/sync', routes);
+
+    const response = await app.request(
+      createJsonRequest({
+        clientId: 'client-1',
+        push: pushPayload('commit-combined-1', 't-combined-1'),
+        pull: pullPayload,
+      })
+    );
+
+    expect(response.status).toBe(200);
+    expect(authCalls).toBe(1);
+  });
+});