@syncular/server-hono 0.0.6-95 → 0.0.6-96

package/src/__tests__/create-server.test.ts

@@ -9,7 +9,7 @@ import {
 import { createPostgresServerDialect } from '@syncular/server-dialect-postgres';
 import { Hono } from 'hono';
 import { defineWebSocketHelper } from 'hono/ws';
-import type { Kysely } from 'kysely';
+import { type Kysely, sql } from 'kysely';
 import { createSyncServer } from '../create-server';
 import { getSyncWebSocketConnectionManager } from '../routes';
 import type { WebSocketConnection } from '../ws';
@@ -100,10 +100,16 @@ describe('createSyncServer console configuration', () => {
     };
   }
 
-  function createPushRequest(): Request {
+  function createPushRequest(args?: {
+    requestId?: string;
+    title?: string;
+  }): Request {
     return new Request('http://localhost/sync', {
       method: 'POST',
-      headers: { 'content-type': 'application/json' },
+      headers: {
+        'content-type': 'application/json',
+        ...(args?.requestId ? { 'x-request-id': args.requestId } : {}),
+      },
       body: JSON.stringify({
         clientId: 'client-1',
         push: {
@@ -117,7 +123,7 @@ describe('createSyncServer console configuration', () => {
               payload: {
                 id: 'task-1',
                 user_id: 'u1',
-                title: 'Task 1',
+                title: args?.title ?? 'Task 1',
                 server_version: 0,
               },
             },
@@ -127,6 +133,62 @@ describe('createSyncServer console configuration', () => {
     });
   }
 
+  function parseSnapshotValue(value: unknown): unknown {
+    if (typeof value !== 'string') {
+      return value;
+    }
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }
+
+  async function waitForRequestEventRow(requestId: string): Promise<{
+    payload_ref: string | null;
+  }> {
+    for (let attempt = 0; attempt < 40; attempt++) {
+      const result = await sql<{ payload_ref: string | null }>`
+        SELECT payload_ref
+        FROM sync_request_events
+        WHERE request_id = ${requestId}
+        ORDER BY event_id DESC
+        LIMIT 1
+      `.execute(db);
+
+      const row = result.rows[0];
+      if (row) {
+        return row;
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+
+    throw new Error(`Timed out waiting for request event: ${requestId}`);
+  }
+
+  async function waitForRequestPayloadSnapshot(
+    payloadRef: string
+  ): Promise<unknown> {
+    for (let attempt = 0; attempt < 40; attempt++) {
+      const result = await sql<{ request_payload: unknown | null }>`
+        SELECT request_payload
+        FROM sync_request_payloads
+        WHERE payload_ref = ${payloadRef}
+        LIMIT 1
+      `.execute(db);
+
+      const row = result.rows[0];
+      if (row && row.request_payload !== null) {
+        return parseSnapshotValue(row.request_payload);
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+
+    throw new Error(`Timed out waiting for payload snapshot: ${payloadRef}`);
+  }
+
   it('keeps console routes disabled when console config is omitted', () => {
     const server = createSyncServer(createOptions());
     expect(server.consoleRoutes).toBeUndefined();
@@ -270,6 +332,87 @@ describe('createSyncServer console configuration', () => {
     });
   });
 
+  it('allows disabling request payload snapshots for privacy-sensitive deployments', async () => {
+    process.env.SYNC_CONSOLE_TOKEN = 'env-token';
+    const options = createOptions();
+    const server = createSyncServer({
+      ...options,
+      console: {},
+      routes: {
+        requestPayloadSnapshots: {
+          enabled: false,
+        },
+      },
+    });
+
+    const app = new Hono();
+    app.route('/sync', server.syncRoutes);
+
+    const requestId = 'req-no-payload-snapshot';
+    const response = await app.request(createPushRequest({ requestId }));
+    expect(response.status).toBe(200);
+
+    const eventRow = await waitForRequestEventRow(requestId);
+    expect(eventRow.payload_ref).toBeNull();
+
+    const payloadCountResult = await sql<{ total: number | string }>`
+      SELECT COUNT(*)::int AS total
+      FROM sync_request_payloads
+    `.execute(db);
+    const payloadCount = Number(payloadCountResult.rows[0]?.total ?? 0);
+    expect(payloadCount).toBe(0);
+  });
+
+  it('supports aggressively reducing stored payload snapshot size', async () => {
+    process.env.SYNC_CONSOLE_TOKEN = 'env-token';
+    const options = createOptions();
+    const server = createSyncServer({
+      ...options,
+      console: {},
+      routes: {
+        requestPayloadSnapshots: {
+          maxBytes: 32,
+        },
+      },
+    });
+
+    const app = new Hono();
+    app.route('/sync', server.syncRoutes);
+
+    const requestId = 'req-small-payload-preview';
+    const response = await app.request(
+      createPushRequest({
+        requestId,
+        title: 'x'.repeat(1024),
+      })
+    );
+    expect(response.status).toBe(200);
+
+    const eventRow = await waitForRequestEventRow(requestId);
+    expect(typeof eventRow.payload_ref).toBe('string');
+    if (!eventRow.payload_ref) {
+      throw new Error('Expected payload_ref to be present.');
+    }
+
+    const storedPayload = await waitForRequestPayloadSnapshot(
+      eventRow.payload_ref
+    );
+    expect(typeof storedPayload).toBe('object');
+    expect(Array.isArray(storedPayload)).toBe(false);
+    if (!storedPayload || typeof storedPayload !== 'object') {
+      throw new Error('Expected stored payload snapshot to be an object.');
+    }
+
+    const truncated = Reflect.get(storedPayload, 'truncated');
+    const preview = Reflect.get(storedPayload, 'preview');
+
+    expect(truncated).toBe(true);
+    expect(typeof preview).toBe('string');
+    if (typeof preview === 'string') {
+      expect(preview.length).toBeLessThanOrEqual(32);
+    }
+  });
+
   it('forwards maxConnectionsTotal from factory to realtime route', async () => {
     const options = createOptions();
     const upgradeWebSocket = defineWebSocketHelper(async () => {});

package/src/console/routes.ts

@@ -391,6 +391,25 @@ const handlersResponseSchema = z.object({
   items: z.array(ConsoleHandlerSchema),
 });
 
+const DEFAULT_REQUEST_EVENTS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+const DEFAULT_REQUEST_EVENTS_MAX_ROWS = 10_000;
+const DEFAULT_OPERATION_EVENTS_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
+const DEFAULT_OPERATION_EVENTS_MAX_ROWS = 5_000;
+const DEFAULT_AUTO_EVENTS_PRUNE_INTERVAL_MS = 5 * 60 * 1000;
+
+function readNonNegativeInteger(
+  value: number | undefined,
+  fallback: number
+): number {
+  if (typeof value !== 'number' || !Number.isFinite(value)) {
+    return fallback;
+  }
+  if (value < 0) {
+    return fallback;
+  }
+  return Math.floor(value);
+}
+
 export function createConsoleRoutes<
   DB extends SyncCoreDb,
   Auth extends SyncServerAuth,
@@ -489,6 +508,27 @@ export function createConsoleRoutes<
     1,
     options.metrics?.rawFallbackMaxEvents ?? 5000
   );
+  const requestEventsMaxAgeMs = readNonNegativeInteger(
+    options.maintenance?.requestEventsMaxAgeMs,
+    DEFAULT_REQUEST_EVENTS_MAX_AGE_MS
+  );
+  const requestEventsMaxRows = readNonNegativeInteger(
+    options.maintenance?.requestEventsMaxRows,
+    DEFAULT_REQUEST_EVENTS_MAX_ROWS
+  );
+  const operationEventsMaxAgeMs = readNonNegativeInteger(
+    options.maintenance?.operationEventsMaxAgeMs,
+    DEFAULT_OPERATION_EVENTS_MAX_AGE_MS
+  );
+  const operationEventsMaxRows = readNonNegativeInteger(
+    options.maintenance?.operationEventsMaxRows,
+    DEFAULT_OPERATION_EVENTS_MAX_ROWS
+  );
+  const autoEventsPruneIntervalMs = readNonNegativeInteger(
+    options.maintenance?.autoPruneIntervalMs,
+    DEFAULT_AUTO_EVENTS_PRUNE_INTERVAL_MS
+  );
+  let lastEventsPruneRunAt = 0;
 
   // Ensure console schema exists before handlers query console tables.
   const consoleSchemaReadyPromise = (
@@ -545,6 +585,13 @@ export function createConsoleRoutes<
     await next();
   });
 
+  routes.use('*', async (c, next) => {
+    if (c.req.method !== 'OPTIONS') {
+      triggerAutomaticEventsPrune();
+    }
+    await next();
+  });
+
   // Auth middleware
   const requireAuth = async (c: Context): Promise<ConsoleAuthResult | null> => {
     const auth = await options.authenticate(c);
@@ -640,6 +687,13 @@ export function createConsoleRoutes<
     createdAt: row.created_at ?? '',
   });
 
+  type PruneEventsRunResult = {
+    requestEventsDeleted: number;
+    operationEventsDeleted: number;
+    payloadSnapshotsDeleted: number;
+    totalDeleted: number;
+  };
+
   const deleteUnreferencedPayloadSnapshots = async (): Promise<number> => {
     const result = await db
       .deleteFrom('sync_request_payloads')
@@ -655,6 +709,180 @@ export function createConsoleRoutes<
     return Number(result?.numDeletedRows ?? 0);
   };
 
+  const pruneRequestEventsByAge = async (): Promise<number> => {
+    if (requestEventsMaxAgeMs <= 0) {
+      return 0;
+    }
+
+    const cutoffDate = new Date(Date.now() - requestEventsMaxAgeMs);
+    const result = await db
+      .deleteFrom('sync_request_events')
+      .where('created_at', '<', cutoffDate.toISOString())
+      .executeTakeFirst();
+
+    return Number(result?.numDeletedRows ?? 0);
+  };
+
+  const pruneRequestEventsByCount = async (): Promise<number> => {
+    if (requestEventsMaxRows <= 0) {
+      return 0;
+    }
+
+    const countRow = await db
+      .selectFrom('sync_request_events')
+      .select(({ fn }) => fn.countAll().as('total'))
+      .executeTakeFirst();
+
+    const total = coerceNumber(countRow?.total) ?? 0;
+    if (total <= requestEventsMaxRows) {
+      return 0;
+    }
+
+    const cutoffRow = await db
+      .selectFrom('sync_request_events')
+      .select(['event_id'])
+      .orderBy('event_id', 'desc')
+      .offset(requestEventsMaxRows)
+      .limit(1)
+      .executeTakeFirst();
+
+    const cutoffEventId = coerceNumber(cutoffRow?.event_id);
+    if (cutoffEventId === null) {
+      return 0;
+    }
+
+    const result = await db
+      .deleteFrom('sync_request_events')
+      .where('event_id', '<=', cutoffEventId)
+      .executeTakeFirst();
+    return Number(result?.numDeletedRows ?? 0);
+  };
+
+  const pruneOperationEventsByAge = async (): Promise<number> => {
+    if (operationEventsMaxAgeMs <= 0) {
+      return 0;
+    }
+
+    const cutoffDate = new Date(Date.now() - operationEventsMaxAgeMs);
+    const result = await db
+      .deleteFrom('sync_operation_events')
+      .where('created_at', '<', cutoffDate.toISOString())
+      .executeTakeFirst();
+    return Number(result?.numDeletedRows ?? 0);
+  };
+
+  const pruneOperationEventsByCount = async (): Promise<number> => {
+    if (operationEventsMaxRows <= 0) {
+      return 0;
+    }
+
+    const countRow = await db
+      .selectFrom('sync_operation_events')
+      .select(({ fn }) => fn.countAll().as('total'))
+      .executeTakeFirst();
+    const total = coerceNumber(countRow?.total) ?? 0;
+    if (total <= operationEventsMaxRows) {
+      return 0;
+    }
+
+    const cutoffRow = await db
+      .selectFrom('sync_operation_events')
+      .select(['operation_id'])
+      .orderBy('operation_id', 'desc')
+      .offset(operationEventsMaxRows)
+      .limit(1)
+      .executeTakeFirst();
+
+    const cutoffOperationId = coerceNumber(cutoffRow?.operation_id);
+    if (cutoffOperationId === null) {
+      return 0;
+    }
+
+    const result = await db
+      .deleteFrom('sync_operation_events')
+      .where('operation_id', '<=', cutoffOperationId)
+      .executeTakeFirst();
+    return Number(result?.numDeletedRows ?? 0);
+  };
+
+  const pruneConsoleEvents = async (): Promise<PruneEventsRunResult> => {
+    const requestEventsDeletedByAge = await pruneRequestEventsByAge();
+    const requestEventsDeletedByCount = await pruneRequestEventsByCount();
+    const requestEventsDeleted =
+      requestEventsDeletedByAge + requestEventsDeletedByCount;
+
+    const operationEventsDeletedByAge = await pruneOperationEventsByAge();
+    const operationEventsDeletedByCount = await pruneOperationEventsByCount();
+    const operationEventsDeleted =
+      operationEventsDeletedByAge + operationEventsDeletedByCount;
+
+    const payloadSnapshotsDeleted = await deleteUnreferencedPayloadSnapshots();
+    const totalDeleted = requestEventsDeleted + operationEventsDeleted;
+
+    return {
+      requestEventsDeleted,
+      operationEventsDeleted,
+      payloadSnapshotsDeleted,
+      totalDeleted,
+    };
+  };
+
+  let eventsPrunePromise: Promise<PruneEventsRunResult> | null = null;
+
+  const runEventsPrune = async (): Promise<PruneEventsRunResult> => {
+    if (eventsPrunePromise) {
+      return eventsPrunePromise;
+    }
+
+    let pending: Promise<PruneEventsRunResult>;
+    pending = pruneConsoleEvents()
+      .then((result) => {
+        lastEventsPruneRunAt = Date.now();
+        return result;
+      })
+      .finally(() => {
+        if (eventsPrunePromise === pending) {
+          eventsPrunePromise = null;
+        }
+      });
+
+    eventsPrunePromise = pending;
+    return pending;
+  };
+
+  const triggerAutomaticEventsPrune = (): void => {
+    if (autoEventsPruneIntervalMs <= 0) {
+      return;
+    }
+    if (eventsPrunePromise) {
+      return;
+    }
+    if (Date.now() - lastEventsPruneRunAt < autoEventsPruneIntervalMs) {
+      return;
+    }
+
+    void runEventsPrune()
+      .then((result) => {
+        if (result.totalDeleted <= 0 && result.payloadSnapshotsDeleted <= 0) {
+          return;
+        }
+
+        logSyncEvent({
+          event: 'console.prune_events_auto',
+          deletedCount: result.totalDeleted,
+          requestEventsDeleted: result.requestEventsDeleted,
+          operationEventsDeleted: result.operationEventsDeleted,
+          payloadDeletedCount: result.payloadSnapshotsDeleted,
+        });
+      })
+      .catch((error) => {
+        logSyncEvent({
+          event: 'console.prune_events_auto_failed',
+          error: error instanceof Error ? error.message : String(error),
+        });
+      });
+  };
+
   const recordOperationEvent = async (event: {
     operationType: ConsoleOperationType;
     consoleUserId?: string;
@@ -2779,56 +3007,16 @@ export function createConsoleRoutes<
       const auth = await requireAuth(c);
       if (!auth) return c.json({ error: 'UNAUTHENTICATED' }, 401);
 
-      // Prune events older than 7 days or keep max 10000 events
-      const cutoffDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
-
-      // Delete by date first
-      const resByDate = await db
-        .deleteFrom('sync_request_events')
-        .where('created_at', '<', cutoffDate.toISOString())
-        .executeTakeFirst();
-
-      let deletedCount = Number(resByDate?.numDeletedRows ?? 0);
-
-      // Then delete oldest if we still have more than 10000 events
-      const countRow = await db
-        .selectFrom('sync_request_events')
-        .select(({ fn }) => fn.countAll().as('total'))
-        .executeTakeFirst();
-
-      const total = coerceNumber(countRow?.total) ?? 0;
-      const maxEvents = 10000;
-
-      if (total > maxEvents) {
-        // Find event_id cutoff to keep only newest maxEvents
-        const cutoffRow = await db
-          .selectFrom('sync_request_events')
-          .select(['event_id'])
-          .orderBy('event_id', 'desc')
-          .offset(maxEvents)
-          .limit(1)
-          .executeTakeFirst();
-
-        if (cutoffRow) {
-          const cutoffEventId = coerceNumber(cutoffRow.event_id);
-          if (cutoffEventId !== null) {
-            const resByCount = await db
-              .deleteFrom('sync_request_events')
-              .where('event_id', '<=', cutoffEventId)
-              .executeTakeFirst();
-
-            deletedCount += Number(resByCount?.numDeletedRows ?? 0);
-          }
-        }
-      }
-
-      const payloadDeletedCount = await deleteUnreferencedPayloadSnapshots();
+      const pruneResult = await runEventsPrune();
+      const deletedCount = pruneResult.totalDeleted;
 
       logSyncEvent({
         event: 'console.prune_events',
         consoleUserId: auth.consoleUserId,
         deletedCount,
-        payloadDeletedCount,
+        requestEventsDeleted: pruneResult.requestEventsDeleted,
+        operationEventsDeleted: pruneResult.operationEventsDeleted,
+        payloadDeletedCount: pruneResult.payloadSnapshotsDeleted,
       });
 
       const result: ConsolePruneEventsResult = { deletedCount };

package/src/console/types.ts

@@ -53,6 +53,39 @@ export interface ConsoleMetricsOptions {
   rawFallbackMaxEvents?: number;
 }
 
+export interface ConsoleMaintenanceOptions {
+  /**
+   * Minimum interval between automatic event-prune runs.
+   * Set to 0 to disable automatic pruning.
+   * Default: 5 minutes.
+   */
+  autoPruneIntervalMs?: number;
+  /**
+   * Max age for request events before pruning.
+   * Set to 0 to disable age-based pruning.
+   * Default: 7 days.
+   */
+  requestEventsMaxAgeMs?: number;
+  /**
+   * Max number of request events to retain.
+   * Set to 0 to disable count-based pruning.
+   * Default: 10000.
+   */
+  requestEventsMaxRows?: number;
+  /**
+   * Max age for operation audit events before pruning.
+   * Set to 0 to disable age-based pruning.
+   * Default: 30 days.
+   */
+  operationEventsMaxAgeMs?: number;
+  /**
+   * Max number of operation audit events to retain.
+   * Set to 0 to disable count-based pruning.
+   * Default: 5000.
+   */
+  operationEventsMaxRows?: number;
+}
+
 export interface ConsoleBlobObject {
   key: string;
   size: number;
@@ -89,6 +122,7 @@ export interface ConsoleSharedOptions {
    */
   corsOrigins?: string[] | '*';
   metrics?: ConsoleMetricsOptions;
+  maintenance?: ConsoleMaintenanceOptions;
   blobBucket?: ConsoleBlobBucket;
 }
 

package/src/create-server.ts

@@ -189,6 +189,7 @@ export function createSyncServer<
     consoleSchemaReady,
     wsConnectionManager: getSyncWebSocketConnectionManager(syncRoutes),
     metrics: resolvedConsoleConfig.metrics,
+    maintenance: resolvedConsoleConfig.maintenance,
     blobBucket: resolvedConsoleConfig.blobBucket,
     ...(upgradeWebSocket && {
       websocket: {