@syncular/server 0.0.1-111 → 0.0.1-114

package/src/pull.ts

@@ -20,7 +20,8 @@ import {
   startSyncSpan,
 } from '@syncular/core';
 import type { Kysely } from 'kysely';
-import type { ServerSyncDialect } from './dialect/types';
+import type { DbExecutor, ServerSyncDialect } from './dialect/types';
+import { EXTERNAL_CLIENT_ID } from './notify';
 import type { SyncCoreDb } from './schema';
 import type { TableRegistry } from './shapes/registry';
 import {
@@ -271,6 +272,33 @@ function recordPullMetrics(args: {
   );
 }
 
+/**
+ * Read synthetic commits created by notifyExternalDataChange() after a given cursor.
+ * Returns commit_seq and affected tables for each external change commit.
+ */
+async function readExternalDataChanges<DB extends SyncCoreDb>(
+  trx: DbExecutor<DB>,
+  dialect: ServerSyncDialect,
+  args: { partitionId: string; afterCursor: number }
+): Promise<Array<{ commitSeq: number; tables: string[] }>> {
+  type SyncExecutor = Pick<Kysely<SyncCoreDb>, 'selectFrom'>;
+  const executor = trx as SyncExecutor;
+
+  const rows = await executor
+    .selectFrom('sync_commits')
+    .select(['commit_seq', 'affected_tables'])
+    .where('partition_id', '=', args.partitionId)
+    .where('client_id', '=', EXTERNAL_CLIENT_ID)
+    .where('commit_seq', '>', args.afterCursor)
+    .orderBy('commit_seq', 'asc')
+    .execute();
+
+  return rows.map((row) => ({
+    commitSeq: Number(row.commit_seq),
+    tables: dialect.dbToArray(row.affected_tables),
+  }));
+}
+
 export async function pull<DB extends SyncCoreDb>(args: {
   db: Kysely<DB>;
   dialect: ServerSyncDialect;
@@ -342,6 +370,29 @@ export async function pull<DB extends SyncCoreDb>(args: {
           const activeSubscriptions: { scopes: ScopeValues }[] = [];
           const nextCursors: number[] = [];
 
+          // Detect external data changes (synthetic commits from notifyExternalDataChange)
+          // Compute minimum cursor across all active subscriptions to scope the query.
+          let minSubCursor = Number.MAX_SAFE_INTEGER;
+          for (const sub of resolved) {
+            if (
+              sub.status === 'revoked' ||
+              Object.keys(sub.scopes).length === 0
+            )
+              continue;
+            const cursor = Math.max(-1, sub.cursor ?? -1);
+            if (cursor >= 0 && cursor < minSubCursor) {
+              minSubCursor = cursor;
+            }
+          }
+
+          const externalDataChanges =
+            minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
+              ? await readExternalDataChanges(trx, dialect, {
+                  partitionId,
+                  afterCursor: minSubCursor,
+                })
+              : [];
+
           for (const sub of resolved) {
             const cursor = Math.max(-1, sub.cursor ?? -1);
             // Validate shape exists (throws if not registered)
@@ -369,7 +420,10 @@ export async function pull<DB extends SyncCoreDb>(args: {
               sub.bootstrapState != null ||
               cursor < 0 ||
               cursor > maxCommitSeq ||
-              (minCommitSeq > 0 && cursor < minCommitSeq - 1);
+              (minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
+              externalDataChanges.some(
+                (c) => c.commitSeq > cursor && c.tables.includes(sub.shape)
+              );
 
             if (needsBootstrap) {
               const tables = args.shapes

package/src/snapshot-chunks/index.ts

@@ -4,6 +4,5 @@
  * Separates chunk metadata (database) from body content (blob storage).
  */
 
-export * from './adapters/s3';
 export * from './db-metadata';
 export * from './types';

package/dist/snapshot-chunks/adapters/s3.d.ts

@@ -1,74 +0,0 @@
-/**
- * @syncular/server - S3-compatible snapshot chunk storage adapter
- *
- * Stores snapshot chunk bodies in S3/R2/MinIO with metadata in database.
- */
-import type { BlobStorageAdapter } from '@syncular/core';
-import type { Kysely } from 'kysely';
-import type { SyncCoreDb } from '../../schema';
-export interface S3SnapshotChunkStorageOptions {
-    /** Database instance for metadata */
-    db: Kysely<SyncCoreDb>;
-    /** S3 blob storage adapter */
-    s3Adapter: BlobStorageAdapter;
-    /** Optional key prefix for all chunks */
-    keyPrefix?: string;
-}
-/**
- * Create S3-compatible snapshot chunk storage.
- *
- * Stores chunk bodies in S3/R2/MinIO and metadata in the database.
- * Supports presigned URLs for direct client downloads.
- *
- * @example
- * ```typescript
- * import { createS3BlobStorageAdapter } from '@syncular/server/blobs/adapters/s3';
- * import { createS3SnapshotChunkStorage } from '@syncular/server/snapshot-chunks/adapters/s3';
- *
- * const s3Adapter = createS3BlobStorageAdapter({
- *   client: new S3Client({ region: 'us-east-1' }),
- *   bucket: 'my-snapshot-chunks',
- *   commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
- *   getSignedUrl,
- * });
- *
- * const chunkStorage = createS3SnapshotChunkStorage({
- *   db: kysely,
- *   s3Adapter,
- *   keyPrefix: 'snapshots/',
- * });
- * ```
- */
-export declare function createS3SnapshotChunkStorage(options: S3SnapshotChunkStorageOptions): {
-    name: string;
-    storeChunk: (metadata: Omit<import("..").SnapshotChunkMetadata, "blobHash" | "byteLength" | "chunkId"> & {
-        body: Uint8Array<ArrayBufferLike>;
-    }) => Promise<{
-        id: string;
-        byteLength: number;
-        sha256: string;
-        encoding: "json-row-frame-v1";
-        compression: "gzip";
-    }>;
-    storeChunkStream: (metadata: Omit<import("..").SnapshotChunkMetadata, "blobHash" | "byteLength" | "chunkId"> & {
-        bodyStream: ReadableStream<Uint8Array<ArrayBufferLike>>;
-        byteLength?: number | undefined;
-    }) => Promise<{
-        id: string;
-        byteLength: number;
-        sha256: string;
-        encoding: "json-row-frame-v1";
-        compression: "gzip";
-    }>;
-    readChunk: (chunkId: string) => Promise<Uint8Array<ArrayBufferLike> | null>;
-    readChunkStream: (chunkId: string) => Promise<ReadableStream<Uint8Array<ArrayBufferLike>> | null>;
-    findChunk: (pageKey: import("..").SnapshotChunkPageKey) => Promise<{
-        id: string;
-        byteLength: number;
-        sha256: string;
-        encoding: "json-row-frame-v1";
-        compression: "gzip";
-    } | null>;
-    cleanupExpired: (beforeIso: string) => Promise<number>;
-};
-//# sourceMappingURL=s3.d.ts.map

package/dist/snapshot-chunks/adapters/s3.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../src/snapshot-chunks/adapters/s3.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,WAAW,6BAA6B;IAC5C,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACvB,8BAA8B;IAC9B,SAAS,EAAE,kBAAkB,CAAC;IAC9B,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqBvC"}

package/dist/snapshot-chunks/adapters/s3.js

@@ -1,50 +0,0 @@
-/**
- * @syncular/server - S3-compatible snapshot chunk storage adapter
- *
- * Stores snapshot chunk bodies in S3/R2/MinIO with metadata in database.
- */
-import { createDbMetadataChunkStorage } from '../db-metadata.js';
-/**
- * Create S3-compatible snapshot chunk storage.
- *
- * Stores chunk bodies in S3/R2/MinIO and metadata in the database.
- * Supports presigned URLs for direct client downloads.
- *
- * @example
- * ```typescript
- * import { createS3BlobStorageAdapter } from '@syncular/server/blobs/adapters/s3';
- * import { createS3SnapshotChunkStorage } from '@syncular/server/snapshot-chunks/adapters/s3';
- *
- * const s3Adapter = createS3BlobStorageAdapter({
- *   client: new S3Client({ region: 'us-east-1' }),
- *   bucket: 'my-snapshot-chunks',
- *   commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
- *   getSignedUrl,
- * });
- *
- * const chunkStorage = createS3SnapshotChunkStorage({
- *   db: kysely,
- *   s3Adapter,
- *   keyPrefix: 'snapshots/',
- * });
- * ```
- */
-export function createS3SnapshotChunkStorage(options) {
-    const { db, s3Adapter, keyPrefix } = options;
-    // Wrap the S3 adapter to use prefixed keys
-    const prefixedAdapter = keyPrefix
-        ? {
-            ...s3Adapter,
-            name: `${s3Adapter.name}+prefixed`,
-            // Keys are already handled by the S3 adapter, prefix is applied there
-        }
-        : s3Adapter;
-    // Use the database metadata storage with S3 for bodies
-    const storage = createDbMetadataChunkStorage({
-        db,
-        blobAdapter: prefixedAdapter,
-        chunkIdPrefix: keyPrefix ? `${keyPrefix.replace(/\/$/, '')}_` : 'chunk_',
-    });
-    return storage;
-}
-//# sourceMappingURL=s3.js.map

package/dist/snapshot-chunks/adapters/s3.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../../src/snapshot-chunks/adapters/s3.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,4BAA4B,EAAE,MAAM,gBAAgB,CAAC;AAW9D;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAAsC,EACtC;IACA,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAE7C,2CAA2C;IAC3C,MAAM,eAAe,GAAuB,SAAS;QACnD,CAAC,CAAC;YACE,GAAG,SAAS;YACZ,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,WAAW;YAClC,sEAAsE;SACvE;QACH,CAAC,CAAC,SAAS,CAAC;IAEd,uDAAuD;IACvD,MAAM,OAAO,GAAG,4BAA4B,CAAC;QAC3C,EAAE;QACF,WAAW,EAAE,eAAe;QAC5B,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ;KACzE,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AAAA,CAChB"}

package/src/snapshot-chunks/adapters/s3.ts

@@ -1,68 +0,0 @@
-/**
- * @syncular/server - S3-compatible snapshot chunk storage adapter
- *
- * Stores snapshot chunk bodies in S3/R2/MinIO with metadata in database.
- */
-
-import type { BlobStorageAdapter } from '@syncular/core';
-import type { Kysely } from 'kysely';
-import type { SyncCoreDb } from '../../schema';
-import { createDbMetadataChunkStorage } from '../db-metadata';
-
-export interface S3SnapshotChunkStorageOptions {
-  /** Database instance for metadata */
-  db: Kysely<SyncCoreDb>;
-  /** S3 blob storage adapter */
-  s3Adapter: BlobStorageAdapter;
-  /** Optional key prefix for all chunks */
-  keyPrefix?: string;
-}
-
-/**
- * Create S3-compatible snapshot chunk storage.
- *
- * Stores chunk bodies in S3/R2/MinIO and metadata in the database.
- * Supports presigned URLs for direct client downloads.
- *
- * @example
- * ```typescript
- * import { createS3BlobStorageAdapter } from '@syncular/server/blobs/adapters/s3';
- * import { createS3SnapshotChunkStorage } from '@syncular/server/snapshot-chunks/adapters/s3';
- *
- * const s3Adapter = createS3BlobStorageAdapter({
- *   client: new S3Client({ region: 'us-east-1' }),
- *   bucket: 'my-snapshot-chunks',
- *   commands: { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand },
- *   getSignedUrl,
- * });
- *
- * const chunkStorage = createS3SnapshotChunkStorage({
- *   db: kysely,
- *   s3Adapter,
- *   keyPrefix: 'snapshots/',
- * });
- * ```
- */
-export function createS3SnapshotChunkStorage(
-  options: S3SnapshotChunkStorageOptions
-) {
-  const { db, s3Adapter, keyPrefix } = options;
-
-  // Wrap the S3 adapter to use prefixed keys
-  const prefixedAdapter: BlobStorageAdapter = keyPrefix
-    ? {
-        ...s3Adapter,
-        name: `${s3Adapter.name}+prefixed`,
-        // Keys are already handled by the S3 adapter, prefix is applied there
-      }
-    : s3Adapter;
-
-  // Use the database metadata storage with S3 for bodies
-  const storage = createDbMetadataChunkStorage({
-    db,
-    blobAdapter: prefixedAdapter,
-    chunkIdPrefix: keyPrefix ? `${keyPrefix.replace(/\/$/, '')}_` : 'chunk_',
-  });
-
-  return storage;
-}