@powersync/service-core 1.14.0 → 1.15.1

package/src/util/protocol-types.ts

@@ -13,9 +13,51 @@ export const BucketRequest = t.object({
 
 export type BucketRequest = t.Decoded<typeof BucketRequest>;
 
+/**
+ * A sync steam that a client has expressed interest in by explicitly opening it on the client side.
+ */
+export const RequestedStreamSubscription = t.object({
+  /**
+   * The defined name of the stream as it appears in sync stream definitions.
+   */
+  stream: t.string,
+  /**
+   * An optional dictionary of parameters to pass to this specific stream.
+   */
+  parameters: t.record(t.any).optional(),
+  /**
+   * Set when the client wishes to re-assign a different priority to this stream.
+   *
+   * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
+   * useful when the priority for partial syncs depends on e.g. the current page opened in a client.
+   */
+  override_priority: t.union(t.number, t.Null)
+});
+
+export type RequestedStreamSubscription = t.Decoded<typeof RequestedStreamSubscription>;
+
+/**
+ * An overview of all subscribed streams as part of a streaming sync request.
+ */
+export const StreamSubscriptionRequest = t.object({
+  /**
+   * Whether to sync default streams.
+   *
+   * When disabled, only explicitly-opened subscriptions are included.
+   */
+  include_defaults: t.boolean.optional(),
+
+  /**
+   * An array of sync streams the client has opened explicitly.
+   */
+  subscriptions: t.array(RequestedStreamSubscription)
+});
+
+export type StreamSubscriptionRequest = t.Decoded<typeof StreamSubscriptionRequest>;
+
 export const StreamingSyncRequest = t.object({
   /**
-   * Existing bucket states.
+   * Existing client-side bucket states.
    */
   buckets: t.array(BucketRequest).optional(),
 
@@ -34,11 +76,6 @@ export const StreamingSyncRequest = t.object({
    */
   raw_data: t.boolean.optional(),
 
-  /**
-   * Data is received in a serialized BSON Buffer
-   */
-  binary_data: t.boolean.optional(),
-
   /**
    * Client parameters to be passed to the sync rules.
    */
@@ -47,7 +84,12 @@ export const StreamingSyncRequest = t.object({
   /**
    * Unique client id.
    */
-  client_id: t.string.optional()
+  client_id: t.string.optional(),
+
+  /**
+   * If the client is aware of streams, an array of streams the client has opened.
+   */
+  streams: StreamSubscriptionRequest.optional()
 });
 
 export type StreamingSyncRequest = t.Decoded<typeof StreamingSyncRequest>;
@@ -60,7 +102,7 @@ export interface StreamingSyncCheckpointDiff {
   checkpoint_diff: {
     last_op_id: ProtocolOpId;
     write_checkpoint?: ProtocolOpId;
-    updated_buckets: BucketChecksumWithDescription[];
+    updated_buckets: CheckpointBucket[];
     removed_buckets: string[];
   };
 }
@@ -99,10 +141,54 @@ export type StreamingSyncLine =
  */
 export type ProtocolOpId = string;
 
+export interface StreamDescription {
+  /**
+   * The name of the stream as it appears in the sync configuration.
+   */
+  name: string;
+
+  /**
+   * Whether this stream is subscribed to by default.
+   *
+   * For default streams, this field is still `true` if clients have an explicit subscription to the stream.
+   */
+  is_default: boolean;
+
+  /**
+   * If some subscriptions on this stream could not be resolved, e.g. due to an error, this array contains the faulty
+   * subscriptions along with an error message.
+   */
+  errors: StreamSubscriptionError[];
+}
+
+export interface StreamSubscriptionError {
+  /**
+   * The subscription that errored - either the default subscription or some of the explicit subscriptions.
+   */
+  subscription: 'default' | number;
+  /**
+   * A message describing the error on the subscription.
+   */
+  message: string;
+}
+
 export interface Checkpoint {
   last_op_id: ProtocolOpId;
   write_checkpoint?: ProtocolOpId;
-  buckets: BucketChecksumWithDescription[];
+  buckets: CheckpointBucket[];
+
+  /**
+   * All streams that the client is subscribed to.
+   *
+   * This field has two purposes:
+   *
+   *  1. It allows clients to determine which of their subscriptions actually works. E.g. if a user does
+   *     `db.syncStream('non_existent_stream').subscribe()`, clients don't immediately know that the stream doesn't
+   *     exist. Only after the next `checkpoint` line can they query this field and mark unresolved subscriptions.
+   *. 2. It allows clients to learn which default streams they have been subscribed to. This is relevant for APIs
+   *     listing all streams on the client-side.
+   */
+  streams: StreamDescription[];
 }
 
 export interface BucketState {
@@ -158,4 +244,46 @@ export interface BucketChecksum {
   count: number;
 }
 
-export interface BucketChecksumWithDescription extends BucketChecksum, BucketDescription {}
+/**
+ * The reason a particular bucket is included in a checkpoint.
+ *
+ * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that
+ * association is useful because it enables clients to track progress for individual sync streams.
+ */
+export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription;
+
+/**
+ * A bucket has been included in a checkpoint because it's part of a default stream.
+ */
+export type BucketDerivedFromDefaultStream = {
+  /**
+   * The index (into {@link Checkpoint.streams}) of the stream defining the bucket.
+   */
+  default: number;
+};
+
+/**
+ * The bucket has been included in a checkpoint because it's part of a stream that a client has explicitly subscribed
+ * to.
+ */
+export type BucketDerivedFromExplicitSubscription = {
+  /**
+   * The index (into {@link StreamSubscriptionRequest.subscriptions}) of the subscription yielding this bucket.
+   */
+  sub: number;
+};
+
+export interface ClientBucketDescription {
+  /**
+   * An opaque id of the bucket.
+   */
+  bucket: string;
+  /**
+   * The priority used to synchronize this bucket, derived from its definition and an optional priority override from
+   * the stream subscription.
+   */
+  priority: BucketPriority;
+  subscriptions: BucketSubscriptionReason[];
+}
+
+export interface CheckpointBucket extends BucketChecksum, ClientBucketDescription {}

package/src/util/utils.ts

@@ -6,11 +6,26 @@ import { BucketChecksum, ProtocolOpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
-import { PartialChecksum } from '../storage/ChecksumCache.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+/**
+ * A partial checksum can never be used on its own - must always be combined with a full BucketChecksum.
+ */
+export interface PartialChecksum {
+  bucket: string;
+  /**
+   * 32-bit unsigned hash.
+   */
+  partialChecksum: number;
+
+  /**
+   * Count of operations - informational only.
+   */
+  partialCount: number;
+}
+
 /**
  * op_id as used internally, for individual operations and checkpoints.
  *
@@ -83,20 +98,53 @@ export function addChecksums(a: number, b: number) {
   return (a + b) & 0xffffffff;
 }
 
-export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null): BucketChecksum {
-  if (b == null) {
+export function isPartialChecksum(c: PartialChecksum | BucketChecksum): c is PartialChecksum {
+  return 'partialChecksum' in c;
+}
+
+export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | BucketChecksum | null): BucketChecksum {
+  const checksum = addPartialChecksums(a.bucket, a, b);
+  if (isPartialChecksum(checksum)) {
+    // Should not happen since a != null
+    throw new ServiceAssertionError('Expected full checksum');
+  }
+  return checksum;
+}
+
+export function addPartialChecksums(
+  bucket: string,
+  a: PartialChecksum | BucketChecksum | null,
+  b: PartialChecksum | BucketChecksum | null
+): PartialChecksum | BucketChecksum {
+  if (a != null && b != null) {
+    if (!isPartialChecksum(b)) {
+      // Replaces a
+      return b;
+    }
+    // merge
+    if (!isPartialChecksum(a)) {
+      return {
+        bucket,
+        checksum: addChecksums(a.checksum, b.partialChecksum),
+        count: a.count + b.partialCount
+      };
+    } else {
+      return {
+        bucket,
+        partialChecksum: addChecksums(a.partialChecksum, b.partialChecksum),
+        partialCount: a.partialCount + b.partialCount
+      };
+    }
+  } else if (a != null) {
     return a;
-  } else if (b.isFullChecksum) {
-    return {
-      bucket: b.bucket,
-      count: b.partialCount,
-      checksum: b.partialChecksum
-    };
+  } else if (b != null) {
+    return b;
   } else {
+    // No data found (may still have a previously-cached checksum).
     return {
-      bucket: a.bucket,
-      count: a.count + b.partialCount,
-      checksum: addChecksums(a.checksum, b.partialChecksum)
+      bucket,
+      partialChecksum: 0,
+      partialCount: 0
     };
   }
 }
@@ -148,7 +196,10 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
  *
  * If we don't store data, we assume we always have a complete row.
  */
-export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+export function isCompleteRow(
+  storeData: boolean,
+  row: sync_rules.ToastableSqliteRow
+): row is sync_rules.SqliteInputRow {
   if (!storeData) {
     // Assume the row is complete - no need to check
     return true;

package/test/src/checksum_cache.test.ts

@@ -1,5 +1,5 @@
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
-import { addChecksums, InternalOpId } from '@/util/util-index.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import { addChecksums, BucketChecksum, InternalOpId, PartialChecksum } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
 import { describe, expect, it } from 'vitest';
 
@@ -12,22 +12,20 @@ function testHash(bucket: string, checkpoint: InternalOpId) {
   return hash;
 }
 
-function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum {
+function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum | BucketChecksum {
   if (request.start) {
     const a = testHash(request.bucket, request.start);
     const b = testHash(request.bucket, request.end);
     return {
       bucket: request.bucket,
       partialCount: Number(request.end) - Number(request.start),
-      partialChecksum: addChecksums(b, -a),
-      isFullChecksum: false
+      partialChecksum: addChecksums(b, -a)
     };
   } else {
     return {
       bucket: request.bucket,
-      partialChecksum: testHash(request.bucket, request.end),
-      partialCount: Number(request.end),
-      isFullChecksum: true
+      checksum: testHash(request.bucket, request.end),
+      count: Number(request.end)
     };
   }
 }

package/test/src/routes/mocks.ts

@@ -0,0 +1,59 @@
+import {
+  BucketStorageFactory,
+  createCoreAPIMetrics,
+  MetricsEngine,
+  OpenTelemetryMetricsFactory,
+  RouteAPI,
+  RouterEngine,
+  ServiceContext,
+  StorageEngine,
+  SyncContext,
+  SyncRulesBucketStorage
+} from '@/index.js';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+
+export function mockServiceContext(storage: Partial<SyncRulesBucketStorage> | null) {
+  // This is very incomplete - just enough to get the current tests passing.
+
+  const storageEngine: StorageEngine = {
+    activeBucketStorage: {
+      async getActiveStorage() {
+        return storage;
+      }
+    } as Partial<BucketStorageFactory>
+  } as any;
+
+  const meterProvider = new MeterProvider({
+    readers: []
+  });
+  const meter = meterProvider.getMeter('powersync-tests');
+  const metricsEngine = new MetricsEngine({
+    disable_telemetry_sharing: true,
+    factory: new OpenTelemetryMetricsFactory(meter)
+  });
+  createCoreAPIMetrics(metricsEngine);
+  const service_context: Partial<ServiceContext> = {
+    syncContext: new SyncContext({ maxBuckets: 1, maxDataFetchConcurrency: 1, maxParameterQueryResults: 1 }),
+    routerEngine: {
+      getAPI() {
+        return {
+          getParseSyncRulesOptions() {
+            return { defaultSchema: 'public' };
+          }
+        } as Partial<RouteAPI>;
+      },
+      addStopHandler() {
+        return () => {};
+      }
+    } as Partial<RouterEngine> as any,
+    storageEngine,
+    metricsEngine: metricsEngine,
+    // Not used
+    configuration: null as any,
+    lifeCycleEngine: null as any,
+    migrations: null as any,
+    replicationEngine: null as any,
+    serviceMode: null as any
+  };
+  return service_context as ServiceContext;
+}

package/test/src/routes/stream.test.ts

@@ -0,0 +1,84 @@
+import { BasicRouterRequest, Context, SyncRulesBucketStorage } from '@/index.js';
+import { logger, RouterResponse, ServiceError } from '@powersync/lib-services-framework';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { Readable, Writable } from 'stream';
+import { pipeline } from 'stream/promises';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { syncStreamed } from '../../../src/routes/endpoints/sync-stream.js';
+import { mockServiceContext } from './mocks.js';
+
+describe('Stream Route', () => {
+  describe('compressed stream', () => {
+    it('handles missing sync rules', async () => {
+      const context: Context = {
+        logger: logger,
+        service_context: mockServiceContext(null)
+      };
+
+      const request: BasicRouterRequest = {
+        headers: {},
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const error = (await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>).catch(
+        (e) => e
+      )) as ServiceError;
+
+      expect(error.errorData.status).toEqual(500);
+      expect(error.errorData.code).toEqual('PSYNC_S2302');
+    });
+
+    it('handles a stream error with compression', async () => {
+      // This primarily tests that an underlying storage error doesn't result in an uncaught error
+      // when compressing the stream.
+
+      const storage = {
+        getParsedSyncRules() {
+          return new SqlSyncRules('bucket_definitions: {}');
+        },
+        watchCheckpointChanges: async function* (options) {
+          throw new Error('Simulated storage error');
+        }
+      } as Partial<SyncRulesBucketStorage>;
+      const serviceContext = mockServiceContext(storage);
+
+      const context: Context = {
+        logger: logger,
+        service_context: serviceContext,
+        token_payload: {
+          exp: new Date().getTime() / 1000 + 10000,
+          iat: new Date().getTime() / 1000 - 10000,
+          sub: 'test-user'
+        }
+      };
+
+      // It may be worth eventually doing this via Fastify to test the full stack
+
+      const request: BasicRouterRequest = {
+        headers: {
+          'accept-encoding': 'gzip'
+        },
+        hostname: '',
+        protocol: 'http'
+      };
+
+      const response = await (syncStreamed.handler({ context, params: {}, request }) as Promise<RouterResponse>);
+      expect(response.status).toEqual(200);
+      const stream = response.data as Readable;
+      const r = await drainWithTimeout(stream).catch((error) => error);
+      expect(r.message).toContain('Simulated storage error');
+    });
+  });
+});
+
+export async function drainWithTimeout(readable: Readable, ms = 2_000) {
+  const devNull = new Writable({
+    write(_chunk, _enc, cb) {
+      cb();
+    } // discard everything
+  });
+
+  // Throws AbortError if it takes longer than ms, and destroys the stream
+  await pipeline(readable, devNull, { signal: AbortSignal.timeout(ms) });
+}