@powersync/service-core 0.0.0-dev-20260225093637 → 0.0.0-dev-20260313100403

package/src/storage/SyncRulesBucketStorage.ts

@@ -1,5 +1,10 @@
 import { Logger, ObserverClient } from '@powersync/lib-services-framework';
-import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
+import {
+  BucketDataSource,
+  HydratedSyncRules,
+  ScopedParameterLookup,
+  SqliteJsonRow
+} from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
 import { BucketStorageFactory } from './BucketStorageFactory.js';
@@ -25,10 +30,17 @@ export interface SyncRulesBucketStorage
   resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
 
   /**
-   * Use this to get access to update storage data.
+   * Create a new writer.
+   *
+   * The writer must be flushed and disposed when done.
+   */
+  createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
+
+  /**
+   * @deprecated Use `createWriter()` with `await using` instead.
    */
   startBatch(
-    options: StartBatchOptions,
+    options: CreateWriterOptions,
     callback: (batch: BucketStorageBatch) => Promise<void>
   ): Promise<FlushedResult | null>;
 
@@ -103,7 +115,7 @@ export interface SyncRulesBucketStorage
    */
   getBucketDataBatch(
     checkpoint: util.InternalOpId,
-    dataBuckets: Map<string, util.InternalOpId>,
+    dataBuckets: BucketDataRequest[],
     options?: BucketDataBatchOptions
   ): AsyncIterable<SyncBucketDataChunk>;
 
@@ -115,7 +127,7 @@ export interface SyncRulesBucketStorage
    * This may be slow, depending on the size of the buckets.
    * The checksums are cached internally to compensate for this, but does not cover all cases.
    */
-  getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
+  getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
 
   /**
    * Clear checksum cache. Primarily intended for tests.
@@ -127,6 +139,16 @@ export interface SyncRulesBucketStorageListener {
   batchStarted: (batch: BucketStorageBatch) => void;
 }
 
+export interface BucketDataRequest {
+  bucket: string;
+  start: util.InternalOpId;
+  source: BucketDataSource;
+}
+export interface BucketChecksumRequest {
+  bucket: string;
+  source: BucketDataSource;
+}
+
 export interface SyncRuleStatus {
   checkpoint_lsn: string | null;
   active: boolean;
@@ -147,7 +169,7 @@ export interface ResolveTableResult {
   dropTables: SourceTable[];
 }
 
-export interface StartBatchOptions extends ParseSyncRulesOptions {
+export interface CreateWriterOptions extends ParseSyncRulesOptions {
   zeroLSN: string;
   /**
    * Whether or not to store a copy of the current data.
@@ -177,6 +199,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
   logger?: Logger;
 }
 
+/**
+ * @deprecated Use `CreateWriterOptions`.
+ */
+export interface StartBatchOptions extends CreateWriterOptions {}
+
 export interface CompactOptions {
   /**
    * Heap memory limit for the compact process.

package/src/sync/BucketChecksumState.ts

@@ -7,7 +7,8 @@ import {
   QuerierError,
   RequestedStream,
   RequestParameters,
-  ResolvedBucket
+  ResolvedBucket,
+  mergeBuckets
 } from '@powersync/service-sync-rules';
 
 import * as storage from '../storage/storage-index.js';
@@ -137,20 +138,20 @@ export class BucketChecksumState {
       }
 
       // Re-check updated buckets only
-      let checksumLookups: string[] = [];
+      let checksumLookups: storage.BucketChecksumRequest[] = [];
 
       let newChecksums = new Map<string, util.BucketChecksum>();
-      for (let bucket of bucketDescriptionMap.keys()) {
-        if (!updatedBuckets.has(bucket)) {
-          const existing = this.lastChecksums.get(bucket);
+      for (let desc of bucketDescriptionMap.values()) {
+        if (!updatedBuckets.has(desc.bucket)) {
+          const existing = this.lastChecksums.get(desc.bucket);
           if (existing == null) {
             // If this happens, it means updatedBuckets did not correctly include all new buckets
-            throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
+            throw new ServiceAssertionError(`Existing checksum not found for bucket ${desc.bucket}`);
           }
           // Bucket is not specifically updated, and we have a previous checksum
-          newChecksums.set(bucket, existing);
+          newChecksums.set(desc.bucket, existing);
         } else {
-          checksumLookups.push(bucket);
+          checksumLookups.push({ bucket: desc.bucket, source: desc.source });
         }
       }
 
@@ -163,12 +164,12 @@ export class BucketChecksumState {
       checksumMap = newChecksums;
     } else {
       // Re-check all buckets
-      const bucketList = [...bucketDescriptionMap.keys()];
+      const bucketList = [...bucketDescriptionMap.values()].map((b) => ({ bucket: b.bucket, source: b.source }));
       checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
     }
 
     // Subset of buckets for which there may be new data in this batch.
-    let bucketsToFetch: BucketDescription[];
+    let bucketsToFetch: ResolvedBucket[];
 
     let checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
 
@@ -207,10 +208,7 @@ export class BucketChecksumState {
         ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
       }));
       bucketsToFetch = [...generateBucketsToFetch].map((b) => {
-        return {
-          priority: bucketDescriptionMap.get(b)!.priority,
-          bucket: b
-        };
+        return bucketDescriptionMap.get(b)!;
       });
 
       deferredLog = () => {
@@ -265,7 +263,7 @@ export class BucketChecksumState {
           totalParamResults
         );
       };
-      bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
+      bucketsToFetch = allBuckets;
 
       const subscriptions: util.StreamDescription[] = [];
       const streamNameToIndex = new Map<string, number>();
@@ -342,17 +340,17 @@ export class BucketChecksumState {
         deferredLog();
       },
 
-      getFilteredBucketPositions: (buckets?: BucketDescription[]): Map<string, util.InternalOpId> => {
+      getFilteredBucketPositions: (buckets?: ResolvedBucket[]): storage.BucketDataRequest[] => {
         if (!hasAdvanced) {
           throw new ServiceAssertionError('Call line.advance() before getFilteredBucketPositions()');
         }
         buckets ??= bucketsToFetch;
-        const filtered = new Map<string, util.InternalOpId>();
+        const filtered: storage.BucketDataRequest[] = [];
 
         for (let bucket of buckets) {
           const state = this.bucketDataPositions.get(bucket.bucket);
           if (state) {
-            filtered.set(bucket.bucket, state.start_op_id);
+            filtered.push({ bucket: bucket.bucket, start: state.start_op_id, source: bucket.source });
           }
         }
         return filtered;
@@ -660,7 +658,7 @@ export class BucketParameterState {
 
 export interface CheckpointLine {
   checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
-  bucketsToFetch: BucketDescription[];
+  bucketsToFetch: ResolvedBucket[];
 
   /**
    * Call when a checkpoint line is being sent to a client, to update the internal state.
@@ -672,7 +670,7 @@ export interface CheckpointLine {
    *
    * @param bucketsToFetch List of buckets to fetch - either this.bucketsToFetch, or a subset of it. Defaults to this.bucketsToFetch.
    */
-  getFilteredBucketPositions(bucketsToFetch?: BucketDescription[]): Map<string, util.InternalOpId>;
+  getFilteredBucketPositions(bucketsToFetch?: ResolvedBucket[]): storage.BucketDataRequest[];
 
   /**
    * Update the position of bucket data the client has, after it was sent to the client.
@@ -762,32 +760,3 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
   const limited = buckets.slice(0, limit);
   return `${JSON.stringify(limited)}...`;
 }
-
-/**
- * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
- *
- * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
- *
- * ```
- * sync_streams:
- *  assets_by_category:
- *    query: select * from assets where category in (request.parameters() -> 'categories')
- * ```
- *
- * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
- * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
- * bucket.
- */
-function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
-  const byBucketId: Record<string, ResolvedBucket> = {};
-
-  for (const bucket of buckets) {
-    if (Object.hasOwn(byBucketId, bucket.bucket)) {
-      byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
-    } else {
-      byBucketId[bucket.bucket] = structuredClone(bucket);
-    }
-  }
-
-  return Object.values(byBucketId);
-}

package/src/sync/sync.ts

@@ -1,5 +1,11 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
-import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
+import {
+  BucketDescription,
+  BucketPriority,
+  HydratedSyncRules,
+  ResolvedBucket,
+  SqliteJsonValue
+} from '@powersync/service-sync-rules';
 
 import { AbortError } from 'ix/aborterror.js';
 
@@ -179,7 +185,7 @@ async function* streamResponseInner(
       // receive a sync complete message after the synchronization is done (which happens in the last
       // bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
       // sync properly.
-      const priorityBatches: [BucketPriority | null, BucketDescription[]][] = bucketsByPriority;
+      const priorityBatches: [BucketPriority | null, ResolvedBucket[]][] = bucketsByPriority;
       if (priorityBatches.length == 0) {
         priorityBatches.push([null, []]);
       }
@@ -257,7 +263,7 @@ interface BucketDataRequest {
   /** Contains current bucket state. Modified by the request as data is sent. */
   checkpointLine: CheckpointLine;
   /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
-  bucketsToFetch: BucketDescription[];
+  bucketsToFetch: ResolvedBucket[];
   /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
   legacyDataLines: boolean;
   /** Signals that the connection was aborted and that streaming should stop ASAP. */

package/src/sync/util.ts

@@ -183,6 +183,16 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
   );
 }
 
+export function unsettledPromise<T>(settled: Promise<PromiseSettledResult<T>>): Promise<T> {
+  return settled.then((result) => {
+    if (result.status === 'fulfilled') {
+      return Promise.resolve(result.value);
+    } else {
+      return Promise.reject(result.reason);
+    }
+  });
+}
+
 export type MapOrSet<T> = Map<T, any> | Set<T>;
 
 /**

package/src/util/env.ts

@@ -17,7 +17,7 @@ export const env = utils.collectEnvironmentVariables({
    */
   POWERSYNC_SYNC_RULES_B64: utils.type.string.optional(),
   /**
-   * Base64 encoded contents of sync rules YAML
+   * Base64 encoded contents of sync config YAML
    */
   POWERSYNC_SYNC_CONFIG_B64: utils.type.string.optional(),
   /**

package/test/src/auth.test.ts

@@ -7,7 +7,7 @@ import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
 import { KeyResult } from '../../src/auth/KeyCollector.js';
 import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
 import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
-import { debugKeyNotFound } from '../../src/auth/utils.js';
+import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
 
 const publicKeyRSA: jose.JWK = {
   use: 'sig',
@@ -496,6 +496,25 @@ describe('JWT Auth', () => {
     expect(verified.parsedPayload.claim).toEqual('test-claim-2');
   });
 
+  test('Supabase connection parsing', () => {
+    const details = getSupabaseJwksUrl({
+      type: 'postgresql',
+      uri: 'postgresql://db.abc123.supabase.co:5432/postgres'
+    });
+    expect(details).toEqual({
+      hostname: 'db.abc123.supabase.co',
+      projectId: 'abc123',
+      url: 'https://abc123.supabase.co/auth/v1/.well-known/jwks.json'
+    });
+
+    // supabase.com is not a supabase URI
+    const other = getSupabaseJwksUrl({
+      type: 'postgresql',
+      uri: 'postgresql://db.abc123.supabase.com:5432/postgres'
+    });
+    expect(other).toEqual(null);
+  });
+
   describe('debugKeyNotFound', () => {
     test('Supabase token with legacy auth not configured', async () => {
       const keys = await StaticSupabaseKeyCollector.importKeys([]);