@powersync/service-core 1.20.0 → 1.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/dist/auth/utils.d.ts +2 -1
  3. package/dist/auth/utils.js +1 -1
  4. package/dist/auth/utils.js.map +1 -1
  5. package/dist/entry/commands/compact-action.js +26 -5
  6. package/dist/entry/commands/compact-action.js.map +1 -1
  7. package/dist/routes/endpoints/admin.js +1 -0
  8. package/dist/routes/endpoints/admin.js.map +1 -1
  9. package/dist/routes/endpoints/sync-stream.js +6 -1
  10. package/dist/routes/endpoints/sync-stream.js.map +1 -1
  11. package/dist/storage/BucketStorageBatch.d.ts +29 -8
  12. package/dist/storage/BucketStorageBatch.js.map +1 -1
  13. package/dist/storage/BucketStorageFactory.d.ts +5 -0
  14. package/dist/storage/ChecksumCache.d.ts +5 -2
  15. package/dist/storage/ChecksumCache.js +8 -4
  16. package/dist/storage/ChecksumCache.js.map +1 -1
  17. package/dist/storage/PersistedSyncRulesContent.d.ts +6 -2
  18. package/dist/storage/PersistedSyncRulesContent.js +2 -1
  19. package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
  20. package/dist/storage/SourceTable.d.ts +7 -2
  21. package/dist/storage/SourceTable.js.map +1 -1
  22. package/dist/storage/StorageVersionConfig.d.ts +33 -0
  23. package/dist/storage/StorageVersionConfig.js +39 -6
  24. package/dist/storage/StorageVersionConfig.js.map +1 -1
  25. package/dist/storage/SyncRulesBucketStorage.d.ts +26 -6
  26. package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
  27. package/dist/sync/BucketChecksumState.d.ts +3 -3
  28. package/dist/sync/BucketChecksumState.js +12 -42
  29. package/dist/sync/BucketChecksumState.js.map +1 -1
  30. package/dist/sync/sync.js.map +1 -1
  31. package/dist/sync/util.d.ts +1 -0
  32. package/dist/sync/util.js +10 -0
  33. package/dist/sync/util.js.map +1 -1
  34. package/package.json +4 -4
  35. package/src/auth/utils.ts +5 -2
  36. package/src/entry/commands/compact-action.ts +29 -5
  37. package/src/routes/endpoints/admin.ts +1 -0
  38. package/src/routes/endpoints/sync-stream.ts +6 -1
  39. package/src/storage/BucketStorageBatch.ts +33 -9
  40. package/src/storage/BucketStorageFactory.ts +6 -0
  41. package/src/storage/ChecksumCache.ts +14 -6
  42. package/src/storage/PersistedSyncRulesContent.ts +7 -2
  43. package/src/storage/SourceTable.ts +7 -1
  44. package/src/storage/StorageVersionConfig.ts +54 -6
  45. package/src/storage/SyncRulesBucketStorage.ts +33 -6
  46. package/src/sync/BucketChecksumState.ts +18 -49
  47. package/src/sync/sync.ts +9 -3
  48. package/src/sync/util.ts +10 -0
  49. package/test/src/auth.test.ts +20 -1
  50. package/test/src/checksum_cache.test.ts +102 -57
  51. package/test/src/config.test.ts +1 -0
  52. package/test/src/sync/BucketChecksumState.test.ts +53 -21
  53. package/test/src/utils.ts +9 -0
  54. package/tsconfig.tsbuildinfo +1 -1
@@ -12,6 +12,16 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio
12
12
  };
13
13
 
14
14
  export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageListener>, AsyncDisposable {
15
+ /**
16
+ * Alias for [Symbol.asyncDispose]
17
+ */
18
+ dispose(): Promise<void>;
19
+
20
+ /**
21
+ * Last written op, if any. This may not reflect a consistent checkpoint.
22
+ */
23
+ last_flushed_op: InternalOpId | null;
24
+
15
25
  /**
16
26
  * Save an op, and potentially flush.
17
27
  *
@@ -45,19 +55,15 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
45
55
  * Flush and commit any saved ops. This creates a new checkpoint by default.
46
56
  *
47
57
  * Only call this after a transaction.
48
- *
49
- * Returns true if either (1) a new checkpoint was created, or (2) there are no changes to commit.
50
58
  */
51
- commit(lsn: string, options?: BucketBatchCommitOptions): Promise<boolean>;
59
+ commit(lsn: string, options?: BucketBatchCommitOptions): Promise<CheckpointResult>;
52
60
 
53
61
  /**
54
62
  * Advance the checkpoint LSN position, without any associated op.
55
63
  *
56
64
  * This must only be called when not inside a transaction.
57
- *
58
- * @returns true if the checkpoint was advanced, false if this was a no-op
59
65
  */
60
- keepalive(lsn: string): Promise<boolean>;
66
+ keepalive(lsn: string): Promise<CheckpointResult>;
61
67
 
62
68
  /**
63
69
  * Set the LSN that replication should resume from.
@@ -83,9 +89,9 @@ export interface BucketStorageBatch extends ObserverClient<BucketBatchStorageLis
83
89
  */
84
90
  resumeFromLsn: string | null;
85
91
 
86
- noCheckpointBeforeLsn: string;
87
-
88
- markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
92
+ markTableSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn?: string): Promise<SourceTable[]>;
93
+ markTableSnapshotRequired(table: SourceTable): Promise<void>;
94
+ markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
89
95
 
90
96
  updateTableProgress(table: SourceTable, progress: Partial<TableSnapshotStatus>): Promise<SourceTable>;
91
97
 
@@ -166,6 +172,24 @@ export interface SaveDelete {
166
172
  afterReplicaId?: undefined;
167
173
  }
168
174
 
175
+ export interface CheckpointResult {
176
+ /**
177
+ * True if any of these are true:
178
+ * 1. A snapshot is in progress.
179
+ * 2. The last checkpoint is older than "no_checkpoint_before" (if provided).
180
+ * 3. Replication was restarted with a lower LSN, and has not caught up yet.
181
+ */
182
+ checkpointBlocked: boolean;
183
+
184
+ /**
185
+ * True if a checkpoint was actually created by this operation. This can be false even if checkpointBlocked is false,
186
+ * if the checkpoint was empty.
187
+ *
188
+ * This is primarily used for testing.
189
+ */
190
+ checkpointCreated: boolean;
191
+ }
192
+
169
193
  export interface BucketBatchStorageListener {
170
194
  replicationEvent: (payload: ReplicationEventPayload) => void;
171
195
  }
@@ -257,3 +257,9 @@ export interface TestStorageOptions {
257
257
  }
258
258
  export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;
259
259
  export type TestReportStorageFactory = (options?: TestStorageOptions) => Promise<ReportStorage>;
260
+
261
+ export interface TestStorageConfig {
262
+ factory: TestStorageFactory;
263
+ tableIdStrings: boolean;
264
+ storageVersion?: number;
265
+ }
@@ -1,7 +1,9 @@
1
1
  import { OrderedSet } from '@js-sdsl/ordered-set';
2
2
  import { LRUCache } from 'lru-cache/min';
3
+ import { BucketDataSource } from '@powersync/service-sync-rules';
3
4
  import { BucketChecksum } from '../util/protocol-types.js';
4
5
  import { addBucketChecksums, ChecksumMap, InternalOpId, PartialChecksum } from '../util/utils.js';
6
+ import { BucketChecksumRequest } from './SyncRulesBucketStorage.js';
5
7
 
6
8
  interface ChecksumFetchContext {
7
9
  fetch(bucket: string): Promise<BucketChecksum>;
@@ -10,6 +12,7 @@ interface ChecksumFetchContext {
10
12
 
11
13
  export interface FetchPartialBucketChecksum {
12
14
  bucket: string;
15
+ source: BucketDataSource;
13
16
  start?: InternalOpId;
14
17
  end: InternalOpId;
15
18
  }
@@ -113,10 +116,10 @@ export class ChecksumCache {
113
116
  this.bucketCheckpoints.clear();
114
117
  }
115
118
 
116
- async getChecksums(checkpoint: InternalOpId, buckets: string[]): Promise<BucketChecksum[]> {
119
+ async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<BucketChecksum[]> {
117
120
  const checksums = await this.getChecksumMap(checkpoint, buckets);
118
121
  // Return results in the same order as the request
119
- return buckets.map((bucket) => checksums.get(bucket)!);
122
+ return buckets.map((bucket) => checksums.get(bucket.bucket)!);
120
123
  }
121
124
 
122
125
  /**
@@ -126,7 +129,7 @@ export class ChecksumCache {
126
129
  *
127
130
  * @returns a Map with exactly one entry for each bucket requested
128
131
  */
129
- async getChecksumMap(checkpoint: InternalOpId, buckets: string[]): Promise<ChecksumMap> {
132
+ async getChecksumMap(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise<ChecksumMap> {
130
133
  // Buckets that don't have a cached checksum for this checkpoint yet
131
134
  let toFetch = new Set<string>();
132
135
 
@@ -164,19 +167,21 @@ export class ChecksumCache {
164
167
  // One promise to await to ensure all fetch requests completed.
165
168
  let settledPromise: Promise<PromiseSettledResult<void>[]> | null = null;
166
169
 
170
+ const sourceMap = new Map<string, BucketDataSource>();
171
+
167
172
  try {
168
173
  // Individual cache fetch promises
169
174
  let cacheFetchPromises: Promise<void>[] = [];
170
175
 
171
176
  for (let bucket of buckets) {
172
- const cacheKey = makeCacheKey(checkpoint, bucket);
177
+ const cacheKey = makeCacheKey(checkpoint, bucket.bucket);
173
178
  let status: LRUCache.Status<BucketChecksum> = {};
174
179
  const p = this.cache.fetch(cacheKey, { context: context, status: status }).then((checksums) => {
175
180
  if (checksums == null) {
176
181
  // Should never happen
177
182
  throw new Error(`Failed to get checksums for ${cacheKey}`);
178
183
  }
179
- finalResults.set(bucket, checksums);
184
+ finalResults.set(bucket.bucket, checksums);
180
185
  });
181
186
  cacheFetchPromises.push(p);
182
187
  if (status.fetch == 'hit' || status.fetch == 'inflight') {
@@ -185,7 +190,8 @@ export class ChecksumCache {
185
190
  // In either case, we don't need to fetch a new checksum.
186
191
  } else {
187
192
  // We need a new request for this checksum.
188
- toFetch.add(bucket);
193
+ toFetch.add(bucket.bucket);
194
+ sourceMap.set(bucket.bucket, bucket.source);
189
195
  }
190
196
  }
191
197
  // We do this directly after creating the promises, otherwise
@@ -220,6 +226,7 @@ export class ChecksumCache {
220
226
  // Partial checksum found - make a partial checksum request
221
227
  bucketRequest = {
222
228
  bucket,
229
+ source: sourceMap.get(bucket)!,
223
230
  start: cp,
224
231
  end: checkpoint
225
232
  };
@@ -240,6 +247,7 @@ export class ChecksumCache {
240
247
  // No partial checksum found - make a new full checksum request
241
248
  bucketRequest = {
242
249
  bucket,
250
+ source: sourceMap.get(bucket)!,
243
251
  end: checkpoint
244
252
  };
245
253
  add.set(bucket, {
@@ -1,3 +1,4 @@
1
+ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
1
2
  import {
2
3
  CompatibilityContext,
3
4
  CompatibilityOption,
@@ -12,10 +13,9 @@ import {
12
13
  SyncConfigWithErrors,
13
14
  versionedHydrationState
14
15
  } from '@powersync/service-sync-rules';
16
+ import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
15
17
  import { ReplicationLock } from './ReplicationLock.js';
16
18
  import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js';
17
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
18
- import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js';
19
19
 
20
20
  export interface ParseSyncRulesOptions {
21
21
  defaultSchema: string;
@@ -120,6 +120,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon
120
120
  id: this.id,
121
121
  slot_name: this.slot_name,
122
122
  sync_rules: config,
123
+ hydrationState,
123
124
  hydratedSyncRules: () => {
124
125
  return config.config.hydrate({ hydrationState });
125
126
  }
@@ -140,6 +141,10 @@ export interface PersistedSyncRules {
140
141
  readonly id: number;
141
142
  readonly sync_rules: SyncConfigWithErrors;
142
143
  readonly slot_name: string;
144
+ /**
145
+ * For testing only.
146
+ */
147
+ readonly hydrationState: HydrationState;
143
148
 
144
149
  hydratedSyncRules(): HydratedSyncRules;
145
150
  }
@@ -1,9 +1,15 @@
1
1
  import { DEFAULT_TAG } from '@powersync/service-sync-rules';
2
2
  import * as util from '../util/util-index.js';
3
3
  import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js';
4
+ import { bson } from '../index.js';
5
+
6
+ /**
7
+ * Format of the id depends on the bucket storage module. It should be consistent within the module.
8
+ */
9
+ export type SourceTableId = string | bson.ObjectId;
4
10
 
5
11
  export interface SourceTableOptions {
6
- id: any;
12
+ id: SourceTableId;
7
13
  connectionTag: string;
8
14
  objectId: number | string | undefined;
9
15
  schema: string;
@@ -1,30 +1,78 @@
1
1
  export interface StorageVersionConfig {
2
+ version: number;
3
+
2
4
  /**
3
5
  * Whether versioned bucket names are automatically enabled.
4
6
  *
5
7
  * If this is false, bucket names may still be versioned depending on the sync config.
8
+ *
9
+ * Introduced in STORAGE_VERSION_2.
6
10
  */
7
11
  versionedBuckets: boolean;
12
+
13
+ /**
14
+ * Whether to use soft deletes for current_data, improving replication concurrency support.
15
+ *
16
+ * Introduced in STORAGE_VERSION_3.
17
+ */
18
+ softDeleteCurrentData: boolean;
8
19
  }
9
20
 
21
+ /**
22
+ * Corresponds to the storage version initially used, before we started explicitly versioning storage.
23
+ */
24
+ export const STORAGE_VERSION_1 = 1;
25
+
26
+ /**
27
+ * First new storage version.
28
+ *
29
+ * Uses versioned bucket names.
30
+ *
31
+ * On MongoDB storage, this always uses Long for checksums.
32
+ */
33
+ export const STORAGE_VERSION_2 = 2;
34
+
35
+ /**
36
+ * This version is currently unstable, and not enabled by default yet.
37
+ *
38
+ * This is used to build towards incremental reprocessing.
39
+ */
40
+ export const STORAGE_VERSION_3 = 3;
41
+
10
42
  /**
11
43
  * Oldest supported storage version.
12
44
  */
13
- export const LEGACY_STORAGE_VERSION = 1;
45
+ export const LEGACY_STORAGE_VERSION = STORAGE_VERSION_1;
14
46
 
15
47
  /**
16
48
  * Default storage version for newly persisted sync rules.
17
49
  */
18
- export const CURRENT_STORAGE_VERSION = 2;
50
+ export const CURRENT_STORAGE_VERSION = STORAGE_VERSION_2;
51
+
52
+ /**
53
+ * All versions that can be loaded.
54
+ *
55
+ * This includes unstable versions.
56
+ */
57
+ export const SUPPORTED_STORAGE_VERSIONS = [STORAGE_VERSION_1, STORAGE_VERSION_2, STORAGE_VERSION_3];
19
58
 
20
59
  /**
21
60
  * Shared storage-version behavior across storage implementations.
22
61
  */
23
62
  export const STORAGE_VERSION_CONFIG: Record<number, StorageVersionConfig | undefined> = {
24
- [LEGACY_STORAGE_VERSION]: {
25
- versionedBuckets: false
63
+ [STORAGE_VERSION_1]: {
64
+ version: STORAGE_VERSION_1,
65
+ versionedBuckets: false,
66
+ softDeleteCurrentData: false
67
+ },
68
+ [STORAGE_VERSION_2]: {
69
+ version: STORAGE_VERSION_2,
70
+ versionedBuckets: true,
71
+ softDeleteCurrentData: false
26
72
  },
27
- [CURRENT_STORAGE_VERSION]: {
28
- versionedBuckets: true
73
+ [STORAGE_VERSION_3]: {
74
+ version: STORAGE_VERSION_3,
75
+ versionedBuckets: true,
76
+ softDeleteCurrentData: true
29
77
  }
30
78
  };
@@ -1,5 +1,10 @@
1
1
  import { Logger, ObserverClient } from '@powersync/lib-services-framework';
2
- import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
2
+ import {
3
+ BucketDataSource,
4
+ HydratedSyncRules,
5
+ ScopedParameterLookup,
6
+ SqliteJsonRow
7
+ } from '@powersync/service-sync-rules';
3
8
  import * as util from '../util/util-index.js';
4
9
  import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
5
10
  import { BucketStorageFactory } from './BucketStorageFactory.js';
@@ -25,10 +30,17 @@ export interface SyncRulesBucketStorage
25
30
  resolveTable(options: ResolveTableOptions): Promise<ResolveTableResult>;
26
31
 
27
32
  /**
28
- * Use this to get access to update storage data.
33
+ * Create a new writer.
34
+ *
35
+ * The writer must be flushed and disposed when done.
36
+ */
37
+ createWriter(options: CreateWriterOptions): Promise<BucketStorageBatch>;
38
+
39
+ /**
40
+ * @deprecated Use `createWriter()` with `await using` instead.
29
41
  */
30
42
  startBatch(
31
- options: StartBatchOptions,
43
+ options: CreateWriterOptions,
32
44
  callback: (batch: BucketStorageBatch) => Promise<void>
33
45
  ): Promise<FlushedResult | null>;
34
46
 
@@ -103,7 +115,7 @@ export interface SyncRulesBucketStorage
103
115
  */
104
116
  getBucketDataBatch(
105
117
  checkpoint: util.InternalOpId,
106
- dataBuckets: Map<string, util.InternalOpId>,
118
+ dataBuckets: BucketDataRequest[],
107
119
  options?: BucketDataBatchOptions
108
120
  ): AsyncIterable<SyncBucketDataChunk>;
109
121
 
@@ -115,7 +127,7 @@ export interface SyncRulesBucketStorage
115
127
  * This may be slow, depending on the size of the buckets.
116
128
  * The checksums are cached internally to compensate for this, but does not cover all cases.
117
129
  */
118
- getChecksums(checkpoint: util.InternalOpId, buckets: string[]): Promise<util.ChecksumMap>;
130
+ getChecksums(checkpoint: util.InternalOpId, buckets: BucketChecksumRequest[]): Promise<util.ChecksumMap>;
119
131
 
120
132
  /**
121
133
  * Clear checksum cache. Primarily intended for tests.
@@ -127,6 +139,16 @@ export interface SyncRulesBucketStorageListener {
127
139
  batchStarted: (batch: BucketStorageBatch) => void;
128
140
  }
129
141
 
142
+ export interface BucketDataRequest {
143
+ bucket: string;
144
+ start: util.InternalOpId;
145
+ source: BucketDataSource;
146
+ }
147
+ export interface BucketChecksumRequest {
148
+ bucket: string;
149
+ source: BucketDataSource;
150
+ }
151
+
130
152
  export interface SyncRuleStatus {
131
153
  checkpoint_lsn: string | null;
132
154
  active: boolean;
@@ -147,7 +169,7 @@ export interface ResolveTableResult {
147
169
  dropTables: SourceTable[];
148
170
  }
149
171
 
150
- export interface StartBatchOptions extends ParseSyncRulesOptions {
172
+ export interface CreateWriterOptions extends ParseSyncRulesOptions {
151
173
  zeroLSN: string;
152
174
  /**
153
175
  * Whether or not to store a copy of the current data.
@@ -177,6 +199,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
177
199
  logger?: Logger;
178
200
  }
179
201
 
202
+ /**
203
+ * @deprecated Use `CreateWriterOptions`.
204
+ */
205
+ export interface StartBatchOptions extends CreateWriterOptions {}
206
+
180
207
  export interface CompactOptions {
181
208
  /**
182
209
  * Heap memory limit for the compact process.
@@ -7,7 +7,8 @@ import {
7
7
  QuerierError,
8
8
  RequestedStream,
9
9
  RequestParameters,
10
- ResolvedBucket
10
+ ResolvedBucket,
11
+ mergeBuckets
11
12
  } from '@powersync/service-sync-rules';
12
13
 
13
14
  import * as storage from '../storage/storage-index.js';
@@ -137,20 +138,20 @@ export class BucketChecksumState {
137
138
  }
138
139
 
139
140
  // Re-check updated buckets only
140
- let checksumLookups: string[] = [];
141
+ let checksumLookups: storage.BucketChecksumRequest[] = [];
141
142
 
142
143
  let newChecksums = new Map<string, util.BucketChecksum>();
143
- for (let bucket of bucketDescriptionMap.keys()) {
144
- if (!updatedBuckets.has(bucket)) {
145
- const existing = this.lastChecksums.get(bucket);
144
+ for (let desc of bucketDescriptionMap.values()) {
145
+ if (!updatedBuckets.has(desc.bucket)) {
146
+ const existing = this.lastChecksums.get(desc.bucket);
146
147
  if (existing == null) {
147
148
  // If this happens, it means updatedBuckets did not correctly include all new buckets
148
- throw new ServiceAssertionError(`Existing checksum not found for bucket ${bucket}`);
149
+ throw new ServiceAssertionError(`Existing checksum not found for bucket ${desc.bucket}`);
149
150
  }
150
151
  // Bucket is not specifically updated, and we have a previous checksum
151
- newChecksums.set(bucket, existing);
152
+ newChecksums.set(desc.bucket, existing);
152
153
  } else {
153
- checksumLookups.push(bucket);
154
+ checksumLookups.push({ bucket: desc.bucket, source: desc.source });
154
155
  }
155
156
  }
156
157
 
@@ -163,12 +164,12 @@ export class BucketChecksumState {
163
164
  checksumMap = newChecksums;
164
165
  } else {
165
166
  // Re-check all buckets
166
- const bucketList = [...bucketDescriptionMap.keys()];
167
+ const bucketList = [...bucketDescriptionMap.values()].map((b) => ({ bucket: b.bucket, source: b.source }));
167
168
  checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
168
169
  }
169
170
 
170
171
  // Subset of buckets for which there may be new data in this batch.
171
- let bucketsToFetch: BucketDescription[];
172
+ let bucketsToFetch: ResolvedBucket[];
172
173
 
173
174
  let checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
174
175
 
@@ -207,10 +208,7 @@ export class BucketChecksumState {
207
208
  ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex)
208
209
  }));
209
210
  bucketsToFetch = [...generateBucketsToFetch].map((b) => {
210
- return {
211
- priority: bucketDescriptionMap.get(b)!.priority,
212
- bucket: b
213
- };
211
+ return bucketDescriptionMap.get(b)!;
214
212
  });
215
213
 
216
214
  deferredLog = () => {
@@ -265,7 +263,7 @@ export class BucketChecksumState {
265
263
  totalParamResults
266
264
  );
267
265
  };
268
- bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority }));
266
+ bucketsToFetch = allBuckets;
269
267
 
270
268
  const subscriptions: util.StreamDescription[] = [];
271
269
  const streamNameToIndex = new Map<string, number>();
@@ -342,17 +340,17 @@ export class BucketChecksumState {
342
340
  deferredLog();
343
341
  },
344
342
 
345
- getFilteredBucketPositions: (buckets?: BucketDescription[]): Map<string, util.InternalOpId> => {
343
+ getFilteredBucketPositions: (buckets?: ResolvedBucket[]): storage.BucketDataRequest[] => {
346
344
  if (!hasAdvanced) {
347
345
  throw new ServiceAssertionError('Call line.advance() before getFilteredBucketPositions()');
348
346
  }
349
347
  buckets ??= bucketsToFetch;
350
- const filtered = new Map<string, util.InternalOpId>();
348
+ const filtered: storage.BucketDataRequest[] = [];
351
349
 
352
350
  for (let bucket of buckets) {
353
351
  const state = this.bucketDataPositions.get(bucket.bucket);
354
352
  if (state) {
355
- filtered.set(bucket.bucket, state.start_op_id);
353
+ filtered.push({ bucket: bucket.bucket, start: state.start_op_id, source: bucket.source });
356
354
  }
357
355
  }
358
356
  return filtered;
@@ -660,7 +658,7 @@ export class BucketParameterState {
660
658
 
661
659
  export interface CheckpointLine {
662
660
  checkpointLine: util.StreamingSyncCheckpointDiff | util.StreamingSyncCheckpoint;
663
- bucketsToFetch: BucketDescription[];
661
+ bucketsToFetch: ResolvedBucket[];
664
662
 
665
663
  /**
666
664
  * Call when a checkpoint line is being sent to a client, to update the internal state.
@@ -672,7 +670,7 @@ export interface CheckpointLine {
672
670
  *
673
671
  * @param bucketsToFetch List of buckets to fetch - either this.bucketsToFetch, or a subset of it. Defaults to this.bucketsToFetch.
674
672
  */
675
- getFilteredBucketPositions(bucketsToFetch?: BucketDescription[]): Map<string, util.InternalOpId>;
673
+ getFilteredBucketPositions(bucketsToFetch?: ResolvedBucket[]): storage.BucketDataRequest[];
676
674
 
677
675
  /**
678
676
  * Update the position of bucket data the client has, after it was sent to the client.
@@ -762,32 +760,3 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number)
762
760
  const limited = buckets.slice(0, limit);
763
761
  return `${JSON.stringify(limited)}...`;
764
762
  }
765
-
766
- /**
767
- * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate.
768
- *
769
- * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g.
770
- *
771
- * ```
772
- * sync_streams:
773
- * assets_by_category:
774
- * query: select * from assets where category in (request.parameters() -> 'categories')
775
- * ```
776
- *
777
- * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each
778
- * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]`
779
- * bucket.
780
- */
781
- function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] {
782
- const byBucketId: Record<string, ResolvedBucket> = {};
783
-
784
- for (const bucket of buckets) {
785
- if (Object.hasOwn(byBucketId, bucket.bucket)) {
786
- byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons);
787
- } else {
788
- byBucketId[bucket.bucket] = structuredClone(bucket);
789
- }
790
- }
791
-
792
- return Object.values(byBucketId);
793
- }
package/src/sync/sync.ts CHANGED
@@ -1,5 +1,11 @@
1
1
  import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
2
- import { BucketDescription, BucketPriority, HydratedSyncRules, SqliteJsonValue } from '@powersync/service-sync-rules';
2
+ import {
3
+ BucketDescription,
4
+ BucketPriority,
5
+ HydratedSyncRules,
6
+ ResolvedBucket,
7
+ SqliteJsonValue
8
+ } from '@powersync/service-sync-rules';
3
9
 
4
10
  import { AbortError } from 'ix/aborterror.js';
5
11
 
@@ -179,7 +185,7 @@ async function* streamResponseInner(
179
185
  // receive a sync complete message after the synchronization is done (which happens in the last
180
186
  // bucketDataInBatches iteration). Without any batch, the line is missing and clients might not complete their
181
187
  // sync properly.
182
- const priorityBatches: [BucketPriority | null, BucketDescription[]][] = bucketsByPriority;
188
+ const priorityBatches: [BucketPriority | null, ResolvedBucket[]][] = bucketsByPriority;
183
189
  if (priorityBatches.length == 0) {
184
190
  priorityBatches.push([null, []]);
185
191
  }
@@ -257,7 +263,7 @@ interface BucketDataRequest {
257
263
  /** Contains current bucket state. Modified by the request as data is sent. */
258
264
  checkpointLine: CheckpointLine;
259
265
  /** Subset of checkpointLine.bucketsToFetch, filtered by priority. */
260
- bucketsToFetch: BucketDescription[];
266
+ bucketsToFetch: ResolvedBucket[];
261
267
  /** Whether data lines should be encoded in a legacy format where {@link util.OplogEntry.data} is a nested object. */
262
268
  legacyDataLines: boolean;
263
269
  /** Signals that the connection was aborted and that streaming should stop ASAP. */
package/src/sync/util.ts CHANGED
@@ -183,6 +183,16 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
183
183
  );
184
184
  }
185
185
 
186
+ export function unsettledPromise<T>(settled: Promise<PromiseSettledResult<T>>): Promise<T> {
187
+ return settled.then((result) => {
188
+ if (result.status === 'fulfilled') {
189
+ return Promise.resolve(result.value);
190
+ } else {
191
+ return Promise.reject(result.reason);
192
+ }
193
+ });
194
+ }
195
+
186
196
  export type MapOrSet<T> = Map<T, any> | Set<T>;
187
197
 
188
198
  /**
@@ -7,7 +7,7 @@ import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
7
7
  import { KeyResult } from '../../src/auth/KeyCollector.js';
8
8
  import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
9
9
  import { JwtPayload, StaticSupabaseKeyCollector } from '@/index.js';
10
- import { debugKeyNotFound } from '../../src/auth/utils.js';
10
+ import { debugKeyNotFound, getSupabaseJwksUrl } from '../../src/auth/utils.js';
11
11
 
12
12
  const publicKeyRSA: jose.JWK = {
13
13
  use: 'sig',
@@ -496,6 +496,25 @@ describe('JWT Auth', () => {
496
496
  expect(verified.parsedPayload.claim).toEqual('test-claim-2');
497
497
  });
498
498
 
499
+ test('Supabase connection parsing', () => {
500
+ const details = getSupabaseJwksUrl({
501
+ type: 'postgresql',
502
+ uri: 'postgresql://db.abc123.supabase.co:5432/postgres'
503
+ });
504
+ expect(details).toEqual({
505
+ hostname: 'db.abc123.supabase.co',
506
+ projectId: 'abc123',
507
+ url: 'https://abc123.supabase.co/auth/v1/.well-known/jwks.json'
508
+ });
509
+
510
+ // supabase.com is not a supabase URI
511
+ const other = getSupabaseJwksUrl({
512
+ type: 'postgresql',
513
+ uri: 'postgresql://db.abc123.supabase.com:5432/postgres'
514
+ });
515
+ expect(other).toEqual(null);
516
+ });
517
+
499
518
  describe('debugKeyNotFound', () => {
500
519
  test('Supabase token with legacy auth not configured', async () => {
501
520
  const keys = await StaticSupabaseKeyCollector.importKeys([]);