@powersync/service-module-mongodb-storage 0.0.0-dev-20261005120023 → 0.0.0-dev-20261007110800

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/CHANGELOG.md +21 -12
  2. package/dist/storage/MongoBucketStorage.d.ts +1 -1
  3. package/dist/storage/MongoBucketStorage.js +7 -7
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +4 -2
  6. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  7. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +1 -2
  8. package/dist/storage/implementation/MongoPersistedReplicationStream.js +4 -7
  9. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  10. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  11. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +6 -6
  12. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  13. package/dist/storage/implementation/MongoStorageProvider.js +1 -1
  14. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  15. package/dist/utils/test-utils.d.ts +1 -2
  16. package/dist/utils/test-utils.js +1 -2
  17. package/dist/utils/test-utils.js.map +1 -1
  18. package/package.json +7 -7
  19. package/src/storage/MongoBucketStorage.ts +6 -8
  20. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +8 -1
  21. package/src/storage/implementation/MongoPersistedReplicationStream.ts +4 -12
  22. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +31 -48
  23. package/src/storage/implementation/MongoStorageProvider.ts +1 -1
  24. package/src/utils/test-utils.ts +2 -3
  25. package/test/src/source_table_config.test.ts +17 -59
  26. package/test/src/storage_sync.test.ts +3 -4
  27. package/tsconfig.tsbuildinfo +1 -1
@@ -98,10 +98,9 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
98
98
 
99
99
  constructor(
100
100
  db: PowerSyncMongo,
101
- syncConfigParser: storage.SyncConfigParser,
102
101
  private options: MongoBucketStorageOptions
103
102
  ) {
104
- super({ syncConfigParser });
103
+ super();
105
104
  this.chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
106
105
  options.chunkCompactionConcurrency,
107
106
  options.objectStorage != null
@@ -424,7 +423,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
424
423
  };
425
424
 
426
425
  await this.db.sync_rules.insertOne(doc, { session });
427
- const rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc], this.syncConfigParser);
426
+ const rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc]);
428
427
  if (options.lock) {
429
428
  // We only lock when creating a new stream - otherwise we'll likely get lock contention
430
429
  // from an existing job on the stream, which would fail the entire transaction.
@@ -557,8 +556,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
557
556
  ...existing,
558
557
  sync_configs: syncConfigStates
559
558
  },
560
- [...existingConfigDocs, syncConfigDoc],
561
- this.syncConfigParser
559
+ [...existingConfigDocs, syncConfigDoc]
562
560
  );
563
561
  // The stream already exists, so an active replication job may already hold the stream lock.
564
562
  // Deployment only persists the appended sync config; replication job locking is handled by
@@ -641,7 +639,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
641
639
  };
642
640
 
643
641
  await this.db.sync_rules.insertOne(doc, { session });
644
- const rules = new MongoPersistedReplicationStream(this.db, doc, [], this.syncConfigParser);
642
+ const rules = new MongoPersistedReplicationStream(this.db, doc);
645
643
  if (options.lock) {
646
644
  // The lock is persisted on rules.current_lock
647
645
  await rules.lock(session);
@@ -733,10 +731,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
733
731
  if (syncConfigDocs.length == 0) {
734
732
  return null;
735
733
  }
736
- return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs, this.syncConfigParser);
734
+ return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs);
737
735
  }
738
736
 
739
- return new MongoPersistedReplicationStream(this.db, doc as SyncRuleDocumentV1, [], this.syncConfigParser);
737
+ return new MongoPersistedReplicationStream(this.db, doc as SyncRuleDocumentV1);
740
738
  }
741
739
 
742
740
  async getDeployingSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
@@ -17,6 +17,7 @@ import {
17
17
  HydratedSyncConfig,
18
18
  HydrationState,
19
19
  nodeSqlite,
20
+ PrecompiledSyncConfig,
20
21
  sourceTableConfigsEqual,
21
22
  SyncConfigWithErrors,
22
23
  versionedHydrationState
@@ -59,7 +60,13 @@ export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet {
59
60
  // Active and processing configs share one source reader and must use the same source-table options.
60
61
  if (
61
62
  this.syncConfigs.some(
62
- ({ config }) => !sourceTableConfigsEqual(firstConfig.config.sourceTableConfig, config.sourceTableConfig)
63
+ ({ config }) =>
64
+ !sourceTableConfigsEqual(
65
+ firstConfig.config instanceof PrecompiledSyncConfig
66
+ ? (firstConfig.config.plan.sourceTableConfig ?? {})
67
+ : {},
68
+ config instanceof PrecompiledSyncConfig ? (config.plan.sourceTableConfig ?? {}) : {}
69
+ )
63
70
  )
64
71
  ) {
65
72
  throw new ReplicationAssertionError(
@@ -21,8 +21,7 @@ export class MongoPersistedReplicationStream extends storage.PersistedReplicatio
21
21
  constructor(
22
22
  private readonly db: PowerSyncMongo,
23
23
  private readonly doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3,
24
- private readonly configs: SyncConfigDefinition[],
25
- private readonly syncConfigParser: storage.SyncConfigParser
24
+ private readonly configs: SyncConfigDefinition[] = []
26
25
  ) {
27
26
  const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION;
28
27
  const replicationJobId =
@@ -54,17 +53,11 @@ export class MongoPersistedReplicationStream extends storage.PersistedReplicatio
54
53
  throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`);
55
54
  }
56
55
  return this.configs.map(
57
- (config) =>
58
- new MongoPersistedSyncConfigContentV3(
59
- this.db,
60
- this.doc as ReplicationStreamDocumentV3,
61
- config,
62
- this.syncConfigParser
63
- )
56
+ (config) => new MongoPersistedSyncConfigContentV3(this.db, this.doc as ReplicationStreamDocumentV3, config)
64
57
  );
65
58
  }
66
59
 
67
- return [new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1, this.syncConfigParser)];
60
+ return [new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1)];
68
61
  }
69
62
 
70
63
  get syncConfigIds(): bson.ObjectId[] {
@@ -103,8 +96,7 @@ export class MongoPersistedReplicationStream extends storage.PersistedReplicatio
103
96
  content: config.content,
104
97
  compiledPlan: config.serialized_plan ?? null,
105
98
  storageVersion: this.storageVersion,
106
- parseOptions: options,
107
- syncConfigParser: this.syncConfigParser
99
+ parseOptions: options
108
100
  }),
109
101
  mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping)
110
102
  };
@@ -20,17 +20,13 @@ export abstract class MongoPersistedSyncConfigContentBase extends storage.Persis
20
20
  options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
21
21
  mapping: SingleSyncConfigBucketDefinitionMapping;
22
22
  syncConfigId: bson.ObjectId | null;
23
- },
24
- syncConfigParser: storage.SyncConfigParser
23
+ }
25
24
  ) {
26
25
  const { mapping, syncConfigId, ...base } = options;
27
- super(
28
- {
29
- ...base,
30
- syncConfigId: syncConfigId?.toHexString() ?? null
31
- },
32
- syncConfigParser
33
- );
26
+ super({
27
+ ...base,
28
+ syncConfigId: syncConfigId?.toHexString() ?? null
29
+ });
34
30
  this.mapping = mapping;
35
31
  this.syncConfigObjectId = syncConfigId;
36
32
  }
@@ -64,23 +60,19 @@ export abstract class MongoPersistedSyncConfigContentBase extends storage.Persis
64
60
  }
65
61
  }
66
62
  export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
67
- constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1, syncConfigParser: storage.SyncConfigParser) {
68
- super(
69
- db,
70
- {
71
- replicationStreamId: doc._id,
72
- sync_rules_content: doc.content,
73
- compiled_plan: doc.serialized_plan ?? null,
74
- // Handle legacy values
75
- replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
76
- storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
77
- mapping: new SingleSyncConfigBucketDefinitionMapping(),
78
- syncConfigId: null,
79
- syncConfigState: doc.state,
80
- version_label: doc.version_label ?? undefined
81
- },
82
- syncConfigParser
83
- );
63
+ constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
64
+ super(db, {
65
+ replicationStreamId: doc._id,
66
+ sync_rules_content: doc.content,
67
+ compiled_plan: doc.serialized_plan ?? null,
68
+ // Handle legacy values
69
+ replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
70
+ storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
71
+ mapping: new SingleSyncConfigBucketDefinitionMapping(),
72
+ syncConfigId: null,
73
+ syncConfigState: doc.state,
74
+ version_label: doc.version_label ?? undefined
75
+ });
84
76
  }
85
77
 
86
78
  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
@@ -95,32 +87,23 @@ export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigC
95
87
  export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
96
88
  declare public readonly syncConfigObjectId: bson.ObjectId;
97
89
 
98
- constructor(
99
- db: PowerSyncMongo,
100
- doc: ReplicationStreamDocumentV3,
101
- config: SyncConfigDefinition,
102
- syncConfigParser: storage.SyncConfigParser
103
- ) {
90
+ constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) {
104
91
  const state = doc.sync_configs.find((c) => c._id.equals(config._id));
105
92
  if (state == null) {
106
93
  throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
107
94
  }
108
- super(
109
- db,
110
- {
111
- replicationStreamId: doc._id,
112
- sync_rules_content: config.content,
113
- compiled_plan: config.serialized_plan ?? null,
114
-
115
- replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
116
- storageVersion: doc.storage_version,
117
- mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
118
- syncConfigId: config._id,
119
- syncConfigState: state.state,
120
- version_label: config.version_label
121
- },
122
- syncConfigParser
123
- );
95
+ super(db, {
96
+ replicationStreamId: doc._id,
97
+ sync_rules_content: config.content,
98
+ compiled_plan: config.serialized_plan ?? null,
99
+
100
+ replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
101
+ storageVersion: doc.storage_version,
102
+ mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
103
+ syncConfigId: config._id,
104
+ syncConfigState: state.state,
105
+ version_label: config.version_label
106
+ });
124
107
  }
125
108
 
126
109
  async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
@@ -74,7 +74,7 @@ export class MongoStorageProvider implements storage.StorageProvider {
74
74
  // 90 is the minimum value.
75
75
  maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
76
76
  });
77
- const syncStorageFactory = new MongoBucketStorage(database, options.syncConfigParser, {
77
+ const syncStorageFactory = new MongoBucketStorage(database, {
78
78
  chunkCompactionConcurrency,
79
79
  replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
80
80
  readPreference,
@@ -1,5 +1,5 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
- import { SqlSyncConfigParser, SyncConfigParser, TestStorageOptions } from '@powersync/service-core';
2
+ import { TestStorageOptions } from '@powersync/service-core';
3
3
  import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js';
4
4
  import { MongoReportStorage } from '../storage/MongoReportStorage.js';
5
5
  import { PowerSyncMongo } from '../storage/implementation/db.js';
@@ -7,7 +7,6 @@ import { PowerSyncMongo } from '../storage/implementation/db.js';
7
7
  export type MongoTestStorageOptions = {
8
8
  url: string;
9
9
  isCI: boolean;
10
- syncConfigParser?: SyncConfigParser;
11
10
  monitorCommands?: boolean;
12
11
  } & Omit<MongoBucketStorageOptions, 'replicationStreamNamePrefix'>;
13
12
 
@@ -28,7 +27,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag
28
27
  await db.clear();
29
28
  }
30
29
 
31
- return new MongoBucketStorage(db, factoryOptions.syncConfigParser ?? new SqlSyncConfigParser(), {
30
+ return new MongoBucketStorage(db, {
32
31
  replicationStreamNamePrefix: 'test_',
33
32
  checksumOptions: factoryOptions.checksumOptions,
34
33
  supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
@@ -1,11 +1,7 @@
1
1
  import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
2
- import { SqlSyncConfigParser, updateSyncRulesFromConfig } from '@powersync/service-core';
2
+ import { updateSyncRulesFromConfig } from '@powersync/service-core';
3
3
  import { test_utils } from '@powersync/service-core-tests';
4
- import {
5
- AdditionalSyncConfigParser,
6
- normalizeSourceTableConfig,
7
- PrecompiledSyncConfig
8
- } from '@powersync/service-sync-rules';
4
+ import { PrecompiledSyncConfig, SqlSyncRules } from '@powersync/service-sync-rules';
9
5
  import { describe, expect, test } from 'vitest';
10
6
  import { env } from './env.js';
11
7
  import { TEST_STORAGE_VERSIONS } from './util.js';
@@ -24,81 +20,43 @@ const CONFIGURED = /* yaml */ `
24
20
  edition: 3
25
21
  source_table_options:
26
22
  orders:
27
- sample: 10
23
+ mongodb_filter_expression: { $eq: ['$$doc.active', true] }
28
24
  streams:
29
25
  orders:
30
26
  query: SELECT * FROM orders
31
27
  `;
32
28
 
33
- // A small module-owned option exercises persistence without introducing MongoDB filter syntax into core.
34
- const TABLE_OPTIONS: AdditionalSyncConfigParser = {
35
- id: 'example.tables',
36
- extendJsonSchema({ schema }) {
37
- const options = (schema.properties as any).config.properties.source_table_options.additionalProperties;
38
- options.properties.sample = { type: 'number' };
39
- },
40
- parse({ config, context }) {
41
- const sourceTables = normalizeSourceTableConfig(
42
- (config as { config?: { source_table_options?: unknown } }).config?.source_table_options
43
- );
44
- for (const [table, options] of Object.entries(sourceTables)) {
45
- if (!Object.hasOwn(options!, 'sample')) continue;
46
- context.parsedConfig.sourceTableConfig = { ...context.parsedConfig.sourceTableConfig, [table]: options };
47
- const { plan } = context.parsedConfig as PrecompiledSyncConfig;
48
- plan.moduleData = { ...plan.moduleData, ['example.tables']: null };
49
- }
50
- }
51
- };
52
-
53
- function storageFactory(withModule: boolean) {
54
- const syncConfigParser = new SqlSyncConfigParser(withModule ? [TABLE_OPTIONS] : []);
29
+ function storageFactory() {
55
30
  return mongoTestStorageFactoryGenerator({
56
31
  url: env.MONGO_TEST_URL,
57
32
  isCI: env.CI,
58
- supportsMultipleSyncConfigs: true,
59
- syncConfigParser
33
+ supportsMultipleSyncConfigs: true
60
34
  });
61
35
  }
62
36
 
63
37
  describe.for(TEST_STORAGE_VERSIONS)('source-table config with storage v%i', (storageVersion) => {
64
- test('preserves module options across closing storage and loading its compiled plan', async () => {
38
+ test('preserves source-table options across closing storage and loading its compiled plan', async () => {
65
39
  let expected;
66
40
  {
67
- await using factory = await storageFactory(true).factory();
68
- const parsed = factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS);
69
- expected = parsed.config.sourceTableConfig;
41
+ await using factory = await storageFactory().factory();
42
+ const parsed = SqlSyncRules.fromYaml(CONFIGURED, test_utils.PARSE_OPTIONS);
43
+ expected = (parsed.config as PrecompiledSyncConfig).plan.sourceTableConfig;
70
44
  await factory.updateSyncRules(updateSyncRulesFromConfig(parsed, { storageVersion }));
71
45
  }
72
46
 
73
47
  // Reopen the same database without clearing it, as a service restart would.
74
- await using reopened = await storageFactory(true).factory({ doNotClear: true });
48
+ await using reopened = await storageFactory().factory({ doNotClear: true });
75
49
  const deploying = (await reopened.getDeployingSyncConfig())!;
76
50
  expect(deploying.content.compiled_plan?.plan.version).toBe(3);
77
51
  const parsed = deploying.content.parsed(test_utils.PARSE_OPTIONS);
78
- expect(parsed.syncConfigs[0].config.sourceTableConfig).toEqual(expected);
52
+ expect((parsed.syncConfigs[0].config as PrecompiledSyncConfig).plan.sourceTableConfig).toEqual(expected);
79
53
  expect(parsed.hydratedSyncConfig.sourceTableConfig).toEqual(expected);
80
54
  });
81
55
 
82
- test('rejects saved options if the module is absent after restart', async () => {
83
- {
84
- await using factory = await storageFactory(true).factory();
85
- await factory.updateSyncRules(
86
- updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(CONFIGURED, test_utils.PARSE_OPTIONS), {
87
- storageVersion
88
- })
89
- );
90
- }
91
- await using reopened = await storageFactory(false).factory({ doNotClear: true });
92
- const deploying = (await reopened.getDeployingSyncConfig())!;
93
- expect(() => deploying.content.parsed(test_utils.PARSE_OPTIONS)).toThrow(
94
- 'Missing required sync config parsers: example.tables'
95
- );
96
- });
97
-
98
- test('continues loading unfiltered plans without an additional parser', async () => {
99
- await using factory = await storageFactory(false).factory();
56
+ test('continues loading unfiltered plans without source options', async () => {
57
+ await using factory = await storageFactory().factory();
100
58
  await factory.updateSyncRules(
101
- updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(STREAMS, test_utils.PARSE_OPTIONS), {
59
+ updateSyncRulesFromConfig(SqlSyncRules.fromYaml(STREAMS, test_utils.PARSE_OPTIONS), {
102
60
  storageVersion
103
61
  })
104
62
  );
@@ -108,10 +66,10 @@ describe.for(TEST_STORAGE_VERSIONS)('source-table config with storage v%i', (sto
108
66
  });
109
67
 
110
68
  test('starts replacement processing when source-table options change', async () => {
111
- await using factory = await storageFactory(true).factory();
69
+ await using factory = await storageFactory().factory();
112
70
  const deploy = (yaml: string) =>
113
71
  factory.updateSyncRules(
114
- updateSyncRulesFromConfig(factory.syncConfigParser.parseContent(yaml, test_utils.PARSE_OPTIONS), {
72
+ updateSyncRulesFromConfig(SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS), {
115
73
  storageVersion
116
74
  })
117
75
  );
@@ -132,7 +90,7 @@ describe.for(TEST_STORAGE_VERSIONS)('source-table config with storage v%i', (sto
132
90
  await writer.markAllSnapshotDone('2/1');
133
91
  await writer.commit('2/1');
134
92
  }
135
- const changed = await deploy(CONFIGURED.replace('sample: 10', 'sample: 20'));
93
+ const changed = await deploy(CONFIGURED.replace('true', 'false'));
136
94
  expect(changed.replicationStreamId).not.toBe(first.replicationStreamId);
137
95
  expect((await factory.getActiveSyncConfig())?.content.replicationStreamId).toBe(first.replicationStreamId);
138
96
  });
@@ -73,7 +73,8 @@ function objectIdGenerator(id: string) {
73
73
  */
74
74
  function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
75
75
  const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
76
- expect(parsed.errors).toEqual([]);
76
+ // These fixtures are legacy Sync Rules, which always report a deprecation warning.
77
+ expect(parsed.errors.filter((error) => error.type !== 'warning')).toEqual([]);
77
78
  return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
78
79
  {
79
80
  syncConfigId: new bson.ObjectId().toHexString(),
@@ -99,9 +100,7 @@ async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory,
99
100
  })
100
101
  .toArray();
101
102
 
102
- return syncConfigDocs.map(
103
- (config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config, mongoFactory.syncConfigParser)
104
- );
103
+ return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
105
104
  }
106
105
 
107
106
  /**