@powersync/service-module-mongodb 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/benchmarks/env.ts +11 -0
- package/benchmarks/replication_branch_throughput.benchmark.ts +240 -0
- package/ci/test-connection.yaml +12 -0
- package/dist/api/MongoRouteAPIAdapter.d.ts +6 -0
- package/dist/api/MongoRouteAPIAdapter.js +33 -4
- package/dist/api/MongoRouteAPIAdapter.js.map +1 -1
- package/dist/api/infer-collection-schema.d.ts +5 -0
- package/dist/api/infer-collection-schema.js +24 -0
- package/dist/api/infer-collection-schema.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/module/MongoModule.d.ts +1 -2
- package/dist/module/MongoModule.js +1 -1
- package/dist/module/MongoModule.js.map +1 -1
- package/dist/replication/ChangeStream.d.ts +13 -3
- package/dist/replication/ChangeStream.js +130 -196
- package/dist/replication/ChangeStream.js.map +1 -1
- package/dist/replication/ChangeStreamReplicationJob.d.ts +4 -1
- package/dist/replication/ChangeStreamReplicationJob.js +19 -8
- package/dist/replication/ChangeStreamReplicationJob.js.map +1 -1
- package/dist/replication/ChangeStreamReplicator.d.ts +4 -0
- package/dist/replication/ChangeStreamReplicator.js +6 -0
- package/dist/replication/ChangeStreamReplicator.js.map +1 -1
- package/dist/replication/MongoRelation.js +5 -2
- package/dist/replication/MongoRelation.js.map +1 -1
- package/dist/replication/MongoReplicationQueryProvider.d.ts +85 -0
- package/dist/replication/MongoReplicationQueryProvider.js +23 -0
- package/dist/replication/MongoReplicationQueryProvider.js.map +1 -0
- package/dist/replication/MongoReplicationStream.d.ts +74 -0
- package/dist/replication/MongoReplicationStream.js +228 -0
- package/dist/replication/MongoReplicationStream.js.map +1 -0
- package/dist/replication/MongoReplicationStreamTypes.d.ts +157 -0
- package/dist/replication/MongoReplicationStreamTypes.js +19 -0
- package/dist/replication/MongoReplicationStreamTypes.js.map +1 -0
- package/dist/replication/MongoSnapshotQuery.d.ts +2 -0
- package/dist/replication/MongoSnapshotQuery.js +12 -2
- package/dist/replication/MongoSnapshotQuery.js.map +1 -1
- package/dist/replication/MongoSnapshotter.d.ts +7 -2
- package/dist/replication/MongoSnapshotter.js +90 -88
- package/dist/replication/MongoSnapshotter.js.map +1 -1
- package/dist/replication/RawChangeStream.d.ts +1 -1
- package/dist/replication/RawChangeStream.js +1 -1
- package/dist/replication/RawChangeStream.js.map +1 -1
- package/dist/replication/SourceRowConverter.d.ts +1 -0
- package/dist/replication/SourceRowConverter.js +4 -2
- package/dist/replication/SourceRowConverter.js.map +1 -1
- package/dist/replication/bufferToSqlite.d.ts +3 -1
- package/dist/replication/bufferToSqlite.js +35 -23
- package/dist/replication/bufferToSqlite.js.map +1 -1
- package/dist/replication/estimateFilteredSnapshotCount.d.ts +12 -0
- package/dist/replication/estimateFilteredSnapshotCount.js +79 -0
- package/dist/replication/estimateFilteredSnapshotCount.js.map +1 -0
- package/dist/replication/replication-index.d.ts +7 -0
- package/dist/replication/replication-index.js +7 -0
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/test-utils/ChangeStreamTestContext.d.ts +131 -0
- package/dist/test-utils/ChangeStreamTestContext.js +390 -0
- package/dist/test-utils/ChangeStreamTestContext.js.map +1 -0
- package/dist/test-utils/test-utils-index.d.ts +1 -0
- package/dist/test-utils/test-utils-index.js +2 -0
- package/dist/test-utils/test-utils-index.js.map +1 -0
- package/package.json +11 -10
- package/src/api/MongoRouteAPIAdapter.ts +38 -4
- package/src/api/infer-collection-schema.ts +26 -0
- package/src/index.ts +1 -0
- package/src/module/MongoModule.ts +2 -3
- package/src/replication/ChangeStream.ts +143 -226
- package/src/replication/ChangeStreamReplicationJob.ts +21 -9
- package/src/replication/ChangeStreamReplicator.ts +9 -0
- package/src/replication/MongoRelation.ts +8 -1
- package/src/replication/MongoReplicationQueryProvider.ts +106 -0
- package/src/replication/MongoReplicationStream.ts +314 -0
- package/src/replication/MongoReplicationStreamTypes.ts +169 -0
- package/src/replication/MongoSnapshotQuery.ts +19 -3
- package/src/replication/MongoSnapshotter.ts +101 -92
- package/src/replication/RawChangeStream.ts +2 -2
- package/src/replication/SourceRowConverter.ts +4 -2
- package/src/replication/bufferToSqlite.ts +43 -23
- package/src/replication/estimateFilteredSnapshotCount.ts +93 -0
- package/src/replication/replication-index.ts +7 -0
- package/src/test-utils/ChangeStreamTestContext.ts +520 -0
- package/src/test-utils/test-utils-index.ts +1 -0
- package/test/src/buffer_to_sqlite.test.ts +9 -0
- package/test/src/change_event_memory.test.ts +33 -0
- package/test/src/change_stream.test.ts +3 -3
- package/test/src/change_stream_abort.test.ts +110 -0
- package/test/src/change_stream_test_setup.ts +38 -0
- package/test/src/chunked_snapshot.test.ts +3 -3
- package/test/src/documentdb_mode.test.ts +3 -3
- package/test/src/filtering_example.test.ts +65 -0
- package/test/src/filtering_lifecycle.test.ts +101 -0
- package/test/src/query_provider_availability.test.ts +77 -0
- package/test/src/query_provider_diagnostics.test.ts +55 -0
- package/test/src/replication_stream.test.ts +688 -0
- package/test/src/resume.test.ts +10 -4
- package/test/src/resuming_snapshots.test.ts +3 -3
- package/test/src/schema.test.ts +62 -19
- package/test/src/slow_tests.test.ts +3 -3
- package/test/src/snapshot_count.test.ts +146 -0
- package/test/src/snapshot_count_integration.test.ts +70 -0
- package/test/src/snapshot_query_provider.test.ts +122 -0
- package/test/src/stream_progress.test.ts +334 -0
- package/test/tsconfig.json +2 -2
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.benchmark.config.ts +15 -0
- package/test/src/change_stream_utils.ts +0 -375
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,79 @@
|
|
|
1
1
|
# @powersync/service-module-mongodb
|
|
2
2
|
|
|
3
|
+
## 0.22.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 44ce423: Add `config.source_table_options` to edition 3 sync configs, including MongoDB pre-filtering expressions. Generate editor schemas with examples and report validation errors at their YAML source locations.
|
|
8
|
+
|
|
9
|
+
MongoDB replication pre-filtering is not supported in the Open Edition; unsupported configurations report `PSYNC_S1348`. For example:
|
|
10
|
+
|
|
11
|
+
```yaml
|
|
12
|
+
config:
|
|
13
|
+
edition: 3
|
|
14
|
+
source_table_options:
|
|
15
|
+
orders:
|
|
16
|
+
mongodb_filter_expression:
|
|
17
|
+
$and:
|
|
18
|
+
- $eq: ['$$doc.active', true]
|
|
19
|
+
- $in: ['$$doc.status', ['pending', 'shipped']]
|
|
20
|
+
streams:
|
|
21
|
+
orders:
|
|
22
|
+
query: SELECT * FROM orders
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Validation and deployment reject filters when the feature is unavailable. Parsing accepts an optional map of connection tags to connection types, rejecting MongoDB pre-filtering expressions on known non-MongoDB sources. Unknown connections or missing connection metadata produce advisory warnings. The service supplies connection metadata during validation, deployment, and filesystem loading. Filesystem-loaded sync configs also run source-capability checks. With `exit_on_error` disabled, configs remain available to diagnostics. Changing source-table options requires replacement processing when deploying a new sync config.
|
|
26
|
+
|
|
27
|
+
Source capability validation returns structured `ValidationDiagnostic` entries, allowing multiple fatal issues and
|
|
28
|
+
advisory warnings. Deployment and reprocessing reject fatal diagnostics while validation continues collecting
|
|
29
|
+
independent findings. Existing `ReplicationError` consumers retain the same diagnostic shape.
|
|
30
|
+
|
|
31
|
+
- fff5024: Add a replication adapter interface for snapshot predicates and ChangeStream event batches. Both default and filtering providers parse events incrementally with shared split-event assembly. Persist safe resume positions after preceding writes have flushed, including when an adapter excludes all data events. Preserve exact-token resumes within transactions and export a pluggable, disposable ChangeStreamTestContext for module integration tests.
|
|
32
|
+
|
|
33
|
+
Expose a shared filtering helper with typed options and keep/exclude/delete decisions. The helper preserves checkpoint and DDL events, source batch boundaries and safe resume progress while counting exclusions. Export the shared stream and filter types from the replication module.
|
|
34
|
+
|
|
35
|
+
Report a replication error if a saved sync config requires MongoDB pre-filtering but no query provider factory is registered. Restore the required module, or remove the filters and deploy a new sync config to restart replication.
|
|
36
|
+
|
|
37
|
+
### Patch Changes
|
|
38
|
+
|
|
39
|
+
- f6ba406: Estimate filtered MongoDB snapshot totals with a total count, falling back to a bounded sample when the count fails or times out.
|
|
40
|
+
- Updated dependencies [44ce423]
|
|
41
|
+
- Updated dependencies [51a1a89]
|
|
42
|
+
- Updated dependencies [e703ca5]
|
|
43
|
+
- Updated dependencies [324093e]
|
|
44
|
+
- Updated dependencies [286b0f8]
|
|
45
|
+
- Updated dependencies [51a1a89]
|
|
46
|
+
- Updated dependencies [57ef2f4]
|
|
47
|
+
- Updated dependencies [6700c71]
|
|
48
|
+
- @powersync/service-sync-rules@0.44.0
|
|
49
|
+
- @powersync/service-core@1.27.1
|
|
50
|
+
- @powersync/service-types@0.20.0
|
|
51
|
+
- @powersync/lib-services-framework@0.10.4
|
|
52
|
+
- @powersync/lib-service-mongodb@0.6.34
|
|
53
|
+
|
|
54
|
+
## 0.21.2
|
|
55
|
+
|
|
56
|
+
### Patch Changes
|
|
57
|
+
|
|
58
|
+
- 155febf: Report a MongoDB collection without columns when sampling it exceeds the query memory limit, instead of failing the whole schema request. Sampling a small collection of large documents can hit this limit, which previously blocked sync config validation and deployment for the entire connection.
|
|
59
|
+
- 54d6f4b: Report a MongoDB view matched by a wildcard table pattern as a warning, instead of failing sync config validation. Previously, a view matching a pattern such as `"orders%"` made the whole validation request fail with `Cannot get name for wildcard table`.
|
|
60
|
+
- 18f0c95: Add `fixed_booleans_in_json` to sync booleans in JSON columns as booleans instead of as 0/1 integer.
|
|
61
|
+
- Updated dependencies [0483d4e]
|
|
62
|
+
- Updated dependencies [28f024b]
|
|
63
|
+
- Updated dependencies [5078032]
|
|
64
|
+
- Updated dependencies [424bbc0]
|
|
65
|
+
- Updated dependencies [e87703a]
|
|
66
|
+
- Updated dependencies [b70f467]
|
|
67
|
+
- Updated dependencies [18f0c95]
|
|
68
|
+
- Updated dependencies [c60db6e]
|
|
69
|
+
- Updated dependencies [0483d4e]
|
|
70
|
+
- Updated dependencies [a8c3f15]
|
|
71
|
+
- @powersync/service-core@1.27.0
|
|
72
|
+
- @powersync/service-types@0.19.0
|
|
73
|
+
- @powersync/service-sync-rules@0.43.0
|
|
74
|
+
- @powersync/lib-services-framework@0.10.3
|
|
75
|
+
- @powersync/lib-service-mongodb@0.6.33
|
|
76
|
+
|
|
3
77
|
## 0.21.1
|
|
4
78
|
|
|
5
79
|
### Patch Changes
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { utils } from '@powersync/lib-services-framework';
|
|
2
|
+
|
|
3
|
+
const POSITIVE_INTEGER = utils.type.string
|
|
4
|
+
.transform(Number)
|
|
5
|
+
.refine((value) => Number.isSafeInteger(value) && value > 0, 'Must be a positive safe integer');
|
|
6
|
+
|
|
7
|
+
export const ENV = utils.collectEnvironmentVariables({
|
|
8
|
+
MEMORY_EAGER: utils.type.boolean.default('false'),
|
|
9
|
+
THROUGHPUT_ROWS: POSITIVE_INTEGER.default('2000'),
|
|
10
|
+
THROUGHPUT_PAYLOAD_BYTES: POSITIVE_INTEGER.default('1048576')
|
|
11
|
+
});
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { createCheckpoint, STANDALONE_CHECKPOINT_ID } from '@module/replication/MongoRelation.js';
|
|
2
|
+
import { DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER } from '@module/replication/MongoReplicationQueryProvider.js';
|
|
3
|
+
import { createMongoReplicationStreamBatch } from '@module/replication/MongoReplicationStream.js';
|
|
4
|
+
import { isBatchEnd, unsettledPromise } from '@powersync/service-core';
|
|
5
|
+
import { bucketRequest } from '@powersync/service-core-tests';
|
|
6
|
+
import { performance } from 'node:perf_hooks';
|
|
7
|
+
import { setTimeout } from 'node:timers/promises';
|
|
8
|
+
import { isMainThread } from 'node:worker_threads';
|
|
9
|
+
import { describe, expect, test } from 'vitest';
|
|
10
|
+
import { openChangeStreamTestContext } from '../test/src/change_stream_test_setup.js';
|
|
11
|
+
import { describeWithStorage, StorageVersionTestContext } from '../test/src/util.js';
|
|
12
|
+
import { ENV } from './env.js';
|
|
13
|
+
|
|
14
|
+
const EAGER = ENV.MEMORY_EAGER;
|
|
15
|
+
const ROW_COUNT = ENV.THROUGHPUT_ROWS;
|
|
16
|
+
const PAYLOAD_BYTES = ENV.THROUGHPUT_PAYLOAD_BYTES;
|
|
17
|
+
const PAYLOAD = 'x'.repeat(PAYLOAD_BYTES);
|
|
18
|
+
const BATCH_SIZE = 1_000;
|
|
19
|
+
const INSERT_BATCH_SIZE = Math.max(1, Math.min(BATCH_SIZE, Math.floor((8 * 1024 * 1024) / (PAYLOAD_BYTES + 64))));
|
|
20
|
+
const MEMORY_FIELDS = ['rss', 'heapUsed', 'arrayBuffers'] as const;
|
|
21
|
+
|
|
22
|
+
const CONFIG = /* yaml */ `
|
|
23
|
+
# Same workload for both parsing modes.
|
|
24
|
+
config:
|
|
25
|
+
edition: 3
|
|
26
|
+
streams:
|
|
27
|
+
global:
|
|
28
|
+
query: SELECT _id AS id, payload FROM documents
|
|
29
|
+
`;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Run one replay in a fresh Node process using Vitest's forks pool, so RSS excludes the parent runner.
|
|
33
|
+
* From this module: TEST_POSTGRES_STORAGE=false pnpm benchmark:replication -t 'storage v2'
|
|
34
|
+
* The benchmark config supplies the forks pool, GC flag and serial execution.
|
|
35
|
+
* Repeat separately with MEMORY_EAGER=true. Run sequentially against otherwise idle test databases.
|
|
36
|
+
* Defaults to 2,000 rows with 1 MiB payloads; THROUGHPUT_ROWS and THROUGHPUT_PAYLOAD_BYTES override them.
|
|
37
|
+
* This measures replay plus storage writes, not just parsing. MongoDB server memory is excluded.
|
|
38
|
+
*/
|
|
39
|
+
describe('replication throughput and memory', () => {
|
|
40
|
+
describeWithStorage({ timeout: 300_000 }, (storage) => {
|
|
41
|
+
test('measures one backlog replay', async () => {
|
|
42
|
+
if (!isMainThread || typeof globalThis.gc != 'function') {
|
|
43
|
+
throw new Error('Run with --pool=forks --execArgv=--expose-gc to isolate worker RSS');
|
|
44
|
+
}
|
|
45
|
+
const result = await measure(storage);
|
|
46
|
+
process.stdout.write(
|
|
47
|
+
JSON.stringify(
|
|
48
|
+
{
|
|
49
|
+
mode: EAGER ? 'eager' : 'lazy',
|
|
50
|
+
pid: process.pid,
|
|
51
|
+
rows: ROW_COUNT,
|
|
52
|
+
payloadBytes: PAYLOAD_BYTES,
|
|
53
|
+
...result
|
|
54
|
+
},
|
|
55
|
+
null,
|
|
56
|
+
2
|
|
57
|
+
) + '\n'
|
|
58
|
+
);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
async function measure({ factory, storageVersion }: StorageVersionTestContext) {
|
|
64
|
+
{
|
|
65
|
+
await using setup = await openChangeStreamTestContext(factory, { storageVersion });
|
|
66
|
+
await setup.updateSyncRules(CONFIG);
|
|
67
|
+
await setup.db.createCollection('documents');
|
|
68
|
+
await setup.replicateSnapshot();
|
|
69
|
+
await setup.getCheckpoint();
|
|
70
|
+
// Leaving this scope aborts and awaits replication, then closes its storage and source connections.
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const batchStats = { batches: 0, events: 0, maxEvents: 0 };
|
|
74
|
+
let sampleBoundary = () => {};
|
|
75
|
+
await using context = await openChangeStreamTestContext(factory, {
|
|
76
|
+
storageVersion,
|
|
77
|
+
doNotClear: true,
|
|
78
|
+
streamOptions: {
|
|
79
|
+
snapshotChunkLength: BATCH_SIZE,
|
|
80
|
+
maxAwaitTimeMS: 200,
|
|
81
|
+
createReplicationQueryProvider: () => ({
|
|
82
|
+
...DEFAULT_MONGO_REPLICATION_QUERY_PROVIDER,
|
|
83
|
+
async *openChangeStream({ open }) {
|
|
84
|
+
for await (const batch of open({})) {
|
|
85
|
+
// Both variants use the same wrapper. Only eager mode materializes parsed envelopes.
|
|
86
|
+
const events = EAGER ? [...batch.events] : batch.events;
|
|
87
|
+
sampleBoundary();
|
|
88
|
+
yield createMongoReplicationStreamBatch({
|
|
89
|
+
events: (function* () {
|
|
90
|
+
let count = 0;
|
|
91
|
+
for (const change of events) {
|
|
92
|
+
count++;
|
|
93
|
+
yield change;
|
|
94
|
+
}
|
|
95
|
+
batchStats.batches++;
|
|
96
|
+
batchStats.events += count;
|
|
97
|
+
batchStats.maxEvents = Math.max(batchStats.maxEvents, count);
|
|
98
|
+
})(),
|
|
99
|
+
resolveResumeToken: () => batch.getResumeToken(),
|
|
100
|
+
resolveMetadata: () => batch.metadata
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
})
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
await context.loadActiveSyncRules();
|
|
108
|
+
// This context has not started replication. Populate the full backlog before the timed streaming phase.
|
|
109
|
+
const collection = context.db.collection<{ _id: string; payload: string }>('documents');
|
|
110
|
+
for (let offset = 0; offset < ROW_COUNT; offset += INSERT_BATCH_SIZE) {
|
|
111
|
+
await collection.insertMany(
|
|
112
|
+
Array.from({ length: Math.min(INSERT_BATCH_SIZE, ROW_COUNT - offset) }, (_, index) => ({
|
|
113
|
+
_id: String(offset + index),
|
|
114
|
+
payload: PAYLOAD
|
|
115
|
+
}))
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
// Create the target barrier before starting the measured replay.
|
|
119
|
+
const target = await createCheckpoint(context.db, STANDALONE_CHECKPOINT_ID);
|
|
120
|
+
// Reclaim unreachable setup allocations before sampling. GC runs naturally during replay.
|
|
121
|
+
await setTimeout(0);
|
|
122
|
+
globalThis.gc!();
|
|
123
|
+
await setTimeout(0);
|
|
124
|
+
globalThis.gc!();
|
|
125
|
+
using memoryMonitor = sampleMemory();
|
|
126
|
+
sampleBoundary = memoryMonitor.sample;
|
|
127
|
+
const start = performance.now();
|
|
128
|
+
const replication = context.startStreaming();
|
|
129
|
+
const polling = new AbortController();
|
|
130
|
+
const wait = (async () => {
|
|
131
|
+
while (performance.now() - start < 120_000) {
|
|
132
|
+
const checkpoint = await context.storage!.getCheckpoint();
|
|
133
|
+
if (checkpoint.lsn != null && checkpoint.lsn >= target) {
|
|
134
|
+
return checkpoint;
|
|
135
|
+
}
|
|
136
|
+
await setTimeout(10, undefined, { signal: polling.signal });
|
|
137
|
+
}
|
|
138
|
+
throw new Error('Timed out waiting for the backlog checkpoint');
|
|
139
|
+
})();
|
|
140
|
+
let checkpoint;
|
|
141
|
+
try {
|
|
142
|
+
checkpoint = await Promise.race([
|
|
143
|
+
wait,
|
|
144
|
+
unsettledPromise(replication).then(() => {
|
|
145
|
+
throw new Error('Replication exited before the target checkpoint');
|
|
146
|
+
})
|
|
147
|
+
]);
|
|
148
|
+
} finally {
|
|
149
|
+
polling.abort();
|
|
150
|
+
await wait.catch(() => {});
|
|
151
|
+
}
|
|
152
|
+
const elapsedMs = performance.now() - start;
|
|
153
|
+
const memory = memoryMonitor.finish();
|
|
154
|
+
sampleBoundary = () => {};
|
|
155
|
+
context.abort();
|
|
156
|
+
await replication;
|
|
157
|
+
// Verify incrementally so large payloads do not accumulate in a multi-gigabyte output array.
|
|
158
|
+
const parsed = context.storage!.getParsedSyncConfigSet({ defaultSchema: context.db.databaseName });
|
|
159
|
+
const ids = new Set<string>();
|
|
160
|
+
let rowCount = 0;
|
|
161
|
+
let after = 0n;
|
|
162
|
+
let hasMore = true;
|
|
163
|
+
while (hasMore) {
|
|
164
|
+
hasMore = false;
|
|
165
|
+
const previousAfter = after;
|
|
166
|
+
for await (const chunk of context.storage!.getBucketDataBatch(
|
|
167
|
+
checkpoint,
|
|
168
|
+
[bucketRequest(parsed, 'global|0[]', after)],
|
|
169
|
+
{ limit: 8, chunkLimitBytes: 8 * 1024 * 1024 }
|
|
170
|
+
)) {
|
|
171
|
+
if (isBatchEnd(chunk)) {
|
|
172
|
+
hasMore = chunk.hasMore;
|
|
173
|
+
} else {
|
|
174
|
+
after = BigInt(chunk.chunkData.next_after);
|
|
175
|
+
hasMore = chunk.chunkData.has_more;
|
|
176
|
+
for (const row of chunk.chunkData.data ?? []) {
|
|
177
|
+
if (row.op != 'PUT') {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
rowCount++;
|
|
181
|
+
ids.add(row.object_id!);
|
|
182
|
+
expect(JSON.parse(row.data!).payload).toBe(PAYLOAD);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
if (hasMore && after <= previousAfter) {
|
|
187
|
+
throw new Error('Output verification made no progress');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
expect(rowCount).toBe(ROW_COUNT);
|
|
191
|
+
expect(ids.size).toBe(ROW_COUNT);
|
|
192
|
+
return { elapsedMs, rowsPerSecond: (ROW_COUNT * 1000) / elapsedMs, memory, batchStats };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Sample process memory during replay, plus the provider boundary after eager materialization.
|
|
197
|
+
* Peaks are per-field samples, not exact allocation maxima; RSS can retain setup allocations.
|
|
198
|
+
*/
|
|
199
|
+
function sampleMemory() {
|
|
200
|
+
const baseline = process.memoryUsage();
|
|
201
|
+
const peak = { ...baseline };
|
|
202
|
+
let samples = 1;
|
|
203
|
+
let lastSample = performance.now();
|
|
204
|
+
let maxSampleGapMs = 0;
|
|
205
|
+
function sample() {
|
|
206
|
+
const now = performance.now();
|
|
207
|
+
maxSampleGapMs = Math.max(maxSampleGapMs, now - lastSample);
|
|
208
|
+
lastSample = now;
|
|
209
|
+
const memory = process.memoryUsage();
|
|
210
|
+
for (const field of MEMORY_FIELDS) {
|
|
211
|
+
peak[field] = Math.max(peak[field], memory[field]);
|
|
212
|
+
}
|
|
213
|
+
samples++;
|
|
214
|
+
}
|
|
215
|
+
const timer = setInterval(sample, 25);
|
|
216
|
+
return {
|
|
217
|
+
sample,
|
|
218
|
+
finish() {
|
|
219
|
+
clearInterval(timer);
|
|
220
|
+
sample();
|
|
221
|
+
return {
|
|
222
|
+
samples,
|
|
223
|
+
maxSampleGapMs,
|
|
224
|
+
MiB: Object.fromEntries(
|
|
225
|
+
MEMORY_FIELDS.map((field) => [
|
|
226
|
+
field,
|
|
227
|
+
{
|
|
228
|
+
baseline: baseline[field] / 1024 ** 2,
|
|
229
|
+
peak: peak[field] / 1024 ** 2,
|
|
230
|
+
growth: (peak[field] - baseline[field]) / 1024 ** 2
|
|
231
|
+
}
|
|
232
|
+
])
|
|
233
|
+
)
|
|
234
|
+
};
|
|
235
|
+
},
|
|
236
|
+
[Symbol.dispose]() {
|
|
237
|
+
clearInterval(timer);
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Used by the MongoDB connection CLI check in .github/workflows/test.yml.
|
|
2
|
+
replication:
|
|
3
|
+
connections:
|
|
4
|
+
- type: mongodb
|
|
5
|
+
uri: mongodb://localhost:27017/powersync_test
|
|
6
|
+
|
|
7
|
+
storage:
|
|
8
|
+
type: mongodb
|
|
9
|
+
uri: mongodb://localhost:27017/powersync_test_connection
|
|
10
|
+
|
|
11
|
+
telemetry:
|
|
12
|
+
disable_telemetry_sharing: true
|
|
@@ -13,6 +13,12 @@ export declare class MongoRouteAPIAdapter implements api.RouteAPI {
|
|
|
13
13
|
private documentDbDetection;
|
|
14
14
|
constructor(config: types.ResolvedConnectionConfig);
|
|
15
15
|
getParseSyncRulesOptions(): ParseSyncConfigOptions;
|
|
16
|
+
validateSourceCapabilities(config: sync_rules.SyncConfig): Promise<service_types.ValidationDiagnostic[]>;
|
|
17
|
+
/**
|
|
18
|
+
* Validate MongoDB pre-filtering support. External modules can override this method to account for
|
|
19
|
+
* their plan parameters and filter capabilities while retaining other source-capability checks.
|
|
20
|
+
*/
|
|
21
|
+
protected validateMongoFilters(config: sync_rules.SyncConfig): Promise<service_types.ValidationDiagnostic[]>;
|
|
16
22
|
shutdown(): Promise<void>;
|
|
17
23
|
[Symbol.asyncDispose](): Promise<void>;
|
|
18
24
|
getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig>;
|
|
@@ -2,13 +2,13 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
3
|
import * as sync_rules from '@powersync/service-sync-rules';
|
|
4
4
|
import * as service_types from '@powersync/service-types';
|
|
5
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
+
import { ErrorCode, logger } from '@powersync/lib-services-framework';
|
|
6
6
|
import { createCheckpointImplementation } from '../replication/checkpoints/create-checkpoint-implementation.js';
|
|
7
7
|
import { MongoManager } from '../replication/MongoManager.js';
|
|
8
8
|
import { CHECKPOINTS_COLLECTION, detectDocumentDb } from '../replication/replication-utils.js';
|
|
9
9
|
import * as types from '../types/types.js';
|
|
10
10
|
import { escapeRegExp } from '../utils.js';
|
|
11
|
-
import { inferCollectionSchema } from './infer-collection-schema.js';
|
|
11
|
+
import { inferCollectionSchema, isSchemaInferenceFailure } from './infer-collection-schema.js';
|
|
12
12
|
const SCHEMA_INFERENCE_CONCURRENCY = 4;
|
|
13
13
|
export class MongoRouteAPIAdapter {
|
|
14
14
|
config;
|
|
@@ -28,9 +28,28 @@ export class MongoRouteAPIAdapter {
|
|
|
28
28
|
}
|
|
29
29
|
getParseSyncRulesOptions() {
|
|
30
30
|
return {
|
|
31
|
-
defaultSchema: this.defaultSchema
|
|
31
|
+
defaultSchema: this.defaultSchema,
|
|
32
|
+
connections: { [this.connectionTag]: { type: this.config.type } }
|
|
32
33
|
};
|
|
33
34
|
}
|
|
35
|
+
async validateSourceCapabilities(config) {
|
|
36
|
+
return this.validateMongoFilters(config);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Validate MongoDB pre-filtering support. External modules can override this method to account for
|
|
40
|
+
* their plan parameters and filter capabilities while retaining other source-capability checks.
|
|
41
|
+
*/
|
|
42
|
+
async validateMongoFilters(config) {
|
|
43
|
+
const filtered = sync_rules.hasMongoFilterExpressions(config instanceof sync_rules.PrecompiledSyncConfig ? (config.plan.sourceTableConfig ?? {}) : {}, this.connectionTag);
|
|
44
|
+
return filtered
|
|
45
|
+
? [
|
|
46
|
+
{
|
|
47
|
+
level: 'fatal',
|
|
48
|
+
message: `[${ErrorCode.PSYNC_S1348}] MongoDB replication pre-filtering is not supported in this edition of the PowerSync service.`
|
|
49
|
+
}
|
|
50
|
+
]
|
|
51
|
+
: [];
|
|
52
|
+
}
|
|
34
53
|
async shutdown() {
|
|
35
54
|
await this.client.close();
|
|
36
55
|
}
|
|
@@ -129,7 +148,7 @@ export class MongoRouteAPIAdapter {
|
|
|
129
148
|
};
|
|
130
149
|
let errors = [];
|
|
131
150
|
if (collection.type == 'view') {
|
|
132
|
-
errors.push({ level: 'warning', message: `Collection ${schema}.${
|
|
151
|
+
errors.push({ level: 'warning', message: `Collection ${schema}.${collection.name} is a view` });
|
|
133
152
|
}
|
|
134
153
|
else {
|
|
135
154
|
errors.push(...validatePostImages(schema, collection));
|
|
@@ -266,6 +285,16 @@ export class MongoRouteAPIAdapter {
|
|
|
266
285
|
// Ignore collections we're not authorized to query
|
|
267
286
|
continue;
|
|
268
287
|
}
|
|
288
|
+
if (isSchemaInferenceFailure(e)) {
|
|
289
|
+
// We know the collection is there, we just could not read its fields. Report it
|
|
290
|
+
// without columns rather than failing: one collection we cannot sample should not
|
|
291
|
+
// block validation and deployment for every sync config on the instance. Queries
|
|
292
|
+
// naming its columns warn about unknown columns; the result is indistinguishable
|
|
293
|
+
// from an empty collection, so a `SELECT *` query still validates cleanly.
|
|
294
|
+
logger.warn(`Could not infer the schema of ${db.name}.${collection.name}`, e);
|
|
295
|
+
tables[index] = { name: collection.name, columns: [] };
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
269
298
|
// Fail the whole request on unexpected errors: the response cannot indicate
|
|
270
299
|
// partial results, so omitting a collection would make an incomplete schema
|
|
271
300
|
// appear complete and could produce an incorrect generated client schema.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/MongoRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"MongoRouteAPIAdapter.js","sourceRoot":"","sources":["../../src/api/MongoRouteAPIAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,KAAK,UAAU,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAEtE,OAAO,EAAE,8BAA8B,EAAE,MAAM,gEAAgE,CAAC;AAChH,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAC/F,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAE/F,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAEvC,MAAM,OAAO,oBAAoB;IAUT;IATZ,MAAM,CAAoB;IAC7B,EAAE,CAAW;IAEpB,aAAa,CAAS;IACtB,aAAa,CAAS;IAEd,wBAAwB,GAAoC,IAAI,CAAC;IACjE,mBAAmB,GAA4B,IAAI,CAAC;IAE5D,YAAsB,MAAsC;QAAtC,WAAM,GAAN,MAAM,CAAgC;QAC1D,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC;IAC5D,CAAC;IAED,wBAAwB;QACtB,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE;SAClE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,MAA6B;QAC5D,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,oBAAoB,CAAC,MAA6B;QAChE,MAAM,QAAQ,GAAG,UAAU,CAAC,yBAAyB,CACnD,MAAM,YAAY,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAC/F,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,OAAO,QAAQ;YACb,CAAC,CAAC;gBACE;oBACE,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,IAAI,SAAS,CAAC,WAAW,gGAAgG;iBACnI;aACF;YACH,CAAC,CAAC,EAAE,CAAC;IACT,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;YAClB,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI;gBACP,SAAS,EAAE,KAAK;gBAChB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,CAAC;QACJ,CAAC;QACD,OAAO;YACL,GAAG,IAAI;YACP,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,MAAa;QAC7C,OAAO,aAAa,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC;YAC7D,OAAO,EAAE;gBACP,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,EAAE;aACT;YACD,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,2CAA2C;SACnD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,aAAwC,EACxC,YAAmC;QAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;QAErC,MAAM,kBAAkB,GAAG,CAAC,MAAc,EAAE,UAAgC,EAAoC,EAAE;YAChH,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;gBACzD,OAAO,EAAE,CAAC;YACZ,CAAC;iBAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,4BAA4B,EAAE,OAAO,EAAE,CAAC;gBACtE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC;oBAC/D,OAAO;wBACL,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,+CAA+C,MAAM,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE;qBACxG,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO;wBACL;4BACE,KAAK,EAAE,SAAS;4BAChB,OAAO,EAAE,+CAA+C,MAAM,IAAI,UAAU,CAAC,IAAI,iCAAiC;yBACnH;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;QAEF,KAAK,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YAEnC,IAAI,aAAa,GAAsB;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,YAAY,CAAC,YAAY;gBAClC,QAAQ,EAAE,YAAY,CAAC,UAAU;aAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAE3B,IAAI,UAA2B,CAAC;YAChC,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,UAAU,GAAG,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACN,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC;YACjC,CAAC;YAED,iCAAiC;YACjC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM;iBAClC,EAAE,CAAC,MAAM,CAAC;iBACV,eAAe,CACd;gBACE,IAAI,EAAE,UAAU;aACjB,EACD,EAAE,QAAQ,EAAE,KAAK,EAAE,CACpB;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,YAAY,CAAC,UAAU,EAAE,CAAC;gBAC5B,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;gBAC1B,KAAK,IAAI,UAAU,IAAI,WAAW,EAAE,CAAC;oBACnC,MAAM,GAAG,GAA8B;wBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;wBACjC,MAAM;wBACN,IAAI,EAAE,UAAU,CAAC,IAAI;qBACtB,CAAC;oBACF,IAAI,MAAM,GAAqC,EAAE,CAAC;oBAClD,IAAI,UAAU,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;wBAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,MAAM,IAAI,UAAU,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;oBAClG,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;oBACzD,CAAC;oBACD,MAAM,QAAQ,GAAG,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;oBAClD,MAAM,cAAc,GAAG,YAAY,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;oBAC9D,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC;wBACxB,MAAM;wBACN,IAAI,EAAE,UAAU,CAAC,IAAI;wBACrB,cAAc,EAAE,CAAC,KAAK,CAAC;wBACvB,YAAY,EAAE,QAAQ;wBACtB,iBAAiB,EAAE,cAAc;wBACjC,MAAM,EAAE,MAAM;qBACf,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAA8B;oBACrC,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,MAAM;oBACN,IAAI,EAAE,YAAY,CAAC,IAAI;iBACxB,CAAC;gBAEF,MAAM,QAAQ,GAAG,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAClD,MAAM,cAAc,GAAG,YAAY,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC;gBAC9D,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;gBAElC,IAAI,MAAM,GAAqC,EAAE,CAAC;gBAClD,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,MAAM,IAAI,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;gBACpG,CAAC;qBAAM,IAAI,UAAU,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,MAAM,IAAI,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;gBACpG,CAAC;qBAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,4BAA4B,EAAE,OAAO,EAAE,CAAC;oBACtE,MAAM,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;gBACzD,CAAC;gBAED,aAAa,CAAC,KAAK,GAAG;oBACpB,MAAM;oBACN,IAAI,EAAE,YAAY,CAAC,IAAI;oBACvB,cAAc,EAAE,CAAC,KAAK,CAAC;oBACvB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,MAAM;iBACP,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,OAAkC;QAC7D,mEAAmE;QACnE,4EAA4E;QAC5E,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAI,QAAoC;QACjE,MAAM,wBAAwB,GAAG,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAC1E,OAAO,wBAAwB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAEO,YAAY;QAClB,8EAA8E;QAC9E,OAAO,CAAC,IAAI,CAAC,mBAAmB,KAAK,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7E,qEAAqE;YACrE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,2BAA2B;QACvC,IAAI,IAAI,CAAC,wBAAwB,IAAI,IAAI,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/C,IAAI,CAAC,wBAAwB,GAAG,8BAA8B,CAAC,YAAY,EAAE;gBAC3E,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,sEAAsE;gBACtE,6DAA6D;gBAC7D,wEAAwE;gBACxE,wEAAwE;gBACxE,0EAA0E;gBAC1E,mEAAmE;gBACnE,kBAAkB,EAAE,IAAI,KAAK,CAAC,QAAQ,EAAE;gBACxC,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,wBAAwB,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,MAAM,iBAAiB,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YAC1D,OAAO,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,MAAM,eAAe,GAAmC,EAAE,CAAC;QAC3D,KAAK,MAAM,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACnC;;;;eAIG;YACH,IAAI,WAAmC,CAAC;YACxC,IAAI,CAAC;gBACH,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,CAAC;YAC1E,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,EAAE,CAAC;oBACpE,iDAAiD;oBACjD,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,CAAC;YACV,CAAC;YAED,MAAM,MAAM,GAA8C,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACxF,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;gBAClC,4DAA4D;gBAC5D,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,kBAAkB,EAAE,CAAC;oBACrD,IAAI,MAAM,EAAE,CAAC;wBACX,OAAO;oBACT,CAAC;oBACD,IAAI,CAAC,sBAAsB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvD,SAAS;oBACX,CAAC;oBACD,IAAI,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC1C,0DAA0D;wBAC1D,oEAAoE;wBACpE,SAAS;oBACX,CAAC;oBACD,IAAI,UAAU,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;wBAC9B,SAAS;oBACX,CAAC;oBACD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,qBAAqB,CACzC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EACnD,YAAY,CACb,CAAC;wBACF,mEAAmE;wBACnE,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;oBACrD,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,IAAI,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,EAAE,CAAC;4BACpE,mDAAmD;4BACnD,SAAS;wBACX,CAAC;wBACD,IAAI,wBAAwB,CAAC,CAAC,CAAC,EAAE,CAAC;4BAChC,gFAAgF;4BAChF,kFAAkF;4BAClF,iFAAiF;4BACjF,iFAAiF;4BACjF,2EAA2E;4BAC3E,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;4BAC9E,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;4BACvD,SAAS;wBACX,CAAC;wBACD,4EAA4E;wBAC5E,4EAA4E;wBAC5E,0EAA0E;wBAC1E,MAAM,GAAG,IAAI,CAAC;wBACd,MAAM,CAAC,CAAC;oBACV,CAAC;gBACH,CAAC;YACH,CAAC,CAAC;YAEF,mFAAmF;YACnF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,4BAA4B,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,EACtE,gBAAgB,CACjB,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,6EAA6E;gBAC7E,uEAAuE;gBACvE,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClC,MAAM,CAAC,CAAC;YACV,CAAC;YAED,eAAe,CAAC,IAAI,CAAC;gBACnB,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;aAChD,CAAC,CAAC;QACL,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { TableSchema } from '@powersync/service-types';
|
|
3
|
+
/**
|
|
4
|
+
* Whether the error means we could not determine this collection's fields, as opposed to
|
|
5
|
+
* something that makes the rest of the schema unreliable too.
|
|
6
|
+
*/
|
|
7
|
+
export declare function isSchemaInferenceFailure(e: unknown): boolean;
|
|
3
8
|
/**
|
|
4
9
|
* Infer top-level fields without transferring sampled document values to the service.
|
|
5
10
|
* Memory on the service scales with the inferred schema, not the size of those values.
|
|
@@ -1,5 +1,29 @@
|
|
|
1
|
+
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
3
|
import { ExpressionType } from '@powersync/service-sync-rules';
|
|
4
|
+
/**
|
|
5
|
+
* Server errors that mean this one collection could not be inspected, rather than the
|
|
6
|
+
* connection being unusable.
|
|
7
|
+
*
|
|
8
|
+
* These come from the memory ceiling {@link inferCollectionSchema} imposes on itself by
|
|
9
|
+
* refusing to spill to disk, and are deterministic per collection. Timeouts are
|
|
10
|
+
* deliberately excluded: they usually indicate an overloaded source, where degrading
|
|
11
|
+
* would silently empty the schema of every collection at once.
|
|
12
|
+
*/
|
|
13
|
+
const SCHEMA_INFERENCE_FAILURE_CODES = new Set([
|
|
14
|
+
// ExceededMemoryLimit - a blocking stage ran out of memory with disk spill disabled.
|
|
15
|
+
146,
|
|
16
|
+
// QueryExceededMemoryLimitNoDiskUseAllowed - as above, reported by the sort $sample
|
|
17
|
+
// uses on collections too small for its random cursor.
|
|
18
|
+
292
|
|
19
|
+
]);
|
|
20
|
+
/**
|
|
21
|
+
* Whether the error means we could not determine this collection's fields, as opposed to
|
|
22
|
+
* something that makes the rest of the schema unreliable too.
|
|
23
|
+
*/
|
|
24
|
+
export function isSchemaInferenceFailure(e) {
|
|
25
|
+
return lib_mongo.isMongoServerError(e) && SCHEMA_INFERENCE_FAILURE_CODES.has(e.code);
|
|
26
|
+
}
|
|
3
27
|
// Match the types exposed after BSON deserialization and conversion to sync rules values.
|
|
4
28
|
const BSON_TYPES = {
|
|
5
29
|
int: { name: 'Integer', sqliteType: ExpressionType.INTEGER },
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"infer-collection-schema.js","sourceRoot":"","sources":["../../src/api/infer-collection-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAG/D,0FAA0F;AAC1F,MAAM,UAAU,GAAiE;IAC/E,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC5D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC1D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC7D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC7D,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC/D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC5D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,SAAS,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IACpE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC1D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACzD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC/D,mBAAmB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACxE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC5D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;CAC5D,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAA4B,EAC5B,YAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,UAAU;SAC5B,SAAS,CACR;QACE,6EAA6E;QAC7E,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;QACzB;YACE,QAAQ,EAAE;gBACR,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,KAAK,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;wBACnC,EAAE,EAAE,OAAO;wBACX,EAAE,EAAE;4BACF,IAAI,EAAE,WAAW;4BACjB,IAAI,EAAE;gCACJ,IAAI,EAAE;oCACJ,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE;oCAC1C,EAAE,EAAE;wCACF,OAAO,EAAE;4CACP,QAAQ,EAAE;gDACR;oDACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE;oDACvC,0DAA0D;oDAC1D,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE;iDAC7E;gDACD;oDACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE;oDACxC,IAAI,EAAE;wDACJ,KAAK,EAAE;4DACL,6DAA6D;4DAC7D,2DAA2D;4DAC3D,6DAA6D;4DAC7D,iDAAiD;4DACjD,oFAAoF;4DACpF;gEACE,IAAI,EAAE;oEACJ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC,EAAE;oEAC/E,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC,EAAE;iEAChF;6DACF;4DACD,MAAM;4DACN,SAAS;yDACV;qDACF;iDACF;6CACF;4CACD,OAAO,EAAE,YAAY;yCACtB;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;SACF;QACD,kFAAkF;QAClF,EAAE,OAAO,EAAE,SAAS,EAAE;QACtB,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,EAAE;KAC1E,EACD;QACE,4EAA4E;QAC5E,yEAAyE;QACzE,SAAS,EAAE,MAAM;QACjB,wEAAwE;QACxE,wEAAwE;QACxE,kEAAkE;QAClE,YAAY,EAAE,KAAK;QACnB,qFAAqF;QACrF,uEAAuE;QACvE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;KAC7D,CACF;SACA,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QAC5B,IAAI,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;YAClC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,aAAa,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,CAAC,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACjH,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC"}
|
|
1
|
+
{"version":3,"file":"infer-collection-schema.js","sourceRoot":"","sources":["../../src/api/infer-collection-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAG/D;;;;;;;;GAQG;AACH,MAAM,8BAA8B,GAAG,IAAI,GAAG,CAAC;IAC7C,qFAAqF;IACrF,GAAG;IACH,oFAAoF;IACpF,uDAAuD;IACvD,GAAG;CACJ,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,CAAU;IACjD,OAAO,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,8BAA8B,CAAC,GAAG,CAAC,CAAC,CAAC,IAAc,CAAC,CAAC;AACjG,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,GAAiE;IAC/E,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC5D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC1D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC7D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IAC7D,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC/D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC5D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,SAAS,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,CAAC,OAAO,EAAE;IACpE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC1D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACzD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC/D,mBAAmB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACxE,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IACvD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC5D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,EAAE;CAC5D,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAA4B,EAC5B,YAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,UAAU;SAC5B,SAAS,CACR;QACE,6EAA6E;QAC7E,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;QACzB;YACE,QAAQ,EAAE;gBACR,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE;oBACN,IAAI,EAAE;wBACJ,KAAK,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;wBACnC,EAAE,EAAE,OAAO;wBACX,EAAE,EAAE;4BACF,IAAI,EAAE,WAAW;4BACjB,IAAI,EAAE;gCACJ,IAAI,EAAE;oCACJ,IAAI,EAAE,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE;oCAC1C,EAAE,EAAE;wCACF,OAAO,EAAE;4CACP,QAAQ,EAAE;gDACR;oDACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC,EAAE;oDACvC,0DAA0D;oDAC1D,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,EAAE;iDAC7E;gDACD;oDACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE;oDACxC,IAAI,EAAE;wDACJ,KAAK,EAAE;4DACL,6DAA6D;4DAC7D,2DAA2D;4DAC3D,6DAA6D;4DAC7D,iDAAiD;4DACjD,oFAAoF;4DACpF;gEACE,IAAI,EAAE;oEACJ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC,EAAE;oEAC/E,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC,EAAE;iEAChF;6DACF;4DACD,MAAM;4DACN,SAAS;yDACV;qDACF;iDACF;6CACF;4CACD,OAAO,EAAE,YAAY;yCACtB;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;SACF;QACD,kFAAkF;QAClF,EAAE,OAAO,EAAE,SAAS,EAAE;QACtB,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,EAAE;KAC1E,EACD;QACE,4EAA4E;QAC5E,yEAAyE;QACzE,SAAS,EAAE,MAAM;QACjB,wEAAwE;QACxE,wEAAwE;QACxE,kEAAkE;QAClE,YAAY,EAAE,KAAK;QACnB,qFAAqF;QACrF,uEAAuE;QACvE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;KAC7D,CACF;SACA,OAAO,EAAE,CAAC;IAEb,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QAC5B,IAAI,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC;QACrC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;YAClC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAChD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,aAAa,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,CAAC,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACjH,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './api/MongoRouteAPIAdapter.js';
|
|
2
2
|
export * from './module/MongoModule.js';
|
|
3
3
|
export * from './replication/replication-index.js';
|
|
4
|
+
export * as test_utils from './test-utils/test-utils-index.js';
|
|
4
5
|
export * from './types/types.js';
|
|
5
6
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,cAAc,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oCAAoC,CAAC;AACnD,OAAO,KAAK,UAAU,MAAM,kCAAkC,CAAC;AAC/D,cAAc,kBAAkB,CAAC"}
|
|
@@ -4,11 +4,10 @@ export declare class MongoModule extends replication.ReplicationModule<types.Mon
|
|
|
4
4
|
constructor();
|
|
5
5
|
protected createRouteAPIAdapter(): api.RouteAPI;
|
|
6
6
|
protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator;
|
|
7
|
-
onInitialized(context: system.ServiceContextContainer): Promise<void>;
|
|
8
7
|
/**
|
|
9
8
|
* Combines base config with normalized connection settings
|
|
10
9
|
*/
|
|
11
|
-
|
|
10
|
+
protected resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig;
|
|
12
11
|
teardown(options: TearDownOptions): Promise<void>;
|
|
13
12
|
testConnection(config: types.MongoConnectionConfig): Promise<ConnectionTestResult>;
|
|
14
13
|
static testConnection(normalizedConfig: types.NormalizedMongoConnectionConfig): Promise<ConnectionTestResult>;
|
|
@@ -26,13 +26,13 @@ export class MongoModule extends replication.ReplicationModule {
|
|
|
26
26
|
id: this.getDefaultId(normalisedConfig.database ?? ''),
|
|
27
27
|
syncRuleProvider: syncRuleProvider,
|
|
28
28
|
storageEngine: context.storageEngine,
|
|
29
|
+
configuration: context.configuration,
|
|
29
30
|
metricsEngine: context.metricsEngine,
|
|
30
31
|
connectionFactory: connectionFactory,
|
|
31
32
|
rateLimiter: new MongoErrorRateLimiter(),
|
|
32
33
|
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
33
34
|
});
|
|
34
35
|
}
|
|
35
|
-
async onInitialized(context) { }
|
|
36
36
|
/**
|
|
37
37
|
* Combines base config with normalized connection settings
|
|
38
38
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;YACxC,wBAAwB,EAAE,gBAAgB,CAAC,0BAA0B;SACtE,CAAC,CAAC;IACL,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"MongoModule.js","sourceRoot":"","sources":["../../src/module/MongoModule.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAEL,kCAAkC,EAElC,WAAW,EAGZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AACtF,OAAO,EAAE,qBAAqB,EAAE,MAAM,yCAAyC,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,KAAK,KAAK,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAY,SAAQ,WAAW,CAAC,iBAA8C;IACzF;QACE,KAAK,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS,CAAC,qBAAqB;YACrC,YAAY,EAAE,KAAK,CAAC,qBAAqB;SAC1C,CAAC,CAAC;IACL,CAAC;IAES,qBAAqB;QAC7B,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAES,gBAAgB,CAAC,OAA8B;QACvD,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,MAAM,gBAAgB,GAAG,IAAI,kCAAkC,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAClG,MAAM,iBAAiB,GAAG,IAAI,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;QAEzE,OAAO,IAAI,sBAAsB,CAAC;YAChC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtD,gBAAgB,EAAE,gBAAgB;YAClC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,iBAAiB,EAAE,iBAAiB;YACpC,WAAW,EAAE,IAAI,qBAAqB,EAAE;YACxC,wBAAwB,EAAE,gBAAgB,CAAC,0BAA0B;SACtE,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,MAAmC;QACzD,OAAO;YACL,GAAG,MAAM;YACT,GAAG,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAwB;QACrC,QAAQ;IACV,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAmC;QACtD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAc,CAAC,CAAC;QACjE,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,gBAAuD;QACjF,MAAM,iBAAiB,GAAG,IAAI,YAAY,CAAC,gBAAgB,EAAE;YAC3D,8CAA8C;YAC9C,0DAA0D;YAC1D,0FAA0F;YAC1F,eAAe,EAAE,KAAK;YACtB,wBAAwB,EAAE,KAAK;SAChC,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,wBAAwB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;gBAAS,CAAC;YACT,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC;QAChC,CAAC;QACD,OAAO;YACL,qBAAqB,EAAE,gBAAgB,CAAC,GAAG;SAC5C,CAAC;IACJ,CAAC;CACF"}
|