@powersync/service-module-mongodb-storage 0.0.0-dev-20251209070120 → 0.0.0-dev-20260114113449
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -5
- package/dist/storage/implementation/MongoBucketBatch.d.ts +2 -2
- package/dist/storage/implementation/MongoBucketBatch.js +3 -5
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.js +4 -0
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +4 -4
- package/dist/storage/implementation/MongoSyncBucketStorage.js +3 -3
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +11 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/types/types.d.ts +3 -0
- package/dist/types/types.js +4 -0
- package/dist/types/types.js.map +1 -1
- package/dist/utils/util.d.ts +23 -0
- package/dist/utils/util.js +45 -0
- package/dist/utils/util.js.map +1 -1
- package/package.json +6 -6
- package/src/storage/implementation/MongoBucketBatch.ts +6 -7
- package/src/storage/implementation/MongoPersistedSyncRules.ts +5 -1
- package/src/storage/implementation/MongoSyncBucketStorage.ts +12 -10
- package/src/storage/implementation/MongoSyncRulesLock.ts +16 -3
- package/src/types/types.ts +4 -0
- package/src/utils/util.ts +51 -1
- package/test/tsconfig.json +1 -7
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +2 -14
package/dist/utils/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAsB,EACtB,IAAY,EACZ,IAAiD;IAEjD,MAAM,eAAe,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,gBAAgB,GAAG,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAG,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IAEjD,uCAAuC;IACvC,IAAI,MAAM,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,MAAM,gBAAgB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,IAAY;IAC7D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,gBAAgB,GAAG,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,gBAAgB,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACpE,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/index.d.ts",
|
|
5
|
-
"version": "0.0.0-dev-
|
|
5
|
+
"version": "0.0.0-dev-20260114113449",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-ALv2",
|
|
8
8
|
"type": "module",
|
|
@@ -27,15 +27,15 @@
|
|
|
27
27
|
"lru-cache": "^10.2.2",
|
|
28
28
|
"ts-codec": "^1.3.0",
|
|
29
29
|
"uuid": "^11.1.0",
|
|
30
|
-
"@powersync/lib-service-mongodb": "0.0.0-dev-
|
|
31
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
32
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
30
|
+
"@powersync/lib-service-mongodb": "0.0.0-dev-20260114113449",
|
|
31
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20260114113449",
|
|
32
|
+
"@powersync/service-core": "0.0.0-dev-20260114113449",
|
|
33
33
|
"@powersync/service-jsonbig": "0.17.12",
|
|
34
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
34
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20260114113449",
|
|
35
35
|
"@powersync/service-types": "0.13.3"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
38
|
+
"@powersync/service-core-tests": "0.0.0-dev-20260114113449"
|
|
39
39
|
},
|
|
40
40
|
"scripts": {
|
|
41
41
|
"build": "tsc -b",
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { SqlEventDescriptor, SqliteRow, SqliteValue,
|
|
2
|
+
import { SqlEventDescriptor, SqliteRow, SqliteValue, HydratedSyncRules } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
6
|
BaseObserver,
|
|
7
7
|
container,
|
|
8
|
+
logger as defaultLogger,
|
|
8
9
|
ErrorCode,
|
|
9
10
|
errors,
|
|
10
11
|
Logger,
|
|
11
|
-
logger as defaultLogger,
|
|
12
12
|
ReplicationAssertionError,
|
|
13
13
|
ServiceError
|
|
14
14
|
} from '@powersync/lib-services-framework';
|
|
@@ -22,13 +22,13 @@ import {
|
|
|
22
22
|
utils
|
|
23
23
|
} from '@powersync/service-core';
|
|
24
24
|
import * as timers from 'node:timers/promises';
|
|
25
|
+
import { idPrefixFilter } from '../../utils/util.js';
|
|
25
26
|
import { PowerSyncMongo } from './db.js';
|
|
26
27
|
import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
|
|
27
28
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
28
29
|
import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
|
|
29
30
|
import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
30
31
|
import { PersistedBatch } from './PersistedBatch.js';
|
|
31
|
-
import { idPrefixFilter } from '../../utils/util.js';
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
34
|
* 15MB
|
|
@@ -44,7 +44,7 @@ const replicationMutex = new utils.Mutex();
|
|
|
44
44
|
|
|
45
45
|
export interface MongoBucketBatchOptions {
|
|
46
46
|
db: PowerSyncMongo;
|
|
47
|
-
syncRules:
|
|
47
|
+
syncRules: HydratedSyncRules;
|
|
48
48
|
groupId: number;
|
|
49
49
|
slotName: string;
|
|
50
50
|
lastCheckpointLsn: string | null;
|
|
@@ -71,7 +71,7 @@ export class MongoBucketBatch
|
|
|
71
71
|
private readonly client: mongo.MongoClient;
|
|
72
72
|
public readonly db: PowerSyncMongo;
|
|
73
73
|
public readonly session: mongo.ClientSession;
|
|
74
|
-
private readonly sync_rules:
|
|
74
|
+
private readonly sync_rules: HydratedSyncRules;
|
|
75
75
|
|
|
76
76
|
private readonly group_id: number;
|
|
77
77
|
|
|
@@ -474,8 +474,7 @@ export class MongoBucketBatch
|
|
|
474
474
|
if (sourceTable.syncData) {
|
|
475
475
|
const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
476
476
|
record: after,
|
|
477
|
-
sourceTable
|
|
478
|
-
bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`)
|
|
477
|
+
sourceTable
|
|
479
478
|
});
|
|
480
479
|
|
|
481
480
|
for (let error of syncErrors) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
1
|
+
import { SqlSyncRules, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
|
|
2
2
|
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
|
|
@@ -13,4 +13,8 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
|
|
|
13
13
|
) {
|
|
14
14
|
this.slot_name = slot_name ?? `powersync_${id}`;
|
|
15
15
|
}
|
|
16
|
+
|
|
17
|
+
hydratedSyncRules(): HydratedSyncRules {
|
|
18
|
+
return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) });
|
|
19
|
+
}
|
|
16
20
|
}
|
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
BroadcastIterable,
|
|
11
11
|
CHECKPOINT_INVALIDATE_ALL,
|
|
12
12
|
CheckpointChanges,
|
|
13
|
-
CompactOptions,
|
|
14
13
|
deserializeParameterLookup,
|
|
15
14
|
GetCheckpointChangesOptions,
|
|
16
15
|
InternalOpId,
|
|
@@ -25,10 +24,11 @@ import {
|
|
|
25
24
|
WatchWriteCheckpointOptions
|
|
26
25
|
} from '@powersync/service-core';
|
|
27
26
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
28
|
-
import {
|
|
27
|
+
import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
|
|
29
28
|
import * as bson from 'bson';
|
|
30
29
|
import { LRUCache } from 'lru-cache';
|
|
31
30
|
import * as timers from 'timers/promises';
|
|
31
|
+
import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
|
|
32
32
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
33
33
|
import { PowerSyncMongo } from './db.js';
|
|
34
34
|
import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, SourceTableDocument } from './models.js';
|
|
@@ -37,7 +37,6 @@ import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
|
|
|
37
37
|
import { MongoCompactor } from './MongoCompactor.js';
|
|
38
38
|
import { MongoParameterCompactor } from './MongoParameterCompactor.js';
|
|
39
39
|
import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
40
|
-
import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
|
|
41
40
|
|
|
42
41
|
export interface MongoSyncBucketStorageOptions {
|
|
43
42
|
checksumOptions?: MongoChecksumOptions;
|
|
@@ -61,7 +60,7 @@ export class MongoSyncBucketStorage
|
|
|
61
60
|
private readonly db: PowerSyncMongo;
|
|
62
61
|
readonly checksums: MongoChecksums;
|
|
63
62
|
|
|
64
|
-
private parsedSyncRulesCache: { parsed:
|
|
63
|
+
private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined;
|
|
65
64
|
private writeCheckpointAPI: MongoWriteCheckpointAPI;
|
|
66
65
|
|
|
67
66
|
constructor(
|
|
@@ -101,14 +100,14 @@ export class MongoSyncBucketStorage
|
|
|
101
100
|
});
|
|
102
101
|
}
|
|
103
102
|
|
|
104
|
-
getParsedSyncRules(options: storage.ParseSyncRulesOptions):
|
|
103
|
+
getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules {
|
|
105
104
|
const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
|
|
106
105
|
/**
|
|
107
106
|
* Check if the cached sync rules, if present, had the same options.
|
|
108
107
|
* Parse sync rules if the options are different or if there is no cached value.
|
|
109
108
|
*/
|
|
110
109
|
if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
|
|
111
|
-
this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).
|
|
110
|
+
this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options };
|
|
112
111
|
}
|
|
113
112
|
|
|
114
113
|
return this.parsedSyncRulesCache!.parsed;
|
|
@@ -170,7 +169,7 @@ export class MongoSyncBucketStorage
|
|
|
170
169
|
await using batch = new MongoBucketBatch({
|
|
171
170
|
logger: options.logger,
|
|
172
171
|
db: this.db,
|
|
173
|
-
syncRules: this.sync_rules.parsed(options).
|
|
172
|
+
syncRules: this.sync_rules.parsed(options).hydratedSyncRules(),
|
|
174
173
|
groupId: this.group_id,
|
|
175
174
|
slotName: this.slot_name,
|
|
176
175
|
lastCheckpointLsn: checkpoint_lsn,
|
|
@@ -293,7 +292,10 @@ export class MongoSyncBucketStorage
|
|
|
293
292
|
return result!;
|
|
294
293
|
}
|
|
295
294
|
|
|
296
|
-
async getParameterSets(
|
|
295
|
+
async getParameterSets(
|
|
296
|
+
checkpoint: MongoReplicationCheckpoint,
|
|
297
|
+
lookups: ScopedParameterLookup[]
|
|
298
|
+
): Promise<SqliteJsonRow[]> {
|
|
297
299
|
return this.db.client.withSession({ snapshot: true }, async (session) => {
|
|
298
300
|
// Set the session's snapshot time to the checkpoint's snapshot time.
|
|
299
301
|
// An alternative would be to create the session when the checkpoint is created, but managing
|
|
@@ -1025,7 +1027,7 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
1025
1027
|
public snapshotTime: mongo.Timestamp
|
|
1026
1028
|
) {}
|
|
1027
1029
|
|
|
1028
|
-
async getParameterSets(lookups:
|
|
1030
|
+
async getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]> {
|
|
1029
1031
|
return this.storage.getParameterSets(this, lookups);
|
|
1030
1032
|
}
|
|
1031
1033
|
}
|
|
@@ -1034,7 +1036,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
1034
1036
|
readonly checkpoint: InternalOpId = 0n;
|
|
1035
1037
|
readonly lsn: string | null = null;
|
|
1036
1038
|
|
|
1037
|
-
async getParameterSets(lookups:
|
|
1039
|
+
async getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]> {
|
|
1038
1040
|
return [];
|
|
1039
1041
|
}
|
|
1040
1042
|
}
|
|
@@ -3,6 +3,8 @@ import crypto from 'crypto';
|
|
|
3
3
|
import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
|
|
4
4
|
import { storage } from '@powersync/service-core';
|
|
5
5
|
import { PowerSyncMongo } from './db.js';
|
|
6
|
+
import { fsCache, syncLockCheck } from '../../utils/util.js';
|
|
7
|
+
import { FsCachePaths } from '../../types/types.js';
|
|
6
8
|
|
|
7
9
|
/**
|
|
8
10
|
* Manages a lock on a sync rules document, so that only one process
|
|
@@ -36,10 +38,21 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
36
38
|
// Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
|
|
37
39
|
const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 } });
|
|
38
40
|
if (heldLock?.lock?.expires_at) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
41
|
+
const alert = await fsCache(
|
|
42
|
+
FsCachePaths.SYNC_RULES_LOCK,
|
|
43
|
+
heldLock.lock.expires_at.toISOString(),
|
|
44
|
+
syncLockCheck
|
|
42
45
|
);
|
|
46
|
+
/** If the date has changed on the expires_at the alert key will be true, we want to notify that another process has a lock */
|
|
47
|
+
if (alert) {
|
|
48
|
+
throw new ServiceError(
|
|
49
|
+
ErrorCode.PSYNC_S1003,
|
|
50
|
+
`Sync rules: ${sync_rules.id} have been locked by another process for replication, expiring at ${heldLock.lock.expires_at.toISOString()}.`
|
|
51
|
+
);
|
|
52
|
+
} else {
|
|
53
|
+
/** We throw undefined as it will be ignored in the catch block */
|
|
54
|
+
throw undefined;
|
|
55
|
+
}
|
|
43
56
|
} else {
|
|
44
57
|
throw new ServiceError(
|
|
45
58
|
ErrorCode.PSYNC_S1003,
|
package/src/types/types.ts
CHANGED
|
@@ -2,6 +2,10 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import * as service_types from '@powersync/service-types';
|
|
3
3
|
import * as t from 'ts-codec';
|
|
4
4
|
|
|
5
|
+
export enum FsCachePaths {
|
|
6
|
+
SYNC_RULES_LOCK = 'sync_rules_lock'
|
|
7
|
+
}
|
|
8
|
+
|
|
5
9
|
export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(
|
|
6
10
|
t.object({
|
|
7
11
|
// Add any mongo specific storage settings here in future
|
package/src/utils/util.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
2
|
import * as crypto from 'crypto';
|
|
3
3
|
import * as uuid from 'uuid';
|
|
4
|
-
|
|
4
|
+
import fs from 'fs';
|
|
5
5
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
6
6
|
import { storage, utils } from '@powersync/service-core';
|
|
7
7
|
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
8
8
|
import { BucketDataDocument } from '../storage/implementation/models.js';
|
|
9
|
+
import { promisify } from 'node:util';
|
|
10
|
+
import { FsCachePaths } from '../types/types.js';
|
|
9
11
|
|
|
10
12
|
export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
|
|
11
13
|
let filter = {
|
|
@@ -154,3 +156,51 @@ export const createPaginatedConnectionQuery = async <T extends mongo.Document>(
|
|
|
154
156
|
more: !(count !== limit)
|
|
155
157
|
};
|
|
156
158
|
};
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Caches data in a file and processes it using a provided function.
|
|
162
|
+
*
|
|
163
|
+
* This function checks if a file exists at the specified `filename` path. If the file exists,
|
|
164
|
+
* it reads the cached data from the file and passes it, along with the provided `text`,
|
|
165
|
+
* to the `func` callback for processing. If the file does not exist, it writes the `text`
|
|
166
|
+
* to the file and then calls the `func` with the `text` as both the cache and input.
|
|
167
|
+
*
|
|
168
|
+
* @template T - The return type of the `func` callback.
|
|
169
|
+
* @param {FsCachePaths} filename - The path to the cache file.
|
|
170
|
+
* @param {string} text - The text to cache or compare against the cached data.
|
|
171
|
+
* @param {(cache: string, text: string) => Promise<T>} func - A callback function that processes the cached data
|
|
172
|
+
* and the provided text. It receives the cached data (or the `text` if no cache exists) as the first argument
|
|
173
|
+
* and the `text` as the second argument.
|
|
174
|
+
* @returns {Promise<T>} - A promise that resolves to the result of the `func` callback.
|
|
175
|
+
*/
|
|
176
|
+
export async function fsCache<T>(
|
|
177
|
+
filename: FsCachePaths,
|
|
178
|
+
text: string,
|
|
179
|
+
func: (cache: string, text: string) => Promise<T>
|
|
180
|
+
): Promise<T> {
|
|
181
|
+
const readFilePromise = promisify(fs.readFile);
|
|
182
|
+
const checkFilePromise = promisify(fs.exists);
|
|
183
|
+
const writeFilePromise = promisify(fs.writeFile);
|
|
184
|
+
|
|
185
|
+
/** Checks to see if the file exists */
|
|
186
|
+
if (await checkFilePromise(filename)) {
|
|
187
|
+
const cache = await readFilePromise(filename, 'utf-8');
|
|
188
|
+
return func(cache, text);
|
|
189
|
+
} else {
|
|
190
|
+
await writeFilePromise(filename, text, 'utf-8');
|
|
191
|
+
return func(text, text);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Compares cached text with new text and updates the cache file if they differ.
|
|
197
|
+
* Returns true if the cache was updated, false otherwise.
|
|
198
|
+
*/
|
|
199
|
+
export async function syncLockCheck(cache: string, text: string): Promise<boolean> {
|
|
200
|
+
if (cache === text) {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
const writeFilePromise = promisify(fs.writeFile);
|
|
204
|
+
await writeFilePromise(FsCachePaths.SYNC_RULES_LOCK, text, 'utf-8');
|
|
205
|
+
return true;
|
|
206
|
+
}
|
package/test/tsconfig.json
CHANGED
|
@@ -19,13 +19,7 @@
|
|
|
19
19
|
"path": "../"
|
|
20
20
|
},
|
|
21
21
|
{
|
|
22
|
-
"path": "../../../packages/service-core
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
"path": "../../../packages/service-core/"
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
"path": "../../../packages/service-core-tests/"
|
|
22
|
+
"path": "../../../packages/service-core-tests"
|
|
29
23
|
}
|
|
30
24
|
]
|
|
31
25
|
}
|