@powersync/service-core 1.18.2 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/metrics/open-telemetry/util.js +3 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +1 -0
- package/dist/replication/AbstractReplicator.js +16 -6
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/routes/endpoints/admin.js +4 -1
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -4
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +1 -1
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -4
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +2 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +4 -4
- package/dist/storage/bson.d.ts +3 -3
- package/dist/storage/bson.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +5 -9
- package/dist/sync/BucketChecksumState.js +13 -12
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/sync.d.ts +2 -2
- package/dist/sync/sync.js.map +1 -1
- package/dist/util/config/collectors/config-collector.js +5 -2
- package/dist/util/config/collectors/config-collector.js.map +1 -1
- package/dist/util/config.d.ts +2 -0
- package/dist/util/config.js +15 -2
- package/dist/util/config.js.map +1 -1
- package/package.json +5 -5
- package/src/metrics/open-telemetry/util.ts +3 -1
- package/src/replication/AbstractReplicator.ts +15 -7
- package/src/routes/endpoints/admin.ts +4 -1
- package/src/routes/endpoints/socket-route.ts +1 -4
- package/src/routes/endpoints/sync-rules.ts +1 -1
- package/src/routes/endpoints/sync-stream.ts +1 -4
- package/src/storage/PersistedSyncRulesContent.ts +3 -1
- package/src/storage/SyncRulesBucketStorage.ts +4 -4
- package/src/storage/bson.ts +3 -3
- package/src/sync/BucketChecksumState.ts +20 -21
- package/src/sync/sync.ts +4 -4
- package/src/util/config/collectors/config-collector.ts +9 -2
- package/src/util/config.ts +20 -2
- package/test/src/config.test.ts +17 -0
- package/test/src/routes/stream.test.ts +2 -2
- package/test/src/sync/BucketChecksumState.test.ts +20 -50
- package/tsconfig.json +0 -3
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +6 -7
package/dist/sync/sync.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
+
import { HydratedSyncRules } from '@powersync/service-sync-rules';
|
|
1
2
|
import * as auth from '../auth/auth-index.js';
|
|
2
3
|
import * as storage from '../storage/storage-index.js';
|
|
3
4
|
import * as util from '../util/util-index.js';
|
|
4
5
|
import { Logger } from '@powersync/lib-services-framework';
|
|
5
|
-
import { VersionedSyncRules } from './BucketChecksumState.js';
|
|
6
6
|
import { RequestTracker } from './RequestTracker.js';
|
|
7
7
|
import { SyncContext } from './SyncContext.js';
|
|
8
8
|
import { TokenStreamOptions } from './util.js';
|
|
9
9
|
export interface SyncStreamParameters {
|
|
10
10
|
syncContext: SyncContext;
|
|
11
11
|
bucketStorage: storage.SyncRulesBucketStorage;
|
|
12
|
-
syncRules:
|
|
12
|
+
syncRules: HydratedSyncRules;
|
|
13
13
|
params: util.StreamingSyncRequest;
|
|
14
14
|
token: auth.JwtPayload;
|
|
15
15
|
logger?: Logger;
|
package/dist/sync/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,mBAAmB,
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAU,MAAM,IAAI,aAAa,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAkB,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAuC,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzF,OAAO,EAAsB,yBAAyB,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAmBvG,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EACJ,WAAW,EACX,aAAa,EACb,SAAS,EACT,MAAM,EACN,KAAK,EACL,kBAAkB,EAClB,OAAO,EACP,MAAM,EACN,gBAAgB,EACjB,GAAG,OAAO,CAAC;IACZ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAE/C,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAChC,WAAW,EACX,aAAa,EACb,SAAS,EACT,MAAM,EACN,KAAK,EACL,OAAO,EACP,UAAU,CAAC,MAAM,EACjB,MAAM,EACN,gBAAgB,CACjB,CAAC;IACF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI,CAAC;QACH,KAAK,CAAC,CAAC,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,UAAU,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;YAAS,CAAC;QACT,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,WAAwB,EACxB,aAA6C,EAC7C,SAA4B,EAC5B,MAAiC,EACjC,YAA+B,EAC/B,OAAuB,EACvB,MAAmB,EACnB,MAAc,EACd,gBAAyB;IAEzB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAE5B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC;IAChC,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEnF,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC;QAC5C,WAAW;QACX,aAAa;QACb,SAAS;QACT,YAAY;QACZ,WAAW,EAAE,MAAM;QACnB,MAAM,EAAE,MAAM;KACf,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,aAAa,CAAC,sBAAsB,CAAC;QAClD,OAAO,EAAE,gBAAgB;QACzB,MAAM;KACP,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IAOtD,KAAK,UAAU,wBAAwB;QACrC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC;IAClF,CAAC;IAED,IAAI,CAAC;QACH,IAAI,qBAAmG,CAAC;QAExG,GAAG,CAAC;YACF,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC3B,oFAAoF;gBACpF,sCAAsC;gBACtC,qBAAqB,GAAG,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC;YACzC,qBAAqB,GAAG,SAAS,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC;YACpB,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;YACnC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,oBAAoB;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;YAEhD,+EAA+E;YAC/E,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,cAAc,CAAC;YAErB,8GAA8G;YAC9G,kHAAkH;YAClH,iHAAiH;YACjH,sBAAsB;YACtB,MAAM,yBAAyB,GAAG,IAAI,eAAe,EAAE,CAAC;YACxD,IAAI,gBAAgB,GAAG,CAAC,CAAC;YAEzB,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,yBAAyB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;YAE1F,MAAM,iBAAiB,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC;YACpF,MAAM,cAAc,GAAG,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAErD,wGAAwG;YACxG,+FAA+F;YAC/F,8GAA8G;YAC9G,iBAAiB;YACjB,MAAM,eAAe,GAAmD,iBAAiB,CAAC;YAC1F,IAAI,eAAe,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,eAAe,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YACnC,CAAC;YAED,SAAS,yBAAyB;gBAChC,IAAI,gBAAgB,IAAI,IAAI,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;oBACpE,qBAAqB,GAAG,CAAC,KAAK,IAAI,EAAE;wBAClC,OAAO,IAAI,EAAE,CAAC;4BACZ,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,wBAAwB,EAAE,CAAC,CAAC;4BAC9D,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gCAC9B,yBAAyB,CAAC,KAAK,EAAE,CAAC;4BACpC,CAAC;iCAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gCAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;oCAClC,2FAA2F;oCAC3F,wBAAwB;oCACxB,SAAS;gCACX,CAAC;gCAED,gGAAgG;gCAChG,sCAAsC;gCACtC,yBAAyB,CAAC,KAAK,EAAE,CAAC;4BACpC,CAAC;4BAED,OAAO,IAAI,CAAC;wBACd,CAAC;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,CAAC;YACH,CAAC;YAED,SAAS,kBAAkB,CAAC,KAA0B;gBACpD,gBAAgB,IAAI,KAAK,CAAC,KAAK,CAAC;gBAChC,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACnC,yBAAyB,EAAE,CAAC;YAC9B,CAAC;YAED,+EAA+E;YAC/E,sFAAsF;YACtF,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC;gBAClD,MAAM,MAAM,GAAG,QAAQ,KAAK,cAAc,CAAC;gBAC3C,IAAI,qBAAqB,CAAC,OAAO,EAAE,CAAC;oBAClC,MAAM;gBACR,CAAC;gBAED,KAAK,CAAC,CAAC,mBAAmB,CAAC;oBACzB,WAAW,EAAE,WAAW;oBACxB,aAAa,EAAE,aAAa;oBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU;oBACvC,cAAc,EAAE,OAAO;oBACvB,cAAc,EAAE,IAAI;oBACpB,eAAe,EAAE,CAAC,gBAAgB,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI;oBAC7D,UAAU,EAAE,kBAAkB;oBAC9B,gBAAgB,EAAE,MAAM;oBACxB,WAAW,EAAE,qBAAqB;oBAClC,OAAO,EAAE,MAAM;oBACf,mHAAmH;oBACnH,iCAAiC;oBACjC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;oBACtC,MAAM;iBACP,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,CAAC;gBACnC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;IAClC,CAAC;AACH,CAAC;AAyBD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/C,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM;gBACR,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE,CAAC;wBACT,MAAM,GAAG,IAAI,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EACJ,WAAW,EACX,aAAa,EAAE,OAAO,EACtB,UAAU,EACV,cAAc,EACd,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,MAAM,EACP,GAAG,OAAO,CAAC;IAEZ,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAElC,IAAI,WAAW,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,yBAAyB,CAAC,WAAW,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC5G,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC;IAClC,IAAI,CAAC;QACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,GAAG,CAAC,EAAE,EAAE;gBAC1D,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,eAAe,EAAE,KAAK,GAAG,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;QACD,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,cAAc,CAAC,0BAA0B,CAAC,cAAc,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAE5E,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,WAAW,EAAE,CAAC;YACzD,qDAAqD;YACrD,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO;YACT,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC9C,qBAAqB,GAAG,IAAI,CAAC;YAC/B,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,MAAM,IAAI,GAAG,eAAe;gBAC1B,CAAC,CAAC,8EAA8E;oBAC9E,4BAA4B;oBAC5B,OAAO,CAAC,SAAS,CAAC;wBAChB,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;qBACA,CAAC;gBACrC,CAAC,CAAC,qGAAqG;oBACpG,EAAE,IAAI,EAAE,CAAC,EAAoC,CAAC;YAEnD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAElC,0EAA0E;YAC1E,8CAA8C;YAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAElC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YAE7B,cAAc,CAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhH,6EAA6E;YAC7E,gBAAgB;YAChB,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,qBAAqB,EAAE,CAAC;gBAC1B,gDAAgD;gBAChD,wDAAwD;gBACxD,kEAAkE;gBAClE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;oBAChC,MAAM,IAAI,GAAkD;wBAC1D,2BAA2B,EAAE;4BAC3B,UAAU,EAAE,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;4BACnD,QAAQ,EAAE,OAAO,CAAC,WAAW;yBAC9B;qBACF,CAAC;oBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAyC;wBACjD,mBAAmB,EAAE;4BACnB,UAAU,EAAE,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;yBACpD;qBACF,CAAC;oBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,6DAA6D;YAC7D,qBAAqB;YACrB,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;gBACjC,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -54,15 +54,18 @@ export class ConfigCollector {
|
|
|
54
54
|
return this.parseJSON(content);
|
|
55
55
|
default: {
|
|
56
56
|
// No content type provided, need to try both
|
|
57
|
+
let yamlError;
|
|
57
58
|
try {
|
|
58
59
|
return this.parseYaml(content);
|
|
59
60
|
}
|
|
60
|
-
catch (ex) {
|
|
61
|
+
catch (ex) {
|
|
62
|
+
yamlError = ex;
|
|
63
|
+
}
|
|
61
64
|
try {
|
|
62
65
|
return this.parseJSON(content);
|
|
63
66
|
}
|
|
64
67
|
catch (ex) {
|
|
65
|
-
throw new Error(`Could not parse PowerSync config file content as JSON or YAML: ${ex}`);
|
|
68
|
+
throw new Error(`Could not parse PowerSync config file content as JSON or YAML: JSON Error: ${ex}${yamlError ? `\nYAML Error: ${yamlError}` : ''}`);
|
|
66
69
|
}
|
|
67
70
|
}
|
|
68
71
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,CAAN,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IAC1B,iCAAa,CAAA;IACb,iCAAa,CAAA;AACf,CAAC,EAHW,gBAAgB,KAAhB,gBAAgB,QAG3B;AAED,uFAAuF;AACvF,MAAM,qBAAqB,GAAG,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,SAAS,EAAE,CAAC;AAEvG,MAAM,OAAgB,eAAe;IASnC;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,aAA2B;QACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED;;;WAGG;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,MAAkC;QACzC,MAAM,KAAK,GAAG,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,MAAM,CAAC,OAA6C;QAClD,IAAI,CAAC;YACH,OAAO,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAES,YAAY,CAAC,OAAe,EAAE,WAA8B;QACpE,QAAQ,WAAW,EAAE,CAAC;YACpB,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,OAAO,CAAC,CAAC,CAAC;gBACR,6CAA6C;gBAC7C,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"config-collector.js","sourceRoot":"","sources":["../../../../src/util/config/collectors/config-collector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,CAAN,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IAC1B,iCAAa,CAAA;IACb,iCAAa,CAAA;AACf,CAAC,EAHW,gBAAgB,KAAhB,gBAAgB,QAG3B;AAED,uFAAuF;AACvF,MAAM,qBAAqB,GAAG,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,SAAS,EAAE,CAAC;AAEvG,MAAM,OAAgB,eAAe;IASnC;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,aAA2B;QACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QACd,CAAC;QAED;;;WAGG;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,MAAkC;QACzC,MAAM,KAAK,GAAG,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,MAAM,CAAC,OAA6C;QAClD,IAAI,CAAC;YACH,OAAO,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAES,YAAY,CAAC,OAAe,EAAE,WAA8B;QACpE,QAAQ,WAAW,EAAE,CAAC;YACpB,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,KAAK,gBAAgB,CAAC,IAAI;gBACxB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,OAAO,CAAC,CAAC,CAAC;gBACR,6CAA6C;gBAC7C,IAAI,SAAkB,CAAC;gBACvB,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,SAAS,GAAG,EAAE,CAAC;gBACjB,CAAC;gBACD,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CACb,8EAA8E,EAAE,GAC9E,SAAS,CAAC,CAAC,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC,CAAC,EAC7C,EAAE,CACH,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,gBAAgB,EAAE,IAAI;YACtB,WAAW;YACX,UAAU,EAAE,CAAC,UAAU,CAAC;SACzB,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,gEAAgE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjH,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAES,SAAS,CAAC,OAAe;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
package/dist/util/config.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { configFile } from '@powersync/service-types';
|
|
1
2
|
import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
|
|
3
|
+
export declare function configureLogger(config?: configFile.LoggingConfig): void;
|
|
2
4
|
/**
|
|
3
5
|
* Loads the resolved config using the registered config collector
|
|
4
6
|
*/
|
package/dist/util/config.js
CHANGED
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
|
-
import
|
|
2
|
+
import winston from 'winston';
|
|
3
|
+
import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
|
|
3
4
|
import { CompoundConfigCollector } from './util-index.js';
|
|
5
|
+
export function configureLogger(config) {
|
|
6
|
+
const level = process.env.PS_LOG_LEVEL ?? config?.level ?? DEFAULT_LOG_LEVEL;
|
|
7
|
+
const format = process.env.PS_LOG_FORMAT ?? config?.format ?? DEFAULT_LOG_FORMAT;
|
|
8
|
+
const winstonFormat = format === 'json' ? LogFormat.production : LogFormat.development;
|
|
9
|
+
// We want the user to always be aware that a log level was configured (they might forget they set it in the config and wonder why they aren't seeing logs)
|
|
10
|
+
// We log this using the configured format, but before we configure the level.
|
|
11
|
+
logger.configure({ level: DEFAULT_LOG_LEVEL, format: winstonFormat, transports: [new winston.transports.Console()] });
|
|
12
|
+
logger.info(`Configured logger with level "${level}" and format "${format}"`);
|
|
13
|
+
logger.configure({ level, format: winstonFormat, transports: [new winston.transports.Console()] });
|
|
14
|
+
}
|
|
4
15
|
/**
|
|
5
16
|
* Loads the resolved config using the registered config collector
|
|
6
17
|
*/
|
|
7
18
|
export async function loadConfig(runnerConfig) {
|
|
8
19
|
const collector = container.getImplementation(CompoundConfigCollector);
|
|
9
|
-
|
|
20
|
+
const config = await collector.collectConfig(runnerConfig);
|
|
21
|
+
configureLogger(config.base_config.system?.logging);
|
|
22
|
+
return config;
|
|
10
23
|
}
|
|
11
24
|
export async function loadSyncRules(config) {
|
|
12
25
|
const sync_rules = config.sync_rules;
|
package/dist/util/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAGxH,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAE1D,MAAM,UAAU,eAAe,CAAC,MAAiC;IAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,MAAM,EAAE,KAAK,IAAI,iBAAiB,CAAC;IAC7E,MAAM,MAAM,GACT,OAAO,CAAC,GAAG,CAAC,aAAoD,IAAI,MAAM,EAAE,MAAM,IAAI,kBAAkB,CAAC;IAC5G,MAAM,aAAa,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC;IAEvF,2JAA2J;IAC3J,8EAA8E;IAC9E,MAAM,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACtH,MAAM,CAAC,IAAI,CAAC,iCAAiC,KAAK,iBAAiB,MAAM,GAAG,CAAC,CAAC;IAE9E,MAAM,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;AACrG,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,YAA0B;IACzD,MAAM,SAAS,GAAG,SAAS,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;IAC3D,eAAe,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAA+B;IACjE,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,UAAU,CAAC,OAAO,CAAC;IAC5B,CAAC;SAAM,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
|
-
"version": "1.
|
|
8
|
+
"version": "1.19.0",
|
|
9
9
|
"main": "dist/index.js",
|
|
10
10
|
"license": "FSL-1.1-ALv2",
|
|
11
11
|
"type": "module",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"uuid": "^11.1.0",
|
|
34
34
|
"winston": "^3.13.0",
|
|
35
35
|
"yaml": "^2.3.2",
|
|
36
|
-
"@powersync/lib-services-framework": "0.
|
|
36
|
+
"@powersync/lib-services-framework": "0.8.0",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.12",
|
|
38
|
-
"@powersync/service-rsocket-router": "0.2.
|
|
39
|
-
"@powersync/service-sync-rules": "0.
|
|
40
|
-
"@powersync/service-types": "0.
|
|
38
|
+
"@powersync/service-rsocket-router": "0.2.12",
|
|
39
|
+
"@powersync/service-sync-rules": "0.30.0",
|
|
40
|
+
"@powersync/service-types": "0.14.0"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/async": "^3.2.24",
|
|
@@ -51,7 +51,7 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
|
|
|
51
51
|
const instanceId = await bucketStorage.getPowerSyncInstanceId();
|
|
52
52
|
resolvedInstanceId(instanceId);
|
|
53
53
|
} catch (err) {
|
|
54
|
-
resolvedInstanceId('
|
|
54
|
+
resolvedInstanceId('unknown');
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
});
|
|
@@ -59,6 +59,8 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr
|
|
|
59
59
|
const resource = resourceFromAttributes({
|
|
60
60
|
['service']: 'PowerSync',
|
|
61
61
|
['service.version']: pkg.version,
|
|
62
|
+
['source_type']: configuration.connections?.[0]?.type ?? 'unknown',
|
|
63
|
+
['storage_type']: configuration.storage.type ?? 'unknown',
|
|
62
64
|
['instance_id']: instanceIdPromise
|
|
63
65
|
});
|
|
64
66
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { container, logger } from '@powersync/lib-services-framework';
|
|
1
|
+
import { container, ErrorCode, logger } from '@powersync/lib-services-framework';
|
|
2
2
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
3
3
|
import { hrtime } from 'node:process';
|
|
4
4
|
import winston from 'winston';
|
|
@@ -36,7 +36,7 @@ export interface AbstractReplicatorOptions {
|
|
|
36
36
|
*/
|
|
37
37
|
export abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
|
|
38
38
|
protected logger: winston.Logger;
|
|
39
|
-
|
|
39
|
+
private lockAlerted: boolean = false;
|
|
40
40
|
/**
|
|
41
41
|
* Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
|
|
42
42
|
* transitioning to a new set of sync rules.
|
|
@@ -225,12 +225,20 @@ export abstract class AbstractReplicator<T extends AbstractReplicationJob = Abst
|
|
|
225
225
|
if (syncRules.active) {
|
|
226
226
|
activeJob = newJob;
|
|
227
227
|
}
|
|
228
|
+
this.lockAlerted = false;
|
|
228
229
|
} catch (e) {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
|
|
231
|
+
if (!this.lockAlerted) {
|
|
232
|
+
this.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
|
|
233
|
+
this.lockAlerted = true;
|
|
234
|
+
}
|
|
235
|
+
} else {
|
|
236
|
+
// Could be a sync rules parse error,
|
|
237
|
+
// for example from stricter validation that was added.
|
|
238
|
+
// This will be retried every couple of seconds.
|
|
239
|
+
// When new (valid) sync rules are deployed and processed, this one be disabled.
|
|
240
|
+
this.logger.error('Failed to start replication for new sync rules', e);
|
|
241
|
+
}
|
|
234
242
|
}
|
|
235
243
|
}
|
|
236
244
|
}
|
|
@@ -177,7 +177,10 @@ export const validate = routeDefinition({
|
|
|
177
177
|
sync_rules: SqlSyncRules.fromYaml(content, {
|
|
178
178
|
...apiHandler.getParseSyncRulesOptions(),
|
|
179
179
|
schema
|
|
180
|
-
})
|
|
180
|
+
}),
|
|
181
|
+
hydratedSyncRules() {
|
|
182
|
+
return this.sync_rules.hydrate();
|
|
183
|
+
}
|
|
181
184
|
};
|
|
182
185
|
},
|
|
183
186
|
sync_rules_content: content,
|
|
@@ -109,10 +109,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
|
|
|
109
109
|
for await (const data of sync.streamResponse({
|
|
110
110
|
syncContext: syncContext,
|
|
111
111
|
bucketStorage: bucketStorage,
|
|
112
|
-
syncRules
|
|
113
|
-
syncRules,
|
|
114
|
-
version: bucketStorage.group_id
|
|
115
|
-
},
|
|
112
|
+
syncRules,
|
|
116
113
|
params: {
|
|
117
114
|
...params
|
|
118
115
|
},
|
|
@@ -202,7 +202,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
|
|
|
202
202
|
|
|
203
203
|
return {
|
|
204
204
|
valid: true,
|
|
205
|
-
bucket_definitions: rules.
|
|
205
|
+
bucket_definitions: rules.debugRepresentation(),
|
|
206
206
|
source_tables: resolved_tables,
|
|
207
207
|
data_tables: rules.debugGetOutputTables()
|
|
208
208
|
};
|
|
@@ -92,10 +92,7 @@ export const syncStreamed = routeDefinition({
|
|
|
92
92
|
const syncLines = sync.streamResponse({
|
|
93
93
|
syncContext: syncContext,
|
|
94
94
|
bucketStorage,
|
|
95
|
-
syncRules
|
|
96
|
-
syncRules,
|
|
97
|
-
version: bucketStorage.group_id
|
|
98
|
-
},
|
|
95
|
+
syncRules,
|
|
99
96
|
params: payload.params,
|
|
100
97
|
token: payload.context.token_payload!,
|
|
101
98
|
tracker,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SqlSyncRules } from '@powersync/service-sync-rules';
|
|
1
|
+
import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules';
|
|
2
2
|
import { ReplicationLock } from './ReplicationLock.js';
|
|
3
3
|
|
|
4
4
|
export interface ParseSyncRulesOptions {
|
|
@@ -30,4 +30,6 @@ export interface PersistedSyncRules {
|
|
|
30
30
|
readonly id: number;
|
|
31
31
|
readonly sync_rules: SqlSyncRules;
|
|
32
32
|
readonly slot_name: string;
|
|
33
|
+
|
|
34
|
+
hydratedSyncRules(): HydratedSyncRules;
|
|
33
35
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Logger, ObserverClient } from '@powersync/lib-services-framework';
|
|
2
|
-
import {
|
|
2
|
+
import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as util from '../util/util-index.js';
|
|
4
4
|
import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js';
|
|
5
5
|
import { BucketStorageFactory } from './BucketStorageFactory.js';
|
|
@@ -32,7 +32,7 @@ export interface SyncRulesBucketStorage
|
|
|
32
32
|
callback: (batch: BucketStorageBatch) => Promise<void>
|
|
33
33
|
): Promise<FlushedResult | null>;
|
|
34
34
|
|
|
35
|
-
getParsedSyncRules(options: ParseSyncRulesOptions):
|
|
35
|
+
getParsedSyncRules(options: ParseSyncRulesOptions): HydratedSyncRules;
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
38
|
* Terminate the sync rules.
|
|
@@ -139,7 +139,7 @@ export interface ResolveTableOptions {
|
|
|
139
139
|
connection_tag: string;
|
|
140
140
|
entity_descriptor: SourceEntityDescriptor;
|
|
141
141
|
|
|
142
|
-
sync_rules:
|
|
142
|
+
sync_rules: HydratedSyncRules;
|
|
143
143
|
}
|
|
144
144
|
|
|
145
145
|
export interface ResolveTableResult {
|
|
@@ -284,7 +284,7 @@ export interface ReplicationCheckpoint {
|
|
|
284
284
|
*
|
|
285
285
|
* This gets parameter sets specific to this checkpoint.
|
|
286
286
|
*/
|
|
287
|
-
getParameterSets(lookups:
|
|
287
|
+
getParameterSets(lookups: ScopedParameterLookup[]): Promise<SqliteJsonRow[]>;
|
|
288
288
|
}
|
|
289
289
|
|
|
290
290
|
export interface WatchWriteCheckpointOptions {
|
package/src/storage/bson.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as bson from 'bson';
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
|
|
4
4
|
import { ReplicaId } from './BucketStorageBatch.js';
|
|
5
5
|
|
|
6
6
|
type NodeBuffer = Buffer<ArrayBuffer>;
|
|
@@ -27,11 +27,11 @@ export const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions = {
|
|
|
27
27
|
* Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
|
|
28
28
|
* @param lookup
|
|
29
29
|
*/
|
|
30
|
-
export const serializeLookupBuffer = (lookup:
|
|
30
|
+
export const serializeLookupBuffer = (lookup: ScopedParameterLookup): NodeBuffer => {
|
|
31
31
|
return bson.serialize({ l: lookup.values }) as NodeBuffer;
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
-
export const serializeLookup = (lookup:
|
|
34
|
+
export const serializeLookup = (lookup: ScopedParameterLookup) => {
|
|
35
35
|
return new bson.Binary(serializeLookupBuffer(lookup));
|
|
36
36
|
};
|
|
37
37
|
|
|
@@ -2,37 +2,32 @@ import {
|
|
|
2
2
|
BucketDescription,
|
|
3
3
|
BucketPriority,
|
|
4
4
|
BucketSource,
|
|
5
|
+
HydratedSyncRules,
|
|
5
6
|
RequestedStream,
|
|
6
7
|
RequestJwtPayload,
|
|
7
8
|
RequestParameters,
|
|
8
|
-
ResolvedBucket
|
|
9
|
-
SqlSyncRules
|
|
9
|
+
ResolvedBucket
|
|
10
10
|
} from '@powersync/service-sync-rules';
|
|
11
11
|
|
|
12
12
|
import * as storage from '../storage/storage-index.js';
|
|
13
13
|
import * as util from '../util/util-index.js';
|
|
14
14
|
|
|
15
15
|
import {
|
|
16
|
+
logger as defaultLogger,
|
|
16
17
|
ErrorCode,
|
|
17
18
|
Logger,
|
|
18
19
|
ServiceAssertionError,
|
|
19
|
-
ServiceError
|
|
20
|
-
logger as defaultLogger
|
|
20
|
+
ServiceError
|
|
21
21
|
} from '@powersync/lib-services-framework';
|
|
22
22
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
23
23
|
import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
|
|
24
24
|
import { SyncContext } from './SyncContext.js';
|
|
25
25
|
import { getIntersection, hasIntersection } from './util.js';
|
|
26
26
|
|
|
27
|
-
export interface VersionedSyncRules {
|
|
28
|
-
syncRules: SqlSyncRules;
|
|
29
|
-
version: number;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
27
|
export interface BucketChecksumStateOptions {
|
|
33
28
|
syncContext: SyncContext;
|
|
34
29
|
bucketStorage: BucketChecksumStateStorage;
|
|
35
|
-
syncRules:
|
|
30
|
+
syncRules: HydratedSyncRules;
|
|
36
31
|
tokenPayload: RequestJwtPayload;
|
|
37
32
|
syncRequest: util.StreamingSyncRequest;
|
|
38
33
|
logger?: Logger;
|
|
@@ -253,7 +248,7 @@ export class BucketChecksumState {
|
|
|
253
248
|
const streamNameToIndex = new Map<string, number>();
|
|
254
249
|
this.streamNameToIndex = streamNameToIndex;
|
|
255
250
|
|
|
256
|
-
for (const source of this.parameterState.syncRules.
|
|
251
|
+
for (const source of this.parameterState.syncRules.definition.bucketSources) {
|
|
257
252
|
if (this.parameterState.isSubscribedToStream(source)) {
|
|
258
253
|
streamNameToIndex.set(source.name, subscriptions.length);
|
|
259
254
|
|
|
@@ -381,7 +376,7 @@ export interface CheckpointUpdate {
|
|
|
381
376
|
export class BucketParameterState {
|
|
382
377
|
private readonly context: SyncContext;
|
|
383
378
|
public readonly bucketStorage: BucketChecksumStateStorage;
|
|
384
|
-
public readonly syncRules:
|
|
379
|
+
public readonly syncRules: HydratedSyncRules;
|
|
385
380
|
public readonly syncParams: RequestParameters;
|
|
386
381
|
private readonly querier: BucketParameterQuerier;
|
|
387
382
|
/**
|
|
@@ -399,12 +394,12 @@ export class BucketParameterState {
|
|
|
399
394
|
private cachedDynamicBuckets: ResolvedBucket[] | null = null;
|
|
400
395
|
private cachedDynamicBucketSet: Set<string> | null = null;
|
|
401
396
|
|
|
402
|
-
private
|
|
397
|
+
private lookupsFromPreviousCheckpoint: Set<string> | null = null;
|
|
403
398
|
|
|
404
399
|
constructor(
|
|
405
400
|
context: SyncContext,
|
|
406
401
|
bucketStorage: BucketChecksumStateStorage,
|
|
407
|
-
syncRules:
|
|
402
|
+
syncRules: HydratedSyncRules,
|
|
408
403
|
tokenPayload: RequestJwtPayload,
|
|
409
404
|
request: util.StreamingSyncRequest,
|
|
410
405
|
logger: Logger
|
|
@@ -436,11 +431,10 @@ export class BucketParameterState {
|
|
|
436
431
|
this.includeDefaultStreams = subscriptions?.include_defaults ?? true;
|
|
437
432
|
this.explicitStreamSubscriptions = explicitStreamSubscriptions;
|
|
438
433
|
|
|
439
|
-
const { querier, errors } = syncRules.
|
|
434
|
+
const { querier, errors } = syncRules.getBucketParameterQuerier({
|
|
440
435
|
globalParameters: this.syncParams,
|
|
441
436
|
hasDefaultStreams: this.includeDefaultStreams,
|
|
442
|
-
streams: streamsByName
|
|
443
|
-
bucketIdTransformer: SqlSyncRules.versionedBucketIdTransformer(`${syncRules.version}`)
|
|
437
|
+
streams: streamsByName
|
|
444
438
|
});
|
|
445
439
|
this.querier = querier;
|
|
446
440
|
this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record<string, QuerierError[]>;
|
|
@@ -448,7 +442,6 @@ export class BucketParameterState {
|
|
|
448
442
|
this.staticBuckets = new Map<string, ResolvedBucket>(
|
|
449
443
|
mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b])
|
|
450
444
|
);
|
|
451
|
-
this.lookups = new Set<string>(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values)));
|
|
452
445
|
this.subscribedStreamNames = new Set(Object.keys(streamsByName));
|
|
453
446
|
}
|
|
454
447
|
|
|
@@ -547,7 +540,6 @@ export class BucketParameterState {
|
|
|
547
540
|
*/
|
|
548
541
|
private async getCheckpointUpdateDynamic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
|
|
549
542
|
const querier = this.querier;
|
|
550
|
-
const storage = this.bucketStorage;
|
|
551
543
|
const staticBuckets = this.staticBuckets.values();
|
|
552
544
|
const update = checkpoint.update;
|
|
553
545
|
|
|
@@ -561,10 +553,10 @@ export class BucketParameterState {
|
|
|
561
553
|
invalidateDataBuckets = true;
|
|
562
554
|
}
|
|
563
555
|
|
|
564
|
-
if (update.invalidateParameterBuckets) {
|
|
556
|
+
if (update.invalidateParameterBuckets || this.lookupsFromPreviousCheckpoint == null) {
|
|
565
557
|
hasParameterChange = true;
|
|
566
558
|
} else {
|
|
567
|
-
if (hasIntersection(this.
|
|
559
|
+
if (hasIntersection(this.lookupsFromPreviousCheckpoint, update.updatedParameterLookups)) {
|
|
568
560
|
// This is a very coarse re-check of all queries
|
|
569
561
|
hasParameterChange = true;
|
|
570
562
|
}
|
|
@@ -572,13 +564,20 @@ export class BucketParameterState {
|
|
|
572
564
|
|
|
573
565
|
let dynamicBuckets: ResolvedBucket[];
|
|
574
566
|
if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
|
|
567
|
+
const recordedLookups = new Set<string>();
|
|
568
|
+
|
|
575
569
|
dynamicBuckets = await querier.queryDynamicBucketDescriptions({
|
|
576
570
|
getParameterSets(lookups) {
|
|
571
|
+
for (const lookup of lookups) {
|
|
572
|
+
recordedLookups.add(lookup.serializedRepresentation);
|
|
573
|
+
}
|
|
574
|
+
|
|
577
575
|
return checkpoint.base.getParameterSets(lookups);
|
|
578
576
|
}
|
|
579
577
|
});
|
|
580
578
|
this.cachedDynamicBuckets = dynamicBuckets;
|
|
581
579
|
this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
|
|
580
|
+
this.lookupsFromPreviousCheckpoint = recordedLookups;
|
|
582
581
|
invalidateDataBuckets = true;
|
|
583
582
|
} else {
|
|
584
583
|
dynamicBuckets = this.cachedDynamicBuckets;
|
package/src/sync/sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
|
|
2
|
-
import { BucketDescription, BucketPriority, RequestJwtPayload } from '@powersync/service-sync-rules';
|
|
2
|
+
import { BucketDescription, BucketPriority, RequestJwtPayload, HydratedSyncRules } from '@powersync/service-sync-rules';
|
|
3
3
|
|
|
4
4
|
import { AbortError } from 'ix/aborterror.js';
|
|
5
5
|
|
|
@@ -9,7 +9,7 @@ import * as util from '../util/util-index.js';
|
|
|
9
9
|
|
|
10
10
|
import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework';
|
|
11
11
|
import { mergeAsyncIterables } from '../streams/streams-index.js';
|
|
12
|
-
import { BucketChecksumState, CheckpointLine
|
|
12
|
+
import { BucketChecksumState, CheckpointLine } from './BucketChecksumState.js';
|
|
13
13
|
import { OperationsSentStats, RequestTracker, statsForBatch } from './RequestTracker.js';
|
|
14
14
|
import { SyncContext } from './SyncContext.js';
|
|
15
15
|
import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStream } from './util.js';
|
|
@@ -17,7 +17,7 @@ import { TokenStreamOptions, acquireSemaphoreAbortable, settledPromise, tokenStr
|
|
|
17
17
|
export interface SyncStreamParameters {
|
|
18
18
|
syncContext: SyncContext;
|
|
19
19
|
bucketStorage: storage.SyncRulesBucketStorage;
|
|
20
|
-
syncRules:
|
|
20
|
+
syncRules: HydratedSyncRules;
|
|
21
21
|
params: util.StreamingSyncRequest;
|
|
22
22
|
token: auth.JwtPayload;
|
|
23
23
|
logger?: Logger;
|
|
@@ -94,7 +94,7 @@ export async function* streamResponse(
|
|
|
94
94
|
async function* streamResponseInner(
|
|
95
95
|
syncContext: SyncContext,
|
|
96
96
|
bucketStorage: storage.SyncRulesBucketStorage,
|
|
97
|
-
syncRules:
|
|
97
|
+
syncRules: HydratedSyncRules,
|
|
98
98
|
params: util.StreamingSyncRequest,
|
|
99
99
|
tokenPayload: RequestJwtPayload,
|
|
100
100
|
tracker: RequestTracker,
|
|
@@ -70,13 +70,20 @@ export abstract class ConfigCollector {
|
|
|
70
70
|
return this.parseJSON(content);
|
|
71
71
|
default: {
|
|
72
72
|
// No content type provided, need to try both
|
|
73
|
+
let yamlError: unknown;
|
|
73
74
|
try {
|
|
74
75
|
return this.parseYaml(content);
|
|
75
|
-
} catch (ex) {
|
|
76
|
+
} catch (ex) {
|
|
77
|
+
yamlError = ex;
|
|
78
|
+
}
|
|
76
79
|
try {
|
|
77
80
|
return this.parseJSON(content);
|
|
78
81
|
} catch (ex) {
|
|
79
|
-
throw new Error(
|
|
82
|
+
throw new Error(
|
|
83
|
+
`Could not parse PowerSync config file content as JSON or YAML: JSON Error: ${ex}${
|
|
84
|
+
yamlError ? `\nYAML Error: ${yamlError}` : ''
|
|
85
|
+
}`
|
|
86
|
+
);
|
|
80
87
|
}
|
|
81
88
|
}
|
|
82
89
|
}
|
package/src/util/config.ts
CHANGED
|
@@ -1,15 +1,33 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
|
+
import winston from 'winston';
|
|
2
3
|
|
|
3
|
-
import { container } from '@powersync/lib-services-framework';
|
|
4
|
+
import { container, logger, LogFormat, DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT } from '@powersync/lib-services-framework';
|
|
5
|
+
import { configFile } from '@powersync/service-types';
|
|
4
6
|
import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
|
|
5
7
|
import { CompoundConfigCollector } from './util-index.js';
|
|
6
8
|
|
|
9
|
+
export function configureLogger(config?: configFile.LoggingConfig): void {
|
|
10
|
+
const level = process.env.PS_LOG_LEVEL ?? config?.level ?? DEFAULT_LOG_LEVEL;
|
|
11
|
+
const format =
|
|
12
|
+
(process.env.PS_LOG_FORMAT as configFile.LoggingConfig['format']) ?? config?.format ?? DEFAULT_LOG_FORMAT;
|
|
13
|
+
const winstonFormat = format === 'json' ? LogFormat.production : LogFormat.development;
|
|
14
|
+
|
|
15
|
+
// We want the user to always be aware that a log level was configured (they might forget they set it in the config and wonder why they aren't seeing logs)
|
|
16
|
+
// We log this using the configured format, but before we configure the level.
|
|
17
|
+
logger.configure({ level: DEFAULT_LOG_LEVEL, format: winstonFormat, transports: [new winston.transports.Console()] });
|
|
18
|
+
logger.info(`Configured logger with level "${level}" and format "${format}"`);
|
|
19
|
+
|
|
20
|
+
logger.configure({ level, format: winstonFormat, transports: [new winston.transports.Console()] });
|
|
21
|
+
}
|
|
22
|
+
|
|
7
23
|
/**
|
|
8
24
|
* Loads the resolved config using the registered config collector
|
|
9
25
|
*/
|
|
10
26
|
export async function loadConfig(runnerConfig: RunnerConfig) {
|
|
11
27
|
const collector = container.getImplementation(CompoundConfigCollector);
|
|
12
|
-
|
|
28
|
+
const config = await collector.collectConfig(runnerConfig);
|
|
29
|
+
configureLogger(config.base_config.system?.logging);
|
|
30
|
+
return config;
|
|
13
31
|
}
|
|
14
32
|
|
|
15
33
|
export async function loadSyncRules(config: ResolvedPowerSyncConfig): Promise<string | undefined> {
|
package/test/src/config.test.ts
CHANGED
|
@@ -69,4 +69,21 @@ describe('Config', () => {
|
|
|
69
69
|
|
|
70
70
|
expect(config.api_parameters.max_buckets_per_connection).toBe(1);
|
|
71
71
|
});
|
|
72
|
+
it('should throw YAML validation error for invalid base64 config', {}, async () => {
|
|
73
|
+
const yamlConfig = /* yaml */ `
|
|
74
|
+
# PowerSync config
|
|
75
|
+
replication:
|
|
76
|
+
connections: []
|
|
77
|
+
storage:
|
|
78
|
+
type: !env INVALID_VAR
|
|
79
|
+
`;
|
|
80
|
+
|
|
81
|
+
const collector = new CompoundConfigCollector();
|
|
82
|
+
|
|
83
|
+
await expect(
|
|
84
|
+
collector.collectConfig({
|
|
85
|
+
config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
|
|
86
|
+
})
|
|
87
|
+
).rejects.toThrow(/YAML Error:[\s\S]*Attempting to substitute environment variable INVALID_VAR/);
|
|
88
|
+
});
|
|
72
89
|
});
|
|
@@ -45,7 +45,7 @@ describe('Stream Route', () => {
|
|
|
45
45
|
|
|
46
46
|
const storage = {
|
|
47
47
|
getParsedSyncRules() {
|
|
48
|
-
return new SqlSyncRules('bucket_definitions: {}');
|
|
48
|
+
return new SqlSyncRules('bucket_definitions: {}').hydrate();
|
|
49
49
|
},
|
|
50
50
|
watchCheckpointChanges: async function* (options) {
|
|
51
51
|
throw new Error('Simulated storage error');
|
|
@@ -83,7 +83,7 @@ describe('Stream Route', () => {
|
|
|
83
83
|
it('logs the application metadata', async () => {
|
|
84
84
|
const storage = {
|
|
85
85
|
getParsedSyncRules() {
|
|
86
|
-
return new SqlSyncRules('bucket_definitions: {}');
|
|
86
|
+
return new SqlSyncRules('bucket_definitions: {}').hydrate();
|
|
87
87
|
},
|
|
88
88
|
watchCheckpointChanges: async function* (options) {
|
|
89
89
|
throw new Error('Simulated storage error');
|