@dxos/echo-pipeline 0.1.53-main.52ca4e9 → 0.1.53-main.5bcfb84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-SCMKAWNW.mjs → chunk-RFRZMQ5T.mjs} +95 -63
- package/dist/lib/browser/chunk-RFRZMQ5T.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +2 -2
- package/dist/lib/browser/testing/index.mjs.map +2 -2
- package/dist/lib/node/index.cjs +96 -64
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +102 -70
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/dbhost/data-service-host.d.ts +3 -0
- package/dist/types/src/dbhost/data-service-host.d.ts.map +1 -1
- package/dist/types/src/dbhost/data-service.d.ts +2 -2
- package/dist/types/src/dbhost/data-service.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +4 -0
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +0 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +3 -3
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +0 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/package.json +30 -30
- package/src/dbhost/data-service-host.ts +14 -1
- package/src/dbhost/data-service.ts +6 -2
- package/src/metadata/metadata-store.ts +15 -2
- package/src/pipeline/pipeline.ts +13 -1
- package/src/space/data-pipeline.ts +25 -19
- package/src/space/space.test.ts +46 -5
- package/src/space/space.ts +3 -12
- package/src/testing/util.ts +1 -1
- package/src/tests/database.test.ts +1 -1
- package/dist/lib/browser/chunk-SCMKAWNW.mjs.map +0 -7
|
@@ -11,8 +11,11 @@ export declare class DataServiceHost {
|
|
|
11
11
|
private readonly _itemManager;
|
|
12
12
|
private readonly _itemDemuxer;
|
|
13
13
|
private readonly _writeStream?;
|
|
14
|
+
private readonly _ctx;
|
|
14
15
|
private readonly _clientTagMap;
|
|
15
16
|
constructor(_itemManager: ItemManager, _itemDemuxer: ItemDemuxer, _writeStream?: FeedWriter<DataMessage> | undefined);
|
|
17
|
+
open(): Promise<void>;
|
|
18
|
+
close(): Promise<void>;
|
|
16
19
|
/**
|
|
17
20
|
* Real-time subscription to data objects in a space.
|
|
18
21
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-service-host.d.ts","sourceRoot":"","sources":["../../../../src/dbhost/data-service-host.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"data-service-host.d.ts","sourceRoot":"","sources":["../../../../src/dbhost/data-service-host.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAuB,WAAW,EAAE,WAAW,EAAuB,MAAM,eAAe,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAG9C,OAAO,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAEnE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,yCAAyC,CAAC;AAGnG;;;GAGG;AAEH,qBAAa,eAAe;IAQxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;IAThC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IAEtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAE5B;gBAGiB,YAAY,EAAE,WAAW,EACzB,YAAY,EAAE,WAAW,EACzB,YAAY,CAAC,qCAAyB;IAGnD,IAAI;IAEJ,KAAK;IAIX;;OAEG;IACH,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IA2CxB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;CAqB7D"}
|
|
@@ -5,8 +5,8 @@ import { DataServiceHost } from './data-service-host';
|
|
|
5
5
|
export declare class DataServiceSubscriptions {
|
|
6
6
|
private readonly _spaces;
|
|
7
7
|
clear(): void;
|
|
8
|
-
registerSpace(spaceKey: PublicKey, host: DataServiceHost): void
|
|
9
|
-
unregisterSpace(spaceKey: PublicKey): void
|
|
8
|
+
registerSpace(spaceKey: PublicKey, host: DataServiceHost): Promise<void>;
|
|
9
|
+
unregisterSpace(spaceKey: PublicKey): Promise<void>;
|
|
10
10
|
getDataService(spaceKey: PublicKey): DataServiceHost | undefined;
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-service.d.ts","sourceRoot":"","sources":["../../../../src/dbhost/data-service.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,YAAY,EACb,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8D;IAEtF,KAAK;
|
|
1
|
+
{"version":3,"file":"data-service.d.ts","sourceRoot":"","sources":["../../../../src/dbhost/data-service.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,YAAY,EACb,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,qBAAa,wBAAwB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8D;IAEtF,KAAK;IAIC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,eAAe;IAOxD,eAAe,CAAC,QAAQ,EAAE,SAAS;IAOzC,cAAc,CAAC,QAAQ,EAAE,SAAS;CAGnC;AAED;;GAEG;AAEH,qBAAa,eAAgB,YAAW,WAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc;gBAAd,cAAc,EAAE,wBAAwB;IAErE,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC;IAOvD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;CAOvD"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Event } from '@dxos/async';
|
|
2
2
|
import { PublicKey } from '@dxos/keys';
|
|
3
|
+
import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
|
|
3
4
|
import { EchoMetadata, SpaceMetadata, IdentityRecord, SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
|
|
4
5
|
import { Directory } from '@dxos/random-access-storage';
|
|
5
6
|
import { Timeframe } from '@dxos/timeframe';
|
|
@@ -7,6 +8,7 @@ export interface AddSpaceOptions {
|
|
|
7
8
|
key: PublicKey;
|
|
8
9
|
genesisFeed: PublicKey;
|
|
9
10
|
}
|
|
11
|
+
declare const EchoMetadata: import("@dxos/codec-protobuf").ProtoCodec<EchoMetadata>;
|
|
10
12
|
export declare class MetadataStore {
|
|
11
13
|
private readonly _directory;
|
|
12
14
|
private _metadata;
|
|
@@ -36,5 +38,7 @@ export declare class MetadataStore {
|
|
|
36
38
|
setSpaceControlLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe): Promise<void>;
|
|
37
39
|
setCache(spaceKey: PublicKey, cache: SpaceCache): Promise<void>;
|
|
38
40
|
setWritableFeedKeys(spaceKey: PublicKey, controlFeedKey: PublicKey, dataFeedKey: PublicKey): Promise<void>;
|
|
41
|
+
setSpaceState(spaceKey: PublicKey, state: SpaceState): Promise<void>;
|
|
39
42
|
}
|
|
43
|
+
export {};
|
|
40
44
|
//# sourceMappingURL=metadata-store.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metadata-store.d.ts","sourceRoot":"","sources":["../../../../src/metadata/metadata-store.ts"],"names":[],"mappings":"AAOA,OAAO,EAAgB,KAAK,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,0CAA0C,CAAC;AACnH,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,SAAS,CAAC;CACxB;AASD,qBAAa,aAAa;IAMtB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAL7B,OAAO,CAAC,SAAS,CAAqC;IACtD,SAAgB,MAAM,sBAA6B;gBAIhC,UAAU,EAAE,SAAS;IAGxC,IAAI,QAAQ,IAAI,YAAY,CAE3B;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;;OAGG;IACH,IAAI,MAAM,IAAI,aAAa,EAAE,CAE5B;IAED;;OAEG;IAEG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"metadata-store.d.ts","sourceRoot":"","sources":["../../../../src/metadata/metadata-store.ts"],"names":[],"mappings":"AAOA,OAAO,EAAgB,KAAK,EAAE,MAAM,aAAa,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,UAAU,EAAE,MAAM,4CAA4C,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,0CAA0C,CAAC;AACnH,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,SAAS,CAAC;IACf,WAAW,EAAE,SAAS,CAAC;CACxB;AASD,QAAA,MAAM,YAAY,yDAA4D,CAAC;AAE/E,qBAAa,aAAa;IAMtB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAL7B,OAAO,CAAC,SAAS,CAAqC;IACtD,SAAgB,MAAM,sBAA6B;gBAIhC,UAAU,EAAE,SAAS;IAGxC,IAAI,QAAQ,IAAI,YAAY,CAE3B;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;;OAGG;IACH,IAAI,MAAM,IAAI,aAAa,EAAE,CAE5B;IAED;;OAEG;IAEG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAsCb,KAAK;IA8BnB,SAAS,CAAC,QAAQ,EAAE,SAAS,GAAG,aAAa;IAW7C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B,iBAAiB,IAAI,cAAc,GAAG,SAAS;IAIzC,iBAAiB,CAAC,MAAM,EAAE,cAAc;IAOxC,QAAQ,CAAC,MAAM,EAAE,aAAa;IAU9B,2BAA2B,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS;IAKrE,8BAA8B,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS;IAKxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU;IAK/C,mBAAmB,CAAC,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS;IAO1F,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU;CAI3D"}
|
|
@@ -30,7 +30,6 @@ export declare class PipelineState {
|
|
|
30
30
|
* Target timeframe we are waiting to reach.
|
|
31
31
|
*/
|
|
32
32
|
private _targetTimeframe;
|
|
33
|
-
private _reachedTargetPromise;
|
|
34
33
|
constructor(_feeds: ComplexMap<PublicKey, FeedWrapper<FeedMessage>>, _timeframeClock: TimeframeClock);
|
|
35
34
|
/**
|
|
36
35
|
* Latest theoretical timeframe based on the last mutation in each feed.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAgC,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,WAAW,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAIxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAE1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAgC,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,WAAW,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAIxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAE1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IA4BtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAtBzB,SAAgB,eAAe,mBAA+B;IAE9D,SAAgB,OAAO,cAAe;IAOtC;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;gBAStC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,EACvD,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,cAAc,cAEjB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,gBAAgB,cAEnB;IAED,IAAI,eAAe,cAElB;IAED,IAAI,KAAK,+BAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,GACpB,GAAE,4BAAiC;CAwCrC;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuC;IAEvE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,gBAAgB,CAAC,CAA+B;IAGxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAE3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAEtD,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO,CAAC,UAAU,CAAS;IAE3B,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,SAAS,CAAS;IAG1B,OAAO,CAAC,OAAO,CAA8C;IAE7D,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAQ5C,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAcrC,KAAK;IASL,IAAI;IAYV;;OAEG;IAEG,SAAS,CAAC,SAAS,EAAE,SAAS;IAkBpC;;OAEG;IAEG,KAAK;IAYL,OAAO;IAQb;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;IAoCjD,OAAO,CAAC,qBAAqB;YAWf,aAAa;CAc5B"}
|
|
@@ -29,7 +29,7 @@ export type DataPipelineParams = {
|
|
|
29
29
|
* Consumes the pipeline and updates the database.
|
|
30
30
|
* Reacts to new epochs to restart the pipeline.
|
|
31
31
|
*/
|
|
32
|
-
export declare class DataPipeline {
|
|
32
|
+
export declare class DataPipeline implements CredentialProcessor {
|
|
33
33
|
private readonly _params;
|
|
34
34
|
private _ctx;
|
|
35
35
|
private _pipeline?;
|
|
@@ -50,13 +50,13 @@ export declare class DataPipeline {
|
|
|
50
50
|
*/
|
|
51
51
|
appliedEpoch?: SpecificCredential<Epoch>;
|
|
52
52
|
private _lastProcessedEpoch;
|
|
53
|
-
onNewEpoch: Event<Credential>;
|
|
54
53
|
private _epochCtx?;
|
|
54
|
+
onNewEpoch: Event<Credential>;
|
|
55
55
|
get isOpen(): boolean;
|
|
56
56
|
get pipeline(): Pipeline | undefined;
|
|
57
57
|
get pipelineState(): import("../pipeline/pipeline").PipelineState | undefined;
|
|
58
58
|
setTargetTimeframe(timeframe: Timeframe): void;
|
|
59
|
-
|
|
59
|
+
processCredential(credential: Credential): Promise<void>;
|
|
60
60
|
open(): Promise<void>;
|
|
61
61
|
close(): Promise<void>;
|
|
62
62
|
private _consumePipeline;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/data-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAA0C,MAAM,aAAa,CAAC;AAE5E,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,kBAAkB,EAAuB,MAAM,mBAAmB,CAAC;AAC3G,OAAO,EAA2B,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CACvD;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,QAAQ,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/B;;OAEG;IACH,iBAAiB,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAiBF;;;;GAIG;AACH,qBACa,
|
|
1
|
+
{"version":3,"file":"data-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/data-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAA0C,MAAM,aAAa,CAAC;AAE5E,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,kBAAkB,EAAuB,MAAM,mBAAmB,CAAC;AAC3G,OAAO,EAA2B,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CACvD;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,SAAS,CAAC;IACpB,gBAAgB,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,QAAQ,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAE/B;;OAEG;IACH,iBAAiB,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAiBF;;;;GAIG;AACH,qBACa,YAAa,YAAW,mBAAmB;IAW1C,OAAO,CAAC,QAAQ,CAAC,OAAO;IAVpC,OAAO,CAAC,IAAI,CAAiB;IAC7B,OAAO,CAAC,SAAS,CAAC,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAwB;IAEjD,OAAO,CAAC,+BAA+B,CAAmB;IAC1D,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,qBAAqB,CAAK;gBAEL,OAAO,EAAE,kBAAkB;IAEjD,WAAW,EAAG,WAAW,CAAC;IAC1B,YAAY,CAAC,EAAE,YAAY,CAAC;IAEnC;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D;;OAEG;IACI,YAAY,CAAC,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAa;IAE5D,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,SAAS,CAAC,CAAU;IAErB,UAAU,oBAA2B;IAE5C,IAAI,MAAM,YAET;IAED,IAAI,QAAQ,yBAEX;IAED,IAAI,aAAa,6DAEhB;IAED,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAKjC,iBAAiB,CAAC,UAAU,EAAE,UAAU;IAaxC,IAAI;IAsCJ,KAAK;YAiCG,gBAAgB;IA6C9B,OAAO,CAAC,eAAe;YAST,oBAAoB;YAMpB,UAAU;YAsBV,wBAAwB;YAiBxB,2BAA2B;YA6B3B,aAAa;IAoBrB,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAMvC,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;IAqB7B,sBAAsB;CAG7B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"space.d.ts","sourceRoot":"","sources":["../../../../src/space/space.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAkC,MAAM,aAAa,CAAC;AACpE,OAAO,
|
|
1
|
+
{"version":3,"file":"space.d.ts","sourceRoot":"","sources":["../../../../src/space/space.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAkC,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAgB,UAAU,EAAE,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGjD,KAAK,YAAY,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;AAElG,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,SAAS,CAAC;IACpB,QAAQ,EAAE,aAAa,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,YAAY,EAAE,YAAY,CAAC;IAC3B,YAAY,EAAE,YAAY,CAAC;IAC3B,aAAa,EAAE,aAAa,CAAC;IAC7B,eAAe,EAAE,eAAe,CAAC;IACjC,SAAS,EAAE,SAAS,CAAC;IAGrB,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,SAAS,CAAC;CAElB,CAAC;AAEF;;GAEG;AAEH,qBACa,KAAK;IAChB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAE3C,SAAgB,qBAAqB,sCAA6C;IAClF,SAAgB,WAAW,cAAe;IAC1C,SAAgB,QAAQ,EAAE,aAAa,CAAC;IAExC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAY;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAY;IAC5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAkB;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;IAC7C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAkB;IAEnD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAC,CAA2B;IAChD,OAAO,CAAC,SAAS,CAAC,CAA2B;gBAEjC,MAAM,EAAE,WAAW;IAsE/B,IACI,GAAG,cAEN;IAED,IAAI,MAAM,YAET;IAED,IAAI,cAAc,IAAI,SAAS,CAE9B;IAED,IAAI,cAAc,0BAEjB;IAED,IAAI,WAAW,0BAEd;IAED,IAAI,UAAU,2CAEb;IAED;;OAEG;IACH,IAAI,eAAe,IAAI,gBAAgB,CAEtC;IAED,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAO7C,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAO1C;;OAEG;IACH,eAAe,IAAI,QAAQ,EAAE;IAI7B;;OAEG;IAKG,IAAI;IAgBJ,KAAK;IAkBL,sBAAsB;CAK7B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/echo-pipeline",
|
|
3
|
-
"version": "0.1.53-main.
|
|
3
|
+
"version": "0.1.53-main.5bcfb84",
|
|
4
4
|
"description": "ECHO database.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -21,41 +21,41 @@
|
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"crc-32": "^1.2.2",
|
|
23
23
|
"tiny-invariant": "^1.3.1",
|
|
24
|
-
"@dxos/async": "0.1.53-main.
|
|
25
|
-
"@dxos/codec-protobuf": "0.1.53-main.
|
|
26
|
-
"@dxos/context": "0.1.53-main.
|
|
27
|
-
"@dxos/credentials": "0.1.53-main.
|
|
28
|
-
"@dxos/crypto": "0.1.53-main.
|
|
29
|
-
"@dxos/debug": "0.1.53-main.
|
|
30
|
-
"@dxos/document-model": "0.1.53-main.
|
|
31
|
-
"@dxos/echo-db": "0.1.53-main.
|
|
32
|
-
"@dxos/errors": "0.1.53-main.
|
|
33
|
-
"@dxos/feed-store": "0.1.53-main.
|
|
34
|
-
"@dxos/hypercore": "0.1.53-main.
|
|
35
|
-
"@dxos/keyring": "0.1.53-main.
|
|
36
|
-
"@dxos/keys": "0.1.53-main.
|
|
37
|
-
"@dxos/log": "0.1.53-main.
|
|
38
|
-
"@dxos/messaging": "0.1.53-main.
|
|
39
|
-
"@dxos/model-factory": "0.1.53-main.
|
|
40
|
-
"@dxos/network-manager": "0.1.53-main.
|
|
41
|
-
"@dxos/node-std": "0.1.53-main.
|
|
42
|
-
"@dxos/protocols": "0.1.53-main.
|
|
43
|
-
"@dxos/random-access-storage": "0.1.53-main.
|
|
44
|
-
"@dxos/rpc": "0.1.53-main.
|
|
45
|
-
"@dxos/teleport": "0.1.53-main.
|
|
46
|
-
"@dxos/teleport-extension-gossip": "0.1.53-main.
|
|
47
|
-
"@dxos/teleport-extension-object-sync": "0.1.53-main.
|
|
48
|
-
"@dxos/teleport-extension-replicator": "0.1.53-main.
|
|
49
|
-
"@dxos/timeframe": "0.1.53-main.
|
|
50
|
-
"@dxos/typings": "0.1.53-main.
|
|
51
|
-
"@dxos/util": "0.1.53-main.
|
|
24
|
+
"@dxos/async": "0.1.53-main.5bcfb84",
|
|
25
|
+
"@dxos/codec-protobuf": "0.1.53-main.5bcfb84",
|
|
26
|
+
"@dxos/context": "0.1.53-main.5bcfb84",
|
|
27
|
+
"@dxos/credentials": "0.1.53-main.5bcfb84",
|
|
28
|
+
"@dxos/crypto": "0.1.53-main.5bcfb84",
|
|
29
|
+
"@dxos/debug": "0.1.53-main.5bcfb84",
|
|
30
|
+
"@dxos/document-model": "0.1.53-main.5bcfb84",
|
|
31
|
+
"@dxos/echo-db": "0.1.53-main.5bcfb84",
|
|
32
|
+
"@dxos/errors": "0.1.53-main.5bcfb84",
|
|
33
|
+
"@dxos/feed-store": "0.1.53-main.5bcfb84",
|
|
34
|
+
"@dxos/hypercore": "0.1.53-main.5bcfb84",
|
|
35
|
+
"@dxos/keyring": "0.1.53-main.5bcfb84",
|
|
36
|
+
"@dxos/keys": "0.1.53-main.5bcfb84",
|
|
37
|
+
"@dxos/log": "0.1.53-main.5bcfb84",
|
|
38
|
+
"@dxos/messaging": "0.1.53-main.5bcfb84",
|
|
39
|
+
"@dxos/model-factory": "0.1.53-main.5bcfb84",
|
|
40
|
+
"@dxos/network-manager": "0.1.53-main.5bcfb84",
|
|
41
|
+
"@dxos/node-std": "0.1.53-main.5bcfb84",
|
|
42
|
+
"@dxos/protocols": "0.1.53-main.5bcfb84",
|
|
43
|
+
"@dxos/random-access-storage": "0.1.53-main.5bcfb84",
|
|
44
|
+
"@dxos/rpc": "0.1.53-main.5bcfb84",
|
|
45
|
+
"@dxos/teleport": "0.1.53-main.5bcfb84",
|
|
46
|
+
"@dxos/teleport-extension-gossip": "0.1.53-main.5bcfb84",
|
|
47
|
+
"@dxos/teleport-extension-object-sync": "0.1.53-main.5bcfb84",
|
|
48
|
+
"@dxos/teleport-extension-replicator": "0.1.53-main.5bcfb84",
|
|
49
|
+
"@dxos/timeframe": "0.1.53-main.5bcfb84",
|
|
50
|
+
"@dxos/typings": "0.1.53-main.5bcfb84",
|
|
51
|
+
"@dxos/util": "0.1.53-main.5bcfb84"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"fast-check": "~3.3.0",
|
|
55
55
|
"hypercore-protocol": "^8.0.7",
|
|
56
56
|
"source-map-support": "^0.5.12",
|
|
57
57
|
"wait-for-expect": "^3.0.2",
|
|
58
|
-
"@dxos/text-model": "0.1.53-main.
|
|
58
|
+
"@dxos/text-model": "0.1.53-main.5bcfb84"
|
|
59
59
|
},
|
|
60
60
|
"publishConfig": {
|
|
61
61
|
"access": "public"
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import invariant from 'tiny-invariant';
|
|
6
6
|
|
|
7
7
|
import { Stream } from '@dxos/codec-protobuf';
|
|
8
|
+
import { Context } from '@dxos/context';
|
|
8
9
|
import { tagMutationsInBatch, ItemDemuxer, ItemManager, setMetadataOnObject } from '@dxos/echo-db';
|
|
9
10
|
import { FeedWriter } from '@dxos/feed-store';
|
|
10
11
|
import { PublicKey } from '@dxos/keys';
|
|
@@ -20,6 +21,8 @@ import { ComplexMap } from '@dxos/util';
|
|
|
20
21
|
*/
|
|
21
22
|
// TODO(burdon): Move to client-services.
|
|
22
23
|
export class DataServiceHost {
|
|
24
|
+
private readonly _ctx = new Context();
|
|
25
|
+
|
|
23
26
|
private readonly _clientTagMap = new ComplexMap<[feedKey: PublicKey, seq: number], string>(
|
|
24
27
|
([feedKey, seq]) => `${feedKey.toHex()}:${seq}`,
|
|
25
28
|
);
|
|
@@ -30,11 +33,20 @@ export class DataServiceHost {
|
|
|
30
33
|
private readonly _writeStream?: FeedWriter<DataMessage>,
|
|
31
34
|
) {}
|
|
32
35
|
|
|
36
|
+
async open() {}
|
|
37
|
+
|
|
38
|
+
async close() {
|
|
39
|
+
await this._ctx.dispose();
|
|
40
|
+
}
|
|
41
|
+
|
|
33
42
|
/**
|
|
34
43
|
* Real-time subscription to data objects in a space.
|
|
35
44
|
*/
|
|
36
45
|
subscribe(): Stream<EchoEvent> {
|
|
37
|
-
return new Stream(({ next, ctx }) => {
|
|
46
|
+
return new Stream(({ next, close, ctx }) => {
|
|
47
|
+
// This looks ridiculous..
|
|
48
|
+
ctx.onDispose(this._ctx.onDispose(close));
|
|
49
|
+
|
|
38
50
|
// send current state
|
|
39
51
|
const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
|
|
40
52
|
next({
|
|
@@ -74,6 +86,7 @@ export class DataServiceHost {
|
|
|
74
86
|
}
|
|
75
87
|
|
|
76
88
|
async write(request: WriteRequest): Promise<MutationReceipt> {
|
|
89
|
+
invariant(!this._ctx.disposed, 'Cannot write to closed DataServiceHost');
|
|
77
90
|
invariant(this._writeStream, 'Cannot write mutations in readonly mode');
|
|
78
91
|
|
|
79
92
|
log('write', { clientTag: request.clientTag, objectCount: request.batch.objects?.length ?? 0 });
|
|
@@ -27,13 +27,17 @@ export class DataServiceSubscriptions {
|
|
|
27
27
|
this._spaces.clear();
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
registerSpace(spaceKey: PublicKey, host: DataServiceHost) {
|
|
30
|
+
async registerSpace(spaceKey: PublicKey, host: DataServiceHost) {
|
|
31
31
|
log('Registering space', { spaceKey });
|
|
32
|
+
invariant(!this._spaces.has(spaceKey));
|
|
33
|
+
await host.open();
|
|
32
34
|
this._spaces.set(spaceKey, host);
|
|
33
35
|
}
|
|
34
36
|
|
|
35
|
-
unregisterSpace(spaceKey: PublicKey) {
|
|
37
|
+
async unregisterSpace(spaceKey: PublicKey) {
|
|
36
38
|
log('Unregistering space', { spaceKey });
|
|
39
|
+
const host = this._spaces.get(spaceKey);
|
|
40
|
+
await host?.close();
|
|
37
41
|
this._spaces.delete(spaceKey);
|
|
38
42
|
}
|
|
39
43
|
|
|
@@ -10,6 +10,7 @@ import { DataCorruptionError } from '@dxos/errors';
|
|
|
10
10
|
import { PublicKey } from '@dxos/keys';
|
|
11
11
|
import { log } from '@dxos/log';
|
|
12
12
|
import { STORAGE_VERSION, schema } from '@dxos/protocols';
|
|
13
|
+
import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
|
|
13
14
|
import { EchoMetadata, SpaceMetadata, IdentityRecord, SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
|
|
14
15
|
import { Directory } from '@dxos/random-access-storage';
|
|
15
16
|
import { Timeframe } from '@dxos/timeframe';
|
|
@@ -27,6 +28,8 @@ const emptyEchoMetadata = (): EchoMetadata => ({
|
|
|
27
28
|
updated: new Date(),
|
|
28
29
|
});
|
|
29
30
|
|
|
31
|
+
const EchoMetadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata');
|
|
32
|
+
|
|
30
33
|
export class MetadataStore {
|
|
31
34
|
private _metadata: EchoMetadata = emptyEchoMetadata();
|
|
32
35
|
public readonly update = new Event<EchoMetadata>();
|
|
@@ -79,7 +82,12 @@ export class MetadataStore {
|
|
|
79
82
|
throw new DataCorruptionError('Metadata checksum is invalid.');
|
|
80
83
|
}
|
|
81
84
|
|
|
82
|
-
this._metadata =
|
|
85
|
+
this._metadata = EchoMetadata.decode(data);
|
|
86
|
+
|
|
87
|
+
// post-processing
|
|
88
|
+
this._metadata.spaces?.forEach((space) => {
|
|
89
|
+
space.state ??= SpaceState.ACTIVE;
|
|
90
|
+
});
|
|
83
91
|
} catch (err: any) {
|
|
84
92
|
log.error('failed to load metadata', { err });
|
|
85
93
|
this._metadata = emptyEchoMetadata();
|
|
@@ -101,7 +109,7 @@ export class MetadataStore {
|
|
|
101
109
|
const file = this._directory.getOrCreateFile('EchoMetadata');
|
|
102
110
|
|
|
103
111
|
try {
|
|
104
|
-
const encoded = arrayToBuffer(
|
|
112
|
+
const encoded = arrayToBuffer(EchoMetadata.encode(data));
|
|
105
113
|
const checksum = CRC32.buf(encoded);
|
|
106
114
|
|
|
107
115
|
const result = Buffer.alloc(8 + encoded.length);
|
|
@@ -181,6 +189,11 @@ export class MetadataStore {
|
|
|
181
189
|
space.dataFeedKey = dataFeedKey;
|
|
182
190
|
await this._save();
|
|
183
191
|
}
|
|
192
|
+
|
|
193
|
+
async setSpaceState(spaceKey: PublicKey, state: SpaceState) {
|
|
194
|
+
this._getSpace(spaceKey).state = state;
|
|
195
|
+
await this._save();
|
|
196
|
+
}
|
|
184
197
|
}
|
|
185
198
|
|
|
186
199
|
const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);
|
package/src/pipeline/pipeline.ts
CHANGED
|
@@ -36,6 +36,11 @@ export type WaitUntilReachedTargetParams = {
|
|
|
36
36
|
* External state accessor.
|
|
37
37
|
*/
|
|
38
38
|
export class PipelineState {
|
|
39
|
+
/**
|
|
40
|
+
* @internal
|
|
41
|
+
*/
|
|
42
|
+
_ctx = new Context();
|
|
43
|
+
|
|
39
44
|
// TODO(dmaretskyi): Remove?.
|
|
40
45
|
public readonly timeframeUpdate = this._timeframeClock.update;
|
|
41
46
|
|
|
@@ -51,7 +56,10 @@ export class PipelineState {
|
|
|
51
56
|
*/
|
|
52
57
|
private _targetTimeframe: Timeframe | undefined;
|
|
53
58
|
|
|
54
|
-
|
|
59
|
+
/**
|
|
60
|
+
* @internal
|
|
61
|
+
*/
|
|
62
|
+
_reachedTargetPromise: Promise<void> | undefined;
|
|
55
63
|
|
|
56
64
|
// prettier-ignore
|
|
57
65
|
constructor(
|
|
@@ -133,6 +141,7 @@ export class PipelineState {
|
|
|
133
141
|
if (timeout) {
|
|
134
142
|
return Promise.race([
|
|
135
143
|
rejectOnDispose(ctx),
|
|
144
|
+
rejectOnDispose(this._ctx),
|
|
136
145
|
this._reachedTargetPromise.then(() => {
|
|
137
146
|
done = true;
|
|
138
147
|
}),
|
|
@@ -274,6 +283,9 @@ export class Pipeline implements PipelineAccessor {
|
|
|
274
283
|
this._isStopping = true;
|
|
275
284
|
await this._feedSetIterator?.close();
|
|
276
285
|
await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
|
|
286
|
+
await this._state._ctx.dispose();
|
|
287
|
+
this._state._ctx = new Context();
|
|
288
|
+
this._state._reachedTargetPromise = undefined;
|
|
277
289
|
log('stopped');
|
|
278
290
|
this._isStarted = false;
|
|
279
291
|
}
|
|
@@ -65,10 +65,10 @@ const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
|
65
65
|
* Reacts to new epochs to restart the pipeline.
|
|
66
66
|
*/
|
|
67
67
|
@trackLeaks('open', 'close')
|
|
68
|
-
export class DataPipeline {
|
|
68
|
+
export class DataPipeline implements CredentialProcessor {
|
|
69
69
|
private _ctx = new Context();
|
|
70
|
-
private _pipeline?: Pipeline;
|
|
71
|
-
private _targetTimeframe?: Timeframe;
|
|
70
|
+
private _pipeline?: Pipeline = undefined;
|
|
71
|
+
private _targetTimeframe?: Timeframe = undefined;
|
|
72
72
|
|
|
73
73
|
private _lastAutomaticSnapshotTimeframe = new Timeframe();
|
|
74
74
|
private _isOpen = false;
|
|
@@ -84,17 +84,18 @@ export class DataPipeline {
|
|
|
84
84
|
/**
|
|
85
85
|
* Current epoch. Might be still processing.
|
|
86
86
|
*/
|
|
87
|
-
public currentEpoch?: SpecificCredential<Epoch
|
|
87
|
+
public currentEpoch?: SpecificCredential<Epoch> = undefined;
|
|
88
88
|
|
|
89
89
|
/**
|
|
90
90
|
* Epoch currently applied.
|
|
91
91
|
*/
|
|
92
|
-
public appliedEpoch?: SpecificCredential<Epoch
|
|
92
|
+
public appliedEpoch?: SpecificCredential<Epoch> = undefined;
|
|
93
93
|
|
|
94
94
|
private _lastProcessedEpoch = -1;
|
|
95
|
-
public onNewEpoch = new Event<Credential>();
|
|
96
95
|
private _epochCtx?: Context;
|
|
97
96
|
|
|
97
|
+
public onNewEpoch = new Event<Credential>();
|
|
98
|
+
|
|
98
99
|
get isOpen() {
|
|
99
100
|
return this._isOpen;
|
|
100
101
|
}
|
|
@@ -112,20 +113,16 @@ export class DataPipeline {
|
|
|
112
113
|
this._pipeline?.state.setTargetTimeframe(timeframe);
|
|
113
114
|
}
|
|
114
115
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
116
|
+
async processCredential(credential: Credential) {
|
|
117
|
+
if (!checkCredentialType(credential, 'dxos.halo.credentials.Epoch')) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
121
120
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
},
|
|
128
|
-
};
|
|
121
|
+
this.currentEpoch = credential;
|
|
122
|
+
if (this._isOpen) {
|
|
123
|
+
// process epoch
|
|
124
|
+
await this._processEpochInSeparateTask(credential);
|
|
125
|
+
}
|
|
129
126
|
}
|
|
130
127
|
|
|
131
128
|
@synchronized
|
|
@@ -189,6 +186,15 @@ export class DataPipeline {
|
|
|
189
186
|
|
|
190
187
|
await this.databaseHost?.close();
|
|
191
188
|
await this.itemManager?.destroy();
|
|
189
|
+
|
|
190
|
+
this._ctx = new Context();
|
|
191
|
+
this._pipeline = undefined;
|
|
192
|
+
this._targetTimeframe = undefined;
|
|
193
|
+
this._lastAutomaticSnapshotTimeframe = new Timeframe();
|
|
194
|
+
this.currentEpoch = undefined;
|
|
195
|
+
this.appliedEpoch = undefined;
|
|
196
|
+
this._lastProcessedEpoch = -1;
|
|
197
|
+
this._epochCtx = undefined;
|
|
192
198
|
}
|
|
193
199
|
|
|
194
200
|
private async _consumePipeline() {
|
package/src/space/space.test.ts
CHANGED
|
@@ -22,7 +22,6 @@ describe('space/space', () => {
|
|
|
22
22
|
const space = await agent.createSpace();
|
|
23
23
|
|
|
24
24
|
await space.open();
|
|
25
|
-
|
|
26
25
|
expect(space.isOpen).toBeTruthy();
|
|
27
26
|
afterTest(() => space.close());
|
|
28
27
|
|
|
@@ -130,7 +129,6 @@ describe('space/space', () => {
|
|
|
130
129
|
const space1 = await agent.createSpace();
|
|
131
130
|
|
|
132
131
|
await space1.open();
|
|
133
|
-
|
|
134
132
|
expect(space1.isOpen).toBeTruthy();
|
|
135
133
|
afterTest(() => space1.close());
|
|
136
134
|
|
|
@@ -152,7 +150,6 @@ describe('space/space', () => {
|
|
|
152
150
|
const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
|
|
153
151
|
|
|
154
152
|
await space2.open();
|
|
155
|
-
|
|
156
153
|
await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
|
|
157
154
|
await space2.initializeDataPipeline();
|
|
158
155
|
await space2.dataPipeline.ensureEpochInitialized();
|
|
@@ -166,6 +163,52 @@ describe('space/space', () => {
|
|
|
166
163
|
await testLocalDatabase(space2.dataPipeline);
|
|
167
164
|
});
|
|
168
165
|
|
|
166
|
+
test('re-open', async () => {
|
|
167
|
+
const builder = new TestAgentBuilder();
|
|
168
|
+
afterTest(async () => await builder.close());
|
|
169
|
+
const agent = await builder.createPeer();
|
|
170
|
+
const space = await agent.createSpace();
|
|
171
|
+
|
|
172
|
+
let objectCount: number;
|
|
173
|
+
{
|
|
174
|
+
await space.open();
|
|
175
|
+
afterTest(() => space.close());
|
|
176
|
+
expect(space.isOpen).toBeTruthy();
|
|
177
|
+
|
|
178
|
+
await agent.spaceGenesis(space);
|
|
179
|
+
|
|
180
|
+
await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
|
|
181
|
+
await space.initializeDataPipeline();
|
|
182
|
+
await space.dataPipeline.ensureEpochInitialized();
|
|
183
|
+
|
|
184
|
+
assert(space.dataPipeline.databaseHost);
|
|
185
|
+
await testLocalDatabase(space.dataPipeline);
|
|
186
|
+
|
|
187
|
+
objectCount = space.dataPipeline.itemManager.entities.size;
|
|
188
|
+
|
|
189
|
+
await space.close();
|
|
190
|
+
expect(space.isOpen).toBeFalsy();
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Re-open.
|
|
194
|
+
{
|
|
195
|
+
await space.open();
|
|
196
|
+
expect(space.isOpen).toBeTruthy();
|
|
197
|
+
|
|
198
|
+
await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
|
|
199
|
+
await space.initializeDataPipeline();
|
|
200
|
+
await space.dataPipeline.ensureEpochInitialized();
|
|
201
|
+
|
|
202
|
+
space.dataPipeline.setTargetTimeframe(space.dataPipeline.pipelineState!.endTimeframe);
|
|
203
|
+
await space.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
|
|
204
|
+
|
|
205
|
+
assert(space.dataPipeline.databaseHost);
|
|
206
|
+
expect(space.dataPipeline.itemManager.entities.size).toEqual(objectCount);
|
|
207
|
+
|
|
208
|
+
await testLocalDatabase(space.dataPipeline);
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
|
|
169
212
|
test('create epoch', async () => {
|
|
170
213
|
const builder = new TestAgentBuilder();
|
|
171
214
|
afterTest(async () => await builder.close());
|
|
@@ -173,7 +216,6 @@ describe('space/space', () => {
|
|
|
173
216
|
const space1 = await agent.createSpace();
|
|
174
217
|
|
|
175
218
|
await space1.open();
|
|
176
|
-
|
|
177
219
|
expect(space1.isOpen).toBeTruthy();
|
|
178
220
|
afterTest(() => space1.close());
|
|
179
221
|
|
|
@@ -214,7 +256,6 @@ describe('space/space', () => {
|
|
|
214
256
|
const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
|
|
215
257
|
|
|
216
258
|
await space2.open();
|
|
217
|
-
|
|
218
259
|
await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
|
|
219
260
|
await space2.initializeDataPipeline();
|
|
220
261
|
await space2.dataPipeline.ensureEpochInitialized();
|
package/src/space/space.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import invariant from 'tiny-invariant';
|
|
6
6
|
|
|
7
7
|
import { Event, synchronized, trackLeaks, Lock } from '@dxos/async';
|
|
8
|
-
import {
|
|
8
|
+
import { FeedInfo } from '@dxos/credentials';
|
|
9
9
|
import { FeedOptions, FeedWrapper } from '@dxos/feed-store';
|
|
10
10
|
import { PublicKey } from '@dxos/keys';
|
|
11
11
|
import { log, logInfo } from '@dxos/log';
|
|
@@ -63,9 +63,6 @@ export class Space {
|
|
|
63
63
|
private readonly _dataPipeline: DataPipeline;
|
|
64
64
|
private readonly _snapshotManager: SnapshotManager;
|
|
65
65
|
|
|
66
|
-
// Processes epoch credentials.
|
|
67
|
-
private _dataPipelineCredentialConsumer?: CredentialConsumer<any> = undefined;
|
|
68
|
-
|
|
69
66
|
private _isOpen = false;
|
|
70
67
|
private _controlFeed?: FeedWrapper<FeedMessage>;
|
|
71
68
|
private _dataFeed?: FeedWrapper<FeedMessage>;
|
|
@@ -217,12 +214,7 @@ export class Space {
|
|
|
217
214
|
// Order is important.
|
|
218
215
|
await this._controlPipeline.start();
|
|
219
216
|
await this.protocol.start();
|
|
220
|
-
|
|
221
|
-
this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(
|
|
222
|
-
this._dataPipeline.createCredentialProcessor(),
|
|
223
|
-
);
|
|
224
|
-
|
|
225
|
-
await this._dataPipelineCredentialConsumer.open();
|
|
217
|
+
await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
|
|
226
218
|
|
|
227
219
|
this._isOpen = true;
|
|
228
220
|
log('opened');
|
|
@@ -234,8 +226,7 @@ export class Space {
|
|
|
234
226
|
if (!this._isOpen) {
|
|
235
227
|
return;
|
|
236
228
|
}
|
|
237
|
-
await this.
|
|
238
|
-
this._dataPipelineCredentialConsumer = undefined;
|
|
229
|
+
await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
|
|
239
230
|
|
|
240
231
|
await this._dataPipeline.close();
|
|
241
232
|
|
package/src/testing/util.ts
CHANGED
|
@@ -45,7 +45,7 @@ export const createRemoteDatabaseFromDataServiceHost = async (
|
|
|
45
45
|
const dataService = new DataServiceImpl(dataServiceSubscriptions);
|
|
46
46
|
|
|
47
47
|
const spaceKey = PublicKey.random();
|
|
48
|
-
dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
|
|
48
|
+
await dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
|
|
49
49
|
|
|
50
50
|
const itemManager = new ItemManager(modelFactory);
|
|
51
51
|
const backend = new DatabaseProxy(dataService, itemManager, spaceKey);
|
|
@@ -42,7 +42,7 @@ const createDatabaseWithFeeds = async () => {
|
|
|
42
42
|
const dataService = new DataServiceImpl(dataServiceSubscriptions);
|
|
43
43
|
|
|
44
44
|
const spaceKey = PublicKey.random();
|
|
45
|
-
dataServiceSubscriptions.registerSpace(spaceKey, host.createDataServiceHost());
|
|
45
|
+
await dataServiceSubscriptions.registerSpace(spaceKey, host.createDataServiceHost());
|
|
46
46
|
|
|
47
47
|
const proxy = new DatabaseProxy(dataService, new ItemManager(modelFactory), spaceKey);
|
|
48
48
|
await proxy.open(new ModelFactory().registerModel(DocumentModel));
|