@dxos/echo-pipeline 0.7.4 → 0.7.5-feature-compute.4d9d99a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-LZK5YFYE.mjs → chunk-QBMTPEMY.mjs} +111 -38
- package/dist/lib/browser/chunk-QBMTPEMY.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +171 -77
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +1 -1
- package/dist/lib/node/{chunk-MACQJ2EP.cjs → chunk-NPZ57MV5.cjs} +110 -40
- package/dist/lib/node/chunk-NPZ57MV5.cjs.map +7 -0
- package/dist/lib/node/index.cjs +184 -94
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +10 -10
- package/dist/lib/node-esm/{chunk-JIZPSASG.mjs → chunk-OY5N3ZIV.mjs} +111 -38
- package/dist/lib/node-esm/chunk-OY5N3ZIV.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +171 -77
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +1 -1
- package/dist/types/src/automerge/automerge-host.d.ts +5 -1
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -0
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +1 -0
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +3 -2
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/edge/inflight-request-limiter.d.ts +24 -0
- package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +9 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +1 -0
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +34 -34
- package/src/automerge/automerge-host.ts +49 -14
- package/src/automerge/collection-synchronizer.ts +8 -4
- package/src/automerge/echo-network-adapter.ts +7 -0
- package/src/automerge/mesh-echo-replicator.ts +2 -2
- package/src/db-host/echo-host.ts +4 -1
- package/src/edge/echo-edge-replicator.ts +34 -18
- package/src/edge/inflight-request-limiter.ts +69 -0
- package/src/pipeline/pipeline.ts +9 -1
- package/src/space/control-pipeline.ts +25 -2
- package/src/space/space-manager.ts +17 -1
- package/dist/lib/browser/chunk-LZK5YFYE.mjs.map +0 -7
- package/dist/lib/node/chunk-MACQJ2EP.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-JIZPSASG.mjs.map +0 -7
|
@@ -108,6 +108,7 @@ export declare class Pipeline implements PipelineAccessor {
|
|
|
108
108
|
getFeeds(): FeedWrapper<FeedMessage>[];
|
|
109
109
|
addFeed(feed: FeedWrapper<FeedMessage>): Promise<void>;
|
|
110
110
|
setWriteFeed(feed: FeedWrapper<FeedMessage>): void;
|
|
111
|
+
retryMessage(message: FeedMessageBlock): void;
|
|
111
112
|
start(): Promise<void>;
|
|
112
113
|
stop(): Promise<void>;
|
|
113
114
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAA2C,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAgCtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IA1BzB,SAAgB,eAAe,mBAA+B;IAE9D,SAAgB,OAAO,cAAe;IAOtC;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;gBAatC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,EACvD,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,cAAc,cAEjB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,gBAAgB,cAEnB;IAED,IAAI,eAAe,cAElB;IAED,IAAI,aAAa,YAEhB;IAED,IAAI,KAAK,+BAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,GACpB,GAAE,4BAAiC;CAyCrC;AAGD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuC;IACvE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAGtD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuF;IAGlH,OAAO,CAAC,gBAAgB,CAAC,CAA+B;IAGxD,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;IAE1B,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAY5C,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAA2C,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAgCtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IA1BzB,SAAgB,eAAe,mBAA+B;IAE9D,SAAgB,OAAO,cAAe;IAOtC;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;gBAatC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,EACvD,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,cAAc,cAEjB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,gBAAgB,cAEnB;IAED,IAAI,eAAe,cAElB;IAED,IAAI,aAAa,YAEhB;IAED,IAAI,KAAK,+BAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,GACpB,GAAE,4BAAiC;CAyCrC;AAGD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuC;IACvE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAGtD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuF;IAGlH,OAAO,CAAC,gBAAgB,CAAC,CAA+B;IAGxD,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;IAE1B,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAY5C,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAa3C,YAAY,CAAC,OAAO,EAAE,gBAAgB;IAMhC,KAAK;IAgBL,IAAI;IAiBV;;;OAGG;IAEG,SAAS,CAAC,SAAS,EAAE,SAAS;IAcpC;;OAEG;IAEG,KAAK;IAWL,OAAO;IAWb;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;IAkCjD,OAAO,CAAC,qBAAqB;YAoBf,aAAa;CAkB5B"}
|
|
@@ -19,6 +19,10 @@ export declare class ControlPipeline {
|
|
|
19
19
|
private readonly _ctx;
|
|
20
20
|
private readonly _pipeline;
|
|
21
21
|
private readonly _spaceStateMachine;
|
|
22
|
+
/**
|
|
23
|
+
* Map to keep track of failed messages.
|
|
24
|
+
*/
|
|
25
|
+
private readonly _failedMessages;
|
|
22
26
|
private readonly _spaceKey;
|
|
23
27
|
private readonly _metadata;
|
|
24
28
|
private _targetTimeframe?;
|
|
@@ -40,6 +44,11 @@ export declare class ControlPipeline {
|
|
|
40
44
|
private _saveSnapshot;
|
|
41
45
|
private _consumePipeline;
|
|
42
46
|
private _processMessage;
|
|
47
|
+
/**
|
|
48
|
+
* If it first failure, it will be retried once the pipeline is processed fully.
|
|
49
|
+
* If it fails again, it will be ignored.
|
|
50
|
+
*/
|
|
51
|
+
private _retryMessage;
|
|
43
52
|
private _noteTargetStateIfNeeded;
|
|
44
53
|
stop(): Promise<void>;
|
|
45
54
|
private _saveTargetTimeframe;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"control-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/control-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,4BAA4B,EAClC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAG5F,OAAO,EAAE,KAAK,aAAa,EAAE,QAAQ,
|
|
1
|
+
{"version":3,"file":"control-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/control-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,4BAA4B,EAClC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAG5F,OAAO,EAAE,KAAK,aAAa,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAE9E,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAY,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,YAAY,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;IACxE,aAAa,EAAE,aAAa,CAAC;CAC9B,CAAC;AAQF;;GAEG;AACH,qBAEa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoB;IAEvD;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAE9B;IAEF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAC,CAAY;IACrC,OAAO,CAAC,sBAAsB,CAAsB;IAEpD,SAAgB,cAAc,oCAA2C;IACzE,SAAgB,mBAAmB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAC7F,SAAgB,4BAA4B,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAGpG,OAAO,CAAC,MAAM,CAA0B;IAGxC,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,aAAa,CAGlB;gBAES,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,EAAE,qBAAqB;IAkCzF,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,QAAQ,IAAI,gBAAgB,CAE/B;IAEK,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAM3C,KAAK;YAgBG,gBAAgB;YAehB,aAAa;YAgBb,gBAAgB;YAgBhB,eAAe;IAqB7B;;;OAGG;IACH,OAAO,CAAC,aAAa;YAWP,wBAAwB;IAUhC,IAAI;YAQI,oBAAoB;CASnC"}
|
|
@@ -53,5 +53,6 @@ export declare class SpaceManager {
|
|
|
53
53
|
close(): Promise<void>;
|
|
54
54
|
constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, onDelegatedInvitationStatusChange, onMemberRolesChanged, memberKey, }: ConstructSpaceParams): Promise<Space>;
|
|
55
55
|
requestSpaceAdmissionCredential(params: RequestSpaceAdmissionCredentialParams): Promise<Credential>;
|
|
56
|
+
findSpaceByRootDocumentId(documentId: string): Space | undefined;
|
|
56
57
|
}
|
|
57
58
|
//# sourceMappingURL=space-manager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"space-manager.d.ts","sourceRoot":"","sources":["../../../../src/space/space-manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"space-manager.d.ts","sourceRoot":"","sources":["../../../../src/space/space-manager.ts"],"names":[],"mappings":"AAMA,OAAO,EAA0B,KAAK,4BAA4B,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/G,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,0CAA0C,CAAC;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAC9E,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,sCAAsC,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAErE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAClC,cAAc,EAAE,mBAAmB,CAAC;IACpC,aAAa,EAAE,aAAa,CAAC;IAE7B,SAAS,EAAE,SAAS,CAAC;IAErB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB;;OAEG;IACH,sBAAsB,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC5C,iCAAiC,EAAE,CAAC,UAAU,EAAE,4BAA4B,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClH,oBAAoB,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D,CAAC;AAEF,MAAM,MAAM,qCAAqC,GAAG;IAClD,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,SAAS,CAAC;IACvB,aAAa,EAAE,aAAa,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBACa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAC1D,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;gBAErC,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,qBAAqB,EAAE,EAAE,kBAAkB;IAU9G,IAAI,MAAM,iCAET;IAGK,IAAI;IAGJ,KAAK;IAIL,cAAc,CAAC,EACnB,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,aAAa,EACb,iCAAiC,EACjC,oBAAoB,EACpB,SAAS,GACV,EAAE,oBAAoB;IAoCV,+BAA+B,CAAC,MAAM,EAAE,qCAAqC,GAAG,OAAO,CAAC,UAAU,CAAC;IAqCzG,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;CAcxE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":"5.7.2"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/echo-pipeline",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.5-feature-compute.4d9d99a",
|
|
4
4
|
"description": "ECHO database.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -37,44 +37,44 @@
|
|
|
37
37
|
"crc-32": "^1.2.2",
|
|
38
38
|
"level-transcoder": "^1.0.1",
|
|
39
39
|
"lodash.isequal": "^4.5.0",
|
|
40
|
-
"@dxos/async": "0.7.
|
|
41
|
-
"@dxos/
|
|
42
|
-
"@dxos/context": "0.7.
|
|
43
|
-
"@dxos/
|
|
44
|
-
"@dxos/credentials": "0.7.
|
|
45
|
-
"@dxos/crypto": "0.7.
|
|
46
|
-
"@dxos/debug": "0.7.
|
|
47
|
-
"@dxos/
|
|
48
|
-
"@dxos/
|
|
49
|
-
"@dxos/
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/keyring": "0.7.
|
|
55
|
-
"@dxos/
|
|
56
|
-
"@dxos/
|
|
57
|
-
"@dxos/keys": "0.7.
|
|
58
|
-
"@dxos/
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/teleport": "0.7.
|
|
64
|
-
"@dxos/teleport-extension-
|
|
65
|
-
"@dxos/teleport-extension-object-sync": "0.7.
|
|
66
|
-
"@dxos/teleport-extension-
|
|
67
|
-
"@dxos/timeframe": "0.7.
|
|
68
|
-
"@dxos/
|
|
69
|
-
"@dxos/
|
|
70
|
-
"@dxos/
|
|
71
|
-
"@dxos/
|
|
40
|
+
"@dxos/async": "0.7.5-feature-compute.4d9d99a",
|
|
41
|
+
"@dxos/codec-protobuf": "0.7.5-feature-compute.4d9d99a",
|
|
42
|
+
"@dxos/context": "0.7.5-feature-compute.4d9d99a",
|
|
43
|
+
"@dxos/automerge": "0.7.5-feature-compute.4d9d99a",
|
|
44
|
+
"@dxos/credentials": "0.7.5-feature-compute.4d9d99a",
|
|
45
|
+
"@dxos/crypto": "0.7.5-feature-compute.4d9d99a",
|
|
46
|
+
"@dxos/debug": "0.7.5-feature-compute.4d9d99a",
|
|
47
|
+
"@dxos/echo-protocol": "0.7.5-feature-compute.4d9d99a",
|
|
48
|
+
"@dxos/echo-schema": "0.7.5-feature-compute.4d9d99a",
|
|
49
|
+
"@dxos/edge-client": "0.7.5-feature-compute.4d9d99a",
|
|
50
|
+
"@dxos/feed-store": "0.7.5-feature-compute.4d9d99a",
|
|
51
|
+
"@dxos/hypercore": "0.7.5-feature-compute.4d9d99a",
|
|
52
|
+
"@dxos/indexing": "0.7.5-feature-compute.4d9d99a",
|
|
53
|
+
"@dxos/invariant": "0.7.5-feature-compute.4d9d99a",
|
|
54
|
+
"@dxos/keyring": "0.7.5-feature-compute.4d9d99a",
|
|
55
|
+
"@dxos/log": "0.7.5-feature-compute.4d9d99a",
|
|
56
|
+
"@dxos/messaging": "0.7.5-feature-compute.4d9d99a",
|
|
57
|
+
"@dxos/keys": "0.7.5-feature-compute.4d9d99a",
|
|
58
|
+
"@dxos/network-manager": "0.7.5-feature-compute.4d9d99a",
|
|
59
|
+
"@dxos/protocols": "0.7.5-feature-compute.4d9d99a",
|
|
60
|
+
"@dxos/random-access-storage": "0.7.5-feature-compute.4d9d99a",
|
|
61
|
+
"@dxos/kv-store": "0.7.5-feature-compute.4d9d99a",
|
|
62
|
+
"@dxos/teleport": "0.7.5-feature-compute.4d9d99a",
|
|
63
|
+
"@dxos/teleport-extension-automerge-replicator": "0.7.5-feature-compute.4d9d99a",
|
|
64
|
+
"@dxos/teleport-extension-gossip": "0.7.5-feature-compute.4d9d99a",
|
|
65
|
+
"@dxos/teleport-extension-object-sync": "0.7.5-feature-compute.4d9d99a",
|
|
66
|
+
"@dxos/teleport-extension-replicator": "0.7.5-feature-compute.4d9d99a",
|
|
67
|
+
"@dxos/timeframe": "0.7.5-feature-compute.4d9d99a",
|
|
68
|
+
"@dxos/tracing": "0.7.5-feature-compute.4d9d99a",
|
|
69
|
+
"@dxos/node-std": "0.7.5-feature-compute.4d9d99a",
|
|
70
|
+
"@dxos/util": "0.7.5-feature-compute.4d9d99a",
|
|
71
|
+
"@dxos/typings": "0.7.5-feature-compute.4d9d99a"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@types/lodash.isequal": "^4.5.0",
|
|
75
75
|
"fast-check": "^3.19.0",
|
|
76
76
|
"get-port-please": "^3.1.1",
|
|
77
|
-
"@dxos/test-utils": "0.7.
|
|
77
|
+
"@dxos/test-utils": "0.7.5-feature-compute.4d9d99a"
|
|
78
78
|
},
|
|
79
79
|
"publishConfig": {
|
|
80
80
|
"access": "public"
|
|
@@ -44,6 +44,8 @@ import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-
|
|
|
44
44
|
|
|
45
45
|
export type PeerIdProvider = () => string | undefined;
|
|
46
46
|
|
|
47
|
+
export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | undefined;
|
|
48
|
+
|
|
47
49
|
export type AutomergeHostParams = {
|
|
48
50
|
db: LevelDB;
|
|
49
51
|
|
|
@@ -54,6 +56,7 @@ export type AutomergeHostParams = {
|
|
|
54
56
|
* Used for creating stable ids. A random key is generated on open, if no value is provided.
|
|
55
57
|
*/
|
|
56
58
|
peerIdProvider?: PeerIdProvider;
|
|
59
|
+
getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
|
|
57
60
|
};
|
|
58
61
|
|
|
59
62
|
export type LoadDocOptions = {
|
|
@@ -90,10 +93,17 @@ export class AutomergeHost extends Resource {
|
|
|
90
93
|
private _peerId!: PeerId;
|
|
91
94
|
|
|
92
95
|
private readonly _peerIdProvider?: PeerIdProvider;
|
|
96
|
+
private readonly _getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
|
|
93
97
|
|
|
94
98
|
public readonly collectionStateUpdated = new Event<{ collectionId: CollectionId }>();
|
|
95
99
|
|
|
96
|
-
constructor({
|
|
100
|
+
constructor({
|
|
101
|
+
db,
|
|
102
|
+
indexMetadataStore,
|
|
103
|
+
dataMonitor,
|
|
104
|
+
peerIdProvider,
|
|
105
|
+
getSpaceKeyByRootDocumentId,
|
|
106
|
+
}: AutomergeHostParams) {
|
|
97
107
|
super();
|
|
98
108
|
this._db = db;
|
|
99
109
|
this._storage = new LevelDBStorageAdapter({
|
|
@@ -114,6 +124,7 @@ export class AutomergeHost extends Resource {
|
|
|
114
124
|
this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
|
|
115
125
|
this._indexMetadataStore = indexMetadataStore;
|
|
116
126
|
this._peerIdProvider = peerIdProvider;
|
|
127
|
+
this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
|
|
117
128
|
}
|
|
118
129
|
|
|
119
130
|
protected override async _open() {
|
|
@@ -132,14 +143,28 @@ export class AutomergeHost extends Resource {
|
|
|
132
143
|
],
|
|
133
144
|
});
|
|
134
145
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
this.
|
|
146
|
+
let updatingAuthScope = false;
|
|
147
|
+
Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(
|
|
148
|
+
this._ctx,
|
|
149
|
+
((e: PeerCandidatePayload) => !updatingAuthScope && this._onPeerConnected(e.peerId)) as any,
|
|
150
|
+
);
|
|
151
|
+
Event.wrap(this._echoNetworkAdapter, 'peer-disconnected').on(
|
|
152
|
+
this._ctx,
|
|
153
|
+
((e: PeerDisconnectedPayload) => !updatingAuthScope && this._onPeerDisconnected(e.peerId)) as any,
|
|
154
|
+
);
|
|
139
155
|
|
|
140
|
-
this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
|
|
156
|
+
this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
|
|
141
157
|
this._onRemoteCollectionStateUpdated(collectionId, peerId);
|
|
142
158
|
this.collectionStateUpdated.emit({ collectionId: collectionId as CollectionId });
|
|
159
|
+
// We use collection lookups during share policy check, so we might need to update share policy for the new doc
|
|
160
|
+
if (newDocsAppeared) {
|
|
161
|
+
updatingAuthScope = true;
|
|
162
|
+
try {
|
|
163
|
+
this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
|
|
164
|
+
} finally {
|
|
165
|
+
updatingAuthScope = false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
143
168
|
});
|
|
144
169
|
|
|
145
170
|
await this._echoNetworkAdapter.open();
|
|
@@ -351,16 +376,23 @@ export class AutomergeHost extends Resource {
|
|
|
351
376
|
|
|
352
377
|
private async _getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null> {
|
|
353
378
|
const doc = this._repo.handles[documentId as any]?.docSync();
|
|
354
|
-
if (
|
|
355
|
-
|
|
379
|
+
if (doc) {
|
|
380
|
+
const spaceKeyHex = getSpaceKeyFromDoc(doc);
|
|
381
|
+
if (spaceKeyHex) {
|
|
382
|
+
return PublicKey.from(spaceKeyHex);
|
|
383
|
+
}
|
|
356
384
|
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
385
|
+
/**
|
|
386
|
+
* Edge case on the initial space setup.
|
|
387
|
+
* A peer is maybe trying to share space root document with us after a successful invitation.
|
|
388
|
+
* We don't have a document to check access block locally, so we need to rely on external sources (space metada).
|
|
389
|
+
*/
|
|
390
|
+
const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
|
|
391
|
+
if (rootDocSpaceKey) {
|
|
392
|
+
return rootDocSpaceKey;
|
|
361
393
|
}
|
|
362
394
|
|
|
363
|
-
return
|
|
395
|
+
return null;
|
|
364
396
|
}
|
|
365
397
|
|
|
366
398
|
/**
|
|
@@ -496,7 +528,10 @@ export class AutomergeHost extends Resource {
|
|
|
496
528
|
return;
|
|
497
529
|
}
|
|
498
530
|
|
|
499
|
-
log.info('
|
|
531
|
+
log.info('replicating documents after collection sync', {
|
|
532
|
+
collectionId,
|
|
533
|
+
peerId,
|
|
534
|
+
toReplicate,
|
|
500
535
|
count: toReplicate.length,
|
|
501
536
|
});
|
|
502
537
|
|
|
@@ -35,7 +35,7 @@ export class CollectionSynchronizer extends Resource {
|
|
|
35
35
|
|
|
36
36
|
private readonly _connectedPeers = new Set<PeerId>();
|
|
37
37
|
|
|
38
|
-
public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId }>();
|
|
38
|
+
public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
|
|
39
39
|
|
|
40
40
|
constructor(params: CollectionSynchronizerParams) {
|
|
41
41
|
super();
|
|
@@ -121,7 +121,7 @@ export class CollectionSynchronizer extends Resource {
|
|
|
121
121
|
return;
|
|
122
122
|
}
|
|
123
123
|
for (const [collectionId, state] of this._perCollectionStates.entries()) {
|
|
124
|
-
if (this._shouldSyncCollection(collectionId, peerId)) {
|
|
124
|
+
if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
|
|
125
125
|
state.interestedPeers.add(peerId);
|
|
126
126
|
state.lastQueried.set(peerId, Date.now());
|
|
127
127
|
this._queryCollectionState(collectionId, peerId);
|
|
@@ -159,8 +159,12 @@ export class CollectionSynchronizer extends Resource {
|
|
|
159
159
|
log('onRemoteStateReceived', { collectionId, peerId, state });
|
|
160
160
|
validateCollectionState(state);
|
|
161
161
|
const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
|
|
162
|
-
perCollectionState.remoteStates.
|
|
163
|
-
|
|
162
|
+
const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
|
|
163
|
+
const diff = diffCollectionState(existingState, state);
|
|
164
|
+
if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
|
|
165
|
+
perCollectionState.remoteStates.set(peerId, state);
|
|
166
|
+
this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
|
|
167
|
+
}
|
|
164
168
|
}
|
|
165
169
|
|
|
166
170
|
private _getOrCreatePerCollectionState(collectionId: string): PerCollectionState {
|
|
@@ -246,6 +246,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
|
|
|
246
246
|
this._params.monitor?.recordMessageReceived(message);
|
|
247
247
|
}
|
|
248
248
|
|
|
249
|
+
public onConnectionAuthScopeChanged(peer: PeerId) {
|
|
250
|
+
const entry = this._connections.get(peer);
|
|
251
|
+
if (entry) {
|
|
252
|
+
this._onConnectionAuthScopeChanged(entry.connection);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
249
256
|
/**
|
|
250
257
|
* Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
|
|
251
258
|
* TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
|
|
@@ -88,10 +88,10 @@ export class MeshEchoReplicator implements EchoReplicator {
|
|
|
88
88
|
documentId: params.documentId,
|
|
89
89
|
peerId: connection.peerId,
|
|
90
90
|
});
|
|
91
|
-
log('document not found locally for share policy check
|
|
91
|
+
log('document not found locally for share policy check', {
|
|
92
92
|
peerId: connection.peerId,
|
|
93
93
|
documentId: params.documentId,
|
|
94
|
-
remoteDocumentExists,
|
|
94
|
+
acceptDocument: remoteDocumentExists,
|
|
95
95
|
});
|
|
96
96
|
// If a document is not present locally return true if another peer claims to have it.
|
|
97
97
|
// Simply returning true will add the peer to "generous peers list" for this document which will
|
package/src/db-host/echo-host.ts
CHANGED
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
type EchoReplicator,
|
|
34
34
|
type EchoDataStats,
|
|
35
35
|
type PeerIdProvider,
|
|
36
|
+
type RootDocumentSpaceKeyProvider,
|
|
36
37
|
} from '../automerge';
|
|
37
38
|
|
|
38
39
|
const INDEXER_CONFIG: IndexConfig = {
|
|
@@ -43,6 +44,7 @@ const INDEXER_CONFIG: IndexConfig = {
|
|
|
43
44
|
export type EchoHostParams = {
|
|
44
45
|
kv: LevelDB;
|
|
45
46
|
peerIdProvider?: PeerIdProvider;
|
|
47
|
+
getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
|
|
46
48
|
};
|
|
47
49
|
|
|
48
50
|
/**
|
|
@@ -60,7 +62,7 @@ export class EchoHost extends Resource {
|
|
|
60
62
|
private readonly _spaceStateManager = new SpaceStateManager();
|
|
61
63
|
private readonly _echoDataMonitor: EchoDataMonitor;
|
|
62
64
|
|
|
63
|
-
constructor({ kv, peerIdProvider }: EchoHostParams) {
|
|
65
|
+
constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
|
|
64
66
|
super();
|
|
65
67
|
|
|
66
68
|
this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
|
|
@@ -72,6 +74,7 @@ export class EchoHost extends Resource {
|
|
|
72
74
|
dataMonitor: this._echoDataMonitor,
|
|
73
75
|
indexMetadataStore: this._indexMetadataStore,
|
|
74
76
|
peerIdProvider,
|
|
77
|
+
getSpaceKeyByRootDocumentId,
|
|
75
78
|
});
|
|
76
79
|
|
|
77
80
|
this._indexer = new Indexer({
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
|
|
6
|
-
import * as A from '@dxos/automerge/automerge';
|
|
7
6
|
import { cbor } from '@dxos/automerge/automerge-repo';
|
|
8
7
|
import { Context, Resource } from '@dxos/context';
|
|
9
8
|
import { randomUUID } from '@dxos/crypto';
|
|
@@ -20,6 +19,7 @@ import {
|
|
|
20
19
|
} from '@dxos/protocols/buf/dxos/edge/messenger_pb';
|
|
21
20
|
import { bufferToArray } from '@dxos/util';
|
|
22
21
|
|
|
22
|
+
import { InflightRequestLimiter } from './inflight-request-limiter';
|
|
23
23
|
import {
|
|
24
24
|
getSpaceIdFromCollectionId,
|
|
25
25
|
type EchoReplicator,
|
|
@@ -184,9 +184,12 @@ type EdgeReplicatorConnectionsParams = {
|
|
|
184
184
|
onRestartRequested: () => Promise<void>;
|
|
185
185
|
};
|
|
186
186
|
|
|
187
|
+
const MAX_INFLIGHT_REQUESTS = 5;
|
|
188
|
+
const MAX_RATE_LIMIT_WAIT_TIME_MS = 3000;
|
|
189
|
+
|
|
187
190
|
class EdgeReplicatorConnection extends Resource implements ReplicatorConnection {
|
|
188
191
|
private readonly _edgeConnection: EdgeConnection;
|
|
189
|
-
private _remotePeerId: string | null = null;
|
|
192
|
+
private readonly _remotePeerId: string | null = null;
|
|
190
193
|
private readonly _targetServiceId: string;
|
|
191
194
|
private readonly _spaceId: SpaceId;
|
|
192
195
|
private readonly _context: EchoReplicatorContext;
|
|
@@ -195,6 +198,11 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
195
198
|
private readonly _onRemoteDisconnected: () => Promise<void>;
|
|
196
199
|
private readonly _onRestartRequested: () => void;
|
|
197
200
|
|
|
201
|
+
private _requestLimiter = new InflightRequestLimiter({
|
|
202
|
+
maxInflightRequests: MAX_INFLIGHT_REQUESTS,
|
|
203
|
+
resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS,
|
|
204
|
+
});
|
|
205
|
+
|
|
198
206
|
private _readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
|
|
199
207
|
|
|
200
208
|
public readable: ReadableStream<AutomergeProtocolMessage>;
|
|
@@ -213,7 +221,6 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
213
221
|
this._edgeConnection = edgeConnection;
|
|
214
222
|
this._spaceId = spaceId;
|
|
215
223
|
this._context = context;
|
|
216
|
-
|
|
217
224
|
// Generate a unique peer id for every connection.
|
|
218
225
|
// This way automerge-repo will have separate sync states for every connection.
|
|
219
226
|
// This is important because the previous connection might have had some messages that failed to deliver
|
|
@@ -233,6 +240,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
233
240
|
|
|
234
241
|
this.writable = new WritableStream<AutomergeProtocolMessage>({
|
|
235
242
|
write: async (message: AutomergeProtocolMessage, controller) => {
|
|
243
|
+
await this._requestLimiter.rateLimit(message);
|
|
244
|
+
|
|
236
245
|
await this._sendMessage(message);
|
|
237
246
|
},
|
|
238
247
|
});
|
|
@@ -240,6 +249,9 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
240
249
|
|
|
241
250
|
protected override async _open(ctx: Context): Promise<void> {
|
|
242
251
|
log('open');
|
|
252
|
+
|
|
253
|
+
await this._requestLimiter.open();
|
|
254
|
+
|
|
243
255
|
// TODO: handle reconnects
|
|
244
256
|
this._ctx.onDispose(
|
|
245
257
|
this._edgeConnection.onMessage((msg: RouterMessage) => {
|
|
@@ -253,6 +265,9 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
253
265
|
protected override async _close(): Promise<void> {
|
|
254
266
|
log('close');
|
|
255
267
|
this._readableStreamController.close();
|
|
268
|
+
|
|
269
|
+
await this._requestLimiter.close();
|
|
270
|
+
|
|
256
271
|
await this._onRemoteDisconnected();
|
|
257
272
|
}
|
|
258
273
|
|
|
@@ -272,9 +287,10 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
272
287
|
peerId: this._remotePeerId as PeerId,
|
|
273
288
|
});
|
|
274
289
|
|
|
275
|
-
log.
|
|
290
|
+
log.verbose('edge-replicator document not found locally for share policy check', {
|
|
276
291
|
documentId: params.documentId,
|
|
277
|
-
remoteDocumentExists,
|
|
292
|
+
acceptDocument: remoteDocumentExists,
|
|
293
|
+
remoteId: this._remotePeerId,
|
|
278
294
|
});
|
|
279
295
|
|
|
280
296
|
// If a document is not present locally return true only if it already exists on edge.
|
|
@@ -290,7 +306,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
290
306
|
return true;
|
|
291
307
|
}
|
|
292
308
|
const spaceId = getSpaceIdFromCollectionId(params.collectionId as CollectionId);
|
|
293
|
-
|
|
309
|
+
// Only sync collections of form space:id:rootDoc, edge ignores legacy space:id collections
|
|
310
|
+
return spaceId === this._spaceId && params.collectionId.split(':').length === 3;
|
|
294
311
|
}
|
|
295
312
|
|
|
296
313
|
private _onMessage(message: RouterMessage) {
|
|
@@ -299,15 +316,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
299
316
|
}
|
|
300
317
|
|
|
301
318
|
const payload = cbor.decode(message.payload!.value) as AutomergeProtocolMessage;
|
|
302
|
-
log('
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
: payload.type === 'collection-state'
|
|
307
|
-
? (payload as any).state
|
|
308
|
-
: payload;
|
|
309
|
-
return { from: message.serviceId, type: payload.type, decodedData };
|
|
319
|
+
log.verbose('edge replicator receive', {
|
|
320
|
+
type: payload.type,
|
|
321
|
+
documentId: payload.type === 'sync' && payload.documentId,
|
|
322
|
+
remoteId: this._remotePeerId,
|
|
310
323
|
});
|
|
324
|
+
|
|
311
325
|
// Fix the peer id.
|
|
312
326
|
payload.senderId = this._remotePeerId! as PeerId;
|
|
313
327
|
this._processMessage(payload);
|
|
@@ -322,6 +336,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
322
336
|
return;
|
|
323
337
|
}
|
|
324
338
|
|
|
339
|
+
this._requestLimiter.handleResponse(message);
|
|
340
|
+
|
|
325
341
|
this._readableStreamController.enqueue(message);
|
|
326
342
|
}
|
|
327
343
|
|
|
@@ -329,12 +345,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
329
345
|
// Fix the peer id.
|
|
330
346
|
(message as any).targetId = this._targetServiceId as PeerId;
|
|
331
347
|
|
|
332
|
-
log('send', {
|
|
348
|
+
log.verbose('edge replicator send', {
|
|
333
349
|
type: message.type,
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
documentId: (message as any).documentId,
|
|
350
|
+
documentId: message.type === 'sync' && message.documentId,
|
|
351
|
+
remoteId: this._remotePeerId,
|
|
337
352
|
});
|
|
353
|
+
|
|
338
354
|
const encoded = cbor.encode(message);
|
|
339
355
|
|
|
340
356
|
await this._edgeConnection.send(
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Trigger } from '@dxos/async';
|
|
6
|
+
import { Resource } from '@dxos/context';
|
|
7
|
+
import { log } from '@dxos/log';
|
|
8
|
+
import { type AutomergeProtocolMessage } from '@dxos/protocols';
|
|
9
|
+
|
|
10
|
+
type InflightRequestLimiterConfig = {
|
|
11
|
+
maxInflightRequests: number;
|
|
12
|
+
resetBalanceTimeoutMs: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export class InflightRequestLimiter extends Resource {
|
|
16
|
+
/**
|
|
17
|
+
* Decrement when we receive a sync message, increment when we send one.
|
|
18
|
+
* Can't exceed _config.maxInflightRequests.
|
|
19
|
+
* Resets after timeout to avoid replicator being stuck.
|
|
20
|
+
*/
|
|
21
|
+
private _inflightRequestBalance = 0;
|
|
22
|
+
private _requestBarrier = new Trigger();
|
|
23
|
+
private _resetBalanceTimeout: NodeJS.Timeout | undefined;
|
|
24
|
+
|
|
25
|
+
constructor(private readonly _config: InflightRequestLimiterConfig) {
|
|
26
|
+
super();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
protected override async _open() {
|
|
30
|
+
this._inflightRequestBalance = 0;
|
|
31
|
+
this._requestBarrier.reset();
|
|
32
|
+
this._requestBarrier.wake();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
protected override async _close() {
|
|
36
|
+
this._inflightRequestBalance = 0;
|
|
37
|
+
this._requestBarrier.throw(new Error('Rate limiter closed.'));
|
|
38
|
+
clearTimeout(this._resetBalanceTimeout);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
public async rateLimit(message: AutomergeProtocolMessage) {
|
|
42
|
+
if (message.type !== 'sync') {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
while (this._inflightRequestBalance >= this._config.maxInflightRequests) {
|
|
46
|
+
await this._requestBarrier.wait();
|
|
47
|
+
}
|
|
48
|
+
this._inflightRequestBalance++;
|
|
49
|
+
if (this._inflightRequestBalance === this._config.maxInflightRequests) {
|
|
50
|
+
this._requestBarrier.reset();
|
|
51
|
+
this._resetBalanceTimeout = setTimeout(() => {
|
|
52
|
+
log.warn('Request balance has not changed during specified timeout, resetting request limiter.');
|
|
53
|
+
this._inflightRequestBalance = 0;
|
|
54
|
+
this._requestBarrier.wake();
|
|
55
|
+
}, this._config.resetBalanceTimeoutMs);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
public handleResponse(message: AutomergeProtocolMessage) {
|
|
60
|
+
if (message.type !== 'sync') {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
this._inflightRequestBalance--;
|
|
64
|
+
if (this._inflightRequestBalance + 1 === this._config.maxInflightRequests) {
|
|
65
|
+
this._requestBarrier.wake();
|
|
66
|
+
clearInterval(this._resetBalanceTimeout);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
package/src/pipeline/pipeline.ts
CHANGED
|
@@ -279,6 +279,11 @@ export class Pipeline implements PipelineAccessor {
|
|
|
279
279
|
);
|
|
280
280
|
}
|
|
281
281
|
|
|
282
|
+
retryMessage(message: FeedMessageBlock) {
|
|
283
|
+
invariant(this._feedSetIterator, 'Iterator not initialized.');
|
|
284
|
+
this._feedSetIterator.reiterateBlock(message);
|
|
285
|
+
}
|
|
286
|
+
|
|
282
287
|
@synchronized
|
|
283
288
|
async start() {
|
|
284
289
|
invariant(!this._isStarted, 'Pipeline is already started.');
|
|
@@ -423,7 +428,10 @@ export class Pipeline implements PipelineAccessor {
|
|
|
423
428
|
});
|
|
424
429
|
|
|
425
430
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
426
|
-
log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds
|
|
431
|
+
log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {
|
|
432
|
+
currentTimeframe: this._timeframeClock.timeframe,
|
|
433
|
+
targetTimeframe: this._state.targetTimeframe,
|
|
434
|
+
});
|
|
427
435
|
this._state.stalled.emit();
|
|
428
436
|
});
|
|
429
437
|
|