@dxos/echo-pipeline 0.1.56-next.a63ce79 → 0.1.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-3OTHGQEY.mjs → chunk-PV6A4BSQ.mjs} +258 -199
- package/dist/lib/browser/chunk-PV6A4BSQ.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +1 -1
- package/dist/lib/node/index.cjs +262 -203
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +264 -205
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts +2 -0
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts.map +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts +4 -3
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +4 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +2 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts +4 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/package.json +32 -32
- package/src/db-host/data-service-host.ts +9 -2
- package/src/pipeline/pipeline-stress.test.ts +226 -0
- package/src/pipeline/pipeline.test.ts +2 -214
- package/src/pipeline/pipeline.ts +47 -22
- package/src/space/control-pipeline.ts +47 -22
- package/src/space/data-pipeline.ts +20 -10
- package/src/space/space-manager.ts +1 -0
- package/src/space/space-protocol.ts +3 -1
- package/src/space/space.ts +1 -0
- package/src/testing/database-test-rig.ts +27 -2
- package/src/tests/database-unit.test.ts +39 -0
- package/dist/lib/browser/chunk-3OTHGQEY.mjs.map +0 -7
|
@@ -8,7 +8,7 @@ var valueEncoding = createCodecEncoding(codec);
|
|
|
8
8
|
|
|
9
9
|
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
10
10
|
import { invariant } from "@dxos/invariant";
|
|
11
|
-
var __dxlog_file = "/
|
|
11
|
+
var __dxlog_file = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/common/feeds.ts";
|
|
12
12
|
var createMappedFeedWriter = (mapper, writer) => {
|
|
13
13
|
invariant(mapper, void 0, {
|
|
14
14
|
F: __dxlog_file,
|
|
@@ -39,8 +39,9 @@ import { Context } from "@dxos/context";
|
|
|
39
39
|
import { tagMutationsInBatch, setMetadataOnObject } from "@dxos/echo-db";
|
|
40
40
|
import { invariant as invariant2 } from "@dxos/invariant";
|
|
41
41
|
import { log } from "@dxos/log";
|
|
42
|
+
import { EchoEvent } from "@dxos/protocols/proto/dxos/echo/service";
|
|
42
43
|
import { ComplexMap } from "@dxos/util";
|
|
43
|
-
var __dxlog_file2 = "/
|
|
44
|
+
var __dxlog_file2 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
44
45
|
var DataServiceHost = class {
|
|
45
46
|
constructor(_itemManager, _itemDemuxer, _writeStream) {
|
|
46
47
|
this._itemManager = _itemManager;
|
|
@@ -66,11 +67,19 @@ var DataServiceHost = class {
|
|
|
66
67
|
objects
|
|
67
68
|
}
|
|
68
69
|
});
|
|
70
|
+
this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
|
|
71
|
+
next({
|
|
72
|
+
action: EchoEvent.DatabaseAction.RESET,
|
|
73
|
+
batch: {
|
|
74
|
+
objects: snapshot.items
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
});
|
|
69
78
|
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
70
79
|
const { batch, meta } = message;
|
|
71
80
|
invariant2(!meta.clientTag, "Unexpected client tag in mutation message", {
|
|
72
81
|
F: __dxlog_file2,
|
|
73
|
-
L:
|
|
82
|
+
L: 68,
|
|
74
83
|
S: this,
|
|
75
84
|
A: [
|
|
76
85
|
"!(meta as any).clientTag",
|
|
@@ -78,11 +87,11 @@ var DataServiceHost = class {
|
|
|
78
87
|
]
|
|
79
88
|
});
|
|
80
89
|
log("message", {
|
|
81
|
-
batch,
|
|
90
|
+
batch: batch.objects?.length,
|
|
82
91
|
meta
|
|
83
92
|
}, {
|
|
84
93
|
F: __dxlog_file2,
|
|
85
|
-
L:
|
|
94
|
+
L: 69,
|
|
86
95
|
S: this,
|
|
87
96
|
C: (f, a) => f(...a)
|
|
88
97
|
});
|
|
@@ -110,7 +119,7 @@ var DataServiceHost = class {
|
|
|
110
119
|
async write(request) {
|
|
111
120
|
invariant2(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
|
|
112
121
|
F: __dxlog_file2,
|
|
113
|
-
L:
|
|
122
|
+
L: 95,
|
|
114
123
|
S: this,
|
|
115
124
|
A: [
|
|
116
125
|
"!this._ctx.disposed",
|
|
@@ -119,7 +128,7 @@ var DataServiceHost = class {
|
|
|
119
128
|
});
|
|
120
129
|
invariant2(this._writeStream, "Cannot write mutations in readonly mode", {
|
|
121
130
|
F: __dxlog_file2,
|
|
122
|
-
L:
|
|
131
|
+
L: 96,
|
|
123
132
|
S: this,
|
|
124
133
|
A: [
|
|
125
134
|
"this._writeStream",
|
|
@@ -131,7 +140,7 @@ var DataServiceHost = class {
|
|
|
131
140
|
objectCount: request.batch.objects?.length ?? 0
|
|
132
141
|
}, {
|
|
133
142
|
F: __dxlog_file2,
|
|
134
|
-
L:
|
|
143
|
+
L: 98,
|
|
135
144
|
S: this,
|
|
136
145
|
C: (f, a) => f(...a)
|
|
137
146
|
});
|
|
@@ -145,7 +154,7 @@ var DataServiceHost = class {
|
|
|
145
154
|
seq: receipt2.seq
|
|
146
155
|
}, {
|
|
147
156
|
F: __dxlog_file2,
|
|
148
|
-
L:
|
|
157
|
+
L: 107,
|
|
149
158
|
S: this,
|
|
150
159
|
C: (f, a) => f(...a)
|
|
151
160
|
});
|
|
@@ -306,7 +315,7 @@ import { invariant as invariant3 } from "@dxos/invariant";
|
|
|
306
315
|
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
307
316
|
import { log as log2 } from "@dxos/log";
|
|
308
317
|
import { ComplexMap as ComplexMap2 } from "@dxos/util";
|
|
309
|
-
var __dxlog_file3 = "/
|
|
318
|
+
var __dxlog_file3 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
|
|
310
319
|
var DataServiceSubscriptions = class {
|
|
311
320
|
constructor() {
|
|
312
321
|
this._spaces = new ComplexMap2(PublicKey2.hash);
|
|
@@ -412,7 +421,7 @@ function _ts_decorate2(decorators, target, key, desc) {
|
|
|
412
421
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
413
422
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
414
423
|
}
|
|
415
|
-
var __dxlog_file4 = "/
|
|
424
|
+
var __dxlog_file4 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
|
|
416
425
|
var emptyEchoMetadata = () => ({
|
|
417
426
|
version: STORAGE_VERSION,
|
|
418
427
|
spaces: [],
|
|
@@ -624,7 +633,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
624
633
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
625
634
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
626
635
|
}
|
|
627
|
-
var __dxlog_file5 = "/
|
|
636
|
+
var __dxlog_file5 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
|
|
628
637
|
var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
|
|
629
638
|
feedKey,
|
|
630
639
|
index
|
|
@@ -721,7 +730,7 @@ import { ComplexMap as ComplexMap3 } from "@dxos/util";
|
|
|
721
730
|
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
722
731
|
import { invariant as invariant5 } from "@dxos/invariant";
|
|
723
732
|
import { log as log5 } from "@dxos/log";
|
|
724
|
-
var __dxlog_file6 = "/
|
|
733
|
+
var __dxlog_file6 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
725
734
|
var createMessageSelector = (timeframeClock) => {
|
|
726
735
|
return (messages) => {
|
|
727
736
|
for (let i = 0; i < messages.length; i++) {
|
|
@@ -759,7 +768,7 @@ function _ts_decorate4(decorators, target, key, desc) {
|
|
|
759
768
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
760
769
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
761
770
|
}
|
|
762
|
-
var __dxlog_file7 = "/
|
|
771
|
+
var __dxlog_file7 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
763
772
|
var PipelineState = class {
|
|
764
773
|
// prettier-ignore
|
|
765
774
|
constructor(_feeds, _timeframeClock) {
|
|
@@ -867,9 +876,11 @@ var Pipeline = class {
|
|
|
867
876
|
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
868
877
|
this._processingTrigger = new Trigger().wake();
|
|
869
878
|
this._pauseTrigger = new Trigger().wake();
|
|
879
|
+
// Pending downloads.
|
|
880
|
+
this._downloads = new ComplexMap3((value) => PublicKey3.hash(value.key));
|
|
870
881
|
this._isStopping = false;
|
|
871
882
|
this._isStarted = false;
|
|
872
|
-
this.
|
|
883
|
+
this._isBeingConsumed = false;
|
|
873
884
|
this._isPaused = false;
|
|
874
885
|
}
|
|
875
886
|
get state() {
|
|
@@ -878,7 +889,7 @@ var Pipeline = class {
|
|
|
878
889
|
get writer() {
|
|
879
890
|
invariant6(this._writer, "Writer not set.", {
|
|
880
891
|
F: __dxlog_file7,
|
|
881
|
-
L:
|
|
892
|
+
L: 233,
|
|
882
893
|
S: this,
|
|
883
894
|
A: [
|
|
884
895
|
"this._writer",
|
|
@@ -900,12 +911,14 @@ var Pipeline = class {
|
|
|
900
911
|
if (this._feedSetIterator) {
|
|
901
912
|
await this._feedSetIterator.addFeed(feed);
|
|
902
913
|
}
|
|
903
|
-
this.
|
|
914
|
+
if (this._isStarted && !this._isPaused) {
|
|
915
|
+
this._setFeedDownloadState(feed);
|
|
916
|
+
}
|
|
904
917
|
}
|
|
905
918
|
setWriteFeed(feed) {
|
|
906
919
|
invariant6(!this._writer, "Writer already set.", {
|
|
907
920
|
F: __dxlog_file7,
|
|
908
|
-
L:
|
|
921
|
+
L: 260,
|
|
909
922
|
S: this,
|
|
910
923
|
A: [
|
|
911
924
|
"!this._writer",
|
|
@@ -914,7 +927,7 @@ var Pipeline = class {
|
|
|
914
927
|
});
|
|
915
928
|
invariant6(feed.properties.writable, "Feed must be writable.", {
|
|
916
929
|
F: __dxlog_file7,
|
|
917
|
-
L:
|
|
930
|
+
L: 261,
|
|
918
931
|
S: this,
|
|
919
932
|
A: [
|
|
920
933
|
"feed.properties.writable",
|
|
@@ -929,7 +942,7 @@ var Pipeline = class {
|
|
|
929
942
|
async start() {
|
|
930
943
|
invariant6(!this._isStarted, "Pipeline is already started.", {
|
|
931
944
|
F: __dxlog_file7,
|
|
932
|
-
L:
|
|
945
|
+
L: 274,
|
|
933
946
|
S: this,
|
|
934
947
|
A: [
|
|
935
948
|
"!this._isStarted",
|
|
@@ -938,7 +951,7 @@ var Pipeline = class {
|
|
|
938
951
|
});
|
|
939
952
|
log6("starting...", void 0, {
|
|
940
953
|
F: __dxlog_file7,
|
|
941
|
-
L:
|
|
954
|
+
L: 275,
|
|
942
955
|
S: this,
|
|
943
956
|
C: (f, a) => f(...a)
|
|
944
957
|
});
|
|
@@ -947,19 +960,28 @@ var Pipeline = class {
|
|
|
947
960
|
this._isStarted = true;
|
|
948
961
|
log6("started", void 0, {
|
|
949
962
|
F: __dxlog_file7,
|
|
950
|
-
L:
|
|
963
|
+
L: 279,
|
|
951
964
|
S: this,
|
|
952
965
|
C: (f, a) => f(...a)
|
|
953
966
|
});
|
|
967
|
+
if (!this._isPaused) {
|
|
968
|
+
for (const feed of this._feeds.values()) {
|
|
969
|
+
this._setFeedDownloadState(feed);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
954
972
|
}
|
|
955
973
|
async stop() {
|
|
956
974
|
log6("stopping...", void 0, {
|
|
957
975
|
F: __dxlog_file7,
|
|
958
|
-
L:
|
|
976
|
+
L: 290,
|
|
959
977
|
S: this,
|
|
960
978
|
C: (f, a) => f(...a)
|
|
961
979
|
});
|
|
962
980
|
this._isStopping = true;
|
|
981
|
+
for (const [feed, handle] of this._downloads.entries()) {
|
|
982
|
+
feed.undownload(handle);
|
|
983
|
+
}
|
|
984
|
+
this._downloads.clear();
|
|
963
985
|
await this._feedSetIterator?.close();
|
|
964
986
|
await this._processingTrigger.wait();
|
|
965
987
|
await this._state._ctx.dispose();
|
|
@@ -968,7 +990,7 @@ var Pipeline = class {
|
|
|
968
990
|
this._isStarted = false;
|
|
969
991
|
log6("stopped", void 0, {
|
|
970
992
|
F: __dxlog_file7,
|
|
971
|
-
L:
|
|
993
|
+
L: 302,
|
|
972
994
|
S: this,
|
|
973
995
|
C: (f, a) => f(...a)
|
|
974
996
|
});
|
|
@@ -980,7 +1002,7 @@ var Pipeline = class {
|
|
|
980
1002
|
async setCursor(timeframe) {
|
|
981
1003
|
invariant6(!this._isStarted || this._isPaused, "Invalid state.", {
|
|
982
1004
|
F: __dxlog_file7,
|
|
983
|
-
L:
|
|
1005
|
+
L: 311,
|
|
984
1006
|
S: this,
|
|
985
1007
|
A: [
|
|
986
1008
|
"!this._isStarted || this._isPaused",
|
|
@@ -989,9 +1011,6 @@ var Pipeline = class {
|
|
|
989
1011
|
});
|
|
990
1012
|
this._state._startTimeframe = timeframe;
|
|
991
1013
|
this._timeframeClock.setTimeframe(timeframe);
|
|
992
|
-
for (const feed of this._feeds.values()) {
|
|
993
|
-
this._setFeedDownloadState(feed);
|
|
994
|
-
}
|
|
995
1014
|
if (this._feedSetIterator) {
|
|
996
1015
|
await this._feedSetIterator.close();
|
|
997
1016
|
await this._initIterator();
|
|
@@ -1002,15 +1021,6 @@ var Pipeline = class {
|
|
|
1002
1021
|
* Calling pause while processing will cause a deadlock.
|
|
1003
1022
|
*/
|
|
1004
1023
|
async pause() {
|
|
1005
|
-
invariant6(this._isStarted, "Pipeline is not open.", {
|
|
1006
|
-
F: __dxlog_file7,
|
|
1007
|
-
L: 316,
|
|
1008
|
-
S: this,
|
|
1009
|
-
A: [
|
|
1010
|
-
"this._isStarted",
|
|
1011
|
-
"'Pipeline is not open.'"
|
|
1012
|
-
]
|
|
1013
|
-
});
|
|
1014
1024
|
if (this._isPaused) {
|
|
1015
1025
|
return;
|
|
1016
1026
|
}
|
|
@@ -1019,18 +1029,9 @@ var Pipeline = class {
|
|
|
1019
1029
|
this._isPaused = true;
|
|
1020
1030
|
}
|
|
1021
1031
|
async unpause() {
|
|
1022
|
-
invariant6(this._isStarted, "Pipeline is not open.", {
|
|
1023
|
-
F: __dxlog_file7,
|
|
1024
|
-
L: 328,
|
|
1025
|
-
S: this,
|
|
1026
|
-
A: [
|
|
1027
|
-
"this._isStarted",
|
|
1028
|
-
"'Pipeline is not open.'"
|
|
1029
|
-
]
|
|
1030
|
-
});
|
|
1031
1032
|
invariant6(this._isPaused, "Pipeline is not paused.", {
|
|
1032
1033
|
F: __dxlog_file7,
|
|
1033
|
-
L:
|
|
1034
|
+
L: 340,
|
|
1034
1035
|
S: this,
|
|
1035
1036
|
A: [
|
|
1036
1037
|
"this._isPaused",
|
|
@@ -1039,25 +1040,28 @@ var Pipeline = class {
|
|
|
1039
1040
|
});
|
|
1040
1041
|
this._pauseTrigger.wake();
|
|
1041
1042
|
this._isPaused = false;
|
|
1043
|
+
for (const feed of this._feeds.values()) {
|
|
1044
|
+
this._setFeedDownloadState(feed);
|
|
1045
|
+
}
|
|
1042
1046
|
}
|
|
1043
1047
|
/**
|
|
1044
1048
|
* Starts to iterate over the ordered messages from the added feeds.
|
|
1045
1049
|
* Updates the timeframe clock after the message has bee processed.
|
|
1046
1050
|
*/
|
|
1047
1051
|
async *consume() {
|
|
1048
|
-
invariant6(!this.
|
|
1052
|
+
invariant6(!this._isBeingConsumed, "Pipeline is already being consumed.", {
|
|
1049
1053
|
F: __dxlog_file7,
|
|
1050
|
-
L:
|
|
1054
|
+
L: 355,
|
|
1051
1055
|
S: this,
|
|
1052
1056
|
A: [
|
|
1053
|
-
"!this.
|
|
1057
|
+
"!this._isBeingConsumed",
|
|
1054
1058
|
"'Pipeline is already being consumed.'"
|
|
1055
1059
|
]
|
|
1056
1060
|
});
|
|
1057
|
-
this.
|
|
1061
|
+
this._isBeingConsumed = true;
|
|
1058
1062
|
invariant6(this._feedSetIterator, "Iterator not initialized.", {
|
|
1059
1063
|
F: __dxlog_file7,
|
|
1060
|
-
L:
|
|
1064
|
+
L: 358,
|
|
1061
1065
|
S: this,
|
|
1062
1066
|
A: [
|
|
1063
1067
|
"this._feedSetIterator",
|
|
@@ -1071,7 +1075,7 @@ var Pipeline = class {
|
|
|
1071
1075
|
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
1072
1076
|
invariant6(this._feedSetIterator, "Iterator not initialized.", {
|
|
1073
1077
|
F: __dxlog_file7,
|
|
1074
|
-
L:
|
|
1078
|
+
L: 367,
|
|
1075
1079
|
S: this,
|
|
1076
1080
|
A: [
|
|
1077
1081
|
"this._feedSetIterator",
|
|
@@ -1091,32 +1095,42 @@ var Pipeline = class {
|
|
|
1091
1095
|
this._timeframeClock.updateTimeframe();
|
|
1092
1096
|
}
|
|
1093
1097
|
}
|
|
1094
|
-
this.
|
|
1098
|
+
this._isBeingConsumed = false;
|
|
1095
1099
|
}
|
|
1096
1100
|
_setFeedDownloadState(feed) {
|
|
1101
|
+
let handle = this._downloads.get(feed);
|
|
1102
|
+
if (handle) {
|
|
1103
|
+
feed.undownload(handle);
|
|
1104
|
+
}
|
|
1097
1105
|
const timeframe = this._state._startTimeframe;
|
|
1098
|
-
const seq = timeframe.get(feed.key) ??
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
+
const seq = timeframe.get(feed.key) ?? -1;
|
|
1107
|
+
log6("download", {
|
|
1108
|
+
feed: feed.key.truncate(),
|
|
1109
|
+
seq,
|
|
1110
|
+
length: feed.length
|
|
1111
|
+
}, {
|
|
1112
|
+
F: __dxlog_file7,
|
|
1113
|
+
L: 396,
|
|
1114
|
+
S: this,
|
|
1115
|
+
C: (f, a) => f(...a)
|
|
1106
1116
|
});
|
|
1107
|
-
feed.download({
|
|
1117
|
+
handle = feed.download({
|
|
1108
1118
|
start: seq + 1,
|
|
1109
1119
|
linear: true
|
|
1110
|
-
}
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1120
|
+
}, (err, data) => {
|
|
1121
|
+
if (err) {
|
|
1122
|
+
} else {
|
|
1123
|
+
log6.info("downloaded", {
|
|
1124
|
+
data
|
|
1125
|
+
}, {
|
|
1126
|
+
F: __dxlog_file7,
|
|
1127
|
+
L: 401,
|
|
1128
|
+
S: this,
|
|
1129
|
+
C: (f, a) => f(...a)
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1119
1132
|
});
|
|
1133
|
+
this._downloads.set(feed, handle);
|
|
1120
1134
|
}
|
|
1121
1135
|
async _initIterator() {
|
|
1122
1136
|
this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
@@ -1126,7 +1140,7 @@ var Pipeline = class {
|
|
|
1126
1140
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1127
1141
|
log6.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1128
1142
|
F: __dxlog_file7,
|
|
1129
|
-
L:
|
|
1143
|
+
L: 415,
|
|
1130
1144
|
S: this,
|
|
1131
1145
|
C: (f, a) => f(...a)
|
|
1132
1146
|
});
|
|
@@ -1161,7 +1175,7 @@ import { invariant as invariant7 } from "@dxos/invariant";
|
|
|
1161
1175
|
import { log as log7 } from "@dxos/log";
|
|
1162
1176
|
import { schema as schema5 } from "@dxos/protocols";
|
|
1163
1177
|
import { RpcExtension } from "@dxos/teleport";
|
|
1164
|
-
var __dxlog_file8 = "/
|
|
1178
|
+
var __dxlog_file8 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/auth.ts";
|
|
1165
1179
|
var AuthExtension = class extends RpcExtension {
|
|
1166
1180
|
constructor(_authParams) {
|
|
1167
1181
|
super({
|
|
@@ -1260,11 +1274,12 @@ var AuthExtension = class extends RpcExtension {
|
|
|
1260
1274
|
import { Event as Event4, scheduleTask as scheduleTask2, synchronized as synchronized3, trackLeaks as trackLeaks2 } from "@dxos/async";
|
|
1261
1275
|
import { Context as Context4 } from "@dxos/context";
|
|
1262
1276
|
import { checkCredentialType } from "@dxos/credentials";
|
|
1263
|
-
import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
|
|
1277
|
+
import { getStateMachineFromItem, ItemManager, TYPE_PROPERTIES } from "@dxos/echo-db";
|
|
1264
1278
|
import { CancelledError } from "@dxos/errors";
|
|
1265
1279
|
import { invariant as invariant8 } from "@dxos/invariant";
|
|
1266
|
-
import { log as log8 } from "@dxos/log";
|
|
1280
|
+
import { log as log8, omit } from "@dxos/log";
|
|
1267
1281
|
import { Timeframe as Timeframe3 } from "@dxos/timeframe";
|
|
1282
|
+
import { TimeSeriesCounter, TimeUsageCounter, trace } from "@dxos/tracing";
|
|
1268
1283
|
import { tracer } from "@dxos/util";
|
|
1269
1284
|
function _ts_decorate5(decorators, target, key, desc) {
|
|
1270
1285
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
@@ -1276,7 +1291,7 @@ function _ts_decorate5(decorators, target, key, desc) {
|
|
|
1276
1291
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1277
1292
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1278
1293
|
}
|
|
1279
|
-
var __dxlog_file9 = "/
|
|
1294
|
+
var __dxlog_file9 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
|
|
1280
1295
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1281
1296
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1282
1297
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
@@ -1291,6 +1306,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1291
1306
|
this._lastTimeframeSaveTime = 0;
|
|
1292
1307
|
this._lastSnapshotSaveTime = 0;
|
|
1293
1308
|
this._lastProcessedEpoch = -1;
|
|
1309
|
+
this._usage = new TimeUsageCounter();
|
|
1310
|
+
this._mutations = new TimeSeriesCounter();
|
|
1294
1311
|
this.currentEpoch = void 0;
|
|
1295
1312
|
this.appliedEpoch = void 0;
|
|
1296
1313
|
this.onNewEpoch = new Event4();
|
|
@@ -1323,8 +1340,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1323
1340
|
}
|
|
1324
1341
|
this._pipeline = new Pipeline();
|
|
1325
1342
|
await this._params.onPipelineCreated(this._pipeline);
|
|
1326
|
-
await this._pipeline.start();
|
|
1327
1343
|
await this._pipeline.pause();
|
|
1344
|
+
await this._pipeline.start();
|
|
1328
1345
|
if (this._targetTimeframe) {
|
|
1329
1346
|
this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
|
|
1330
1347
|
}
|
|
@@ -1332,7 +1349,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1332
1349
|
write: (data, options) => {
|
|
1333
1350
|
invariant8(this._pipeline, "Pipeline is not initialized.", {
|
|
1334
1351
|
F: __dxlog_file9,
|
|
1335
|
-
L:
|
|
1352
|
+
L: 154,
|
|
1336
1353
|
S: this,
|
|
1337
1354
|
A: [
|
|
1338
1355
|
"this._pipeline",
|
|
@@ -1341,7 +1358,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1341
1358
|
});
|
|
1342
1359
|
invariant8(this.currentEpoch, "Epoch is not initialized.", {
|
|
1343
1360
|
F: __dxlog_file9,
|
|
1344
|
-
L:
|
|
1361
|
+
L: 155,
|
|
1345
1362
|
S: this,
|
|
1346
1363
|
A: [
|
|
1347
1364
|
"this.currentEpoch",
|
|
@@ -1367,7 +1384,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1367
1384
|
}
|
|
1368
1385
|
log8("close", void 0, {
|
|
1369
1386
|
F: __dxlog_file9,
|
|
1370
|
-
L:
|
|
1387
|
+
L: 179,
|
|
1371
1388
|
S: this,
|
|
1372
1389
|
C: (f, a) => f(...a)
|
|
1373
1390
|
});
|
|
@@ -1382,7 +1399,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1382
1399
|
} catch (err) {
|
|
1383
1400
|
log8.catch(err, void 0, {
|
|
1384
1401
|
F: __dxlog_file9,
|
|
1385
|
-
L:
|
|
1402
|
+
L: 192,
|
|
1386
1403
|
S: this,
|
|
1387
1404
|
C: (f, a) => f(...a)
|
|
1388
1405
|
});
|
|
@@ -1406,7 +1423,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1406
1423
|
}
|
|
1407
1424
|
invariant8(this._pipeline, "Pipeline is not initialized.", {
|
|
1408
1425
|
F: __dxlog_file9,
|
|
1409
|
-
L:
|
|
1426
|
+
L: 215,
|
|
1410
1427
|
S: this,
|
|
1411
1428
|
A: [
|
|
1412
1429
|
"this._pipeline",
|
|
@@ -1414,13 +1431,15 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1414
1431
|
]
|
|
1415
1432
|
});
|
|
1416
1433
|
for await (const msg of this._pipeline.consume()) {
|
|
1434
|
+
const span = this._usage.beginRecording();
|
|
1435
|
+
this._mutations.inc();
|
|
1417
1436
|
const { feedKey, seq, data } = msg;
|
|
1418
1437
|
log8("processing message", {
|
|
1419
1438
|
feedKey,
|
|
1420
1439
|
seq
|
|
1421
1440
|
}, {
|
|
1422
1441
|
F: __dxlog_file9,
|
|
1423
|
-
L:
|
|
1442
|
+
L: 221,
|
|
1424
1443
|
S: this,
|
|
1425
1444
|
C: (f, a) => f(...a)
|
|
1426
1445
|
});
|
|
@@ -1432,7 +1451,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1432
1451
|
feedKey
|
|
1433
1452
|
}, {
|
|
1434
1453
|
F: __dxlog_file9,
|
|
1435
|
-
L:
|
|
1454
|
+
L: 227,
|
|
1436
1455
|
S: this,
|
|
1437
1456
|
C: (f, a) => f(...a)
|
|
1438
1457
|
});
|
|
@@ -1455,7 +1474,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1455
1474
|
spaceKey: this._params.spaceKey.toHex()
|
|
1456
1475
|
}, {
|
|
1457
1476
|
F: __dxlog_file9,
|
|
1458
|
-
L:
|
|
1477
|
+
L: 244,
|
|
1459
1478
|
S: this,
|
|
1460
1479
|
C: (f, a) => f(...a)
|
|
1461
1480
|
});
|
|
@@ -1464,17 +1483,18 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1464
1483
|
} catch (err) {
|
|
1465
1484
|
log8.catch(err, void 0, {
|
|
1466
1485
|
F: __dxlog_file9,
|
|
1467
|
-
L:
|
|
1486
|
+
L: 254,
|
|
1468
1487
|
S: this,
|
|
1469
1488
|
C: (f, a) => f(...a)
|
|
1470
1489
|
});
|
|
1471
1490
|
}
|
|
1491
|
+
span.end();
|
|
1472
1492
|
}
|
|
1473
1493
|
}
|
|
1474
1494
|
_createSnapshot() {
|
|
1475
1495
|
invariant8(this.databaseHost, "Database backend is not initialized.", {
|
|
1476
1496
|
F: __dxlog_file9,
|
|
1477
|
-
L:
|
|
1497
|
+
L: 262,
|
|
1478
1498
|
S: this,
|
|
1479
1499
|
A: [
|
|
1480
1500
|
"this.databaseHost",
|
|
@@ -1496,14 +1516,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1496
1516
|
const cache = {};
|
|
1497
1517
|
try {
|
|
1498
1518
|
const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos:model/document" && // TODO(burdon): Document?
|
|
1499
|
-
(getStateMachineFromItem(item)?.snapshot()).type ===
|
|
1519
|
+
(getStateMachineFromItem(item)?.snapshot()).type === TYPE_PROPERTIES);
|
|
1500
1520
|
if (propertiesItem) {
|
|
1501
1521
|
cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot();
|
|
1502
1522
|
}
|
|
1503
1523
|
} catch (err) {
|
|
1504
1524
|
log8.warn("Failed to cache properties", err, {
|
|
1505
1525
|
F: __dxlog_file9,
|
|
1506
|
-
L:
|
|
1526
|
+
L: 291,
|
|
1507
1527
|
S: this,
|
|
1508
1528
|
C: (f, a) => f(...a)
|
|
1509
1529
|
});
|
|
@@ -1529,14 +1549,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1529
1549
|
if (err instanceof CancelledError) {
|
|
1530
1550
|
log8("Epoch processing cancelled.", void 0, {
|
|
1531
1551
|
F: __dxlog_file9,
|
|
1532
|
-
L:
|
|
1552
|
+
L: 323,
|
|
1533
1553
|
S: this,
|
|
1534
1554
|
C: (f, a) => f(...a)
|
|
1535
1555
|
});
|
|
1536
1556
|
} else {
|
|
1537
1557
|
log8.catch(err, void 0, {
|
|
1538
1558
|
F: __dxlog_file9,
|
|
1539
|
-
L:
|
|
1559
|
+
L: 325,
|
|
1540
1560
|
S: this,
|
|
1541
1561
|
C: (f, a) => f(...a)
|
|
1542
1562
|
});
|
|
@@ -1548,14 +1568,6 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1548
1568
|
if (!this._isOpen) {
|
|
1549
1569
|
return;
|
|
1550
1570
|
}
|
|
1551
|
-
log8("process epoch", {
|
|
1552
|
-
epoch
|
|
1553
|
-
}, {
|
|
1554
|
-
F: __dxlog_file9,
|
|
1555
|
-
L: 321,
|
|
1556
|
-
S: this,
|
|
1557
|
-
C: (f, a) => f(...a)
|
|
1558
|
-
});
|
|
1559
1571
|
await this._processEpoch(ctx, epoch.subject.assertion);
|
|
1560
1572
|
this.appliedEpoch = epoch;
|
|
1561
1573
|
this.onNewEpoch.emit(epoch);
|
|
@@ -1564,7 +1576,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1564
1576
|
async _processEpoch(ctx, epoch) {
|
|
1565
1577
|
invariant8(this._isOpen, "Space is closed.", {
|
|
1566
1578
|
F: __dxlog_file9,
|
|
1567
|
-
L:
|
|
1579
|
+
L: 344,
|
|
1568
1580
|
S: this,
|
|
1569
1581
|
A: [
|
|
1570
1582
|
"this._isOpen",
|
|
@@ -1573,7 +1585,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1573
1585
|
});
|
|
1574
1586
|
invariant8(this._pipeline, void 0, {
|
|
1575
1587
|
F: __dxlog_file9,
|
|
1576
|
-
L:
|
|
1588
|
+
L: 345,
|
|
1577
1589
|
S: this,
|
|
1578
1590
|
A: [
|
|
1579
1591
|
"this._pipeline",
|
|
@@ -1581,11 +1593,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1581
1593
|
]
|
|
1582
1594
|
});
|
|
1583
1595
|
this._lastProcessedEpoch = epoch.number;
|
|
1584
|
-
log8("
|
|
1585
|
-
epoch
|
|
1596
|
+
log8("processing", {
|
|
1597
|
+
epoch: omit(epoch, "proof")
|
|
1586
1598
|
}, {
|
|
1587
1599
|
F: __dxlog_file9,
|
|
1588
|
-
L:
|
|
1600
|
+
L: 348,
|
|
1589
1601
|
S: this,
|
|
1590
1602
|
C: (f, a) => f(...a)
|
|
1591
1603
|
});
|
|
@@ -1593,9 +1605,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1593
1605
|
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1594
1606
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1595
1607
|
}
|
|
1596
|
-
log8("restarting pipeline
|
|
1608
|
+
log8("restarting pipeline from epoch", void 0, {
|
|
1597
1609
|
F: __dxlog_file9,
|
|
1598
|
-
L:
|
|
1610
|
+
L: 354,
|
|
1599
1611
|
S: this,
|
|
1600
1612
|
C: (f, a) => f(...a)
|
|
1601
1613
|
});
|
|
@@ -1606,7 +1618,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1606
1618
|
async waitUntilTimeframe(timeframe) {
|
|
1607
1619
|
invariant8(this._pipeline, "Pipeline is not initialized.", {
|
|
1608
1620
|
F: __dxlog_file9,
|
|
1609
|
-
L:
|
|
1621
|
+
L: 361,
|
|
1610
1622
|
S: this,
|
|
1611
1623
|
A: [
|
|
1612
1624
|
"this._pipeline",
|
|
@@ -1618,7 +1630,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1618
1630
|
async createEpoch() {
|
|
1619
1631
|
invariant8(this._pipeline, void 0, {
|
|
1620
1632
|
F: __dxlog_file9,
|
|
1621
|
-
L:
|
|
1633
|
+
L: 367,
|
|
1622
1634
|
S: this,
|
|
1623
1635
|
A: [
|
|
1624
1636
|
"this._pipeline",
|
|
@@ -1627,7 +1639,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1627
1639
|
});
|
|
1628
1640
|
invariant8(this.currentEpoch, void 0, {
|
|
1629
1641
|
F: __dxlog_file9,
|
|
1630
|
-
L:
|
|
1642
|
+
L: 368,
|
|
1631
1643
|
S: this,
|
|
1632
1644
|
A: [
|
|
1633
1645
|
"this.currentEpoch",
|
|
@@ -1650,6 +1662,12 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1650
1662
|
await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
|
|
1651
1663
|
}
|
|
1652
1664
|
};
|
|
1665
|
+
_ts_decorate5([
|
|
1666
|
+
trace.metricsCounter()
|
|
1667
|
+
], DataPipeline.prototype, "_usage", void 0);
|
|
1668
|
+
_ts_decorate5([
|
|
1669
|
+
trace.metricsCounter()
|
|
1670
|
+
], DataPipeline.prototype, "_mutations", void 0);
|
|
1653
1671
|
_ts_decorate5([
|
|
1654
1672
|
synchronized3
|
|
1655
1673
|
], DataPipeline.prototype, "open", null);
|
|
@@ -1663,7 +1681,8 @@ _ts_decorate5([
|
|
|
1663
1681
|
synchronized3
|
|
1664
1682
|
], DataPipeline.prototype, "createEpoch", null);
|
|
1665
1683
|
DataPipeline = _ts_decorate5([
|
|
1666
|
-
trackLeaks2("open", "close")
|
|
1684
|
+
trackLeaks2("open", "close"),
|
|
1685
|
+
trace.resource()
|
|
1667
1686
|
], DataPipeline);
|
|
1668
1687
|
|
|
1669
1688
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
@@ -1671,22 +1690,36 @@ import { Event as Event5, synchronized as synchronized4, trackLeaks as trackLeak
|
|
|
1671
1690
|
import { invariant as invariant9 } from "@dxos/invariant";
|
|
1672
1691
|
import { log as log10, logInfo } from "@dxos/log";
|
|
1673
1692
|
import { AdmittedFeed as AdmittedFeed2 } from "@dxos/protocols/proto/dxos/halo/credentials";
|
|
1674
|
-
import { trace } from "@dxos/tracing";
|
|
1693
|
+
import { trace as trace3 } from "@dxos/tracing";
|
|
1675
1694
|
import { Callback as Callback2 } from "@dxos/util";
|
|
1676
1695
|
|
|
1677
1696
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1697
|
+
import { Context as Context5 } from "@dxos/context";
|
|
1678
1698
|
import { SpaceStateMachine } from "@dxos/credentials";
|
|
1679
1699
|
import { PublicKey as PublicKey4 } from "@dxos/keys";
|
|
1680
1700
|
import { log as log9 } from "@dxos/log";
|
|
1681
1701
|
import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
|
|
1682
1702
|
import { Timeframe as Timeframe4 } from "@dxos/timeframe";
|
|
1703
|
+
import { TimeSeriesCounter as TimeSeriesCounter2, TimeUsageCounter as TimeUsageCounter2, trace as trace2 } from "@dxos/tracing";
|
|
1683
1704
|
import { Callback, tracer as tracer2 } from "@dxos/util";
|
|
1684
|
-
|
|
1705
|
+
function _ts_decorate6(decorators, target, key, desc) {
|
|
1706
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1707
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1708
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
1709
|
+
else
|
|
1710
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
1711
|
+
if (d = decorators[i])
|
|
1712
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1713
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1714
|
+
}
|
|
1715
|
+
var __dxlog_file10 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1685
1716
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
1686
|
-
var ControlPipeline = class {
|
|
1717
|
+
var ControlPipeline = class ControlPipeline2 {
|
|
1687
1718
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
1688
1719
|
this._lastTimeframeSaveTime = Date.now();
|
|
1689
1720
|
this.onFeedAdmitted = new Callback();
|
|
1721
|
+
this._usage = new TimeUsageCounter2();
|
|
1722
|
+
this._mutations = new TimeSeriesCounter2();
|
|
1690
1723
|
this._spaceKey = spaceKey;
|
|
1691
1724
|
this._metadata = metadataStore;
|
|
1692
1725
|
this._pipeline = new Pipeline();
|
|
@@ -1697,7 +1730,7 @@ var ControlPipeline = class {
|
|
|
1697
1730
|
key: info.key
|
|
1698
1731
|
}, {
|
|
1699
1732
|
F: __dxlog_file10,
|
|
1700
|
-
L:
|
|
1733
|
+
L: 61,
|
|
1701
1734
|
S: this,
|
|
1702
1735
|
C: (f, a) => f(...a)
|
|
1703
1736
|
});
|
|
@@ -1708,7 +1741,7 @@ var ControlPipeline = class {
|
|
|
1708
1741
|
} catch (err) {
|
|
1709
1742
|
log9.catch(err, void 0, {
|
|
1710
1743
|
F: __dxlog_file10,
|
|
1711
|
-
L:
|
|
1744
|
+
L: 69,
|
|
1712
1745
|
S: this,
|
|
1713
1746
|
C: (f, a) => f(...a)
|
|
1714
1747
|
});
|
|
@@ -1731,57 +1764,66 @@ var ControlPipeline = class {
|
|
|
1731
1764
|
async start() {
|
|
1732
1765
|
log9("starting...", void 0, {
|
|
1733
1766
|
F: __dxlog_file10,
|
|
1734
|
-
L:
|
|
1767
|
+
L: 93,
|
|
1735
1768
|
S: this,
|
|
1736
1769
|
C: (f, a) => f(...a)
|
|
1737
1770
|
});
|
|
1738
1771
|
setTimeout(async () => {
|
|
1739
|
-
|
|
1740
|
-
try {
|
|
1741
|
-
log9("processing", {
|
|
1742
|
-
key: msg.feedKey,
|
|
1743
|
-
seq: msg.seq
|
|
1744
|
-
}, {
|
|
1745
|
-
F: __dxlog_file10,
|
|
1746
|
-
L: 88,
|
|
1747
|
-
S: this,
|
|
1748
|
-
C: (f, a) => f(...a)
|
|
1749
|
-
});
|
|
1750
|
-
if (msg.data.payload.credential) {
|
|
1751
|
-
const timer = tracer2.mark("dxos.echo.pipeline.control");
|
|
1752
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
|
|
1753
|
-
timer.end();
|
|
1754
|
-
if (!result) {
|
|
1755
|
-
log9.warn("processing failed", {
|
|
1756
|
-
msg
|
|
1757
|
-
}, {
|
|
1758
|
-
F: __dxlog_file10,
|
|
1759
|
-
L: 98,
|
|
1760
|
-
S: this,
|
|
1761
|
-
C: (f, a) => f(...a)
|
|
1762
|
-
});
|
|
1763
|
-
} else {
|
|
1764
|
-
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
} catch (err) {
|
|
1768
|
-
log9.catch(err, void 0, {
|
|
1769
|
-
F: __dxlog_file10,
|
|
1770
|
-
L: 104,
|
|
1771
|
-
S: this,
|
|
1772
|
-
C: (f, a) => f(...a)
|
|
1773
|
-
});
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1772
|
+
void this._consumePipeline(new Context5());
|
|
1776
1773
|
});
|
|
1777
1774
|
await this._pipeline.start();
|
|
1778
1775
|
log9("started", void 0, {
|
|
1779
1776
|
F: __dxlog_file10,
|
|
1780
|
-
L:
|
|
1777
|
+
L: 99,
|
|
1781
1778
|
S: this,
|
|
1782
1779
|
C: (f, a) => f(...a)
|
|
1783
1780
|
});
|
|
1784
1781
|
}
|
|
1782
|
+
async _consumePipeline(ctx) {
|
|
1783
|
+
for await (const msg of this._pipeline.consume()) {
|
|
1784
|
+
const span = this._usage.beginRecording();
|
|
1785
|
+
this._mutations.inc();
|
|
1786
|
+
try {
|
|
1787
|
+
await this._processMessage(ctx, msg);
|
|
1788
|
+
} catch (err) {
|
|
1789
|
+
log9.catch(err, void 0, {
|
|
1790
|
+
F: __dxlog_file10,
|
|
1791
|
+
L: 111,
|
|
1792
|
+
S: this,
|
|
1793
|
+
C: (f, a) => f(...a)
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
span.end();
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
async _processMessage(ctx, msg) {
|
|
1800
|
+
log9("processing", {
|
|
1801
|
+
key: msg.feedKey,
|
|
1802
|
+
seq: msg.seq
|
|
1803
|
+
}, {
|
|
1804
|
+
F: __dxlog_file10,
|
|
1805
|
+
L: 121,
|
|
1806
|
+
S: this,
|
|
1807
|
+
C: (f, a) => f(...a)
|
|
1808
|
+
});
|
|
1809
|
+
if (msg.data.payload.credential) {
|
|
1810
|
+
const timer = tracer2.mark("dxos.echo.pipeline.control");
|
|
1811
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
|
|
1812
|
+
timer.end();
|
|
1813
|
+
if (!result) {
|
|
1814
|
+
log9.warn("processing failed", {
|
|
1815
|
+
msg
|
|
1816
|
+
}, {
|
|
1817
|
+
F: __dxlog_file10,
|
|
1818
|
+
L: 131,
|
|
1819
|
+
S: this,
|
|
1820
|
+
C: (f, a) => f(...a)
|
|
1821
|
+
});
|
|
1822
|
+
} else {
|
|
1823
|
+
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1785
1827
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
1786
1828
|
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
|
|
1787
1829
|
this._lastTimeframeSaveTime = Date.now();
|
|
@@ -1791,7 +1833,7 @@ var ControlPipeline = class {
|
|
|
1791
1833
|
async stop() {
|
|
1792
1834
|
log9("stopping...", void 0, {
|
|
1793
1835
|
F: __dxlog_file10,
|
|
1794
|
-
L:
|
|
1836
|
+
L: 149,
|
|
1795
1837
|
S: this,
|
|
1796
1838
|
C: (f, a) => f(...a)
|
|
1797
1839
|
});
|
|
@@ -1799,7 +1841,7 @@ var ControlPipeline = class {
|
|
|
1799
1841
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1800
1842
|
log9("stopped", void 0, {
|
|
1801
1843
|
F: __dxlog_file10,
|
|
1802
|
-
L:
|
|
1844
|
+
L: 152,
|
|
1803
1845
|
S: this,
|
|
1804
1846
|
C: (f, a) => f(...a)
|
|
1805
1847
|
});
|
|
@@ -1812,16 +1854,31 @@ var ControlPipeline = class {
|
|
|
1812
1854
|
} catch (err) {
|
|
1813
1855
|
log9(err, void 0, {
|
|
1814
1856
|
F: __dxlog_file10,
|
|
1815
|
-
L:
|
|
1857
|
+
L: 161,
|
|
1816
1858
|
S: this,
|
|
1817
1859
|
C: (f, a) => f(...a)
|
|
1818
1860
|
});
|
|
1819
1861
|
}
|
|
1820
1862
|
}
|
|
1821
1863
|
};
|
|
1864
|
+
_ts_decorate6([
|
|
1865
|
+
trace2.metricsCounter()
|
|
1866
|
+
], ControlPipeline.prototype, "_usage", void 0);
|
|
1867
|
+
_ts_decorate6([
|
|
1868
|
+
trace2.metricsCounter()
|
|
1869
|
+
], ControlPipeline.prototype, "_mutations", void 0);
|
|
1870
|
+
_ts_decorate6([
|
|
1871
|
+
trace2.span()
|
|
1872
|
+
], ControlPipeline.prototype, "_consumePipeline", null);
|
|
1873
|
+
_ts_decorate6([
|
|
1874
|
+
trace2.span()
|
|
1875
|
+
], ControlPipeline.prototype, "_processMessage", null);
|
|
1876
|
+
ControlPipeline = _ts_decorate6([
|
|
1877
|
+
trace2.resource()
|
|
1878
|
+
], ControlPipeline);
|
|
1822
1879
|
|
|
1823
1880
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1824
|
-
function
|
|
1881
|
+
function _ts_decorate7(decorators, target, key, desc) {
|
|
1825
1882
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1826
1883
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1827
1884
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1831,7 +1888,7 @@ function _ts_decorate6(decorators, target, key, desc) {
|
|
|
1831
1888
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1832
1889
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1833
1890
|
}
|
|
1834
|
-
var __dxlog_file11 = "/
|
|
1891
|
+
var __dxlog_file11 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space.ts";
|
|
1835
1892
|
var Space = class Space2 {
|
|
1836
1893
|
constructor(params) {
|
|
1837
1894
|
this._addFeedLock = new Lock();
|
|
@@ -1947,7 +2004,7 @@ var Space = class Space2 {
|
|
|
1947
2004
|
setControlFeed(feed) {
|
|
1948
2005
|
invariant9(!this._controlFeed, "Control feed already set.", {
|
|
1949
2006
|
F: __dxlog_file11,
|
|
1950
|
-
L:
|
|
2007
|
+
L: 184,
|
|
1951
2008
|
S: this,
|
|
1952
2009
|
A: [
|
|
1953
2010
|
"!this._controlFeed",
|
|
@@ -1961,7 +2018,7 @@ var Space = class Space2 {
|
|
|
1961
2018
|
setDataFeed(feed) {
|
|
1962
2019
|
invariant9(!this._dataFeed, "Data feed already set.", {
|
|
1963
2020
|
F: __dxlog_file11,
|
|
1964
|
-
L:
|
|
2021
|
+
L: 191,
|
|
1965
2022
|
S: this,
|
|
1966
2023
|
A: [
|
|
1967
2024
|
"!this._dataFeed",
|
|
@@ -1987,7 +2044,7 @@ var Space = class Space2 {
|
|
|
1987
2044
|
async open(ctx) {
|
|
1988
2045
|
log10("opening...", void 0, {
|
|
1989
2046
|
F: __dxlog_file11,
|
|
1990
|
-
L:
|
|
2047
|
+
L: 213,
|
|
1991
2048
|
S: this,
|
|
1992
2049
|
C: (f, a) => f(...a)
|
|
1993
2050
|
});
|
|
@@ -2000,7 +2057,7 @@ var Space = class Space2 {
|
|
|
2000
2057
|
this._isOpen = true;
|
|
2001
2058
|
log10("opened", void 0, {
|
|
2002
2059
|
F: __dxlog_file11,
|
|
2003
|
-
L:
|
|
2060
|
+
L: 224,
|
|
2004
2061
|
S: this,
|
|
2005
2062
|
C: (f, a) => f(...a)
|
|
2006
2063
|
});
|
|
@@ -2010,7 +2067,7 @@ var Space = class Space2 {
|
|
|
2010
2067
|
key: this._key
|
|
2011
2068
|
}, {
|
|
2012
2069
|
F: __dxlog_file11,
|
|
2013
|
-
L:
|
|
2070
|
+
L: 229,
|
|
2014
2071
|
S: this,
|
|
2015
2072
|
C: (f, a) => f(...a)
|
|
2016
2073
|
});
|
|
@@ -2024,7 +2081,7 @@ var Space = class Space2 {
|
|
|
2024
2081
|
this._isOpen = false;
|
|
2025
2082
|
log10("closed", void 0, {
|
|
2026
2083
|
F: __dxlog_file11,
|
|
2027
|
-
L:
|
|
2084
|
+
L: 242,
|
|
2028
2085
|
S: this,
|
|
2029
2086
|
C: (f, a) => f(...a)
|
|
2030
2087
|
});
|
|
@@ -2032,13 +2089,13 @@ var Space = class Space2 {
|
|
|
2032
2089
|
async initializeDataPipeline() {
|
|
2033
2090
|
log10("initializeDataPipeline", void 0, {
|
|
2034
2091
|
F: __dxlog_file11,
|
|
2035
|
-
L:
|
|
2092
|
+
L: 247,
|
|
2036
2093
|
S: this,
|
|
2037
2094
|
C: (f, a) => f(...a)
|
|
2038
2095
|
});
|
|
2039
2096
|
invariant9(this._isOpen, "Space must be open to initialize data pipeline.", {
|
|
2040
2097
|
F: __dxlog_file11,
|
|
2041
|
-
L:
|
|
2098
|
+
L: 248,
|
|
2042
2099
|
S: this,
|
|
2043
2100
|
A: [
|
|
2044
2101
|
"this._isOpen",
|
|
@@ -2048,22 +2105,23 @@ var Space = class Space2 {
|
|
|
2048
2105
|
await this._dataPipeline.open();
|
|
2049
2106
|
}
|
|
2050
2107
|
};
|
|
2051
|
-
|
|
2052
|
-
logInfo
|
|
2108
|
+
_ts_decorate7([
|
|
2109
|
+
logInfo,
|
|
2110
|
+
trace3.info()
|
|
2053
2111
|
], Space.prototype, "key", null);
|
|
2054
|
-
|
|
2112
|
+
_ts_decorate7([
|
|
2055
2113
|
synchronized4,
|
|
2056
|
-
|
|
2114
|
+
trace3.span()
|
|
2057
2115
|
], Space.prototype, "open", null);
|
|
2058
|
-
|
|
2116
|
+
_ts_decorate7([
|
|
2059
2117
|
synchronized4
|
|
2060
2118
|
], Space.prototype, "close", null);
|
|
2061
|
-
|
|
2119
|
+
_ts_decorate7([
|
|
2062
2120
|
synchronized4
|
|
2063
2121
|
], Space.prototype, "initializeDataPipeline", null);
|
|
2064
|
-
Space =
|
|
2122
|
+
Space = _ts_decorate7([
|
|
2065
2123
|
trackLeaks3("open", "close"),
|
|
2066
|
-
|
|
2124
|
+
trace3.resource()
|
|
2067
2125
|
], Space);
|
|
2068
2126
|
|
|
2069
2127
|
// packages/core/echo/echo-pipeline/src/space/space-protocol.ts
|
|
@@ -2075,7 +2133,7 @@ import { Teleport } from "@dxos/teleport";
|
|
|
2075
2133
|
import { BlobSync } from "@dxos/teleport-extension-object-sync";
|
|
2076
2134
|
import { ReplicatorExtension } from "@dxos/teleport-extension-replicator";
|
|
2077
2135
|
import { ComplexMap as ComplexMap4 } from "@dxos/util";
|
|
2078
|
-
function
|
|
2136
|
+
function _ts_decorate8(decorators, target, key, desc) {
|
|
2079
2137
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2080
2138
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2081
2139
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2085,7 +2143,7 @@ function _ts_decorate7(decorators, target, key, desc) {
|
|
|
2085
2143
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2086
2144
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2087
2145
|
}
|
|
2088
|
-
var __dxlog_file12 = "/
|
|
2146
|
+
var __dxlog_file12 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
|
|
2089
2147
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
2090
2148
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
2091
2149
|
var SpaceProtocol = class {
|
|
@@ -2116,7 +2174,7 @@ var SpaceProtocol = class {
|
|
|
2116
2174
|
key: feed.key
|
|
2117
2175
|
}, {
|
|
2118
2176
|
F: __dxlog_file12,
|
|
2119
|
-
L:
|
|
2177
|
+
L: 96,
|
|
2120
2178
|
S: this,
|
|
2121
2179
|
C: (f, a) => f(...a)
|
|
2122
2180
|
});
|
|
@@ -2125,6 +2183,7 @@ var SpaceProtocol = class {
|
|
|
2125
2183
|
session.replicator.addFeed(feed);
|
|
2126
2184
|
}
|
|
2127
2185
|
}
|
|
2186
|
+
// TODO(burdon): Rename open? Common open/close interfaces for all services?
|
|
2128
2187
|
async start() {
|
|
2129
2188
|
if (this._connection) {
|
|
2130
2189
|
return;
|
|
@@ -2138,7 +2197,7 @@ var SpaceProtocol = class {
|
|
|
2138
2197
|
await this.blobSync.open();
|
|
2139
2198
|
log11("starting...", void 0, {
|
|
2140
2199
|
F: __dxlog_file12,
|
|
2141
|
-
L:
|
|
2200
|
+
L: 122,
|
|
2142
2201
|
S: this,
|
|
2143
2202
|
C: (f, a) => f(...a)
|
|
2144
2203
|
});
|
|
@@ -2148,11 +2207,11 @@ var SpaceProtocol = class {
|
|
|
2148
2207
|
peerId: this._swarmIdentity.peerKey,
|
|
2149
2208
|
topic,
|
|
2150
2209
|
topology: new MMSTTopology(topologyConfig),
|
|
2151
|
-
label: `
|
|
2210
|
+
label: `space swarm ${topic.truncate()}`
|
|
2152
2211
|
});
|
|
2153
2212
|
log11("started", void 0, {
|
|
2154
2213
|
F: __dxlog_file12,
|
|
2155
|
-
L:
|
|
2214
|
+
L: 132,
|
|
2156
2215
|
S: this,
|
|
2157
2216
|
C: (f, a) => f(...a)
|
|
2158
2217
|
});
|
|
@@ -2162,14 +2221,14 @@ var SpaceProtocol = class {
|
|
|
2162
2221
|
if (this._connection) {
|
|
2163
2222
|
log11("stopping...", void 0, {
|
|
2164
2223
|
F: __dxlog_file12,
|
|
2165
|
-
L:
|
|
2224
|
+
L: 139,
|
|
2166
2225
|
S: this,
|
|
2167
2226
|
C: (f, a) => f(...a)
|
|
2168
2227
|
});
|
|
2169
2228
|
await this._connection.close();
|
|
2170
2229
|
log11("stopped", void 0, {
|
|
2171
2230
|
F: __dxlog_file12,
|
|
2172
|
-
L:
|
|
2231
|
+
L: 141,
|
|
2173
2232
|
S: this,
|
|
2174
2233
|
C: (f, a) => f(...a)
|
|
2175
2234
|
});
|
|
@@ -2192,10 +2251,10 @@ var SpaceProtocol = class {
|
|
|
2192
2251
|
};
|
|
2193
2252
|
}
|
|
2194
2253
|
};
|
|
2195
|
-
|
|
2254
|
+
_ts_decorate8([
|
|
2196
2255
|
logInfo2
|
|
2197
2256
|
], SpaceProtocol.prototype, "_topic", void 0);
|
|
2198
|
-
|
|
2257
|
+
_ts_decorate8([
|
|
2199
2258
|
logInfo2
|
|
2200
2259
|
], SpaceProtocol.prototype, "_ownPeerKey", null);
|
|
2201
2260
|
var AuthStatus;
|
|
@@ -2236,7 +2295,7 @@ var SpaceProtocolSession = class {
|
|
|
2236
2295
|
onAuthSuccess: () => {
|
|
2237
2296
|
log11("Peer authenticated", void 0, {
|
|
2238
2297
|
F: __dxlog_file12,
|
|
2239
|
-
L:
|
|
2298
|
+
L: 238,
|
|
2240
2299
|
S: this,
|
|
2241
2300
|
C: (f, a) => f(...a)
|
|
2242
2301
|
});
|
|
@@ -2255,10 +2314,10 @@ var SpaceProtocolSession = class {
|
|
|
2255
2314
|
await this._teleport.close();
|
|
2256
2315
|
}
|
|
2257
2316
|
};
|
|
2258
|
-
|
|
2317
|
+
_ts_decorate8([
|
|
2259
2318
|
logInfo2
|
|
2260
2319
|
], SpaceProtocolSession.prototype, "_wireParams", void 0);
|
|
2261
|
-
|
|
2320
|
+
_ts_decorate8([
|
|
2262
2321
|
logInfo2
|
|
2263
2322
|
], SpaceProtocolSession.prototype, "authStatus", null);
|
|
2264
2323
|
|
|
@@ -2267,9 +2326,9 @@ import { synchronized as synchronized5, trackLeaks as trackLeaks4 } from "@dxos/
|
|
|
2267
2326
|
import { failUndefined as failUndefined2 } from "@dxos/debug";
|
|
2268
2327
|
import { PublicKey as PublicKey6 } from "@dxos/keys";
|
|
2269
2328
|
import { log as log12 } from "@dxos/log";
|
|
2270
|
-
import { trace as
|
|
2329
|
+
import { trace as trace4 } from "@dxos/protocols";
|
|
2271
2330
|
import { ComplexMap as ComplexMap5 } from "@dxos/util";
|
|
2272
|
-
function
|
|
2331
|
+
function _ts_decorate9(decorators, target, key, desc) {
|
|
2273
2332
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2274
2333
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2275
2334
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2279,7 +2338,7 @@ function _ts_decorate8(decorators, target, key, desc) {
|
|
|
2279
2338
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2280
2339
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2281
2340
|
}
|
|
2282
|
-
var __dxlog_file13 = "/
|
|
2341
|
+
var __dxlog_file13 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
|
|
2283
2342
|
var SpaceManager = class SpaceManager2 {
|
|
2284
2343
|
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
2285
2344
|
this._spaces = new ComplexMap5(PublicKey6.hash);
|
|
@@ -2303,7 +2362,7 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2303
2362
|
].map((space) => space.close()));
|
|
2304
2363
|
}
|
|
2305
2364
|
async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
|
|
2306
|
-
log12.trace("dxos.echo.space-manager.construct-space",
|
|
2365
|
+
log12.trace("dxos.echo.space-manager.construct-space", trace4.begin({
|
|
2307
2366
|
id: this._instanceId
|
|
2308
2367
|
}), {
|
|
2309
2368
|
F: __dxlog_file13,
|
|
@@ -2341,24 +2400,24 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2341
2400
|
memberKey
|
|
2342
2401
|
});
|
|
2343
2402
|
this._spaces.set(space.key, space);
|
|
2344
|
-
log12.trace("dxos.echo.space-manager.construct-space",
|
|
2403
|
+
log12.trace("dxos.echo.space-manager.construct-space", trace4.end({
|
|
2345
2404
|
id: this._instanceId
|
|
2346
2405
|
}), {
|
|
2347
2406
|
F: __dxlog_file13,
|
|
2348
|
-
L:
|
|
2407
|
+
L: 126,
|
|
2349
2408
|
S: this,
|
|
2350
2409
|
C: (f, a) => f(...a)
|
|
2351
2410
|
});
|
|
2352
2411
|
return space;
|
|
2353
2412
|
}
|
|
2354
2413
|
};
|
|
2355
|
-
|
|
2414
|
+
_ts_decorate9([
|
|
2356
2415
|
synchronized5
|
|
2357
2416
|
], SpaceManager.prototype, "open", null);
|
|
2358
|
-
|
|
2417
|
+
_ts_decorate9([
|
|
2359
2418
|
synchronized5
|
|
2360
2419
|
], SpaceManager.prototype, "close", null);
|
|
2361
|
-
SpaceManager =
|
|
2420
|
+
SpaceManager = _ts_decorate9([
|
|
2362
2421
|
trackLeaks4("open", "close")
|
|
2363
2422
|
], SpaceManager);
|
|
2364
2423
|
|
|
@@ -2388,4 +2447,4 @@ export {
|
|
|
2388
2447
|
SpaceProtocolSession,
|
|
2389
2448
|
SpaceManager
|
|
2390
2449
|
};
|
|
2391
|
-
//# sourceMappingURL=chunk-
|
|
2450
|
+
//# sourceMappingURL=chunk-PV6A4BSQ.mjs.map
|