@dxos/echo-pipeline 0.1.51-next.03aa1a8 → 0.1.51-next.5787f10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-KTQITKUX.mjs → chunk-AXGWKSM3.mjs} +147 -164
- package/dist/lib/browser/chunk-AXGWKSM3.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +9 -2
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +138 -155
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +191 -201
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/dbhost/snapshot-manager.d.ts +5 -6
- package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +7 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +7 -2
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +3 -0
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/package.json +30 -30
- package/src/dbhost/snapshot-manager.ts +27 -42
- package/src/pipeline/pipeline.ts +14 -11
- package/src/space/data-pipeline.ts +0 -1
- package/src/space/space-manager.ts +19 -2
- package/src/space/space-protocol.test.ts +4 -0
- package/src/space/space-protocol.ts +20 -2
- package/src/testing/test-agent-builder.ts +8 -0
- package/dist/lib/browser/chunk-KTQITKUX.mjs.map +0 -7
|
@@ -463,6 +463,7 @@ var Pipeline = class {
|
|
|
463
463
|
if (this._feedSetIterator) {
|
|
464
464
|
await this._feedSetIterator.addFeed(feed);
|
|
465
465
|
}
|
|
466
|
+
this._setFeedDownloadState(feed);
|
|
466
467
|
}
|
|
467
468
|
hasFeed(feedKey) {
|
|
468
469
|
return this._feeds.has(feedKey);
|
|
@@ -478,7 +479,7 @@ var Pipeline = class {
|
|
|
478
479
|
async start() {
|
|
479
480
|
log4("starting...", {}, {
|
|
480
481
|
file: "pipeline.ts",
|
|
481
|
-
line:
|
|
482
|
+
line: 264,
|
|
482
483
|
scope: this,
|
|
483
484
|
callSite: (f, a) => f(...a)
|
|
484
485
|
});
|
|
@@ -486,7 +487,7 @@ var Pipeline = class {
|
|
|
486
487
|
await this._feedSetIterator.open();
|
|
487
488
|
log4("started", {}, {
|
|
488
489
|
file: "pipeline.ts",
|
|
489
|
-
line:
|
|
490
|
+
line: 267,
|
|
490
491
|
scope: this,
|
|
491
492
|
callSite: (f, a) => f(...a)
|
|
492
493
|
});
|
|
@@ -496,7 +497,7 @@ var Pipeline = class {
|
|
|
496
497
|
var _a;
|
|
497
498
|
log4("stopping...", {}, {
|
|
498
499
|
file: "pipeline.ts",
|
|
499
|
-
line:
|
|
500
|
+
line: 273,
|
|
500
501
|
scope: this,
|
|
501
502
|
callSite: (f, a) => f(...a)
|
|
502
503
|
});
|
|
@@ -505,7 +506,7 @@ var Pipeline = class {
|
|
|
505
506
|
await this._processingTrigger.wait();
|
|
506
507
|
log4("stopped", {}, {
|
|
507
508
|
file: "pipeline.ts",
|
|
508
|
-
line:
|
|
509
|
+
line: 277,
|
|
509
510
|
scope: this,
|
|
510
511
|
callSite: (f, a) => f(...a)
|
|
511
512
|
});
|
|
@@ -518,32 +519,8 @@ var Pipeline = class {
|
|
|
518
519
|
assert4(!this._isStarted || this._isPaused, "Invalid state.");
|
|
519
520
|
this._state._startTimeframe = timeframe;
|
|
520
521
|
this._timeframeClock.setTimeframe(timeframe);
|
|
521
|
-
for (const
|
|
522
|
-
|
|
523
|
-
if (!feed) {
|
|
524
|
-
throw new Error("Feed not found");
|
|
525
|
-
}
|
|
526
|
-
feed.undownload({
|
|
527
|
-
callback: () => log4("Undownloaded", {}, {
|
|
528
|
-
file: "pipeline.ts",
|
|
529
|
-
line: 297,
|
|
530
|
-
scope: this,
|
|
531
|
-
callSite: (f, a) => f(...a)
|
|
532
|
-
})
|
|
533
|
-
});
|
|
534
|
-
feed.download({
|
|
535
|
-
start: seq + 1,
|
|
536
|
-
linear: true
|
|
537
|
-
}).catch((err) => {
|
|
538
|
-
log4("failed to download feed", {
|
|
539
|
-
err
|
|
540
|
-
}, {
|
|
541
|
-
file: "pipeline.ts",
|
|
542
|
-
line: 300,
|
|
543
|
-
scope: this,
|
|
544
|
-
callSite: (f, a) => f(...a)
|
|
545
|
-
});
|
|
546
|
-
});
|
|
522
|
+
for (const feed of this._feeds.values()) {
|
|
523
|
+
this._setFeedDownloadState(feed);
|
|
547
524
|
}
|
|
548
525
|
if (this._feedSetIterator) {
|
|
549
526
|
await this._feedSetIterator.close();
|
|
@@ -599,6 +576,32 @@ var Pipeline = class {
|
|
|
599
576
|
}
|
|
600
577
|
this._isOpen = false;
|
|
601
578
|
}
|
|
579
|
+
_setFeedDownloadState(feed) {
|
|
580
|
+
var _a;
|
|
581
|
+
const timeframe = this._state._startTimeframe;
|
|
582
|
+
const seq = (_a = timeframe.get(feed.key)) != null ? _a : 0;
|
|
583
|
+
feed.undownload({
|
|
584
|
+
callback: () => log4("Undownloaded", {}, {
|
|
585
|
+
file: "pipeline.ts",
|
|
586
|
+
line: 371,
|
|
587
|
+
scope: this,
|
|
588
|
+
callSite: (f, a) => f(...a)
|
|
589
|
+
})
|
|
590
|
+
});
|
|
591
|
+
feed.download({
|
|
592
|
+
start: seq + 1,
|
|
593
|
+
linear: true
|
|
594
|
+
}).catch((err) => {
|
|
595
|
+
log4("failed to download feed", {
|
|
596
|
+
err
|
|
597
|
+
}, {
|
|
598
|
+
file: "pipeline.ts",
|
|
599
|
+
line: 374,
|
|
600
|
+
scope: this,
|
|
601
|
+
callSite: (f, a) => f(...a)
|
|
602
|
+
});
|
|
603
|
+
});
|
|
604
|
+
}
|
|
602
605
|
async _initIterator() {
|
|
603
606
|
this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
604
607
|
start: startAfter(this._timeframeClock.timeframe),
|
|
@@ -607,7 +610,7 @@ var Pipeline = class {
|
|
|
607
610
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
608
611
|
log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
|
|
609
612
|
file: "pipeline.ts",
|
|
610
|
-
line:
|
|
613
|
+
line: 384,
|
|
611
614
|
scope: this,
|
|
612
615
|
callSite: (f, a) => f(...a)
|
|
613
616
|
});
|
|
@@ -869,9 +872,9 @@ var DatabaseHost = class {
|
|
|
869
872
|
import { trackLeaks } from "@dxos/async";
|
|
870
873
|
import { cancelWithContext } from "@dxos/context";
|
|
871
874
|
import { timed as timed2 } from "@dxos/debug";
|
|
872
|
-
import {
|
|
875
|
+
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
873
876
|
import { schema as schema4 } from "@dxos/protocols";
|
|
874
|
-
import {
|
|
877
|
+
import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
875
878
|
var __decorate4 = function(decorators, target, key, desc) {
|
|
876
879
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
877
880
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -884,62 +887,32 @@ var __decorate4 = function(decorators, target, key, desc) {
|
|
|
884
887
|
};
|
|
885
888
|
var SnapshotManager = class SnapshotManager2 {
|
|
886
889
|
// prettier-ignore
|
|
887
|
-
constructor(_snapshotStore) {
|
|
890
|
+
constructor(_snapshotStore, _blobStore, _blobSync) {
|
|
888
891
|
this._snapshotStore = _snapshotStore;
|
|
889
|
-
this.
|
|
890
|
-
|
|
891
|
-
const snapshot = await this._snapshotStore.loadSnapshot(id);
|
|
892
|
-
log7("getObject", {
|
|
893
|
-
id,
|
|
894
|
-
snapshot
|
|
895
|
-
}, {
|
|
896
|
-
file: "snapshot-manager.ts",
|
|
897
|
-
line: 31,
|
|
898
|
-
scope: this,
|
|
899
|
-
callSite: (f, a) => f(...a)
|
|
900
|
-
});
|
|
901
|
-
if (!snapshot) {
|
|
902
|
-
return void 0;
|
|
903
|
-
}
|
|
904
|
-
return {
|
|
905
|
-
id,
|
|
906
|
-
payload: schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encodeAsAny(snapshot)
|
|
907
|
-
};
|
|
908
|
-
},
|
|
909
|
-
setObject: async (data) => {
|
|
910
|
-
log7("setObject", {
|
|
911
|
-
data
|
|
912
|
-
}, {
|
|
913
|
-
file: "snapshot-manager.ts",
|
|
914
|
-
line: 41,
|
|
915
|
-
scope: this,
|
|
916
|
-
callSite: (f, a) => f(...a)
|
|
917
|
-
});
|
|
918
|
-
const snapshot = schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(data.payload.value);
|
|
919
|
-
await this._snapshotStore.saveSnapshot(snapshot);
|
|
920
|
-
}
|
|
921
|
-
});
|
|
922
|
-
}
|
|
923
|
-
get objectSync() {
|
|
924
|
-
return this._objectSync;
|
|
925
|
-
}
|
|
926
|
-
async open() {
|
|
927
|
-
await this._objectSync.open();
|
|
892
|
+
this._blobStore = _blobStore;
|
|
893
|
+
this._blobSync = _blobSync;
|
|
928
894
|
}
|
|
929
|
-
async
|
|
930
|
-
await this.
|
|
895
|
+
async _getBlob(blobId) {
|
|
896
|
+
const blob = await this._blobStore.get(blobId);
|
|
897
|
+
return schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(blob);
|
|
931
898
|
}
|
|
932
899
|
async load(ctx, id) {
|
|
933
|
-
const
|
|
934
|
-
|
|
935
|
-
|
|
900
|
+
const blobId = PublicKey2.fromHex(id).asUint8Array();
|
|
901
|
+
const blobMeta = await this._blobStore.getMeta(blobId);
|
|
902
|
+
if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
|
|
903
|
+
return this._getBlob(blobId);
|
|
904
|
+
}
|
|
905
|
+
const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
|
|
906
|
+
if (fallbackStore) {
|
|
907
|
+
return fallbackStore;
|
|
936
908
|
}
|
|
937
|
-
|
|
938
|
-
return
|
|
909
|
+
await this._blobSync.download(ctx, blobId);
|
|
910
|
+
return this._getBlob(blobId);
|
|
939
911
|
}
|
|
940
912
|
async store(snapshot) {
|
|
941
|
-
const id = await this.
|
|
942
|
-
|
|
913
|
+
const { id } = await this._blobStore.set(schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encode(snapshot));
|
|
914
|
+
await this._blobSync.notifyBlobAdded(id);
|
|
915
|
+
return PublicKey2.from(id).toHex();
|
|
943
916
|
}
|
|
944
917
|
};
|
|
945
918
|
__decorate4([
|
|
@@ -997,18 +970,18 @@ var SnapshotStore = class {
|
|
|
997
970
|
// packages/core/echo/echo-pipeline/src/dbhost/data-service.ts
|
|
998
971
|
import assert7 from "@dxos/node-std/assert";
|
|
999
972
|
import { raise } from "@dxos/debug";
|
|
1000
|
-
import { PublicKey as
|
|
1001
|
-
import { log as
|
|
973
|
+
import { PublicKey as PublicKey3 } from "@dxos/keys";
|
|
974
|
+
import { log as log7 } from "@dxos/log";
|
|
1002
975
|
import { ComplexMap as ComplexMap3 } from "@dxos/util";
|
|
1003
976
|
var DataServiceSubscriptions = class {
|
|
1004
977
|
constructor() {
|
|
1005
|
-
this._spaces = new ComplexMap3(
|
|
978
|
+
this._spaces = new ComplexMap3(PublicKey3.hash);
|
|
1006
979
|
}
|
|
1007
980
|
clear() {
|
|
1008
981
|
this._spaces.clear();
|
|
1009
982
|
}
|
|
1010
983
|
registerSpace(spaceKey, host) {
|
|
1011
|
-
|
|
984
|
+
log7("Registering space", {
|
|
1012
985
|
spaceKey
|
|
1013
986
|
}, {
|
|
1014
987
|
file: "data-service.ts",
|
|
@@ -1019,7 +992,7 @@ var DataServiceSubscriptions = class {
|
|
|
1019
992
|
this._spaces.set(spaceKey, host);
|
|
1020
993
|
}
|
|
1021
994
|
unregisterSpace(spaceKey) {
|
|
1022
|
-
|
|
995
|
+
log7("Unregistering space", {
|
|
1023
996
|
spaceKey
|
|
1024
997
|
}, {
|
|
1025
998
|
file: "data-service.ts",
|
|
@@ -1059,7 +1032,7 @@ import { Context as Context3 } from "@dxos/context";
|
|
|
1059
1032
|
import { checkCredentialType } from "@dxos/credentials";
|
|
1060
1033
|
import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
|
|
1061
1034
|
import { CancelledError } from "@dxos/errors";
|
|
1062
|
-
import { log as
|
|
1035
|
+
import { log as log8 } from "@dxos/log";
|
|
1063
1036
|
import { Timeframe as Timeframe3 } from "@dxos/timeframe";
|
|
1064
1037
|
var __decorate5 = function(decorators, target, key, desc) {
|
|
1065
1038
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
@@ -1146,7 +1119,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1146
1119
|
if (!this._isOpen) {
|
|
1147
1120
|
return;
|
|
1148
1121
|
}
|
|
1149
|
-
|
|
1122
|
+
log8("close", {}, {
|
|
1150
1123
|
file: "data-pipeline.ts",
|
|
1151
1124
|
line: 174,
|
|
1152
1125
|
scope: this,
|
|
@@ -1161,7 +1134,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1161
1134
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1162
1135
|
}
|
|
1163
1136
|
} catch (err) {
|
|
1164
|
-
|
|
1137
|
+
log8.catch(err, {}, {
|
|
1165
1138
|
file: "data-pipeline.ts",
|
|
1166
1139
|
line: 187,
|
|
1167
1140
|
scope: this,
|
|
@@ -1170,7 +1143,6 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1170
1143
|
}
|
|
1171
1144
|
await ((_b = this.databaseHost) == null ? void 0 : _b.close());
|
|
1172
1145
|
await ((_c = this.itemManager) == null ? void 0 : _c.destroy());
|
|
1173
|
-
await this._params.snapshotManager.close();
|
|
1174
1146
|
}
|
|
1175
1147
|
async _consumePipeline() {
|
|
1176
1148
|
if (this.currentEpoch) {
|
|
@@ -1181,12 +1153,12 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1181
1153
|
assert8(this._pipeline, "Pipeline is not initialized.");
|
|
1182
1154
|
for await (const msg of this._pipeline.consume()) {
|
|
1183
1155
|
const { feedKey, seq, data } = msg;
|
|
1184
|
-
|
|
1156
|
+
log8("processing message", {
|
|
1185
1157
|
feedKey,
|
|
1186
1158
|
seq
|
|
1187
1159
|
}, {
|
|
1188
1160
|
file: "data-pipeline.ts",
|
|
1189
|
-
line:
|
|
1161
|
+
line: 204,
|
|
1190
1162
|
scope: this,
|
|
1191
1163
|
callSite: (f, a) => f(...a)
|
|
1192
1164
|
});
|
|
@@ -1194,11 +1166,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1194
1166
|
if (data.payload.data) {
|
|
1195
1167
|
const feedInfo = this._params.feedInfoProvider(feedKey);
|
|
1196
1168
|
if (!feedInfo) {
|
|
1197
|
-
|
|
1169
|
+
log8.error("Could not find feed.", {
|
|
1198
1170
|
feedKey
|
|
1199
1171
|
}, {
|
|
1200
1172
|
file: "data-pipeline.ts",
|
|
1201
|
-
line:
|
|
1173
|
+
line: 210,
|
|
1202
1174
|
scope: this,
|
|
1203
1175
|
callSite: (f, a) => f(...a)
|
|
1204
1176
|
});
|
|
@@ -1213,22 +1185,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1213
1185
|
memberKey: feedInfo.assertion.identityKey
|
|
1214
1186
|
}
|
|
1215
1187
|
});
|
|
1216
|
-
|
|
1188
|
+
log8.trace("dxos.echo.data-pipeline.processed", {
|
|
1217
1189
|
feedKey: feedKey.toHex(),
|
|
1218
1190
|
seq,
|
|
1219
1191
|
spaceKey: this._params.spaceKey.toHex()
|
|
1220
1192
|
}, {
|
|
1221
1193
|
file: "data-pipeline.ts",
|
|
1222
|
-
line:
|
|
1194
|
+
line: 224,
|
|
1223
1195
|
scope: this,
|
|
1224
1196
|
callSite: (f, a) => f(...a)
|
|
1225
1197
|
});
|
|
1226
1198
|
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1227
1199
|
}
|
|
1228
1200
|
} catch (err) {
|
|
1229
|
-
|
|
1201
|
+
log8.catch(err, {}, {
|
|
1230
1202
|
file: "data-pipeline.ts",
|
|
1231
|
-
line:
|
|
1203
|
+
line: 234,
|
|
1232
1204
|
scope: this,
|
|
1233
1205
|
callSite: (f, a) => f(...a)
|
|
1234
1206
|
});
|
|
@@ -1261,9 +1233,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1261
1233
|
cache.properties = (_a = getStateMachineFromItem(propertiesItem)) == null ? void 0 : _a.snapshot();
|
|
1262
1234
|
}
|
|
1263
1235
|
} catch (err) {
|
|
1264
|
-
|
|
1236
|
+
log8.warn("Failed to cache properties", err, {
|
|
1265
1237
|
file: "data-pipeline.ts",
|
|
1266
|
-
line:
|
|
1238
|
+
line: 268,
|
|
1267
1239
|
scope: this,
|
|
1268
1240
|
callSite: (f, a) => f(...a)
|
|
1269
1241
|
});
|
|
@@ -1288,16 +1260,16 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1288
1260
|
const ctx = new Context3({
|
|
1289
1261
|
onError: (err) => {
|
|
1290
1262
|
if (err instanceof CancelledError) {
|
|
1291
|
-
|
|
1263
|
+
log8("Epoch processing cancelled.", {}, {
|
|
1292
1264
|
file: "data-pipeline.ts",
|
|
1293
|
-
line:
|
|
1265
|
+
line: 300,
|
|
1294
1266
|
scope: this,
|
|
1295
1267
|
callSite: (f, a) => f(...a)
|
|
1296
1268
|
});
|
|
1297
1269
|
} else {
|
|
1298
|
-
|
|
1270
|
+
log8.catch(err, {}, {
|
|
1299
1271
|
file: "data-pipeline.ts",
|
|
1300
|
-
line:
|
|
1272
|
+
line: 302,
|
|
1301
1273
|
scope: this,
|
|
1302
1274
|
callSite: (f, a) => f(...a)
|
|
1303
1275
|
});
|
|
@@ -1309,11 +1281,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1309
1281
|
if (!this._isOpen) {
|
|
1310
1282
|
return;
|
|
1311
1283
|
}
|
|
1312
|
-
|
|
1284
|
+
log8("process epoch", {
|
|
1313
1285
|
epoch
|
|
1314
1286
|
}, {
|
|
1315
1287
|
file: "data-pipeline.ts",
|
|
1316
|
-
line:
|
|
1288
|
+
line: 312,
|
|
1317
1289
|
scope: this,
|
|
1318
1290
|
callSite: (f, a) => f(...a)
|
|
1319
1291
|
});
|
|
@@ -1326,11 +1298,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1326
1298
|
assert8(this._isOpen, "Space is closed.");
|
|
1327
1299
|
assert8(this._pipeline);
|
|
1328
1300
|
this._lastProcessedEpoch = epoch.number;
|
|
1329
|
-
|
|
1301
|
+
log8("Processing epoch", {
|
|
1330
1302
|
epoch
|
|
1331
1303
|
}, {
|
|
1332
1304
|
file: "data-pipeline.ts",
|
|
1333
|
-
line:
|
|
1305
|
+
line: 326,
|
|
1334
1306
|
scope: this,
|
|
1335
1307
|
callSite: (f, a) => f(...a)
|
|
1336
1308
|
});
|
|
@@ -1338,9 +1310,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1338
1310
|
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1339
1311
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1340
1312
|
}
|
|
1341
|
-
|
|
1313
|
+
log8("restarting pipeline for epoch", {}, {
|
|
1342
1314
|
file: "data-pipeline.ts",
|
|
1343
|
-
line:
|
|
1315
|
+
line: 334,
|
|
1344
1316
|
scope: this,
|
|
1345
1317
|
callSite: (f, a) => f(...a)
|
|
1346
1318
|
});
|
|
@@ -1390,14 +1362,14 @@ DataPipeline = __decorate5([
|
|
|
1390
1362
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1391
1363
|
import assert9 from "@dxos/node-std/assert";
|
|
1392
1364
|
import { Event as Event5, synchronized as synchronized4, trackLeaks as trackLeaks3, Lock } from "@dxos/async";
|
|
1393
|
-
import { log as
|
|
1365
|
+
import { log as log10, logInfo } from "@dxos/log";
|
|
1394
1366
|
import { AdmittedFeed as AdmittedFeed2 } from "@dxos/protocols/proto/dxos/halo/credentials";
|
|
1395
1367
|
import { Callback as Callback2 } from "@dxos/util";
|
|
1396
1368
|
|
|
1397
1369
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1398
1370
|
import { SpaceStateMachine } from "@dxos/credentials";
|
|
1399
|
-
import { PublicKey as
|
|
1400
|
-
import { log as
|
|
1371
|
+
import { PublicKey as PublicKey4 } from "@dxos/keys";
|
|
1372
|
+
import { log as log9 } from "@dxos/log";
|
|
1401
1373
|
import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
|
|
1402
1374
|
import { Timeframe as Timeframe4 } from "@dxos/timeframe";
|
|
1403
1375
|
import { Callback } from "@dxos/util";
|
|
@@ -1412,7 +1384,7 @@ var ControlPipeline = class {
|
|
|
1412
1384
|
void this._pipeline.addFeed(genesisFeed);
|
|
1413
1385
|
this._spaceStateMachine = new SpaceStateMachine(spaceKey);
|
|
1414
1386
|
this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
|
|
1415
|
-
|
|
1387
|
+
log9("feed admitted", {
|
|
1416
1388
|
key: info.key
|
|
1417
1389
|
}, {
|
|
1418
1390
|
file: "control-pipeline.ts",
|
|
@@ -1425,7 +1397,7 @@ var ControlPipeline = class {
|
|
|
1425
1397
|
const feed = await feedProvider(info.key);
|
|
1426
1398
|
await this._pipeline.addFeed(feed);
|
|
1427
1399
|
} catch (err) {
|
|
1428
|
-
|
|
1400
|
+
log9.catch(err, {}, {
|
|
1429
1401
|
file: "control-pipeline.ts",
|
|
1430
1402
|
line: 59,
|
|
1431
1403
|
scope: this,
|
|
@@ -1448,7 +1420,7 @@ var ControlPipeline = class {
|
|
|
1448
1420
|
this._pipeline.setWriteFeed(feed);
|
|
1449
1421
|
}
|
|
1450
1422
|
async start() {
|
|
1451
|
-
|
|
1423
|
+
log9("starting...", {}, {
|
|
1452
1424
|
file: "control-pipeline.ts",
|
|
1453
1425
|
line: 83,
|
|
1454
1426
|
scope: this,
|
|
@@ -1457,7 +1429,7 @@ var ControlPipeline = class {
|
|
|
1457
1429
|
setTimeout(async () => {
|
|
1458
1430
|
for await (const msg of this._pipeline.consume()) {
|
|
1459
1431
|
try {
|
|
1460
|
-
|
|
1432
|
+
log9("processing", {
|
|
1461
1433
|
key: msg.feedKey,
|
|
1462
1434
|
seq: msg.seq
|
|
1463
1435
|
}, {
|
|
@@ -1467,9 +1439,9 @@ var ControlPipeline = class {
|
|
|
1467
1439
|
callSite: (f, a) => f(...a)
|
|
1468
1440
|
});
|
|
1469
1441
|
if (msg.data.payload.credential) {
|
|
1470
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential,
|
|
1442
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
|
|
1471
1443
|
if (!result) {
|
|
1472
|
-
|
|
1444
|
+
log9.warn("processing failed", {
|
|
1473
1445
|
msg
|
|
1474
1446
|
}, {
|
|
1475
1447
|
file: "control-pipeline.ts",
|
|
@@ -1482,7 +1454,7 @@ var ControlPipeline = class {
|
|
|
1482
1454
|
}
|
|
1483
1455
|
}
|
|
1484
1456
|
} catch (err) {
|
|
1485
|
-
|
|
1457
|
+
log9.catch(err, {}, {
|
|
1486
1458
|
file: "control-pipeline.ts",
|
|
1487
1459
|
line: 101,
|
|
1488
1460
|
scope: this,
|
|
@@ -1492,7 +1464,7 @@ var ControlPipeline = class {
|
|
|
1492
1464
|
}
|
|
1493
1465
|
});
|
|
1494
1466
|
await this._pipeline.start();
|
|
1495
|
-
|
|
1467
|
+
log9("started", {}, {
|
|
1496
1468
|
file: "control-pipeline.ts",
|
|
1497
1469
|
line: 107,
|
|
1498
1470
|
scope: this,
|
|
@@ -1506,7 +1478,7 @@ var ControlPipeline = class {
|
|
|
1506
1478
|
}
|
|
1507
1479
|
}
|
|
1508
1480
|
async stop() {
|
|
1509
|
-
|
|
1481
|
+
log9("stopping...", {}, {
|
|
1510
1482
|
file: "control-pipeline.ts",
|
|
1511
1483
|
line: 121,
|
|
1512
1484
|
scope: this,
|
|
@@ -1514,7 +1486,7 @@ var ControlPipeline = class {
|
|
|
1514
1486
|
});
|
|
1515
1487
|
await this._pipeline.stop();
|
|
1516
1488
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1517
|
-
|
|
1489
|
+
log9("stopped", {}, {
|
|
1518
1490
|
file: "control-pipeline.ts",
|
|
1519
1491
|
line: 124,
|
|
1520
1492
|
scope: this,
|
|
@@ -1528,7 +1500,7 @@ var ControlPipeline = class {
|
|
|
1528
1500
|
await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
|
|
1529
1501
|
this._targetTimeframe = newTimeframe;
|
|
1530
1502
|
} catch (err) {
|
|
1531
|
-
|
|
1503
|
+
log9(err, {}, {
|
|
1532
1504
|
file: "control-pipeline.ts",
|
|
1533
1505
|
line: 133,
|
|
1534
1506
|
scope: this,
|
|
@@ -1589,7 +1561,7 @@ var Space = class Space2 {
|
|
|
1589
1561
|
});
|
|
1590
1562
|
this._controlPipeline.onCredentialProcessed.set(async (credential) => {
|
|
1591
1563
|
await this.onCredentialProcessed.callIfSet(credential);
|
|
1592
|
-
|
|
1564
|
+
log10("onCredentialProcessed", {
|
|
1593
1565
|
credential
|
|
1594
1566
|
}, {
|
|
1595
1567
|
file: "space.ts",
|
|
@@ -1683,7 +1655,7 @@ var Space = class Space2 {
|
|
|
1683
1655
|
// return this._dataPipeline?.getFeeds();
|
|
1684
1656
|
// }
|
|
1685
1657
|
async open() {
|
|
1686
|
-
|
|
1658
|
+
log10("opening...", {}, {
|
|
1687
1659
|
file: "space.ts",
|
|
1688
1660
|
line: 212,
|
|
1689
1661
|
scope: this,
|
|
@@ -1697,7 +1669,7 @@ var Space = class Space2 {
|
|
|
1697
1669
|
this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
|
|
1698
1670
|
await this._dataPipelineCredentialConsumer.open();
|
|
1699
1671
|
this._isOpen = true;
|
|
1700
|
-
|
|
1672
|
+
log10("opened", {}, {
|
|
1701
1673
|
file: "space.ts",
|
|
1702
1674
|
line: 228,
|
|
1703
1675
|
scope: this,
|
|
@@ -1706,7 +1678,7 @@ var Space = class Space2 {
|
|
|
1706
1678
|
}
|
|
1707
1679
|
async close() {
|
|
1708
1680
|
var _a;
|
|
1709
|
-
|
|
1681
|
+
log10("closing...", {
|
|
1710
1682
|
key: this._key
|
|
1711
1683
|
}, {
|
|
1712
1684
|
file: "space.ts",
|
|
@@ -1723,7 +1695,7 @@ var Space = class Space2 {
|
|
|
1723
1695
|
await this.protocol.stop();
|
|
1724
1696
|
await this._controlPipeline.stop();
|
|
1725
1697
|
this._isOpen = false;
|
|
1726
|
-
|
|
1698
|
+
log10("closed", {}, {
|
|
1727
1699
|
file: "space.ts",
|
|
1728
1700
|
line: 247,
|
|
1729
1701
|
scope: this,
|
|
@@ -1731,7 +1703,7 @@ var Space = class Space2 {
|
|
|
1731
1703
|
});
|
|
1732
1704
|
}
|
|
1733
1705
|
async initializeDataPipeline() {
|
|
1734
|
-
|
|
1706
|
+
log10("initializeDataPipeline", {}, {
|
|
1735
1707
|
file: "space.ts",
|
|
1736
1708
|
line: 252,
|
|
1737
1709
|
scope: this,
|
|
@@ -1759,10 +1731,11 @@ Space = __decorate6([
|
|
|
1759
1731
|
|
|
1760
1732
|
// packages/core/echo/echo-pipeline/src/space/space-protocol.ts
|
|
1761
1733
|
import { discoveryKey, sha256 } from "@dxos/crypto";
|
|
1762
|
-
import { PublicKey as
|
|
1763
|
-
import { log as
|
|
1734
|
+
import { PublicKey as PublicKey5 } from "@dxos/keys";
|
|
1735
|
+
import { log as log11, logInfo as logInfo2 } from "@dxos/log";
|
|
1764
1736
|
import { MMSTTopology } from "@dxos/network-manager";
|
|
1765
1737
|
import { Teleport } from "@dxos/teleport";
|
|
1738
|
+
import { BlobSync } from "@dxos/teleport-extension-object-sync";
|
|
1766
1739
|
import { ReplicatorExtension } from "@dxos/teleport-extension-replicator";
|
|
1767
1740
|
import { ComplexMap as ComplexMap4 } from "@dxos/util";
|
|
1768
1741
|
var __decorate7 = function(decorators, target, key, desc) {
|
|
@@ -1778,14 +1751,17 @@ var __decorate7 = function(decorators, target, key, desc) {
|
|
|
1778
1751
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
1779
1752
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
1780
1753
|
var SpaceProtocol = class {
|
|
1781
|
-
constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure }) {
|
|
1754
|
+
constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
|
|
1782
1755
|
this._feeds = /* @__PURE__ */ new Set();
|
|
1783
|
-
this._sessions = new ComplexMap4(
|
|
1756
|
+
this._sessions = new ComplexMap4(PublicKey5.hash);
|
|
1784
1757
|
this._networkManager = networkManager;
|
|
1785
1758
|
this._swarmIdentity = swarmIdentity;
|
|
1786
1759
|
this._onSessionAuth = onSessionAuth;
|
|
1787
1760
|
this._onAuthFailure = onAuthFailure;
|
|
1788
|
-
this.
|
|
1761
|
+
this.blobSync = new BlobSync({
|
|
1762
|
+
blobStore
|
|
1763
|
+
});
|
|
1764
|
+
this._topic = PublicKey5.from(discoveryKey(sha256(topic.toHex())));
|
|
1789
1765
|
}
|
|
1790
1766
|
get sessions() {
|
|
1791
1767
|
return this._sessions;
|
|
@@ -1798,11 +1774,11 @@ var SpaceProtocol = class {
|
|
|
1798
1774
|
}
|
|
1799
1775
|
// TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
|
|
1800
1776
|
addFeed(feed) {
|
|
1801
|
-
|
|
1777
|
+
log11("addFeed", {
|
|
1802
1778
|
key: feed.key
|
|
1803
1779
|
}, {
|
|
1804
1780
|
file: "space-protocol.ts",
|
|
1805
|
-
line:
|
|
1781
|
+
line: 94,
|
|
1806
1782
|
scope: this,
|
|
1807
1783
|
callSite: (f, a) => f(...a)
|
|
1808
1784
|
});
|
|
@@ -1821,9 +1797,10 @@ var SpaceProtocol = class {
|
|
|
1821
1797
|
maxPeers: 10,
|
|
1822
1798
|
sampleSize: 20
|
|
1823
1799
|
};
|
|
1824
|
-
|
|
1800
|
+
await this.blobSync.open();
|
|
1801
|
+
log11("starting...", {}, {
|
|
1825
1802
|
file: "space-protocol.ts",
|
|
1826
|
-
line:
|
|
1803
|
+
line: 119,
|
|
1827
1804
|
scope: this,
|
|
1828
1805
|
callSite: (f, a) => f(...a)
|
|
1829
1806
|
});
|
|
@@ -1834,25 +1811,26 @@ var SpaceProtocol = class {
|
|
|
1834
1811
|
topology: new MMSTTopology(topologyConfig),
|
|
1835
1812
|
label: `Protocol swarm: ${this._topic}`
|
|
1836
1813
|
});
|
|
1837
|
-
|
|
1814
|
+
log11("started", {}, {
|
|
1838
1815
|
file: "space-protocol.ts",
|
|
1839
|
-
line:
|
|
1816
|
+
line: 128,
|
|
1840
1817
|
scope: this,
|
|
1841
1818
|
callSite: (f, a) => f(...a)
|
|
1842
1819
|
});
|
|
1843
1820
|
}
|
|
1844
1821
|
async stop() {
|
|
1822
|
+
await this.blobSync.close();
|
|
1845
1823
|
if (this._connection) {
|
|
1846
|
-
|
|
1824
|
+
log11("stopping...", {}, {
|
|
1847
1825
|
file: "space-protocol.ts",
|
|
1848
|
-
line:
|
|
1826
|
+
line: 135,
|
|
1849
1827
|
scope: this,
|
|
1850
1828
|
callSite: (f, a) => f(...a)
|
|
1851
1829
|
});
|
|
1852
1830
|
await this._connection.close();
|
|
1853
|
-
|
|
1831
|
+
log11("stopped", {}, {
|
|
1854
1832
|
file: "space-protocol.ts",
|
|
1855
|
-
line:
|
|
1833
|
+
line: 137,
|
|
1856
1834
|
scope: this,
|
|
1857
1835
|
callSite: (f, a) => f(...a)
|
|
1858
1836
|
});
|
|
@@ -1864,7 +1842,8 @@ var SpaceProtocol = class {
|
|
|
1864
1842
|
wireParams,
|
|
1865
1843
|
swarmIdentity: this._swarmIdentity,
|
|
1866
1844
|
onSessionAuth: this._onSessionAuth,
|
|
1867
|
-
onAuthFailure: this._onAuthFailure
|
|
1845
|
+
onAuthFailure: this._onAuthFailure,
|
|
1846
|
+
blobSync: this.blobSync
|
|
1868
1847
|
});
|
|
1869
1848
|
this._sessions.set(wireParams.remotePeerId, session);
|
|
1870
1849
|
for (const feed of this._feeds) {
|
|
@@ -1888,7 +1867,7 @@ var AuthStatus;
|
|
|
1888
1867
|
})(AuthStatus || (AuthStatus = {}));
|
|
1889
1868
|
var SpaceProtocolSession = class {
|
|
1890
1869
|
// TODO(dmaretskyi): Allow to pass in extra extensions.
|
|
1891
|
-
constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure }) {
|
|
1870
|
+
constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
|
|
1892
1871
|
// TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
|
|
1893
1872
|
this.replicator = new ReplicatorExtension().setOptions({
|
|
1894
1873
|
upload: true
|
|
@@ -1898,6 +1877,7 @@ var SpaceProtocolSession = class {
|
|
|
1898
1877
|
this._swarmIdentity = swarmIdentity;
|
|
1899
1878
|
this._onSessionAuth = onSessionAuth;
|
|
1900
1879
|
this._onAuthFailure = onAuthFailure;
|
|
1880
|
+
this._blobSync = blobSync;
|
|
1901
1881
|
this._teleport = new Teleport(wireParams);
|
|
1902
1882
|
}
|
|
1903
1883
|
get authStatus() {
|
|
@@ -1913,9 +1893,9 @@ var SpaceProtocolSession = class {
|
|
|
1913
1893
|
verifier: this._swarmIdentity.credentialAuthenticator,
|
|
1914
1894
|
onAuthSuccess: () => {
|
|
1915
1895
|
var _a;
|
|
1916
|
-
|
|
1896
|
+
log11("Peer authenticated", {}, {
|
|
1917
1897
|
file: "space-protocol.ts",
|
|
1918
|
-
line:
|
|
1898
|
+
line: 230,
|
|
1919
1899
|
scope: this,
|
|
1920
1900
|
callSite: (f, a) => f(...a)
|
|
1921
1901
|
});
|
|
@@ -1929,6 +1909,7 @@ var SpaceProtocolSession = class {
|
|
|
1929
1909
|
}
|
|
1930
1910
|
}));
|
|
1931
1911
|
this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
|
|
1912
|
+
this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
|
|
1932
1913
|
}
|
|
1933
1914
|
async destroy() {
|
|
1934
1915
|
await this._teleport.close();
|
|
@@ -1944,8 +1925,8 @@ __decorate7([
|
|
|
1944
1925
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
1945
1926
|
import { synchronized as synchronized5, trackLeaks as trackLeaks4 } from "@dxos/async";
|
|
1946
1927
|
import { failUndefined as failUndefined2 } from "@dxos/debug";
|
|
1947
|
-
import { PublicKey as
|
|
1948
|
-
import { log as
|
|
1928
|
+
import { PublicKey as PublicKey6 } from "@dxos/keys";
|
|
1929
|
+
import { log as log12 } from "@dxos/log";
|
|
1949
1930
|
import { trace } from "@dxos/protocols";
|
|
1950
1931
|
import { ComplexMap as ComplexMap5 } from "@dxos/util";
|
|
1951
1932
|
var __decorate8 = function(decorators, target, key, desc) {
|
|
@@ -1959,14 +1940,15 @@ var __decorate8 = function(decorators, target, key, desc) {
|
|
|
1959
1940
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1960
1941
|
};
|
|
1961
1942
|
var SpaceManager = class SpaceManager2 {
|
|
1962
|
-
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore }) {
|
|
1963
|
-
this._spaces = new ComplexMap5(
|
|
1964
|
-
this._instanceId =
|
|
1943
|
+
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
1944
|
+
this._spaces = new ComplexMap5(PublicKey6.hash);
|
|
1945
|
+
this._instanceId = PublicKey6.random().toHex();
|
|
1965
1946
|
this._feedStore = feedStore;
|
|
1966
1947
|
this._networkManager = networkManager;
|
|
1967
1948
|
this._modelFactory = modelFactory;
|
|
1968
1949
|
this._metadataStore = metadataStore;
|
|
1969
1950
|
this._snapshotStore = snapshotStore;
|
|
1951
|
+
this._blobStore = blobStore;
|
|
1970
1952
|
}
|
|
1971
1953
|
// TODO(burdon): Remove.
|
|
1972
1954
|
get spaces() {
|
|
@@ -1981,19 +1963,19 @@ var SpaceManager = class SpaceManager2 {
|
|
|
1981
1963
|
}
|
|
1982
1964
|
async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
|
|
1983
1965
|
var _a;
|
|
1984
|
-
|
|
1966
|
+
log12.trace("dxos.echo.space-manager.construct-space", trace.begin({
|
|
1985
1967
|
id: this._instanceId
|
|
1986
1968
|
}), {
|
|
1987
1969
|
file: "space-manager.ts",
|
|
1988
|
-
line:
|
|
1970
|
+
line: 97,
|
|
1989
1971
|
scope: this,
|
|
1990
1972
|
callSite: (f, a) => f(...a)
|
|
1991
1973
|
});
|
|
1992
|
-
|
|
1974
|
+
log12("constructing space...", {
|
|
1993
1975
|
spaceKey: metadata.genesisFeedKey
|
|
1994
1976
|
}, {
|
|
1995
1977
|
file: "space-manager.ts",
|
|
1996
|
-
line:
|
|
1978
|
+
line: 98,
|
|
1997
1979
|
scope: this,
|
|
1998
1980
|
callSite: (f, a) => f(...a)
|
|
1999
1981
|
});
|
|
@@ -2004,9 +1986,10 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2004
1986
|
swarmIdentity,
|
|
2005
1987
|
networkManager: this._networkManager,
|
|
2006
1988
|
onSessionAuth: onNetworkConnection,
|
|
2007
|
-
onAuthFailure
|
|
1989
|
+
onAuthFailure,
|
|
1990
|
+
blobStore: this._blobStore
|
|
2008
1991
|
});
|
|
2009
|
-
const snapshotManager = new SnapshotManager(this._snapshotStore);
|
|
1992
|
+
const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
|
|
2010
1993
|
const space = new Space({
|
|
2011
1994
|
spaceKey,
|
|
2012
1995
|
protocol,
|
|
@@ -2018,11 +2001,11 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2018
2001
|
memberKey
|
|
2019
2002
|
});
|
|
2020
2003
|
this._spaces.set(space.key, space);
|
|
2021
|
-
|
|
2004
|
+
log12.trace("dxos.echo.space-manager.construct-space", trace.end({
|
|
2022
2005
|
id: this._instanceId
|
|
2023
2006
|
}), {
|
|
2024
2007
|
file: "space-manager.ts",
|
|
2025
|
-
line:
|
|
2008
|
+
line: 125,
|
|
2026
2009
|
scope: this,
|
|
2027
2010
|
callSite: (f, a) => f(...a)
|
|
2028
2011
|
});
|
|
@@ -2065,4 +2048,4 @@ export {
|
|
|
2065
2048
|
SpaceProtocolSession,
|
|
2066
2049
|
SpaceManager
|
|
2067
2050
|
};
|
|
2068
|
-
//# sourceMappingURL=chunk-
|
|
2051
|
+
//# sourceMappingURL=chunk-AXGWKSM3.mjs.map
|