@dxos/echo-pipeline 0.1.55-main.ff36828 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-OJGL5427.mjs → chunk-4VJRJ4SY.mjs} +877 -872
- package/dist/lib/browser/chunk-4VJRJ4SY.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +1 -1
- package/dist/lib/browser/testing/index.mjs.map +1 -1
- package/dist/lib/node/index.cjs +787 -784
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +78 -73
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -0
- package/dist/types/src/db-host/data-service.d.ts.map +1 -0
- package/dist/types/src/db-host/database-host.d.ts.map +1 -0
- package/dist/types/src/db-host/index.d.ts.map +1 -0
- package/dist/types/src/db-host/snapshot-manager.d.ts.map +1 -0
- package/dist/types/src/db-host/snapshot-store.d.ts.map +1 -0
- package/dist/types/src/db-host/snapshot-store.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +2 -2
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +3 -2
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +4 -4
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +2 -3
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/util.d.ts +1 -1
- package/dist/types/src/testing/util.d.ts.map +1 -1
- package/package.json +30 -30
- package/src/{dbhost → db-host}/database-host.ts +0 -2
- package/src/index.ts +2 -2
- package/src/pipeline/pipeline.ts +19 -26
- package/src/space/control-pipeline.ts +4 -1
- package/src/space/data-pipeline.ts +12 -9
- package/src/space/space-manager.ts +1 -1
- package/src/space/space-protocol.ts +2 -3
- package/src/space/space.ts +1 -1
- package/src/testing/database-test-rig.ts +1 -1
- package/src/testing/test-agent-builder.ts +1 -1
- package/src/testing/util.ts +1 -1
- package/src/tests/database.test.ts +1 -1
- package/dist/lib/browser/chunk-OJGL5427.mjs.map +0 -7
- package/dist/types/src/dbhost/data-service-host.d.ts.map +0 -1
- package/dist/types/src/dbhost/data-service.d.ts.map +0 -1
- package/dist/types/src/dbhost/database-host.d.ts.map +0 -1
- package/dist/types/src/dbhost/index.d.ts.map +0 -1
- package/dist/types/src/dbhost/snapshot-manager.d.ts.map +0 -1
- package/dist/types/src/dbhost/snapshot-store.d.ts.map +0 -1
- package/dist/types/src/dbhost/snapshot-store.test.d.ts.map +0 -1
- /package/dist/types/src/{dbhost → db-host}/data-service-host.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/data-service.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/database-host.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/index.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/snapshot-manager.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/snapshot-store.d.ts +0 -0
- /package/dist/types/src/{dbhost → db-host}/snapshot-store.test.d.ts +0 -0
- /package/src/{dbhost → db-host}/data-service-host.ts +0 -0
- /package/src/{dbhost → db-host}/data-service.ts +0 -0
- /package/src/{dbhost → db-host}/index.ts +0 -0
- /package/src/{dbhost → db-host}/snapshot-manager.ts +0 -0
- /package/src/{dbhost → db-host}/snapshot-store.test.ts +0 -0
- /package/src/{dbhost → db-host}/snapshot-store.ts +0 -0
|
@@ -1,14 +1,183 @@
|
|
|
1
1
|
import "@dxos/node-std/globals";
|
|
2
2
|
|
|
3
|
-
// packages/core/echo/echo-pipeline/src/
|
|
4
|
-
import
|
|
3
|
+
// packages/core/echo/echo-pipeline/src/common/codec.ts
|
|
4
|
+
import { createCodecEncoding } from "@dxos/hypercore";
|
|
5
|
+
import { schema } from "@dxos/protocols";
|
|
6
|
+
var codec = schema.getCodecForType("dxos.echo.feed.FeedMessage");
|
|
7
|
+
var valueEncoding = createCodecEncoding(codec);
|
|
8
|
+
|
|
9
|
+
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
5
10
|
import invariant from "tiny-invariant";
|
|
6
|
-
|
|
7
|
-
|
|
11
|
+
var createMappedFeedWriter = (mapper, writer) => {
|
|
12
|
+
invariant(mapper);
|
|
13
|
+
invariant(writer);
|
|
14
|
+
return {
|
|
15
|
+
write: async (data, options) => await writer.write(await mapper(data), options)
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
// packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
|
|
20
|
+
import invariant2 from "tiny-invariant";
|
|
21
|
+
import { Stream } from "@dxos/codec-protobuf";
|
|
22
|
+
import { Context } from "@dxos/context";
|
|
23
|
+
import { tagMutationsInBatch, setMetadataOnObject } from "@dxos/echo-db";
|
|
8
24
|
import { log } from "@dxos/log";
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
|
|
25
|
+
import { ComplexMap } from "@dxos/util";
|
|
26
|
+
var __dxlog_file = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
27
|
+
var DataServiceHost = class {
|
|
28
|
+
constructor(_itemManager, _itemDemuxer, _writeStream) {
|
|
29
|
+
this._itemManager = _itemManager;
|
|
30
|
+
this._itemDemuxer = _itemDemuxer;
|
|
31
|
+
this._writeStream = _writeStream;
|
|
32
|
+
this._ctx = new Context();
|
|
33
|
+
this._clientTagMap = new ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
|
|
34
|
+
}
|
|
35
|
+
async open() {
|
|
36
|
+
}
|
|
37
|
+
async close() {
|
|
38
|
+
await this._ctx.dispose();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Real-time subscription to data objects in a space.
|
|
42
|
+
*/
|
|
43
|
+
subscribe() {
|
|
44
|
+
return new Stream(({ next, close, ctx }) => {
|
|
45
|
+
ctx.onDispose(this._ctx.onDispose(close));
|
|
46
|
+
const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
|
|
47
|
+
next({
|
|
48
|
+
batch: {
|
|
49
|
+
objects
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
53
|
+
var _a;
|
|
54
|
+
const { batch, meta } = message;
|
|
55
|
+
invariant2(!meta.clientTag, "Unexpected client tag in mutation message");
|
|
56
|
+
log("message", {
|
|
57
|
+
batch,
|
|
58
|
+
meta
|
|
59
|
+
}, {
|
|
60
|
+
F: __dxlog_file,
|
|
61
|
+
L: 63,
|
|
62
|
+
S: this,
|
|
63
|
+
C: (f, a) => f(...a)
|
|
64
|
+
});
|
|
65
|
+
const clientTag = this._clientTagMap.get([
|
|
66
|
+
message.meta.feedKey,
|
|
67
|
+
message.meta.seq
|
|
68
|
+
]);
|
|
69
|
+
(_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
|
|
70
|
+
setMetadataOnObject(object, {
|
|
71
|
+
...meta
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
if (clientTag) {
|
|
75
|
+
tagMutationsInBatch(batch, clientTag, 0);
|
|
76
|
+
}
|
|
77
|
+
next({
|
|
78
|
+
clientTag,
|
|
79
|
+
feedKey: message.meta.feedKey,
|
|
80
|
+
seq: message.meta.seq,
|
|
81
|
+
batch
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
async write(request) {
|
|
87
|
+
var _a, _b;
|
|
88
|
+
invariant2(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
|
|
89
|
+
invariant2(this._writeStream, "Cannot write mutations in readonly mode");
|
|
90
|
+
log("write", {
|
|
91
|
+
clientTag: request.clientTag,
|
|
92
|
+
objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
|
|
93
|
+
}, {
|
|
94
|
+
F: __dxlog_file,
|
|
95
|
+
L: 92,
|
|
96
|
+
S: this,
|
|
97
|
+
C: (f, a) => f(...a)
|
|
98
|
+
});
|
|
99
|
+
const message = createDataMessage(request.batch);
|
|
100
|
+
const receipt = await this._writeStream.write(message, {
|
|
101
|
+
afterWrite: async (receipt2) => {
|
|
102
|
+
if (request.clientTag) {
|
|
103
|
+
log("tag", {
|
|
104
|
+
clientTag: request.clientTag,
|
|
105
|
+
feedKey: receipt2.feedKey,
|
|
106
|
+
seq: receipt2.seq
|
|
107
|
+
}, {
|
|
108
|
+
F: __dxlog_file,
|
|
109
|
+
L: 101,
|
|
110
|
+
S: this,
|
|
111
|
+
C: (f, a) => f(...a)
|
|
112
|
+
});
|
|
113
|
+
this._clientTagMap.set([
|
|
114
|
+
receipt2.feedKey,
|
|
115
|
+
receipt2.seq
|
|
116
|
+
], request.clientTag);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
return receipt;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
var createDataMessage = (batch) => {
|
|
124
|
+
var _a;
|
|
125
|
+
return {
|
|
126
|
+
batch: {
|
|
127
|
+
objects: (_a = batch.objects) == null ? void 0 : _a.map((object) => {
|
|
128
|
+
var _a2;
|
|
129
|
+
return {
|
|
130
|
+
...object,
|
|
131
|
+
mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((mutation) => ({
|
|
132
|
+
...mutation,
|
|
133
|
+
meta: void 0
|
|
134
|
+
})),
|
|
135
|
+
meta: void 0
|
|
136
|
+
};
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
// packages/core/echo/echo-pipeline/src/db-host/database-host.ts
|
|
143
|
+
import { ItemDemuxer } from "@dxos/echo-db";
|
|
144
|
+
var DatabaseHost = class {
|
|
145
|
+
constructor(_outboundStream) {
|
|
146
|
+
this._outboundStream = _outboundStream;
|
|
147
|
+
}
|
|
148
|
+
get isReadOnly() {
|
|
149
|
+
return !!this._outboundStream;
|
|
150
|
+
}
|
|
151
|
+
get echoProcessor() {
|
|
152
|
+
return this._echoProcessor;
|
|
153
|
+
}
|
|
154
|
+
async open(itemManager, modelFactory) {
|
|
155
|
+
this._itemManager = itemManager;
|
|
156
|
+
this._itemManager._debugLabel = "host";
|
|
157
|
+
this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
|
|
158
|
+
this._echoProcessor = this._itemDemuxer.open();
|
|
159
|
+
}
|
|
160
|
+
async close() {
|
|
161
|
+
}
|
|
162
|
+
getWriteStream() {
|
|
163
|
+
return this._outboundStream;
|
|
164
|
+
}
|
|
165
|
+
createSnapshot() {
|
|
166
|
+
return this._itemDemuxer.createSnapshot();
|
|
167
|
+
}
|
|
168
|
+
createDataServiceHost() {
|
|
169
|
+
var _a;
|
|
170
|
+
return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
// packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
|
|
175
|
+
import { trackLeaks } from "@dxos/async";
|
|
176
|
+
import { cancelWithContext } from "@dxos/context";
|
|
177
|
+
import { timed } from "@dxos/debug";
|
|
178
|
+
import { PublicKey } from "@dxos/keys";
|
|
179
|
+
import { schema as schema2 } from "@dxos/protocols";
|
|
180
|
+
import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
12
181
|
function _ts_decorate(decorators, target, key, desc) {
|
|
13
182
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
14
183
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -19,57 +188,223 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
19
188
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
20
189
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
21
190
|
}
|
|
22
|
-
var
|
|
23
|
-
var
|
|
24
|
-
version: STORAGE_VERSION,
|
|
25
|
-
spaces: [],
|
|
26
|
-
created: new Date(),
|
|
27
|
-
updated: new Date()
|
|
28
|
-
});
|
|
29
|
-
var EchoMetadata = schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
30
|
-
var MetadataStore = class {
|
|
191
|
+
var SpaceSnapshot = schema2.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
192
|
+
var SnapshotManager = class SnapshotManager2 {
|
|
31
193
|
// prettier-ignore
|
|
32
|
-
constructor(
|
|
33
|
-
this.
|
|
34
|
-
this.
|
|
35
|
-
this.
|
|
194
|
+
constructor(_snapshotStore, _blobStore, _blobSync) {
|
|
195
|
+
this._snapshotStore = _snapshotStore;
|
|
196
|
+
this._blobStore = _blobStore;
|
|
197
|
+
this._blobSync = _blobSync;
|
|
36
198
|
}
|
|
37
|
-
|
|
38
|
-
|
|
199
|
+
async _getBlob(blobId) {
|
|
200
|
+
const blob = await this._blobStore.get(blobId);
|
|
201
|
+
return SpaceSnapshot.decode(blob);
|
|
39
202
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
203
|
+
async load(ctx, id) {
|
|
204
|
+
const blobId = PublicKey.fromHex(id).asUint8Array();
|
|
205
|
+
const blobMeta = await this._blobStore.getMeta(blobId);
|
|
206
|
+
if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
|
|
207
|
+
return this._getBlob(blobId);
|
|
208
|
+
}
|
|
209
|
+
const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
|
|
210
|
+
if (fallbackStore) {
|
|
211
|
+
return fallbackStore;
|
|
212
|
+
}
|
|
213
|
+
await this._blobSync.download(ctx, blobId);
|
|
214
|
+
return this._getBlob(blobId);
|
|
43
215
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
get spaces() {
|
|
49
|
-
var _a;
|
|
50
|
-
return (_a = this._metadata.spaces) != null ? _a : [];
|
|
216
|
+
async store(snapshot) {
|
|
217
|
+
const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
|
|
218
|
+
await this._blobSync.notifyBlobAdded(id);
|
|
219
|
+
return PublicKey.from(id).toHex();
|
|
51
220
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
221
|
+
};
|
|
222
|
+
_ts_decorate([
|
|
223
|
+
timed(1e4)
|
|
224
|
+
], SnapshotManager.prototype, "load", null);
|
|
225
|
+
SnapshotManager = _ts_decorate([
|
|
226
|
+
trackLeaks("open", "close")
|
|
227
|
+
], SnapshotManager);
|
|
228
|
+
|
|
229
|
+
// packages/core/echo/echo-pipeline/src/db-host/snapshot-store.ts
|
|
230
|
+
import { subtleCrypto } from "@dxos/crypto";
|
|
231
|
+
import { schema as schema3 } from "@dxos/protocols";
|
|
232
|
+
var SpaceSnapshot2 = schema3.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
233
|
+
var SnapshotStore = class {
|
|
234
|
+
// prettier-ignore
|
|
235
|
+
constructor(_directory) {
|
|
236
|
+
this._directory = _directory;
|
|
237
|
+
}
|
|
238
|
+
async saveSnapshot(snapshot) {
|
|
239
|
+
const encoded = SpaceSnapshot2.encode(snapshot);
|
|
240
|
+
const key = await subtleCrypto.digest("SHA-256", encoded);
|
|
241
|
+
const keyString = Buffer.from(key).toString("hex");
|
|
242
|
+
const file = await this._directory.getOrCreateFile(keyString);
|
|
58
243
|
try {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
244
|
+
await file.write(0, Buffer.from(encoded));
|
|
245
|
+
} finally {
|
|
246
|
+
await file.close();
|
|
247
|
+
}
|
|
248
|
+
return keyString;
|
|
249
|
+
}
|
|
250
|
+
async loadSnapshot(key) {
|
|
251
|
+
const file = await this._directory.getOrCreateFile(key);
|
|
252
|
+
try {
|
|
253
|
+
const { size } = await file.stat();
|
|
254
|
+
if (size === 0) {
|
|
255
|
+
return void 0;
|
|
62
256
|
}
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
257
|
+
const buffer = await file.read(0, size);
|
|
258
|
+
return SpaceSnapshot2.decode(buffer);
|
|
259
|
+
} finally {
|
|
260
|
+
await file.close();
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
async listSnapshots() {
|
|
264
|
+
const entries = await this._directory.list();
|
|
265
|
+
return await Promise.all(entries.map(async (key) => {
|
|
266
|
+
const { size } = await this._directory.getOrCreateFile(key).stat();
|
|
267
|
+
return {
|
|
268
|
+
key,
|
|
269
|
+
size
|
|
270
|
+
};
|
|
271
|
+
}));
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
276
|
+
import invariant3 from "tiny-invariant";
|
|
277
|
+
import { raise } from "@dxos/debug";
|
|
278
|
+
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
279
|
+
import { log as log2 } from "@dxos/log";
|
|
280
|
+
import { ComplexMap as ComplexMap2 } from "@dxos/util";
|
|
281
|
+
var __dxlog_file2 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
|
|
282
|
+
var DataServiceSubscriptions = class {
|
|
283
|
+
constructor() {
|
|
284
|
+
this._spaces = new ComplexMap2(PublicKey2.hash);
|
|
285
|
+
}
|
|
286
|
+
clear() {
|
|
287
|
+
this._spaces.clear();
|
|
288
|
+
}
|
|
289
|
+
async registerSpace(spaceKey, host) {
|
|
290
|
+
log2("Registering space", {
|
|
291
|
+
spaceKey
|
|
292
|
+
}, {
|
|
293
|
+
F: __dxlog_file2,
|
|
294
|
+
L: 31,
|
|
295
|
+
S: this,
|
|
296
|
+
C: (f, a) => f(...a)
|
|
297
|
+
});
|
|
298
|
+
invariant3(!this._spaces.has(spaceKey));
|
|
299
|
+
await host.open();
|
|
300
|
+
this._spaces.set(spaceKey, host);
|
|
301
|
+
}
|
|
302
|
+
async unregisterSpace(spaceKey) {
|
|
303
|
+
log2("Unregistering space", {
|
|
304
|
+
spaceKey
|
|
305
|
+
}, {
|
|
306
|
+
F: __dxlog_file2,
|
|
307
|
+
L: 38,
|
|
308
|
+
S: this,
|
|
309
|
+
C: (f, a) => f(...a)
|
|
310
|
+
});
|
|
311
|
+
const host = this._spaces.get(spaceKey);
|
|
312
|
+
await (host == null ? void 0 : host.close());
|
|
313
|
+
this._spaces.delete(spaceKey);
|
|
314
|
+
}
|
|
315
|
+
getDataService(spaceKey) {
|
|
316
|
+
return this._spaces.get(spaceKey);
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
var DataServiceImpl = class {
|
|
320
|
+
constructor(_subscriptions) {
|
|
321
|
+
this._subscriptions = _subscriptions;
|
|
322
|
+
}
|
|
323
|
+
subscribe(request) {
|
|
324
|
+
var _a;
|
|
325
|
+
invariant3(request.spaceKey);
|
|
326
|
+
const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
|
|
327
|
+
return host.subscribe();
|
|
328
|
+
}
|
|
329
|
+
write(request) {
|
|
330
|
+
var _a;
|
|
331
|
+
invariant3(request.spaceKey);
|
|
332
|
+
invariant3(request.batch);
|
|
333
|
+
const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
|
|
334
|
+
return host.write(request);
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
// packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
|
|
339
|
+
import CRC32 from "crc-32";
|
|
340
|
+
import invariant4 from "tiny-invariant";
|
|
341
|
+
import { synchronized, Event } from "@dxos/async";
|
|
342
|
+
import { DataCorruptionError } from "@dxos/errors";
|
|
343
|
+
import { log as log3 } from "@dxos/log";
|
|
344
|
+
import { STORAGE_VERSION, schema as schema4 } from "@dxos/protocols";
|
|
345
|
+
import { SpaceState } from "@dxos/protocols/proto/dxos/client/services";
|
|
346
|
+
import { arrayToBuffer } from "@dxos/util";
|
|
347
|
+
function _ts_decorate2(decorators, target, key, desc) {
|
|
348
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
349
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
350
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
351
|
+
else
|
|
352
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
353
|
+
if (d = decorators[i])
|
|
354
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
355
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
356
|
+
}
|
|
357
|
+
var __dxlog_file3 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
|
|
358
|
+
var emptyEchoMetadata = () => ({
|
|
359
|
+
version: STORAGE_VERSION,
|
|
360
|
+
spaces: [],
|
|
361
|
+
created: new Date(),
|
|
362
|
+
updated: new Date()
|
|
363
|
+
});
|
|
364
|
+
var EchoMetadata = schema4.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
365
|
+
var MetadataStore = class {
|
|
366
|
+
// prettier-ignore
|
|
367
|
+
constructor(_directory) {
|
|
368
|
+
this._directory = _directory;
|
|
369
|
+
this._metadata = emptyEchoMetadata();
|
|
370
|
+
this.update = new Event();
|
|
371
|
+
}
|
|
372
|
+
get metadata() {
|
|
373
|
+
return this._metadata;
|
|
374
|
+
}
|
|
375
|
+
get version() {
|
|
376
|
+
var _a;
|
|
377
|
+
return (_a = this._metadata.version) != null ? _a : 0;
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
|
|
381
|
+
* addSpaceFeed functions.
|
|
382
|
+
*/
|
|
383
|
+
get spaces() {
|
|
384
|
+
var _a;
|
|
385
|
+
return (_a = this._metadata.spaces) != null ? _a : [];
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Loads metadata from persistent storage.
|
|
389
|
+
*/
|
|
390
|
+
async load() {
|
|
391
|
+
var _a;
|
|
392
|
+
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
393
|
+
try {
|
|
394
|
+
const { size: fileLength } = await file.stat();
|
|
395
|
+
if (fileLength < 8) {
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
const dataSize = fromBytesInt32(await file.read(0, 4));
|
|
399
|
+
const checksum = fromBytesInt32(await file.read(4, 4));
|
|
400
|
+
log3("loaded", {
|
|
401
|
+
size: dataSize,
|
|
402
|
+
checksum
|
|
403
|
+
}, {
|
|
404
|
+
F: __dxlog_file3,
|
|
405
|
+
L: 72,
|
|
406
|
+
S: this,
|
|
407
|
+
C: (f, a) => f(...a)
|
|
73
408
|
});
|
|
74
409
|
if (fileLength < dataSize + 8) {
|
|
75
410
|
throw new DataCorruptionError("Metadata size is smaller than expected.");
|
|
@@ -85,10 +420,10 @@ var MetadataStore = class {
|
|
|
85
420
|
(_a2 = space.state) != null ? _a2 : space.state = SpaceState.ACTIVE;
|
|
86
421
|
});
|
|
87
422
|
} catch (err) {
|
|
88
|
-
|
|
423
|
+
log3.error("failed to load metadata", {
|
|
89
424
|
err
|
|
90
425
|
}, {
|
|
91
|
-
F:
|
|
426
|
+
F: __dxlog_file3,
|
|
92
427
|
L: 92,
|
|
93
428
|
S: this,
|
|
94
429
|
C: (f, a) => f(...a)
|
|
@@ -116,11 +451,11 @@ var MetadataStore = class {
|
|
|
116
451
|
result.writeInt32LE(checksum, 4);
|
|
117
452
|
encoded.copy(result, 8);
|
|
118
453
|
await file.write(0, result);
|
|
119
|
-
|
|
454
|
+
log3("saved", {
|
|
120
455
|
size: encoded.length,
|
|
121
456
|
checksum
|
|
122
457
|
}, {
|
|
123
|
-
F:
|
|
458
|
+
F: __dxlog_file3,
|
|
124
459
|
L: 124,
|
|
125
460
|
S: this,
|
|
126
461
|
C: (f, a) => f(...a)
|
|
@@ -135,15 +470,15 @@ var MetadataStore = class {
|
|
|
135
470
|
return this._metadata.identity.haloSpace;
|
|
136
471
|
}
|
|
137
472
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
138
|
-
|
|
473
|
+
invariant4(space, "Space not found");
|
|
139
474
|
return space;
|
|
140
475
|
}
|
|
141
476
|
/**
|
|
142
477
|
* Clears storage - doesn't work for now.
|
|
143
478
|
*/
|
|
144
479
|
async clear() {
|
|
145
|
-
|
|
146
|
-
F:
|
|
480
|
+
log3("clearing all metadata", void 0, {
|
|
481
|
+
F: __dxlog_file3,
|
|
147
482
|
L: 145,
|
|
148
483
|
S: this,
|
|
149
484
|
C: (f, a) => f(...a)
|
|
@@ -155,13 +490,13 @@ var MetadataStore = class {
|
|
|
155
490
|
return this._metadata.identity;
|
|
156
491
|
}
|
|
157
492
|
async setIdentityRecord(record) {
|
|
158
|
-
|
|
493
|
+
invariant4(!this._metadata.identity, "Cannot overwrite existing identity in metadata");
|
|
159
494
|
this._metadata.identity = record;
|
|
160
495
|
await this._save();
|
|
161
496
|
}
|
|
162
497
|
async addSpace(record) {
|
|
163
498
|
var _a, _b, _c;
|
|
164
|
-
|
|
499
|
+
invariant4(!((_a = this._metadata.spaces) != null ? _a : []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata");
|
|
165
500
|
((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
|
|
166
501
|
await this._save();
|
|
167
502
|
}
|
|
@@ -188,36 +523,20 @@ var MetadataStore = class {
|
|
|
188
523
|
await this._save();
|
|
189
524
|
}
|
|
190
525
|
};
|
|
191
|
-
|
|
526
|
+
_ts_decorate2([
|
|
192
527
|
synchronized
|
|
193
528
|
], MetadataStore.prototype, "load", null);
|
|
194
|
-
|
|
529
|
+
_ts_decorate2([
|
|
195
530
|
synchronized
|
|
196
531
|
], MetadataStore.prototype, "_save", null);
|
|
197
532
|
var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
198
533
|
|
|
199
|
-
// packages/core/echo/echo-pipeline/src/common/codec.ts
|
|
200
|
-
import { createCodecEncoding } from "@dxos/hypercore";
|
|
201
|
-
import { schema as schema2 } from "@dxos/protocols";
|
|
202
|
-
var codec = schema2.getCodecForType("dxos.echo.feed.FeedMessage");
|
|
203
|
-
var valueEncoding = createCodecEncoding(codec);
|
|
204
|
-
|
|
205
|
-
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
206
|
-
import invariant2 from "tiny-invariant";
|
|
207
|
-
var createMappedFeedWriter = (mapper, writer) => {
|
|
208
|
-
invariant2(mapper);
|
|
209
|
-
invariant2(writer);
|
|
210
|
-
return {
|
|
211
|
-
write: async (data, options) => await writer.write(await mapper(data), options)
|
|
212
|
-
};
|
|
213
|
-
};
|
|
214
|
-
|
|
215
534
|
// packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
|
|
216
535
|
import { Event as Event2 } from "@dxos/async";
|
|
217
|
-
import { timed } from "@dxos/debug";
|
|
218
|
-
import { log as
|
|
536
|
+
import { timed as timed2 } from "@dxos/debug";
|
|
537
|
+
import { log as log4 } from "@dxos/log";
|
|
219
538
|
import { Timeframe } from "@dxos/timeframe";
|
|
220
|
-
function
|
|
539
|
+
function _ts_decorate3(decorators, target, key, desc) {
|
|
221
540
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
222
541
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
223
542
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -227,7 +546,7 @@ function _ts_decorate2(decorators, target, key, desc) {
|
|
|
227
546
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
228
547
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
229
548
|
}
|
|
230
|
-
var
|
|
549
|
+
var __dxlog_file4 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
|
|
231
550
|
var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
|
|
232
551
|
feedKey,
|
|
233
552
|
index
|
|
@@ -242,829 +561,510 @@ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
|
|
|
242
561
|
}));
|
|
243
562
|
var TimeframeClock = class {
|
|
244
563
|
// prettier-ignore
|
|
245
|
-
constructor(_timeframe = new Timeframe()) {
|
|
246
|
-
this._timeframe = _timeframe;
|
|
247
|
-
this.update = new Event2();
|
|
248
|
-
this._pendingTimeframe = _timeframe;
|
|
249
|
-
}
|
|
250
|
-
/**
|
|
251
|
-
* Timeframe that was processed by ECHO.
|
|
252
|
-
*/
|
|
253
|
-
get timeframe() {
|
|
254
|
-
return this._timeframe;
|
|
255
|
-
}
|
|
256
|
-
/**
|
|
257
|
-
* Timeframe that is currently being processed by ECHO.
|
|
258
|
-
* Will be equal to `timeframe` after the processing is complete.
|
|
259
|
-
*/
|
|
260
|
-
get pendingTimeframe() {
|
|
261
|
-
return this._pendingTimeframe;
|
|
262
|
-
}
|
|
263
|
-
setTimeframe(timeframe) {
|
|
264
|
-
this._timeframe = timeframe;
|
|
265
|
-
this._pendingTimeframe = timeframe;
|
|
266
|
-
this.update.emit(this._timeframe);
|
|
267
|
-
}
|
|
268
|
-
updatePendingTimeframe(key, seq) {
|
|
269
|
-
this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
|
|
270
|
-
[
|
|
271
|
-
key,
|
|
272
|
-
seq
|
|
273
|
-
]
|
|
274
|
-
]));
|
|
275
|
-
}
|
|
276
|
-
updateTimeframe() {
|
|
277
|
-
this._timeframe = this._pendingTimeframe;
|
|
278
|
-
this.update.emit(this._timeframe);
|
|
279
|
-
}
|
|
280
|
-
hasGaps(timeframe) {
|
|
281
|
-
const gaps = Timeframe.dependencies(timeframe, this._timeframe);
|
|
282
|
-
return !gaps.isEmpty();
|
|
283
|
-
}
|
|
284
|
-
async waitUntilReached(target) {
|
|
285
|
-
log2("waitUntilReached", {
|
|
286
|
-
target,
|
|
287
|
-
current: this._timeframe
|
|
288
|
-
}, {
|
|
289
|
-
F: __dxlog_file2,
|
|
290
|
-
L: 73,
|
|
291
|
-
S: this,
|
|
292
|
-
C: (f, a) => f(...a)
|
|
293
|
-
});
|
|
294
|
-
await this.update.waitForCondition(() => {
|
|
295
|
-
log2("check if reached", {
|
|
296
|
-
target,
|
|
297
|
-
current: this._timeframe,
|
|
298
|
-
deps: Timeframe.dependencies(target, this._timeframe)
|
|
299
|
-
}, {
|
|
300
|
-
F: __dxlog_file2,
|
|
301
|
-
L: 75,
|
|
302
|
-
S: this,
|
|
303
|
-
C: (f, a) => f(...a)
|
|
304
|
-
});
|
|
305
|
-
return Timeframe.dependencies(target, this._timeframe).isEmpty();
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
};
|
|
309
|
-
_ts_decorate2([
|
|
310
|
-
timed(5e3)
|
|
311
|
-
], TimeframeClock.prototype, "waitUntilReached", null);
|
|
312
|
-
|
|
313
|
-
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
314
|
-
import invariant4 from "tiny-invariant";
|
|
315
|
-
import { Event as Event3, sleep, synchronized as synchronized2, Trigger } from "@dxos/async";
|
|
316
|
-
import { Context, rejectOnDispose } from "@dxos/context";
|
|
317
|
-
import { failUndefined } from "@dxos/debug";
|
|
318
|
-
import { FeedSetIterator } from "@dxos/feed-store";
|
|
319
|
-
import { PublicKey } from "@dxos/keys";
|
|
320
|
-
import { log as log4 } from "@dxos/log";
|
|
321
|
-
import { Timeframe as Timeframe2 } from "@dxos/timeframe";
|
|
322
|
-
import { ComplexMap } from "@dxos/util";
|
|
323
|
-
|
|
324
|
-
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
325
|
-
import invariant3 from "tiny-invariant";
|
|
326
|
-
import { log as log3 } from "@dxos/log";
|
|
327
|
-
var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
328
|
-
var createMessageSelector = (timeframeClock) => {
|
|
329
|
-
return (messages) => {
|
|
330
|
-
for (let i = 0; i < messages.length; i++) {
|
|
331
|
-
const { data: { timeframe } } = messages[i];
|
|
332
|
-
invariant3(timeframe);
|
|
333
|
-
if (!timeframeClock.hasGaps(timeframe)) {
|
|
334
|
-
return i;
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
log3("Skipping...", void 0, {
|
|
338
|
-
F: __dxlog_file3,
|
|
339
|
-
L: 34,
|
|
340
|
-
S: void 0,
|
|
341
|
-
C: (f, a) => f(...a)
|
|
342
|
-
});
|
|
343
|
-
};
|
|
344
|
-
};
|
|
345
|
-
|
|
346
|
-
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
347
|
-
function _ts_decorate3(decorators, target, key, desc) {
|
|
348
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
349
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
350
|
-
r = Reflect.decorate(decorators, target, key, desc);
|
|
351
|
-
else
|
|
352
|
-
for (var i = decorators.length - 1; i >= 0; i--)
|
|
353
|
-
if (d = decorators[i])
|
|
354
|
-
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
355
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
356
|
-
}
|
|
357
|
-
var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
358
|
-
var PipelineState = class {
|
|
359
|
-
// prettier-ignore
|
|
360
|
-
constructor(_feeds, _timeframeClock) {
|
|
361
|
-
this._feeds = _feeds;
|
|
362
|
-
this._timeframeClock = _timeframeClock;
|
|
363
|
-
this._ctx = new Context();
|
|
364
|
-
this.timeframeUpdate = this._timeframeClock.update;
|
|
365
|
-
this.stalled = new Event3();
|
|
366
|
-
this._startTimeframe = new Timeframe2();
|
|
367
|
-
}
|
|
368
|
-
/**
|
|
369
|
-
* Latest theoretical timeframe based on the last mutation in each feed.
|
|
370
|
-
* NOTE: This might never be reached if the mutation dependencies
|
|
371
|
-
*/
|
|
372
|
-
// TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
|
|
373
|
-
get endTimeframe() {
|
|
374
|
-
return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
|
|
375
|
-
feedKey: feed.key,
|
|
376
|
-
index: feed.length - 1
|
|
377
|
-
})));
|
|
378
|
-
}
|
|
379
|
-
get startTimeframe() {
|
|
380
|
-
return this._startTimeframe;
|
|
381
|
-
}
|
|
382
|
-
get timeframe() {
|
|
383
|
-
return this._timeframeClock.timeframe;
|
|
384
|
-
}
|
|
385
|
-
get pendingTimeframe() {
|
|
386
|
-
return this._timeframeClock.pendingTimeframe;
|
|
387
|
-
}
|
|
388
|
-
get targetTimeframe() {
|
|
389
|
-
return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
|
|
390
|
-
}
|
|
391
|
-
get feeds() {
|
|
392
|
-
return Array.from(this._feeds.values());
|
|
393
|
-
}
|
|
394
|
-
async waitUntilTimeframe(target) {
|
|
395
|
-
await this._timeframeClock.waitUntilReached(target);
|
|
396
|
-
}
|
|
397
|
-
setTargetTimeframe(target) {
|
|
398
|
-
this._targetTimeframe = target;
|
|
399
|
-
}
|
|
400
|
-
/**
|
|
401
|
-
* Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
|
|
402
|
-
*
|
|
403
|
-
* This function will resolve immediately if the pipeline is stalled.
|
|
404
|
-
*
|
|
405
|
-
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
406
|
-
*/
|
|
407
|
-
async waitUntilReachedTargetTimeframe({ ctx = new Context(), timeout, breakOnStall = true } = {}) {
|
|
408
|
-
var _a;
|
|
409
|
-
log4("waitUntilReachedTargetTimeframe", {
|
|
410
|
-
timeout,
|
|
411
|
-
current: this.timeframe,
|
|
412
|
-
target: this.targetTimeframe
|
|
413
|
-
}, {
|
|
414
|
-
F: __dxlog_file4,
|
|
415
|
-
L: 126,
|
|
416
|
-
S: this,
|
|
417
|
-
C: (f, a) => f(...a)
|
|
418
|
-
});
|
|
419
|
-
(_a = this._reachedTargetPromise) != null ? _a : this._reachedTargetPromise = Promise.race([
|
|
420
|
-
this._timeframeClock.update.waitForCondition(() => {
|
|
421
|
-
return Timeframe2.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
422
|
-
}),
|
|
423
|
-
...breakOnStall ? [
|
|
424
|
-
this.stalled.discardParameter().waitForCount(1)
|
|
425
|
-
] : []
|
|
426
|
-
]);
|
|
427
|
-
let done = false;
|
|
428
|
-
if (timeout) {
|
|
429
|
-
return Promise.race([
|
|
430
|
-
rejectOnDispose(ctx),
|
|
431
|
-
rejectOnDispose(this._ctx),
|
|
432
|
-
this._reachedTargetPromise.then(() => {
|
|
433
|
-
done = true;
|
|
434
|
-
}),
|
|
435
|
-
sleep(timeout).then(() => {
|
|
436
|
-
if (done) {
|
|
437
|
-
return;
|
|
438
|
-
}
|
|
439
|
-
log4.warn("waitUntilReachedTargetTimeframe timed out", {
|
|
440
|
-
timeout,
|
|
441
|
-
current: this.timeframe,
|
|
442
|
-
target: this.targetTimeframe,
|
|
443
|
-
dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
|
|
444
|
-
}, {
|
|
445
|
-
F: __dxlog_file4,
|
|
446
|
-
L: 153,
|
|
447
|
-
S: this,
|
|
448
|
-
C: (f, a) => f(...a)
|
|
449
|
-
});
|
|
450
|
-
})
|
|
451
|
-
]);
|
|
452
|
-
} else {
|
|
453
|
-
return this._reachedTargetPromise;
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
};
|
|
457
|
-
var Pipeline = class {
|
|
458
|
-
constructor() {
|
|
459
|
-
this._timeframeClock = new TimeframeClock(new Timeframe2());
|
|
460
|
-
this._feeds = new ComplexMap(PublicKey.hash);
|
|
461
|
-
// External state accessor.
|
|
462
|
-
this._state = new PipelineState(this._feeds, this._timeframeClock);
|
|
463
|
-
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
464
|
-
this._processingTrigger = new Trigger().wake();
|
|
465
|
-
this._pauseTrigger = new Trigger().wake();
|
|
466
|
-
this._isStopping = false;
|
|
467
|
-
this._isStarted = false;
|
|
468
|
-
this._isOpen = false;
|
|
469
|
-
this._isPaused = false;
|
|
470
|
-
}
|
|
471
|
-
get state() {
|
|
472
|
-
return this._state;
|
|
473
|
-
}
|
|
474
|
-
get writer() {
|
|
475
|
-
invariant4(this._writer, "Writer not set.");
|
|
476
|
-
return this._writer;
|
|
477
|
-
}
|
|
478
|
-
getFeeds() {
|
|
479
|
-
return this._feedSetIterator.feeds;
|
|
480
|
-
}
|
|
481
|
-
// NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
|
|
482
|
-
// which might be opening feeds during the mutation processing, which w
|
|
483
|
-
async addFeed(feed) {
|
|
484
|
-
this._feeds.set(feed.key, feed);
|
|
485
|
-
if (this._feedSetIterator) {
|
|
486
|
-
await this._feedSetIterator.addFeed(feed);
|
|
487
|
-
}
|
|
488
|
-
this._setFeedDownloadState(feed);
|
|
489
|
-
}
|
|
490
|
-
hasFeed(feedKey) {
|
|
491
|
-
return this._feeds.has(feedKey);
|
|
492
|
-
}
|
|
493
|
-
setWriteFeed(feed) {
|
|
494
|
-
invariant4(!this._writer, "Writer already set.");
|
|
495
|
-
invariant4(feed.properties.writable, "Feed must be writable.");
|
|
496
|
-
this._writer = createMappedFeedWriter((payload) => ({
|
|
497
|
-
timeframe: this._timeframeClock.timeframe,
|
|
498
|
-
payload
|
|
499
|
-
}), feed.createFeedWriter());
|
|
500
|
-
}
|
|
501
|
-
async start() {
|
|
502
|
-
log4("starting...", {}, {
|
|
503
|
-
F: __dxlog_file4,
|
|
504
|
-
L: 273,
|
|
505
|
-
S: this,
|
|
506
|
-
C: (f, a) => f(...a)
|
|
507
|
-
});
|
|
508
|
-
await this._initIterator();
|
|
509
|
-
await this._feedSetIterator.open();
|
|
510
|
-
log4("started", void 0, {
|
|
511
|
-
F: __dxlog_file4,
|
|
512
|
-
L: 276,
|
|
513
|
-
S: this,
|
|
514
|
-
C: (f, a) => f(...a)
|
|
515
|
-
});
|
|
516
|
-
this._isStarted = true;
|
|
517
|
-
}
|
|
518
|
-
async stop() {
|
|
519
|
-
var _a;
|
|
520
|
-
log4("stopping...", {}, {
|
|
521
|
-
F: __dxlog_file4,
|
|
522
|
-
L: 282,
|
|
523
|
-
S: this,
|
|
524
|
-
C: (f, a) => f(...a)
|
|
525
|
-
});
|
|
526
|
-
this._isStopping = true;
|
|
527
|
-
await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
|
|
528
|
-
await this._processingTrigger.wait();
|
|
529
|
-
await this._state._ctx.dispose();
|
|
530
|
-
this._state._ctx = new Context();
|
|
531
|
-
this._state._reachedTargetPromise = void 0;
|
|
532
|
-
log4("stopped", void 0, {
|
|
533
|
-
F: __dxlog_file4,
|
|
534
|
-
L: 289,
|
|
535
|
-
S: this,
|
|
536
|
-
C: (f, a) => f(...a)
|
|
537
|
-
});
|
|
538
|
-
this._isStarted = false;
|
|
539
|
-
}
|
|
540
|
-
/**
|
|
541
|
-
* @param timeframe Timeframe of already processed messages. The pipeline will start processing messages AFTER this timeframe.
|
|
542
|
-
*/
|
|
543
|
-
async setCursor(timeframe) {
|
|
544
|
-
invariant4(!this._isStarted || this._isPaused, "Invalid state.");
|
|
545
|
-
this._state._startTimeframe = timeframe;
|
|
546
|
-
this._timeframeClock.setTimeframe(timeframe);
|
|
547
|
-
for (const feed of this._feeds.values()) {
|
|
548
|
-
this._setFeedDownloadState(feed);
|
|
549
|
-
}
|
|
550
|
-
if (this._feedSetIterator) {
|
|
551
|
-
await this._feedSetIterator.close();
|
|
552
|
-
await this._initIterator();
|
|
553
|
-
await this._feedSetIterator.open();
|
|
554
|
-
}
|
|
564
|
+
constructor(_timeframe = new Timeframe()) {
|
|
565
|
+
this._timeframe = _timeframe;
|
|
566
|
+
this.update = new Event2();
|
|
567
|
+
this._pendingTimeframe = _timeframe;
|
|
555
568
|
}
|
|
556
569
|
/**
|
|
557
|
-
*
|
|
570
|
+
* Timeframe that was processed by ECHO.
|
|
558
571
|
*/
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
if (this._isPaused) {
|
|
562
|
-
return;
|
|
563
|
-
}
|
|
564
|
-
this._pauseTrigger.reset();
|
|
565
|
-
await this._processingTrigger.wait();
|
|
566
|
-
this._isPaused = true;
|
|
567
|
-
}
|
|
568
|
-
async unpause() {
|
|
569
|
-
invariant4(this._isStarted, "Pipeline is not open.");
|
|
570
|
-
invariant4(this._isPaused, "Pipeline is not paused.");
|
|
571
|
-
this._pauseTrigger.wake();
|
|
572
|
-
this._isPaused = false;
|
|
572
|
+
get timeframe() {
|
|
573
|
+
return this._timeframe;
|
|
573
574
|
}
|
|
574
575
|
/**
|
|
575
|
-
*
|
|
576
|
-
*
|
|
576
|
+
* Timeframe that is currently being processed by ECHO.
|
|
577
|
+
* Will be equal to `timeframe` after the processing is complete.
|
|
577
578
|
*/
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
this._isOpen = true;
|
|
581
|
-
invariant4(this._feedSetIterator, "Iterator not initialized.");
|
|
582
|
-
let lastFeedSetIterator = this._feedSetIterator;
|
|
583
|
-
let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
584
|
-
while (!this._isStopping) {
|
|
585
|
-
await this._pauseTrigger.wait();
|
|
586
|
-
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
587
|
-
invariant4(this._feedSetIterator, "Iterator not initialized.");
|
|
588
|
-
lastFeedSetIterator = this._feedSetIterator;
|
|
589
|
-
iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
590
|
-
}
|
|
591
|
-
const { done, value } = await iterable.next();
|
|
592
|
-
if (done) {
|
|
593
|
-
continue;
|
|
594
|
-
}
|
|
595
|
-
const block = value != null ? value : failUndefined();
|
|
596
|
-
this._processingTrigger.reset();
|
|
597
|
-
this._timeframeClock.updatePendingTimeframe(PublicKey.from(block.feedKey), block.seq);
|
|
598
|
-
yield block;
|
|
599
|
-
this._processingTrigger.wake();
|
|
600
|
-
this._timeframeClock.updateTimeframe();
|
|
601
|
-
}
|
|
602
|
-
this._isOpen = false;
|
|
579
|
+
get pendingTimeframe() {
|
|
580
|
+
return this._pendingTimeframe;
|
|
603
581
|
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
feed.undownload({
|
|
609
|
-
callback: () => log4("Undownloaded", void 0, {
|
|
610
|
-
F: __dxlog_file4,
|
|
611
|
-
L: 383,
|
|
612
|
-
S: this,
|
|
613
|
-
C: (f, a) => f(...a)
|
|
614
|
-
})
|
|
615
|
-
});
|
|
616
|
-
feed.download({
|
|
617
|
-
start: seq + 1,
|
|
618
|
-
linear: true
|
|
619
|
-
}).catch((err) => {
|
|
620
|
-
log4("failed to download feed", {
|
|
621
|
-
err
|
|
622
|
-
}, {
|
|
623
|
-
F: __dxlog_file4,
|
|
624
|
-
L: 386,
|
|
625
|
-
S: this,
|
|
626
|
-
C: (f, a) => f(...a)
|
|
627
|
-
});
|
|
628
|
-
});
|
|
582
|
+
setTimeframe(timeframe) {
|
|
583
|
+
this._timeframe = timeframe;
|
|
584
|
+
this._pendingTimeframe = timeframe;
|
|
585
|
+
this.update.emit(this._timeframe);
|
|
629
586
|
}
|
|
630
|
-
|
|
631
|
-
this.
|
|
632
|
-
|
|
633
|
-
|
|
587
|
+
updatePendingTimeframe(key, seq) {
|
|
588
|
+
this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
|
|
589
|
+
[
|
|
590
|
+
key,
|
|
591
|
+
seq
|
|
592
|
+
]
|
|
593
|
+
]));
|
|
594
|
+
}
|
|
595
|
+
updateTimeframe() {
|
|
596
|
+
this._timeframe = this._pendingTimeframe;
|
|
597
|
+
this.update.emit(this._timeframe);
|
|
598
|
+
}
|
|
599
|
+
hasGaps(timeframe) {
|
|
600
|
+
const gaps = Timeframe.dependencies(timeframe, this._timeframe);
|
|
601
|
+
return !gaps.isEmpty();
|
|
602
|
+
}
|
|
603
|
+
async waitUntilReached(target) {
|
|
604
|
+
log4("waitUntilReached", {
|
|
605
|
+
target,
|
|
606
|
+
current: this._timeframe
|
|
607
|
+
}, {
|
|
608
|
+
F: __dxlog_file4,
|
|
609
|
+
L: 73,
|
|
610
|
+
S: this,
|
|
611
|
+
C: (f, a) => f(...a)
|
|
634
612
|
});
|
|
635
|
-
this.
|
|
636
|
-
log4
|
|
613
|
+
await this.update.waitForCondition(() => {
|
|
614
|
+
log4("check if reached", {
|
|
615
|
+
target,
|
|
616
|
+
current: this._timeframe,
|
|
617
|
+
deps: Timeframe.dependencies(target, this._timeframe)
|
|
618
|
+
}, {
|
|
637
619
|
F: __dxlog_file4,
|
|
638
|
-
L:
|
|
620
|
+
L: 75,
|
|
639
621
|
S: this,
|
|
640
622
|
C: (f, a) => f(...a)
|
|
641
623
|
});
|
|
642
|
-
this.
|
|
624
|
+
return Timeframe.dependencies(target, this._timeframe).isEmpty();
|
|
643
625
|
});
|
|
644
|
-
for (const feed of this._feeds.values()) {
|
|
645
|
-
await this._feedSetIterator.addFeed(feed);
|
|
646
|
-
}
|
|
647
626
|
}
|
|
648
627
|
};
|
|
649
628
|
_ts_decorate3([
|
|
650
|
-
|
|
651
|
-
],
|
|
652
|
-
_ts_decorate3([
|
|
653
|
-
synchronized2
|
|
654
|
-
], Pipeline.prototype, "stop", null);
|
|
655
|
-
_ts_decorate3([
|
|
656
|
-
synchronized2
|
|
657
|
-
], Pipeline.prototype, "setCursor", null);
|
|
658
|
-
_ts_decorate3([
|
|
659
|
-
synchronized2
|
|
660
|
-
], Pipeline.prototype, "pause", null);
|
|
661
|
-
_ts_decorate3([
|
|
662
|
-
synchronized2
|
|
663
|
-
], Pipeline.prototype, "unpause", null);
|
|
629
|
+
timed2(5e3)
|
|
630
|
+
], TimeframeClock.prototype, "waitUntilReached", null);
|
|
664
631
|
|
|
665
|
-
// packages/core/echo/echo-pipeline/src/
|
|
632
|
+
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
633
|
+
import invariant6 from "tiny-invariant";
|
|
634
|
+
import { Event as Event3, sleep, synchronized as synchronized2, Trigger } from "@dxos/async";
|
|
635
|
+
import { Context as Context2, rejectOnDispose } from "@dxos/context";
|
|
636
|
+
import { failUndefined } from "@dxos/debug";
|
|
637
|
+
import { FeedSetIterator } from "@dxos/feed-store";
|
|
638
|
+
import { PublicKey as PublicKey3 } from "@dxos/keys";
|
|
639
|
+
import { log as log6 } from "@dxos/log";
|
|
640
|
+
import { Timeframe as Timeframe2 } from "@dxos/timeframe";
|
|
641
|
+
import { ComplexMap as ComplexMap3 } from "@dxos/util";
|
|
642
|
+
|
|
643
|
+
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
666
644
|
import invariant5 from "tiny-invariant";
|
|
667
|
-
import { runInContext, scheduleTask } from "@dxos/async";
|
|
668
|
-
import { Context as Context2 } from "@dxos/context";
|
|
669
|
-
import { randomBytes } from "@dxos/crypto";
|
|
670
645
|
import { log as log5 } from "@dxos/log";
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
},
|
|
680
|
-
exposed: {
|
|
681
|
-
AuthService: schema3.getService("dxos.mesh.teleport.auth.AuthService")
|
|
682
|
-
},
|
|
683
|
-
timeout: 60 * 1e3
|
|
684
|
-
});
|
|
685
|
-
this._authParams = _authParams;
|
|
686
|
-
this._ctx = new Context2({
|
|
687
|
-
onError: (err) => {
|
|
688
|
-
log5.catch(err, void 0, {
|
|
689
|
-
F: __dxlog_file5,
|
|
690
|
-
L: 29,
|
|
691
|
-
S: this,
|
|
692
|
-
C: (f, a) => f(...a)
|
|
693
|
-
});
|
|
646
|
+
var __dxlog_file5 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
647
|
+
var createMessageSelector = (timeframeClock) => {
|
|
648
|
+
return (messages) => {
|
|
649
|
+
for (let i = 0; i < messages.length; i++) {
|
|
650
|
+
const { data: { timeframe } } = messages[i];
|
|
651
|
+
invariant5(timeframe);
|
|
652
|
+
if (!timeframeClock.hasGaps(timeframe)) {
|
|
653
|
+
return i;
|
|
694
654
|
}
|
|
655
|
+
}
|
|
656
|
+
log5("Skipping...", void 0, {
|
|
657
|
+
F: __dxlog_file5,
|
|
658
|
+
L: 34,
|
|
659
|
+
S: void 0,
|
|
660
|
+
C: (f, a) => f(...a)
|
|
695
661
|
});
|
|
662
|
+
};
|
|
663
|
+
};
|
|
664
|
+
|
|
665
|
+
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
666
|
+
function _ts_decorate4(decorators, target, key, desc) {
|
|
667
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
668
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
669
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
670
|
+
else
|
|
671
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
672
|
+
if (d = decorators[i])
|
|
673
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
674
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
675
|
+
}
|
|
676
|
+
var __dxlog_file6 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
677
|
+
var PipelineState = class {
|
|
678
|
+
// prettier-ignore
|
|
679
|
+
constructor(_feeds, _timeframeClock) {
|
|
680
|
+
this._feeds = _feeds;
|
|
681
|
+
this._timeframeClock = _timeframeClock;
|
|
682
|
+
this._ctx = new Context2();
|
|
683
|
+
this.timeframeUpdate = this._timeframeClock.update;
|
|
684
|
+
this.stalled = new Event3();
|
|
685
|
+
this._startTimeframe = new Timeframe2();
|
|
696
686
|
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
credential
|
|
708
|
-
};
|
|
709
|
-
} catch (err) {
|
|
710
|
-
log5.error("failed to generate auth credentials", err, {
|
|
711
|
-
F: __dxlog_file5,
|
|
712
|
-
L: 56,
|
|
713
|
-
S: this,
|
|
714
|
-
C: (f, a) => f(...a)
|
|
715
|
-
});
|
|
716
|
-
throw new Error("auth rejected");
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
};
|
|
687
|
+
/**
|
|
688
|
+
* Latest theoretical timeframe based on the last mutation in each feed.
|
|
689
|
+
* NOTE: This might never be reached if the mutation dependencies
|
|
690
|
+
*/
|
|
691
|
+
// TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
|
|
692
|
+
get endTimeframe() {
|
|
693
|
+
return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
|
|
694
|
+
feedKey: feed.key,
|
|
695
|
+
index: feed.length - 1
|
|
696
|
+
})));
|
|
721
697
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
scheduleTask(this._ctx, async () => {
|
|
725
|
-
try {
|
|
726
|
-
const challenge = randomBytes(32);
|
|
727
|
-
const { credential } = await this.rpc.AuthService.authenticate({
|
|
728
|
-
challenge
|
|
729
|
-
});
|
|
730
|
-
invariant5((credential == null ? void 0 : credential.length) > 0, "invalid credential");
|
|
731
|
-
const success = await this._authParams.verifier(challenge, credential);
|
|
732
|
-
invariant5(success, "credential not verified");
|
|
733
|
-
runInContext(this._ctx, () => this._authParams.onAuthSuccess());
|
|
734
|
-
} catch (err) {
|
|
735
|
-
log5("auth failed", err, {
|
|
736
|
-
F: __dxlog_file5,
|
|
737
|
-
L: 75,
|
|
738
|
-
S: this,
|
|
739
|
-
C: (f, a) => f(...a)
|
|
740
|
-
});
|
|
741
|
-
this.close();
|
|
742
|
-
this._authParams.onAuthFailure();
|
|
743
|
-
}
|
|
744
|
-
});
|
|
698
|
+
get startTimeframe() {
|
|
699
|
+
return this._startTimeframe;
|
|
745
700
|
}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
await super.onClose();
|
|
701
|
+
get timeframe() {
|
|
702
|
+
return this._timeframeClock.timeframe;
|
|
749
703
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
import { Context as Context3 } from "@dxos/context";
|
|
756
|
-
import { tagMutationsInBatch, setMetadataOnObject } from "@dxos/echo-db";
|
|
757
|
-
import { log as log6 } from "@dxos/log";
|
|
758
|
-
import { ComplexMap as ComplexMap2 } from "@dxos/util";
|
|
759
|
-
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts";
|
|
760
|
-
var DataServiceHost = class {
|
|
761
|
-
constructor(_itemManager, _itemDemuxer, _writeStream) {
|
|
762
|
-
this._itemManager = _itemManager;
|
|
763
|
-
this._itemDemuxer = _itemDemuxer;
|
|
764
|
-
this._writeStream = _writeStream;
|
|
765
|
-
this._ctx = new Context3();
|
|
766
|
-
this._clientTagMap = new ComplexMap2(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
|
|
704
|
+
get pendingTimeframe() {
|
|
705
|
+
return this._timeframeClock.pendingTimeframe;
|
|
706
|
+
}
|
|
707
|
+
get targetTimeframe() {
|
|
708
|
+
return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
|
|
767
709
|
}
|
|
768
|
-
|
|
710
|
+
get feeds() {
|
|
711
|
+
return Array.from(this._feeds.values());
|
|
769
712
|
}
|
|
770
|
-
async
|
|
771
|
-
await this.
|
|
713
|
+
async waitUntilTimeframe(target) {
|
|
714
|
+
await this._timeframeClock.waitUntilReached(target);
|
|
715
|
+
}
|
|
716
|
+
setTargetTimeframe(target) {
|
|
717
|
+
this._targetTimeframe = target;
|
|
772
718
|
}
|
|
773
719
|
/**
|
|
774
|
-
*
|
|
720
|
+
* Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
|
|
721
|
+
*
|
|
722
|
+
* This function will resolve immediately if the pipeline is stalled.
|
|
723
|
+
*
|
|
724
|
+
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
775
725
|
*/
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
objects
|
|
783
|
-
}
|
|
784
|
-
});
|
|
785
|
-
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
786
|
-
var _a;
|
|
787
|
-
const { batch, meta } = message;
|
|
788
|
-
invariant6(!meta.clientTag, "Unexpected client tag in mutation message");
|
|
789
|
-
log6("message", {
|
|
790
|
-
batch,
|
|
791
|
-
meta
|
|
792
|
-
}, {
|
|
793
|
-
F: __dxlog_file6,
|
|
794
|
-
L: 63,
|
|
795
|
-
S: this,
|
|
796
|
-
C: (f, a) => f(...a)
|
|
797
|
-
});
|
|
798
|
-
const clientTag = this._clientTagMap.get([
|
|
799
|
-
message.meta.feedKey,
|
|
800
|
-
message.meta.seq
|
|
801
|
-
]);
|
|
802
|
-
(_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
|
|
803
|
-
setMetadataOnObject(object, {
|
|
804
|
-
...meta
|
|
805
|
-
});
|
|
806
|
-
});
|
|
807
|
-
if (clientTag) {
|
|
808
|
-
tagMutationsInBatch(batch, clientTag, 0);
|
|
809
|
-
}
|
|
810
|
-
next({
|
|
811
|
-
clientTag,
|
|
812
|
-
feedKey: message.meta.feedKey,
|
|
813
|
-
seq: message.meta.seq,
|
|
814
|
-
batch
|
|
815
|
-
});
|
|
816
|
-
});
|
|
817
|
-
});
|
|
818
|
-
}
|
|
819
|
-
async write(request) {
|
|
820
|
-
var _a, _b;
|
|
821
|
-
invariant6(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
|
|
822
|
-
invariant6(this._writeStream, "Cannot write mutations in readonly mode");
|
|
823
|
-
log6("write", {
|
|
824
|
-
clientTag: request.clientTag,
|
|
825
|
-
objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
|
|
726
|
+
async waitUntilReachedTargetTimeframe({ ctx = new Context2(), timeout, breakOnStall = true } = {}) {
|
|
727
|
+
var _a;
|
|
728
|
+
log6("waitUntilReachedTargetTimeframe", {
|
|
729
|
+
timeout,
|
|
730
|
+
current: this.timeframe,
|
|
731
|
+
target: this.targetTimeframe
|
|
826
732
|
}, {
|
|
827
733
|
F: __dxlog_file6,
|
|
828
|
-
L:
|
|
734
|
+
L: 126,
|
|
829
735
|
S: this,
|
|
830
736
|
C: (f, a) => f(...a)
|
|
831
737
|
});
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
738
|
+
(_a = this._reachedTargetPromise) != null ? _a : this._reachedTargetPromise = Promise.race([
|
|
739
|
+
this._timeframeClock.update.waitForCondition(() => {
|
|
740
|
+
return Timeframe2.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
741
|
+
}),
|
|
742
|
+
...breakOnStall ? [
|
|
743
|
+
this.stalled.discardParameter().waitForCount(1)
|
|
744
|
+
] : []
|
|
745
|
+
]);
|
|
746
|
+
let done = false;
|
|
747
|
+
if (timeout) {
|
|
748
|
+
return Promise.race([
|
|
749
|
+
rejectOnDispose(ctx),
|
|
750
|
+
rejectOnDispose(this._ctx),
|
|
751
|
+
this._reachedTargetPromise.then(() => {
|
|
752
|
+
done = true;
|
|
753
|
+
}),
|
|
754
|
+
sleep(timeout).then(() => {
|
|
755
|
+
if (done) {
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
log6.warn("waitUntilReachedTargetTimeframe timed out", {
|
|
759
|
+
timeout,
|
|
760
|
+
current: this.timeframe,
|
|
761
|
+
target: this.targetTimeframe,
|
|
762
|
+
dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
|
|
840
763
|
}, {
|
|
841
764
|
F: __dxlog_file6,
|
|
842
|
-
L:
|
|
765
|
+
L: 153,
|
|
843
766
|
S: this,
|
|
844
767
|
C: (f, a) => f(...a)
|
|
845
768
|
});
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
});
|
|
853
|
-
return receipt;
|
|
854
|
-
}
|
|
855
|
-
};
|
|
856
|
-
var createDataMessage = (batch) => {
|
|
857
|
-
var _a;
|
|
858
|
-
return {
|
|
859
|
-
batch: {
|
|
860
|
-
objects: (_a = batch.objects) == null ? void 0 : _a.map((object) => {
|
|
861
|
-
var _a2;
|
|
862
|
-
return {
|
|
863
|
-
...object,
|
|
864
|
-
mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((mutation) => ({
|
|
865
|
-
...mutation,
|
|
866
|
-
meta: void 0
|
|
867
|
-
})),
|
|
868
|
-
meta: void 0
|
|
869
|
-
};
|
|
870
|
-
})
|
|
769
|
+
})
|
|
770
|
+
]);
|
|
771
|
+
} else {
|
|
772
|
+
return this._reachedTargetPromise;
|
|
871
773
|
}
|
|
872
|
-
};
|
|
873
|
-
};
|
|
874
|
-
|
|
875
|
-
// packages/core/echo/echo-pipeline/src/dbhost/database-host.ts
|
|
876
|
-
import { ItemDemuxer } from "@dxos/echo-db";
|
|
877
|
-
var DatabaseHost = class {
|
|
878
|
-
constructor(_outboundStream) {
|
|
879
|
-
this._outboundStream = _outboundStream;
|
|
880
774
|
}
|
|
881
|
-
|
|
882
|
-
|
|
775
|
+
};
|
|
776
|
+
var Pipeline = class {
|
|
777
|
+
constructor() {
|
|
778
|
+
this._timeframeClock = new TimeframeClock(new Timeframe2());
|
|
779
|
+
this._feeds = new ComplexMap3(PublicKey3.hash);
|
|
780
|
+
// External state accessor.
|
|
781
|
+
this._state = new PipelineState(this._feeds, this._timeframeClock);
|
|
782
|
+
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
783
|
+
this._processingTrigger = new Trigger().wake();
|
|
784
|
+
this._pauseTrigger = new Trigger().wake();
|
|
785
|
+
this._isStopping = false;
|
|
786
|
+
this._isStarted = false;
|
|
787
|
+
this._isOpen = false;
|
|
788
|
+
this._isPaused = false;
|
|
883
789
|
}
|
|
884
|
-
get
|
|
885
|
-
return this.
|
|
790
|
+
get state() {
|
|
791
|
+
return this._state;
|
|
886
792
|
}
|
|
887
|
-
|
|
888
|
-
this.
|
|
889
|
-
this.
|
|
890
|
-
this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
|
|
891
|
-
this._echoProcessor = this._itemDemuxer.open();
|
|
793
|
+
get writer() {
|
|
794
|
+
invariant6(this._writer, "Writer not set.");
|
|
795
|
+
return this._writer;
|
|
892
796
|
}
|
|
893
|
-
|
|
797
|
+
hasFeed(feedKey) {
|
|
798
|
+
return this._feeds.has(feedKey);
|
|
894
799
|
}
|
|
895
|
-
|
|
896
|
-
return this.
|
|
800
|
+
getFeeds() {
|
|
801
|
+
return this._feedSetIterator.feeds;
|
|
897
802
|
}
|
|
898
|
-
|
|
899
|
-
|
|
803
|
+
// NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
|
|
804
|
+
// which might be opening feeds during the mutation processing, which w
|
|
805
|
+
async addFeed(feed) {
|
|
806
|
+
this._feeds.set(feed.key, feed);
|
|
807
|
+
if (this._feedSetIterator) {
|
|
808
|
+
await this._feedSetIterator.addFeed(feed);
|
|
809
|
+
}
|
|
810
|
+
this._setFeedDownloadState(feed);
|
|
900
811
|
}
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
812
|
+
setWriteFeed(feed) {
|
|
813
|
+
invariant6(!this._writer, "Writer already set.");
|
|
814
|
+
invariant6(feed.properties.writable, "Feed must be writable.");
|
|
815
|
+
this._writer = createMappedFeedWriter((payload) => ({
|
|
816
|
+
timeframe: this._timeframeClock.timeframe,
|
|
817
|
+
payload
|
|
818
|
+
}), feed.createFeedWriter());
|
|
904
819
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
922
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
923
|
-
}
|
|
924
|
-
var SpaceSnapshot = schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
925
|
-
var SnapshotManager = class SnapshotManager2 {
|
|
926
|
-
// prettier-ignore
|
|
927
|
-
constructor(_snapshotStore, _blobStore, _blobSync) {
|
|
928
|
-
this._snapshotStore = _snapshotStore;
|
|
929
|
-
this._blobStore = _blobStore;
|
|
930
|
-
this._blobSync = _blobSync;
|
|
820
|
+
async start() {
|
|
821
|
+
log6("starting...", void 0, {
|
|
822
|
+
F: __dxlog_file6,
|
|
823
|
+
L: 268,
|
|
824
|
+
S: this,
|
|
825
|
+
C: (f, a) => f(...a)
|
|
826
|
+
});
|
|
827
|
+
await this._initIterator();
|
|
828
|
+
await this._feedSetIterator.open();
|
|
829
|
+
this._isStarted = true;
|
|
830
|
+
log6("started", void 0, {
|
|
831
|
+
F: __dxlog_file6,
|
|
832
|
+
L: 272,
|
|
833
|
+
S: this,
|
|
834
|
+
C: (f, a) => f(...a)
|
|
835
|
+
});
|
|
931
836
|
}
|
|
932
|
-
async
|
|
933
|
-
|
|
934
|
-
|
|
837
|
+
async stop() {
|
|
838
|
+
var _a;
|
|
839
|
+
log6("stopping...", void 0, {
|
|
840
|
+
F: __dxlog_file6,
|
|
841
|
+
L: 277,
|
|
842
|
+
S: this,
|
|
843
|
+
C: (f, a) => f(...a)
|
|
844
|
+
});
|
|
845
|
+
this._isStopping = true;
|
|
846
|
+
await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
|
|
847
|
+
await this._processingTrigger.wait();
|
|
848
|
+
await this._state._ctx.dispose();
|
|
849
|
+
this._state._ctx = new Context2();
|
|
850
|
+
this._state._reachedTargetPromise = void 0;
|
|
851
|
+
this._isStarted = false;
|
|
852
|
+
log6("stopped", void 0, {
|
|
853
|
+
F: __dxlog_file6,
|
|
854
|
+
L: 285,
|
|
855
|
+
S: this,
|
|
856
|
+
C: (f, a) => f(...a)
|
|
857
|
+
});
|
|
935
858
|
}
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
859
|
+
/**
|
|
860
|
+
* @param timeframe Timeframe of already processed messages.
|
|
861
|
+
* The pipeline will start processing messages AFTER this timeframe.
|
|
862
|
+
*/
|
|
863
|
+
async setCursor(timeframe) {
|
|
864
|
+
invariant6(!this._isStarted || this._isPaused, "Invalid state.");
|
|
865
|
+
this._state._startTimeframe = timeframe;
|
|
866
|
+
this._timeframeClock.setTimeframe(timeframe);
|
|
867
|
+
for (const feed of this._feeds.values()) {
|
|
868
|
+
this._setFeedDownloadState(feed);
|
|
941
869
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
870
|
+
if (this._feedSetIterator) {
|
|
871
|
+
await this._feedSetIterator.close();
|
|
872
|
+
await this._initIterator();
|
|
873
|
+
await this._feedSetIterator.open();
|
|
945
874
|
}
|
|
946
|
-
await this._blobSync.download(ctx, blobId);
|
|
947
|
-
return this._getBlob(blobId);
|
|
948
|
-
}
|
|
949
|
-
async store(snapshot) {
|
|
950
|
-
const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
|
|
951
|
-
await this._blobSync.notifyBlobAdded(id);
|
|
952
|
-
return PublicKey2.from(id).toHex();
|
|
953
|
-
}
|
|
954
|
-
};
|
|
955
|
-
_ts_decorate4([
|
|
956
|
-
timed2(1e4)
|
|
957
|
-
], SnapshotManager.prototype, "load", null);
|
|
958
|
-
SnapshotManager = _ts_decorate4([
|
|
959
|
-
trackLeaks("open", "close")
|
|
960
|
-
], SnapshotManager);
|
|
961
|
-
|
|
962
|
-
// packages/core/echo/echo-pipeline/src/dbhost/snapshot-store.ts
|
|
963
|
-
import { subtleCrypto } from "@dxos/crypto";
|
|
964
|
-
import { schema as schema5 } from "@dxos/protocols";
|
|
965
|
-
var SpaceSnapshot2 = schema5.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
966
|
-
var SnapshotStore = class {
|
|
967
|
-
// prettier-ignore
|
|
968
|
-
constructor(_directory) {
|
|
969
|
-
this._directory = _directory;
|
|
970
875
|
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
} finally {
|
|
979
|
-
await file.close();
|
|
876
|
+
/**
|
|
877
|
+
* Calling pause while processing will cause a deadlock.
|
|
878
|
+
*/
|
|
879
|
+
async pause() {
|
|
880
|
+
invariant6(this._isStarted, "Pipeline is not open.");
|
|
881
|
+
if (this._isPaused) {
|
|
882
|
+
return;
|
|
980
883
|
}
|
|
981
|
-
|
|
884
|
+
this._pauseTrigger.reset();
|
|
885
|
+
await this._processingTrigger.wait();
|
|
886
|
+
this._isPaused = true;
|
|
982
887
|
}
|
|
983
|
-
async
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
888
|
+
async unpause() {
|
|
889
|
+
invariant6(this._isStarted, "Pipeline is not open.");
|
|
890
|
+
invariant6(this._isPaused, "Pipeline is not paused.");
|
|
891
|
+
this._pauseTrigger.wake();
|
|
892
|
+
this._isPaused = false;
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Starts to iterate over the ordered messages from the added feeds.
|
|
896
|
+
* Updates the timeframe clock after the message has bee processed.
|
|
897
|
+
*/
|
|
898
|
+
async *consume() {
|
|
899
|
+
invariant6(!this._isOpen, "Pipeline is already being consumed.");
|
|
900
|
+
this._isOpen = true;
|
|
901
|
+
invariant6(this._feedSetIterator, "Iterator not initialized.");
|
|
902
|
+
let lastFeedSetIterator = this._feedSetIterator;
|
|
903
|
+
let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
904
|
+
while (!this._isStopping) {
|
|
905
|
+
await this._pauseTrigger.wait();
|
|
906
|
+
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
907
|
+
invariant6(this._feedSetIterator, "Iterator not initialized.");
|
|
908
|
+
lastFeedSetIterator = this._feedSetIterator;
|
|
909
|
+
iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
910
|
+
}
|
|
911
|
+
const { done, value } = await iterable.next();
|
|
912
|
+
if (!done) {
|
|
913
|
+
const block = value != null ? value : failUndefined();
|
|
914
|
+
this._processingTrigger.reset();
|
|
915
|
+
this._timeframeClock.updatePendingTimeframe(PublicKey3.from(block.feedKey), block.seq);
|
|
916
|
+
yield block;
|
|
917
|
+
this._processingTrigger.wake();
|
|
918
|
+
this._timeframeClock.updateTimeframe();
|
|
989
919
|
}
|
|
990
|
-
const buffer = await file.read(0, size);
|
|
991
|
-
return SpaceSnapshot2.decode(buffer);
|
|
992
|
-
} finally {
|
|
993
|
-
await file.close();
|
|
994
920
|
}
|
|
921
|
+
this._isOpen = false;
|
|
995
922
|
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
923
|
+
_setFeedDownloadState(feed) {
|
|
924
|
+
var _a;
|
|
925
|
+
const timeframe = this._state._startTimeframe;
|
|
926
|
+
const seq = (_a = timeframe.get(feed.key)) != null ? _a : 0;
|
|
927
|
+
feed.undownload({
|
|
928
|
+
callback: () => log6("undownload", void 0, {
|
|
929
|
+
F: __dxlog_file6,
|
|
930
|
+
L: 376,
|
|
931
|
+
S: this,
|
|
932
|
+
C: (f, a) => f(...a)
|
|
933
|
+
})
|
|
934
|
+
});
|
|
935
|
+
feed.download({
|
|
936
|
+
start: seq + 1,
|
|
937
|
+
linear: true
|
|
938
|
+
}).catch((err) => {
|
|
939
|
+
log6("failed to download feed", {
|
|
940
|
+
err
|
|
941
|
+
}, {
|
|
942
|
+
F: __dxlog_file6,
|
|
943
|
+
L: 378,
|
|
944
|
+
S: this,
|
|
945
|
+
C: (f, a) => f(...a)
|
|
946
|
+
});
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
async _initIterator() {
|
|
950
|
+
this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
951
|
+
start: startAfter(this._timeframeClock.timeframe),
|
|
952
|
+
stallTimeout: 1e3
|
|
953
|
+
});
|
|
954
|
+
this._feedSetIterator.stalled.on((iterator) => {
|
|
955
|
+
log6.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
956
|
+
F: __dxlog_file6,
|
|
957
|
+
L: 389,
|
|
958
|
+
S: this,
|
|
959
|
+
C: (f, a) => f(...a)
|
|
960
|
+
});
|
|
961
|
+
this._state.stalled.emit();
|
|
962
|
+
});
|
|
963
|
+
for (const feed of this._feeds.values()) {
|
|
964
|
+
await this._feedSetIterator.addFeed(feed);
|
|
965
|
+
}
|
|
1005
966
|
}
|
|
1006
967
|
};
|
|
968
|
+
_ts_decorate4([
|
|
969
|
+
synchronized2
|
|
970
|
+
], Pipeline.prototype, "start", null);
|
|
971
|
+
_ts_decorate4([
|
|
972
|
+
synchronized2
|
|
973
|
+
], Pipeline.prototype, "stop", null);
|
|
974
|
+
_ts_decorate4([
|
|
975
|
+
synchronized2
|
|
976
|
+
], Pipeline.prototype, "setCursor", null);
|
|
977
|
+
_ts_decorate4([
|
|
978
|
+
synchronized2
|
|
979
|
+
], Pipeline.prototype, "pause", null);
|
|
980
|
+
_ts_decorate4([
|
|
981
|
+
synchronized2
|
|
982
|
+
], Pipeline.prototype, "unpause", null);
|
|
1007
983
|
|
|
1008
|
-
// packages/core/echo/echo-pipeline/src/
|
|
984
|
+
// packages/core/echo/echo-pipeline/src/space/auth.ts
|
|
1009
985
|
import invariant7 from "tiny-invariant";
|
|
1010
|
-
import {
|
|
1011
|
-
import {
|
|
986
|
+
import { runInContext, scheduleTask } from "@dxos/async";
|
|
987
|
+
import { Context as Context3 } from "@dxos/context";
|
|
988
|
+
import { randomBytes } from "@dxos/crypto";
|
|
1012
989
|
import { log as log7 } from "@dxos/log";
|
|
1013
|
-
import {
|
|
1014
|
-
|
|
1015
|
-
var
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
F: __dxlog_file7,
|
|
1027
|
-
L: 31,
|
|
1028
|
-
S: this,
|
|
1029
|
-
C: (f, a) => f(...a)
|
|
990
|
+
import { schema as schema5 } from "@dxos/protocols";
|
|
991
|
+
import { RpcExtension } from "@dxos/teleport";
|
|
992
|
+
var __dxlog_file7 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/auth.ts";
|
|
993
|
+
var AuthExtension = class extends RpcExtension {
|
|
994
|
+
constructor(_authParams) {
|
|
995
|
+
super({
|
|
996
|
+
requested: {
|
|
997
|
+
AuthService: schema5.getService("dxos.mesh.teleport.auth.AuthService")
|
|
998
|
+
},
|
|
999
|
+
exposed: {
|
|
1000
|
+
AuthService: schema5.getService("dxos.mesh.teleport.auth.AuthService")
|
|
1001
|
+
},
|
|
1002
|
+
timeout: 60 * 1e3
|
|
1030
1003
|
});
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
S: this,
|
|
1042
|
-
C: (f, a) => f(...a)
|
|
1004
|
+
this._authParams = _authParams;
|
|
1005
|
+
this._ctx = new Context3({
|
|
1006
|
+
onError: (err) => {
|
|
1007
|
+
log7.catch(err, void 0, {
|
|
1008
|
+
F: __dxlog_file7,
|
|
1009
|
+
L: 29,
|
|
1010
|
+
S: this,
|
|
1011
|
+
C: (f, a) => f(...a)
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1043
1014
|
});
|
|
1044
|
-
const host = this._spaces.get(spaceKey);
|
|
1045
|
-
await (host == null ? void 0 : host.close());
|
|
1046
|
-
this._spaces.delete(spaceKey);
|
|
1047
|
-
}
|
|
1048
|
-
getDataService(spaceKey) {
|
|
1049
|
-
return this._spaces.get(spaceKey);
|
|
1050
1015
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1016
|
+
async getHandlers() {
|
|
1017
|
+
return {
|
|
1018
|
+
AuthService: {
|
|
1019
|
+
authenticate: async ({ challenge }) => {
|
|
1020
|
+
try {
|
|
1021
|
+
const credential = await this._authParams.provider(challenge);
|
|
1022
|
+
if (!credential) {
|
|
1023
|
+
throw new Error("auth rejected");
|
|
1024
|
+
}
|
|
1025
|
+
return {
|
|
1026
|
+
credential
|
|
1027
|
+
};
|
|
1028
|
+
} catch (err) {
|
|
1029
|
+
log7.error("failed to generate auth credentials", err, {
|
|
1030
|
+
F: __dxlog_file7,
|
|
1031
|
+
L: 56,
|
|
1032
|
+
S: this,
|
|
1033
|
+
C: (f, a) => f(...a)
|
|
1034
|
+
});
|
|
1035
|
+
throw new Error("auth rejected");
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
};
|
|
1055
1040
|
}
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1041
|
+
async onOpen(context) {
|
|
1042
|
+
await super.onOpen(context);
|
|
1043
|
+
scheduleTask(this._ctx, async () => {
|
|
1044
|
+
try {
|
|
1045
|
+
const challenge = randomBytes(32);
|
|
1046
|
+
const { credential } = await this.rpc.AuthService.authenticate({
|
|
1047
|
+
challenge
|
|
1048
|
+
});
|
|
1049
|
+
invariant7((credential == null ? void 0 : credential.length) > 0, "invalid credential");
|
|
1050
|
+
const success = await this._authParams.verifier(challenge, credential);
|
|
1051
|
+
invariant7(success, "credential not verified");
|
|
1052
|
+
runInContext(this._ctx, () => this._authParams.onAuthSuccess());
|
|
1053
|
+
} catch (err) {
|
|
1054
|
+
log7("auth failed", err, {
|
|
1055
|
+
F: __dxlog_file7,
|
|
1056
|
+
L: 75,
|
|
1057
|
+
S: this,
|
|
1058
|
+
C: (f, a) => f(...a)
|
|
1059
|
+
});
|
|
1060
|
+
this.close();
|
|
1061
|
+
this._authParams.onAuthFailure();
|
|
1062
|
+
}
|
|
1063
|
+
});
|
|
1061
1064
|
}
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
invariant7(request.batch);
|
|
1066
|
-
const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
|
|
1067
|
-
return host.write(request);
|
|
1065
|
+
async onClose() {
|
|
1066
|
+
await this._ctx.dispose();
|
|
1067
|
+
await super.onClose();
|
|
1068
1068
|
}
|
|
1069
1069
|
};
|
|
1070
1070
|
|
|
@@ -1077,6 +1077,7 @@ import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
|
|
|
1077
1077
|
import { CancelledError } from "@dxos/errors";
|
|
1078
1078
|
import { log as log8 } from "@dxos/log";
|
|
1079
1079
|
import { Timeframe as Timeframe3 } from "@dxos/timeframe";
|
|
1080
|
+
import { tracer } from "@dxos/util";
|
|
1080
1081
|
function _ts_decorate5(decorators, target, key, desc) {
|
|
1081
1082
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1082
1083
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -1087,7 +1088,7 @@ function _ts_decorate5(decorators, target, key, desc) {
|
|
|
1087
1088
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1088
1089
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1089
1090
|
}
|
|
1090
|
-
var __dxlog_file8 = "/
|
|
1091
|
+
var __dxlog_file8 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
|
|
1091
1092
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1092
1093
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1093
1094
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
@@ -1101,9 +1102,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1101
1102
|
this._isOpen = false;
|
|
1102
1103
|
this._lastTimeframeSaveTime = 0;
|
|
1103
1104
|
this._lastSnapshotSaveTime = 0;
|
|
1105
|
+
this._lastProcessedEpoch = -1;
|
|
1104
1106
|
this.currentEpoch = void 0;
|
|
1105
1107
|
this.appliedEpoch = void 0;
|
|
1106
|
-
this._lastProcessedEpoch = -1;
|
|
1107
1108
|
this.onNewEpoch = new Event4();
|
|
1108
1109
|
}
|
|
1109
1110
|
get isOpen() {
|
|
@@ -1218,7 +1219,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1218
1219
|
if (data.payload.data) {
|
|
1219
1220
|
const feedInfo = this._params.feedInfoProvider(feedKey);
|
|
1220
1221
|
if (!feedInfo) {
|
|
1221
|
-
log8.
|
|
1222
|
+
log8.warn("Could not find feed", {
|
|
1222
1223
|
feedKey
|
|
1223
1224
|
}, {
|
|
1224
1225
|
F: __dxlog_file8,
|
|
@@ -1228,7 +1229,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1228
1229
|
});
|
|
1229
1230
|
continue;
|
|
1230
1231
|
}
|
|
1231
|
-
|
|
1232
|
+
const timer = tracer.mark("dxos.echo.pipeline.data");
|
|
1233
|
+
this.databaseHost.echoProcessor({
|
|
1232
1234
|
batch: data.payload.data.batch,
|
|
1233
1235
|
meta: {
|
|
1234
1236
|
feedKey,
|
|
@@ -1237,13 +1239,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1237
1239
|
memberKey: feedInfo.assertion.identityKey
|
|
1238
1240
|
}
|
|
1239
1241
|
});
|
|
1242
|
+
timer.end();
|
|
1240
1243
|
log8.trace("dxos.echo.data-pipeline.processed", {
|
|
1241
1244
|
feedKey: feedKey.toHex(),
|
|
1242
1245
|
seq,
|
|
1243
1246
|
spaceKey: this._params.spaceKey.toHex()
|
|
1244
1247
|
}, {
|
|
1245
1248
|
F: __dxlog_file8,
|
|
1246
|
-
L:
|
|
1249
|
+
L: 233,
|
|
1247
1250
|
S: this,
|
|
1248
1251
|
C: (f, a) => f(...a)
|
|
1249
1252
|
});
|
|
@@ -1252,7 +1255,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1252
1255
|
} catch (err) {
|
|
1253
1256
|
log8.catch(err, void 0, {
|
|
1254
1257
|
F: __dxlog_file8,
|
|
1255
|
-
L:
|
|
1258
|
+
L: 243,
|
|
1256
1259
|
S: this,
|
|
1257
1260
|
C: (f, a) => f(...a)
|
|
1258
1261
|
});
|
|
@@ -1287,7 +1290,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1287
1290
|
} catch (err) {
|
|
1288
1291
|
log8.warn("Failed to cache properties", err, {
|
|
1289
1292
|
F: __dxlog_file8,
|
|
1290
|
-
L:
|
|
1293
|
+
L: 278,
|
|
1291
1294
|
S: this,
|
|
1292
1295
|
C: (f, a) => f(...a)
|
|
1293
1296
|
});
|
|
@@ -1314,14 +1317,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1314
1317
|
if (err instanceof CancelledError) {
|
|
1315
1318
|
log8("Epoch processing cancelled.", void 0, {
|
|
1316
1319
|
F: __dxlog_file8,
|
|
1317
|
-
L:
|
|
1320
|
+
L: 310,
|
|
1318
1321
|
S: this,
|
|
1319
1322
|
C: (f, a) => f(...a)
|
|
1320
1323
|
});
|
|
1321
1324
|
} else {
|
|
1322
1325
|
log8.catch(err, void 0, {
|
|
1323
1326
|
F: __dxlog_file8,
|
|
1324
|
-
L:
|
|
1327
|
+
L: 312,
|
|
1325
1328
|
S: this,
|
|
1326
1329
|
C: (f, a) => f(...a)
|
|
1327
1330
|
});
|
|
@@ -1337,7 +1340,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1337
1340
|
epoch
|
|
1338
1341
|
}, {
|
|
1339
1342
|
F: __dxlog_file8,
|
|
1340
|
-
L:
|
|
1343
|
+
L: 322,
|
|
1341
1344
|
S: this,
|
|
1342
1345
|
C: (f, a) => f(...a)
|
|
1343
1346
|
});
|
|
@@ -1354,7 +1357,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1354
1357
|
epoch
|
|
1355
1358
|
}, {
|
|
1356
1359
|
F: __dxlog_file8,
|
|
1357
|
-
L:
|
|
1360
|
+
L: 336,
|
|
1358
1361
|
S: this,
|
|
1359
1362
|
C: (f, a) => f(...a)
|
|
1360
1363
|
});
|
|
@@ -1364,7 +1367,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1364
1367
|
}
|
|
1365
1368
|
log8("restarting pipeline for epoch", void 0, {
|
|
1366
1369
|
F: __dxlog_file8,
|
|
1367
|
-
L:
|
|
1370
|
+
L: 344,
|
|
1368
1371
|
S: this,
|
|
1369
1372
|
C: (f, a) => f(...a)
|
|
1370
1373
|
});
|
|
@@ -1424,8 +1427,8 @@ import { PublicKey as PublicKey4 } from "@dxos/keys";
|
|
|
1424
1427
|
import { log as log9 } from "@dxos/log";
|
|
1425
1428
|
import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
|
|
1426
1429
|
import { Timeframe as Timeframe4 } from "@dxos/timeframe";
|
|
1427
|
-
import { Callback } from "@dxos/util";
|
|
1428
|
-
var __dxlog_file9 = "/
|
|
1430
|
+
import { Callback, tracer as tracer2 } from "@dxos/util";
|
|
1431
|
+
var __dxlog_file9 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1429
1432
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
1430
1433
|
var ControlPipeline = class {
|
|
1431
1434
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
@@ -1492,13 +1495,15 @@ var ControlPipeline = class {
|
|
|
1492
1495
|
C: (f, a) => f(...a)
|
|
1493
1496
|
});
|
|
1494
1497
|
if (msg.data.payload.credential) {
|
|
1498
|
+
const timer = tracer2.mark("dxos.echo.pipeline.control");
|
|
1495
1499
|
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
|
|
1500
|
+
timer.end();
|
|
1496
1501
|
if (!result) {
|
|
1497
1502
|
log9.warn("processing failed", {
|
|
1498
1503
|
msg
|
|
1499
1504
|
}, {
|
|
1500
1505
|
F: __dxlog_file9,
|
|
1501
|
-
L:
|
|
1506
|
+
L: 98,
|
|
1502
1507
|
S: this,
|
|
1503
1508
|
C: (f, a) => f(...a)
|
|
1504
1509
|
});
|
|
@@ -1509,7 +1514,7 @@ var ControlPipeline = class {
|
|
|
1509
1514
|
} catch (err) {
|
|
1510
1515
|
log9.catch(err, void 0, {
|
|
1511
1516
|
F: __dxlog_file9,
|
|
1512
|
-
L:
|
|
1517
|
+
L: 104,
|
|
1513
1518
|
S: this,
|
|
1514
1519
|
C: (f, a) => f(...a)
|
|
1515
1520
|
});
|
|
@@ -1519,7 +1524,7 @@ var ControlPipeline = class {
|
|
|
1519
1524
|
await this._pipeline.start();
|
|
1520
1525
|
log9("started", void 0, {
|
|
1521
1526
|
F: __dxlog_file9,
|
|
1522
|
-
L:
|
|
1527
|
+
L: 110,
|
|
1523
1528
|
S: this,
|
|
1524
1529
|
C: (f, a) => f(...a)
|
|
1525
1530
|
});
|
|
@@ -1533,7 +1538,7 @@ var ControlPipeline = class {
|
|
|
1533
1538
|
async stop() {
|
|
1534
1539
|
log9("stopping...", void 0, {
|
|
1535
1540
|
F: __dxlog_file9,
|
|
1536
|
-
L:
|
|
1541
|
+
L: 124,
|
|
1537
1542
|
S: this,
|
|
1538
1543
|
C: (f, a) => f(...a)
|
|
1539
1544
|
});
|
|
@@ -1541,7 +1546,7 @@ var ControlPipeline = class {
|
|
|
1541
1546
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1542
1547
|
log9("stopped", void 0, {
|
|
1543
1548
|
F: __dxlog_file9,
|
|
1544
|
-
L:
|
|
1549
|
+
L: 127,
|
|
1545
1550
|
S: this,
|
|
1546
1551
|
C: (f, a) => f(...a)
|
|
1547
1552
|
});
|
|
@@ -1555,7 +1560,7 @@ var ControlPipeline = class {
|
|
|
1555
1560
|
} catch (err) {
|
|
1556
1561
|
log9(err, void 0, {
|
|
1557
1562
|
F: __dxlog_file9,
|
|
1558
|
-
L:
|
|
1563
|
+
L: 136,
|
|
1559
1564
|
S: this,
|
|
1560
1565
|
C: (f, a) => f(...a)
|
|
1561
1566
|
});
|
|
@@ -1574,7 +1579,7 @@ function _ts_decorate6(decorators, target, key, desc) {
|
|
|
1574
1579
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1575
1580
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1576
1581
|
}
|
|
1577
|
-
var __dxlog_file10 = "/
|
|
1582
|
+
var __dxlog_file10 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space.ts";
|
|
1578
1583
|
var Space = class Space2 {
|
|
1579
1584
|
constructor(params) {
|
|
1580
1585
|
this._addFeedLock = new Lock();
|
|
@@ -1797,7 +1802,7 @@ function _ts_decorate7(decorators, target, key, desc) {
|
|
|
1797
1802
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1798
1803
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1799
1804
|
}
|
|
1800
|
-
var __dxlog_file11 = "/
|
|
1805
|
+
var __dxlog_file11 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
|
|
1801
1806
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
1802
1807
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
1803
1808
|
var SpaceProtocol = class {
|
|
@@ -1828,7 +1833,7 @@ var SpaceProtocol = class {
|
|
|
1828
1833
|
key: feed.key
|
|
1829
1834
|
}, {
|
|
1830
1835
|
F: __dxlog_file11,
|
|
1831
|
-
L:
|
|
1836
|
+
L: 95,
|
|
1832
1837
|
S: this,
|
|
1833
1838
|
C: (f, a) => f(...a)
|
|
1834
1839
|
});
|
|
@@ -1850,7 +1855,7 @@ var SpaceProtocol = class {
|
|
|
1850
1855
|
await this.blobSync.open();
|
|
1851
1856
|
log11("starting...", void 0, {
|
|
1852
1857
|
F: __dxlog_file11,
|
|
1853
|
-
L:
|
|
1858
|
+
L: 120,
|
|
1854
1859
|
S: this,
|
|
1855
1860
|
C: (f, a) => f(...a)
|
|
1856
1861
|
});
|
|
@@ -1863,7 +1868,7 @@ var SpaceProtocol = class {
|
|
|
1863
1868
|
});
|
|
1864
1869
|
log11("started", void 0, {
|
|
1865
1870
|
F: __dxlog_file11,
|
|
1866
|
-
L:
|
|
1871
|
+
L: 129,
|
|
1867
1872
|
S: this,
|
|
1868
1873
|
C: (f, a) => f(...a)
|
|
1869
1874
|
});
|
|
@@ -1873,14 +1878,14 @@ var SpaceProtocol = class {
|
|
|
1873
1878
|
if (this._connection) {
|
|
1874
1879
|
log11("stopping...", void 0, {
|
|
1875
1880
|
F: __dxlog_file11,
|
|
1876
|
-
L:
|
|
1881
|
+
L: 136,
|
|
1877
1882
|
S: this,
|
|
1878
1883
|
C: (f, a) => f(...a)
|
|
1879
1884
|
});
|
|
1880
1885
|
await this._connection.close();
|
|
1881
1886
|
log11("stopped", void 0, {
|
|
1882
1887
|
F: __dxlog_file11,
|
|
1883
|
-
L:
|
|
1888
|
+
L: 138,
|
|
1884
1889
|
S: this,
|
|
1885
1890
|
C: (f, a) => f(...a)
|
|
1886
1891
|
});
|
|
@@ -1948,7 +1953,7 @@ var SpaceProtocolSession = class {
|
|
|
1948
1953
|
var _a;
|
|
1949
1954
|
log11("Peer authenticated", void 0, {
|
|
1950
1955
|
F: __dxlog_file11,
|
|
1951
|
-
L:
|
|
1956
|
+
L: 235,
|
|
1952
1957
|
S: this,
|
|
1953
1958
|
C: (f, a) => f(...a)
|
|
1954
1959
|
});
|
|
@@ -1992,7 +1997,7 @@ function _ts_decorate8(decorators, target, key, desc) {
|
|
|
1992
1997
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1993
1998
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1994
1999
|
}
|
|
1995
|
-
var __dxlog_file12 = "/
|
|
2000
|
+
var __dxlog_file12 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
|
|
1996
2001
|
var SpaceManager = class SpaceManager2 {
|
|
1997
2002
|
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
1998
2003
|
this._spaces = new ComplexMap5(PublicKey6.hash);
|
|
@@ -2077,22 +2082,22 @@ SpaceManager = _ts_decorate8([
|
|
|
2077
2082
|
], SpaceManager);
|
|
2078
2083
|
|
|
2079
2084
|
export {
|
|
2080
|
-
MetadataStore,
|
|
2081
2085
|
codec,
|
|
2082
2086
|
valueEncoding,
|
|
2083
2087
|
createMappedFeedWriter,
|
|
2084
|
-
mapTimeframeToFeedIndexes,
|
|
2085
|
-
mapFeedIndexesToTimeframe,
|
|
2086
|
-
startAfter,
|
|
2087
|
-
TimeframeClock,
|
|
2088
|
-
Pipeline,
|
|
2089
|
-
AuthExtension,
|
|
2090
2088
|
DataServiceHost,
|
|
2091
2089
|
DatabaseHost,
|
|
2092
2090
|
SnapshotManager,
|
|
2093
2091
|
SnapshotStore,
|
|
2094
2092
|
DataServiceSubscriptions,
|
|
2095
2093
|
DataServiceImpl,
|
|
2094
|
+
MetadataStore,
|
|
2095
|
+
mapTimeframeToFeedIndexes,
|
|
2096
|
+
mapFeedIndexesToTimeframe,
|
|
2097
|
+
startAfter,
|
|
2098
|
+
TimeframeClock,
|
|
2099
|
+
Pipeline,
|
|
2100
|
+
AuthExtension,
|
|
2096
2101
|
DataPipeline,
|
|
2097
2102
|
Space,
|
|
2098
2103
|
MOCK_AUTH_PROVIDER,
|
|
@@ -2102,4 +2107,4 @@ export {
|
|
|
2102
2107
|
SpaceProtocolSession,
|
|
2103
2108
|
SpaceManager
|
|
2104
2109
|
};
|
|
2105
|
-
//# sourceMappingURL=chunk-
|
|
2110
|
+
//# sourceMappingURL=chunk-4VJRJ4SY.mjs.map
|