@typeberry/lib 0.6.0-41cb030 → 0.6.0-4df87ee
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/workers/api-node/config.d.ts +9 -0
- package/packages/workers/api-node/config.d.ts.map +1 -1
- package/packages/workers/api-node/config.js +10 -0
- package/packages/workers/api-node/config.test.d.ts +2 -0
- package/packages/workers/api-node/config.test.d.ts.map +1 -0
- package/packages/workers/api-node/config.test.js +41 -0
- package/packages/workers/api-node/protocol.d.ts.map +1 -1
- package/packages/workers/api-node/protocol.js +5 -3
package/package.json
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { MessagePort } from "node:worker_threads";
|
|
1
2
|
import { type Decode, type Encode } from "#@typeberry/codec";
|
|
2
3
|
import { ChainSpec } from "#@typeberry/config";
|
|
3
4
|
import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
|
|
@@ -37,6 +38,14 @@ export type TransferableConfig = {
|
|
|
37
38
|
dbPath: string;
|
|
38
39
|
workerPorts: [string, TransferablePort][];
|
|
39
40
|
};
|
|
41
|
+
/**
|
|
42
|
+
* Collect the transferable objects (communication ports) embedded in a config.
|
|
43
|
+
*
|
|
44
|
+
* `MessagePort`s can only be transferred, not structurally cloned, so they have to
|
|
45
|
+
* be listed in the `postMessage` transfer list. Omitting them results in a
|
|
46
|
+
* `DataCloneError`.
|
|
47
|
+
*/
|
|
48
|
+
export declare function configTransferList(config: TransferableConfig): MessagePort[];
|
|
40
49
|
/**
|
|
41
50
|
* In-memory (direct) worker using serialized state database.
|
|
42
51
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;IA3ChD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACjC;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IASP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
|
|
@@ -57,6 +57,16 @@ export class LmdbWorkerConfig {
|
|
|
57
57
|
};
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Collect the transferable objects (communication ports) embedded in a config.
|
|
62
|
+
*
|
|
63
|
+
* `MessagePort`s can only be transferred, not structurally cloned, so they have to
|
|
64
|
+
* be listed in the `postMessage` transfer list. Omitting them results in a
|
|
65
|
+
* `DataCloneError`.
|
|
66
|
+
*/
|
|
67
|
+
export function configTransferList(config) {
|
|
68
|
+
return config.workerPorts.map(([, transferable]) => transferable.port);
|
|
69
|
+
}
|
|
60
70
|
/**
|
|
61
71
|
* In-memory (direct) worker using serialized state database.
|
|
62
72
|
*
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import assert from "node:assert";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { MessageChannel } from "node:worker_threads";
|
|
4
|
+
import { codec } from "#@typeberry/codec";
|
|
5
|
+
import { tinyChainSpec } from "#@typeberry/config";
|
|
6
|
+
import { Blake2b } from "#@typeberry/hash";
|
|
7
|
+
import { tryAsU32 } from "#@typeberry/numbers";
|
|
8
|
+
import { configTransferList, LmdbWorkerConfig } from "./config.js";
|
|
9
|
+
import { ThreadPort } from "./port.js";
|
|
10
|
+
const spec = tinyChainSpec;
|
|
11
|
+
describe("LmdbWorkerConfig transfer list", () => {
|
|
12
|
+
it("surfaces embedded worker ports so they can be transferred", async () => {
|
|
13
|
+
const blake2b = await Blake2b.createHasher();
|
|
14
|
+
const [portA, portB] = ThreadPort.pair(spec);
|
|
15
|
+
const config = LmdbWorkerConfig.new({
|
|
16
|
+
nodeName: "node",
|
|
17
|
+
chainSpec: spec,
|
|
18
|
+
workerParams: tryAsU32(7),
|
|
19
|
+
dbPath: "db",
|
|
20
|
+
blake2b,
|
|
21
|
+
ports: new Map([["authorship-network", portA]]),
|
|
22
|
+
});
|
|
23
|
+
const transferable = config.intoTransferable(codec.varU32);
|
|
24
|
+
const transferList = configTransferList(transferable);
|
|
25
|
+
// the single embedded comms port must be reported for transfer
|
|
26
|
+
assert.strictEqual(transferList.length, 1);
|
|
27
|
+
const sink = new MessageChannel();
|
|
28
|
+
try {
|
|
29
|
+
// reproduces the bug: a config carrying a port cannot be cloned without
|
|
30
|
+
// listing that port in the transfer list.
|
|
31
|
+
assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
|
|
32
|
+
// with the ports surfaced, posting succeeds.
|
|
33
|
+
assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
sink.port1.close();
|
|
37
|
+
sink.port2.close();
|
|
38
|
+
portB.close();
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAsB,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAM5F,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAoCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAmDD"}
|
|
@@ -3,7 +3,7 @@ import { Listener } from "#@typeberry/listener";
|
|
|
3
3
|
import { Level, Logger } from "#@typeberry/logger";
|
|
4
4
|
import { assertNever } from "#@typeberry/utils";
|
|
5
5
|
import { Channel } from "#@typeberry/workers-api";
|
|
6
|
-
import { LmdbWorkerConfig } from "./config.js";
|
|
6
|
+
import { configTransferList, LmdbWorkerConfig } from "./config.js";
|
|
7
7
|
import { logHeapLimit } from "./host-environment.js";
|
|
8
8
|
import { ThreadPort } from "./port.js";
|
|
9
9
|
const logger = Logger.new(import.meta.filename, "workers");
|
|
@@ -38,8 +38,10 @@ export function spawnWorker(protocol, bootstrapPath, config, paramsEncoder) {
|
|
|
38
38
|
config: config.intoTransferable(paramsEncoder),
|
|
39
39
|
};
|
|
40
40
|
logger.trace `(${protocol.name}) <-- config`;
|
|
41
|
-
// send the config down to the worker
|
|
42
|
-
worker
|
|
41
|
+
// send the config down to the worker. We need to transfer the parent
|
|
42
|
+
// communication port as well as any inter-worker ports carried in the config,
|
|
43
|
+
// otherwise structured clone fails with a `DataCloneError`.
|
|
44
|
+
worker.postMessage(msg, [msg.parentPort, ...configTransferList(msg.config)]);
|
|
43
45
|
const workerFinished = new Promise((resolve, reject) => {
|
|
44
46
|
worker.once("error", reject);
|
|
45
47
|
worker.once("exit", (exitCode) => {
|