@typeberry/lib 0.5.1-aff8b4f → 0.5.1-eb84786

package/packages/workers/api/port.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=port.test.d.ts.map

package/packages/workers/api/port.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"port.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api/port.test.ts"],"names":[],"mappings":""}

package/packages/workers/api/port.test.js

@@ -0,0 +1,28 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { codec } from "#@typeberry/codec";
+import { tryAsU32 } from "#@typeberry/numbers";
+import { DirectPort } from "./port.js";
+describe("DirectPort", () => {
+    it("should communicate over direct port", () => {
+        const [txPort, rxPort] = DirectPort.pair();
+        let receivedMessage = null;
+        rxPort.on("hello", codec.u32, (msg) => {
+            receivedMessage = msg;
+        });
+        assert.deepStrictEqual(receivedMessage, null);
+        txPort.postMessage("nothello", codec.nothing, {
+            responseId: "10",
+            data: undefined,
+        });
+        assert.deepStrictEqual(receivedMessage, null);
+        txPort.postMessage("hello", codec.u32, {
+            responseId: "10",
+            data: tryAsU32(10),
+        });
+        assert.deepStrictEqual(receivedMessage, {
+            responseId: "10",
+            data: tryAsU32(10),
+        });
+    });
+});

package/packages/workers/api/protocol.d.ts

@@ -0,0 +1,10 @@
+import type { Api, Internal, LousyProtocol, MessagesList } from "./types.js";
+export declare function createProtocol<To, From>(name: string, { toWorker, fromWorker, }: {
+    toWorker: To & MessagesList<To>;
+    fromWorker: From & MessagesList<From>;
+}): LousyProtocol<To, From>;
+export declare function startSameThread<To, From>(protocol: LousyProtocol<To, From>): {
+    api: Api<LousyProtocol<To, From>>;
+    internal: Internal<LousyProtocol<To, From>>;
+};
+//# sourceMappingURL=protocol.d.ts.map

package/packages/workers/api/protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api/protocol.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7E,wBAAgB,cAAc,CAAC,EAAE,EAAE,IAAI,EACrC,IAAI,EAAE,MAAM,EACZ,EACE,QAAQ,EACR,UAAU,GACX,EAAE;IACD,QAAQ,EAAE,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;IAChC,UAAU,EAAE,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;CACvC,GACA,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,CAEzB;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,IAAI,EACtC,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,GAChC;IACD,GAAG,EAAE,GAAG,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;IAClC,QAAQ,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;CAC7C,CASA"}

package/packages/workers/api/protocol.js

@@ -0,0 +1,14 @@
+import { Channel } from "./channel.js";
+import { DirectPort } from "./port.js";
+export function createProtocol(name, { toWorker, fromWorker, }) {
+    return { name, toWorker, fromWorker };
+}
+export function startSameThread(protocol) {
+    const [txPort, rxPort] = DirectPort.pair();
+    const api = Channel.tx(protocol, txPort);
+    const internal = Channel.rx(protocol, rxPort);
+    return {
+        api,
+        internal,
+    };
+}

package/packages/workers/api/types.d.ts

@@ -0,0 +1,52 @@
+import type { Codec } from "#@typeberry/codec";
+/**
+ * Definition of communication protocol between workers.
+ */
+export type Protocol<To, From> = {
+    name: string;
+    toWorker: MessagesList<To>;
+    fromWorker: MessagesList<From>;
+};
+/** Relaxed version of protocol definition to ensure proper type inference. */
+export type LousyProtocol<To, From> = Protocol<To, From> & {
+    toWorker: To & {};
+    fromWorker: From & {};
+};
+/**
+ * A one-way definition of the protocol, containing only request & response codecs
+ * for supported messages.
+ */
+export type MessagesList<T> = {
+    [K in keyof T]: T[K] extends MessageCodecs<infer Req, infer Res> ? MessageCodecs<Req, Res> : never;
+};
+/** Request and response codec definitions for protocol message. */
+export type MessageCodecs<Req, Res> = {
+    request: Codec<Req>;
+    response: Codec<Res>;
+};
+/** Request handler. Takes a `Req` and converts it into `Promise<Res>` */
+export type Handler<Req, Res> = (x: (req: Req) => Promise<Res>) => void;
+/** Collection of request handlers with naming based on protocol messages. */
+export type Handlers<In> = {
+    [K in keyof In as HandlerKey<K>]: In[K] extends MessageCodecs<infer Req, infer Res> ? Handler<Req, Res> : never;
+};
+/** Request handler key type. */
+export type HandlerKey<K> = `setOn${Capitalize<string & K>}`;
+/** Request sender. Takes a `Req` and promises a response. */
+export type Sender<Req, Res> = (req: Req) => Promise<Res>;
+/** Collection of request senders with naming based on protocol messages. */
+export type Senders<Out> = {
+    [K in keyof Out as SenderKey<K>]: Out[K] extends MessageCodecs<infer Req, infer Res> ? Sender<Req, Res> : never;
+};
+export type Destroy = {
+    destroy(): void;
+};
+/** Request sender key type. */
+export type SenderKey<K> = `send${Capitalize<string & K>}`;
+/** Receiving end of the protocol communication channel. */
+export type Rx<To, From> = Handlers<To> & Senders<From> & Destroy;
+/** Transmitting end of the protocol communication channel. */
+export type Tx<To, From> = Senders<To> & Handlers<From> & Destroy;
+export type Api<T> = T extends LousyProtocol<infer To, infer From> ? Tx<To, From> : never;
+export type Internal<T> = T extends LousyProtocol<infer To, infer From> ? Rx<To, From> : never;
+//# sourceMappingURL=types.d.ts.map

package/packages/workers/api/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE9C;;GAEG;AACH,MAAM,MAAM,QAAQ,CAAC,EAAE,EAAE,IAAI,IAAI;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IAC3B,UAAU,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;CAChC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,aAAa,CAAC,EAAE,EAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG;IACzD,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC;IAClB,UAAU,EAAE,IAAI,GAAG,EAAE,CAAC;CACvB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,CAAC,CAAC,IAAI;KAC3B,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,MAAM,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK;CACnG,CAAC;AAEF,mEAAmE;AACnE,MAAM,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,IAAI;IACpC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACpB,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;CACtB,CAAC;AAEF,yEAAyE;AACzE,MAAM,MAAM,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;AAExE,6EAA6E;AAC7E,MAAM,MAAM,QAAQ,CAAC,EAAE,IAAI;KACxB,CAAC,IAAI,MAAM,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,MAAM,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK;CAChH,CAAC;AACF,gCAAgC;AAChC,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI,QAAQ,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAE7D,6DAA6D;AAC7D,MAAM,MAAM,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,MAAM,OAAO,CAAC,GAAG,IAAI;KACxB,CAAC,IAAI,MAAM,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,MAAM,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,KAAK;CAChH,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG;IAAE,OAAO,IAAI,IAAI,CAAA;CAAE,CAAC;AAE1C,+BAA+B;AAC/B,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAE3D,2DAA2D;AAC3D,MAAM,MAAM,EAAE,CAAC,EAAE,EAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;AAElE,8DAA8D;AAC9D,MAAM,MAAM,EAAE,CAAC,EAAE,EAAE,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;AAElE,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,MAAM,EAAE,EAAE,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC;AAC1F,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,MAAM,EAAE,EAAE,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC"}

package/packages/workers/api/types.js

@@ -0,0 +1 @@
+export {};

package/packages/workers/api-node/config.d.ts

@@ -0,0 +1,59 @@
+import { type Decode, type Encode } from "#@typeberry/codec";
+import { ChainSpec } from "#@typeberry/config";
+import { type BlocksDb, type RootDb, type SerializedStatesDb } from "#@typeberry/database";
+import { Blake2b } from "#@typeberry/hash";
+import type { WorkerConfig } from "#@typeberry/workers-api";
+/** A worker config that's usable in node.js and uses LMDB database backend. */
+export declare class LmdbWorkerConfig<T = void> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
+    readonly nodeName: string;
+    readonly chainSpec: ChainSpec;
+    readonly workerParams: T;
+    readonly dbPath: string;
+    readonly blake2b: Blake2b;
+    static new<T>({ nodeName, chainSpec, workerParams, dbPath, blake2b, }: {
+        nodeName: string;
+        chainSpec: ChainSpec;
+        workerParams: T;
+        dbPath: string;
+        blake2b: Blake2b;
+    }): LmdbWorkerConfig<T>;
+    /** Restore node config from a transferable config object. */
+    static fromTransferable<T>(decodeParams: Decode<T>, config: TransferableConfig): Promise<LmdbWorkerConfig<T>>;
+    private constructor();
+    openDatabase(options?: {
+        readonly: boolean;
+    }): RootDb<BlocksDb, SerializedStatesDb>;
+    /** Convert this config into a thread-transferable object. */
+    intoTransferable(paramsCodec: Encode<T>): TransferableConfig;
+}
+/** Config that's safe to transfer between worker threads. */
+export type TransferableConfig = {
+    nodeName: string;
+    chainSpec: ChainSpec;
+    workerParams: Uint8Array;
+    dbPath: string;
+};
+/**
+ * In-memory (direct) worker using serialized state database.
+ *
+ * Note the database is always empty, and needs to be initialized.
+ */
+export declare class InMemWorkerConfig<T = undefined> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
+    readonly nodeName: string;
+    readonly chainSpec: ChainSpec;
+    readonly workerParams: T;
+    readonly blake2b: Blake2b;
+    static new<T>({ nodeName, chainSpec, workerParams, blake2b, }: {
+        nodeName: string;
+        chainSpec: ChainSpec;
+        workerParams: T;
+        blake2b: Blake2b;
+    }): InMemWorkerConfig<T>;
+    private readonly blocks;
+    private readonly states;
+    private constructor();
+    openDatabase(_options?: {
+        readonly: boolean;
+    }): RootDb<BlocksDb, SerializedStatesDb>;
+}
+//# sourceMappingURL=config.d.ts.map

package/packages/workers/api-node/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAiC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;IApClC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAcpF,OAAO;IAQP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAQ7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}

package/packages/workers/api-node/config.js

@@ -0,0 +1,85 @@
+import { Decoder, Encoder } from "#@typeberry/codec";
+import { ChainSpec } from "#@typeberry/config";
+import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
+import { LmdbBlocks, LmdbRoot, LmdbStates } from "#@typeberry/database-lmdb";
+import { Blake2b } from "#@typeberry/hash";
+/** A worker config that's usable in node.js and uses LMDB database backend. */
+export class LmdbWorkerConfig {
+    nodeName;
+    chainSpec;
+    workerParams;
+    dbPath;
+    blake2b;
+    static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, }) {
+        return new LmdbWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b);
+    }
+    /** Restore node config from a transferable config object. */
+    static async fromTransferable(decodeParams, config) {
+        const blake2b = await Blake2b.createHasher();
+        const chainSpec = new ChainSpec(config.chainSpec);
+        const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
+        return LmdbWorkerConfig.new({
+            nodeName: config.nodeName,
+            chainSpec,
+            workerParams,
+            dbPath: config.dbPath,
+            blake2b,
+        });
+    }
+    constructor(nodeName, chainSpec, workerParams, dbPath, blake2b) {
+        this.nodeName = nodeName;
+        this.chainSpec = chainSpec;
+        this.workerParams = workerParams;
+        this.dbPath = dbPath;
+        this.blake2b = blake2b;
+    }
+    openDatabase(options = { readonly: true }) {
+        const lmdb = new LmdbRoot(this.dbPath, options.readonly);
+        return {
+            getBlocksDb: () => new LmdbBlocks(this.chainSpec, lmdb),
+            getStatesDb: () => new LmdbStates(this.chainSpec, this.blake2b, lmdb),
+            close: async () => await lmdb.close(),
+        };
+    }
+    /** Convert this config into a thread-transferable object. */
+    intoTransferable(paramsCodec) {
+        return {
+            nodeName: this.nodeName,
+            chainSpec: this.chainSpec,
+            workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
+            dbPath: this.dbPath,
+        };
+    }
+}
+/**
+ * In-memory (direct) worker using serialized state database.
+ *
+ * Note the database is always empty, and needs to be initialized.
+ */
+export class InMemWorkerConfig {
+    nodeName;
+    chainSpec;
+    workerParams;
+    blake2b;
+    static new({ nodeName, chainSpec, workerParams, blake2b, }) {
+        return new InMemWorkerConfig(nodeName, chainSpec, workerParams, blake2b);
+    }
+    blocks;
+    states;
+    constructor(nodeName, chainSpec, workerParams, blake2b) {
+        this.nodeName = nodeName;
+        this.chainSpec = chainSpec;
+        this.workerParams = workerParams;
+        this.blake2b = blake2b;
+        this.blocks = InMemoryBlocks.new();
+        this.states = InMemorySerializedStates.withHasher({ chainSpec, blake2b });
+    }
+    openDatabase(_options = { readonly: true }) {
+        // opening/closing db doesn't do anything, we persist the state.
+        return {
+            getBlocksDb: () => this.blocks,
+            getStatesDb: () => this.states,
+            close: async () => { },
+        };
+    }
+}

package/packages/workers/api-node/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./config.js";
+export * from "./port.js";
+export * from "./protocol.js";
+//# sourceMappingURL=index.d.ts.map

package/packages/workers/api-node/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}

package/packages/workers/api-node/index.js

@@ -0,0 +1,3 @@
+export * from "./config.js";
+export * from "./port.js";
+export * from "./protocol.js";

package/packages/workers/api-node/port.d.ts

@@ -0,0 +1,23 @@
+import { type MessagePort } from "node:worker_threads";
+import { type Codec } from "#@typeberry/codec";
+import type { ChainSpec } from "#@typeberry/config";
+import type { Envelope, Port } from "#@typeberry/workers-api/port.js";
+export type Message = {
+    eventName: string;
+    responseId: string;
+    data: Uint8Array;
+};
+export declare class ThreadPort implements Port {
+    private readonly spec;
+    private readonly port;
+    readonly threadId: number;
+    private readonly events;
+    constructor(spec: ChainSpec, port: MessagePort);
+    close(): void;
+    private createListener;
+    onClose(callback: (e: Error) => void): void;
+    on<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
+    once<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
+    postMessage<T>(event: string, codec: Codec<T>, msg: Envelope<T>): void;
+}
+//# sourceMappingURL=port.d.ts.map

package/packages/workers/api-node/port.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAKF,qBAAa,UAAW,YAAW,IAAI;IAKnC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IALvB,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAG1B,IAAI,EAAE,SAAS,EACf,IAAI,EAAE,WAAW;IAcpC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;CAWvE"}

package/packages/workers/api-node/port.js

@@ -0,0 +1,88 @@
+import EventEmitter from "node:events";
+import { threadId } from "node:worker_threads";
+import { Decoder, Encoder } from "#@typeberry/codec";
+import { Logger } from "#@typeberry/logger";
+import { check } from "#@typeberry/utils";
+const MESSAGE_KEYS = ["eventName", "responseId", "data"];
+const logger = Logger.new(import.meta.filename, "workers/api");
+export class ThreadPort {
+    spec;
+    port;
+    threadId = threadId;
+    events = new EventEmitter();
+    constructor(spec, port) {
+        this.spec = spec;
+        this.port = port;
+        this.port.on("message", (input) => {
+            if (!isMessage(input)) {
+                logger.error `Received a malformed data from another thread: ${input}`;
+                return;
+            }
+            const { eventName, responseId, data } = input;
+            check `${this.events.listeners(eventName).length > 0} No listeners for received event ${eventName}!`;
+            this.events.emit(eventName, responseId, data);
+        });
+    }
+    close() {
+        this.port.close();
+    }
+    createListener(codec, callback) {
+        return (responseId, data) => {
+            let decoded;
+            try {
+                decoded = Decoder.decodeObject(codec, data, this.spec);
+            }
+            catch (e) {
+                logger.error `Unable to decode expected from thread: ${"name" in codec ? codec.name : codec}: ${e}`;
+                return;
+            }
+            try {
+                callback({ responseId, data: decoded });
+            }
+            catch (e) {
+                logger.error `Error while processing thread message: ${e}`;
+                throw e;
+            }
+        };
+    }
+    onClose(callback) {
+        this.port.on("error", callback);
+    }
+    on(event, codec, callback) {
+        const listener = this.createListener(codec, callback);
+        this.events.on(event, listener);
+        return () => {
+            this.events.off(event, listener);
+        };
+    }
+    once(event, codec, callback) {
+        const listener = this.createListener(codec, callback);
+        this.events.once(event, listener);
+        return () => {
+            this.events.off(event, listener);
+        };
+    }
+    postMessage(event, codec, msg) {
+        const encoded = Encoder.encodeObject(codec, msg.data, this.spec);
+        const message = {
+            eventName: event,
+            responseId: msg.responseId,
+            data: encoded.raw,
+        };
+        // casting to transferable is safe here, since we know that encoder
+        // always returns owned uint8arrays.
+        this.port.postMessage(message, [encoded.raw.buffer]);
+    }
+}
+function isMessage(data) {
+    const isObject = data !== null && typeof data === "object";
+    if (!isObject) {
+        return false;
+    }
+    for (const k of MESSAGE_KEYS) {
+        if (!(k in data)) {
+            return false;
+        }
+    }
+    return true;
+}

package/packages/workers/api-node/port.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=port.test.d.ts.map

package/packages/workers/api-node/port.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"port.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.test.ts"],"names":[],"mappings":""}

package/packages/workers/api-node/port.test.js

@@ -0,0 +1,39 @@
+import assert from "node:assert";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { setTimeout } from "node:timers/promises";
+import { MessageChannel } from "node:worker_threads";
+import { codec } from "#@typeberry/codec";
+import { tinyChainSpec } from "#@typeberry/config";
+import { tryAsU32 } from "#@typeberry/numbers";
+import { ThreadPort } from "./port.js";
+const spec = tinyChainSpec;
+describe("ThreadPort", () => {
+    let channel;
+    beforeEach(() => {
+        channel = new MessageChannel();
+    });
+    afterEach(() => {
+        channel.port1.close();
+        channel.port2.close();
+    });
+    it("should successfuly send messages", async () => {
+        const tx = new ThreadPort(spec, channel.port1);
+        const rx = new ThreadPort(spec, channel.port2);
+        let received = null;
+        // attach listener
+        rx.on("hello", codec.varU32, (msg) => {
+            received = msg;
+        });
+        assert.strictEqual(received, null);
+        // send some message
+        tx.postMessage("hello", codec.varU32, {
+            responseId: "10",
+            data: tryAsU32(42),
+        });
+        await setTimeout(1_000);
+        assert.deepStrictEqual(received, {
+            responseId: "10",
+            data: tryAsU32(42),
+        });
+    });
+});

package/packages/workers/api-node/protocol.d.ts

@@ -0,0 +1,47 @@
+import { type MessagePort, Worker } from "node:worker_threads";
+import type { Decode, Encode } from "#@typeberry/codec";
+import { Listener } from "#@typeberry/listener";
+import type { Api, Internal, LousyProtocol } from "#@typeberry/workers-api/types.js";
+import { LmdbWorkerConfig, type TransferableConfig } from "./config.js";
+/** Type of the control plane message. */
+export declare enum WorkerControlPlaneMsg {
+    /** Transfers a communication port to some other thread. */
+    CommunicationPort = 0,
+    /** Transfers worker configuration. */
+    Config = 1
+}
+export type ThreadComms = {
+    /** Name of the thread we can communicate with. */
+    threadName: string;
+    /** Communication port. */
+    port: MessagePort;
+};
+/** Control plane message. Received received from parent thread, only on `parentPort`. */
+export type WorkerControlPlane = ({
+    /** Direct communication port with some other thread. */
+    kind: WorkerControlPlaneMsg.CommunicationPort;
+} & ThreadComms) | {
+    /** Configuration object for a worker and parent-thread communication port. */
+    kind: WorkerControlPlaneMsg.Config;
+    /** Main thread communication port. */
+    parentPort: MessagePort;
+    /** Configuration object. */
+    config: TransferableConfig;
+};
+/**
+ * Invoked by the main thread, to spawn a worker thread and initiate communication channel.
+ */
+export declare function spawnWorker<To, From, Params>(protocol: LousyProtocol<To, From>, bootstrapPath: URL, config: LmdbWorkerConfig<Params>, paramsEncoder: Encode<Params>): {
+    api: Api<typeof protocol>;
+    worker: Worker;
+    workerFinished: Promise<void>;
+};
+/**
+ * Initialize worker thread by awaiting the config message.
+ */
+export declare function initWorker<To, From, Params>(protocol: LousyProtocol<To, From>, paramsDecoder: Decode<Params>): Promise<{
+    config: LmdbWorkerConfig<Params>;
+    comms: Internal<typeof protocol>;
+    threadComms: Listener<ThreadComms>;
+}>;
+//# sourceMappingURL=protocol.d.ts.map

package/packages/workers/api-node/protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKxE,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAkCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAkDD"}

package/packages/workers/api-node/protocol.js

@@ -0,0 +1,105 @@
+import { MessageChannel, parentPort, Worker } from "node:worker_threads";
+import { Listener } from "#@typeberry/listener";
+import { Level, Logger } from "#@typeberry/logger";
+import { assertNever } from "#@typeberry/utils";
+import { Channel } from "#@typeberry/workers-api";
+import { LmdbWorkerConfig } from "./config.js";
+import { ThreadPort } from "./port.js";
+const logger = Logger.new(import.meta.filename, "workers");
+/** Type of the control plane message. */
+export var WorkerControlPlaneMsg;
+(function (WorkerControlPlaneMsg) {
+    /** Transfers a communication port to some other thread. */
+    WorkerControlPlaneMsg[WorkerControlPlaneMsg["CommunicationPort"] = 0] = "CommunicationPort";
+    /** Transfers worker configuration. */
+    WorkerControlPlaneMsg[WorkerControlPlaneMsg["Config"] = 1] = "Config";
+})(WorkerControlPlaneMsg || (WorkerControlPlaneMsg = {}));
+function isControlPlane(data) {
+    const isObject = data !== null && typeof data === "object";
+    if (!isObject) {
+        return false;
+    }
+    if ("kind" in data && typeof data.kind === "number" && WorkerControlPlaneMsg[data.kind] !== undefined) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Invoked by the main thread, to spawn a worker thread and initiate communication channel.
+ */
+export function spawnWorker(protocol, bootstrapPath, config, paramsEncoder) {
+    logger.trace `Spawning ${protocol.name} child worker.`;
+    const channel = new MessageChannel();
+    const worker = new Worker(bootstrapPath);
+    const msg = {
+        kind: WorkerControlPlaneMsg.Config,
+        parentPort: channel.port2,
+        config: config.intoTransferable(paramsEncoder),
+    };
+    logger.trace `(${protocol.name}) <-- config`;
+    // send the config down to the worker
+    worker.postMessage(msg, [msg.parentPort]);
+    const workerFinished = new Promise((resolve, reject) => {
+        worker.once("error", reject);
+        worker.once("exit", (exitCode) => {
+            if (exitCode === 0) {
+                resolve();
+            }
+            else {
+                reject(new Error(`(${protocol.name}) exit code: ${exitCode}`));
+            }
+        });
+    });
+    // now return communication channel with that worker
+    const txPort = new ThreadPort(config.chainSpec, channel.port1);
+    return {
+        api: Channel.tx(protocol, txPort),
+        worker,
+        workerFinished,
+    };
+}
+/**
+ * Initialize worker thread by awaiting the config message.
+ */
+export async function initWorker(protocol, paramsDecoder) {
+    // configure logger inside a worker thread
+    Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
+    logger.trace `Worker ${protocol.name} starting.`;
+    return new Promise((resolve, reject) => {
+        if (parentPort === null) {
+            throw new Error(`Unable to start ${protocol.name} worker. Not running in a worker thread!`);
+        }
+        parentPort.once("close", () => reject(new Error(`(${protocol.name}) parent port closed too early`)));
+        let isResolved = false;
+        const threadComms = new Listener();
+        parentPort.on("message", async (msg) => {
+            if (!isControlPlane(msg)) {
+                logger.error `--> (${protocol.name}) received unexpected message: ${msg}`;
+                return;
+            }
+            if (msg.kind === WorkerControlPlaneMsg.CommunicationPort) {
+                logger.trace `--> (${protocol.name}) received comms port with ${msg.threadName}.`;
+                threadComms.emit(msg);
+                return;
+            }
+            if (msg.kind === WorkerControlPlaneMsg.Config) {
+                if (isResolved) {
+                    logger.error `--> (${protocol.name}) ignoring duplicated config message.`;
+                    return;
+                }
+                logger.trace `--> (${protocol.name}) received configuration.`;
+                isResolved = true;
+                const config = await LmdbWorkerConfig.fromTransferable(paramsDecoder, msg.config);
+                const rxPort = new ThreadPort(config.chainSpec, msg.parentPort);
+                const comms = Channel.rx(protocol, rxPort);
+                resolve({
+                    config,
+                    comms,
+                    threadComms,
+                });
+                return;
+            }
+            assertNever(msg.kind);
+        });
+    });
+}

package/packages/workers/block-authorship/bootstrap-main.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=bootstrap-main.d.ts.map

package/packages/workers/block-authorship/bootstrap-main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap-main.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/bootstrap-main.ts"],"names":[],"mappings":""}

package/packages/workers/block-authorship/bootstrap-main.js

@@ -0,0 +1,12 @@
+import { Telemetry } from "#@typeberry/telemetry";
+import { initWorker } from "#@typeberry/workers-api-node";
+import { main } from "./main.js";
+import { BlockAuthorshipConfig, protocol } from "./protocol.js";
+const { config, comms } = await initWorker(protocol, BlockAuthorshipConfig.Codec);
+// Initialize OpenTelemetry for this worker
+const tele = Telemetry.initialize({
+    worker: "generator",
+    nodeName: config.nodeName,
+});
+await main(config, comms);
+await tele?.close();