opnet 1.7.21 → 1.7.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser/229.index.js +1 -0
- package/browser/_version.d.ts +1 -1
- package/browser/block/Block.d.ts +8 -2
- package/browser/cache/LRUCaching.d.ts +7 -0
- package/browser/cache/P2OPCache.d.ts +2 -0
- package/browser/fetch/fetch-browser.d.ts +1 -0
- package/browser/fetch/fetch.d.ts +2 -2
- package/browser/fetch/fetcher-type.d.ts +4 -0
- package/browser/index.js +1 -1
- package/browser/providers/JSONRpcProvider.d.ts +5 -2
- package/browser/threading/JSONThreader.d.ts +20 -0
- package/browser/threading/SharedThreader.d.ts +36 -0
- package/browser/threading/WorkerCreator.d.ts +3 -0
- package/browser/threading/interfaces/IThread.d.ts +28 -0
- package/browser/threading/worker-scripts/JSONWorker.d.ts +2 -0
- package/build/_version.d.ts +1 -1
- package/build/_version.js +1 -1
- package/build/block/Block.d.ts +8 -2
- package/build/block/Block.js +24 -10
- package/build/cache/LRUCaching.d.ts +7 -0
- package/build/cache/LRUCaching.js +28 -0
- package/build/cache/P2OPCache.d.ts +2 -0
- package/build/cache/P2OPCache.js +19 -0
- package/build/fetch/fetch.d.ts +2 -2
- package/build/fetch/fetch.js +9 -5
- package/build/fetch/fetcher-type.d.ts +4 -0
- package/build/providers/JSONRpcProvider.d.ts +5 -2
- package/build/providers/JSONRpcProvider.js +22 -6
- package/build/threading/JSONThreader.d.ts +20 -0
- package/build/threading/JSONThreader.js +38 -0
- package/build/threading/SharedThreader.d.ts +36 -0
- package/build/threading/SharedThreader.js +184 -0
- package/build/threading/WorkerCreator.d.ts +3 -0
- package/build/threading/WorkerCreator.js +57 -0
- package/build/threading/interfaces/IThread.d.ts +28 -0
- package/build/threading/interfaces/IThread.js +1 -0
- package/build/threading/worker-scripts/JSONWorker.d.ts +2 -0
- package/build/threading/worker-scripts/JSONWorker.js +76 -0
- package/build/transactions/metadata/TransactionReceipt.js +4 -8
- package/package.json +3 -2
- package/src/_version.ts +1 -1
- package/src/block/Block.ts +32 -14
- package/src/cache/LRUCaching.ts +30 -0
- package/src/cache/P2OPCache.ts +22 -0
- package/src/fetch/fetch-browser.js +3 -1
- package/src/fetch/fetch.ts +14 -9
- package/src/fetch/fetcher-type.ts +5 -0
- package/src/providers/JSONRpcProvider.ts +24 -10
- package/src/threading/JSONThreader.ts +62 -0
- package/src/threading/SharedThreader.ts +255 -0
- package/src/threading/WorkerCreator.ts +67 -0
- package/src/threading/interfaces/IThread.ts +38 -0
- package/src/threading/worker-scripts/JSONWorker.ts +78 -0
- package/src/transactions/metadata/TransactionReceipt.ts +5 -14
- package/webpack.config.js +2 -0
|
@@ -7,11 +7,14 @@ export declare class JSONRpcProvider extends AbstractRpcProvider {
|
|
|
7
7
|
private readonly timeout;
|
|
8
8
|
private readonly fetcherConfigurations;
|
|
9
9
|
private useRESTAPI;
|
|
10
|
+
private readonly useThreadedParsing;
|
|
10
11
|
readonly url: string;
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
private _fetcherWithCleanup;
|
|
13
|
+
constructor(url: string, network: Network, timeout?: number, fetcherConfigurations?: Agent.Options, useRESTAPI?: boolean, useThreadedParsing?: boolean);
|
|
13
14
|
private get fetcher();
|
|
15
|
+
close(): Promise<void>;
|
|
14
16
|
setFetchMode(useRESTAPI: boolean): void;
|
|
15
17
|
_send(payload: JsonRpcPayload | JsonRpcPayload[]): Promise<JsonRpcCallResult>;
|
|
16
18
|
protected providerUrl(url: string): string;
|
|
19
|
+
private parseResponse;
|
|
17
20
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { ThreaderOptions, WorkerScript } from './interfaces/IThread.js';
|
|
2
|
+
import { BaseThreader } from './SharedThreader.js';
|
|
3
|
+
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
4
|
+
[key: string]: JsonValue;
|
|
5
|
+
};
|
|
6
|
+
type JsonOp = 'parse' | 'stringify';
|
|
7
|
+
type JsonInput = string | JsonValue | ArrayBuffer;
|
|
8
|
+
type JsonOutput = string | JsonValue;
|
|
9
|
+
export declare class JsonThreader extends BaseThreader<JsonOp, JsonInput, JsonOutput> {
|
|
10
|
+
protected readonly workerScript: WorkerScript;
|
|
11
|
+
private readonly threadingThreshold;
|
|
12
|
+
constructor(options?: ThreaderOptions & {
|
|
13
|
+
threadingThreshold?: number;
|
|
14
|
+
});
|
|
15
|
+
parse<T = JsonValue>(json: string): Promise<T>;
|
|
16
|
+
parseBuffer<T = JsonValue>(buffer: ArrayBuffer): Promise<T>;
|
|
17
|
+
stringify(data: JsonValue): Promise<string>;
|
|
18
|
+
}
|
|
19
|
+
export declare const jsonThreader: JsonThreader;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Logger } from '@btc-vision/logger';
|
|
2
|
+
import { ThreaderOptions, WorkerScript } from './interfaces/IThread.js';
|
|
3
|
+
export declare abstract class BaseThreader<TOp extends string, TData, TResult> extends Logger {
|
|
4
|
+
readonly logColor: string;
|
|
5
|
+
protected abstract readonly workerScript: WorkerScript;
|
|
6
|
+
private workers;
|
|
7
|
+
private available;
|
|
8
|
+
private pending;
|
|
9
|
+
private queue;
|
|
10
|
+
private idCounter;
|
|
11
|
+
private readonly poolSize;
|
|
12
|
+
private initialized;
|
|
13
|
+
private initializing;
|
|
14
|
+
private tasksProcessed;
|
|
15
|
+
private tasksFailed;
|
|
16
|
+
private lastStatsLog;
|
|
17
|
+
private readonly statsInterval;
|
|
18
|
+
private cleanupBound;
|
|
19
|
+
protected constructor(options?: ThreaderOptions);
|
|
20
|
+
get stats(): {
|
|
21
|
+
pending: number;
|
|
22
|
+
queued: number;
|
|
23
|
+
available: number;
|
|
24
|
+
total: number;
|
|
25
|
+
processed: number;
|
|
26
|
+
failed: number;
|
|
27
|
+
};
|
|
28
|
+
terminate(): Promise<void>;
|
|
29
|
+
drain(): Promise<void>;
|
|
30
|
+
protected runWithTransfer(op: TOp, data: TData, transferables: ArrayBuffer[]): Promise<TResult>;
|
|
31
|
+
protected run(op: TOp, data: TData): Promise<TResult>;
|
|
32
|
+
private bindCleanupHandlers;
|
|
33
|
+
private init;
|
|
34
|
+
private logStatsIfNeeded;
|
|
35
|
+
private processQueue;
|
|
36
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export interface TaskMessage<TOp extends string = string, TData = unknown> {
|
|
2
|
+
id: number;
|
|
3
|
+
op: TOp;
|
|
4
|
+
data: TData;
|
|
5
|
+
}
|
|
6
|
+
export interface ResultMessage<TResult = unknown> {
|
|
7
|
+
id: number;
|
|
8
|
+
result?: TResult;
|
|
9
|
+
error?: string;
|
|
10
|
+
}
|
|
11
|
+
export interface PendingTask<TResult = unknown> {
|
|
12
|
+
resolve: (value: TResult) => void;
|
|
13
|
+
reject: (error: Error) => void;
|
|
14
|
+
}
|
|
15
|
+
export interface QueuedTask<TOp extends string = string, TData = unknown, TResult = unknown> extends PendingTask<TResult> {
|
|
16
|
+
op: TOp;
|
|
17
|
+
data: TData;
|
|
18
|
+
transferables?: ArrayBuffer[];
|
|
19
|
+
}
|
|
20
|
+
export interface ThreaderOptions {
|
|
21
|
+
poolSize?: number;
|
|
22
|
+
}
|
|
23
|
+
export interface UniversalWorker<TOp extends string = string, TData = unknown, TResult = unknown> {
|
|
24
|
+
postMessage(msg: TaskMessage<TOp, TData>, transferables?: readonly Transferable[]): void;
|
|
25
|
+
onMessage(callback: (msg: ResultMessage<TResult>) => void): void;
|
|
26
|
+
terminate(): void | Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
export type WorkerScript = string;
|
package/build/_version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const version = "1.7.
|
|
1
|
+
export declare const version = "1.7.22";
|
package/build/_version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = '1.7.
|
|
1
|
+
export const version = '1.7.22';
|
package/build/block/Block.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { Network } from '@btc-vision/bitcoin';
|
|
|
2
2
|
import { Address } from '@btc-vision/transaction';
|
|
3
3
|
import { BigNumberish } from '../common/CommonTypes.js';
|
|
4
4
|
import { OPNetTransactionTypes } from '../interfaces/opnet/OPNetTransactionTypes.js';
|
|
5
|
+
import { ITransaction } from '../transactions/interfaces/ITransaction.js';
|
|
5
6
|
import { TransactionBase } from '../transactions/Transaction.js';
|
|
6
7
|
import { BlockHeaderChecksumProof, IBlock } from './interfaces/IBlock.js';
|
|
7
8
|
export declare class Block implements Omit<IBlock, 'gasUsed' | 'ema' | 'baseGas' | 'deployments'> {
|
|
@@ -26,7 +27,12 @@ export declare class Block implements Omit<IBlock, 'gasUsed' | 'ema' | 'baseGas'
|
|
|
26
27
|
readonly baseGas: bigint;
|
|
27
28
|
readonly gasUsed: bigint;
|
|
28
29
|
readonly checksumProofs: BlockHeaderChecksumProof;
|
|
29
|
-
readonly
|
|
30
|
-
readonly
|
|
30
|
+
private readonly _rawBlock;
|
|
31
|
+
private readonly _network;
|
|
31
32
|
constructor(block: IBlock, network: Network);
|
|
33
|
+
private _transactions?;
|
|
34
|
+
get transactions(): TransactionBase<OPNetTransactionTypes>[];
|
|
35
|
+
private _deployments?;
|
|
36
|
+
get deployments(): Address[];
|
|
37
|
+
get rawTransactions(): ITransaction[];
|
|
32
38
|
}
|
package/build/block/Block.js
CHANGED
|
@@ -22,12 +22,13 @@ export class Block {
|
|
|
22
22
|
baseGas;
|
|
23
23
|
gasUsed;
|
|
24
24
|
checksumProofs;
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
_rawBlock;
|
|
26
|
+
_network;
|
|
27
27
|
constructor(block, network) {
|
|
28
|
-
if (!block)
|
|
28
|
+
if (!block)
|
|
29
29
|
throw new Error('Invalid block.');
|
|
30
|
-
|
|
30
|
+
this._rawBlock = block;
|
|
31
|
+
this._network = network;
|
|
31
32
|
this.height = BigInt(block.height.toString());
|
|
32
33
|
this.hash = block.hash;
|
|
33
34
|
this.previousBlockHash = block.previousBlockHash;
|
|
@@ -49,11 +50,24 @@ export class Block {
|
|
|
49
50
|
this.storageRoot = block.storageRoot;
|
|
50
51
|
this.receiptRoot = block.receiptRoot;
|
|
51
52
|
this.checksumProofs = block.checksumProofs;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
53
|
+
}
|
|
54
|
+
_transactions;
|
|
55
|
+
get transactions() {
|
|
56
|
+
if (!this._transactions) {
|
|
57
|
+
this._transactions = TransactionParser.parseTransactions(this._rawBlock.transactions, this._network);
|
|
58
|
+
}
|
|
59
|
+
return this._transactions;
|
|
60
|
+
}
|
|
61
|
+
_deployments;
|
|
62
|
+
get deployments() {
|
|
63
|
+
if (!this._deployments) {
|
|
64
|
+
this._deployments = this._rawBlock.deployments
|
|
65
|
+
? this._rawBlock.deployments.map((address) => Address.fromString(address))
|
|
66
|
+
: [];
|
|
67
|
+
}
|
|
68
|
+
return this._deployments;
|
|
69
|
+
}
|
|
70
|
+
get rawTransactions() {
|
|
71
|
+
return this._rawBlock.transactions;
|
|
58
72
|
}
|
|
59
73
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export class LRUCache {
|
|
2
|
+
cache;
|
|
3
|
+
maxSize;
|
|
4
|
+
constructor(maxSize) {
|
|
5
|
+
this.cache = new Map();
|
|
6
|
+
this.maxSize = maxSize;
|
|
7
|
+
}
|
|
8
|
+
get(key) {
|
|
9
|
+
const value = this.cache.get(key);
|
|
10
|
+
if (value !== undefined) {
|
|
11
|
+
this.cache.delete(key);
|
|
12
|
+
this.cache.set(key, value);
|
|
13
|
+
}
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
set(key, value) {
|
|
17
|
+
if (this.cache.has(key)) {
|
|
18
|
+
this.cache.delete(key);
|
|
19
|
+
}
|
|
20
|
+
else if (this.cache.size >= this.maxSize) {
|
|
21
|
+
const firstKey = this.cache.keys().next().value;
|
|
22
|
+
if (firstKey !== undefined) {
|
|
23
|
+
this.cache.delete(firstKey);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
this.cache.set(key, value);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Address } from '@btc-vision/transaction';
|
|
2
|
+
import { LRUCache } from './LRUCaching.js';
|
|
3
|
+
const P2OP_CACHE_MAX_SIZE = 5_000;
|
|
4
|
+
const p2opCache = new LRUCache(P2OP_CACHE_MAX_SIZE);
|
|
5
|
+
const addressCache = new LRUCache(P2OP_CACHE_MAX_SIZE);
|
|
6
|
+
export const getP2op = (rawAddress, network) => {
|
|
7
|
+
const cacheKey = `${network.bip32}:${network.pubKeyHash}:${network.bech32}:${rawAddress}`;
|
|
8
|
+
let cached = p2opCache.get(cacheKey);
|
|
9
|
+
if (cached === undefined) {
|
|
10
|
+
let addr = addressCache.get(rawAddress);
|
|
11
|
+
if (addr === undefined) {
|
|
12
|
+
addr = Address.fromString(rawAddress);
|
|
13
|
+
addressCache.set(rawAddress, addr);
|
|
14
|
+
}
|
|
15
|
+
cached = addr.p2op(network);
|
|
16
|
+
p2opCache.set(cacheKey, cached);
|
|
17
|
+
}
|
|
18
|
+
return cached;
|
|
19
|
+
};
|
package/build/fetch/fetch.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { Agent } from 'undici';
|
|
2
|
-
import {
|
|
3
|
-
export declare function getFetcher(configs: Agent.Options):
|
|
2
|
+
import { FetcherWithCleanup } from './fetcher-type.js';
|
|
3
|
+
export declare function getFetcher(configs: Agent.Options): FetcherWithCleanup;
|
|
4
4
|
export default getFetcher;
|
package/build/fetch/fetch.js
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import pLimit from 'p-limit';
|
|
2
|
-
import { Agent, fetch as undiciFetch
|
|
2
|
+
import { Agent, fetch as undiciFetch } from 'undici';
|
|
3
3
|
export function getFetcher(configs) {
|
|
4
4
|
const agent = new Agent(configs);
|
|
5
|
-
setGlobalDispatcher(agent);
|
|
6
5
|
const limit = pLimit(500);
|
|
7
|
-
async function limitedFetch(
|
|
8
|
-
return limit(() => undiciFetch(...
|
|
6
|
+
async function limitedFetch(input, init) {
|
|
7
|
+
return limit(() => undiciFetch(input, { ...init, dispatcher: agent }));
|
|
9
8
|
}
|
|
10
|
-
return
|
|
9
|
+
return {
|
|
10
|
+
fetch: limitedFetch,
|
|
11
|
+
close: async () => {
|
|
12
|
+
await agent.close();
|
|
13
|
+
},
|
|
14
|
+
};
|
|
11
15
|
}
|
|
12
16
|
export default getFetcher;
|
|
@@ -7,11 +7,14 @@ export declare class JSONRpcProvider extends AbstractRpcProvider {
|
|
|
7
7
|
private readonly timeout;
|
|
8
8
|
private readonly fetcherConfigurations;
|
|
9
9
|
private useRESTAPI;
|
|
10
|
+
private readonly useThreadedParsing;
|
|
10
11
|
readonly url: string;
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
private _fetcherWithCleanup;
|
|
13
|
+
constructor(url: string, network: Network, timeout?: number, fetcherConfigurations?: Agent.Options, useRESTAPI?: boolean, useThreadedParsing?: boolean);
|
|
13
14
|
private get fetcher();
|
|
15
|
+
close(): Promise<void>;
|
|
14
16
|
setFetchMode(useRESTAPI: boolean): void;
|
|
15
17
|
_send(payload: JsonRpcPayload | JsonRpcPayload[]): Promise<JsonRpcCallResult>;
|
|
16
18
|
protected providerUrl(url: string): string;
|
|
19
|
+
private parseResponse;
|
|
17
20
|
}
|
|
@@ -1,28 +1,37 @@
|
|
|
1
1
|
import getFetcher from '../fetch/fetch.js';
|
|
2
|
+
import { jsonThreader } from '../threading/JSONThreader.js';
|
|
2
3
|
import { AbstractRpcProvider } from './AbstractRpcProvider.js';
|
|
3
4
|
export class JSONRpcProvider extends AbstractRpcProvider {
|
|
4
5
|
timeout;
|
|
5
6
|
fetcherConfigurations;
|
|
6
7
|
useRESTAPI;
|
|
8
|
+
useThreadedParsing;
|
|
7
9
|
url;
|
|
10
|
+
_fetcherWithCleanup;
|
|
8
11
|
constructor(url, network, timeout = 20_000, fetcherConfigurations = {
|
|
9
12
|
keepAliveTimeout: 30_000,
|
|
10
13
|
keepAliveTimeoutThreshold: 30_000,
|
|
11
14
|
connections: 128,
|
|
12
15
|
pipelining: 2,
|
|
13
|
-
}, useRESTAPI = true) {
|
|
16
|
+
}, useRESTAPI = true, useThreadedParsing = true) {
|
|
14
17
|
super(network);
|
|
15
18
|
this.timeout = timeout;
|
|
16
19
|
this.fetcherConfigurations = fetcherConfigurations;
|
|
17
20
|
this.useRESTAPI = useRESTAPI;
|
|
21
|
+
this.useThreadedParsing = useThreadedParsing;
|
|
18
22
|
this.url = this.providerUrl(url);
|
|
19
23
|
}
|
|
20
|
-
_fetcher;
|
|
21
24
|
get fetcher() {
|
|
22
|
-
if (!this.
|
|
23
|
-
this.
|
|
25
|
+
if (!this._fetcherWithCleanup) {
|
|
26
|
+
this._fetcherWithCleanup = getFetcher(this.fetcherConfigurations);
|
|
27
|
+
}
|
|
28
|
+
return this._fetcherWithCleanup.fetch;
|
|
29
|
+
}
|
|
30
|
+
async close() {
|
|
31
|
+
if (this._fetcherWithCleanup) {
|
|
32
|
+
await this._fetcherWithCleanup.close();
|
|
33
|
+
this._fetcherWithCleanup = undefined;
|
|
24
34
|
}
|
|
25
|
-
return this._fetcher;
|
|
26
35
|
}
|
|
27
36
|
setFetchMode(useRESTAPI) {
|
|
28
37
|
this.useRESTAPI = useRESTAPI;
|
|
@@ -50,7 +59,7 @@ export class JSONRpcProvider extends AbstractRpcProvider {
|
|
|
50
59
|
if (!resp.ok) {
|
|
51
60
|
throw new Error(`Failed to fetch: ${resp.statusText}`);
|
|
52
61
|
}
|
|
53
|
-
const fetchedData =
|
|
62
|
+
const fetchedData = await this.parseResponse(resp);
|
|
54
63
|
if (!fetchedData) {
|
|
55
64
|
throw new Error('No data fetched');
|
|
56
65
|
}
|
|
@@ -79,4 +88,11 @@ export class JSONRpcProvider extends AbstractRpcProvider {
|
|
|
79
88
|
return `${url}/api/v1/json-rpc`;
|
|
80
89
|
}
|
|
81
90
|
}
|
|
91
|
+
async parseResponse(resp) {
|
|
92
|
+
if (this.useThreadedParsing) {
|
|
93
|
+
const buffer = await resp.arrayBuffer();
|
|
94
|
+
return jsonThreader.parseBuffer(buffer);
|
|
95
|
+
}
|
|
96
|
+
return (await resp.json());
|
|
97
|
+
}
|
|
82
98
|
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { ThreaderOptions, WorkerScript } from './interfaces/IThread.js';
|
|
2
|
+
import { BaseThreader } from './SharedThreader.js';
|
|
3
|
+
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
4
|
+
[key: string]: JsonValue;
|
|
5
|
+
};
|
|
6
|
+
type JsonOp = 'parse' | 'stringify';
|
|
7
|
+
type JsonInput = string | JsonValue | ArrayBuffer;
|
|
8
|
+
type JsonOutput = string | JsonValue;
|
|
9
|
+
export declare class JsonThreader extends BaseThreader<JsonOp, JsonInput, JsonOutput> {
|
|
10
|
+
protected readonly workerScript: WorkerScript;
|
|
11
|
+
private readonly threadingThreshold;
|
|
12
|
+
constructor(options?: ThreaderOptions & {
|
|
13
|
+
threadingThreshold?: number;
|
|
14
|
+
});
|
|
15
|
+
parse<T = JsonValue>(json: string): Promise<T>;
|
|
16
|
+
parseBuffer<T = JsonValue>(buffer: ArrayBuffer): Promise<T>;
|
|
17
|
+
stringify(data: JsonValue): Promise<string>;
|
|
18
|
+
}
|
|
19
|
+
export declare const jsonThreader: JsonThreader;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { BaseThreader } from './SharedThreader.js';
|
|
2
|
+
import { jsonWorkerScript } from './worker-scripts/JSONWorker.js';
|
|
3
|
+
export class JsonThreader extends BaseThreader {
|
|
4
|
+
workerScript = jsonWorkerScript;
|
|
5
|
+
threadingThreshold;
|
|
6
|
+
constructor(options = {}) {
|
|
7
|
+
super(options);
|
|
8
|
+
this.threadingThreshold = options.threadingThreshold ?? 16_384;
|
|
9
|
+
}
|
|
10
|
+
async parse(json) {
|
|
11
|
+
if (json.length < this.threadingThreshold) {
|
|
12
|
+
return JSON.parse(json);
|
|
13
|
+
}
|
|
14
|
+
const result = await this.run('parse', json);
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
async parseBuffer(buffer) {
|
|
18
|
+
if (buffer.byteLength < this.threadingThreshold) {
|
|
19
|
+
const text = new TextDecoder().decode(buffer);
|
|
20
|
+
return JSON.parse(text);
|
|
21
|
+
}
|
|
22
|
+
const result = await this.runWithTransfer('parse', buffer, [buffer]);
|
|
23
|
+
return result;
|
|
24
|
+
}
|
|
25
|
+
async stringify(data) {
|
|
26
|
+
const result = JSON.stringify(data);
|
|
27
|
+
if (result.length < this.threadingThreshold) {
|
|
28
|
+
return result;
|
|
29
|
+
}
|
|
30
|
+
return (await this.run('stringify', data));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const GLOBAL_KEY = Symbol.for('opnet.jsonThreader');
|
|
34
|
+
const globalObj = (typeof globalThis !== 'undefined' ? globalThis : global);
|
|
35
|
+
if (!globalObj[GLOBAL_KEY]) {
|
|
36
|
+
globalObj[GLOBAL_KEY] = new JsonThreader();
|
|
37
|
+
}
|
|
38
|
+
export const jsonThreader = globalObj[GLOBAL_KEY];
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Logger } from '@btc-vision/logger';
|
|
2
|
+
import { ThreaderOptions, WorkerScript } from './interfaces/IThread.js';
|
|
3
|
+
export declare abstract class BaseThreader<TOp extends string, TData, TResult> extends Logger {
|
|
4
|
+
readonly logColor: string;
|
|
5
|
+
protected abstract readonly workerScript: WorkerScript;
|
|
6
|
+
private workers;
|
|
7
|
+
private available;
|
|
8
|
+
private pending;
|
|
9
|
+
private queue;
|
|
10
|
+
private idCounter;
|
|
11
|
+
private readonly poolSize;
|
|
12
|
+
private initialized;
|
|
13
|
+
private initializing;
|
|
14
|
+
private tasksProcessed;
|
|
15
|
+
private tasksFailed;
|
|
16
|
+
private lastStatsLog;
|
|
17
|
+
private readonly statsInterval;
|
|
18
|
+
private cleanupBound;
|
|
19
|
+
protected constructor(options?: ThreaderOptions);
|
|
20
|
+
get stats(): {
|
|
21
|
+
pending: number;
|
|
22
|
+
queued: number;
|
|
23
|
+
available: number;
|
|
24
|
+
total: number;
|
|
25
|
+
processed: number;
|
|
26
|
+
failed: number;
|
|
27
|
+
};
|
|
28
|
+
terminate(): Promise<void>;
|
|
29
|
+
drain(): Promise<void>;
|
|
30
|
+
protected runWithTransfer(op: TOp, data: TData, transferables: ArrayBuffer[]): Promise<TResult>;
|
|
31
|
+
protected run(op: TOp, data: TData): Promise<TResult>;
|
|
32
|
+
private bindCleanupHandlers;
|
|
33
|
+
private init;
|
|
34
|
+
private logStatsIfNeeded;
|
|
35
|
+
private processQueue;
|
|
36
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { Logger } from '@btc-vision/logger';
|
|
2
|
+
import { createWorker, isNode } from './WorkerCreator.js';
|
|
3
|
+
export class BaseThreader extends Logger {
|
|
4
|
+
logColor = '#FF5733';
|
|
5
|
+
workers = [];
|
|
6
|
+
available = [];
|
|
7
|
+
pending = new Map();
|
|
8
|
+
queue = [];
|
|
9
|
+
idCounter = 0;
|
|
10
|
+
poolSize;
|
|
11
|
+
initialized = false;
|
|
12
|
+
initializing = null;
|
|
13
|
+
tasksProcessed = 0;
|
|
14
|
+
tasksFailed = 0;
|
|
15
|
+
lastStatsLog = 0;
|
|
16
|
+
statsInterval = 30_000;
|
|
17
|
+
cleanupBound = false;
|
|
18
|
+
constructor(options = {}) {
|
|
19
|
+
super();
|
|
20
|
+
this.poolSize = options.poolSize ?? (isNode ? 6 : (navigator?.hardwareConcurrency ?? 4));
|
|
21
|
+
}
|
|
22
|
+
get stats() {
|
|
23
|
+
return {
|
|
24
|
+
pending: this.pending.size,
|
|
25
|
+
queued: this.queue.length,
|
|
26
|
+
available: this.available.length,
|
|
27
|
+
total: this.workers.length,
|
|
28
|
+
processed: this.tasksProcessed,
|
|
29
|
+
failed: this.tasksFailed,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
async terminate() {
|
|
33
|
+
if (!this.initialized && !this.initializing)
|
|
34
|
+
return;
|
|
35
|
+
const queuedCount = this.queue.length;
|
|
36
|
+
const pendingCount = this.pending.size;
|
|
37
|
+
for (const task of this.queue) {
|
|
38
|
+
task.reject(new Error('Threader terminated'));
|
|
39
|
+
}
|
|
40
|
+
for (const [, handler] of this.pending) {
|
|
41
|
+
handler.reject(new Error('Threader terminated'));
|
|
42
|
+
}
|
|
43
|
+
for (const worker of this.workers) {
|
|
44
|
+
await worker.terminate();
|
|
45
|
+
}
|
|
46
|
+
this.queue = [];
|
|
47
|
+
this.pending.clear();
|
|
48
|
+
this.workers = [];
|
|
49
|
+
this.available = [];
|
|
50
|
+
this.initialized = false;
|
|
51
|
+
this.initializing = null;
|
|
52
|
+
if (queuedCount > 0 || pendingCount > 0) {
|
|
53
|
+
this.info(`Terminated. Rejected ${queuedCount} queued and ${pendingCount} pending tasks. Total processed: ${this.tasksProcessed}, failed: ${this.tasksFailed}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
async drain() {
|
|
57
|
+
if (!this.initialized)
|
|
58
|
+
return;
|
|
59
|
+
const queuedCount = this.queue.length;
|
|
60
|
+
const pendingCount = this.pending.size;
|
|
61
|
+
this.info(`Draining. Rejecting ${queuedCount} queued, waiting for ${pendingCount} pending...`);
|
|
62
|
+
for (const task of this.queue) {
|
|
63
|
+
task.reject(new Error('Threader draining'));
|
|
64
|
+
}
|
|
65
|
+
this.queue = [];
|
|
66
|
+
if (this.pending.size > 0) {
|
|
67
|
+
await new Promise((resolve) => {
|
|
68
|
+
const checkDone = () => {
|
|
69
|
+
if (this.pending.size === 0) {
|
|
70
|
+
resolve();
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
const originalPending = new Map(this.pending);
|
|
74
|
+
for (const [id, handler] of originalPending) {
|
|
75
|
+
const origResolve = handler.resolve;
|
|
76
|
+
const origReject = handler.reject;
|
|
77
|
+
handler.resolve = (v) => {
|
|
78
|
+
origResolve(v);
|
|
79
|
+
checkDone();
|
|
80
|
+
};
|
|
81
|
+
handler.reject = (e) => {
|
|
82
|
+
origReject(e);
|
|
83
|
+
checkDone();
|
|
84
|
+
};
|
|
85
|
+
this.pending.set(id, handler);
|
|
86
|
+
}
|
|
87
|
+
checkDone();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
await this.terminate();
|
|
91
|
+
}
|
|
92
|
+
runWithTransfer(op, data, transferables) {
|
|
93
|
+
return new Promise(async (resolve, reject) => {
|
|
94
|
+
await this.init();
|
|
95
|
+
this.queue.push({ resolve, reject, op, data, transferables });
|
|
96
|
+
this.processQueue();
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
run(op, data) {
|
|
100
|
+
return new Promise(async (resolve, reject) => {
|
|
101
|
+
await this.init();
|
|
102
|
+
this.queue.push({ resolve, reject, op, data });
|
|
103
|
+
this.processQueue();
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
bindCleanupHandlers() {
|
|
107
|
+
if (this.cleanupBound)
|
|
108
|
+
return;
|
|
109
|
+
this.cleanupBound = true;
|
|
110
|
+
const cleanup = () => {
|
|
111
|
+
this.terminate().catch(() => { });
|
|
112
|
+
};
|
|
113
|
+
if (isNode) {
|
|
114
|
+
process.once('beforeExit', cleanup);
|
|
115
|
+
process.once('SIGINT', cleanup);
|
|
116
|
+
process.once('SIGTERM', cleanup);
|
|
117
|
+
}
|
|
118
|
+
else if (typeof window !== 'undefined') {
|
|
119
|
+
window.addEventListener('beforeunload', cleanup);
|
|
120
|
+
window.addEventListener('unload', cleanup);
|
|
121
|
+
}
|
|
122
|
+
else if (typeof self !== 'undefined') {
|
|
123
|
+
self.addEventListener('beforeunload', cleanup);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async init() {
|
|
127
|
+
if (this.initialized)
|
|
128
|
+
return;
|
|
129
|
+
if (this.initializing)
|
|
130
|
+
return this.initializing;
|
|
131
|
+
this.bindCleanupHandlers();
|
|
132
|
+
this.initializing = (async () => {
|
|
133
|
+
const startTime = Date.now();
|
|
134
|
+
const workers = await Promise.all(Array.from({ length: this.poolSize }, () => createWorker(this.workerScript)));
|
|
135
|
+
for (const worker of workers) {
|
|
136
|
+
worker.onMessage((msg) => {
|
|
137
|
+
const handler = this.pending.get(msg.id);
|
|
138
|
+
if (handler) {
|
|
139
|
+
this.pending.delete(msg.id);
|
|
140
|
+
if (msg.error) {
|
|
141
|
+
this.tasksFailed++;
|
|
142
|
+
handler.reject(new Error(msg.error));
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
this.tasksProcessed++;
|
|
146
|
+
handler.resolve(msg.result);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
this.available.push(worker);
|
|
150
|
+
this.logStatsIfNeeded();
|
|
151
|
+
this.processQueue();
|
|
152
|
+
});
|
|
153
|
+
this.workers.push(worker);
|
|
154
|
+
this.available.push(worker);
|
|
155
|
+
}
|
|
156
|
+
this.initialized = true;
|
|
157
|
+
})();
|
|
158
|
+
return this.initializing;
|
|
159
|
+
}
|
|
160
|
+
logStatsIfNeeded() {
|
|
161
|
+
const now = Date.now();
|
|
162
|
+
if (now - this.lastStatsLog < this.statsInterval)
|
|
163
|
+
return;
|
|
164
|
+
this.lastStatsLog = now;
|
|
165
|
+
const s = this.stats;
|
|
166
|
+
this.debug(`Stats: ${s.processed} processed, ${s.failed} failed, ${s.pending} pending, ${s.queued} queued, ${s.available}/${s.total} workers available`);
|
|
167
|
+
}
|
|
168
|
+
processQueue() {
|
|
169
|
+
if (this.queue.length > 100 && this.available.length === 0) {
|
|
170
|
+
this.warn(`Queue backing up: ${this.queue.length} tasks waiting, no workers available`);
|
|
171
|
+
}
|
|
172
|
+
while (this.queue.length > 0 && this.available.length > 0) {
|
|
173
|
+
const task = this.queue.shift();
|
|
174
|
+
if (!task)
|
|
175
|
+
break;
|
|
176
|
+
const worker = this.available.pop();
|
|
177
|
+
if (!worker)
|
|
178
|
+
break;
|
|
179
|
+
const id = this.idCounter++;
|
|
180
|
+
this.pending.set(id, task);
|
|
181
|
+
worker.postMessage({ id, op: task.op, data: task.data }, task.transferables);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|