@isidorus/cpu 0.0.0-alpha.3 → 0.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +1 -1
- package/dist/graph.d.ts +19 -12
- package/dist/graph.d.ts.map +1 -1
- package/dist/graph.js +23 -16
- package/dist/graph.js.map +1 -1
- package/dist/inference-pool.d.ts +35 -12
- package/dist/inference-pool.d.ts.map +1 -1
- package/dist/inference-pool.js +226 -259
- package/dist/inference-pool.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -3
- package/prebuilds/darwin-arm64/@isidorus+cpu.node +0 -0
- package/prebuilds/linux-x64/@isidorus+cpu.node +0 -0
- package/prebuilds/win32-x64/@isidorus+cpu.node +0 -0
- package/scripts/test-install.js +1 -1
- package/src/native/graph.cc +356 -255
- package/src/native/graph.h +2 -0
package/binding.gyp
CHANGED
package/dist/graph.d.ts
CHANGED
|
@@ -9,6 +9,9 @@ export type AttrValue = {
|
|
|
9
9
|
} | {
|
|
10
10
|
kind: "bool";
|
|
11
11
|
value: boolean;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "string";
|
|
14
|
+
value: string;
|
|
12
15
|
} | {
|
|
13
16
|
kind: "type";
|
|
14
17
|
value: DType;
|
|
@@ -21,9 +24,6 @@ export type AttrValue = {
|
|
|
21
24
|
} | {
|
|
22
25
|
kind: "list_int";
|
|
23
26
|
value: number[];
|
|
24
|
-
} | {
|
|
25
|
-
kind: "string";
|
|
26
|
-
value: string;
|
|
27
27
|
} | {
|
|
28
28
|
kind: "tensor";
|
|
29
29
|
value: InlineTensor;
|
|
@@ -57,14 +57,14 @@ export declare class Graph {
|
|
|
57
57
|
/**
|
|
58
58
|
* Add a raw op to the graph.
|
|
59
59
|
*
|
|
60
|
-
* @param type
|
|
61
|
-
* @param inputs
|
|
62
|
-
* @param attrs
|
|
63
|
-
* @param name
|
|
60
|
+
* @param type TF op type string, e.g. "MatMul", "Placeholder"
|
|
61
|
+
* @param inputs Data inputs — output references from prior ops
|
|
62
|
+
* @param attrs Op attributes
|
|
63
|
+
* @param name Optional explicit op name (auto-generated if omitted)
|
|
64
64
|
* @param controlInputs Op names that must complete before this op runs.
|
|
65
65
|
* Used by globalVariablesInitializer to sequence init
|
|
66
66
|
* ops before the NoOp target that callers wait on.
|
|
67
|
-
* @returns
|
|
67
|
+
* @returns Array of output Tensors (one per op output)
|
|
68
68
|
*/
|
|
69
69
|
addOp(type: string, inputs: TFOutput[], attrs?: Record<string, AttrValue>, name?: string, controlInputs?: string[]): Tensor[];
|
|
70
70
|
/**
|
|
@@ -89,10 +89,7 @@ export declare class Graph {
|
|
|
89
89
|
hasOp(name: string): boolean;
|
|
90
90
|
/** Total number of ops in the graph. */
|
|
91
91
|
get numOps(): number;
|
|
92
|
-
/**
|
|
93
|
-
* Serialise the graph to a binary GraphDef proto.
|
|
94
|
-
* Useful for saving, inspecting with Netron, or freezing.
|
|
95
|
-
*/
|
|
92
|
+
/** Serialise the graph to a binary GraphDef proto. */
|
|
96
93
|
toGraphDef(): Buffer;
|
|
97
94
|
/**
|
|
98
95
|
* importGraphDef — deserialise a binary GraphDef proto into this graph.
|
|
@@ -110,6 +107,16 @@ export declare class Graph {
|
|
|
110
107
|
* const sess = session(g, { strategy: "tf-parallel", reserveCores: 2 });
|
|
111
108
|
*/
|
|
112
109
|
importGraphDef(buffer: Buffer): void;
|
|
110
|
+
/**
|
|
111
|
+
* listOpsOfType — return the names of all ops whose type matches `type`.
|
|
112
|
+
* Used to auto-discover Placeholder (input) op names in frozen graphs.
|
|
113
|
+
*/
|
|
114
|
+
listOpsOfType(type: string): string[];
|
|
115
|
+
/**
|
|
116
|
+
* listSinkOps — return op names whose outputs are not consumed by any
|
|
117
|
+
* other op in the graph. These are the natural output ops of a frozen graph.
|
|
118
|
+
*/
|
|
119
|
+
listSinkOps(): string[];
|
|
113
120
|
getOp(name: string, index?: number): Tensor | null;
|
|
114
121
|
}
|
|
115
122
|
//# sourceMappingURL=graph.d.ts.map
|
package/dist/graph.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graph.d.ts","sourceRoot":"","sources":["../src/ts/graph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"graph.d.ts","sourceRoot":"","sources":["../src/ts/graph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAS,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,KAAK,EAA2B,MAAM,gBAAgB,CAAC;AAEhE,MAAM,MAAM,SAAS,GACjB;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,GAClC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,KAAK,EAAE,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,YAAY,CAAA;CAAE,CAAC;AAE5C,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,QAAQ,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD;;;;;;;;;;;;GAYG;AACH,qBAAa,KAAK;IAChB,gBAAgB;IAChB,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC;gBAEV,MAAM,EAAE,GAAG;IAIvB;;;;;;;;;;;OAWG;IACH,KAAK,CACH,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,QAAQ,EAAE,EAClB,KAAK,GAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAM,EACrC,IAAI,CAAC,EAAE,MAAM,EACb,aAAa,GAAE,MAAM,EAAO,GAC3B,MAAM,EAAE;IAsCX;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,MAAM,EAAE;IAgBrE,8DAA8D;IAC9D,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAI5B,wCAAwC;IACxC,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,sDAAsD;IACtD,UAAU,IAAI,MAAM;IAIpB;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIpC;;;OAGG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAIrC;;;OAGG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAI,GAAG,MAAM,GAAG,IAAI;CAW9C"}
|
package/dist/graph.js
CHANGED
|
@@ -21,27 +21,24 @@ export class Graph {
|
|
|
21
21
|
/**
|
|
22
22
|
* Add a raw op to the graph.
|
|
23
23
|
*
|
|
24
|
-
* @param type
|
|
25
|
-
* @param inputs
|
|
26
|
-
* @param attrs
|
|
27
|
-
* @param name
|
|
24
|
+
* @param type TF op type string, e.g. "MatMul", "Placeholder"
|
|
25
|
+
* @param inputs Data inputs — output references from prior ops
|
|
26
|
+
* @param attrs Op attributes
|
|
27
|
+
* @param name Optional explicit op name (auto-generated if omitted)
|
|
28
28
|
* @param controlInputs Op names that must complete before this op runs.
|
|
29
29
|
* Used by globalVariablesInitializer to sequence init
|
|
30
30
|
* ops before the NoOp target that callers wait on.
|
|
31
|
-
* @returns
|
|
31
|
+
* @returns Array of output Tensors (one per op output)
|
|
32
32
|
*/
|
|
33
33
|
addOp(type, inputs, attrs = {}, name, controlInputs = []) {
|
|
34
34
|
const nativeAttrs = {};
|
|
35
35
|
for (const [k, v] of Object.entries(attrs)) {
|
|
36
|
-
if (v.kind === "list_type")
|
|
36
|
+
if (v.kind === "list_type")
|
|
37
37
|
nativeAttrs[k] = { kind: "list_type", value: v.value.map(Number) };
|
|
38
|
-
|
|
39
|
-
else if (v.kind === "type") {
|
|
38
|
+
else if (v.kind === "type")
|
|
40
39
|
nativeAttrs[k] = { kind: "type", value: Number(v.value) };
|
|
41
|
-
|
|
42
|
-
else {
|
|
40
|
+
else
|
|
43
41
|
nativeAttrs[k] = v;
|
|
44
|
-
}
|
|
45
42
|
}
|
|
46
43
|
const result = this._native.addOp(type, inputs, nativeAttrs, name, controlInputs);
|
|
47
44
|
const { opName, numOutputs } = result;
|
|
@@ -86,10 +83,7 @@ export class Graph {
|
|
|
86
83
|
get numOps() {
|
|
87
84
|
return this._native.numOps();
|
|
88
85
|
}
|
|
89
|
-
/**
|
|
90
|
-
* Serialise the graph to a binary GraphDef proto.
|
|
91
|
-
* Useful for saving, inspecting with Netron, or freezing.
|
|
92
|
-
*/
|
|
86
|
+
/** Serialise the graph to a binary GraphDef proto. */
|
|
93
87
|
toGraphDef() {
|
|
94
88
|
return this._native.toGraphDef();
|
|
95
89
|
}
|
|
@@ -111,7 +105,20 @@ export class Graph {
|
|
|
111
105
|
importGraphDef(buffer) {
|
|
112
106
|
this._native.importGraphDef(buffer);
|
|
113
107
|
}
|
|
114
|
-
|
|
108
|
+
/**
|
|
109
|
+
* listOpsOfType — return the names of all ops whose type matches `type`.
|
|
110
|
+
* Used to auto-discover Placeholder (input) op names in frozen graphs.
|
|
111
|
+
*/
|
|
112
|
+
listOpsOfType(type) {
|
|
113
|
+
return this._native.listOpsOfType(type);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* listSinkOps — return op names whose outputs are not consumed by any
|
|
117
|
+
* other op in the graph. These are the natural output ops of a frozen graph.
|
|
118
|
+
*/
|
|
119
|
+
listSinkOps() {
|
|
120
|
+
return this._native.listSinkOps();
|
|
121
|
+
}
|
|
115
122
|
getOp(name, index = 0) {
|
|
116
123
|
if (!this._native.hasOp(name))
|
|
117
124
|
return null;
|
package/dist/graph.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graph.js","sourceRoot":"","sources":["../src/ts/graph.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"graph.js","sourceRoot":"","sources":["../src/ts/graph.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAqBhE;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,KAAK;IAChB,gBAAgB;IACP,OAAO,CAAM;IAEtB,YAAY,MAAW;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CACH,IAAY,EACZ,MAAkB,EAClB,QAAmC,EAAE,EACrC,IAAa,EACb,gBAA0B,EAAE;QAE5B,MAAM,WAAW,GAAwB,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;gBACxB,WAAW,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;iBAChE,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBACxB,WAAW,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;;gBACvD,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAC/B,IAAI,EACJ,MAAM,EACN,WAAW,EACX,IAAI,EACJ,aAAa,CACd,CAAC;QACF,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAG9B,CAAC;QAEF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAkB,CAAC;YACtE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAoB,CAAC;YACzE,OAAO,CAAC,IAAI,CACV,UAAU,CACR,MAAM,EACN,CAAC,EACD,OAAO,IAAI,IAAI,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAC,IAAI,EAC3C,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAC9C,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,CAAa,EAAE,CAAa,EAAE,EAAe;QACxD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,IAAI,CAAe,CAAC;QACtE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAkB,CAAC;YAC1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAEhD,CAAC;YACT,OAAO,UAAU,CACf,MAAM,EACN,KAAK,EACL,OAAO,IAAI,IAAI,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAC,IAAI,EAC3C,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8DAA8D;IAC9D,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC7C,CAAC;IAED,wCAAwC;IACxC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAY,CAAC;IACzC,CAAC;IAED,sDAAsD;IACtD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAY,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,MAAc;QAC3B,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAa,CAAC;IACtD,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,KAAK,GAAG,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAkB,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAoB,CAAC;QAC3E,OAAO,UAAU,CACf,IAAI,EACJ,KAAK,EACL,OAAO,IAAI,IAAI,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAC,IAAI,EAC3C,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAC9C,CAAC;IACJ,CAAC;CACF"}
|
package/dist/inference-pool.d.ts
CHANGED
|
@@ -2,45 +2,69 @@
|
|
|
2
2
|
* InferencePool — strategy-aware inference execution with CPU affinity.
|
|
3
3
|
*
|
|
4
4
|
* Strategies:
|
|
5
|
-
*
|
|
6
|
-
* worker-pool N Workers × Session(intra=1, inter=1)
|
|
5
|
+
* worker-pool N Workers × native Session(intra=1, inter=1)
|
|
7
6
|
* JS controls parallelism. Each Worker owns one Session.
|
|
8
7
|
* N concurrent requests run on N cores simultaneously.
|
|
9
8
|
* Best: small/medium models, high concurrency.
|
|
10
9
|
*
|
|
11
|
-
* tf-parallel 1 Session ×
|
|
10
|
+
* tf-parallel 1 native Session × (intra=hw−reserveCores, inter=1)
|
|
12
11
|
* TF's eigen threadpool owns all TF cores for one request.
|
|
13
12
|
* Concurrent requests queue behind each other.
|
|
14
13
|
* Best: large models where one matmul fills all cores.
|
|
15
14
|
*
|
|
16
15
|
* auto Probe-based selection:
|
|
17
|
-
* model < 150 MB
|
|
18
|
-
* model ≥ 150 MB + probeShape
|
|
19
|
-
* model ≥ 150 MB, no probeShape
|
|
16
|
+
* model < 150 MB → worker-pool (no probe)
|
|
17
|
+
* model ≥ 150 MB + probeShape → warm probe → threshold
|
|
18
|
+
* model ≥ 150 MB, no probeShape → tf-parallel (fallback)
|
|
20
19
|
*
|
|
21
20
|
* CPU affinity (reserveCores):
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
21
|
+
* Pins TF compute to the LAST (N−reserveCores) cores via OS affinity fence
|
|
22
|
+
* applied in OnRunWork immediately before/after TF_SessionRun.
|
|
23
|
+
*
|
|
24
|
+
* Transport:
|
|
25
|
+
* Control plane (state machine) — 4-slot Int32Array over a SharedArrayBuffer.
|
|
26
|
+
* Main stores WORK, Worker Atomics.wait()s and observes it. Tiny, exact.
|
|
27
|
+
*
|
|
28
|
+
* Data plane (tensor bytes) — one SharedTensorSegment (jude-map) per slot.
|
|
29
|
+
* Main calls seg.write(shape, dtype, bytes) — seqlock write, zero copy.
|
|
30
|
+
* Worker calls seg.read() — seqlock read, zero copy.
|
|
31
|
+
* No postMessage for input data. No SAB size limit concern (control SAB
|
|
32
|
+
* is 16 bytes per worker; data segments are sized to the model's input).
|
|
33
|
+
* Atomics are slower than seqlocks — jude-map's seqlock handles the data
|
|
34
|
+
* plane; Atomics handle only the 4-slot state machine.
|
|
25
35
|
*/
|
|
26
36
|
export type ExecutionStrategy = "worker-pool" | "tf-parallel" | "auto";
|
|
27
37
|
export interface PoolOptions {
|
|
28
38
|
modelPath: string;
|
|
39
|
+
/**
|
|
40
|
+
* Input op name. Optional — if omitted the first Placeholder op in the
|
|
41
|
+
* frozen graph is used (auto-discover via native graph scan).
|
|
42
|
+
*/
|
|
29
43
|
inputOp?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Output op names. Optional — if omitted the sink ops (ops whose outputs
|
|
46
|
+
* are not consumed by any other op) are used.
|
|
47
|
+
*/
|
|
30
48
|
outputOps?: string[];
|
|
31
49
|
/**
|
|
32
50
|
* Reserve the first R cores for the event loop and other native libs.
|
|
33
|
-
* TF inference is pinned to the last (N
|
|
51
|
+
* TF inference is pinned to the last (N−R) cores via OS affinity.
|
|
34
52
|
* Default: 0 (no reservation).
|
|
35
53
|
*/
|
|
36
54
|
reserveCores?: number;
|
|
37
55
|
strategy?: ExecutionStrategy;
|
|
38
|
-
/** Worker thread count (worker-pool
|
|
56
|
+
/** Worker thread count (worker-pool). Default: os.availableParallelism(). */
|
|
39
57
|
concurrency?: number;
|
|
40
58
|
/** Input shape for auto probe. Required when model >= 150 MB. */
|
|
41
59
|
probeShape?: number[];
|
|
42
60
|
/** Crossover threshold for auto strategy (ms). Default: 20. */
|
|
43
61
|
autoThresholdMs?: number;
|
|
62
|
+
/**
|
|
63
|
+
* Capacity of each Worker's SharedTensorSegment data buffer (bytes).
|
|
64
|
+
* Must be >= the byte size of the largest input tensor you will feed.
|
|
65
|
+
* Default: 4 MB.
|
|
66
|
+
*/
|
|
67
|
+
maxInputBytes?: number;
|
|
44
68
|
}
|
|
45
69
|
export interface PoolResult {
|
|
46
70
|
workerId: number;
|
|
@@ -72,7 +96,6 @@ export declare class InferencePool {
|
|
|
72
96
|
infer(inputBuf: Buffer, inputShape: number[], inputDtype?: number): Promise<PoolResult>;
|
|
73
97
|
private inferWorkerPool;
|
|
74
98
|
private dispatchToWorker;
|
|
75
|
-
/** Settle a worker slot's in-flight promise and drain the queue. */
|
|
76
99
|
private settleSlot;
|
|
77
100
|
private inferTfParallel;
|
|
78
101
|
private runTfParallel;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"inference-pool.d.ts","sourceRoot":"","sources":["../src/ts/inference-pool.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"inference-pool.d.ts","sourceRoot":"","sources":["../src/ts/inference-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAgCH,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG,aAAa,GAAG,MAAM,CAAC;AAEvE,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,+DAA+D;IAC/D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,aAAa,GAAG,aAAa,CAAC;IACxC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5D,WAAW,EAAE,MAAM,CAAC;CACrB;AAwLD,qBAAa,aAAa;IACxB,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,aAAa,CAAC;IACjD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAe;IAC3C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IACnD,OAAO,CAAC,eAAe,CAAe;IACtC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IAErC,OAAO;WA4BM,MAAM,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;mBAiHzC,gBAAgB;mBA0FhB,gBAAgB;IA2CrC,KAAK,CACH,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,SAAI,GACb,OAAO,CAAC,UAAU,CAAC;IAMtB,OAAO,CAAC,eAAe;IAqBvB,OAAO,CAAC,gBAAgB;IAmDxB,OAAO,CAAC,UAAU;IA4BlB,OAAO,CAAC,eAAe;IAoBvB,OAAO,CAAC,aAAa;IAwErB,IAAI,SAAS,IAAI,MAAM,CAItB;IAED,IAAI,UAAU,IAAI,MAAM,CAIvB;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAIK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAkD/B"}
|