@pond-ts/process 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5914 -0
- package/LICENSE +21 -0
- package/README.md +345 -0
- package/dist/cjs-fallback.cjs +15 -0
- package/dist/column.d.ts +197 -0
- package/dist/column.js +306 -0
- package/dist/errors.d.ts +22 -0
- package/dist/errors.js +25 -0
- package/dist/graph.d.ts +89 -0
- package/dist/graph.js +133 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.js +45 -0
- package/dist/node.d.ts +151 -0
- package/dist/node.js +268 -0
- package/dist/plan/builder.d.ts +138 -0
- package/dist/plan/builder.js +166 -0
- package/dist/plan/fluent.d.ts +93 -0
- package/dist/plan/fluent.js +140 -0
- package/dist/plan/folds.d.ts +25 -0
- package/dist/plan/folds.js +190 -0
- package/dist/plan/graph.d.ts +171 -0
- package/dist/plan/graph.js +658 -0
- package/dist/plan/history.d.ts +61 -0
- package/dist/plan/history.js +82 -0
- package/dist/plan/host.d.ts +173 -0
- package/dist/plan/host.js +234 -0
- package/dist/plan/identity.d.ts +81 -0
- package/dist/plan/identity.js +158 -0
- package/dist/plan/params.d.ts +15 -0
- package/dist/plan/params.js +26 -0
- package/dist/plan/registry.d.ts +162 -0
- package/dist/plan/registry.js +422 -0
- package/dist/plan/run.d.ts +211 -0
- package/dist/plan/run.js +360 -0
- package/dist/plan/slots.d.ts +65 -0
- package/dist/plan/slots.js +114 -0
- package/dist/plan/source.d.ts +49 -0
- package/dist/plan/source.js +54 -0
- package/dist/plan/types.d.ts +376 -0
- package/dist/plan/types.js +20 -0
- package/dist/pool/index.d.ts +15 -0
- package/dist/pool/index.js +12 -0
- package/dist/pool/pool.d.ts +92 -0
- package/dist/pool/pool.js +237 -0
- package/dist/pool/protocol.d.ts +48 -0
- package/dist/pool/protocol.js +9 -0
- package/dist/pool/wire.d.ts +52 -0
- package/dist/pool/wire.js +95 -0
- package/dist/pool/worker.d.ts +22 -0
- package/dist/pool/worker.js +80 -0
- package/dist/port.d.ts +79 -0
- package/dist/port.js +222 -0
- package/dist/source.d.ts +161 -0
- package/dist/source.js +182 -0
- package/dist/types.d.ts +77 -0
- package/dist/types.js +26 -0
- package/package.json +50 -0
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `HostPool` — whole requests across resident workers ([PND-PROCPAR]).
|
|
3
|
+
*
|
|
4
|
+
* ## Which parallelism this is
|
|
5
|
+
*
|
|
6
|
+
* The worker-threads assessment
|
|
7
|
+
* (`docs/notes/worker-threads-assessment-2026-07.md`) found two distinct
|
|
8
|
+
* wins and warned against conflating them:
|
|
9
|
+
*
|
|
10
|
+
* - **Latency** of one composite query — split its nodes across workers.
|
|
11
|
+
* Measured 2.42× on a 5-study stack, and it needs an engine change: a
|
|
12
|
+
* node's value can only be produced by its own `compute`, so a result
|
|
13
|
+
* computed elsewhere has nowhere to land. Still ahead of us.
|
|
14
|
+
* - **Throughput** under concurrent queries — run *whole* requests, each
|
|
15
|
+
* single-threaded, on a pool of resident hosts. Near-linear, no
|
|
16
|
+
* decomposition, no numeric-semantics questions, **and no engine
|
|
17
|
+
* change at all**.
|
|
18
|
+
*
|
|
19
|
+
* This is the second. It is deliberately first: the agent workload is
|
|
20
|
+
* many overlapping questions, so throughput is what it feels, and this
|
|
21
|
+
* shape cannot get an answer wrong — each request runs the same
|
|
22
|
+
* single-threaded code it runs today, in a different isolate.
|
|
23
|
+
*
|
|
24
|
+
* ## What makes it cheap
|
|
25
|
+
*
|
|
26
|
+
* Each worker holds a **long-lived `Host`**, so compiled nodes and
|
|
27
|
+
* cached columns survive between requests exactly as they do in-process.
|
|
28
|
+
* The pool is only a router. What crosses the boundary is a plan
|
|
29
|
+
* (JSON by construction) and a result whose columns travel as
|
|
30
|
+
* transferable buffers rather than boxed values.
|
|
31
|
+
*
|
|
32
|
+
* ## What it is not
|
|
33
|
+
*
|
|
34
|
+
* Not a cache-sharing scheme. Each worker warms its own graph, so N
|
|
35
|
+
* workers hold up to N copies of a hot column and a question already
|
|
36
|
+
* answered by worker 2 is cold on worker 3. That is the honest cost of
|
|
37
|
+
* the simple shape: it buys throughput, not deduplication. Route related
|
|
38
|
+
* requests to the same worker (see {@link HostPool.run}'s `affinity`) if
|
|
39
|
+
* repeat-hit rate matters more than even spread.
|
|
40
|
+
*/
|
|
41
|
+
import { Worker } from 'node:worker_threads';
|
|
42
|
+
import { existsSync } from 'node:fs';
|
|
43
|
+
import { fileURLToPath } from 'node:url';
|
|
44
|
+
import { ProcessError } from '../errors.js';
|
|
45
|
+
import { fromWire } from './wire.js';
|
|
46
|
+
/** Default pool size: leave a core for the main thread. */
|
|
47
|
+
async function defaultSize() {
|
|
48
|
+
const os = await import('node:os');
|
|
49
|
+
const cores = os.availableParallelism?.() ?? os.cpus().length;
|
|
50
|
+
return Math.max(1, cores - 1);
|
|
51
|
+
}
|
|
52
|
+
export class HostPool {
|
|
53
|
+
#slots;
|
|
54
|
+
#pending = new Map();
|
|
55
|
+
#nextId = 1;
|
|
56
|
+
#closed = false;
|
|
57
|
+
/**
|
|
58
|
+
* Set when a worker dies unexpectedly. A dead worker cannot answer, so
|
|
59
|
+
* every *later* request must fail too — rejecting only the in-flight
|
|
60
|
+
* ones left a pool that looked alive and swallowed everything sent to
|
|
61
|
+
* it. A pool is not self-healing: replacing a worker would silently
|
|
62
|
+
* discard its warm graph, so failing loudly is the honest response.
|
|
63
|
+
*/
|
|
64
|
+
#fatal;
|
|
65
|
+
constructor(slots) {
|
|
66
|
+
this.#slots = slots;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Starts the pool and waits for every worker to be listening.
|
|
70
|
+
*
|
|
71
|
+
* Setup modules are imported lazily *inside* each worker on its first
|
|
72
|
+
* request, so `start` resolving does not mean the registry loaded — a
|
|
73
|
+
* broken setup module surfaces as that first request's error, where a
|
|
74
|
+
* caller can read it, rather than as a start-up crash with no
|
|
75
|
+
* request to attach it to.
|
|
76
|
+
*/
|
|
77
|
+
static async start(options) {
|
|
78
|
+
const size = options.size ?? (await defaultSize());
|
|
79
|
+
if (!Number.isInteger(size) || size < 1) {
|
|
80
|
+
throw new ProcessError(`HostPool: size must be >= 1, got ${size}`);
|
|
81
|
+
}
|
|
82
|
+
const setup = options.setup instanceof URL ? options.setup.href : String(options.setup);
|
|
83
|
+
// The worker entry sits next to this module in `dist`. Checked
|
|
84
|
+
// rather than assumed: a caller running the package from TypeScript
|
|
85
|
+
// source (a test runner, a bundler) resolves this to a `.ts` file no
|
|
86
|
+
// Node worker can load, and `new Worker` on a missing file fails
|
|
87
|
+
// asynchronously — which used to leave a pool whose workers were all
|
|
88
|
+
// dead and whose requests hung forever. A named error at start beats
|
|
89
|
+
// a silent hang later.
|
|
90
|
+
const entry = fileURLToPath(new URL('./worker.js', import.meta.url));
|
|
91
|
+
if (!existsSync(entry)) {
|
|
92
|
+
throw new ProcessError(`HostPool: worker entry not found at '${entry}'. The pool needs the ` +
|
|
93
|
+
`built package — run \`npm run build\` and import ` +
|
|
94
|
+
`'@pond-ts/process/pool' (or dist/pool/index.js) rather than the ` +
|
|
95
|
+
`TypeScript source, which a worker thread cannot load.`);
|
|
96
|
+
}
|
|
97
|
+
const slots = [];
|
|
98
|
+
for (let i = 0; i < size; i += 1) {
|
|
99
|
+
const worker = new Worker(entry, {
|
|
100
|
+
workerData: {
|
|
101
|
+
setup,
|
|
102
|
+
...(options.setupOptions !== undefined && {
|
|
103
|
+
setupOptions: options.setupOptions,
|
|
104
|
+
}),
|
|
105
|
+
},
|
|
106
|
+
});
|
|
107
|
+
// Idle workers must not hold the process open — a pool someone
|
|
108
|
+
// forgot to close should not stop `node script.js` from exiting.
|
|
109
|
+
// Re-`ref`'d while a request is in flight (see `#dispatch`),
|
|
110
|
+
// because a pending promise does *not* keep the event loop alive
|
|
111
|
+
// on its own: unref'd throughout, a program awaiting `run()` could
|
|
112
|
+
// exit before its answer arrived.
|
|
113
|
+
worker.unref();
|
|
114
|
+
slots.push({ worker, inFlight: 0 });
|
|
115
|
+
}
|
|
116
|
+
const pool = new HostPool(slots);
|
|
117
|
+
for (const slot of slots) {
|
|
118
|
+
slot.worker.on('message', (m) => pool.#settle(slot, m));
|
|
119
|
+
slot.worker.on('error', (e) => pool.#die(e));
|
|
120
|
+
// ANY unexpected exit is fatal, not just a non-zero one. A worker
|
|
121
|
+
// that left cleanly is exactly as unable to answer as one that
|
|
122
|
+
// crashed, and treating `code === 0` as benign left every later
|
|
123
|
+
// request routed to that slot hanging forever — the same silent
|
|
124
|
+
// hang the fatal latch exists to stop.
|
|
125
|
+
slot.worker.on('exit', (code) => {
|
|
126
|
+
if (!pool.#closed) {
|
|
127
|
+
pool.#die(new ProcessError(`HostPool: worker exited with code ${code}`));
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
return pool;
|
|
132
|
+
}
|
|
133
|
+
/** Workers in the pool. */
|
|
134
|
+
get size() {
|
|
135
|
+
return this.#slots.length;
|
|
136
|
+
}
|
|
137
|
+
/** Requests dispatched and not yet answered. */
|
|
138
|
+
get inFlight() {
|
|
139
|
+
return this.#pending.size;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Runs one envelope on the least-busy worker.
|
|
143
|
+
*
|
|
144
|
+
* `assemble` is forced off — the pool answers `columns`, and the caller
|
|
145
|
+
* assembles a `TimeSeries` from them if it wants one. Assembling
|
|
146
|
+
* worker-side would build an object that cannot cross the boundary.
|
|
147
|
+
*
|
|
148
|
+
* Pass `affinity` to pin related requests to one worker. Requests
|
|
149
|
+
* sharing an affinity key land on the same host, so its warm nodes are
|
|
150
|
+
* reused — worth it when a caller re-asks overlapping questions about
|
|
151
|
+
* one dataset, and pointless when every request is unrelated.
|
|
152
|
+
*/
|
|
153
|
+
async run(envelope, affinity) {
|
|
154
|
+
if (this.#closed)
|
|
155
|
+
throw new ProcessError('HostPool: pool is closed');
|
|
156
|
+
if (this.#fatal !== undefined)
|
|
157
|
+
throw this.#fatal;
|
|
158
|
+
const slot = this.#pick(affinity);
|
|
159
|
+
const id = this.#nextId++;
|
|
160
|
+
const request = { id, envelope };
|
|
161
|
+
return new Promise((resolve, reject) => {
|
|
162
|
+
this.#pending.set(id, { resolve, reject });
|
|
163
|
+
if (slot.inFlight === 0)
|
|
164
|
+
slot.worker.ref();
|
|
165
|
+
slot.inFlight += 1;
|
|
166
|
+
slot.worker.postMessage(request);
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
/** Terminates every worker. Outstanding requests reject. */
|
|
170
|
+
async close() {
|
|
171
|
+
if (this.#closed)
|
|
172
|
+
return;
|
|
173
|
+
this.#closed = true;
|
|
174
|
+
this.#failAll(new ProcessError('HostPool: pool closed'));
|
|
175
|
+
await Promise.all(this.#slots.map((s) => s.worker.terminate()));
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Least-in-flight, not round-robin: requests here differ by orders of
|
|
179
|
+
* magnitude (a cached fact against a cold 500k-row study), so spreading
|
|
180
|
+
* by count would queue short work behind long work on the same worker.
|
|
181
|
+
*/
|
|
182
|
+
#pick(affinity) {
|
|
183
|
+
if (affinity !== undefined) {
|
|
184
|
+
let hash = 0;
|
|
185
|
+
for (let i = 0; i < affinity.length; i += 1) {
|
|
186
|
+
hash = (hash * 31 + affinity.charCodeAt(i)) | 0;
|
|
187
|
+
}
|
|
188
|
+
return this.#slots[Math.abs(hash) % this.#slots.length];
|
|
189
|
+
}
|
|
190
|
+
let best = this.#slots[0];
|
|
191
|
+
for (const slot of this.#slots) {
|
|
192
|
+
if (slot.inFlight < best.inFlight)
|
|
193
|
+
best = slot;
|
|
194
|
+
}
|
|
195
|
+
return best;
|
|
196
|
+
}
|
|
197
|
+
#settle(slot, message) {
|
|
198
|
+
// Identify the request FIRST. A worker's `parentPort` is in scope for
|
|
199
|
+
// the caller's own setup module, so a stray `postMessage` from it
|
|
200
|
+
// reaches here — and decrementing on that would `unref` a worker with
|
|
201
|
+
// real work outstanding, letting the process exit before the answer
|
|
202
|
+
// arrived. That is exactly the hang ref/unref exists to prevent, and
|
|
203
|
+
// a `Math.max(0, …)` guard hides it as an early unref rather than a
|
|
204
|
+
// negative count. Only a message we are actually waiting on counts.
|
|
205
|
+
const pending = this.#pending.get(message.id);
|
|
206
|
+
if (pending === undefined)
|
|
207
|
+
return;
|
|
208
|
+
this.#pending.delete(message.id);
|
|
209
|
+
slot.inFlight = Math.max(0, slot.inFlight - 1);
|
|
210
|
+
if (slot.inFlight === 0)
|
|
211
|
+
slot.worker.unref();
|
|
212
|
+
if (message.ok) {
|
|
213
|
+
pending.resolve(fromWire(message.wire));
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
const error = new ProcessError(message.error);
|
|
217
|
+
if (message.name !== undefined)
|
|
218
|
+
error.name = message.name;
|
|
219
|
+
pending.reject(error);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/** A worker died: fail what is outstanding, and everything after it. */
|
|
223
|
+
#die(error) {
|
|
224
|
+
this.#fatal ??= error;
|
|
225
|
+
this.#failAll(error);
|
|
226
|
+
}
|
|
227
|
+
#failAll(error) {
|
|
228
|
+
for (const pending of this.#pending.values())
|
|
229
|
+
pending.reject(error);
|
|
230
|
+
this.#pending.clear();
|
|
231
|
+
for (const slot of this.#slots) {
|
|
232
|
+
slot.inFlight = 0;
|
|
233
|
+
slot.worker.unref();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
//# sourceMappingURL=pool.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pool's worker protocol — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* Deliberately tiny: one request shape, one response shape, correlated
|
|
5
|
+
* by a monotonic id. Everything interesting is already carried by the
|
|
6
|
+
* plan format.
|
|
7
|
+
*/
|
|
8
|
+
import type { Registry } from '../plan/registry.js';
|
|
9
|
+
import type { SourceRegistry } from '../plan/source.js';
|
|
10
|
+
import type { Units } from '../plan/types.js';
|
|
11
|
+
import type { AsyncEnvelope } from '../plan/host.js';
|
|
12
|
+
import type { WireResult } from './wire.js';
|
|
13
|
+
import type { SeriesSchema, TimeSeries } from 'pond-ts';
|
|
14
|
+
/**
|
|
15
|
+
* What a worker's setup module returns — the same shape {@link Host}
|
|
16
|
+
* takes, plus datasets to seed.
|
|
17
|
+
*
|
|
18
|
+
* Named as a module specifier rather than passed as a value because a
|
|
19
|
+
* registry is functions, and functions do not survive structured clone.
|
|
20
|
+
* Both isolates import the same module; the ops are shared code.
|
|
21
|
+
*/
|
|
22
|
+
export interface PoolSetupConfig {
|
|
23
|
+
readonly registry: Registry;
|
|
24
|
+
readonly units?: Units;
|
|
25
|
+
readonly sources?: SourceRegistry;
|
|
26
|
+
/**
|
|
27
|
+
* Datasets to `add` at start-up. Optional — a host with a
|
|
28
|
+
* `SourceRegistry` can instead load datasets on demand by identity,
|
|
29
|
+
* which is the shape that avoids sending data to workers at all.
|
|
30
|
+
*/
|
|
31
|
+
readonly datasets?: Readonly<Record<string, TimeSeries<SeriesSchema>>>;
|
|
32
|
+
}
|
|
33
|
+
export type PoolSetup = (options?: unknown) => PoolSetupConfig | Promise<PoolSetupConfig>;
|
|
34
|
+
export interface WorkerRequest {
|
|
35
|
+
readonly id: number;
|
|
36
|
+
readonly envelope: AsyncEnvelope;
|
|
37
|
+
}
|
|
38
|
+
export type WorkerResponse = {
|
|
39
|
+
readonly id: number;
|
|
40
|
+
readonly ok: true;
|
|
41
|
+
readonly wire: WireResult;
|
|
42
|
+
} | {
|
|
43
|
+
readonly id: number;
|
|
44
|
+
readonly ok: false;
|
|
45
|
+
readonly error: string;
|
|
46
|
+
readonly name?: string;
|
|
47
|
+
};
|
|
48
|
+
//# sourceMappingURL=protocol.d.ts.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pool's worker protocol — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* Deliberately tiny: one request shape, one response shape, correlated
|
|
5
|
+
* by a monotonic id. Everything interesting is already carried by the
|
|
6
|
+
* plan format.
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=protocol.js.map
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `RunResult` wire shape — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* A `RunResult` is already almost JSON: `outputs`, `facts`, `explain`,
|
|
5
|
+
* `skipped` and `nodes` are plain data by construction, because the plan
|
|
6
|
+
* layer exists to be spoken over a wire. Two fields are not:
|
|
7
|
+
*
|
|
8
|
+
* - **`columns`** — `Column` instances. Sent as their buffers
|
|
9
|
+
* ({@link columnBuffers}) and rebuilt on arrival, so a 500k-row answer
|
|
10
|
+
* crosses as two transferable buffers rather than 500k boxed values.
|
|
11
|
+
* - **`series`** — an assembled `TimeSeries`. **Not sent at all.** A pool
|
|
12
|
+
* request runs with `assemble: false`; the caller assembles from
|
|
13
|
+
* `columns` if it wants one, which is [PND-PROCTERM]'s position
|
|
14
|
+
* anyway (assembly is requested, not assumed) and avoids shipping a
|
|
15
|
+
* whole schema-bearing object to rebuild something the columns already
|
|
16
|
+
* describe.
|
|
17
|
+
*
|
|
18
|
+
* A **numeric** column this cannot express as buffers (chunked storage)
|
|
19
|
+
* falls back to a boxed array rather than failing the request — a
|
|
20
|
+
* correct slow answer beats a fast error. A **non-numeric** column has
|
|
21
|
+
* no wire form at all and is refused; see {@link toWireColumn} for why
|
|
22
|
+
* boxing one would produce plausible-looking nonsense rather than an
|
|
23
|
+
* error.
|
|
24
|
+
*/
|
|
25
|
+
import type { ColumnBuffers } from '../column.js';
|
|
26
|
+
import type { RunResult } from '../plan/run.js';
|
|
27
|
+
/** A column on the wire: buffers where possible, boxed where not. */
|
|
28
|
+
export type WireColumn = {
|
|
29
|
+
readonly kind: 'buffers';
|
|
30
|
+
readonly data: ColumnBuffers;
|
|
31
|
+
} | {
|
|
32
|
+
readonly kind: 'boxed';
|
|
33
|
+
readonly values: readonly (number | undefined)[];
|
|
34
|
+
};
|
|
35
|
+
/** A `RunResult` with its columns flattened. `series` is dropped. */
|
|
36
|
+
export interface WireResult extends Omit<RunResult, 'series' | 'columns'> {
|
|
37
|
+
readonly columns?: Readonly<Record<string, WireColumn>>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Flattens a result for `postMessage`, collecting the buffers that should
|
|
41
|
+
* be **transferred** rather than copied.
|
|
42
|
+
*
|
|
43
|
+
* The buffers are already private copies (see {@link columnBuffers}), so
|
|
44
|
+
* transferring them detaches nothing the sender still needs.
|
|
45
|
+
*/
|
|
46
|
+
export declare function toWire(result: RunResult): {
|
|
47
|
+
wire: WireResult;
|
|
48
|
+
transfer: ArrayBuffer[];
|
|
49
|
+
};
|
|
50
|
+
/** Rebuilds a `RunResult` from the wire. Adopts the arrived buffers. */
|
|
51
|
+
export declare function fromWire(wire: WireResult): RunResult;
|
|
52
|
+
//# sourceMappingURL=wire.d.ts.map
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `RunResult` wire shape — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* A `RunResult` is already almost JSON: `outputs`, `facts`, `explain`,
|
|
5
|
+
* `skipped` and `nodes` are plain data by construction, because the plan
|
|
6
|
+
* layer exists to be spoken over a wire. Two fields are not:
|
|
7
|
+
*
|
|
8
|
+
* - **`columns`** — `Column` instances. Sent as their buffers
|
|
9
|
+
* ({@link columnBuffers}) and rebuilt on arrival, so a 500k-row answer
|
|
10
|
+
* crosses as two transferable buffers rather than 500k boxed values.
|
|
11
|
+
* - **`series`** — an assembled `TimeSeries`. **Not sent at all.** A pool
|
|
12
|
+
* request runs with `assemble: false`; the caller assembles from
|
|
13
|
+
* `columns` if it wants one, which is [PND-PROCTERM]'s position
|
|
14
|
+
* anyway (assembly is requested, not assumed) and avoids shipping a
|
|
15
|
+
* whole schema-bearing object to rebuild something the columns already
|
|
16
|
+
* describe.
|
|
17
|
+
*
|
|
18
|
+
* A **numeric** column this cannot express as buffers (chunked storage)
|
|
19
|
+
* falls back to a boxed array rather than failing the request — a
|
|
20
|
+
* correct slow answer beats a fast error. A **non-numeric** column has
|
|
21
|
+
* no wire form at all and is refused; see {@link toWireColumn} for why
|
|
22
|
+
* boxing one would produce plausible-looking nonsense rather than an
|
|
23
|
+
* error.
|
|
24
|
+
*/
|
|
25
|
+
import { columnBuffers, columnFromBuffers, packColumn } from '../column.js';
|
|
26
|
+
import { ProcessError } from '../errors.js';
|
|
27
|
+
function toWireColumn(column) {
|
|
28
|
+
const data = columnBuffers(column);
|
|
29
|
+
if (data !== undefined)
|
|
30
|
+
return { kind: 'buffers', data };
|
|
31
|
+
// The fallback is for a **numeric** column `columnBuffers` cannot take
|
|
32
|
+
// as buffers — chunked storage, in practice. It is not a general
|
|
33
|
+
// escape hatch: the boxed form is rebuilt with `packColumn`, which
|
|
34
|
+
// reads numbers, so a `string` / `boolean` / `array` column would pack
|
|
35
|
+
// every cell as a defined `NaN` (`Number.isNaN('x')` is false) and
|
|
36
|
+
// arrive as plausible-looking nonsense. Refusing is the only honest
|
|
37
|
+
// answer until the wire grows those kinds.
|
|
38
|
+
if (column.kind !== 'number') {
|
|
39
|
+
throw new ProcessError(`HostPool: cannot send a '${column.kind}' column across a worker ` +
|
|
40
|
+
`boundary — only numeric columns have a wire form. Select numeric ` +
|
|
41
|
+
`outputs, or run this request in-process.`);
|
|
42
|
+
}
|
|
43
|
+
const at = column;
|
|
44
|
+
const values = new Array(column.length);
|
|
45
|
+
for (let i = 0; i < column.length; i += 1)
|
|
46
|
+
values[i] = at.at(i);
|
|
47
|
+
return { kind: 'boxed', values };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Flattens a result for `postMessage`, collecting the buffers that should
|
|
51
|
+
* be **transferred** rather than copied.
|
|
52
|
+
*
|
|
53
|
+
* The buffers are already private copies (see {@link columnBuffers}), so
|
|
54
|
+
* transferring them detaches nothing the sender still needs.
|
|
55
|
+
*/
|
|
56
|
+
export function toWire(result) {
|
|
57
|
+
const transfer = [];
|
|
58
|
+
let columns;
|
|
59
|
+
if (result.columns !== undefined) {
|
|
60
|
+
columns = {};
|
|
61
|
+
for (const [name, column] of Object.entries(result.columns)) {
|
|
62
|
+
const wire = toWireColumn(column);
|
|
63
|
+
columns[name] = wire;
|
|
64
|
+
if (wire.kind === 'buffers') {
|
|
65
|
+
transfer.push(wire.data.values.buffer);
|
|
66
|
+
if (wire.data.bits !== undefined) {
|
|
67
|
+
transfer.push(wire.data.bits.buffer);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const { series: _series, columns: _columns, ...rest } = result;
|
|
73
|
+
return {
|
|
74
|
+
wire: { ...rest, ...(columns !== undefined && { columns }) },
|
|
75
|
+
transfer,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/** Rebuilds a `RunResult` from the wire. Adopts the arrived buffers. */
|
|
79
|
+
export function fromWire(wire) {
|
|
80
|
+
// No `columns` key at all ⇒ the two shapes already coincide (every
|
|
81
|
+
// other field is plain data), so there is nothing to rebuild.
|
|
82
|
+
if (wire.columns === undefined) {
|
|
83
|
+
const { columns: _absent, ...rest } = wire;
|
|
84
|
+
return rest;
|
|
85
|
+
}
|
|
86
|
+
const columns = {};
|
|
87
|
+
for (const [name, value] of Object.entries(wire.columns)) {
|
|
88
|
+
columns[name] =
|
|
89
|
+
value.kind === 'buffers'
|
|
90
|
+
? columnFromBuffers(value.data)
|
|
91
|
+
: packColumn(value.values);
|
|
92
|
+
}
|
|
93
|
+
return { ...wire, columns };
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=wire.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pool's worker entry — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* One long-lived {@link Host} per worker, built once from the caller's
|
|
5
|
+
* setup module, then answering whole envelopes. The **long-lived** part
|
|
6
|
+
* is the point: a per-request graph would discard every compiled node
|
|
7
|
+
* and cached column between questions, which is precisely the cost this
|
|
8
|
+
* package exists to avoid. A worker that rebuilt its host per request
|
|
9
|
+
* would be slower than not having a pool at all.
|
|
10
|
+
*
|
|
11
|
+
* Why a *setup module specifier* rather than a config object: a registry
|
|
12
|
+
* is functions (`OpDef.run`), and functions do not survive structured
|
|
13
|
+
* clone. The plan is data and crosses freely; the code behind it has to
|
|
14
|
+
* be imported at both ends. So the caller names a module, both isolates
|
|
15
|
+
* import it, and the ops are ordinary shared code.
|
|
16
|
+
*
|
|
17
|
+
* This file is only ever loaded *as* a worker. It is Node-only by nature
|
|
18
|
+
* (`node:worker_threads`), which is why it sits behind its own entry
|
|
19
|
+
* point rather than in the package index.
|
|
20
|
+
*/
|
|
21
|
+
export {};
|
|
22
|
+
//# sourceMappingURL=worker.d.ts.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pool's worker entry — [PND-PROCPAR].
|
|
3
|
+
*
|
|
4
|
+
* One long-lived {@link Host} per worker, built once from the caller's
|
|
5
|
+
* setup module, then answering whole envelopes. The **long-lived** part
|
|
6
|
+
* is the point: a per-request graph would discard every compiled node
|
|
7
|
+
* and cached column between questions, which is precisely the cost this
|
|
8
|
+
* package exists to avoid. A worker that rebuilt its host per request
|
|
9
|
+
* would be slower than not having a pool at all.
|
|
10
|
+
*
|
|
11
|
+
* Why a *setup module specifier* rather than a config object: a registry
|
|
12
|
+
* is functions (`OpDef.run`), and functions do not survive structured
|
|
13
|
+
* clone. The plan is data and crosses freely; the code behind it has to
|
|
14
|
+
* be imported at both ends. So the caller names a module, both isolates
|
|
15
|
+
* import it, and the ops are ordinary shared code.
|
|
16
|
+
*
|
|
17
|
+
* This file is only ever loaded *as* a worker. It is Node-only by nature
|
|
18
|
+
* (`node:worker_threads`), which is why it sits behind its own entry
|
|
19
|
+
* point rather than in the package index.
|
|
20
|
+
*/
|
|
21
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
22
|
+
import { Host } from '../plan/host.js';
|
|
23
|
+
import { toWire } from './wire.js';
|
|
24
|
+
if (parentPort === null) {
|
|
25
|
+
throw new Error('@pond-ts/process worker: loaded outside a worker thread');
|
|
26
|
+
}
|
|
27
|
+
const port = parentPort;
|
|
28
|
+
const data = workerData;
|
|
29
|
+
/**
|
|
30
|
+
* Built once, awaited by every request. Errors here are reported against
|
|
31
|
+
* the first request rather than crashing the worker silently — a bad
|
|
32
|
+
* setup module is a caller mistake and should read as one.
|
|
33
|
+
*/
|
|
34
|
+
const ready = (async () => {
|
|
35
|
+
const module = (await import(data.setup));
|
|
36
|
+
const factory = module.default ?? module.setup;
|
|
37
|
+
if (typeof factory !== 'function') {
|
|
38
|
+
throw new Error(`@pond-ts/process worker: '${data.setup}' must export a setup function ` +
|
|
39
|
+
`(as \`default\` or \`setup\`) returning { registry, units?, sources?, datasets? }`);
|
|
40
|
+
}
|
|
41
|
+
const config = await factory(data.setupOptions);
|
|
42
|
+
const host = new Host({
|
|
43
|
+
registry: config.registry,
|
|
44
|
+
...(config.units !== undefined && { units: config.units }),
|
|
45
|
+
...(config.sources !== undefined && { sources: config.sources }),
|
|
46
|
+
});
|
|
47
|
+
for (const [id, series] of Object.entries(config.datasets ?? {})) {
|
|
48
|
+
host.add(id, series);
|
|
49
|
+
}
|
|
50
|
+
return host;
|
|
51
|
+
})();
|
|
52
|
+
// Surfaced through the first request instead of as an unhandled rejection.
|
|
53
|
+
ready.catch(() => undefined);
|
|
54
|
+
port.on('message', (message) => {
|
|
55
|
+
void (async () => {
|
|
56
|
+
try {
|
|
57
|
+
const host = await ready;
|
|
58
|
+
// `assemble: false` always. The pool answers columns; assembling a
|
|
59
|
+
// `TimeSeries` here would build something that cannot cross the
|
|
60
|
+
// boundary and that the caller can rebuild from the columns.
|
|
61
|
+
const result = await host.runAsync({
|
|
62
|
+
...message.envelope,
|
|
63
|
+
assemble: false,
|
|
64
|
+
});
|
|
65
|
+
const { wire, transfer } = toWire(result);
|
|
66
|
+
const response = { id: message.id, ok: true, wire };
|
|
67
|
+
port.postMessage(response, transfer);
|
|
68
|
+
}
|
|
69
|
+
catch (e) {
|
|
70
|
+
const response = {
|
|
71
|
+
id: message.id,
|
|
72
|
+
ok: false,
|
|
73
|
+
error: e instanceof Error ? e.message : String(e),
|
|
74
|
+
...(e instanceof Error && e.name !== 'Error' && { name: e.name }),
|
|
75
|
+
};
|
|
76
|
+
port.postMessage(response);
|
|
77
|
+
}
|
|
78
|
+
})();
|
|
79
|
+
});
|
|
80
|
+
//# sourceMappingURL=worker.js.map
|
package/dist/port.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed ports and the wiring between them.
|
|
3
|
+
*
|
|
4
|
+
* An `Outlet<T>` is a node's typed output; an `Inlet<T>` is a typed
|
|
5
|
+
* input. Because both are generic in the value type and the connect
|
|
6
|
+
* signatures require a matching `T`, wiring a `string` output into a
|
|
7
|
+
* `number` input is a compile error rather than a runtime surprise.
|
|
8
|
+
*
|
|
9
|
+
* Ports also carry the evaluation state. Each outlet holds the cached
|
|
10
|
+
* value plus a **version stamp** that increments only when a recomputed
|
|
11
|
+
* value actually differs (per the port's `equals`). Nodes record the
|
|
12
|
+
* input versions they last computed against, which is what lets a node
|
|
13
|
+
* that was marked dirty skip recomputation when nothing upstream really
|
|
14
|
+
* changed. See `node.ts` for the pull algorithm that uses this.
|
|
15
|
+
*/
|
|
16
|
+
import type { Node } from './node.js';
|
|
17
|
+
/** A node's typed output port. */
|
|
18
|
+
export declare class Outlet<T> {
|
|
19
|
+
#private;
|
|
20
|
+
readonly name: string;
|
|
21
|
+
/** The node this output belongs to. */
|
|
22
|
+
readonly node: Node<any, any>;
|
|
23
|
+
/**
|
|
24
|
+
* Evaluates this outlet and returns its value.
|
|
25
|
+
*
|
|
26
|
+
* Pull-based: the owning node recomputes only if it is dirty *and* an
|
|
27
|
+
* upstream version actually changed. A clean node returns the cached
|
|
28
|
+
* value without touching the rest of the graph.
|
|
29
|
+
*/
|
|
30
|
+
get(): T;
|
|
31
|
+
/**
|
|
32
|
+
* Returns the cached value without evaluating anything, or `undefined`
|
|
33
|
+
* if this outlet has never produced one. Use it to inspect graph state
|
|
34
|
+
* (a debug view, a node inspector) without forcing computation.
|
|
35
|
+
*/
|
|
36
|
+
peek(): T | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* How many times this outlet's value has actually changed. Downstream
|
|
39
|
+
* nodes compare against this to decide whether a dirty mark was a real
|
|
40
|
+
* change or a false alarm. `0` means nothing has been produced yet.
|
|
41
|
+
*/
|
|
42
|
+
get version(): number;
|
|
43
|
+
/** The inlets this outlet feeds. */
|
|
44
|
+
get connections(): readonly Inlet<T>[];
|
|
45
|
+
/**
|
|
46
|
+
* Wires this output into `inlet`.
|
|
47
|
+
*
|
|
48
|
+
* @throws {CycleError} if the edge would close a cycle.
|
|
49
|
+
*/
|
|
50
|
+
connect(inlet: Inlet<T>): this;
|
|
51
|
+
/** Removes the edge to `inlet`, if present. */
|
|
52
|
+
disconnect(inlet: Inlet<T>): this;
|
|
53
|
+
}
|
|
54
|
+
/** A node's typed input port. */
|
|
55
|
+
export declare class Inlet<T> {
|
|
56
|
+
#private;
|
|
57
|
+
readonly name: string;
|
|
58
|
+
/** The node this input belongs to. */
|
|
59
|
+
readonly node: Node<any, any>;
|
|
60
|
+
/**
|
|
61
|
+
* Wires `outlet` into this input, replacing any existing connection.
|
|
62
|
+
*
|
|
63
|
+
* @throws {CycleError} if the edge would close a cycle.
|
|
64
|
+
*/
|
|
65
|
+
connect(outlet: Outlet<T>): this;
|
|
66
|
+
/** Removes the incoming edge, if any. Falls back to the default value. */
|
|
67
|
+
disconnect(): this;
|
|
68
|
+
/** Whether an outlet is wired into this input. */
|
|
69
|
+
get connected(): boolean;
|
|
70
|
+
/** The outlet feeding this input, or `undefined` if unconnected. */
|
|
71
|
+
get source(): Outlet<T> | undefined;
|
|
72
|
+
/**
|
|
73
|
+
* Pulls the upstream value, evaluating it if stale.
|
|
74
|
+
*
|
|
75
|
+
* @throws {UnconnectedInputError} if unconnected and no default was declared.
|
|
76
|
+
*/
|
|
77
|
+
get(): T;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=port.d.ts.map
|