@pond-ts/process 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5914 -0
- package/LICENSE +21 -0
- package/README.md +345 -0
- package/dist/cjs-fallback.cjs +15 -0
- package/dist/column.d.ts +197 -0
- package/dist/column.js +306 -0
- package/dist/errors.d.ts +22 -0
- package/dist/errors.js +25 -0
- package/dist/graph.d.ts +89 -0
- package/dist/graph.js +133 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.js +45 -0
- package/dist/node.d.ts +151 -0
- package/dist/node.js +268 -0
- package/dist/plan/builder.d.ts +138 -0
- package/dist/plan/builder.js +166 -0
- package/dist/plan/fluent.d.ts +93 -0
- package/dist/plan/fluent.js +140 -0
- package/dist/plan/folds.d.ts +25 -0
- package/dist/plan/folds.js +190 -0
- package/dist/plan/graph.d.ts +171 -0
- package/dist/plan/graph.js +658 -0
- package/dist/plan/history.d.ts +61 -0
- package/dist/plan/history.js +82 -0
- package/dist/plan/host.d.ts +173 -0
- package/dist/plan/host.js +234 -0
- package/dist/plan/identity.d.ts +81 -0
- package/dist/plan/identity.js +158 -0
- package/dist/plan/params.d.ts +15 -0
- package/dist/plan/params.js +26 -0
- package/dist/plan/registry.d.ts +162 -0
- package/dist/plan/registry.js +422 -0
- package/dist/plan/run.d.ts +211 -0
- package/dist/plan/run.js +360 -0
- package/dist/plan/slots.d.ts +65 -0
- package/dist/plan/slots.js +114 -0
- package/dist/plan/source.d.ts +49 -0
- package/dist/plan/source.js +54 -0
- package/dist/plan/types.d.ts +376 -0
- package/dist/plan/types.js +20 -0
- package/dist/pool/index.d.ts +15 -0
- package/dist/pool/index.js +12 -0
- package/dist/pool/pool.d.ts +92 -0
- package/dist/pool/pool.js +237 -0
- package/dist/pool/protocol.d.ts +48 -0
- package/dist/pool/protocol.js +9 -0
- package/dist/pool/wire.d.ts +52 -0
- package/dist/pool/wire.js +95 -0
- package/dist/pool/worker.d.ts +22 -0
- package/dist/pool/worker.js +80 -0
- package/dist/port.d.ts +79 -0
- package/dist/port.js +222 -0
- package/dist/source.d.ts +161 -0
- package/dist/source.js +182 -0
- package/dist/types.d.ts +77 -0
- package/dist/types.js +26 -0
- package/package.json +50 -0
|
@@ -0,0 +1,658 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `bind` and plan compilation — [PND-DEMOM0].
|
|
3
|
+
*
|
|
4
|
+
* A **bound graph** is one source plus the nodes compiled against it.
|
|
5
|
+
* Identity is scoped to the binding, which is what makes `specId` safe as
|
|
6
|
+
* a cache key: the id names the computation, not the data, so two
|
|
7
|
+
* instruments sharing one id-space would answer for each other. One
|
|
8
|
+
* graph per binding; hosts own graph lifecycle, the graph owns
|
|
9
|
+
* memoization.
|
|
10
|
+
*/
|
|
11
|
+
import { appendColumn, columnBytes, columnView, packColumn, prepareRange, sealRange, } from '../column.js';
|
|
12
|
+
import { ProcessError } from '../errors.js';
|
|
13
|
+
import { defineNode } from '../node.js';
|
|
14
|
+
import { port } from '../types.js';
|
|
15
|
+
import { source } from '../source.js';
|
|
16
|
+
import { requiredHistory } from './history.js';
|
|
17
|
+
import { columnsOf, specId, unitOf } from './identity.js';
|
|
18
|
+
import { isFold, isPicked, specOf, } from './types.js';
|
|
19
|
+
/** Thrown when an op demands an input unit its source does not carry. */
|
|
20
|
+
export class UnitError extends ProcessError {
|
|
21
|
+
}
|
|
22
|
+
/** The per-output key a node's outlets are addressed by. */
|
|
23
|
+
function outletKey(output) {
|
|
24
|
+
return output.id === '' ? 'value' : output.id;
|
|
25
|
+
}
|
|
26
|
+
/** Normalizes an op's return into one column per declared output. */
|
|
27
|
+
function toColumns(op, id, length, result) {
|
|
28
|
+
const list = Array.isArray(result) && op.outputs.length > 1 ? result : [result];
|
|
29
|
+
if (list.length !== op.outputs.length) {
|
|
30
|
+
throw new ProcessError(`op '${op.name}' declares ${op.outputs.length} output(s) but returned ${list.length} for '${id}'`);
|
|
31
|
+
}
|
|
32
|
+
return list.map((v, n) => {
|
|
33
|
+
// A Column is already packed; loose values are packed once here
|
|
34
|
+
// rather than retained boxed ([PND-PROCCOL]).
|
|
35
|
+
const column = v !== null && typeof v === 'object' && 'kind' in v
|
|
36
|
+
? v
|
|
37
|
+
: packColumn(v);
|
|
38
|
+
// Every output must match the bound series row-for-row — a warm-up
|
|
39
|
+
// is expressed as gaps, never as a shorter column. Checked here, at
|
|
40
|
+
// the producer, because the only other thing that catches it is
|
|
41
|
+
// assembly's own length check, and `assemble: false` skips assembly:
|
|
42
|
+
// a one-row result from a three-row source came back as a success.
|
|
43
|
+
if (column.length !== length) {
|
|
44
|
+
throw new ProcessError(`op '${op.name}' returned ${column.length} row(s) for output '${outletKey(op.outputs[n])}' of '${id}', expected ${length} — an output must match the bound series length, with warm-up as gaps`);
|
|
45
|
+
}
|
|
46
|
+
return column;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/** The outlet a fold's fact arrives on. Not a column, so not in `outlets`. */
|
|
50
|
+
const FACT = 'fact';
|
|
51
|
+
/** Reads a column into a dense array — done once per version, inside the memo. */
|
|
52
|
+
function densify(column) {
|
|
53
|
+
const out = new Array(column.length).fill(undefined);
|
|
54
|
+
const anyCol = column;
|
|
55
|
+
for (let i = 0; i < column.length; i += 1) {
|
|
56
|
+
const v = anyCol.at(i);
|
|
57
|
+
if (v !== undefined && !Number.isNaN(v))
|
|
58
|
+
out[i] = v;
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* A source plus every node compiled against it.
|
|
64
|
+
*
|
|
65
|
+
* ## The budget — [PND-PROCCACHE]
|
|
66
|
+
*
|
|
67
|
+
* Every distinct spec ever compiled used to be retained forever, so
|
|
68
|
+
* memory scaled with *questions asked* rather than with anything
|
|
69
|
+
* bounded. A session that walks a slider from period 20 to 200 leaves
|
|
70
|
+
* 180 nodes holding 180 result columns, and nothing ever drops one.
|
|
71
|
+
*
|
|
72
|
+
* The ticket framed this as an op-level cache: an op declares which of
|
|
73
|
+
* its inputs key a result, and the engine memoizes around `compute`.
|
|
74
|
+
* **Half of that is already true here and should not be rebuilt.** A
|
|
75
|
+
* spec's `specId` is content-addressed over its op, params and inputs,
|
|
76
|
+
* so asking the same question twice hits the same node by construction —
|
|
77
|
+
* there is nothing for an op to declare, and a per-op cache would be a
|
|
78
|
+
* second key beside a correct one.
|
|
79
|
+
*
|
|
80
|
+
* What was genuinely missing is the other half, and the ticket is right
|
|
81
|
+
* that it does not belong to the op: **a per-op capacity is a per-op
|
|
82
|
+
* promise, and nothing supervises the total.** Measured at 20 nodes ×
|
|
83
|
+
* 5 entries of a 200k-row result, a per-op cap held 100 entries and
|
|
84
|
+
* 157 MB where one engine-wide cap held 10 and 35 MB.
|
|
85
|
+
*
|
|
86
|
+
* So the budget is graph-wide, in **bytes** rather than entries — the
|
|
87
|
+
* unit that means anything, and only knowable since [PND-PROCCOL] made
|
|
88
|
+
* node values columns with a reportable `columnBytes`. Eviction is LRU
|
|
89
|
+
* with one constraint: a node feeding a retained node is skipped,
|
|
90
|
+
* because dropping it frees nothing while its consumer still holds the
|
|
91
|
+
* outlet.
|
|
92
|
+
*/
|
|
93
|
+
export class BoundGraph {
|
|
94
|
+
registry;
|
|
95
|
+
units;
|
|
96
|
+
#source;
|
|
97
|
+
#nodes = new Map();
|
|
98
|
+
/** Insertion order is LRU order: a touch deletes and re-adds. */
|
|
99
|
+
#lru = new Set();
|
|
100
|
+
#budgetBytes;
|
|
101
|
+
#evicted = 0;
|
|
102
|
+
/**
|
|
103
|
+
* First changed row still owed to each node, by id.
|
|
104
|
+
*
|
|
105
|
+
* Per node, not graph-wide, and accumulated as a running **minimum**
|
|
106
|
+
* until that node actually recomputes. A single global marker is wrong
|
|
107
|
+
* because nodes compute lazily: one that is not pulled at V1 and is
|
|
108
|
+
* pulled at V2 would patch its V0 output using V2's boundary and skip
|
|
109
|
+
* everything V1 changed — silently, since the result is still defined.
|
|
110
|
+
* Found by a Codex pass on PR #571, with a repro: rows 20–22 kept
|
|
111
|
+
* `[57,60,63]` where a from-scratch pass gave `[1036,1039,1042]`.
|
|
112
|
+
*/
|
|
113
|
+
#pendingFrom = new Map();
|
|
114
|
+
/** Nodes owed a whole recompute, which no partial claim can downgrade. */
|
|
115
|
+
#fullDirty = new Set();
|
|
116
|
+
/** Ranged recomputes and full ones, for tests and for `explain`. */
|
|
117
|
+
#ranged = 0;
|
|
118
|
+
#full = 0;
|
|
119
|
+
constructor(series, options) {
|
|
120
|
+
this.registry = options.registry;
|
|
121
|
+
this.units = options.units ?? {};
|
|
122
|
+
this.#budgetBytes = options.budgetBytes ?? Number.POSITIVE_INFINITY;
|
|
123
|
+
this.#source = source({ initial: series, kind: 'source' });
|
|
124
|
+
}
|
|
125
|
+
/** Bytes currently retained across every materialized node value. */
|
|
126
|
+
get retainedBytes() {
|
|
127
|
+
let total = 0;
|
|
128
|
+
for (const c of this.#nodes.values())
|
|
129
|
+
total += this.#bytesOf(c);
|
|
130
|
+
return total;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* A node's retained bytes, read from its ports rather than from a
|
|
134
|
+
* field written when a value happens to be surfaced.
|
|
135
|
+
*
|
|
136
|
+
* Caching this in `columnOf` left an escape hatch: pulling a value
|
|
137
|
+
* through `compile(spec).node.out[…].get()` materialized a column that
|
|
138
|
+
* the budget could not see, so `retainedBytes` reported 0 while the
|
|
139
|
+
* memory was real (Codex, PR #571). `peek()` never forces a compute,
|
|
140
|
+
* so asking every node is cheap and cannot itself cause work.
|
|
141
|
+
*/
|
|
142
|
+
#bytesOf(compiled) {
|
|
143
|
+
let total = 0;
|
|
144
|
+
for (const outlet of Object.values(compiled.outlets)) {
|
|
145
|
+
const held = compiled.node.out[outlet].peek?.();
|
|
146
|
+
if (held !== undefined)
|
|
147
|
+
total += columnBytes(held);
|
|
148
|
+
}
|
|
149
|
+
return total;
|
|
150
|
+
}
|
|
151
|
+
/** How many nodes the budget has dropped over this graph's life. */
|
|
152
|
+
get evictions() {
|
|
153
|
+
return this.#evicted;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Recomputes that ran ranged, and that ran whole — [PND-PROCRANGE].
|
|
157
|
+
*
|
|
158
|
+
* Worth having as a counter rather than inferring it from timings: a
|
|
159
|
+
* node silently falling back to a full recompute is the failure mode
|
|
160
|
+
* here, and it looks exactly like "the optimisation did not help much".
|
|
161
|
+
*/
|
|
162
|
+
get recomputes() {
|
|
163
|
+
return { ranged: this.#ranged, full: this.#full };
|
|
164
|
+
}
|
|
165
|
+
/** Rows still owed per node, for tests and for `explain`. */
|
|
166
|
+
get pendingFrom() {
|
|
167
|
+
return this.#pendingFrom;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Drops least-recently-used nodes until the graph is inside its byte
|
|
171
|
+
* budget. Called after a run resolves; safe to call at any time.
|
|
172
|
+
*
|
|
173
|
+
* A node that feeds a retained node is skipped — its consumer holds
|
|
174
|
+
* the outlet, so dropping the lookup frees nothing and would only
|
|
175
|
+
* force a recompile on the next pull.
|
|
176
|
+
*/
|
|
177
|
+
enforceBudget() {
|
|
178
|
+
if (!Number.isFinite(this.#budgetBytes))
|
|
179
|
+
return;
|
|
180
|
+
let total = this.retainedBytes;
|
|
181
|
+
// Repeat until a whole pass frees nothing. Evicting a consumer
|
|
182
|
+
// unpins its inputs, and a single pass over a snapshot of the LRU
|
|
183
|
+
// has already walked past them — so a two-node chain under a 1-byte
|
|
184
|
+
// budget kept 8,000 bytes and reported success (Codex, PR #571). The
|
|
185
|
+
// loop terminates because every iteration that continues has evicted
|
|
186
|
+
// at least one node, and nodes are finite.
|
|
187
|
+
let progress = true;
|
|
188
|
+
while (progress && total > this.#budgetBytes) {
|
|
189
|
+
progress = false;
|
|
190
|
+
for (const id of [...this.#lru]) {
|
|
191
|
+
if (total <= this.#budgetBytes)
|
|
192
|
+
break;
|
|
193
|
+
const compiled = this.#nodes.get(id);
|
|
194
|
+
if (compiled === undefined)
|
|
195
|
+
continue;
|
|
196
|
+
if (compiled.dependents.size > 0)
|
|
197
|
+
continue;
|
|
198
|
+
total -= this.#bytesOf(compiled);
|
|
199
|
+
// Dropping the lookup is not eviction. `Outlet.#downstream` is a
|
|
200
|
+
// strong `Set<Inlet>` and `Inlet.node` points back, so a node
|
|
201
|
+
// left connected stays reachable from the source forever —
|
|
202
|
+
// `#nodes.delete` alone freed NOTHING, and re-asking an evicted
|
|
203
|
+
// spec compiled a second node onto the same source, so churn grew
|
|
204
|
+
// memory without bound while `ids.length` stayed flat. Found by a
|
|
205
|
+
// Layer 2 review of PR #571.
|
|
206
|
+
for (const inlet of Object.values(compiled.node.in)) {
|
|
207
|
+
inlet.disconnect();
|
|
208
|
+
}
|
|
209
|
+
this.#nodes.delete(id);
|
|
210
|
+
this.#lru.delete(id);
|
|
211
|
+
this.#pendingFrom.delete(id);
|
|
212
|
+
this.#fullDirty.delete(id);
|
|
213
|
+
this.#evicted += 1;
|
|
214
|
+
progress = true;
|
|
215
|
+
// Its inputs may now be evictable in turn — the chain unwinds
|
|
216
|
+
// from the consumer end, which is the only end that frees
|
|
217
|
+
// anything.
|
|
218
|
+
for (const upstream of compiled.upstream) {
|
|
219
|
+
this.#nodes.get(upstream)?.dependents.delete(id);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
#touch(id) {
|
|
225
|
+
this.#lru.delete(id);
|
|
226
|
+
this.#lru.add(id);
|
|
227
|
+
}
|
|
228
|
+
/** Replaces the bound data. Every node downstream goes dirty. */
|
|
229
|
+
setSource(series) {
|
|
230
|
+
// "No claim", owed to every node — and it DOMINATES any later partial
|
|
231
|
+
// claim. If a node misses a full replacement and is then handed a
|
|
232
|
+
// `setSourceFrom`, it must still recompute wholly: the rows before
|
|
233
|
+
// that boundary changed too, and nothing remembers by how much.
|
|
234
|
+
for (const id of this.#nodes.keys()) {
|
|
235
|
+
this.#fullDirty.add(id);
|
|
236
|
+
this.#pendingFrom.delete(id);
|
|
237
|
+
}
|
|
238
|
+
this.#source.set(series);
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Replaces the bound data, declaring that rows before `changedFrom`
|
|
242
|
+
* are unchanged — [PND-PROCRANGE].
|
|
243
|
+
*
|
|
244
|
+
* This is the whole input to ranged recompute: a node that declares a
|
|
245
|
+
* `lookback` and a `runRange` then rebuilds only
|
|
246
|
+
* `[changedFrom - lookback, length)` instead of the whole column.
|
|
247
|
+
*
|
|
248
|
+
* **The claim is the caller's to keep.** Nothing here verifies that
|
|
249
|
+
* the earlier rows really are untouched, because verifying costs the
|
|
250
|
+
* scan the whole feature exists to avoid. Pass a row that is genuinely
|
|
251
|
+
* at or before the first difference — a live feed appending a bar
|
|
252
|
+
* passes the old length, which is the case this is built for. Getting
|
|
253
|
+
* it wrong yields a stale prefix rather than an error, so when in
|
|
254
|
+
* doubt use {@link setSource}, which recomputes everything.
|
|
255
|
+
*/
|
|
256
|
+
setSourceFrom(series, changedFrom) {
|
|
257
|
+
const from = Math.max(0, Math.floor(changedFrom));
|
|
258
|
+
for (const id of this.#nodes.keys()) {
|
|
259
|
+
if (this.#fullDirty.has(id))
|
|
260
|
+
continue; // a full dirty outranks this
|
|
261
|
+
const owed = this.#pendingFrom.get(id);
|
|
262
|
+
this.#pendingFrom.set(id, owed === undefined ? from : Math.min(owed, from));
|
|
263
|
+
}
|
|
264
|
+
this.#source.set(series);
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* The row a node must recompute from, and clears the debt.
|
|
268
|
+
*
|
|
269
|
+
* `undefined` means "no claim was made", which forces a full recompute
|
|
270
|
+
* — the safe answer, and what a freshly compiled node gets.
|
|
271
|
+
*/
|
|
272
|
+
#takePending(id) {
|
|
273
|
+
const owed = this.#pendingFrom.get(id);
|
|
274
|
+
this.#pendingFrom.delete(id);
|
|
275
|
+
if (this.#fullDirty.delete(id))
|
|
276
|
+
return undefined;
|
|
277
|
+
return owed;
|
|
278
|
+
}
|
|
279
|
+
get series() {
|
|
280
|
+
return this.#source.out.value.get();
|
|
281
|
+
}
|
|
282
|
+
/** Ids currently compiled. Node lifetime is a budget question — see [PND-PROCCACHE]. */
|
|
283
|
+
get ids() {
|
|
284
|
+
return [...this.#nodes.keys()];
|
|
285
|
+
}
|
|
286
|
+
get(id) {
|
|
287
|
+
const hit = this.#nodes.get(id);
|
|
288
|
+
if (hit !== undefined)
|
|
289
|
+
this.#touch(id);
|
|
290
|
+
return hit;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Compiles a spec (and its inputs) into nodes, memoized by `specId`.
|
|
294
|
+
*
|
|
295
|
+
* **A returned handle is not durable under a byte budget.** Eviction
|
|
296
|
+
* disconnects a node's inlets, so a `Compiled` held across a `run` can
|
|
297
|
+
* throw `UnconnectedInputError` on a later pull, naming the input
|
|
298
|
+
* rather than the budget that took it. `run` re-resolves through
|
|
299
|
+
* `columnOf` and never hits this; a caller holding its own handle
|
|
300
|
+
* should re-`compile` after any run, which is a memoized lookup when
|
|
301
|
+
* the node survived. Only relevant with `budgetBytes` set — without
|
|
302
|
+
* one, nothing is ever evicted.
|
|
303
|
+
*
|
|
304
|
+
* Validation happens here rather than at pull time so a bad plan is
|
|
305
|
+
* rejected before any work: params first, then arity, then the typed
|
|
306
|
+
* input check.
|
|
307
|
+
*/
|
|
308
|
+
compile(spec) {
|
|
309
|
+
const id = specId(this.registry, spec);
|
|
310
|
+
const existing = this.#nodes.get(id);
|
|
311
|
+
if (existing) {
|
|
312
|
+
this.#touch(id);
|
|
313
|
+
return existing;
|
|
314
|
+
}
|
|
315
|
+
const op = this.registry.get(spec.op);
|
|
316
|
+
const params = this.registry.resolveParams(op, spec.params);
|
|
317
|
+
if (spec.inputs.length !== op.inputs.length) {
|
|
318
|
+
throw new ProcessError(`${spec.op} takes ${op.inputs.length} input(s), got ${spec.inputs.length}`);
|
|
319
|
+
}
|
|
320
|
+
// Typed inputs: an op may demand a unit its source must already
|
|
321
|
+
// carry. Checked before compiling so the reason names both sides.
|
|
322
|
+
op.inputs.forEach((def, i) => {
|
|
323
|
+
if (def.unit === undefined)
|
|
324
|
+
return;
|
|
325
|
+
const raw = spec.inputs[i];
|
|
326
|
+
let got;
|
|
327
|
+
if (typeof raw === 'string') {
|
|
328
|
+
got = this.units[raw] ?? null;
|
|
329
|
+
}
|
|
330
|
+
else {
|
|
331
|
+
// The unit of the output actually picked, not output 0. Dropping
|
|
332
|
+
// the pick failed both ways: a picked `variance` was refused by
|
|
333
|
+
// an op requiring variance, and accepted by one requiring price —
|
|
334
|
+
// the second silently, which is the worse half. An output name
|
|
335
|
+
// the upstream op does not declare defers to the binding pass
|
|
336
|
+
// below, whose error names both sides.
|
|
337
|
+
const from = specOf(raw);
|
|
338
|
+
const declaredUp = this.registry.outputsOf(this.registry.get(from.op));
|
|
339
|
+
const index = isPicked(raw)
|
|
340
|
+
? declaredUp.findIndex((o) => o.id === raw.output)
|
|
341
|
+
: 0;
|
|
342
|
+
if (isPicked(raw) && index === -1)
|
|
343
|
+
return;
|
|
344
|
+
got = unitOf(this.registry, from, this.units, index);
|
|
345
|
+
}
|
|
346
|
+
if (got !== def.unit) {
|
|
347
|
+
const name = typeof raw === 'string'
|
|
348
|
+
? raw
|
|
349
|
+
: specId(this.registry, specOf(raw)) +
|
|
350
|
+
(isPicked(raw) ? `#${raw.output}` : '');
|
|
351
|
+
throw new UnitError(`${spec.op} needs a '${def.unit}' input for '${def.role}', but '${name}' is '${got ?? 'unitless'}'`);
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
// Bind each input: a raw column reads off the source; a nested spec
|
|
355
|
+
// reads its upstream node's first output.
|
|
356
|
+
const bound = spec.inputs.map((raw, i) => {
|
|
357
|
+
const role = op.inputs[i].role;
|
|
358
|
+
if (typeof raw === 'string') {
|
|
359
|
+
return { role, column: raw, outlet: undefined, nested: false };
|
|
360
|
+
}
|
|
361
|
+
// A fold ends in a fact, so it has nothing to hand onward. Caught
|
|
362
|
+
// here rather than at pull time, and named on both sides, because
|
|
363
|
+
// a caller composing from the schema has no other way to learn it.
|
|
364
|
+
const from = specOf(raw);
|
|
365
|
+
const upstreamDef = this.registry.get(from.op);
|
|
366
|
+
if (isFold(upstreamDef)) {
|
|
367
|
+
throw new ProcessError(`'${from.op}' produces a fact, not a series, so it cannot be the '${role}' input of '${spec.op}' — surface it in outputs instead`);
|
|
368
|
+
}
|
|
369
|
+
const declaredUp = upstreamDef.outputs;
|
|
370
|
+
const index = isPicked(raw)
|
|
371
|
+
? declaredUp.findIndex((o) => o.id === raw.output)
|
|
372
|
+
: 0;
|
|
373
|
+
if (index === -1) {
|
|
374
|
+
const have = declaredUp.map((o) => `'${o.id}'`).join(', ');
|
|
375
|
+
throw new ProcessError(`'${from.op}' has no output '${raw.output}' (has ${have})`);
|
|
376
|
+
}
|
|
377
|
+
const upstream = this.compile(from);
|
|
378
|
+
const column = columnsOf(this.registry, from, upstream.id)[index];
|
|
379
|
+
const key = outletKey(declaredUp[index]);
|
|
380
|
+
return {
|
|
381
|
+
role,
|
|
382
|
+
column,
|
|
383
|
+
outlet: upstream.node.out[key],
|
|
384
|
+
nested: true,
|
|
385
|
+
upstreamId: upstream.id,
|
|
386
|
+
};
|
|
387
|
+
});
|
|
388
|
+
const inlets = { src: this.#source.out.value };
|
|
389
|
+
bound.forEach((b, i) => {
|
|
390
|
+
if (b.nested)
|
|
391
|
+
inlets[`in${i}`] = b.outlet;
|
|
392
|
+
});
|
|
393
|
+
const terminal = isFold(op);
|
|
394
|
+
const declared = this.registry.outputsOf(op);
|
|
395
|
+
const outputs = terminal
|
|
396
|
+
? { [FACT]: port() }
|
|
397
|
+
: Object.fromEntries(declared.map((o) => [outletKey(o), port()]));
|
|
398
|
+
// `undefined` when any op in this chain declares no lookback, which
|
|
399
|
+
// is what disables ranging for it — the boundary is unknowable.
|
|
400
|
+
const history = requiredHistory(this.registry, [spec]);
|
|
401
|
+
const chainLookback = history.known ? history.rows : undefined;
|
|
402
|
+
// The node's last output, held by the GRAPH and handed to `runRange`
|
|
403
|
+
// as an argument. That split is the design: the graph is a cache and
|
|
404
|
+
// was already stateful, while the op stays a pure function — of more
|
|
405
|
+
// things than before, but declared ones, so `explain` still describes
|
|
406
|
+
// what a value depends on.
|
|
407
|
+
let previous;
|
|
408
|
+
const factory = defineNode({
|
|
409
|
+
kind: spec.op,
|
|
410
|
+
inputs: Object.fromEntries(Object.keys(inlets).map((k) => [k, port()])),
|
|
411
|
+
outputs,
|
|
412
|
+
compute: (vals) => {
|
|
413
|
+
const src = vals['src'];
|
|
414
|
+
const inputsByRole = Object.fromEntries(bound.map((b) => [b.role, b.column]));
|
|
415
|
+
if (isFold(op)) {
|
|
416
|
+
// A fold reads columns; it never needs a series ([PND-PROCTERM]).
|
|
417
|
+
//
|
|
418
|
+
// Every node used to widen the source with `appendColumn` for
|
|
419
|
+
// each nested input, so an op could call the corpus normally —
|
|
420
|
+
// the studies take `(series, { column })`. For a fold that was
|
|
421
|
+
// pure waste twice over: the column it wants is already sitting
|
|
422
|
+
// in `vals`, and it was being packed into a `TimeSeries` only to
|
|
423
|
+
// be read straight back out. Worse, `appendColumn` boxes a
|
|
424
|
+
// GAPPED column on the way in (core's `withColumn` takes values,
|
|
425
|
+
// not a column), which is 22.4 ms at 1M rows — and every rolling
|
|
426
|
+
// study is gapped, so the expensive path was the common one.
|
|
427
|
+
const columnOfRole = new Map(bound.map((b, i) => [
|
|
428
|
+
b.role,
|
|
429
|
+
b.nested
|
|
430
|
+
? vals[`in${i}`]
|
|
431
|
+
: src.column(b.column),
|
|
432
|
+
]));
|
|
433
|
+
// Both accessors resolve inside the memo, so whatever a fold
|
|
434
|
+
// reads is prepared once per version rather than once per
|
|
435
|
+
// request. The difference is what "prepared" costs.
|
|
436
|
+
//
|
|
437
|
+
// `numeric` is a zero-copy view: no allocation at any length.
|
|
438
|
+
// `values` densifies into a boxed array and is therefore LAZY —
|
|
439
|
+
// a getter per role, memoized — because it was the graph's
|
|
440
|
+
// largest heap cost and `latest`, which reads a single cell,
|
|
441
|
+
// was paying for 500,000 of them ([PND-PROCCOL]).
|
|
442
|
+
const views = new Map();
|
|
443
|
+
const numeric = (role) => {
|
|
444
|
+
if (views.has(role))
|
|
445
|
+
return views.get(role);
|
|
446
|
+
const col = columnOfRole.get(role);
|
|
447
|
+
const view = col === undefined ? undefined : columnView(col);
|
|
448
|
+
views.set(role, view);
|
|
449
|
+
return view;
|
|
450
|
+
};
|
|
451
|
+
const values = {};
|
|
452
|
+
const densified = new Map();
|
|
453
|
+
for (const b of bound) {
|
|
454
|
+
Object.defineProperty(values, b.role, {
|
|
455
|
+
enumerable: true,
|
|
456
|
+
get: () => {
|
|
457
|
+
let dense = densified.get(b.role);
|
|
458
|
+
if (dense === undefined) {
|
|
459
|
+
dense = densify(columnOfRole.get(b.role));
|
|
460
|
+
densified.set(b.role, dense);
|
|
461
|
+
}
|
|
462
|
+
return dense;
|
|
463
|
+
},
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
// The source's key column, not a widened copy's — appending a
|
|
467
|
+
// value column never changes it.
|
|
468
|
+
const keyColumn = src.keyColumn();
|
|
469
|
+
return {
|
|
470
|
+
[FACT]: op.fold({
|
|
471
|
+
values,
|
|
472
|
+
numeric,
|
|
473
|
+
at: (i) => keyColumn.at(i),
|
|
474
|
+
params,
|
|
475
|
+
id,
|
|
476
|
+
}),
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
// Only a column-producing op needs the widened series, because the
|
|
480
|
+
// corpus studies take `(series, { column })`.
|
|
481
|
+
let series = src;
|
|
482
|
+
bound.forEach((b, i) => {
|
|
483
|
+
if (b.nested) {
|
|
484
|
+
series = appendColumn(series, b.column, vals[`in${i}`]);
|
|
485
|
+
}
|
|
486
|
+
});
|
|
487
|
+
const ctx = { series, inputs: inputsByRole, params, id };
|
|
488
|
+
// Ranged, or whole? Four things must hold, and any one missing
|
|
489
|
+
// falls back to a full recompute — which is always correct and
|
|
490
|
+
// merely slower ([PND-PROCRANGE]).
|
|
491
|
+
//
|
|
492
|
+
// 1. the caller declared which row changed (`setSourceFrom`),
|
|
493
|
+
// 2. this op opted in with `runRange` — meaning it claims a
|
|
494
|
+
// patched result is bit-identical to a from-scratch one,
|
|
495
|
+
// 3. it declared a `lookback`, which is what widens an upstream
|
|
496
|
+
// dirty range into this node's,
|
|
497
|
+
// 4. there is a previous output to patch.
|
|
498
|
+
//
|
|
499
|
+
// The lookback is the ACCUMULATED one down this spec's whole
|
|
500
|
+
// input chain, not this node's own. Using its own is wrong for a
|
|
501
|
+
// non-causal chain: with `win(5)` over `win(30)`, a change at row
|
|
502
|
+
// 70 reaches the outer node's row 37, but its own lookback only
|
|
503
|
+
// walks back to 66. Found by a Codex pass on PR #571, which
|
|
504
|
+
// produced exactly that — incremental 70, from-scratch 999. The
|
|
505
|
+
// trailing-window tests all passed because a trailing change
|
|
506
|
+
// propagates forward, where `to = length` already covers it.
|
|
507
|
+
//
|
|
508
|
+
// `requiredHistory` over this one spec is precisely that sum, so
|
|
509
|
+
// the two tickets compose rather than each carrying their own
|
|
510
|
+
// arithmetic.
|
|
511
|
+
const changedFrom = this.#takePending(id);
|
|
512
|
+
// Ranged, or whole? Every precondition below must hold, and any
|
|
513
|
+
// one missing falls back to a full recompute — always correct,
|
|
514
|
+
// merely slower ([PND-PROCRANGE]).
|
|
515
|
+
const from = changedFrom === undefined || chainLookback === undefined
|
|
516
|
+
? undefined
|
|
517
|
+
: Math.max(0, changedFrom - chainLookback);
|
|
518
|
+
const to = series.length;
|
|
519
|
+
// A prefix that cannot be VIEWED cannot be carried. `columnView`
|
|
520
|
+
// declines anything not packed numeric — a chunked column, say —
|
|
521
|
+
// and preparing from `undefined` seals `[0, from)` as ALL MISSING
|
|
522
|
+
// and returns it as an answer: 16 of 21 cells wrong, no error
|
|
523
|
+
// raised (Layer 2, PR #573). Every other precondition here falls
|
|
524
|
+
// back; this one must too. Silence is the bug, not slowness.
|
|
525
|
+
const views = from === undefined || previous === undefined
|
|
526
|
+
? undefined
|
|
527
|
+
: previous.map((c) => columnView(c));
|
|
528
|
+
const rangeable = from !== undefined &&
|
|
529
|
+
op.runRange !== undefined &&
|
|
530
|
+
previous !== undefined &&
|
|
531
|
+
views !== undefined &&
|
|
532
|
+
(from === 0 || views.every((v) => v !== undefined));
|
|
533
|
+
let columns;
|
|
534
|
+
if (rangeable) {
|
|
535
|
+
const priors = views;
|
|
536
|
+
// `out` is LAZY, and that does two jobs. A return-style
|
|
537
|
+
// `runRange` never touches it, so it stops paying to prepare
|
|
538
|
+
// and copy a prefix per output that is then discarded. And
|
|
539
|
+
// touching an entry is the op's statement of intent, which is
|
|
540
|
+
// what lets a partial write be caught below instead of
|
|
541
|
+
// silently shipping.
|
|
542
|
+
const prepared = new Array(op.outputs.length);
|
|
543
|
+
const out = [];
|
|
544
|
+
op.outputs.forEach((_, n) => {
|
|
545
|
+
Object.defineProperty(out, n, {
|
|
546
|
+
enumerable: true,
|
|
547
|
+
configurable: true,
|
|
548
|
+
get: () => (prepared[n] ??= prepareRange(to, from, priors[n])),
|
|
549
|
+
});
|
|
550
|
+
});
|
|
551
|
+
out.length = op.outputs.length;
|
|
552
|
+
const produced = op.runRange({
|
|
553
|
+
...ctx,
|
|
554
|
+
from,
|
|
555
|
+
to,
|
|
556
|
+
previous: previous,
|
|
557
|
+
previousView: priors,
|
|
558
|
+
out,
|
|
559
|
+
});
|
|
560
|
+
// Returning nothing means "written into `ctx.out`" — the path
|
|
561
|
+
// that carries the prefix as a block. An op may still return a
|
|
562
|
+
// whole result, which is simply the slower way to say it.
|
|
563
|
+
if (produced === undefined || produced === null) {
|
|
564
|
+
// Every declared output must have been written. Sealing an
|
|
565
|
+
// untouched buffer produces a column that keeps its prefix
|
|
566
|
+
// and reports the new rows as MISSING — a plausible, silent,
|
|
567
|
+
// incomplete answer (Codex, PR #573). An op that writes
|
|
568
|
+
// `out[0]` of three declared outputs is a contract error, so
|
|
569
|
+
// it is one here rather than a wrong number downstream.
|
|
570
|
+
const missing = op.outputs
|
|
571
|
+
.map((o, n) => (prepared[n] === undefined ? o.id || `${n}` : ''))
|
|
572
|
+
.filter(Boolean);
|
|
573
|
+
if (missing.length > 0) {
|
|
574
|
+
throw new ProcessError(`op '${spec.op}' wrote no ranged output for ${missing
|
|
575
|
+
.map((m) => `'${m}'`)
|
|
576
|
+
.join(', ')} — a \`runRange\` that returns nothing must ` +
|
|
577
|
+
`write every declared output through \`ctx.out\`, or ` +
|
|
578
|
+
`return a whole result instead`);
|
|
579
|
+
}
|
|
580
|
+
columns = prepared.map((o) => sealRange(o, to));
|
|
581
|
+
}
|
|
582
|
+
else {
|
|
583
|
+
// The same length contract as a whole result: a ranged op
|
|
584
|
+
// that returns a short column is as wrong as a full one.
|
|
585
|
+
// (The `ctx.out` path above is exact by construction —
|
|
586
|
+
// `sealRange(…, to)` cannot produce another length.)
|
|
587
|
+
columns = toColumns(op, id, to, produced);
|
|
588
|
+
}
|
|
589
|
+
this.#ranged += 1;
|
|
590
|
+
}
|
|
591
|
+
else {
|
|
592
|
+
columns = toColumns(op, id, series.length, op.run(ctx));
|
|
593
|
+
this.#full += 1;
|
|
594
|
+
}
|
|
595
|
+
previous = columns;
|
|
596
|
+
return Object.fromEntries(op.outputs.map((o, n) => [outletKey(o), columns[n]]));
|
|
597
|
+
},
|
|
598
|
+
});
|
|
599
|
+
const node = factory();
|
|
600
|
+
for (const [key, outlet] of Object.entries(inlets)) {
|
|
601
|
+
outlet.connect(node.in[key]);
|
|
602
|
+
}
|
|
603
|
+
const upstream = bound
|
|
604
|
+
.filter((b) => b.nested)
|
|
605
|
+
.map((b) => b.upstreamId);
|
|
606
|
+
const compiled = {
|
|
607
|
+
id,
|
|
608
|
+
spec,
|
|
609
|
+
params,
|
|
610
|
+
node,
|
|
611
|
+
fold: terminal,
|
|
612
|
+
outlets: Object.fromEntries(declared.map((o) => [o.id, outletKey(o)])),
|
|
613
|
+
upstream,
|
|
614
|
+
dependents: new Set(),
|
|
615
|
+
};
|
|
616
|
+
for (const up of upstream)
|
|
617
|
+
this.#nodes.get(up)?.dependents.add(id);
|
|
618
|
+
this.#nodes.set(id, compiled);
|
|
619
|
+
this.#touch(id);
|
|
620
|
+
return compiled;
|
|
621
|
+
}
|
|
622
|
+
/** Reads one output column of a compiled spec, by output suffix. */
|
|
623
|
+
columnOf(compiled, suffix) {
|
|
624
|
+
const key = compiled.outlets[suffix];
|
|
625
|
+
if (key === undefined) {
|
|
626
|
+
const have = Object.keys(compiled.outlets)
|
|
627
|
+
.map((s) => `'${s}'`)
|
|
628
|
+
.join(', ');
|
|
629
|
+
throw new ProcessError(compiled.fold
|
|
630
|
+
? `'${compiled.spec.op}' is a fold — it produces a fact, not columns`
|
|
631
|
+
: `'${compiled.spec.op}' has no output '${suffix}' (has ${have})`);
|
|
632
|
+
}
|
|
633
|
+
return compiled.node.out[key].get();
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* Reads a fold's fact.
|
|
637
|
+
*
|
|
638
|
+
* The same memoized pull `columnOf` does, which is the whole change:
|
|
639
|
+
* the value is cached against the node's version like any column, so
|
|
640
|
+
* asking twice costs a version check rather than a rescan.
|
|
641
|
+
*/
|
|
642
|
+
factOf(compiled) {
|
|
643
|
+
if (!compiled.fold) {
|
|
644
|
+
throw new ProcessError(`'${compiled.spec.op}' produces columns, not a fact`);
|
|
645
|
+
}
|
|
646
|
+
return compiled.node.out[FACT].get();
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Binds a dataset, producing a graph its plans resolve against.
|
|
651
|
+
*
|
|
652
|
+
* One graph per data binding — two instruments get two graphs and share
|
|
653
|
+
* no nodes, even though their specs produce identical ids.
|
|
654
|
+
*/
|
|
655
|
+
export function bind(series, options) {
|
|
656
|
+
return new BoundGraph(series, options);
|
|
657
|
+
}
|
|
658
|
+
//# sourceMappingURL=graph.js.map
|