@pond-ts/process 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5914 -0
- package/LICENSE +21 -0
- package/README.md +345 -0
- package/dist/cjs-fallback.cjs +15 -0
- package/dist/column.d.ts +197 -0
- package/dist/column.js +306 -0
- package/dist/errors.d.ts +22 -0
- package/dist/errors.js +25 -0
- package/dist/graph.d.ts +89 -0
- package/dist/graph.js +133 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.js +45 -0
- package/dist/node.d.ts +151 -0
- package/dist/node.js +268 -0
- package/dist/plan/builder.d.ts +138 -0
- package/dist/plan/builder.js +166 -0
- package/dist/plan/fluent.d.ts +93 -0
- package/dist/plan/fluent.js +140 -0
- package/dist/plan/folds.d.ts +25 -0
- package/dist/plan/folds.js +190 -0
- package/dist/plan/graph.d.ts +171 -0
- package/dist/plan/graph.js +658 -0
- package/dist/plan/history.d.ts +61 -0
- package/dist/plan/history.js +82 -0
- package/dist/plan/host.d.ts +173 -0
- package/dist/plan/host.js +234 -0
- package/dist/plan/identity.d.ts +81 -0
- package/dist/plan/identity.js +158 -0
- package/dist/plan/params.d.ts +15 -0
- package/dist/plan/params.js +26 -0
- package/dist/plan/registry.d.ts +162 -0
- package/dist/plan/registry.js +422 -0
- package/dist/plan/run.d.ts +211 -0
- package/dist/plan/run.js +360 -0
- package/dist/plan/slots.d.ts +65 -0
- package/dist/plan/slots.js +114 -0
- package/dist/plan/source.d.ts +49 -0
- package/dist/plan/source.js +54 -0
- package/dist/plan/types.d.ts +376 -0
- package/dist/plan/types.js +20 -0
- package/dist/pool/index.d.ts +15 -0
- package/dist/pool/index.js +12 -0
- package/dist/pool/pool.d.ts +92 -0
- package/dist/pool/pool.js +237 -0
- package/dist/pool/protocol.d.ts +48 -0
- package/dist/pool/protocol.js +9 -0
- package/dist/pool/wire.d.ts +52 -0
- package/dist/pool/wire.js +95 -0
- package/dist/pool/worker.d.ts +22 -0
- package/dist/pool/worker.js +80 -0
- package/dist/port.d.ts +79 -0
- package/dist/port.js +222 -0
- package/dist/source.d.ts +161 -0
- package/dist/source.js +182 -0
- package/dist/types.d.ts +77 -0
- package/dist/types.js +26 -0
- package/package.json +50 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { Registry } from './registry.js';
|
|
2
|
+
import { type Spec } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* The minimum safe tail for a plan, in rows — [PND-PROCHIST].
|
|
5
|
+
*
|
|
6
|
+
* ## Why this is not the consumer's call
|
|
7
|
+
*
|
|
8
|
+
* The hot leading edge is the design's worst cliff: an 8-study stack over
|
|
9
|
+
* 500k rows costs **765 ms/tick**, saturating at ~1.3 ticks/sec. The same
|
|
10
|
+
* stack over a 5,000-row tail runs at **5.4 ms/tick**. So a consumer
|
|
11
|
+
* watching a live edge should slice, and the only question is where.
|
|
12
|
+
*
|
|
13
|
+
* The RFC left that to the consumer. It should not be a guess in either
|
|
14
|
+
* direction: too short silently truncates a study's warm-up and reports
|
|
15
|
+
* `undefined` where a value exists, too long gives the cliff back. The
|
|
16
|
+
* registry already knows every op's lookback, so `display range +
|
|
17
|
+
* requiredHistory(plan)` is *provably* sufficient.
|
|
18
|
+
*
|
|
19
|
+
* ## Lookbacks sum along a chain; they do not max
|
|
20
|
+
*
|
|
21
|
+
* This is the part worth getting right. `sma(20)` over `sma(50)` does not
|
|
22
|
+
* need 50 rows of history — it needs 50 for the inner study to produce
|
|
23
|
+
* anything, and then a further 19 rows of *that output* before the outer
|
|
24
|
+
* one does. 69, not 50. Taking the max across a nested chain
|
|
25
|
+
* under-provisions by exactly the amount that makes the bug subtle: the
|
|
26
|
+
* answer is defined, plausible, and computed from a truncated window.
|
|
27
|
+
*
|
|
28
|
+
* Across *independent* specs in a plan it is a max, because the plan
|
|
29
|
+
* needs whichever branch reaches furthest back.
|
|
30
|
+
*
|
|
31
|
+
* ## Unknown is not zero
|
|
32
|
+
*
|
|
33
|
+
* An op that does not declare `lookback` makes this return
|
|
34
|
+
* `{ known: false }` with the offending ops named, rather than a number.
|
|
35
|
+
* A missing declaration and a genuinely element-wise op are the same
|
|
36
|
+
* value and opposite meanings, and defaulting to zero would hand back a
|
|
37
|
+
* confidently wrong slice. An element-wise op should declare `() => 0`.
|
|
38
|
+
*/
|
|
39
|
+
export interface HistoryResult {
|
|
40
|
+
/** False when any op in the plan does not declare a lookback. */
|
|
41
|
+
readonly known: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Rows of history the plan needs before a requested range. Present
|
|
44
|
+
* only when {@link known} — there is no safe default to report.
|
|
45
|
+
*/
|
|
46
|
+
readonly rows?: number;
|
|
47
|
+
/** Ops with no declared lookback, deduplicated, in encounter order. */
|
|
48
|
+
readonly undeclared: readonly string[];
|
|
49
|
+
/**
|
|
50
|
+
* The deepest history any spec of a given op needed, by op name.
|
|
51
|
+
*
|
|
52
|
+
* Not per spec: `sma(5)` and `sma(100)` in one plan collapse to
|
|
53
|
+
* `{ sma: 99 }`. That is a diagnostic aid for `explain`-style output,
|
|
54
|
+
* not something to slice against — {@link rows} is the number a caller
|
|
55
|
+
* uses. The doc previously said "per-spec depth … keyed by op name",
|
|
56
|
+
* which is self-contradictory and was flagged in review.
|
|
57
|
+
*/
|
|
58
|
+
readonly byOp: Readonly<Record<string, number>>;
|
|
59
|
+
}
|
|
60
|
+
export declare function requiredHistory(registry: Registry, plan: readonly Spec[]): HistoryResult;
|
|
61
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { ProcessError } from '../errors.js';
|
|
2
|
+
import { isFold, isPicked, } from './types.js';
|
|
3
|
+
export function requiredHistory(registry, plan) {
|
|
4
|
+
const undeclared = [];
|
|
5
|
+
const seen = new Set();
|
|
6
|
+
const byOp = {};
|
|
7
|
+
// A plan is a DAG of shared specs, so the same subtree is reachable by
|
|
8
|
+
// many paths. Memoize on identity — two structurally equal specs are
|
|
9
|
+
// separate objects here, and both give the same answer anyway.
|
|
10
|
+
const depth = new Map();
|
|
11
|
+
const of = (spec) => {
|
|
12
|
+
const cached = depth.get(spec);
|
|
13
|
+
if (cached !== undefined)
|
|
14
|
+
return cached;
|
|
15
|
+
const def = registry.get(spec.op);
|
|
16
|
+
let own = 0;
|
|
17
|
+
if (isFold(def)) {
|
|
18
|
+
// A fold reads a whole column and emits a fact. It adds no warm-up
|
|
19
|
+
// of its own — but a truncated tail still changes its answer, which
|
|
20
|
+
// is a windowing question rather than a history one.
|
|
21
|
+
own = 0;
|
|
22
|
+
}
|
|
23
|
+
else if (def.lookback === undefined) {
|
|
24
|
+
if (!seen.has(spec.op)) {
|
|
25
|
+
seen.add(spec.op);
|
|
26
|
+
undeclared.push(spec.op);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
own = def.lookback(paramsOf(registry, spec));
|
|
31
|
+
if (!Number.isFinite(own) || own < 0) {
|
|
32
|
+
throw new ProcessError(`op '${spec.op}' declared a lookback of ${String(own)}; ` +
|
|
33
|
+
`it must be a non-negative finite row count`);
|
|
34
|
+
}
|
|
35
|
+
own = Math.ceil(own);
|
|
36
|
+
}
|
|
37
|
+
// Sum along nesting, max across sibling inputs: a spec with two
|
|
38
|
+
// nested inputs waits for whichever arrives latest, and then needs
|
|
39
|
+
// its own warm-up on top of that.
|
|
40
|
+
let deepest = 0;
|
|
41
|
+
for (const input of spec.inputs ?? []) {
|
|
42
|
+
if (typeof input === 'string')
|
|
43
|
+
continue; // a source column: no warm-up
|
|
44
|
+
const nested = of(specOf(input));
|
|
45
|
+
if (nested > deepest)
|
|
46
|
+
deepest = nested;
|
|
47
|
+
}
|
|
48
|
+
const total = own + deepest;
|
|
49
|
+
depth.set(spec, total);
|
|
50
|
+
byOp[spec.op] = Math.max(byOp[spec.op] ?? 0, total);
|
|
51
|
+
return total;
|
|
52
|
+
};
|
|
53
|
+
let rows = 0;
|
|
54
|
+
for (const spec of plan) {
|
|
55
|
+
const d = of(spec);
|
|
56
|
+
if (d > rows)
|
|
57
|
+
rows = d;
|
|
58
|
+
}
|
|
59
|
+
return undeclared.length > 0
|
|
60
|
+
? { known: false, undeclared, byOp }
|
|
61
|
+
: { known: true, rows, undeclared: [], byOp };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* A nested input is either a spec or a picked output of one.
|
|
65
|
+
*
|
|
66
|
+
* The picked form nests under `from`, not `spec` — an earlier guess at
|
|
67
|
+
* the field name silently produced `undefined` here, which only surfaced
|
|
68
|
+
* once this ran on a plan using multi-output studies.
|
|
69
|
+
*/
|
|
70
|
+
function specOf(input) {
|
|
71
|
+
return isPicked(input) ? input.from : input;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Params with defaults applied, which is what a lookback must see: `sma`
|
|
75
|
+
* with no `period` still has one, and reading `spec.params` directly
|
|
76
|
+
* would hand the lookback an `undefined` to multiply.
|
|
77
|
+
*/
|
|
78
|
+
function paramsOf(registry, spec) {
|
|
79
|
+
const def = registry.get(spec.op);
|
|
80
|
+
return registry.resolveParams(def, spec.params);
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The long-lived host — [PND-DEMOM1].
|
|
3
|
+
*
|
|
4
|
+
* A `Host` owns `Map<datasetId, BoundGraph>` and **outlives requests**.
|
|
5
|
+
* That is the whole architectural claim: a graph built per request starts
|
|
6
|
+
* cold, and a cold graph is a fold with extra steps. Every caching figure
|
|
7
|
+
* behind this design assumes a warm binding.
|
|
8
|
+
*
|
|
9
|
+
* Where the host runs is a separate question — a long-lived worker proves
|
|
10
|
+
* client-side execution with an unblocked main thread; a server process
|
|
11
|
+
* proves one cache shared across sessions. Both satisfy the invariant;
|
|
12
|
+
* neither is expressed here, because this is the part they share.
|
|
13
|
+
*/
|
|
14
|
+
import { ProcessError } from '../errors.js';
|
|
15
|
+
import type { SeriesSchema, TimeSeries } from 'pond-ts';
|
|
16
|
+
import { BoundGraph } from './graph.js';
|
|
17
|
+
import type { Registry } from './registry.js';
|
|
18
|
+
import { type RunResult, type Select, type ErrorPolicy } from './run.js';
|
|
19
|
+
import type { Slots } from './slots.js';
|
|
20
|
+
import { type SourceRef, type SourceRegistry } from './source.js';
|
|
21
|
+
import type { Plan, Units } from './types.js';
|
|
22
|
+
/** Thrown when a request names a dataset the host has not been given. */
|
|
23
|
+
export declare class UnknownDatasetError extends ProcessError {
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* A request as it arrives over a wire.
|
|
27
|
+
*
|
|
28
|
+
* `from` is the binding key, and the multi-source hook — widening it to
|
|
29
|
+
* an array later ([PND-PROCJOIN]) should be an addition rather than a
|
|
30
|
+
* break. `as` **names** the output so a later request can refer to it; it
|
|
31
|
+
* deliberately does not window one, or the request would have two places
|
|
32
|
+
* that slice time and they would eventually disagree.
|
|
33
|
+
*/
|
|
34
|
+
interface EnvelopeBase<From extends string | SourceRef> {
|
|
35
|
+
readonly from: From;
|
|
36
|
+
readonly as?: string;
|
|
37
|
+
readonly onError?: ErrorPolicy;
|
|
38
|
+
/** See {@link RunOptions.assemble}. A wire consumer wants `false`. */
|
|
39
|
+
readonly assemble?: boolean;
|
|
40
|
+
}
|
|
41
|
+
/** The original form: a plan of nested specs, selected inline. */
|
|
42
|
+
export interface PlanEnvelope extends EnvelopeBase<string> {
|
|
43
|
+
readonly process: Plan;
|
|
44
|
+
readonly select?: readonly Select[];
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The slot form — [PND-PROCSLOT].
|
|
48
|
+
*
|
|
49
|
+
* `nodes` is keyed by names the caller owns and `outputs` by names it
|
|
50
|
+
* will read back, so both survive a param edit that moves every derived
|
|
51
|
+
* id. What a `PlanBuilder` emits.
|
|
52
|
+
*/
|
|
53
|
+
export interface SlotEnvelope extends EnvelopeBase<string> {
|
|
54
|
+
readonly nodes: Slots;
|
|
55
|
+
readonly outputs?: Readonly<Record<string, Select>>;
|
|
56
|
+
}
|
|
57
|
+
export type Envelope = PlanEnvelope | SlotEnvelope;
|
|
58
|
+
/** Nested-plan request whose source must be resolved asynchronously. */
|
|
59
|
+
export interface AsyncPlanEnvelope extends EnvelopeBase<SourceRef> {
|
|
60
|
+
readonly process: Plan;
|
|
61
|
+
readonly select?: readonly Select[];
|
|
62
|
+
}
|
|
63
|
+
/** Slot request whose source must be resolved asynchronously. */
|
|
64
|
+
export interface AsyncSlotEnvelope extends EnvelopeBase<SourceRef> {
|
|
65
|
+
readonly nodes: Slots;
|
|
66
|
+
readonly outputs?: Readonly<Record<string, Select>>;
|
|
67
|
+
}
|
|
68
|
+
/** What `Host.runAsync` accepts: local requests too, for composable callers. */
|
|
69
|
+
export type AsyncEnvelope = Envelope | AsyncPlanEnvelope | AsyncSlotEnvelope;
|
|
70
|
+
export interface DatasetInfo {
|
|
71
|
+
readonly id: string;
|
|
72
|
+
readonly rows: number;
|
|
73
|
+
readonly columns: readonly string[];
|
|
74
|
+
/** Nodes compiled against this binding so far. */
|
|
75
|
+
readonly nodes: number;
|
|
76
|
+
}
|
|
77
|
+
export declare class Host {
|
|
78
|
+
#private;
|
|
79
|
+
readonly registry: Registry;
|
|
80
|
+
constructor(options: {
|
|
81
|
+
registry: Registry;
|
|
82
|
+
units?: Units;
|
|
83
|
+
sources?: SourceRegistry;
|
|
84
|
+
/**
|
|
85
|
+
* Cap on retained node values **per bound graph**, in bytes — see
|
|
86
|
+
* {@link bind}. Without one, a host accepting runtime plans retains
|
|
87
|
+
* every distinct spec ever asked of every dataset, so memory scales
|
|
88
|
+
* with questions asked rather than with anything bounded. A host
|
|
89
|
+
* whose plans arrive from callers it does not control should set
|
|
90
|
+
* this.
|
|
91
|
+
*
|
|
92
|
+
* What bounds the *number* of graphs is a separate question with two
|
|
93
|
+
* answers. Datasets registered with {@link add} are the host
|
|
94
|
+
* author's own, bounded by what the author adds and released with
|
|
95
|
+
* {@link remove}. Sources resolved through a source registry by
|
|
96
|
+
* `runAsync` are **request-driven** — every distinct `SourceRef` a
|
|
97
|
+
* caller supplies binds another graph — so a host exposed to
|
|
98
|
+
* untrusted callers must also set {@link maxSources}. The retained
|
|
99
|
+
* total is then at most `budgetBytes × (author datasets +
|
|
100
|
+
* maxSources)`.
|
|
101
|
+
*/
|
|
102
|
+
budgetBytes?: number;
|
|
103
|
+
/**
|
|
104
|
+
* Cap on sources resolved through the source registry, LRU — the
|
|
105
|
+
* request-driven half of the host's footprint. `runAsync` binds a
|
|
106
|
+
* graph per distinct `SourceRef`, and a caller chooses the refs, so
|
|
107
|
+
* without a cap an untrusted caller grows the host without bound.
|
|
108
|
+
* Author-added datasets ({@link add}) are never evicted by this.
|
|
109
|
+
* Unbounded when omitted — acceptable only when every `SourceRef`
|
|
110
|
+
* the host will see comes from code the author controls.
|
|
111
|
+
*/
|
|
112
|
+
maxSources?: number;
|
|
113
|
+
});
|
|
114
|
+
/**
|
|
115
|
+
* Registers a dataset. The graph is built lazily on first use, so
|
|
116
|
+
* seeding many datasets is cheap.
|
|
117
|
+
*/
|
|
118
|
+
add(id: string, series: TimeSeries<SeriesSchema>): this;
|
|
119
|
+
has(id: string): boolean;
|
|
120
|
+
get datasets(): DatasetInfo[];
|
|
121
|
+
/** The bound graph for a dataset, built on first use and kept. */
|
|
122
|
+
graphFor(id: string): BoundGraph;
|
|
123
|
+
/**
|
|
124
|
+
* Forgets a dataset: its source, its graph, and every cached node
|
|
125
|
+
* value with them. The explicit end of a binding's lifecycle — the
|
|
126
|
+
* counterpart to {@link add} for a host that cycles datasets, where
|
|
127
|
+
* "kept until the process dies" is not a policy.
|
|
128
|
+
*
|
|
129
|
+
* Returns whether the dataset was known. A load in flight for the
|
|
130
|
+
* same source id **discards its result** rather than resurrecting the
|
|
131
|
+
* removed dataset — its `runAsync` callers get `UnknownDatasetError`,
|
|
132
|
+
* which is what asking for a concurrently-removed dataset means.
|
|
133
|
+
*/
|
|
134
|
+
remove(id: string): boolean;
|
|
135
|
+
/** Resolves an envelope against its dataset's long-lived graph. */
|
|
136
|
+
run(envelope: Envelope): RunResult;
|
|
137
|
+
/**
|
|
138
|
+
* Resolves either a preloaded dataset or an opaque asynchronous source.
|
|
139
|
+
*
|
|
140
|
+
* Equal revisions preserve the graph untouched. A changed revision updates
|
|
141
|
+
* its source in place, retaining compiled nodes while normal invalidation
|
|
142
|
+
* propagates from the new value.
|
|
143
|
+
*/
|
|
144
|
+
runAsync(envelope: AsyncEnvelope): Promise<RunResult>;
|
|
145
|
+
}
|
|
146
|
+
export declare function createHost(options: {
|
|
147
|
+
registry: Registry;
|
|
148
|
+
units?: Units;
|
|
149
|
+
sources?: SourceRegistry;
|
|
150
|
+
/** Per-graph cap on retained node values, in bytes — see {@link Host}. */
|
|
151
|
+
budgetBytes?: number;
|
|
152
|
+
/** LRU cap on registry-loaded sources — see {@link Host}. */
|
|
153
|
+
maxSources?: number;
|
|
154
|
+
}): Host;
|
|
155
|
+
/** A response with the in-process values dropped — what crosses a wire. */
|
|
156
|
+
export interface WireResult extends Omit<RunResult, 'series' | 'columns'> {
|
|
157
|
+
readonly as?: string;
|
|
158
|
+
/** Whether a `columns` selector asked for drawable values. */
|
|
159
|
+
readonly hasSeries: boolean;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Projects a result for transport.
|
|
163
|
+
*
|
|
164
|
+
* The renderer path hands back a live `TimeSeries` on purpose — a chart
|
|
165
|
+
* draws every point, and serializing 10⁶ of them per frame is the failure
|
|
166
|
+
* this split exists to avoid. `toWire` is for the other caller, and the
|
|
167
|
+
* useful property is that it is **only lossy when columns were asked
|
|
168
|
+
* for**: a facts-only response is already JSON-safe, so this is a no-op
|
|
169
|
+
* on exactly the requests that cross a wire.
|
|
170
|
+
*/
|
|
171
|
+
export declare function toWire(result: RunResult, as?: string): WireResult;
|
|
172
|
+
export {};
|
|
173
|
+
//# sourceMappingURL=host.d.ts.map
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The long-lived host — [PND-DEMOM1].
|
|
3
|
+
*
|
|
4
|
+
* A `Host` owns `Map<datasetId, BoundGraph>` and **outlives requests**.
|
|
5
|
+
* That is the whole architectural claim: a graph built per request starts
|
|
6
|
+
* cold, and a cold graph is a fold with extra steps. Every caching figure
|
|
7
|
+
* behind this design assumes a warm binding.
|
|
8
|
+
*
|
|
9
|
+
* Where the host runs is a separate question — a long-lived worker proves
|
|
10
|
+
* client-side execution with an unblocked main thread; a server process
|
|
11
|
+
* proves one cache shared across sessions. Both satisfy the invariant;
|
|
12
|
+
* neither is expressed here, because this is the part they share.
|
|
13
|
+
*/
|
|
14
|
+
import { ProcessError } from '../errors.js';
|
|
15
|
+
import { bind } from './graph.js';
|
|
16
|
+
import { run } from './run.js';
|
|
17
|
+
import { sourceId, } from './source.js';
|
|
18
|
+
/** Thrown when a request names a dataset the host has not been given. */
|
|
19
|
+
export class UnknownDatasetError extends ProcessError {
|
|
20
|
+
}
|
|
21
|
+
function isLocalEnvelope(envelope) {
|
|
22
|
+
return typeof envelope.from === 'string';
|
|
23
|
+
}
|
|
24
|
+
export class Host {
|
|
25
|
+
registry;
|
|
26
|
+
#units;
|
|
27
|
+
#budgetBytes;
|
|
28
|
+
#maxSources;
|
|
29
|
+
#graphs = new Map();
|
|
30
|
+
#sources = new Map();
|
|
31
|
+
#sourceRegistry;
|
|
32
|
+
/** Insertion order is LRU order for {@link Host.#maxSources} eviction. */
|
|
33
|
+
#loadedSources = new Map();
|
|
34
|
+
#loadingSources = new Map();
|
|
35
|
+
/**
|
|
36
|
+
* Bumped by {@link remove} while a load for that id is in flight, so
|
|
37
|
+
* the landing load discards its result instead of resurrecting the
|
|
38
|
+
* dataset. Entries are consumed by the discarding load — the map only
|
|
39
|
+
* holds ids removed mid-flight.
|
|
40
|
+
*/
|
|
41
|
+
#sourceEpochs = new Map();
|
|
42
|
+
constructor(options) {
|
|
43
|
+
this.registry = options.registry;
|
|
44
|
+
this.#units = options.units ?? {};
|
|
45
|
+
this.#budgetBytes = options.budgetBytes;
|
|
46
|
+
this.#maxSources = options.maxSources;
|
|
47
|
+
this.#sourceRegistry = options.sources;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Registers a dataset. The graph is built lazily on first use, so
|
|
51
|
+
* seeding many datasets is cheap.
|
|
52
|
+
*/
|
|
53
|
+
add(id, series) {
|
|
54
|
+
this.#sources.set(id, series);
|
|
55
|
+
const existing = this.#graphs.get(id);
|
|
56
|
+
// Rebinding an existing dataset updates it in place rather than
|
|
57
|
+
// discarding the graph — the nodes stay, and dirty marking handles
|
|
58
|
+
// the rest. Dropping the graph would throw away the cache on every
|
|
59
|
+
// data refresh, which is exactly what this class exists to avoid.
|
|
60
|
+
if (existing)
|
|
61
|
+
existing.setSource(series);
|
|
62
|
+
return this;
|
|
63
|
+
}
|
|
64
|
+
has(id) {
|
|
65
|
+
return this.#sources.has(id);
|
|
66
|
+
}
|
|
67
|
+
get datasets() {
|
|
68
|
+
return [...this.#sources.entries()].map(([id, series]) => ({
|
|
69
|
+
id,
|
|
70
|
+
rows: series.length,
|
|
71
|
+
columns: series.schema.slice(1).map((c) => c.name),
|
|
72
|
+
nodes: this.#graphs.get(id)?.ids.length ?? 0,
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
/** The bound graph for a dataset, built on first use and kept. */
|
|
76
|
+
graphFor(id) {
|
|
77
|
+
const existing = this.#graphs.get(id);
|
|
78
|
+
if (existing)
|
|
79
|
+
return existing;
|
|
80
|
+
const series = this.#sources.get(id);
|
|
81
|
+
if (series === undefined) {
|
|
82
|
+
const have = [...this.#sources.keys()].map((k) => `'${k}'`).join(', ');
|
|
83
|
+
throw new UnknownDatasetError(`unknown dataset '${id}'${have ? ` — have ${have}` : ''}`);
|
|
84
|
+
}
|
|
85
|
+
const graph = bind(series, {
|
|
86
|
+
registry: this.registry,
|
|
87
|
+
units: this.#units,
|
|
88
|
+
...(this.#budgetBytes !== undefined && {
|
|
89
|
+
budgetBytes: this.#budgetBytes,
|
|
90
|
+
}),
|
|
91
|
+
});
|
|
92
|
+
this.#graphs.set(id, graph);
|
|
93
|
+
return graph;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Forgets a dataset: its source, its graph, and every cached node
|
|
97
|
+
* value with them. The explicit end of a binding's lifecycle — the
|
|
98
|
+
* counterpart to {@link add} for a host that cycles datasets, where
|
|
99
|
+
* "kept until the process dies" is not a policy.
|
|
100
|
+
*
|
|
101
|
+
* Returns whether the dataset was known. A load in flight for the
|
|
102
|
+
* same source id **discards its result** rather than resurrecting the
|
|
103
|
+
* removed dataset — its `runAsync` callers get `UnknownDatasetError`,
|
|
104
|
+
* which is what asking for a concurrently-removed dataset means.
|
|
105
|
+
*/
|
|
106
|
+
remove(id) {
|
|
107
|
+
const known = this.#sources.delete(id);
|
|
108
|
+
this.#graphs.delete(id);
|
|
109
|
+
this.#loadedSources.delete(id);
|
|
110
|
+
if (this.#loadingSources.has(id)) {
|
|
111
|
+
this.#sourceEpochs.set(id, (this.#sourceEpochs.get(id) ?? 0) + 1);
|
|
112
|
+
}
|
|
113
|
+
return known;
|
|
114
|
+
}
|
|
115
|
+
/** Resolves an envelope against its dataset's long-lived graph. */
|
|
116
|
+
run(envelope) {
|
|
117
|
+
const graph = this.graphFor(envelope.from);
|
|
118
|
+
const options = {
|
|
119
|
+
...(envelope.onError !== undefined && { onError: envelope.onError }),
|
|
120
|
+
...(envelope.assemble !== undefined && { assemble: envelope.assemble }),
|
|
121
|
+
};
|
|
122
|
+
return 'nodes' in envelope
|
|
123
|
+
? run(graph, {
|
|
124
|
+
nodes: envelope.nodes,
|
|
125
|
+
...(envelope.outputs !== undefined && { outputs: envelope.outputs }),
|
|
126
|
+
...options,
|
|
127
|
+
})
|
|
128
|
+
: run(graph, {
|
|
129
|
+
plan: envelope.process,
|
|
130
|
+
...(envelope.select !== undefined && { select: envelope.select }),
|
|
131
|
+
...options,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Resolves either a preloaded dataset or an opaque asynchronous source.
|
|
136
|
+
*
|
|
137
|
+
* Equal revisions preserve the graph untouched. A changed revision updates
|
|
138
|
+
* its source in place, retaining compiled nodes while normal invalidation
|
|
139
|
+
* propagates from the new value.
|
|
140
|
+
*/
|
|
141
|
+
async runAsync(envelope) {
|
|
142
|
+
if (isLocalEnvelope(envelope))
|
|
143
|
+
return this.run(envelope);
|
|
144
|
+
if (this.#sourceRegistry === undefined) {
|
|
145
|
+
throw new ProcessError(`source '${envelope.from.source}' cannot load — this host has no source registry`);
|
|
146
|
+
}
|
|
147
|
+
const id = sourceId(envelope.from);
|
|
148
|
+
let loading = this.#loadingSources.get(id);
|
|
149
|
+
if (loading === undefined) {
|
|
150
|
+
loading = this.#refreshSource(id, envelope.from);
|
|
151
|
+
this.#loadingSources.set(id, loading);
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
await loading;
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
if (this.#loadingSources.get(id) === loading) {
|
|
158
|
+
this.#loadingSources.delete(id);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return this.run({ ...envelope, from: id });
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Loads and applies one source revision as a single in-flight operation.
|
|
165
|
+
*
|
|
166
|
+
* `runAsync` shares this promise per canonical source identity. Keeping the
|
|
167
|
+
* state update inside it means concurrent callers cannot each invalidate the
|
|
168
|
+
* graph or race to install an older revision.
|
|
169
|
+
*/
|
|
170
|
+
async #refreshSource(id, ref) {
|
|
171
|
+
const epoch = this.#sourceEpochs.get(id);
|
|
172
|
+
const previous = this.#loadedSources.get(id);
|
|
173
|
+
const loaded = await this.#sourceRegistry.load(ref, previous);
|
|
174
|
+
if (this.#sourceEpochs.get(id) !== epoch) {
|
|
175
|
+
// Removed while loading. Installing now would resurrect a dataset
|
|
176
|
+
// the author explicitly ended; dropping the result is the only
|
|
177
|
+
// reading of `remove` that means anything. The entry is consumed
|
|
178
|
+
// so the map holds nothing once the flight lands.
|
|
179
|
+
this.#sourceEpochs.delete(id);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
// Delete-then-set keeps `#loadedSources` in recency order whether or
|
|
183
|
+
// not the revision moved — the LRU the source cap evicts from.
|
|
184
|
+
this.#loadedSources.delete(id);
|
|
185
|
+
if (previous === undefined || previous.revision !== loaded.revision) {
|
|
186
|
+
this.#loadedSources.set(id, loaded);
|
|
187
|
+
this.add(id, loaded.value);
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
this.#loadedSources.set(id, previous);
|
|
191
|
+
}
|
|
192
|
+
this.#enforceSourceCap(id);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Evicts least-recently-used registry-loaded sources over the cap —
|
|
196
|
+
* whole datasets, via {@link remove}, so the graph and its cached
|
|
197
|
+
* nodes go with them. Never touches an author-added dataset: those
|
|
198
|
+
* are not in `#loadedSources`. `current` is exempt — evicting the
|
|
199
|
+
* source this call just loaded would fail the very request that paid
|
|
200
|
+
* for it.
|
|
201
|
+
*/
|
|
202
|
+
#enforceSourceCap(current) {
|
|
203
|
+
if (this.#maxSources === undefined)
|
|
204
|
+
return;
|
|
205
|
+
while (this.#loadedSources.size > this.#maxSources) {
|
|
206
|
+
const oldest = this.#loadedSources.keys().next().value;
|
|
207
|
+
if (oldest === undefined || oldest === current)
|
|
208
|
+
break;
|
|
209
|
+
this.remove(oldest);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
export function createHost(options) {
|
|
214
|
+
return new Host(options);
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Projects a result for transport.
|
|
218
|
+
*
|
|
219
|
+
* The renderer path hands back a live `TimeSeries` on purpose — a chart
|
|
220
|
+
* draws every point, and serializing 10⁶ of them per frame is the failure
|
|
221
|
+
* this split exists to avoid. `toWire` is for the other caller, and the
|
|
222
|
+
* useful property is that it is **only lossy when columns were asked
|
|
223
|
+
* for**: a facts-only response is already JSON-safe, so this is a no-op
|
|
224
|
+
* on exactly the requests that cross a wire.
|
|
225
|
+
*/
|
|
226
|
+
export function toWire(result, as) {
|
|
227
|
+
const { series, columns, ...rest } = result;
|
|
228
|
+
return {
|
|
229
|
+
...rest,
|
|
230
|
+
...(as !== undefined && { as }),
|
|
231
|
+
hasSeries: series !== undefined || columns !== undefined,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=host.js.map
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Identity, lineage, and units — and the [PND-PROCIDENT] decision.
|
|
3
|
+
*
|
|
4
|
+
* ## Identity is content-addressed, and that is forced
|
|
5
|
+
*
|
|
6
|
+
* The investigation weighed content-addressed ids against params-as-Ins
|
|
7
|
+
* and measured the latter far cheaper under a parameter sweep (1 node vs
|
|
8
|
+
* 200, 6 MB of buffers vs 310 MB). But writing `run()` settles it the
|
|
9
|
+
* other way, because the *plan format* decides it:
|
|
10
|
+
*
|
|
11
|
+
* A plan may hold `sma(px, 20)` **and** `sma(px, 50)` at once. They are
|
|
12
|
+
* two entries, they need two nodes, and — decisively — they need two
|
|
13
|
+
* distinct output **column names**. A column is named by its spec's id,
|
|
14
|
+
* so an id that excluded params would collide. Params must be in the id.
|
|
15
|
+
*
|
|
16
|
+
* The sweep result is still real; it is just an argument about
|
|
17
|
+
* **lifetime**, not identity. Dragging a slider mints a node per
|
|
18
|
+
* position, and those nodes stop being referenced by the current plan the
|
|
19
|
+
* moment it moves on. The answer is a retained set plus a budget
|
|
20
|
+
* ([PND-PROCCACHE]) — not a different identity model.
|
|
21
|
+
*
|
|
22
|
+
* So: **content-addressed identity, budgeted lifetime.**
|
|
23
|
+
*
|
|
24
|
+
* ## Two properties that must not regress
|
|
25
|
+
*
|
|
26
|
+
* Both are requirements rather than accidents, because a persisted saved
|
|
27
|
+
* view and a freshly composed request have to land on the same cache
|
|
28
|
+
* entry:
|
|
29
|
+
*
|
|
30
|
+
* - `specId` is **invariant under param key order** — a caller's JSON
|
|
31
|
+
* preserves insertion order and two callers will not agree on one.
|
|
32
|
+
* - An **omitted param collides with its explicit default**, so
|
|
33
|
+
* `{op:'sma'}` and `{op:'sma',params:{period:20}}` are one node.
|
|
34
|
+
*
|
|
35
|
+
* Both are pinned by tests.
|
|
36
|
+
*/
|
|
37
|
+
import type { Registry } from './registry.js';
|
|
38
|
+
import type { Params, Spec, SpecRef, Units } from './types.js';
|
|
39
|
+
/**
|
|
40
|
+
* Canonical, versioned id for a spec — simultaneously the **column
|
|
41
|
+
* name**, the **cache key**, and the **provenance citation**.
|
|
42
|
+
*
|
|
43
|
+
* Params are sorted by key and materialized post-defaults, so two
|
|
44
|
+
* spellings of one computation collide deliberately.
|
|
45
|
+
*/
|
|
46
|
+
export declare function specId(registry: Registry, spec: Spec): string;
|
|
47
|
+
/** Resolves a reference that may be an inline spec or an id string. */
|
|
48
|
+
export declare function refToId(registry: Registry, ref: SpecRef): string;
|
|
49
|
+
/**
|
|
50
|
+
* Human lineage, folded from the plan and the registry.
|
|
51
|
+
*
|
|
52
|
+
* Derived rather than reconstructed by hand — hand-built lineage is
|
|
53
|
+
* exactly what loses the inner `sma` in `ema(sma(x))`, which the RFC
|
|
54
|
+
* cites as a live consumer bug.
|
|
55
|
+
*/
|
|
56
|
+
export declare function explain(registry: Registry, spec: Spec): string;
|
|
57
|
+
/**
|
|
58
|
+
* Unit of a spec's output `n` — declared outright, or folded from input 0.
|
|
59
|
+
*
|
|
60
|
+
* `null` means unitless: the consumer supplied no unit for the raw column
|
|
61
|
+
* at the root of the chain. That is reported rather than guessed.
|
|
62
|
+
*/
|
|
63
|
+
export declare function unitOf(registry: Registry, spec: Spec, units: Units, outputIndex?: number): string | null;
|
|
64
|
+
/**
|
|
65
|
+
* Column names a spec produces, in output-declaration order.
|
|
66
|
+
*
|
|
67
|
+
* A single-output op declares suffix `''`, so its column *is* the id; a
|
|
68
|
+
* band's three columns share the id as a prefix, which is the corpus's
|
|
69
|
+
* own convention and needs no mapping layer.
|
|
70
|
+
*/
|
|
71
|
+
export declare function columnsOf(registry: Registry, spec: Spec, id: string): string[];
|
|
72
|
+
/**
|
|
73
|
+
* Which params an output depends on, defaulting to all of them.
|
|
74
|
+
*
|
|
75
|
+
* Declaring a narrower set is what lets a change to one param leave
|
|
76
|
+
* another output's version untouched ([PND-PROCSEL]).
|
|
77
|
+
*/
|
|
78
|
+
export declare function dependsOn(registry: Registry, spec: Spec, outputIndex: number): string[];
|
|
79
|
+
/** Params reduced to the subset an output depends on — its cache key. */
|
|
80
|
+
export declare function outputKey(params: Params, keys: readonly string[]): string;
|
|
81
|
+
//# sourceMappingURL=identity.d.ts.map
|