@jarenjs/linq 0.56.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +10 -0
- package/README.md +93 -2
- package/docs/APP-PEN.md +3 -3
- package/docs/CONTRACT-PEN.md +10 -6
- package/docs/DB-CLIENT.md +98 -19
- package/docs/FLOW-PEN.md +12 -5
- package/docs/FORMS-PEN.md +2 -2
- package/docs/JSLT-PEN.md +4 -4
- package/docs/LINQ-FORMAT.md +42 -34
- package/docs/MIGRATION-PEN.md +2 -2
- package/docs/MODEL-PEN.md +15 -6
- package/docs/QUERY-PEN.md +107 -19
- package/docs/SCHEMA-PEN.md +2 -2
- package/package.json +6 -6
- package/src/app/action.js +4 -8
- package/src/app/define.js +8 -13
- package/src/async.js +58 -10
- package/src/concurrency.js +40 -8
- package/src/contract/define.js +23 -10
- package/src/contract/index.js +5 -5
- package/src/contract/operation.js +10 -14
- package/src/db/handle.js +3 -0
- package/src/db/include.js +40 -5
- package/src/db/index.js +6 -0
- package/src/db/ledger.js +195 -0
- package/src/db/open.js +59 -11
- package/src/db/replication.js +20 -0
- package/src/errors.js +10 -1
- package/src/expression.js +30 -4
- package/src/federate.js +531 -0
- package/src/flow/dag.js +28 -14
- package/src/flow/fsm.js +6 -11
- package/src/index.js +1 -0
- package/src/jslt/rules.js +7 -12
- package/src/migration/define.js +9 -14
- package/src/migration/steps.js +5 -9
- package/src/model/collection.js +102 -0
- package/src/model/index.js +1 -1
- package/types/contract.d.ts +115 -18
- package/types/db.d.ts +189 -11
- package/types/index.d.ts +65 -0
- package/types/model.d.ts +34 -1
package/src/federate.js
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/**
|
|
3
|
+
* @file The federation boundary (QUERY-PEN.md §13): an EXPLICIT opt-in
|
|
4
|
+
* to reading two different provider sources under one query document.
|
|
5
|
+
*
|
|
6
|
+
* The rule everywhere else in this package is that a query document
|
|
7
|
+
* reads ONE input, and a join whose sides come from two unrelated
|
|
8
|
+
* sources is `JL0005` at build time. That refusal is not a limitation
|
|
9
|
+
* to route around — it is what keeps a chain honest about where the
|
|
10
|
+
* work happens. A cross-source join cannot be pushed anywhere: someone
|
|
11
|
+
* has to hold rows in memory, and a surface that did it implicitly
|
|
12
|
+
* would turn a one-line chain into an unbounded fetch of two
|
|
13
|
+
* databases.
|
|
14
|
+
*
|
|
15
|
+
* So it is spelled out instead. `federate()` takes the sources by
|
|
16
|
+
* name, takes the budgets that make the fetch finite, and hands back
|
|
17
|
+
* one provider-compatible source per name, all sharing one scope — so
|
|
18
|
+
* the join the chain already knows how to build is admitted, and the
|
|
19
|
+
* federation is what executes it:
|
|
20
|
+
*
|
|
21
|
+
* 1. each binding's own document — the filters and the projection the
|
|
22
|
+
* chain already packed per side — runs against ITS source;
|
|
23
|
+
* 2. the smaller side (declared estimate, else the first named) is
|
|
24
|
+
* streamed into a hash table keyed by the join key, counting rows
|
|
25
|
+
* and serialized bytes against the budget as it fills;
|
|
26
|
+
* 3. the other side is streamed and PROBED: a row whose key no build
|
|
27
|
+
* row carries cannot join, so it is dropped before it costs
|
|
28
|
+
* anything;
|
|
29
|
+
* 4. the caller's own document runs in the engine over the two
|
|
30
|
+
* reduced sets — the resident join, which is what decides.
|
|
31
|
+
*
|
|
32
|
+
* Step 4 is why this file spells no join semantics of its own. The
|
|
33
|
+
* engine's `$eq` decides which rows pair, its ordering orders them and
|
|
34
|
+
* its projection shapes them; the hash table exists to bound the FETCH,
|
|
35
|
+
* never to answer the query. A reduction that dropped a row the engine
|
|
36
|
+
* would have joined would be a wrong answer, so the probe keeps
|
|
37
|
+
* anything it cannot key (a compound key value) rather than guessing.
|
|
38
|
+
*
|
|
39
|
+
* Non-goals, named rather than discovered: no spill (a budget is a
|
|
40
|
+
* refusal, not a disk), no distributed transaction, no cross-source
|
|
41
|
+
* write, and no non-equality join — without an equality key the fetch
|
|
42
|
+
* is the cross product, which is exactly what the budget exists to
|
|
43
|
+
* refuse.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { LinqBuildError, LinqRuntimeError } from './errors.js';
|
|
47
|
+
import { compileDocument, isProviderSource, providerRoot } from './provider.js';
|
|
48
|
+
|
|
49
|
+
/** The strategies this boundary knows. One, for now, and it says so. */
|
|
50
|
+
const STRATEGIES = new Set(['hash']);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A positive integer budget, or the refusal that names it.
|
|
54
|
+
* @param {any} value
|
|
55
|
+
* @param {string} what
|
|
56
|
+
* @returns {number}
|
|
57
|
+
*/
|
|
58
|
+
function budgetOf(value, what) {
|
|
59
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
60
|
+
throw new LinqBuildError('JL0005',
|
|
61
|
+
`federate() needs a positive integer ${what} — a federated fetch holds rows in memory, `
|
|
62
|
+
+ 'and a bound is what makes that finite');
|
|
63
|
+
}
|
|
64
|
+
return value;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The one member path a `'$it.a.b'` operand names, or `null` for
|
|
69
|
+
* anything else (an operator call, a literal, the binding itself).
|
|
70
|
+
* @param {any} node
|
|
71
|
+
* @returns {{ binding: string, path: string[] } | null}
|
|
72
|
+
*/
|
|
73
|
+
function memberPathOf(node) {
|
|
74
|
+
if (typeof node !== 'string' || !node.startsWith('$')) return null;
|
|
75
|
+
const parts = node.slice(1).split('.');
|
|
76
|
+
if (parts.length < 2) return null;
|
|
77
|
+
const [binding, ...path] = parts;
|
|
78
|
+
if (binding === '' || path.some((name) => name === '')) return null;
|
|
79
|
+
return { binding, path };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Read one member path out of a row; `undefined` where it is absent. */
|
|
83
|
+
function valueAt(row, path) {
|
|
84
|
+
let value = row;
|
|
85
|
+
for (const name of path) {
|
|
86
|
+
if (value === null || typeof value !== 'object') return undefined;
|
|
87
|
+
value = value[name];
|
|
88
|
+
}
|
|
89
|
+
return value;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The hash key of one join-key value, or `null` when the value cannot
|
|
94
|
+
* be keyed at all.
|
|
95
|
+
*
|
|
96
|
+
* `null` is not "no key": it is the answer that this row must be kept
|
|
97
|
+
* whatever the other side holds, because the engine's own comparison
|
|
98
|
+
* decides it and a reduction may only ever drop rows that CANNOT pair.
|
|
99
|
+
* A compound value is the case — `$eq` over two objects is a deep
|
|
100
|
+
* comparison this table does not reproduce — so it opts out instead of
|
|
101
|
+
* approximating.
|
|
102
|
+
* @param {any} value
|
|
103
|
+
* @returns {string | null}
|
|
104
|
+
*/
|
|
105
|
+
function hashKey(value) {
|
|
106
|
+
if (value === undefined) return 'absent';
|
|
107
|
+
if (value === null) return 'null';
|
|
108
|
+
const type = typeof value;
|
|
109
|
+
if (type === 'string') return `s:${value}`;
|
|
110
|
+
if (type === 'number') return Object.is(value, -0) ? 'n:0' : `n:${value}`;
|
|
111
|
+
if (type === 'boolean') return `b:${value}`;
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** The serialized size of one row, in bytes, as the budget counts it.
|
|
116
|
+
* The encoder is built on first use: a module-level `new` is a side
|
|
117
|
+
* effect, and a bundle that never federates should not pay for one. */
|
|
118
|
+
let encoder = null;
|
|
119
|
+
function byteSize(row) {
|
|
120
|
+
const text = JSON.stringify(row);
|
|
121
|
+
if (text === undefined) return 0;
|
|
122
|
+
encoder ??= new TextEncoder();
|
|
123
|
+
return encoder.encode(text).length;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The root NAME a root expression carries: `'$.Post[*]'` → `'Post'`.
|
|
128
|
+
* A federated source's root is always this shape, because the
|
|
129
|
+
* federation names it.
|
|
130
|
+
* @param {string} root
|
|
131
|
+
* @returns {string}
|
|
132
|
+
*/
|
|
133
|
+
function rootName(root) {
|
|
134
|
+
const match = /^\$\.([^.[\]]+)\[\*\]$/.exec(root);
|
|
135
|
+
return match === null ? '' : match[1];
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* The FLWOR a terminal wrapped, and how to put a rewritten one back.
|
|
140
|
+
*
|
|
141
|
+
* An element terminal emits an ARRAY constructor around the query
|
|
142
|
+
* (`[{ $for … }]`) so a provider answers exactly one array; an
|
|
143
|
+
* aggregate wraps it in its own operator instead. The federation plans
|
|
144
|
+
* the query and hands the WRAPPER back untouched, because what the
|
|
145
|
+
* caller asked for around the join — one array, a count, a sum — is
|
|
146
|
+
* the engine's to answer over the rows this boundary fetched.
|
|
147
|
+
* @param {any} document
|
|
148
|
+
* @returns {{ flwor: any, rewrap: (flwor: any) => any } | null}
|
|
149
|
+
*/
|
|
150
|
+
function locateFlwor(document) {
|
|
151
|
+
if (document !== null && typeof document === 'object' && !Array.isArray(document)) {
|
|
152
|
+
if (Object.hasOwn(document, '$for')) return { flwor: document, rewrap: (next) => next };
|
|
153
|
+
const keys = Object.keys(document);
|
|
154
|
+
if (keys.length === 1) {
|
|
155
|
+
const inner = locateFlwor(document[keys[0]]);
|
|
156
|
+
if (inner !== null) {
|
|
157
|
+
return { flwor: inner.flwor,
|
|
158
|
+
rewrap: (next) => ({ [keys[0]]: inner.rewrap(next) }) };
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
if (Array.isArray(document) && document.length === 1) {
|
|
164
|
+
const inner = locateFlwor(document[0]);
|
|
165
|
+
if (inner !== null) return { flwor: inner.flwor, rewrap: (next) => [inner.rewrap(next)] };
|
|
166
|
+
}
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Split one federated document into the per-source work and the
|
|
172
|
+
* resident document that decides over it, or refuse by name.
|
|
173
|
+
*
|
|
174
|
+
* The chain has already done the hard half: a side carrying filters or
|
|
175
|
+
* a projection arrives PACKED as its own sub-document under the
|
|
176
|
+
* binding, and a bare side arrives as its root path. Either way the
|
|
177
|
+
* binding's value IS the child document, once its root is rewritten
|
|
178
|
+
* from the federated name to the source's own.
|
|
179
|
+
* @param {any} document
|
|
180
|
+
* @param {Map<string, any>} members - federated name → member record
|
|
181
|
+
* @returns {any}
|
|
182
|
+
*/
|
|
183
|
+
function planFederation(document, members) {
|
|
184
|
+
const located = locateFlwor(document);
|
|
185
|
+
const query = located === null ? document : located.flwor;
|
|
186
|
+
const bindings = query?.$for;
|
|
187
|
+
if (bindings === null || typeof bindings !== 'object' || Array.isArray(bindings)) {
|
|
188
|
+
throw new LinqBuildError('JL0005',
|
|
189
|
+
'a federated document ranges over its sources with $for — this one has no bindings');
|
|
190
|
+
}
|
|
191
|
+
const names = Object.keys(bindings);
|
|
192
|
+
if (names.length !== 2) {
|
|
193
|
+
throw new LinqBuildError('JL0005',
|
|
194
|
+
`a federated fetch joins exactly two sources, not ${names.length} — `
|
|
195
|
+
+ 'federate one pair at a time, or load the third side yourself');
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** @type {any[]} */
|
|
199
|
+
const sides = [];
|
|
200
|
+
for (const binding of names) {
|
|
201
|
+
const value = bindings[binding];
|
|
202
|
+
// the two spellings the chain builds: a bare root, or the side's
|
|
203
|
+
// own packed document (its `$where`, its `$orderby`, its `$return`)
|
|
204
|
+
const packed = Array.isArray(value) && value.length === 1 ? value[0] : null;
|
|
205
|
+
// a packed side ranges over ONE root: a side that is itself a join
|
|
206
|
+
// is a federation of a federation, which this boundary does not
|
|
207
|
+
// plan and will not guess at
|
|
208
|
+
const inner = packed === null ? [] : Object.keys(packed.$for ?? {});
|
|
209
|
+
const root = packed === null ? value
|
|
210
|
+
: (inner.length === 1 ? packed.$for[inner[0]] : null);
|
|
211
|
+
if (typeof root !== 'string') {
|
|
212
|
+
throw new LinqBuildError('JL0005',
|
|
213
|
+
`the binding '${binding}' does not range over a federated source`);
|
|
214
|
+
}
|
|
215
|
+
const member = members.get(root);
|
|
216
|
+
if (member === undefined) {
|
|
217
|
+
throw new LinqBuildError('JL0005',
|
|
218
|
+
`'${root}' is not one of this federation's sources `
|
|
219
|
+
+ `(${[...members.keys()].join(', ')})`);
|
|
220
|
+
}
|
|
221
|
+
sides.push({ binding, member, packed, root });
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const key = joinKey(query.$where, sides);
|
|
225
|
+
// the estimate decides which side fills the table: a hash join holds
|
|
226
|
+
// the BUILD side whole, so the smaller declared side is the one to
|
|
227
|
+
// hold. Undeclared estimates keep the caller's own order, which is
|
|
228
|
+
// stable and says so in `explain()`
|
|
229
|
+
const [first, second] = sides;
|
|
230
|
+
const build = (second.member.estimatedRows ?? Infinity)
|
|
231
|
+
< (first.member.estimatedRows ?? Infinity) ? second : first;
|
|
232
|
+
const probe = build === first ? second : first;
|
|
233
|
+
return {
|
|
234
|
+
strategy: 'hash',
|
|
235
|
+
build: { ...build, key: key[build.binding] },
|
|
236
|
+
probe: { ...probe, key: key[probe.binding] },
|
|
237
|
+
// the resident document is the caller's own with each side reduced
|
|
238
|
+
// to its root: the packed work has already run at the source, and
|
|
239
|
+
// whatever the terminal wrapped around the query is put back
|
|
240
|
+
resident: located.rewrap({ ...query,
|
|
241
|
+
$for: Object.fromEntries(sides.map((side) => [side.binding, side.root])) }),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The equality key that links the two sides, or the refusal.
|
|
247
|
+
* @param {any} where
|
|
248
|
+
* @param {any[]} sides
|
|
249
|
+
* @returns {Record<string, string[]>}
|
|
250
|
+
*/
|
|
251
|
+
function joinKey(where, sides) {
|
|
252
|
+
const conjuncts = where === undefined || where === null ? []
|
|
253
|
+
: (Array.isArray(where?.$and) ? where.$and : [where]);
|
|
254
|
+
const [a, b] = sides;
|
|
255
|
+
for (const conjunct of conjuncts) {
|
|
256
|
+
const operands = conjunct?.$eq;
|
|
257
|
+
if (!Array.isArray(operands) || operands.length !== 2) continue;
|
|
258
|
+
const left = memberPathOf(operands[0]);
|
|
259
|
+
const right = memberPathOf(operands[1]);
|
|
260
|
+
if (left === null || right === null) continue;
|
|
261
|
+
if (left.binding === a.binding && right.binding === b.binding)
|
|
262
|
+
return { [a.binding]: left.path, [b.binding]: right.path };
|
|
263
|
+
if (left.binding === b.binding && right.binding === a.binding)
|
|
264
|
+
return { [b.binding]: left.path, [a.binding]: right.path };
|
|
265
|
+
}
|
|
266
|
+
throw new LinqBuildError('JL0005',
|
|
267
|
+
'a federated join needs an equality between one member of each side — without one the '
|
|
268
|
+
+ 'fetch is the cross product of two sources, which is what the budget exists to refuse '
|
|
269
|
+
+ '(a non-equality condition still applies, but it cannot bound the fetch)');
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Stream one side's rows, under its own budget.
|
|
274
|
+
*
|
|
275
|
+
* A source that offers a cursor is pulled one row at a time, so the
|
|
276
|
+
* budget REFUSES before the row that would break it is held; one that
|
|
277
|
+
* offers only `execute` answers whole, and the count is checked over
|
|
278
|
+
* what came back — the budget is still honoured, but the memory was
|
|
279
|
+
* already spent at the source, which `explain()` says.
|
|
280
|
+
* @param {any} member
|
|
281
|
+
* @param {any} document
|
|
282
|
+
* @param {any} options
|
|
283
|
+
* @param {{ maxRows: number, maxBytes: number }} budget
|
|
284
|
+
* @param {(row: any) => boolean} keep
|
|
285
|
+
* @param {any[]} open - cursors to close, in order
|
|
286
|
+
* @returns {Promise<{ rows: any[], read: number, bytes: number, streamed: boolean }>}
|
|
287
|
+
*/
|
|
288
|
+
async function fetchSide(member, document, options, budget, keep, open) {
|
|
289
|
+
const kept = [];
|
|
290
|
+
let read = 0;
|
|
291
|
+
let bytes = 0;
|
|
292
|
+
const admit = (row) => {
|
|
293
|
+
read++;
|
|
294
|
+
if (!keep(row)) return;
|
|
295
|
+
if (kept.length + 1 > budget.maxRows) {
|
|
296
|
+
throw new LinqRuntimeError('JL2008',
|
|
297
|
+
`the federated fetch of '${member.name}' reached its ${budget.maxRows}-row budget — `
|
|
298
|
+
+ 'narrow the sides, or raise maxRows');
|
|
299
|
+
}
|
|
300
|
+
const size = byteSize(row);
|
|
301
|
+
if (bytes + size > budget.maxBytes) {
|
|
302
|
+
throw new LinqRuntimeError('JL2008',
|
|
303
|
+
`the federated fetch of '${member.name}' reached its ${budget.maxBytes}-byte budget `
|
|
304
|
+
+ `at row ${kept.length + 1} — narrow the sides, or raise maxBytes`);
|
|
305
|
+
}
|
|
306
|
+
bytes += size;
|
|
307
|
+
kept.push(row);
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
const signal = options?.signal;
|
|
311
|
+
const stopIfAborted = () => {
|
|
312
|
+
if (signal?.aborted === true) {
|
|
313
|
+
throw signal.reason instanceof Error ? signal.reason
|
|
314
|
+
: new LinqRuntimeError('JL2008',
|
|
315
|
+
`the federated fetch of '${member.name}' was aborted`);
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
const cursor = typeof member.provider.cursor === 'function'
|
|
320
|
+
? member.provider.cursor(document, options) : null;
|
|
321
|
+
if (cursor !== null) {
|
|
322
|
+
const opened = await cursor;
|
|
323
|
+
open.push(opened);
|
|
324
|
+
for (;;) {
|
|
325
|
+
// between rows, because that is where a cursor can be let go
|
|
326
|
+
// without abandoning a pull the source is still inside
|
|
327
|
+
stopIfAborted();
|
|
328
|
+
const next = await opened.next();
|
|
329
|
+
if (next.done === true) break;
|
|
330
|
+
admit(next.value);
|
|
331
|
+
}
|
|
332
|
+
return { rows: kept, read, bytes, streamed: true };
|
|
333
|
+
}
|
|
334
|
+
stopIfAborted();
|
|
335
|
+
const answer = await member.provider.execute(document, options);
|
|
336
|
+
for (const row of itemsOf(answer)) admit(row);
|
|
337
|
+
return { rows: kept, read, bytes, streamed: false };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** The engine's answer shape as a row list: none, one, or many. */
|
|
341
|
+
function itemsOf(answer) {
|
|
342
|
+
if (answer === undefined) return [];
|
|
343
|
+
return Array.isArray(answer) ? answer : [answer];
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** One side's document, with the federated root rewritten to the source's own. */
|
|
347
|
+
function childDocument(side) {
|
|
348
|
+
const own = providerRoot(side.member.provider);
|
|
349
|
+
if (side.packed === null) return { $for: { it: own }, $return: '$it' };
|
|
350
|
+
const binding = Object.keys(side.packed.$for)[0];
|
|
351
|
+
return { ...side.packed, $for: { ...side.packed.$for, [binding]: own } };
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* An explicit federation boundary over two or more provider sources.
|
|
356
|
+
*
|
|
357
|
+
* @param {{ sources: Record<string, any>, maxRows: number, maxBytes: number,
|
|
358
|
+
* strategy?: string }} spec
|
|
359
|
+
* @returns {{ source: (name: string) => any, names: readonly string[] }}
|
|
360
|
+
* @example
|
|
361
|
+
* const fed = federate({
|
|
362
|
+
* sources: { orders: shop.entity('Order'), events: analytics },
|
|
363
|
+
* maxRows: 50_000,
|
|
364
|
+
* maxBytes: 32 * 1024 * 1024,
|
|
365
|
+
* });
|
|
366
|
+
* const rows = await fromAsync(fed.source('orders'))
|
|
367
|
+
* .join(fromAsync(fed.source('events')), (o) => o.id, (e) => e.orderId,
|
|
368
|
+
* (o, e) => ({ id: o.id, at: e.at }))
|
|
369
|
+
* .toArray();
|
|
370
|
+
*/
|
|
371
|
+
export function federate(spec) {
|
|
372
|
+
const sources = spec?.sources;
|
|
373
|
+
if (sources === null || typeof sources !== 'object' || Array.isArray(sources)) {
|
|
374
|
+
throw new LinqBuildError('JL0005',
|
|
375
|
+
'federate() takes its sources as an object of name → provider');
|
|
376
|
+
}
|
|
377
|
+
const names = Object.keys(sources);
|
|
378
|
+
if (names.length < 2) {
|
|
379
|
+
throw new LinqBuildError('JL0005',
|
|
380
|
+
'federate() needs at least two named sources — one source is an ordinary chain');
|
|
381
|
+
}
|
|
382
|
+
const strategy = spec.strategy ?? 'hash';
|
|
383
|
+
if (!STRATEGIES.has(strategy)) {
|
|
384
|
+
throw new LinqBuildError('JL0005',
|
|
385
|
+
`federate() knows the strategies ${[...STRATEGIES].join(', ')}, not '${strategy}'`);
|
|
386
|
+
}
|
|
387
|
+
const budget = {
|
|
388
|
+
maxRows: budgetOf(spec.maxRows, 'maxRows'),
|
|
389
|
+
maxBytes: budgetOf(spec.maxBytes, 'maxBytes'),
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
/** The scope every member shares: what admits the join, and nothing else. */
|
|
393
|
+
const scope = Object.freeze({ federation: true });
|
|
394
|
+
/** @type {Map<string, any>} federated root → member */
|
|
395
|
+
const members = new Map();
|
|
396
|
+
/** @type {Map<string, any>} name → the source handle */
|
|
397
|
+
const handles = new Map();
|
|
398
|
+
|
|
399
|
+
for (const name of names) {
|
|
400
|
+
const declared = sources[name];
|
|
401
|
+
const provider = declared !== null && typeof declared === 'object'
|
|
402
|
+
&& 'provider' in declared ? declared.provider : declared;
|
|
403
|
+
if (!isProviderSource(provider)) {
|
|
404
|
+
throw new LinqBuildError('JL0001',
|
|
405
|
+
`federate()'s source '${name}' is not a provider: it exposes no execute(document, options)`);
|
|
406
|
+
}
|
|
407
|
+
const estimatedRows = declared?.estimatedRows;
|
|
408
|
+
if (estimatedRows !== undefined && (!Number.isInteger(estimatedRows) || estimatedRows < 0)) {
|
|
409
|
+
throw new LinqBuildError('JL0005',
|
|
410
|
+
`federate()'s source '${name}' declares a non-integer estimatedRows`);
|
|
411
|
+
}
|
|
412
|
+
// the federated root, which is what the chain binds and what the
|
|
413
|
+
// resident document ranges over; the source's OWN root is what its
|
|
414
|
+
// child document uses, and `childDocument` rewrites between them
|
|
415
|
+
const root = `$.${name}[*]`;
|
|
416
|
+
members.set(root, { name, root, provider, estimatedRows });
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Run one federated document: fetch each side under its budget, then
|
|
421
|
+
* let the engine decide over what came back.
|
|
422
|
+
* @param {any} document
|
|
423
|
+
* @param {any} options
|
|
424
|
+
*/
|
|
425
|
+
const execute = async (document, options) => {
|
|
426
|
+
const plan = planFederation(document, members);
|
|
427
|
+
/** @type {any[]} */
|
|
428
|
+
const open = [];
|
|
429
|
+
try {
|
|
430
|
+
const buildDoc = childDocument(plan.build);
|
|
431
|
+
const built = await fetchSide(plan.build.member, buildDoc, options, budget,
|
|
432
|
+
() => true, open);
|
|
433
|
+
// the table is the REDUCTION, never the answer: it says which
|
|
434
|
+
// keys can pair, and the engine decides which rows do
|
|
435
|
+
const keys = new Set();
|
|
436
|
+
let unkeyed = false;
|
|
437
|
+
for (const row of built.rows) {
|
|
438
|
+
const key = hashKey(valueAt(row, plan.build.key));
|
|
439
|
+
if (key === null) unkeyed = true;
|
|
440
|
+
else keys.add(key);
|
|
441
|
+
}
|
|
442
|
+
const probeDoc = childDocument(plan.probe);
|
|
443
|
+
const probed = await fetchSide(plan.probe.member, probeDoc, options, budget,
|
|
444
|
+
(row) => {
|
|
445
|
+
if (unkeyed) return true;
|
|
446
|
+
const key = hashKey(valueAt(row, plan.probe.key));
|
|
447
|
+
return key === null || keys.has(key);
|
|
448
|
+
}, open);
|
|
449
|
+
|
|
450
|
+
const input = {
|
|
451
|
+
[rootName(plan.build.root)]: built.rows,
|
|
452
|
+
[rootName(plan.probe.root)]: probed.rows,
|
|
453
|
+
};
|
|
454
|
+
const compiled = compileDocument(plan.resident,
|
|
455
|
+
{ ...options, externals: options?.externalNames ?? [] });
|
|
456
|
+
const answer = compiled(input, options?.externals ?? {});
|
|
457
|
+
// the fetch answered, so a cursor that will not close IS this
|
|
458
|
+
// call's failure rather than something to swallow
|
|
459
|
+
await closeAll(open);
|
|
460
|
+
return answer;
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
// the call's own failure is the one the caller gets: a cursor
|
|
464
|
+
// that also fails to close must not replace the budget's refusal
|
|
465
|
+
await closeAll(open);
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Every cursor a call opened, closed exactly once — whether it
|
|
472
|
+
* answered, refused, failed or was aborted. One that will not close
|
|
473
|
+
* does not strand the others; the first failure is raised after all
|
|
474
|
+
* of them have been tried.
|
|
475
|
+
* @param {any[]} open
|
|
476
|
+
*/
|
|
477
|
+
const closeAll = async (open) => {
|
|
478
|
+
/** @type {unknown[]} */
|
|
479
|
+
const failures = [];
|
|
480
|
+
for (const cursor of open.splice(0)) {
|
|
481
|
+
try {
|
|
482
|
+
if (typeof cursor?.return === 'function') await cursor.return();
|
|
483
|
+
}
|
|
484
|
+
catch (error) {
|
|
485
|
+
failures.push(error);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
if (failures.length > 0) throw failures[0];
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
/** What the federation will do, without doing any of it. */
|
|
492
|
+
const explain = (document) => {
|
|
493
|
+
const plan = planFederation(document, members);
|
|
494
|
+
const describe = (side) => ({
|
|
495
|
+
source: side.member.name,
|
|
496
|
+
root: side.root,
|
|
497
|
+
estimatedRows: side.member.estimatedRows ?? null,
|
|
498
|
+
key: `$${side.binding}.${side.key.join('.')}`,
|
|
499
|
+
document: childDocument(side),
|
|
500
|
+
streaming: typeof side.member.provider.cursor === 'function' ? 'row' : 'buffered',
|
|
501
|
+
});
|
|
502
|
+
return {
|
|
503
|
+
strategy: plan.strategy,
|
|
504
|
+
budget: { ...budget },
|
|
505
|
+
build: describe(plan.build),
|
|
506
|
+
probe: describe(plan.probe),
|
|
507
|
+
// the join itself is the engine's, over what the two fetches
|
|
508
|
+
// brought back — this boundary bounds the fetch and nothing else
|
|
509
|
+
resident: { document: plan.resident },
|
|
510
|
+
};
|
|
511
|
+
};
|
|
512
|
+
|
|
513
|
+
const handleFor = (name) => {
|
|
514
|
+
let handle = handles.get(name);
|
|
515
|
+
if (handle === undefined) {
|
|
516
|
+
const root = `$.${name}[*]`;
|
|
517
|
+
if (!members.has(root)) {
|
|
518
|
+
throw new LinqBuildError('JL0005',
|
|
519
|
+
`'${name}' is not one of this federation's sources (${names.join(', ')})`);
|
|
520
|
+
}
|
|
521
|
+
handle = Object.freeze({ execute, explain, root, scope });
|
|
522
|
+
handles.set(name, handle);
|
|
523
|
+
}
|
|
524
|
+
return handle;
|
|
525
|
+
};
|
|
526
|
+
|
|
527
|
+
return Object.freeze({
|
|
528
|
+
source: handleFor,
|
|
529
|
+
names: Object.freeze([...names]),
|
|
530
|
+
});
|
|
531
|
+
}
|
package/src/flow/dag.js
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* task-registry resolution (`JF0018`).
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
-
import { cloneJson, deepFreeze, setObjectMember } from '@jarenjs/core/object';
|
|
23
|
+
import { cloneJson, deepFreeze, setObjectMember, isJsonObject } from '@jarenjs/core/object';
|
|
24
24
|
|
|
25
25
|
import { LinqBuildError } from '../errors.js';
|
|
26
26
|
import { describeValue, requireJson, requireNameMap } from '../json-boundary.js';
|
|
@@ -41,11 +41,6 @@ const EDGE_MEMBERS = Object.freeze(['port', 'select']);
|
|
|
41
41
|
/** The members `defineDag()` takes. */
|
|
42
42
|
const DAG_MEMBERS = Object.freeze(['nodes', 'edges']);
|
|
43
43
|
|
|
44
|
-
/** @param {any} value */
|
|
45
|
-
function isPlainObject(value) {
|
|
46
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
44
|
/**
|
|
50
45
|
* A member set the pen knows, or `JL0101` naming the one it does not.
|
|
51
46
|
* @param {any} spec
|
|
@@ -75,7 +70,15 @@ function node(members) {
|
|
|
75
70
|
* The value must be JSON — `JF2008` at save time otherwise, never
|
|
76
71
|
* a silent skip.
|
|
77
72
|
*/
|
|
78
|
-
checkpoint() {
|
|
73
|
+
checkpoint() {
|
|
74
|
+
if (members.kind === 'task' && members.version === undefined) {
|
|
75
|
+
throw new LinqBuildError('JL0101',
|
|
76
|
+
`checkpoint() on task('${members.run}') needs the handler's declared version — `
|
|
77
|
+
+ "write task(run, props, { version }); a recorded value is replayed only while "
|
|
78
|
+
+ 'the handler that produced it is the same one', '/version');
|
|
79
|
+
}
|
|
80
|
+
return node({ ...members, checkpoint: true });
|
|
81
|
+
},
|
|
79
82
|
};
|
|
80
83
|
Object.defineProperty(out, NODE, { value: members, enumerable: false });
|
|
81
84
|
return Object.freeze(out);
|
|
@@ -151,16 +154,27 @@ export function jslt(document) {
|
|
|
151
154
|
* cannot).
|
|
152
155
|
* @param {string} run - the registry handler name
|
|
153
156
|
* @param {any} [props] - `(v) => ({ … })` over the input scope, or a query document
|
|
157
|
+
* @param {{ version?: string }} [options] - the declared identity of the
|
|
158
|
+
* handler implementation (§7.8), which the registry must supply too.
|
|
159
|
+
* REQUIRED on a `.checkpoint()` node: a recorded value is replayed only
|
|
160
|
+
* while the handler that produced it is the same one.
|
|
154
161
|
* @returns {any} the node declaration
|
|
155
162
|
* @example
|
|
156
|
-
* task('llm', (v) => ({ prompt: v.instruction }));
|
|
163
|
+
* task('llm', (v) => ({ prompt: v.instruction }), { version: '2026-09-05' });
|
|
157
164
|
*/
|
|
158
|
-
export function task(run, props = undefined) {
|
|
165
|
+
export function task(run, props = undefined, options = undefined) {
|
|
159
166
|
if (typeof run !== 'string' || run === '') {
|
|
160
167
|
throw new LinqBuildError('JL0101',
|
|
161
168
|
`task() takes the handler name as a non-empty string, got ${describeValue(run)}`, '/run');
|
|
162
169
|
}
|
|
170
|
+
const version = options?.version;
|
|
171
|
+
if (version !== undefined && (typeof version !== 'string' || version === '')) {
|
|
172
|
+
throw new LinqBuildError('JL0101',
|
|
173
|
+
`task() takes the handler version as a non-empty string, got ${describeValue(version)}`,
|
|
174
|
+
'/version');
|
|
175
|
+
}
|
|
163
176
|
const members = { kind: 'task', run };
|
|
177
|
+
if (version !== undefined) members.version = version;
|
|
164
178
|
if (props !== undefined) members.with = queryMember('task() with', SCOPE, props);
|
|
165
179
|
return node(members);
|
|
166
180
|
}
|
|
@@ -190,7 +204,7 @@ export function edge(from, to, options = undefined) {
|
|
|
190
204
|
}
|
|
191
205
|
const members = { from, to };
|
|
192
206
|
if (options !== undefined) {
|
|
193
|
-
if (!
|
|
207
|
+
if (!isJsonObject(options)) {
|
|
194
208
|
throw new LinqBuildError('JL0101',
|
|
195
209
|
`edge() options are { port?, select? }, got ${describeValue(options)}`);
|
|
196
210
|
}
|
|
@@ -231,12 +245,12 @@ export function edge(from, to, options = undefined) {
|
|
|
231
245
|
* await compileDag(graph).run([{ age: 20 }]);
|
|
232
246
|
*/
|
|
233
247
|
export function defineDag(spec) {
|
|
234
|
-
if (!
|
|
248
|
+
if (!isJsonObject(spec)) {
|
|
235
249
|
throw new LinqBuildError('JL0101',
|
|
236
250
|
`defineDag() takes { nodes, edges }, got ${describeValue(spec)}`);
|
|
237
251
|
}
|
|
238
252
|
closedTo(spec, DAG_MEMBERS, 'defineDag()');
|
|
239
|
-
if (!
|
|
253
|
+
if (!isJsonObject(spec.nodes)) {
|
|
240
254
|
throw new LinqBuildError('JL0101',
|
|
241
255
|
`defineDag() nodes is a plain object of id → node declaration, got ${describeValue(spec.nodes)}`,
|
|
242
256
|
'/nodes');
|
|
@@ -249,7 +263,7 @@ export function defineDag(spec) {
|
|
|
249
263
|
const nodes = {};
|
|
250
264
|
for (const id of ids) {
|
|
251
265
|
const declared = spec.nodes[id];
|
|
252
|
-
const members =
|
|
266
|
+
const members = isJsonObject(declared) ? declared[NODE] : undefined;
|
|
253
267
|
if (members === undefined) {
|
|
254
268
|
throw new LinqBuildError('JL0101',
|
|
255
269
|
`defineDag() node '${id}' is input(), constant(), query(), jslt(), task() or `
|
|
@@ -265,7 +279,7 @@ export function defineDag(spec) {
|
|
|
265
279
|
}
|
|
266
280
|
const edges = spec.edges.map((declared, i) => {
|
|
267
281
|
const at = `/edges/${i}`;
|
|
268
|
-
const members =
|
|
282
|
+
const members = isJsonObject(declared) ? declared[EDGE] : undefined;
|
|
269
283
|
if (members === undefined) {
|
|
270
284
|
throw new LinqBuildError('JL0101',
|
|
271
285
|
`defineDag() edges[${i}] is edge(from, to, options?), got ${describeValue(declared)}`, at);
|