@elaraai/e3-types 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +829 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +301 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +17 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +28 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/intake.d.ts +75 -0
- package/dist/src/intake.d.ts.map +1 -0
- package/dist/src/intake.js +46 -0
- package/dist/src/intake.js.map +1 -0
- package/dist/src/lock.d.ts +253 -21
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +55 -21
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +38 -26
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +22 -26
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +326 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +141 -39
- package/src/intake.ts +53 -0
- package/src/lock.ts +67 -21
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +22 -26
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/dist/src/stream.spec.js
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
-
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
-
*/
|
|
5
|
-
/**
|
|
6
|
-
* The stream and merge command builders (issue #770): the argv a stream
|
|
7
|
-
* execution spawns for each merge mode and streamed-input choice, and the
|
|
8
|
-
* argv a merge execution spawns over sorted partials within a key range.
|
|
9
|
-
*/
|
|
10
|
-
import { describe, it } from 'node:test';
|
|
11
|
-
import assert from 'node:assert/strict';
|
|
12
|
-
import { variant } from '@elaraai/east';
|
|
13
|
-
import { mergeCommandIr, streamCommandIr } from './stream.js';
|
|
14
|
-
const EAST_C = variant('east_c', { platforms: ['east-c-std'] });
|
|
15
|
-
/** The argv a stream command evaluates to for the staged `inputs`. */
|
|
16
|
-
function argvOf(runner, spec, inputs) {
|
|
17
|
-
return streamCommandIr(runner, spec).compile([])(inputs, 'out.beast2');
|
|
18
|
-
}
|
|
19
|
-
/** The argv a merge command evaluates to for the staged `inputs`. */
|
|
20
|
-
function mergeArgvOf(runner, mode, inputs) {
|
|
21
|
-
return mergeCommandIr(runner, mode).compile([])(inputs, 'out.beast2');
|
|
22
|
-
}
|
|
23
|
-
describe('streamCommandIr', () => {
|
|
24
|
-
it('builds a folding dict argv: the merge IR as --merge, the first input streamed', () => {
|
|
25
|
-
assert.deepEqual(argvOf(EAST_C, { emit: 'dict', merge: 'function', stream: 'first' }, ['body.beast2', 'merge.beast2', 'rows.beast2', 'other.beast2']), ['east-c', 'run', '-p', 'east-c-std', '--emit', 'dict', '--merge', 'merge.beast2', '--stream', '0',
|
|
26
|
-
'-i', 'rows.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2']);
|
|
27
|
-
});
|
|
28
|
-
it('builds a union set argv streaming the first input', () => {
|
|
29
|
-
assert.deepEqual(argvOf(EAST_C, { emit: 'set', merge: 'union', stream: 'first' }, ['body.beast2', 'stream.beast2', 'other.beast2']), ['east-c', 'run', '-p', 'east-c-std', '--emit', 'set', '--union', '--stream', '0',
|
|
30
|
-
'-i', 'stream.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2']);
|
|
31
|
-
});
|
|
32
|
-
it('builds a producer argv with no merge flag and no streamed input', () => {
|
|
33
|
-
assert.deepEqual(argvOf(EAST_C, { emit: 'array', merge: 'none', stream: 'none' }, ['body.beast2', 'config.beast2']), ['east-c', 'run', '-p', 'east-c-std', '--emit', 'array', '-i', 'config.beast2', '-o', 'out.beast2', 'body.beast2']);
|
|
34
|
-
});
|
|
35
|
-
it('starts from the runner argv of every stock runtime', () => {
|
|
36
|
-
const node = variant('east_node', { platforms: ['@elaraai/east-node-std'] });
|
|
37
|
-
const argv = argvOf(node, { emit: 'set', merge: 'none', stream: 'first' }, ['body.beast2', 'rows.beast2']);
|
|
38
|
-
assert.deepEqual(argv.slice(0, 6), ['east-node', 'run', '-p', '@elaraai/east-node-std', '--emit', 'set']);
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
describe('mergeCommandIr', () => {
|
|
42
|
-
it('builds a dict merge argv: the merge IR as --merge, the key range as --range, every other input a partial', () => {
|
|
43
|
-
assert.deepEqual(mergeArgvOf(EAST_C, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2', 'p1.beast2', 'p2.beast2']), ['east-c', 'merge', '-p', 'east-c-std', '--merge', 'merge.beast2', '--range', 'range.beast2',
|
|
44
|
-
'-i', 'p0.beast2', '-i', 'p1.beast2', '-i', 'p2.beast2', '-o', 'out.beast2']);
|
|
45
|
-
});
|
|
46
|
-
it('builds a set union argv: the key range first, every other input a partial', () => {
|
|
47
|
-
assert.deepEqual(mergeArgvOf(EAST_C, 'union', ['range.beast2', 'p0.beast2', 'p1.beast2']), ['east-c', 'merge', '-p', 'east-c-std', '--union', '--range', 'range.beast2', '-i', 'p0.beast2', '-i', 'p1.beast2', '-o', 'out.beast2']);
|
|
48
|
-
});
|
|
49
|
-
it('runs the merge command of every stock runtime', () => {
|
|
50
|
-
const py = variant('east_py', { platforms: ['east-py-std'] });
|
|
51
|
-
assert.deepEqual(mergeArgvOf(py, 'union', ['range.beast2', 'p0.beast2']).slice(0, 4), ['east-py', 'merge', '-p', 'east-py-std']);
|
|
52
|
-
const node = variant('east_node', { platforms: [] });
|
|
53
|
-
assert.deepEqual(mergeArgvOf(node, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2']).slice(0, 2), ['east-node', 'merge']);
|
|
54
|
-
});
|
|
55
|
-
it('refuses a custom runner, which has no merge command', () => {
|
|
56
|
-
const custom = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
|
|
57
|
-
assert.throws(() => mergeCommandIr(custom, 'union'), { message: 'a custom runner has no merge command' });
|
|
58
|
-
});
|
|
59
|
-
});
|
|
60
|
-
//# sourceMappingURL=stream.spec.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"stream.spec.js","sourceRoot":"","sources":["../../src/stream.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,eAAe,EAA0B,MAAM,aAAa,CAAC;AAGtF,MAAM,MAAM,GAAgB,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;AAE7E,sEAAsE;AACtE,SAAS,MAAM,CAAC,MAAmB,EAAE,IAAuB,EAAE,MAAgB;IAC5E,OAAO,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACzE,CAAC;AAED,qEAAqE;AACrE,SAAS,WAAW,CAAC,MAAmB,EAAE,IAA0B,EAAE,MAAgB;IACpF,OAAO,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACxE,CAAC;AAED,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,+EAA+E,EAAE,GAAG,EAAE;QACvF,MAAM,CAAC,SAAS,CACd,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC,EACpI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG;YAChG,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,aAAa,CAAC,CAChF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,SAAS,CACd,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,aAAa,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC,EAClH,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG;YAC/E,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,aAAa,CAAC,CAClF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,SAAS,CACd,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,EAClG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE,aAAa,CAAC,CACnH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,IAAI,GAAgB,OAAO,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,CAAC,wBAAwB,CAAC,EAAE,CAAC,CAAC;QAC1F,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC;QAC3G,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,wBAAwB,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IAC5G,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,0GAA0G,EAAE,GAAG,EAAE;QAClH,MAAM,CAAC,SAAS,CACd,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,EACxG,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc;YAC1F,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,CAAC,CAC/E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,CAAC,SAAS,CACd,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,cAAc,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,EACxE,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,CAAC,CACxI,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,EAAE,GAAgB,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC3E,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC;QACjI,MAAM,IAAI,GAAgB,OAAO,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,cAAc,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACrI,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,MAAM,GAAgB,OAAO,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5F,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAC;IAC5G,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/src/stream.spec.ts
DELETED
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
-
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* The stream and merge command builders (issue #770): the argv a stream
|
|
8
|
-
* execution spawns for each merge mode and streamed-input choice, and the
|
|
9
|
-
* argv a merge execution spawns over sorted partials within a key range.
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
import { describe, it } from 'node:test';
|
|
13
|
-
import assert from 'node:assert/strict';
|
|
14
|
-
import { variant } from '@elaraai/east';
|
|
15
|
-
import { mergeCommandIr, streamCommandIr, type StreamCommandSpec } from './stream.js';
|
|
16
|
-
import type { RunnerValue } from './runner.js';
|
|
17
|
-
|
|
18
|
-
const EAST_C: RunnerValue = variant('east_c', { platforms: ['east-c-std'] });
|
|
19
|
-
|
|
20
|
-
/** The argv a stream command evaluates to for the staged `inputs`. */
|
|
21
|
-
function argvOf(runner: RunnerValue, spec: StreamCommandSpec, inputs: string[]): string[] {
|
|
22
|
-
return streamCommandIr(runner, spec).compile([])(inputs, 'out.beast2');
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/** The argv a merge command evaluates to for the staged `inputs`. */
|
|
26
|
-
function mergeArgvOf(runner: RunnerValue, mode: 'function' | 'union', inputs: string[]): string[] {
|
|
27
|
-
return mergeCommandIr(runner, mode).compile([])(inputs, 'out.beast2');
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
describe('streamCommandIr', () => {
|
|
31
|
-
it('builds a folding dict argv: the merge IR as --merge, the first input streamed', () => {
|
|
32
|
-
assert.deepEqual(
|
|
33
|
-
argvOf(EAST_C, { emit: 'dict', merge: 'function', stream: 'first' }, ['body.beast2', 'merge.beast2', 'rows.beast2', 'other.beast2']),
|
|
34
|
-
['east-c', 'run', '-p', 'east-c-std', '--emit', 'dict', '--merge', 'merge.beast2', '--stream', '0',
|
|
35
|
-
'-i', 'rows.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2'],
|
|
36
|
-
);
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
it('builds a union set argv streaming the first input', () => {
|
|
40
|
-
assert.deepEqual(
|
|
41
|
-
argvOf(EAST_C, { emit: 'set', merge: 'union', stream: 'first' }, ['body.beast2', 'stream.beast2', 'other.beast2']),
|
|
42
|
-
['east-c', 'run', '-p', 'east-c-std', '--emit', 'set', '--union', '--stream', '0',
|
|
43
|
-
'-i', 'stream.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2'],
|
|
44
|
-
);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it('builds a producer argv with no merge flag and no streamed input', () => {
|
|
48
|
-
assert.deepEqual(
|
|
49
|
-
argvOf(EAST_C, { emit: 'array', merge: 'none', stream: 'none' }, ['body.beast2', 'config.beast2']),
|
|
50
|
-
['east-c', 'run', '-p', 'east-c-std', '--emit', 'array', '-i', 'config.beast2', '-o', 'out.beast2', 'body.beast2'],
|
|
51
|
-
);
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
it('starts from the runner argv of every stock runtime', () => {
|
|
55
|
-
const node: RunnerValue = variant('east_node', { platforms: ['@elaraai/east-node-std'] });
|
|
56
|
-
const argv = argvOf(node, { emit: 'set', merge: 'none', stream: 'first' }, ['body.beast2', 'rows.beast2']);
|
|
57
|
-
assert.deepEqual(argv.slice(0, 6), ['east-node', 'run', '-p', '@elaraai/east-node-std', '--emit', 'set']);
|
|
58
|
-
});
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
describe('mergeCommandIr', () => {
|
|
62
|
-
it('builds a dict merge argv: the merge IR as --merge, the key range as --range, every other input a partial', () => {
|
|
63
|
-
assert.deepEqual(
|
|
64
|
-
mergeArgvOf(EAST_C, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2', 'p1.beast2', 'p2.beast2']),
|
|
65
|
-
['east-c', 'merge', '-p', 'east-c-std', '--merge', 'merge.beast2', '--range', 'range.beast2',
|
|
66
|
-
'-i', 'p0.beast2', '-i', 'p1.beast2', '-i', 'p2.beast2', '-o', 'out.beast2'],
|
|
67
|
-
);
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
it('builds a set union argv: the key range first, every other input a partial', () => {
|
|
71
|
-
assert.deepEqual(
|
|
72
|
-
mergeArgvOf(EAST_C, 'union', ['range.beast2', 'p0.beast2', 'p1.beast2']),
|
|
73
|
-
['east-c', 'merge', '-p', 'east-c-std', '--union', '--range', 'range.beast2', '-i', 'p0.beast2', '-i', 'p1.beast2', '-o', 'out.beast2'],
|
|
74
|
-
);
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
it('runs the merge command of every stock runtime', () => {
|
|
78
|
-
const py: RunnerValue = variant('east_py', { platforms: ['east-py-std'] });
|
|
79
|
-
assert.deepEqual(mergeArgvOf(py, 'union', ['range.beast2', 'p0.beast2']).slice(0, 4), ['east-py', 'merge', '-p', 'east-py-std']);
|
|
80
|
-
const node: RunnerValue = variant('east_node', { platforms: [] });
|
|
81
|
-
assert.deepEqual(mergeArgvOf(node, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2']).slice(0, 2), ['east-node', 'merge']);
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
it('refuses a custom runner, which has no merge command', () => {
|
|
85
|
-
const custom: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
|
|
86
|
-
assert.throws(() => mergeCommandIr(custom, 'union'), { message: 'a custom runner has no merge command' });
|
|
87
|
-
});
|
|
88
|
-
});
|
package/src/stream.ts
DELETED
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
-
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* The stream and merge commands (issue #770) — the builders of every argv a
|
|
8
|
-
* stream execution spawns (an authored `streamTask`'s) and of the merge
|
|
9
|
-
* command a partitioned task's keyed fan-in runs per key range, both built
|
|
10
|
-
* at export time and carried in the package like any other command IR.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
import { ArrayType, East, EastIR, StringType } from '@elaraai/east';
|
|
14
|
-
import { runnerToArgv, type RunnerValue } from './runner.js';
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* How a stream execution's emit sink treats equal keys: `none` — a duplicate
|
|
18
|
-
* is an error; `function` — the merge IR (wire input 1) folds adjacent equal
|
|
19
|
-
* Dict keys, passed to the runner as `--merge`; `union` — adjacent equal Set
|
|
20
|
-
* elements collapse to the first, `--union`.
|
|
21
|
-
*/
|
|
22
|
-
export type StreamMergeMode = 'none' | 'function' | 'union';
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* The shape of a stream command.
|
|
26
|
-
*
|
|
27
|
-
* @remarks
|
|
28
|
-
* The wire inputs a stream execution stages are laid out `[body IR, merge IR
|
|
29
|
-
* (function mode only), ...-i inputs]`; `stream` names whether the first of
|
|
30
|
-
* the `-i` inputs is the one the runner opens lazily.
|
|
31
|
-
*/
|
|
32
|
-
export interface StreamCommandSpec {
|
|
33
|
-
/** The output collection kind the emit sink writes. */
|
|
34
|
-
emit: 'array' | 'set' | 'dict';
|
|
35
|
-
/** How equal keys fold — see {@link StreamMergeMode}. */
|
|
36
|
-
merge: StreamMergeMode;
|
|
37
|
-
/** The `-i` input the runner feeds lazily: none, or the first. */
|
|
38
|
-
stream: 'none' | 'first';
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Builds the command IR of a stream execution: `(input_paths, output_path)
|
|
43
|
-
* -> argv`.
|
|
44
|
-
*
|
|
45
|
-
* The argv is `[...runner, '--emit', <emit>, '--merge' <merge IR path> |
|
|
46
|
-
* '--union', '--stream' '0' when the first -i input streams, '-i' <path> for
|
|
47
|
-
* each -i input, '-o', <output>, <body IR path>]` — the body IR is wire
|
|
48
|
-
* input 0, the merge IR wire input 1 in function mode, and the `-i` inputs
|
|
49
|
-
* follow. Built with `East.function` exactly as a task command is.
|
|
50
|
-
*
|
|
51
|
-
* @param runner - the stock runner the execution spawns
|
|
52
|
-
* @param spec - the emit kind, the merge mode and the streamed input
|
|
53
|
-
* @returns the command IR
|
|
54
|
-
*
|
|
55
|
-
* @example
|
|
56
|
-
* ```ts
|
|
57
|
-
* import { variant } from '@elaraai/east';
|
|
58
|
-
*
|
|
59
|
-
* const command = streamCommandIr(variant('east_c', { platforms: [] }), { emit: 'dict', merge: 'function', stream: 'first' });
|
|
60
|
-
* command.compile([])(['body.beast2', 'merge.beast2', 'a.beast2'], 'out.beast2');
|
|
61
|
-
* // ['east-c', 'run', '--emit', 'dict', '--merge', 'merge.beast2', '--stream', '0',
|
|
62
|
-
* // '-i', 'a.beast2', '-o', 'out.beast2', 'body.beast2']
|
|
63
|
-
* ```
|
|
64
|
-
*/
|
|
65
|
-
export function streamCommandIr(runner: RunnerValue, spec: StreamCommandSpec): EastIR<[string[], string], string[]> {
|
|
66
|
-
const prefix = [
|
|
67
|
-
...runnerToArgv(runner),
|
|
68
|
-
'--emit', spec.emit,
|
|
69
|
-
...(spec.merge === 'union' ? ['--union'] : []),
|
|
70
|
-
];
|
|
71
|
-
// The merge mode and the streamed input are known here, so the flags they
|
|
72
|
-
// add are emitted by branching at build time — only the merge IR's PATH is
|
|
73
|
-
// a run-time value, and only the -i inputs need a loop.
|
|
74
|
-
const mergeInput = spec.merge === 'function' ? 1n : null;
|
|
75
|
-
const firstInput = mergeInput === null ? 1n : 2n;
|
|
76
|
-
const command = East.function(
|
|
77
|
-
[ArrayType(StringType), StringType],
|
|
78
|
-
ArrayType(StringType),
|
|
79
|
-
($, input_paths, output_path) => {
|
|
80
|
-
const argv = $.let(prefix, ArrayType(StringType));
|
|
81
|
-
if (mergeInput !== null) {
|
|
82
|
-
$(argv.pushLast('--merge'));
|
|
83
|
-
$(argv.pushLast(input_paths.get(mergeInput)));
|
|
84
|
-
}
|
|
85
|
-
if (spec.stream === 'first') {
|
|
86
|
-
$(argv.pushLast('--stream'));
|
|
87
|
-
$(argv.pushLast('0'));
|
|
88
|
-
}
|
|
89
|
-
const i = $.let(firstInput);
|
|
90
|
-
$.while(East.less(i, input_paths.size()), $ => {
|
|
91
|
-
$(argv.pushLast('-i'));
|
|
92
|
-
$(argv.pushLast(input_paths.get(i)));
|
|
93
|
-
$.assign(i, i.add(1n));
|
|
94
|
-
});
|
|
95
|
-
$(argv.pushLast('-o'));
|
|
96
|
-
$(argv.pushLast(output_path));
|
|
97
|
-
$(argv.pushLast(input_paths.get(0n)));
|
|
98
|
-
$.return(argv);
|
|
99
|
-
},
|
|
100
|
-
);
|
|
101
|
-
return command.toIR();
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Builds the command IR of a merge execution — the runner's `merge` command
|
|
106
|
-
* over sorted partials of one Set or Dict type, within a key range:
|
|
107
|
-
* `(input_paths, output_path) -> argv`.
|
|
108
|
-
*
|
|
109
|
-
* The argv is `[...runner merge, '--merge' <merge IR path> | '--union',
|
|
110
|
-
* '--range' <range path>, '-i' <path> for each partial, '-o', <output>]`. In
|
|
111
|
-
* `function` mode wire input 0 is the merge IR `(K, V, V) -> V`, wire input
|
|
112
|
-
* 1 the key range and the partials follow; in `union` mode wire input 0 is
|
|
113
|
-
* the key range and every other wire input is a partial. The key range is a
|
|
114
|
-
* blob of `Struct{from: Option<K>, to: Option<K>}` over the output's key
|
|
115
|
-
* type: only the keys in `[from, to)` merge, each partial sought to the
|
|
116
|
-
* segment owning `from`, and an absent bound is open — a component that
|
|
117
|
-
* merges whole takes the open range. Equal keys across partials fold in
|
|
118
|
-
* input order, and the output is byte-identical to what the runner's emit
|
|
119
|
-
* sink writes for the same entries emitted ascending. A partitioned task's
|
|
120
|
-
* SDK writes this IR into its metadata at export, and the orchestrator runs
|
|
121
|
-
* it as an ordinary execution per key range of a group of partials.
|
|
122
|
-
*
|
|
123
|
-
* @param runner - the stock runner whose `merge` command runs
|
|
124
|
-
* @param mode - `function` (a Dict output, folded with the merge IR) or
|
|
125
|
-
* `union` (a Set output, the first of equal elements standing)
|
|
126
|
-
* @returns the command IR
|
|
127
|
-
* @throws {Error} When the runner is `custom` — it has no merge command.
|
|
128
|
-
*
|
|
129
|
-
* @example
|
|
130
|
-
* ```ts
|
|
131
|
-
* import { variant } from '@elaraai/east';
|
|
132
|
-
*
|
|
133
|
-
* const command = mergeCommandIr(variant('east_c', { platforms: [] }), 'function');
|
|
134
|
-
* command.compile([])(['merge.beast2', 'range.beast2', 'p0.beast2', 'p1.beast2'], 'out.beast2');
|
|
135
|
-
* // ['east-c', 'merge', '--merge', 'merge.beast2', '--range', 'range.beast2',
|
|
136
|
-
* // '-i', 'p0.beast2', '-i', 'p1.beast2', '-o', 'out.beast2']
|
|
137
|
-
* ```
|
|
138
|
-
*/
|
|
139
|
-
export function mergeCommandIr(runner: RunnerValue, mode: 'function' | 'union'): EastIR<[string[], string], string[]> {
|
|
140
|
-
const prefix = [
|
|
141
|
-
...runnerToArgv(runner, 'merge'),
|
|
142
|
-
...(mode === 'union' ? ['--union'] : []),
|
|
143
|
-
];
|
|
144
|
-
const mergeInput = mode === 'function' ? 0n : null;
|
|
145
|
-
const rangeInput = mergeInput === null ? 0n : 1n;
|
|
146
|
-
const firstInput = rangeInput + 1n;
|
|
147
|
-
const command = East.function(
|
|
148
|
-
[ArrayType(StringType), StringType],
|
|
149
|
-
ArrayType(StringType),
|
|
150
|
-
($, input_paths, output_path) => {
|
|
151
|
-
const argv = $.let(prefix, ArrayType(StringType));
|
|
152
|
-
if (mergeInput !== null) {
|
|
153
|
-
$(argv.pushLast('--merge'));
|
|
154
|
-
$(argv.pushLast(input_paths.get(mergeInput)));
|
|
155
|
-
}
|
|
156
|
-
$(argv.pushLast('--range'));
|
|
157
|
-
$(argv.pushLast(input_paths.get(rangeInput)));
|
|
158
|
-
const i = $.let(firstInput);
|
|
159
|
-
$.while(East.less(i, input_paths.size()), $ => {
|
|
160
|
-
$(argv.pushLast('-i'));
|
|
161
|
-
$(argv.pushLast(input_paths.get(i)));
|
|
162
|
-
$.assign(i, i.add(1n));
|
|
163
|
-
});
|
|
164
|
-
$(argv.pushLast('-o'));
|
|
165
|
-
$(argv.pushLast(output_path));
|
|
166
|
-
$.return(argv);
|
|
167
|
-
},
|
|
168
|
-
);
|
|
169
|
-
return command.toIR();
|
|
170
|
-
}
|