@elaraai/e3-types 1.0.79 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +569 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +257 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +16 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +25 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/lock.d.ts +68 -22
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +33 -22
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +16 -22
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +16 -22
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +276 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +122 -39
- package/src/lock.ts +39 -22
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +16 -22
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A repository's own records: the record of the forms it keeps — the release
|
|
8
|
+
* of e3 that last wrote it, and the store upgrades it has had — and its
|
|
9
|
+
* metadata: its name, its lifecycle status and when each began.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
ArrayType,
|
|
14
|
+
VariantType,
|
|
15
|
+
StructType,
|
|
16
|
+
StringType,
|
|
17
|
+
DateTimeType,
|
|
18
|
+
NullType,
|
|
19
|
+
ValueTypeOf,
|
|
20
|
+
} from '@elaraai/east';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The repository record: the release of e3 that last wrote it, and the store
|
|
24
|
+
* upgrades the repository has had, which an e3 opening it reads before
|
|
25
|
+
* anything else.
|
|
26
|
+
*/
|
|
27
|
+
export const RepositoryRecordType = StructType({
|
|
28
|
+
/** The release of e3 that last wrote the record */
|
|
29
|
+
release: StringType,
|
|
30
|
+
/** The store upgrades the repository has had, in the order they were
|
|
31
|
+
* applied: each by its name, with the release of e3 that applied it */
|
|
32
|
+
upgrades: ArrayType(StructType({ name: StringType, release: StringType })),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
export type RepositoryRecord = ValueTypeOf<typeof RepositoryRecordType>;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Where a repository is in its lifecycle.
|
|
39
|
+
*/
|
|
40
|
+
export const RepoStatusType = VariantType({
|
|
41
|
+
/** Being initialised */
|
|
42
|
+
creating: NullType,
|
|
43
|
+
/** Ready for use */
|
|
44
|
+
active: NullType,
|
|
45
|
+
/** A garbage collection is running */
|
|
46
|
+
gc: NullType,
|
|
47
|
+
/** Being deleted */
|
|
48
|
+
deleting: NullType,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
export type RepoStatus = ValueTypeOf<typeof RepoStatusType>;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* A repository's metadata.
|
|
55
|
+
*/
|
|
56
|
+
export const RepoMetadataType = StructType({
|
|
57
|
+
/** The repository's name */
|
|
58
|
+
name: StringType,
|
|
59
|
+
/** Where it is in its lifecycle */
|
|
60
|
+
status: RepoStatusType,
|
|
61
|
+
/** When it was created */
|
|
62
|
+
createdAt: DateTimeType,
|
|
63
|
+
/** When its status last changed */
|
|
64
|
+
statusChangedAt: DateTimeType,
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
export type RepoMetadata = ValueTypeOf<typeof RepoMetadataType>;
|
package/src/runner.spec.ts
CHANGED
|
@@ -4,25 +4,18 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Gating contract for {@link
|
|
7
|
+
* Gating contract for {@link withRunnerLifeline}.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
* before spawn. These tests pin the two things that
|
|
11
|
-
*
|
|
12
|
-
* never touches a `custom` runner's user-authored command.
|
|
13
|
-
* rewrites the argv (never the runner variant or any hash), it is
|
|
14
|
-
* cache-irrelevant by construction.
|
|
9
|
+
* `--exit-with-parent` is a pure runtime toggle: it is spliced into an
|
|
10
|
+
* already-built argv just before spawn. These tests pin the two things that
|
|
11
|
+
* make it safe — it reaches the known runtimes at the right argv position, and
|
|
12
|
+
* it never touches a `custom` runner's user-authored command.
|
|
15
13
|
*/
|
|
16
14
|
|
|
17
15
|
import { describe, it } from 'node:test';
|
|
18
16
|
import * as assert from 'node:assert';
|
|
19
17
|
import { variant } from '@elaraai/east';
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
/** A realistic fully-built argv: runner prefix + the `-i`/`-o`/`<ir>` suffix. */
|
|
23
|
-
function fullArgv(runner: RunnerValue): string[] {
|
|
24
|
-
return [...runnerToArgv(runner), '-i', 'in.beast2', '-o', 'out.beast2', 'body.beast2'];
|
|
25
|
-
}
|
|
18
|
+
import { withRunnerLifeline, type RunnerValue } from './runner.js';
|
|
26
19
|
|
|
27
20
|
const KNOWN: Array<[string, RunnerValue, string]> = [
|
|
28
21
|
['east_node', variant('east_node', { platforms: ['@elaraai/east-node-std'] }), 'east-node'],
|
|
@@ -30,100 +23,24 @@ const KNOWN: Array<[string, RunnerValue, string]> = [
|
|
|
30
23
|
['east_c', variant('east_c', { platforms: ['east-c-std'] }), 'east-c'],
|
|
31
24
|
];
|
|
32
25
|
|
|
33
|
-
describe('withRunnerVerbose', () => {
|
|
34
|
-
for (const [tag, runner, bin] of KNOWN) {
|
|
35
|
-
it(`inserts -v after the 'run' subcommand for ${tag}`, () => {
|
|
36
|
-
const args = fullArgv(runner);
|
|
37
|
-
const out = withRunnerVerbose(runner, args, true);
|
|
38
|
-
// `[<bin>, 'run', '-v', …rest]` — position 2, ahead of -p/-i/-o and the IR.
|
|
39
|
-
assert.deepStrictEqual(out.slice(0, 3), [bin, 'run', '-v']);
|
|
40
|
-
// Exactly one -v, and the rest of the argv is preserved in order.
|
|
41
|
-
assert.strictEqual(out.filter((a) => a === '-v').length, 1);
|
|
42
|
-
assert.deepStrictEqual(out.slice(3), args.slice(2));
|
|
43
|
-
// The IR path is still last.
|
|
44
|
-
assert.strictEqual(out[out.length - 1], 'body.beast2');
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it(`leaves the argv untouched when verbose is off for ${tag}`, () => {
|
|
48
|
-
const args = fullArgv(runner);
|
|
49
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, args, false), args);
|
|
50
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, args, undefined), args);
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
it('never splices -v into a custom runner (user-authored argv)', () => {
|
|
55
|
-
const runner: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
|
|
56
|
-
const args = fullArgv(runner);
|
|
57
|
-
// Even with verbose requested, a custom command is passed through verbatim —
|
|
58
|
-
// we cannot know it understands `-v`.
|
|
59
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, args, true), args);
|
|
60
|
-
assert.ok(!withRunnerVerbose(runner, args, true).includes('-v'));
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
it('does not mutate the input argv (returns a new array)', () => {
|
|
64
|
-
const runner = variant('east_node', { platforms: ['@elaraai/east-node-std'] });
|
|
65
|
-
const args = fullArgv(runner);
|
|
66
|
-
const before = [...args];
|
|
67
|
-
withRunnerVerbose(runner, args, true);
|
|
68
|
-
assert.deepStrictEqual(args, before, 'input array must be unchanged');
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
it('is a no-op on a degenerate (too-short) argv rather than throwing', () => {
|
|
72
|
-
const runner = variant('east_node', { platforms: [] });
|
|
73
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, ['east-node'], true), ['east-node']);
|
|
74
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, [], true), []);
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
it('inserts -v after the merge subcommand too', () => {
|
|
78
|
-
const runner = variant('east_c', { platforms: ['east-c-std'] });
|
|
79
|
-
const args = [...runnerToArgv(runner, 'merge'), '--union', '-i', 'p0.beast2', '-o', 'out.beast2'];
|
|
80
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, args, true).slice(0, 3), ['east-c', 'merge', '-v']);
|
|
81
|
-
});
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
describe('runnerToArgv', () => {
|
|
85
|
-
for (const [tag, runner, bin] of KNOWN) {
|
|
86
|
-
it(`resolves the run and merge commands of ${tag}`, () => {
|
|
87
|
-
assert.deepStrictEqual(runnerToArgv(runner).slice(0, 2), [bin, 'run']);
|
|
88
|
-
assert.deepStrictEqual(runnerToArgv(runner, 'run'), runnerToArgv(runner));
|
|
89
|
-
const merge = runnerToArgv(runner, 'merge');
|
|
90
|
-
assert.deepStrictEqual(merge.slice(0, 2), [bin, 'merge']);
|
|
91
|
-
// The platform flags follow either command unchanged.
|
|
92
|
-
assert.deepStrictEqual(merge.slice(2), runnerToArgv(runner).slice(2));
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
it('resolves a custom runner to its own command, and has no merge command for it', () => {
|
|
97
|
-
const runner: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
|
|
98
|
-
assert.deepStrictEqual(runnerToArgv(runner), ['uv', 'run', 'east-py', 'run']);
|
|
99
|
-
assert.throws(() => runnerToArgv(runner, 'merge'), { message: 'a custom runner has no merge command' });
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
|
|
103
26
|
describe('withRunnerLifeline', () => {
|
|
104
27
|
for (const [tag, runner, bin] of KNOWN) {
|
|
105
|
-
it(`inserts --exit-with-parent after the subcommand for ${tag}
|
|
106
|
-
const args =
|
|
107
|
-
|
|
108
|
-
assert.deepStrictEqual(out.slice(0, 3), [bin, 'run', '--exit-with-parent']);
|
|
109
|
-
assert.deepStrictEqual(out.slice(3), args.slice(2));
|
|
110
|
-
const merge = [...runnerToArgv(runner, 'merge'), '--union', '-i', 'p0.beast2', '-o', 'out.beast2'];
|
|
111
|
-
assert.deepStrictEqual(withRunnerLifeline(runner, merge).slice(0, 3), [bin, 'merge', '--exit-with-parent']);
|
|
112
|
-
// Composed with -v, both sit ahead of the flags, in either order.
|
|
113
|
-
assert.deepStrictEqual(withRunnerVerbose(runner, withRunnerLifeline(runner, args), true).slice(0, 4), [bin, 'run', '-v', '--exit-with-parent']);
|
|
28
|
+
it(`inserts --exit-with-parent after the subcommand for ${tag}`, () => {
|
|
29
|
+
const args = [bin, 'exec', 'unit.beast2', '-v'];
|
|
30
|
+
assert.deepStrictEqual(withRunnerLifeline(runner, args), [bin, 'exec', '--exit-with-parent', 'unit.beast2', '-v']);
|
|
114
31
|
});
|
|
115
32
|
}
|
|
116
33
|
|
|
117
34
|
it('never splices the flag into a custom runner, which keeps an ignored stdin', () => {
|
|
118
35
|
const runner: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
|
|
119
|
-
const args =
|
|
36
|
+
const args = ['uv', 'run', 'east-py', 'run', '-i', 'in.beast2', '-o', 'out.beast2', 'program.beast2'];
|
|
120
37
|
assert.deepStrictEqual(withRunnerLifeline(runner, args), args);
|
|
121
38
|
});
|
|
122
39
|
|
|
123
40
|
it('is a no-op on a degenerate argv and does not mutate its input', () => {
|
|
124
41
|
const runner = variant('east_c', { platforms: [] });
|
|
125
42
|
assert.deepStrictEqual(withRunnerLifeline(runner, ['east-c']), ['east-c']);
|
|
126
|
-
const args =
|
|
43
|
+
const args = ['east-c', 'exec', 'unit.beast2'];
|
|
127
44
|
const before = [...args];
|
|
128
45
|
withRunnerLifeline(runner, args);
|
|
129
46
|
assert.deepStrictEqual(args, before);
|
package/src/runner.ts
CHANGED
|
@@ -4,31 +4,25 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Runner wire types for e3 functions.
|
|
7
|
+
* Runner wire types for e3 tasks, functions and records.
|
|
8
8
|
*
|
|
9
|
-
* `RunnerType` is the wire image of the SDK's `Runner` union
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* name its own executable; only platform flags vary.
|
|
13
|
-
*
|
|
14
|
-
* A `platforms` entry is just a `-p <name>` flag on the wire; the SDK's
|
|
15
|
-
* `Platform<Known> | { custom }` distinction is authoring sugar that
|
|
16
|
-
* collapses to a string.
|
|
9
|
+
* `RunnerType` is the wire image of the SDK's `Runner` union. A known-runtime
|
|
10
|
+
* tag pins the executable to one of the stock runners, which runs units through
|
|
11
|
+
* its `exec` command; `custom` names a command of the author's own.
|
|
17
12
|
*/
|
|
18
13
|
|
|
19
14
|
import { VariantType, StructType, ArrayType, StringType, ValueTypeOf } from '@elaraai/east';
|
|
20
15
|
|
|
21
16
|
/**
|
|
22
|
-
* Wire representation of a runner — shared by tasks and
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
* command
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* so `custom` grants no capability that tasks don't have.
|
|
17
|
+
* Wire representation of a runner — shared by tasks, functions and records.
|
|
18
|
+
*
|
|
19
|
+
* A known-runtime tag names a stock runner, and `platforms` names the platform
|
|
20
|
+
* packages the units it runs load. `custom` carries a command, which e3 runs
|
|
21
|
+
* with the arguments of a stock runner's `run`: `-i` for each input, `-o` for
|
|
22
|
+
* the output, then the program's file. A custom task's body is its own
|
|
23
|
+
* command, and its runner's is empty. Package authors can already execute
|
|
24
|
+
* arbitrary commands via custom tasks, so `custom` grants no capability that
|
|
25
|
+
* tasks don't have.
|
|
32
26
|
*/
|
|
33
27
|
export const RunnerType = VariantType({
|
|
34
28
|
east_node: StructType({ platforms: ArrayType(StringType) }),
|
|
@@ -40,63 +34,6 @@ export type RunnerType = typeof RunnerType;
|
|
|
40
34
|
|
|
41
35
|
export type RunnerValue = ValueTypeOf<typeof RunnerType>;
|
|
42
36
|
|
|
43
|
-
function flags(platforms: string[]): string[] {
|
|
44
|
-
return platforms.flatMap((p) => ['-p', p]);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Resolve a {@link RunnerType} value to the argv prefix of one of its
|
|
49
|
-
* commands (the wire-value analogue of the SDK's `runnerToCommand`): `run`,
|
|
50
|
-
* the default — `[<bin>, 'run', -p…]` — or `merge`, the blob merge every
|
|
51
|
-
* stock runner ships — `[<bin>, 'merge', -p…]`, the fan-in of a partitioned
|
|
52
|
-
* task's keyed partials (issue #770). Lives in e3-types so both e3-core
|
|
53
|
-
* (local) and the cloud execution kernel import the one resolver.
|
|
54
|
-
*
|
|
55
|
-
* Variant tags use underscores (`east_node`) mapped to the binary name
|
|
56
|
-
* (`east-node`) here. A `custom` runner's command is its `run`; it has no
|
|
57
|
-
* merge command.
|
|
58
|
-
*
|
|
59
|
-
* @param r - the runner
|
|
60
|
-
* @param command - the runner command, `run` by default
|
|
61
|
-
* @returns the argv prefix
|
|
62
|
-
* @throws {Error} When `merge` is asked of a custom runner.
|
|
63
|
-
*/
|
|
64
|
-
export function runnerToArgv(r: RunnerValue, command: 'run' | 'merge' = 'run'): string[] {
|
|
65
|
-
switch (r.type) {
|
|
66
|
-
case 'east_node': return ['east-node', command, ...flags(r.value.platforms)];
|
|
67
|
-
case 'east_py': return ['east-py', command, ...flags(r.value.platforms)];
|
|
68
|
-
case 'east_c': return ['east-c', command, ...flags(r.value.platforms)];
|
|
69
|
-
case 'custom':
|
|
70
|
-
if (command !== 'run') throw new Error(`a custom runner has no ${command} command`);
|
|
71
|
-
return [...r.value.command];
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Insert the runner's `-v/--verbose` flag into a fully-built argv, for the
|
|
77
|
-
* known runtimes only.
|
|
78
|
-
*
|
|
79
|
-
* All three known runners (`east-node`/`east-py`/`east-c`, `run` and `merge`)
|
|
80
|
-
* accept `-v` among their options and print timing/perf detail to stderr; a
|
|
81
|
-
* `custom` runner's argv is user-authored, so a flag is never spliced into
|
|
82
|
-
* it. A known-runtime argv always starts `[<bin>, <command>, …]` (see
|
|
83
|
-
* {@link runnerToArgv} and the SDK's `runnerToCommand`), so `-v` goes at
|
|
84
|
-
* index 2 — ahead of the `-p`/`-i`/`-o` flags and the trailing IR path.
|
|
85
|
-
*
|
|
86
|
-
* This is a pure **runtime** toggle: it is applied to the *evaluated* argv
|
|
87
|
-
* immediately before spawn and never touches the task's `commandIr`, the
|
|
88
|
-
* {@link RunnerType}, or any hash — so it cannot affect caching.
|
|
89
|
-
*
|
|
90
|
-
* @param runner - the runner the argv was built for (gates the injection)
|
|
91
|
-
* @param args - the fully-built argv (runner prefix + `-i`/`-o`/`<ir>` suffix)
|
|
92
|
-
* @param verbose - when true, request the runner's verbose output
|
|
93
|
-
* @returns the argv, with `-v` inserted for known runtimes when verbose
|
|
94
|
-
*/
|
|
95
|
-
export function withRunnerVerbose(runner: RunnerValue, args: string[], verbose?: boolean): string[] {
|
|
96
|
-
if (!verbose || runner.type === 'custom' || args.length < 2) return args;
|
|
97
|
-
return [...args.slice(0, 2), '-v', ...args.slice(2)];
|
|
98
|
-
}
|
|
99
|
-
|
|
100
37
|
/**
|
|
101
38
|
* Insert the runner's `--exit-with-parent` flag into a fully-built argv, for
|
|
102
39
|
* the known runtimes only — the stdin lifeline (issue #770).
|
|
@@ -104,10 +41,9 @@ export function withRunnerVerbose(runner: RunnerValue, args: string[], verbose?:
|
|
|
104
41
|
* A stock runner given the flag and a stdin pipe its parent never writes to
|
|
105
42
|
* exits as soon as the pipe reaches end of file: when the parent dies. A
|
|
106
43
|
* `custom` runner's argv is user-authored, so the flag is never spliced into
|
|
107
|
-
* it, and it keeps an ignored stdin.
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
* hash.
|
|
44
|
+
* it, and it keeps an ignored stdin. The flag goes at index 2, after
|
|
45
|
+
* `[<bin>, <command>]`, and is a pure runtime toggle applied just before
|
|
46
|
+
* spawn — never part of the task object or any hash.
|
|
111
47
|
*
|
|
112
48
|
* @param runner - the runner the argv was built for (gates the injection)
|
|
113
49
|
* @param args - the fully-built argv
|
package/src/structure.ts
CHANGED
|
@@ -61,14 +61,6 @@ export type StructureType = typeof StructureType;
|
|
|
61
61
|
|
|
62
62
|
export type Structure = ValueTypeOf<typeof StructureType>;
|
|
63
63
|
|
|
64
|
-
// Backwards compatibility alias
|
|
65
|
-
/** @deprecated Use StructureType instead */
|
|
66
|
-
export const DatasetSchemaType = StructureType;
|
|
67
|
-
/** @deprecated Use Structure instead */
|
|
68
|
-
export type DatasetSchemaType = StructureType;
|
|
69
|
-
/** @deprecated Use Structure instead */
|
|
70
|
-
export type DatasetSchema = Structure;
|
|
71
|
-
|
|
72
64
|
/**
|
|
73
65
|
* Path segment for navigating data trees.
|
|
74
66
|
*
|
package/src/task.spec.ts
CHANGED
|
@@ -4,205 +4,84 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
8
|
-
* metadata's merge fields and its triple decoder, the stream metadata's
|
|
9
|
-
* `merge` mode and its dual decoder, the partition plan round trip, and the
|
|
10
|
-
* `by` projection shapes the orchestrator evaluates by reading key fields
|
|
11
|
-
* instead of compiling the projection.
|
|
7
|
+
* Task object wire: the typed task object's round trip and its refusals.
|
|
12
8
|
*/
|
|
13
9
|
|
|
14
10
|
import { describe, it } from 'node:test';
|
|
15
11
|
import assert from 'node:assert/strict';
|
|
16
|
-
import { ArrayType, BlobType,
|
|
17
|
-
import type { FunctionIR } from '@elaraai/east';
|
|
12
|
+
import { ArrayType, BlobType, IntegerType, OptionType, StringType, StructType, encodeBeast2For, none, some, variant } from '@elaraai/east';
|
|
18
13
|
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
encodePartitionTaskMetadata,
|
|
24
|
-
encodeStreamTaskMetadata,
|
|
25
|
-
partitionProjectionShape,
|
|
26
|
-
projectKey,
|
|
27
|
-
projectedKeyType,
|
|
28
|
-
type PartitionPlan,
|
|
29
|
-
type PartitionTaskMetadata,
|
|
30
|
-
type ProjectionShape,
|
|
31
|
-
type StreamTaskMetadata,
|
|
14
|
+
TASK_OBJECT_KIND,
|
|
15
|
+
TaskObjectType,
|
|
16
|
+
decodeTaskObject,
|
|
17
|
+
type TaskObject,
|
|
32
18
|
} from './task.js';
|
|
33
|
-
import {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
19
|
+
import { RunnerType } from './runner.js';
|
|
20
|
+
import { TreePathType } from './structure.js';
|
|
21
|
+
|
|
22
|
+
describe('TaskObjectType', () => {
|
|
23
|
+
const sales = [variant('field', 'inputs'), variant('field', 'sales')];
|
|
24
|
+
const rates = [variant('field', 'inputs'), variant('field', 'rates')];
|
|
25
|
+
const totals = [variant('field', 'tasks'), variant('field', 'totals'), variant('field', 'output')];
|
|
26
|
+
const base: TaskObject = {
|
|
27
|
+
kind: TASK_OBJECT_KIND,
|
|
28
|
+
body: variant('east', { program: 'a'.repeat(64) }),
|
|
29
|
+
runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
|
|
30
|
+
inputs: [{ path: sales, partition: none }],
|
|
31
|
+
output: { path: totals, kind: variant('value', null) },
|
|
32
|
+
role: variant('data', null),
|
|
33
|
+
environment: none,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
it('round-trips every body, output kind and role', () => {
|
|
37
|
+
const tasks: TaskObject[] = [
|
|
38
|
+
base,
|
|
39
|
+
{
|
|
40
|
+
...base,
|
|
41
|
+
body: variant('command', { commandIr: 'b'.repeat(64) }),
|
|
42
|
+
runner: variant('custom', { command: [] }),
|
|
43
|
+
environment: some('c'.repeat(64)),
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
...base,
|
|
47
|
+
inputs: [{ path: sales, partition: some({ by: ['account', 'at.day'] }) }, { path: rates, partition: none }],
|
|
48
|
+
output: { path: totals, kind: variant('dict', { merge: some('d'.repeat(64)) }) },
|
|
49
|
+
},
|
|
50
|
+
{ ...base, output: { path: totals, kind: variant('fold', { zero: 'e'.repeat(64), combine: 'f'.repeat(64) }) } },
|
|
51
|
+
{ ...base, output: { path: totals, kind: variant('set', null) } },
|
|
52
|
+
{ ...base, role: variant('ui', { paths: [sales], functions: ['forecast'], records: ['plans'], pages: [rates] }) },
|
|
53
|
+
];
|
|
54
|
+
const encode = encodeBeast2For(TaskObjectType);
|
|
55
|
+
for (const original of tasks) assert.deepEqual(decodeTaskObject(encode(original)), original);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('refuses a task object an older SDK exported, saying to re-export the package', () => {
|
|
59
|
+
const PreCutoverTaskObjectType = StructType({
|
|
60
|
+
commandIr: StringType,
|
|
61
|
+
inputs: ArrayType(TreePathType),
|
|
62
|
+
output: TreePathType,
|
|
63
|
+
kind: OptionType(StringType),
|
|
64
|
+
metadata: OptionType(BlobType),
|
|
65
|
+
runner: RunnerType,
|
|
66
|
+
environment: OptionType(StringType),
|
|
59
67
|
});
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
68
|
+
const older = encodeBeast2For(PreCutoverTaskObjectType)({
|
|
69
|
+
commandIr: 'a'.repeat(64),
|
|
70
|
+
inputs: [[variant('field', 'tasks'), variant('field', 'totals'), variant('field', 'function_ir')], sales],
|
|
71
|
+
output: totals,
|
|
72
|
+
kind: none,
|
|
73
|
+
metadata: none,
|
|
74
|
+
runner: variant('east_node', { platforms: [] }),
|
|
75
|
+
environment: none,
|
|
63
76
|
});
|
|
77
|
+
assert.throws(() => decodeTaskObject(older), /exported by an older e3 SDK — re-export it with the current one/);
|
|
64
78
|
});
|
|
65
79
|
|
|
66
|
-
it('
|
|
67
|
-
|
|
68
|
-
const legacy = encodeBeast2For(shape)({ partitions: 1n, by: none, combine: none, targetPartitionBytes: 256n });
|
|
69
|
-
assert.deepEqual(decodePartitionTaskMetadata(legacy), {
|
|
70
|
-
partitions: 1n, by: none, combine: none, targetPartitionBytes: 256n, merge: none, mergeSets: false, mergeCommand: none,
|
|
71
|
-
});
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
it('throws for bytes of no known metadata shape', () => {
|
|
75
|
-
assert.throws(() => decodePartitionTaskMetadata(encodeBeast2For(IntegerType)(7n)));
|
|
76
|
-
});
|
|
77
|
-
});
|
|
78
|
-
|
|
79
|
-
describe('StreamTaskMetadataType', () => {
|
|
80
|
-
it('round-trips the merge mode', () => {
|
|
81
|
-
const meta: StreamTaskMetadata = { stream: true, emit: 'dict', merge: 'function' };
|
|
82
|
-
assert.deepEqual(decodeStreamTaskMetadata(encodeStreamTaskMetadata(meta)), meta);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
it('decodes metadata exported before merge modes as merge none', () => {
|
|
86
|
-
const legacy = encodeBeast2For(StructType({ stream: BooleanType, emit: StringType }))({ stream: false, emit: 'set' });
|
|
87
|
-
assert.deepEqual(decodeStreamTaskMetadata(legacy), { stream: false, emit: 'set', merge: 'none' });
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
it('throws for bytes of no known metadata shape', () => {
|
|
91
|
-
assert.throws(() => decodeStreamTaskMetadata(encodeBeast2For(IntegerType)(7n)));
|
|
92
|
-
});
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
describe('PartitionPlanType', () => {
|
|
96
|
-
it('round-trips a plan, before and after its slices are carved and its merge ranges planned', () => {
|
|
97
|
-
const plan: PartitionPlan = {
|
|
98
|
-
partitions: ['a'.repeat(64), 'b'.repeat(64)],
|
|
99
|
-
boundaries: [0n, 4n, 9n],
|
|
100
|
-
splits: [[{ seg: 0n, offset: 0n }, { seg: 3n, offset: 17n }, { seg: 8n, offset: 0n }, { seg: 12n, offset: 0n }]],
|
|
101
|
-
slices: [],
|
|
102
|
-
merges: [],
|
|
103
|
-
};
|
|
104
|
-
assert.deepEqual(decodePartitionPlan(encodePartitionPlan(plan)), plan);
|
|
105
|
-
|
|
106
|
-
const carved: PartitionPlan = { ...plan, slices: [['c'.repeat(64), 'd'.repeat(64), 'e'.repeat(64)], ['f'.repeat(64), '0'.repeat(64), '1'.repeat(64)]] };
|
|
107
|
-
assert.deepEqual(decodePartitionPlan(encodePartitionPlan(carved)), carved);
|
|
108
|
-
|
|
109
|
-
const merged: PartitionPlan = {
|
|
110
|
-
...carved,
|
|
111
|
-
merges: [{ partials: ['2'.repeat(64), '3'.repeat(64)], ranges: ['4'.repeat(64), '5'.repeat(64)] }],
|
|
112
|
-
};
|
|
113
|
-
assert.deepEqual(decodePartitionPlan(encodePartitionPlan(merged)), merged);
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
it('decodes a plan recorded before merge ranges existed, with none', () => {
|
|
117
|
-
// The pre-`merges` wire shape, encoded directly.
|
|
118
|
-
const PreMergesPlanType = StructType({
|
|
119
|
-
partitions: ArrayType(StringType),
|
|
120
|
-
boundaries: ArrayType(IntegerType),
|
|
121
|
-
splits: ArrayType(ArrayType(StructType({ seg: IntegerType, offset: IntegerType }))),
|
|
122
|
-
slices: ArrayType(ArrayType(StringType)),
|
|
123
|
-
});
|
|
124
|
-
const older = encodeBeast2For(PreMergesPlanType)({
|
|
125
|
-
partitions: ['a'.repeat(64)],
|
|
126
|
-
boundaries: [0n, 2n],
|
|
127
|
-
splits: [],
|
|
128
|
-
slices: [['b'.repeat(64), 'c'.repeat(64)]],
|
|
129
|
-
});
|
|
130
|
-
assert.deepEqual(decodePartitionPlan(older), {
|
|
131
|
-
partitions: ['a'.repeat(64)],
|
|
132
|
-
boundaries: [0n, 2n],
|
|
133
|
-
splits: [],
|
|
134
|
-
slices: [['b'.repeat(64), 'c'.repeat(64)]],
|
|
135
|
-
merges: [],
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
describe('partition projections', () => {
|
|
141
|
-
const AtType = StructType({ day: IntegerType, hour: IntegerType });
|
|
142
|
-
const KeyType = StructType({ region: StringType, at: AtType, sku: StringType });
|
|
143
|
-
const keyTypeValue = toEastTypeValue(KeyType);
|
|
144
|
-
type Key = { region: string; at: { day: bigint; hour: bigint }; sku: string };
|
|
145
|
-
const keys: Key[] = [
|
|
146
|
-
{ region: 'eu', at: { day: 2n, hour: 7n }, sku: 'b' },
|
|
147
|
-
{ region: 'eu', at: { day: 2n, hour: 7n }, sku: 'a' },
|
|
148
|
-
{ region: 'eu', at: { day: 1n, hour: 9n }, sku: 'c' },
|
|
149
|
-
{ region: 'ap', at: { day: 3n, hour: 0n }, sku: 'a' },
|
|
150
|
-
];
|
|
151
|
-
|
|
152
|
-
/** A `by` projection over the key, compiled, and its shape. */
|
|
153
|
-
function projection(by: ($: any, key: any) => any): { compiled: (key: Key) => unknown; shape: ProjectionShape | null } {
|
|
154
|
-
const bundle = East.function([KeyType], undefined, by).toIR();
|
|
155
|
-
return {
|
|
156
|
-
compiled: bundle.compile([]) as (key: Key) => unknown,
|
|
157
|
-
shape: partitionProjectionShape(bundle.ir as FunctionIR),
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/** Whether two comparisons agree in sign. */
|
|
162
|
-
const sameSign = (a: number, b: number): boolean => Math.sign(a) === Math.sign(b);
|
|
163
|
-
|
|
164
|
-
it('reads the identity, a leading field, a struct literal of leading fields and a nested first-field path', () => {
|
|
165
|
-
assert.deepEqual(projection(($, key) => key).shape, { kind: 'fields', names: [] });
|
|
166
|
-
assert.deepEqual(projection(($, key) => key.region).shape, { kind: 'fields', names: ['region'] });
|
|
167
|
-
assert.deepEqual(projection(($, key) => ({ region: key.region, at: key.at })).shape, { kind: 'fields', names: ['region', 'at'] });
|
|
168
|
-
assert.deepEqual(projection(($, key) => key.at.day).shape, { kind: 'path', names: ['at', 'day'] });
|
|
169
|
-
});
|
|
170
|
-
|
|
171
|
-
it('returns null for a projection that computes', () => {
|
|
172
|
-
assert.equal(projection(($, key) => key.region.concat('-')).shape, null);
|
|
173
|
-
});
|
|
174
|
-
|
|
175
|
-
it('projects the identity to the key itself', () => {
|
|
176
|
-
const shape: ProjectionShape = { kind: 'fields', names: [] };
|
|
177
|
-
assert.ok(isTypeValueEqual(projectedKeyType(shape, keyTypeValue), keyTypeValue));
|
|
178
|
-
assert.equal(projectKey(shape, keys[0]), keys[0]);
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
it('projects a struct literal of leading fields exactly as the compiled projection does', () => {
|
|
182
|
-
const { compiled, shape } = projection(($, key) => ({ region: key.region, at: key.at }));
|
|
183
|
-
assert.ok(isTypeValueEqual(projectedKeyType(shape!, keyTypeValue), toEastTypeValue(StructType({ region: StringType, at: AtType }))));
|
|
184
|
-
for (const key of keys) assert.deepEqual(projectKey(shape!, key), compiled(key));
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
it('projects a nested first-field path exactly as the compiled projection does', () => {
|
|
188
|
-
const { compiled, shape } = projection(($, key) => key.at.day);
|
|
189
|
-
assert.ok(isTypeValueEqual(projectedKeyType(shape!, keyTypeValue), toEastTypeValue(IntegerType)));
|
|
190
|
-
for (const key of keys) assert.equal(projectKey(shape!, key), compiled(key));
|
|
191
|
-
});
|
|
192
|
-
|
|
193
|
-
it('orders a single leading field as the compiled projection does', () => {
|
|
194
|
-
const { compiled, shape } = projection(($, key) => key.region);
|
|
195
|
-
const cmp = compareFor(projectedKeyType(shape!, keyTypeValue) as any) as (a: unknown, b: unknown) => number;
|
|
196
|
-
const cmpCompiled = compareFor(StringType);
|
|
197
|
-
for (const a of keys) {
|
|
198
|
-
for (const b of keys) {
|
|
199
|
-
assert.ok(sameSign(cmp(projectKey(shape!, a), projectKey(shape!, b)), cmpCompiled(compiled(a) as string, compiled(b) as string)));
|
|
200
|
-
}
|
|
201
|
-
}
|
|
80
|
+
it('refuses an object of the task shape that carries another kind tag', () => {
|
|
81
|
+
assert.throws(() => decodeTaskObject(encodeBeast2For(TaskObjectType)({ ...base, kind: '$plan' })), /its kind is '\$plan', not '\$task'/);
|
|
202
82
|
});
|
|
203
83
|
|
|
204
|
-
it('throws
|
|
205
|
-
assert.throws(() =>
|
|
206
|
-
assert.throws(() => projectedKeyType({ kind: 'path', names: ['at', 'minute'] }, keyTypeValue), /field 'minute'/);
|
|
84
|
+
it('throws for bytes of no known task shape', () => {
|
|
85
|
+
assert.throws(() => decodeTaskObject(encodeBeast2For(IntegerType)(7n)), /re-export/);
|
|
207
86
|
});
|
|
208
87
|
});
|