@elaraai/e3-types 1.0.76 → 1.0.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/src/api.d.ts +25 -0
  2. package/dist/src/api.d.ts.map +1 -1
  3. package/dist/src/api.js +18 -0
  4. package/dist/src/api.js.map +1 -1
  5. package/dist/src/dataflow.d.ts +10 -6
  6. package/dist/src/dataflow.d.ts.map +1 -1
  7. package/dist/src/dataflow.js.map +1 -1
  8. package/dist/src/dataset-type.d.ts +71 -0
  9. package/dist/src/dataset-type.d.ts.map +1 -0
  10. package/dist/src/dataset-type.js +101 -0
  11. package/dist/src/dataset-type.js.map +1 -0
  12. package/dist/src/execution.d.ts +17 -0
  13. package/dist/src/execution.d.ts.map +1 -1
  14. package/dist/src/index.d.ts +8 -6
  15. package/dist/src/index.d.ts.map +1 -1
  16. package/dist/src/index.js +10 -5
  17. package/dist/src/index.js.map +1 -1
  18. package/dist/src/package.d.ts +33 -3
  19. package/dist/src/package.d.ts.map +1 -1
  20. package/dist/src/package.js +47 -8
  21. package/dist/src/package.js.map +1 -1
  22. package/dist/src/runner.d.ts +35 -9
  23. package/dist/src/runner.d.ts.map +1 -1
  24. package/dist/src/runner.js +46 -13
  25. package/dist/src/runner.js.map +1 -1
  26. package/dist/src/runner.spec.js +50 -1
  27. package/dist/src/runner.spec.js.map +1 -1
  28. package/dist/src/stream.d.ts +97 -0
  29. package/dist/src/stream.d.ts.map +1 -0
  30. package/dist/src/stream.js +134 -0
  31. package/dist/src/stream.js.map +1 -0
  32. package/dist/src/stream.spec.d.ts +6 -0
  33. package/dist/src/stream.spec.d.ts.map +1 -0
  34. package/dist/src/stream.spec.js +60 -0
  35. package/dist/src/stream.spec.js.map +1 -0
  36. package/dist/src/task.d.ts +198 -10
  37. package/dist/src/task.d.ts.map +1 -1
  38. package/dist/src/task.js +340 -11
  39. package/dist/src/task.js.map +1 -1
  40. package/dist/src/task.spec.d.ts +6 -0
  41. package/dist/src/task.spec.d.ts.map +1 -0
  42. package/dist/src/task.spec.js +169 -0
  43. package/dist/src/task.spec.js.map +1 -0
  44. package/dist/src/transfer.d.ts +70 -3
  45. package/dist/src/transfer.d.ts.map +1 -1
  46. package/dist/src/transfer.js +76 -3
  47. package/dist/src/transfer.js.map +1 -1
  48. package/dist/src/transfer.spec.d.ts +6 -0
  49. package/dist/src/transfer.spec.d.ts.map +1 -0
  50. package/dist/src/transfer.spec.js +75 -0
  51. package/dist/src/transfer.spec.js.map +1 -0
  52. package/package.json +2 -2
  53. package/src/api.ts +18 -0
  54. package/src/dataflow.ts +10 -6
  55. package/src/dataset-type.ts +134 -0
  56. package/src/execution.ts +18 -0
  57. package/src/index.ts +37 -0
  58. package/src/package.ts +50 -8
  59. package/src/runner.spec.ts +56 -1
  60. package/src/runner.ts +45 -13
  61. package/src/stream.spec.ts +88 -0
  62. package/src/stream.ts +170 -0
  63. package/src/task.spec.ts +208 -0
  64. package/src/task.ts +374 -13
  65. package/src/transfer.spec.ts +98 -0
  66. package/src/transfer.ts +84 -3
package/src/index.ts CHANGED
@@ -46,6 +46,7 @@ export {
46
46
  decodeTaskObject,
47
47
  TASK_KIND_PARTITION,
48
48
  TASK_KIND_STREAM,
49
+ TASK_KIND_MERGE,
49
50
  PartitionTaskMetadataType,
50
51
  type PartitionTaskMetadata,
51
52
  encodePartitionTaskMetadata,
@@ -54,8 +55,26 @@ export {
54
55
  type StreamTaskMetadata,
55
56
  encodeStreamTaskMetadata,
56
57
  decodeStreamTaskMetadata,
58
+ PartitionPlanType,
59
+ MergeRangePlanType,
60
+ type PartitionPlan,
61
+ type MergeRangePlan,
62
+ encodePartitionPlan,
63
+ decodePartitionPlan,
64
+ type ProjectionShape,
65
+ partitionProjectionShape,
66
+ projectedKeyType,
67
+ projectKey,
57
68
  } from './task.js';
58
69
 
70
+ // Stream and merge commands
71
+ export {
72
+ type StreamMergeMode,
73
+ type StreamCommandSpec,
74
+ streamCommandIr,
75
+ mergeCommandIr,
76
+ } from './stream.js';
77
+
59
78
  // Execution environments
60
79
  export {
61
80
  EnvironmentSpecType,
@@ -101,6 +120,7 @@ export {
101
120
  type RunnerValue,
102
121
  runnerToArgv,
103
122
  withRunnerVerbose,
123
+ withRunnerLifeline,
104
124
  } from './runner.js';
105
125
 
106
126
  // Function objects
@@ -126,6 +146,8 @@ export {
126
146
  type PackageData,
127
147
  PackageObjectType,
128
148
  type PackageObject,
149
+ DatasetSourceWireType,
150
+ type DatasetSourceWire,
129
151
  decodePackageObject,
130
152
  // Backwards compatibility
131
153
  PackageDatasetsType,
@@ -161,6 +183,7 @@ export {
161
183
  export {
162
184
  ExecutionStatusType,
163
185
  type ExecutionStatus,
186
+ type ExecutionOwner,
164
187
  } from './execution.js';
165
188
 
166
189
  // Lock state
@@ -175,10 +198,15 @@ export {
175
198
 
176
199
  // Dataset transfer types
177
200
  export {
201
+ TRANSFER_PROTOCOL_VERSION,
202
+ transferPartCount,
203
+ transferPartRange,
178
204
  TransferUploadRequestType,
179
205
  type TransferUploadRequest,
180
206
  TransferUploadResponseType,
181
207
  type TransferUploadResponse,
208
+ TransferPartResponseType,
209
+ type TransferPartResponse,
182
210
  TransferDoneResponseType,
183
211
  type TransferDoneResponse,
184
212
  } from './transfer.js';
@@ -190,6 +218,14 @@ export {
190
218
  encodeDatasetBlob,
191
219
  } from './dataset-blob.js';
192
220
 
221
+ // The ONE declared-type-vs-wire-type check, shared by every door a value
222
+ // enters a dataset through (the export, the set, the adopt, the API)
223
+ export {
224
+ checkDatasetType,
225
+ datasetAddress,
226
+ type DatasetTypeMismatch,
227
+ } from './dataset-type.js';
228
+
193
229
  // Wire format constants
194
230
  export { BEAST2_CONTENT_TYPE } from './constants.js';
195
231
 
@@ -212,6 +248,7 @@ export {
212
248
  PermissionDeniedErrorType,
213
249
  InternalErrorType,
214
250
  RepositoryNotFoundErrorType,
251
+ DatasetTypeMismatchErrorType,
215
252
  ErrorType,
216
253
  ResponseType,
217
254
  // Repository
package/src/package.ts CHANGED
@@ -72,6 +72,27 @@ export type PackageDatasets = PackageData;
72
72
  * };
73
73
  * ```
74
74
  */
75
+ /**
76
+ * Where a dataset's initial value comes from, when it is not in the package.
77
+ *
78
+ * @remarks
79
+ * A package carries the DESCRIPTOR, never the bytes: `e3.input('table', T,
80
+ * variant('file', './TABLE.beast2'))` puts `{ file: { path } }` here and leaves
81
+ * the ref `unassigned`, and deploy resolves it — adopting the file into the
82
+ * object store by hash, so a new delivery under the same path is a new hash
83
+ * whose consumers re-run. An inline `variant('value', v)` needs no descriptor:
84
+ * its bytes are an object in the bundle like any other.
85
+ *
86
+ * A variant so that further source kinds can be appended later without a wire
87
+ * break; `file` is the only one today.
88
+ */
89
+ export const DatasetSourceWireType = VariantType({
90
+ /** A path on the machine that DEPLOYS the package, resolved at deploy. */
91
+ file: StructType({ path: StringType }),
92
+ });
93
+ export type DatasetSourceWireType = typeof DatasetSourceWireType;
94
+ export type DatasetSourceWire = ValueTypeOf<typeof DatasetSourceWireType>;
95
+
75
96
  export const PackageObjectType = StructType({
76
97
  /** Tasks defined in this package: name -> task object hash */
77
98
  tasks: DictType(StringType, StringType),
@@ -83,11 +104,26 @@ export const PackageObjectType = StructType({
83
104
  * BEAST2 encodes struct fields positionally in declaration order, so new
84
105
  * fields MUST be appended LAST — never inserted between existing fields. */
85
106
  records: DictType(StringType, StringType),
107
+ /** Unresolved dataset sources: refPath (e.g. "inputs/table") -> descriptor.
108
+ * Appended LAST, per the positional rule above. */
109
+ sources: DictType(StringType, DatasetSourceWireType),
86
110
  });
87
111
  export type PackageObjectType = typeof PackageObjectType;
88
112
 
89
113
  export type PackageObject = ValueTypeOf<typeof PackageObjectType>;
90
114
 
115
+ /**
116
+ * The pre-`sources` package object wire shape (tasks, data, functions,
117
+ * records), kept only so {@link decodePackageObject} can read packages exported
118
+ * before path-initialised inputs existed.
119
+ */
120
+ const RecordsEraPackageObjectType = StructType({
121
+ tasks: DictType(StringType, StringType),
122
+ data: PackageDataType,
123
+ functions: DictType(StringType, StringType),
124
+ records: DictType(StringType, StringType),
125
+ });
126
+
91
127
  /**
92
128
  * The pre-`records` package object wire shape (tasks, data, functions), kept
93
129
  * only so {@link decodePackageObject} can read packages exported before records
@@ -110,31 +146,37 @@ const LegacyPackageObjectType = StructType({
110
146
  });
111
147
 
112
148
  const decodeCurrent = decodeBeast2For(PackageObjectType);
149
+ const decodeRecordsEra = decodeBeast2For(RecordsEraPackageObjectType);
113
150
  const decodeFunctionsEra = decodeBeast2For(FunctionsEraPackageObjectType);
114
151
  const decodeLegacy = decodeBeast2For(LegacyPackageObjectType);
115
152
 
116
153
  /**
117
- * Decode a `PackageObject` from BEAST2 bytes, tolerating the two older wire
154
+ * Decode a `PackageObject` from BEAST2 bytes, tolerating the older wire
118
155
  * formats (dual-decode migration).
119
156
  *
120
157
  * Every package-read path — local AND cloud — must use this instead of
121
158
  * `decodeBeast2For(PackageObjectType)` directly, so packages exported before
122
- * the `records`/`functions` fields existed keep decoding. Older bytes decode
123
- * with the missing maps defaulted to empty.
159
+ * the `sources`/`records`/`functions` fields existed keep decoding. Older
160
+ * bytes decode with the missing maps defaulted to empty.
124
161
  */
125
162
  export function decodePackageObject(data: Uint8Array): PackageObject {
126
163
  try {
127
164
  return decodeCurrent(data);
128
165
  } catch (err) {
129
166
  try {
130
- const fnEra = decodeFunctionsEra(data);
131
- return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map() };
167
+ const recEra = decodeRecordsEra(data);
168
+ return { ...recEra, sources: new Map() };
132
169
  } catch {
133
170
  try {
134
- const legacy = decodeLegacy(data);
135
- return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map() };
171
+ const fnEra = decodeFunctionsEra(data);
172
+ return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map(), sources: new Map() };
136
173
  } catch {
137
- throw err; // no known shape — surface the current-format error
174
+ try {
175
+ const legacy = decodeLegacy(data);
176
+ return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map(), sources: new Map() };
177
+ } catch {
178
+ throw err; // no known shape — surface the current-format error
179
+ }
138
180
  }
139
181
  }
140
182
  }
@@ -17,7 +17,7 @@
17
17
  import { describe, it } from 'node:test';
18
18
  import * as assert from 'node:assert';
19
19
  import { variant } from '@elaraai/east';
20
- import { runnerToArgv, withRunnerVerbose, type RunnerValue } from './runner.js';
20
+ import { runnerToArgv, withRunnerLifeline, withRunnerVerbose, type RunnerValue } from './runner.js';
21
21
 
22
22
  /** A realistic fully-built argv: runner prefix + the `-i`/`-o`/`<ir>` suffix. */
23
23
  function fullArgv(runner: RunnerValue): string[] {
@@ -73,4 +73,59 @@ describe('withRunnerVerbose', () => {
73
73
  assert.deepStrictEqual(withRunnerVerbose(runner, ['east-node'], true), ['east-node']);
74
74
  assert.deepStrictEqual(withRunnerVerbose(runner, [], true), []);
75
75
  });
76
+
77
+ it('inserts -v after the merge subcommand too', () => {
78
+ const runner = variant('east_c', { platforms: ['east-c-std'] });
79
+ const args = [...runnerToArgv(runner, 'merge'), '--union', '-i', 'p0.beast2', '-o', 'out.beast2'];
80
+ assert.deepStrictEqual(withRunnerVerbose(runner, args, true).slice(0, 3), ['east-c', 'merge', '-v']);
81
+ });
82
+ });
83
+
84
+ describe('runnerToArgv', () => {
85
+ for (const [tag, runner, bin] of KNOWN) {
86
+ it(`resolves the run and merge commands of ${tag}`, () => {
87
+ assert.deepStrictEqual(runnerToArgv(runner).slice(0, 2), [bin, 'run']);
88
+ assert.deepStrictEqual(runnerToArgv(runner, 'run'), runnerToArgv(runner));
89
+ const merge = runnerToArgv(runner, 'merge');
90
+ assert.deepStrictEqual(merge.slice(0, 2), [bin, 'merge']);
91
+ // The platform flags follow either command unchanged.
92
+ assert.deepStrictEqual(merge.slice(2), runnerToArgv(runner).slice(2));
93
+ });
94
+ }
95
+
96
+ it('resolves a custom runner to its own command, and has no merge command for it', () => {
97
+ const runner: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
98
+ assert.deepStrictEqual(runnerToArgv(runner), ['uv', 'run', 'east-py', 'run']);
99
+ assert.throws(() => runnerToArgv(runner, 'merge'), { message: 'a custom runner has no merge command' });
100
+ });
101
+ });
102
+
103
+ describe('withRunnerLifeline', () => {
104
+ for (const [tag, runner, bin] of KNOWN) {
105
+ it(`inserts --exit-with-parent after the subcommand for ${tag}, on run and merge`, () => {
106
+ const args = fullArgv(runner);
107
+ const out = withRunnerLifeline(runner, args);
108
+ assert.deepStrictEqual(out.slice(0, 3), [bin, 'run', '--exit-with-parent']);
109
+ assert.deepStrictEqual(out.slice(3), args.slice(2));
110
+ const merge = [...runnerToArgv(runner, 'merge'), '--union', '-i', 'p0.beast2', '-o', 'out.beast2'];
111
+ assert.deepStrictEqual(withRunnerLifeline(runner, merge).slice(0, 3), [bin, 'merge', '--exit-with-parent']);
112
+ // Composed with -v, both sit ahead of the flags, in either order.
113
+ assert.deepStrictEqual(withRunnerVerbose(runner, withRunnerLifeline(runner, args), true).slice(0, 4), [bin, 'run', '-v', '--exit-with-parent']);
114
+ });
115
+ }
116
+
117
+ it('never splices the flag into a custom runner, which keeps an ignored stdin', () => {
118
+ const runner: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
119
+ const args = fullArgv(runner);
120
+ assert.deepStrictEqual(withRunnerLifeline(runner, args), args);
121
+ });
122
+
123
+ it('is a no-op on a degenerate argv and does not mutate its input', () => {
124
+ const runner = variant('east_c', { platforms: [] });
125
+ assert.deepStrictEqual(withRunnerLifeline(runner, ['east-c']), ['east-c']);
126
+ const args = fullArgv(runner);
127
+ const before = [...args];
128
+ withRunnerLifeline(runner, args);
129
+ assert.deepStrictEqual(args, before);
130
+ });
76
131
  });
package/src/runner.ts CHANGED
@@ -45,19 +45,30 @@ function flags(platforms: string[]): string[] {
45
45
  }
46
46
 
47
47
  /**
48
- * Resolve a {@link RunnerType} value to the argv prefix (the wire-value
49
- * analogue of the SDK's `runnerToCommand`). Lives in e3-types so both
50
- * e3-core (local) and the cloud execution kernel import the one resolver.
48
+ * Resolve a {@link RunnerType} value to the argv prefix of one of its
49
+ * commands (the wire-value analogue of the SDK's `runnerToCommand`): `run`,
50
+ * the default — `[<bin>, 'run', -p…]` — or `merge`, the blob merge every
51
+ * stock runner ships — `[<bin>, 'merge', -p…]`, the fan-in of a partitioned
52
+ * task's keyed partials (issue #770). Lives in e3-types so both e3-core
53
+ * (local) and the cloud execution kernel import the one resolver.
51
54
  *
52
55
  * Variant tags use underscores (`east_node`) mapped to the binary name
53
- * (`east-node`) here.
56
+ * (`east-node`) here. A `custom` runner's command is its `run`; it has no
57
+ * merge command.
58
+ *
59
+ * @param r - the runner
60
+ * @param command - the runner command, `run` by default
61
+ * @returns the argv prefix
62
+ * @throws {Error} When `merge` is asked of a custom runner.
54
63
  */
55
- export function runnerToArgv(r: RunnerValue): string[] {
64
+ export function runnerToArgv(r: RunnerValue, command: 'run' | 'merge' = 'run'): string[] {
56
65
  switch (r.type) {
57
- case 'east_node': return ['east-node', 'run', ...flags(r.value.platforms)];
58
- case 'east_py': return ['east-py', 'run', ...flags(r.value.platforms)];
59
- case 'east_c': return ['east-c', 'run', ...flags(r.value.platforms)];
60
- case 'custom': return [...r.value.command];
66
+ case 'east_node': return ['east-node', command, ...flags(r.value.platforms)];
67
+ case 'east_py': return ['east-py', command, ...flags(r.value.platforms)];
68
+ case 'east_c': return ['east-c', command, ...flags(r.value.platforms)];
69
+ case 'custom':
70
+ if (command !== 'run') throw new Error(`a custom runner has no ${command} command`);
71
+ return [...r.value.command];
61
72
  }
62
73
  }
63
74
 
@@ -65,10 +76,10 @@ export function runnerToArgv(r: RunnerValue): string[] {
65
76
  * Insert the runner's `-v/--verbose` flag into a fully-built argv, for the
66
77
  * known runtimes only.
67
78
  *
68
- * All three known runners (`east-node`/`east-py`/`east-c run`) accept `-v`
69
- * among their options and print timing/perf detail to stderr; a `custom`
70
- * runner's argv is user-authored, so a flag is never spliced into it. A
71
- * known-runtime argv always starts `[<bin>, 'run', …]` (see
79
+ * All three known runners (`east-node`/`east-py`/`east-c`, `run` and `merge`)
80
+ * accept `-v` among their options and print timing/perf detail to stderr; a
81
+ * `custom` runner's argv is user-authored, so a flag is never spliced into
82
+ * it. A known-runtime argv always starts `[<bin>, <command>, …]` (see
72
83
  * {@link runnerToArgv} and the SDK's `runnerToCommand`), so `-v` goes at
73
84
  * index 2 — ahead of the `-p`/`-i`/`-o` flags and the trailing IR path.
74
85
  *
@@ -85,3 +96,24 @@ export function withRunnerVerbose(runner: RunnerValue, args: string[], verbose?:
85
96
  if (!verbose || runner.type === 'custom' || args.length < 2) return args;
86
97
  return [...args.slice(0, 2), '-v', ...args.slice(2)];
87
98
  }
99
+
100
+ /**
101
+ * Insert the runner's `--exit-with-parent` flag into a fully-built argv, for
102
+ * the known runtimes only — the stdin lifeline (issue #770).
103
+ *
104
+ * A stock runner given the flag and a stdin pipe its parent never writes to
105
+ * exits as soon as the pipe reaches end of file: when the parent dies. A
106
+ * `custom` runner's argv is user-authored, so the flag is never spliced into
107
+ * it, and it keeps an ignored stdin. Like {@link withRunnerVerbose} the flag
108
+ * goes at index 2, after `[<bin>, <command>]`, and is a pure runtime toggle
109
+ * applied just before spawn — never part of the task's `commandIr` or any
110
+ * hash.
111
+ *
112
+ * @param runner - the runner the argv was built for (gates the injection)
113
+ * @param args - the fully-built argv
114
+ * @returns the argv, with `--exit-with-parent` inserted for known runtimes
115
+ */
116
+ export function withRunnerLifeline(runner: RunnerValue, args: string[]): string[] {
117
+ if (runner.type === 'custom' || args.length < 2) return args;
118
+ return [...args.slice(0, 2), '--exit-with-parent', ...args.slice(2)];
119
+ }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
4
+ */
5
+
6
+ /**
7
+ * The stream and merge command builders (issue #770): the argv a stream
8
+ * execution spawns for each merge mode and streamed-input choice, and the
9
+ * argv a merge execution spawns over sorted partials within a key range.
10
+ */
11
+
12
+ import { describe, it } from 'node:test';
13
+ import assert from 'node:assert/strict';
14
+ import { variant } from '@elaraai/east';
15
+ import { mergeCommandIr, streamCommandIr, type StreamCommandSpec } from './stream.js';
16
+ import type { RunnerValue } from './runner.js';
17
+
18
+ const EAST_C: RunnerValue = variant('east_c', { platforms: ['east-c-std'] });
19
+
20
+ /** The argv a stream command evaluates to for the staged `inputs`. */
21
+ function argvOf(runner: RunnerValue, spec: StreamCommandSpec, inputs: string[]): string[] {
22
+ return streamCommandIr(runner, spec).compile([])(inputs, 'out.beast2');
23
+ }
24
+
25
+ /** The argv a merge command evaluates to for the staged `inputs`. */
26
+ function mergeArgvOf(runner: RunnerValue, mode: 'function' | 'union', inputs: string[]): string[] {
27
+ return mergeCommandIr(runner, mode).compile([])(inputs, 'out.beast2');
28
+ }
29
+
30
+ describe('streamCommandIr', () => {
31
+ it('builds a folding dict argv: the merge IR as --merge, the first input streamed', () => {
32
+ assert.deepEqual(
33
+ argvOf(EAST_C, { emit: 'dict', merge: 'function', stream: 'first' }, ['body.beast2', 'merge.beast2', 'rows.beast2', 'other.beast2']),
34
+ ['east-c', 'run', '-p', 'east-c-std', '--emit', 'dict', '--merge', 'merge.beast2', '--stream', '0',
35
+ '-i', 'rows.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2'],
36
+ );
37
+ });
38
+
39
+ it('builds a union set argv streaming the first input', () => {
40
+ assert.deepEqual(
41
+ argvOf(EAST_C, { emit: 'set', merge: 'union', stream: 'first' }, ['body.beast2', 'stream.beast2', 'other.beast2']),
42
+ ['east-c', 'run', '-p', 'east-c-std', '--emit', 'set', '--union', '--stream', '0',
43
+ '-i', 'stream.beast2', '-i', 'other.beast2', '-o', 'out.beast2', 'body.beast2'],
44
+ );
45
+ });
46
+
47
+ it('builds a producer argv with no merge flag and no streamed input', () => {
48
+ assert.deepEqual(
49
+ argvOf(EAST_C, { emit: 'array', merge: 'none', stream: 'none' }, ['body.beast2', 'config.beast2']),
50
+ ['east-c', 'run', '-p', 'east-c-std', '--emit', 'array', '-i', 'config.beast2', '-o', 'out.beast2', 'body.beast2'],
51
+ );
52
+ });
53
+
54
+ it('starts from the runner argv of every stock runtime', () => {
55
+ const node: RunnerValue = variant('east_node', { platforms: ['@elaraai/east-node-std'] });
56
+ const argv = argvOf(node, { emit: 'set', merge: 'none', stream: 'first' }, ['body.beast2', 'rows.beast2']);
57
+ assert.deepEqual(argv.slice(0, 6), ['east-node', 'run', '-p', '@elaraai/east-node-std', '--emit', 'set']);
58
+ });
59
+ });
60
+
61
+ describe('mergeCommandIr', () => {
62
+ it('builds a dict merge argv: the merge IR as --merge, the key range as --range, every other input a partial', () => {
63
+ assert.deepEqual(
64
+ mergeArgvOf(EAST_C, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2', 'p1.beast2', 'p2.beast2']),
65
+ ['east-c', 'merge', '-p', 'east-c-std', '--merge', 'merge.beast2', '--range', 'range.beast2',
66
+ '-i', 'p0.beast2', '-i', 'p1.beast2', '-i', 'p2.beast2', '-o', 'out.beast2'],
67
+ );
68
+ });
69
+
70
+ it('builds a set union argv: the key range first, every other input a partial', () => {
71
+ assert.deepEqual(
72
+ mergeArgvOf(EAST_C, 'union', ['range.beast2', 'p0.beast2', 'p1.beast2']),
73
+ ['east-c', 'merge', '-p', 'east-c-std', '--union', '--range', 'range.beast2', '-i', 'p0.beast2', '-i', 'p1.beast2', '-o', 'out.beast2'],
74
+ );
75
+ });
76
+
77
+ it('runs the merge command of every stock runtime', () => {
78
+ const py: RunnerValue = variant('east_py', { platforms: ['east-py-std'] });
79
+ assert.deepEqual(mergeArgvOf(py, 'union', ['range.beast2', 'p0.beast2']).slice(0, 4), ['east-py', 'merge', '-p', 'east-py-std']);
80
+ const node: RunnerValue = variant('east_node', { platforms: [] });
81
+ assert.deepEqual(mergeArgvOf(node, 'function', ['merge.beast2', 'range.beast2', 'p0.beast2']).slice(0, 2), ['east-node', 'merge']);
82
+ });
83
+
84
+ it('refuses a custom runner, which has no merge command', () => {
85
+ const custom: RunnerValue = variant('custom', { command: ['uv', 'run', 'east-py', 'run'] });
86
+ assert.throws(() => mergeCommandIr(custom, 'union'), { message: 'a custom runner has no merge command' });
87
+ });
88
+ });
package/src/stream.ts ADDED
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
4
+ */
5
+
6
+ /**
7
+ * The stream and merge commands (issue #770) — the builders of every argv a
8
+ * stream execution spawns (an authored `streamTask`'s) and of the merge
9
+ * command a partitioned task's keyed fan-in runs per key range, both built
10
+ * at export time and carried in the package like any other command IR.
11
+ */
12
+
13
+ import { ArrayType, East, EastIR, StringType } from '@elaraai/east';
14
+ import { runnerToArgv, type RunnerValue } from './runner.js';
15
+
16
+ /**
17
+ * How a stream execution's emit sink treats equal keys: `none` — a duplicate
18
+ * is an error; `function` — the merge IR (wire input 1) folds adjacent equal
19
+ * Dict keys, passed to the runner as `--merge`; `union` — adjacent equal Set
20
+ * elements collapse to the first, `--union`.
21
+ */
22
+ export type StreamMergeMode = 'none' | 'function' | 'union';
23
+
24
+ /**
25
+ * The shape of a stream command.
26
+ *
27
+ * @remarks
28
+ * The wire inputs a stream execution stages are laid out `[body IR, merge IR
29
+ * (function mode only), ...-i inputs]`; `stream` names whether the first of
30
+ * the `-i` inputs is the one the runner opens lazily.
31
+ */
32
+ export interface StreamCommandSpec {
33
+ /** The output collection kind the emit sink writes. */
34
+ emit: 'array' | 'set' | 'dict';
35
+ /** How equal keys fold — see {@link StreamMergeMode}. */
36
+ merge: StreamMergeMode;
37
+ /** The `-i` input the runner feeds lazily: none, or the first. */
38
+ stream: 'none' | 'first';
39
+ }
40
+
41
+ /**
42
+ * Builds the command IR of a stream execution: `(input_paths, output_path)
43
+ * -> argv`.
44
+ *
45
+ * The argv is `[...runner, '--emit', <emit>, '--merge' <merge IR path> |
46
+ * '--union', '--stream' '0' when the first -i input streams, '-i' <path> for
47
+ * each -i input, '-o', <output>, <body IR path>]` — the body IR is wire
48
+ * input 0, the merge IR wire input 1 in function mode, and the `-i` inputs
49
+ * follow. Built with `East.function` exactly as a task command is.
50
+ *
51
+ * @param runner - the stock runner the execution spawns
52
+ * @param spec - the emit kind, the merge mode and the streamed input
53
+ * @returns the command IR
54
+ *
55
+ * @example
56
+ * ```ts
57
+ * import { variant } from '@elaraai/east';
58
+ *
59
+ * const command = streamCommandIr(variant('east_c', { platforms: [] }), { emit: 'dict', merge: 'function', stream: 'first' });
60
+ * command.compile([])(['body.beast2', 'merge.beast2', 'a.beast2'], 'out.beast2');
61
+ * // ['east-c', 'run', '--emit', 'dict', '--merge', 'merge.beast2', '--stream', '0',
62
+ * // '-i', 'a.beast2', '-o', 'out.beast2', 'body.beast2']
63
+ * ```
64
+ */
65
+ export function streamCommandIr(runner: RunnerValue, spec: StreamCommandSpec): EastIR<[string[], string], string[]> {
66
+ const prefix = [
67
+ ...runnerToArgv(runner),
68
+ '--emit', spec.emit,
69
+ ...(spec.merge === 'union' ? ['--union'] : []),
70
+ ];
71
+ // The merge mode and the streamed input are known here, so the flags they
72
+ // add are emitted by branching at build time — only the merge IR's PATH is
73
+ // a run-time value, and only the -i inputs need a loop.
74
+ const mergeInput = spec.merge === 'function' ? 1n : null;
75
+ const firstInput = mergeInput === null ? 1n : 2n;
76
+ const command = East.function(
77
+ [ArrayType(StringType), StringType],
78
+ ArrayType(StringType),
79
+ ($, input_paths, output_path) => {
80
+ const argv = $.let(prefix, ArrayType(StringType));
81
+ if (mergeInput !== null) {
82
+ $(argv.pushLast('--merge'));
83
+ $(argv.pushLast(input_paths.get(mergeInput)));
84
+ }
85
+ if (spec.stream === 'first') {
86
+ $(argv.pushLast('--stream'));
87
+ $(argv.pushLast('0'));
88
+ }
89
+ const i = $.let(firstInput);
90
+ $.while(East.less(i, input_paths.size()), $ => {
91
+ $(argv.pushLast('-i'));
92
+ $(argv.pushLast(input_paths.get(i)));
93
+ $.assign(i, i.add(1n));
94
+ });
95
+ $(argv.pushLast('-o'));
96
+ $(argv.pushLast(output_path));
97
+ $(argv.pushLast(input_paths.get(0n)));
98
+ $.return(argv);
99
+ },
100
+ );
101
+ return command.toIR();
102
+ }
103
+
104
+ /**
105
+ * Builds the command IR of a merge execution — the runner's `merge` command
106
+ * over sorted partials of one Set or Dict type, within a key range:
107
+ * `(input_paths, output_path) -> argv`.
108
+ *
109
+ * The argv is `[...runner merge, '--merge' <merge IR path> | '--union',
110
+ * '--range' <range path>, '-i' <path> for each partial, '-o', <output>]`. In
111
+ * `function` mode wire input 0 is the merge IR `(K, V, V) -> V`, wire input
112
+ * 1 the key range and the partials follow; in `union` mode wire input 0 is
113
+ * the key range and every other wire input is a partial. The key range is a
114
+ * blob of `Struct{from: Option<K>, to: Option<K>}` over the output's key
115
+ * type: only the keys in `[from, to)` merge, each partial sought to the
116
+ * segment owning `from`, and an absent bound is open — a component that
117
+ * merges whole takes the open range. Equal keys across partials fold in
118
+ * input order, and the output is byte-identical to what the runner's emit
119
+ * sink writes for the same entries emitted ascending. A partitioned task's
120
+ * SDK writes this IR into its metadata at export, and the orchestrator runs
121
+ * it as an ordinary execution per key range of a group of partials.
122
+ *
123
+ * @param runner - the stock runner whose `merge` command runs
124
+ * @param mode - `function` (a Dict output, folded with the merge IR) or
125
+ * `union` (a Set output, the first of equal elements standing)
126
+ * @returns the command IR
127
+ * @throws {Error} When the runner is `custom` — it has no merge command.
128
+ *
129
+ * @example
130
+ * ```ts
131
+ * import { variant } from '@elaraai/east';
132
+ *
133
+ * const command = mergeCommandIr(variant('east_c', { platforms: [] }), 'function');
134
+ * command.compile([])(['merge.beast2', 'range.beast2', 'p0.beast2', 'p1.beast2'], 'out.beast2');
135
+ * // ['east-c', 'merge', '--merge', 'merge.beast2', '--range', 'range.beast2',
136
+ * // '-i', 'p0.beast2', '-i', 'p1.beast2', '-o', 'out.beast2']
137
+ * ```
138
+ */
139
+ export function mergeCommandIr(runner: RunnerValue, mode: 'function' | 'union'): EastIR<[string[], string], string[]> {
140
+ const prefix = [
141
+ ...runnerToArgv(runner, 'merge'),
142
+ ...(mode === 'union' ? ['--union'] : []),
143
+ ];
144
+ const mergeInput = mode === 'function' ? 0n : null;
145
+ const rangeInput = mergeInput === null ? 0n : 1n;
146
+ const firstInput = rangeInput + 1n;
147
+ const command = East.function(
148
+ [ArrayType(StringType), StringType],
149
+ ArrayType(StringType),
150
+ ($, input_paths, output_path) => {
151
+ const argv = $.let(prefix, ArrayType(StringType));
152
+ if (mergeInput !== null) {
153
+ $(argv.pushLast('--merge'));
154
+ $(argv.pushLast(input_paths.get(mergeInput)));
155
+ }
156
+ $(argv.pushLast('--range'));
157
+ $(argv.pushLast(input_paths.get(rangeInput)));
158
+ const i = $.let(firstInput);
159
+ $.while(East.less(i, input_paths.size()), $ => {
160
+ $(argv.pushLast('-i'));
161
+ $(argv.pushLast(input_paths.get(i)));
162
+ $.assign(i, i.add(1n));
163
+ });
164
+ $(argv.pushLast('-o'));
165
+ $(argv.pushLast(output_path));
166
+ $.return(argv);
167
+ },
168
+ );
169
+ return command.toIR();
170
+ }