@elaraai/e3-types 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +829 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +301 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +17 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +28 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/intake.d.ts +75 -0
- package/dist/src/intake.d.ts.map +1 -0
- package/dist/src/intake.js +46 -0
- package/dist/src/intake.js.map +1 -0
- package/dist/src/lock.d.ts +253 -21
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +55 -21
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +38 -26
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +22 -26
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +326 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +141 -39
- package/src/intake.ts +53 -0
- package/src/lock.ts +67 -21
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +22 -26
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/dist/src/task.js
CHANGED
|
@@ -5,510 +5,155 @@
|
|
|
5
5
|
/**
|
|
6
6
|
* Task object types for e3.
|
|
7
7
|
*
|
|
8
|
-
* A task object
|
|
9
|
-
*
|
|
8
|
+
* A task object says what a task runs, on which runtime, over which datasets,
|
|
9
|
+
* and what its output is: every field a mode used to be implied by is typed.
|
|
10
|
+
* Task objects are stored in the object store and referenced by packages, so
|
|
11
|
+
* they are content-addressed, which is what memoizes their executions.
|
|
10
12
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* Input and output types are inferred from the package's structure at the
|
|
15
|
-
* specified paths - the task just references locations, not types.
|
|
13
|
+
* Input and output types are not stored here: they are the package structure's
|
|
14
|
+
* at the paths the task names.
|
|
16
15
|
*/
|
|
17
|
-
import { StructType, StringType, ArrayType,
|
|
16
|
+
import { StructType, StringType, ArrayType, NullType, OptionType, VariantType, decodeBeast2For } from '@elaraai/east';
|
|
18
17
|
import { TreePathType } from './structure.js';
|
|
19
18
|
import { RunnerType } from './runner.js';
|
|
19
|
+
/** The kind tag every task object carries, which the garbage collector
|
|
20
|
+
* dispatches on. */
|
|
21
|
+
export const TASK_OBJECT_KIND = '$task';
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* A task is a complete executable unit that reads from input dataset paths
|
|
24
|
-
* and writes to an output dataset path. The commandIr is evaluated at runtime
|
|
25
|
-
* to produce the exec args.
|
|
26
|
-
*
|
|
27
|
-
* @remarks
|
|
28
|
-
* - `commandIr`: Hash of East IR object that produces exec args
|
|
29
|
-
* - IR signature: (inputs: Array<String>, output: String) -> Array<String>
|
|
30
|
-
* - `inputs` are paths to staged input .beast2 files
|
|
31
|
-
* - `output` is the path where output should be written
|
|
32
|
-
* - Returns array of strings to exec (e.g., ["sh", "-c", "python ..."])
|
|
33
|
-
* - `inputs`: Paths to input datasets in the data tree
|
|
34
|
-
* - `output`: Path to the output dataset in the data tree
|
|
35
|
-
*
|
|
36
|
-
* Types are not stored in the task - they are inferred from the package's
|
|
37
|
-
* structure at the specified paths. This keeps tasks simple and avoids
|
|
38
|
-
* redundant type information.
|
|
23
|
+
* What a task runs.
|
|
39
24
|
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* const task: TaskObject = {
|
|
46
|
-
* commandIr: '5e7a3b...', // hash of compiled IR
|
|
47
|
-
* inputs: [
|
|
48
|
-
* [variant('field', 'inputs'), variant('field', 'sales')],
|
|
49
|
-
* ],
|
|
50
|
-
* output: [variant('field', 'tasks'), variant('field', 'train'), variant('field', 'output')],
|
|
51
|
-
* };
|
|
52
|
-
* ```
|
|
25
|
+
* - `east`: an East program, which a stock runner executes as a unit
|
|
26
|
+
* (`exec`): `program` is the hash of its IR bundle.
|
|
27
|
+
* - `command`: a command, which the custom runtime spawns. `commandIr` is the
|
|
28
|
+
* hash of the IR bundle of the East function that builds its argv,
|
|
29
|
+
* `(inputs: Array<String>, output: String) -> Array<String>`.
|
|
53
30
|
*/
|
|
54
|
-
export const
|
|
55
|
-
|
|
56
|
-
commandIr: StringType,
|
|
57
|
-
/** Input paths: where to read each input dataset from the data tree */
|
|
58
|
-
inputs: ArrayType(TreePathType),
|
|
59
|
-
/** Output path: where to write the output dataset in the data tree */
|
|
60
|
-
output: TreePathType,
|
|
61
|
-
/** Task kind: "data" (default), "ui", or future extensions. None for old packages. */
|
|
62
|
-
kind: OptionType(StringType),
|
|
63
|
-
/** Opaque extension metadata (beast2-encoded). Interpreted by the kind-specific consumer. */
|
|
64
|
-
metadata: OptionType(BlobType),
|
|
65
|
-
/**
|
|
66
|
-
* The task's runner, as routing metadata (symmetric with
|
|
67
|
-
* FunctionObject.runner). For `custom` runners, `commandIr` remains
|
|
68
|
-
* authoritative for execution — the wire command is informational.
|
|
69
|
-
*
|
|
70
|
-
* NOTE: added as a hard cutover (no dual decoder) — packages exported by
|
|
71
|
-
* older SDKs must be re-exported.
|
|
72
|
-
*/
|
|
73
|
-
runner: RunnerType,
|
|
74
|
-
/**
|
|
75
|
-
* Hash of an {@link EnvironmentSpecType} object the task executes in;
|
|
76
|
-
* `none` ⇒ the stock runtime image. Appended LAST (BEAST2 encodes struct
|
|
77
|
-
* fields positionally) with a legacy dual decoder — see
|
|
78
|
-
* {@link decodeTaskObject}.
|
|
79
|
-
*/
|
|
80
|
-
environment: OptionType(StringType),
|
|
31
|
+
export const TaskBodyType = VariantType({
|
|
32
|
+
east: StructType({ program: StringType }),
|
|
33
|
+
command: StructType({ commandIr: StringType }),
|
|
81
34
|
});
|
|
82
35
|
/**
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
36
|
+
* An input the work may be split over: `by` names the leading key fields
|
|
37
|
+
* whose equal values are never split across pieces — a field, or a
|
|
38
|
+
* first-field path written `a.b`. Empty when any key may start a piece.
|
|
86
39
|
*/
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
metadata: OptionType(BlobType),
|
|
93
|
-
runner: RunnerType,
|
|
40
|
+
export const TaskPartitionType = StructType({ by: ArrayType(StringType) });
|
|
41
|
+
/** A dataset a task reads, and whether the work may be split over it. */
|
|
42
|
+
export const TaskInputType = StructType({
|
|
43
|
+
path: TreePathType,
|
|
44
|
+
partition: OptionType(TaskPartitionType),
|
|
94
45
|
});
|
|
95
|
-
const decodeCurrentTask = decodeBeast2For(TaskObjectType);
|
|
96
|
-
const decodePreEnvironmentTask = decodeBeast2For(PreEnvironmentTaskObjectType);
|
|
97
46
|
/**
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
102
|
-
* `
|
|
103
|
-
* `
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
throw err; // no known shape — surface the current-format error
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
// =============================================================================
|
|
120
|
-
// Partition / stream task kinds
|
|
121
|
-
// =============================================================================
|
|
122
|
-
/** Task kind of a partitioned task — the orchestrator carves its partitioned
|
|
123
|
-
* input(s) into key-range slices, runs each slice as an ordinary
|
|
124
|
-
* content-addressed execution, and assembles the output by splice or by
|
|
125
|
-
* combining partials. The spec rides {@link TaskObjectType}'s `metadata`
|
|
126
|
-
* slot as a beast2-encoded {@link PartitionTaskMetadataType}. */
|
|
127
|
-
export const TASK_KIND_PARTITION = 'partition';
|
|
128
|
-
/** Task kind of a streaming task — one execution whose runner feeds the
|
|
129
|
-
* stream input lazily and writes the output incrementally through an `emit`
|
|
130
|
-
* capability. The spec rides {@link TaskObjectType}'s `metadata` slot as a
|
|
131
|
-
* beast2-encoded {@link StreamTaskMetadataType}. */
|
|
132
|
-
export const TASK_KIND_STREAM = 'stream';
|
|
133
|
-
/** Task kind of a merge unit — the runner's `merge` command over sorted
|
|
134
|
-
* partials of a partitioned task's keyed output (issue #770). The
|
|
135
|
-
* orchestrator writes one such task per partitioned task from the
|
|
136
|
-
* package's {@link PartitionTaskMetadataType} `mergeCommand`, and runs it
|
|
137
|
-
* as an ordinary content-addressed execution per group of partials. It
|
|
138
|
-
* carries no metadata. */
|
|
139
|
-
export const TASK_KIND_MERGE = 'merge';
|
|
140
|
-
/**
|
|
141
|
-
* Metadata of a {@link TASK_KIND_PARTITION} task.
|
|
142
|
-
*
|
|
143
|
-
* The task's wire `inputs` are laid out `[function_ir, ...partitions,
|
|
144
|
-
* ...inputs]`, so `partitions` counts how many entries after the function IR
|
|
145
|
-
* are partitioned datasets; the rest are ordinary (broadcast) inputs that
|
|
146
|
-
* hash into every partition execution's identity.
|
|
147
|
-
*
|
|
148
|
-
* `by` and `combine` are carried as `encodeEastIR` bundles (capture-free IR +
|
|
149
|
-
* its source map), not as FunctionType values or object-store hashes: IR is
|
|
150
|
-
* how executable code travels everywhere on the e3 wire (`commandIr`,
|
|
151
|
-
* `function_ir`, `bodyIr`), a FunctionType value could smuggle captures the
|
|
152
|
-
* orchestrator must not evaluate, and inline bytes stay reachable where a
|
|
153
|
-
* hash inside an opaque metadata blob would be invisible to GC.
|
|
154
|
-
*/
|
|
155
|
-
export const PartitionTaskMetadataType = StructType({
|
|
156
|
-
/** Number of partitioned inputs (wire input indices `1..1+partitions`). */
|
|
157
|
-
partitions: IntegerType,
|
|
158
|
-
/** `encodeEastIR` bundle of the boundary-alignment projection
|
|
159
|
-
* `(Key) -> Projection`; `none` when partitioning is free per row/segment. */
|
|
160
|
-
by: OptionType(BlobType),
|
|
161
|
-
/** `encodeEastIR` bundle of the associative fold `(Out, Out) -> Out`;
|
|
162
|
-
* `none` in splice mode (shards concatenate). */
|
|
163
|
-
combine: OptionType(BlobType),
|
|
164
|
-
/** Target carved-slice size in wire bytes — the only sizing knob. */
|
|
165
|
-
targetPartitionBytes: IntegerType,
|
|
166
|
-
/**
|
|
167
|
-
* `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
|
|
168
|
-
* for a Dict output; `none` otherwise.
|
|
169
|
-
*
|
|
170
|
-
* Its presence (or {@link mergeSets}) selects the MERGE-TREE assembly:
|
|
171
|
-
* partials whose key ranges overlap are merged by the task's own runner, in
|
|
172
|
-
* a tree of merge executions ({@link mergeCommand}) folding equal keys with
|
|
173
|
-
* this function; disjoint partials are spliced; the orchestrator never
|
|
174
|
-
* decodes a partial. Appended after `targetPartitionBytes` (BEAST2 encodes
|
|
175
|
-
* struct fields positionally) with a dual decoder — see
|
|
176
|
-
* {@link decodePartitionTaskMetadata}.
|
|
177
|
-
*/
|
|
178
|
-
merge: OptionType(BlobType),
|
|
179
|
-
/** Whether a Set output assembles by the merge tree, keeping one of equal
|
|
180
|
-
* elements (`--union`). The Set twin of {@link merge}, which needs no
|
|
181
|
-
* function. */
|
|
182
|
-
mergeSets: BooleanType,
|
|
183
|
-
/**
|
|
184
|
-
* `encodeEastIR` bundle of the merge command `(input_paths, output_path) ->
|
|
185
|
-
* argv` — `mergeCommandIr` over the task's runner, in `function` mode with
|
|
186
|
-
* {@link merge} and `union` mode with {@link mergeSets} — that the
|
|
187
|
-
* orchestrator's merge units execute; `none` in splice and combine modes.
|
|
188
|
-
* Built at export, so the merge unit task is a package object like any
|
|
189
|
-
* other and nothing is synthesized at run time. Appended LAST with a
|
|
190
|
-
* triple decoder — see {@link decodePartitionTaskMetadata}.
|
|
191
|
-
*/
|
|
192
|
-
mergeCommand: OptionType(BlobType),
|
|
47
|
+
* How a task's output is made, which fixes how the parts of it that units
|
|
48
|
+
* write combine.
|
|
49
|
+
*
|
|
50
|
+
* - `value`: the program returns it.
|
|
51
|
+
* - `array`: emitted elements, concatenated in emission order.
|
|
52
|
+
* - `set`: emitted elements, in any order, unioned.
|
|
53
|
+
* - `dict`: emitted entries, in any order; equal keys fold with `merge`, the
|
|
54
|
+
* hash of a `(K, V, V) -> V` IR bundle, and without it are refused.
|
|
55
|
+
* - `fold`: emitted values folded with `combine`, the hash of a `(T, T) -> T`
|
|
56
|
+
* IR bundle, starting at `zero`, the hash of a stored value.
|
|
57
|
+
*/
|
|
58
|
+
export const TaskOutputKindType = VariantType({
|
|
59
|
+
value: NullType,
|
|
60
|
+
array: NullType,
|
|
61
|
+
set: NullType,
|
|
62
|
+
dict: StructType({ merge: OptionType(StringType) }),
|
|
63
|
+
fold: StructType({ zero: StringType, combine: StringType }),
|
|
193
64
|
});
|
|
194
|
-
/**
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
*/
|
|
199
|
-
const PreMergePartitionTaskMetadataType = StructType({
|
|
200
|
-
partitions: IntegerType,
|
|
201
|
-
by: OptionType(BlobType),
|
|
202
|
-
combine: OptionType(BlobType),
|
|
203
|
-
targetPartitionBytes: IntegerType,
|
|
204
|
-
});
|
|
205
|
-
/**
|
|
206
|
-
* The v1.0.77 partition metadata wire shape — `merge` and `mergeSets` but no
|
|
207
|
-
* `mergeCommand` — kept only so {@link decodePartitionTaskMetadata} can read
|
|
208
|
-
* tasks exported by that SDK; their merge units cannot run.
|
|
209
|
-
*/
|
|
210
|
-
const PreMergeCommandPartitionTaskMetadataType = StructType({
|
|
211
|
-
partitions: IntegerType,
|
|
212
|
-
by: OptionType(BlobType),
|
|
213
|
-
combine: OptionType(BlobType),
|
|
214
|
-
targetPartitionBytes: IntegerType,
|
|
215
|
-
merge: OptionType(BlobType),
|
|
216
|
-
mergeSets: BooleanType,
|
|
65
|
+
/** Where a task's output goes, and how it is made. */
|
|
66
|
+
export const TaskOutputType = StructType({
|
|
67
|
+
path: TreePathType,
|
|
68
|
+
kind: TaskOutputKindType,
|
|
217
69
|
});
|
|
218
|
-
/** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
|
|
219
|
-
export const encodePartitionTaskMetadata = encodeBeast2For(PartitionTaskMetadataType);
|
|
220
|
-
const decodeCurrentPartitionMetadata = decodeBeast2For(PartitionTaskMetadataType);
|
|
221
|
-
const decodePreMergeCommandPartitionMetadata = decodeBeast2For(PreMergeCommandPartitionTaskMetadataType);
|
|
222
|
-
const decodePreMergePartitionMetadata = decodeBeast2For(PreMergePartitionTaskMetadataType);
|
|
223
70
|
/**
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
* @
|
|
230
|
-
*
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
catch {
|
|
241
|
-
try {
|
|
242
|
-
return { ...decodePreMergePartitionMetadata(data), merge: none, mergeSets: false, mergeCommand: none };
|
|
243
|
-
}
|
|
244
|
-
catch {
|
|
245
|
-
throw err; // no known shape — surface the current-format error
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
/**
|
|
251
|
-
* Metadata of a {@link TASK_KIND_STREAM} task.
|
|
252
|
-
*
|
|
253
|
-
* The task's wire `inputs` are laid out `[function_ir, merge_ir?, stream?,
|
|
254
|
-
* ...inputs]` — `merge_ir` only in `function` merge mode; the compiled body
|
|
255
|
-
* takes one trailing `emit` parameter beyond the wire inputs, and the runner
|
|
256
|
-
* writes the `-o` file from the emit sink instead of the body's (Null) return
|
|
257
|
-
* value.
|
|
258
|
-
*/
|
|
259
|
-
export const StreamTaskMetadataType = StructType({
|
|
260
|
-
/** Whether the first input after the IRs is the streamed input (producer
|
|
261
|
-
* tasks have no streamed input). */
|
|
262
|
-
stream: BooleanType,
|
|
263
|
-
/** The output collection kind the emit sink writes: `"array"`, `"set"`, or
|
|
264
|
-
* `"dict"` — element/key/value types come from the body IR's emit
|
|
265
|
-
* parameter. */
|
|
266
|
-
emit: StringType,
|
|
267
|
-
/**
|
|
268
|
-
* How the emit sink treats equal keys, a `StreamMergeMode`: `"none"` —
|
|
269
|
-
* a duplicate key is an error; `"function"` — wire input 1 is the merge IR
|
|
270
|
-
* `(K, V, V) -> V`, folding equal Dict keys; `"union"` — equal Set elements
|
|
271
|
-
* collapse. Appended LAST (BEAST2 encodes struct fields positionally) with a
|
|
272
|
-
* dual decoder — see {@link decodeStreamTaskMetadata}.
|
|
273
|
-
*/
|
|
274
|
-
merge: StringType,
|
|
71
|
+
* What a ui task binds as it renders.
|
|
72
|
+
*
|
|
73
|
+
* @property paths - Dataset paths it reads or writes through `Data.bind`,
|
|
74
|
+
* including each bound record's own path, so the record's current value is
|
|
75
|
+
* preloaded and polled like any dataset.
|
|
76
|
+
* @property functions - Package functions it calls through `Func.bind`.
|
|
77
|
+
* @property records - Records it binds through `Record.bind`.
|
|
78
|
+
* @property pages - Dataset paths it reads a window at a time through
|
|
79
|
+
* `Data.bindPaged`: declared apart from `paths` because they are never
|
|
80
|
+
* preloaded or polled whole.
|
|
81
|
+
*/
|
|
82
|
+
export const DataManifestType = StructType({
|
|
83
|
+
paths: ArrayType(TreePathType),
|
|
84
|
+
functions: ArrayType(StringType),
|
|
85
|
+
records: ArrayType(StringType),
|
|
86
|
+
pages: ArrayType(TreePathType),
|
|
275
87
|
});
|
|
276
88
|
/**
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
* emit sinks existed.
|
|
89
|
+
* What a task's output is for: `data`, or a `ui` whose output is a component
|
|
90
|
+
* tree rendered against the datasets, functions and records it binds.
|
|
280
91
|
*/
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
92
|
+
export const TaskRoleType = VariantType({
|
|
93
|
+
data: NullType,
|
|
94
|
+
ui: DataManifestType,
|
|
284
95
|
});
|
|
285
|
-
/** Encode a {@link StreamTaskMetadataType} value for `TaskObject.metadata`. */
|
|
286
|
-
export const encodeStreamTaskMetadata = encodeBeast2For(StreamTaskMetadataType);
|
|
287
|
-
const decodeCurrentStreamMetadata = decodeBeast2For(StreamTaskMetadataType);
|
|
288
|
-
const decodePreMergeStreamMetadata = decodeBeast2For(PreMergeStreamTaskMetadataType);
|
|
289
96
|
/**
|
|
290
|
-
*
|
|
291
|
-
* tolerating the pre-`merge` wire format (dual-decode migration).
|
|
292
|
-
*
|
|
293
|
-
* @param data - the metadata blob
|
|
294
|
-
* @returns the decoded metadata, with `merge` defaulted to `"none"` for older
|
|
295
|
-
* bytes
|
|
296
|
-
*/
|
|
297
|
-
export function decodeStreamTaskMetadata(data) {
|
|
298
|
-
try {
|
|
299
|
-
return decodeCurrentStreamMetadata(data);
|
|
300
|
-
}
|
|
301
|
-
catch (err) {
|
|
302
|
-
try {
|
|
303
|
-
return { ...decodePreMergeStreamMetadata(data), merge: 'none' };
|
|
304
|
-
}
|
|
305
|
-
catch {
|
|
306
|
-
throw err; // no known shape — surface the current-format error
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
// =============================================================================
|
|
311
|
-
// Partition plan
|
|
312
|
-
// =============================================================================
|
|
313
|
-
/** A carve position: a segment index and an element offset within it. */
|
|
314
|
-
const SplitPointType = StructType({ seg: IntegerType, offset: IntegerType });
|
|
315
|
-
/**
|
|
316
|
-
* The plan of one merged component's fan-in (issue #770): the partials whose
|
|
317
|
-
* key ranges overlap, and the key ranges its merge units run over, as the
|
|
318
|
-
* hashes of the range blobs the units take as their input.
|
|
97
|
+
* Task object stored in the object store.
|
|
319
98
|
*
|
|
320
99
|
* @remarks
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
* partial — so a re-run whose partials are the same objects plans the same
|
|
324
|
-
* ranges; recording them lets it skip the planning probes, and because
|
|
325
|
-
* every unit's inputs are then the same objects, every merge unit
|
|
326
|
-
* cache-hits.
|
|
327
|
-
*/
|
|
328
|
-
export const MergeRangePlanType = StructType({
|
|
329
|
-
/** The component's partial hashes, in partition order. */
|
|
330
|
-
partials: ArrayType(StringType),
|
|
331
|
-
/** The hashes of the component's range blobs — `Struct{from: Option<K>,
|
|
332
|
-
* to: Option<K>}` over the output's key type — in key order; one open
|
|
333
|
-
* range for a component that merges whole. */
|
|
334
|
-
ranges: ArrayType(StringType),
|
|
335
|
-
});
|
|
336
|
-
/**
|
|
337
|
-
* The plan of a partitioned execution (issue #770): the partitioned inputs,
|
|
338
|
-
* where each partition starts in every one of them, the carved slices, and
|
|
339
|
-
* the ranged fan-in of each merged component.
|
|
100
|
+
* A package-borne wire, so it changes by hard cutover: a package exported by
|
|
101
|
+
* an older SDK is re-exported, and {@link decodeTaskObject} says so.
|
|
340
102
|
*
|
|
341
|
-
* @
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
345
|
-
*
|
|
346
|
-
*
|
|
347
|
-
*
|
|
348
|
-
*
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
/** Per secondary (partitions[1..]): the split point of every partition plus the end; length boundaries.length + 1. */
|
|
356
|
-
splits: ArrayType(ArrayType(SplitPointType)),
|
|
357
|
-
/** slices[input][partition] object hashes; empty until carved. */
|
|
358
|
-
slices: ArrayType(ArrayType(StringType)),
|
|
359
|
-
/** The ranged fan-in of each merged component, once its ranges are
|
|
360
|
-
* planned. Appended LAST (BEAST2 encodes struct fields positionally) with
|
|
361
|
-
* a dual decoder — see {@link decodePartitionPlan}. */
|
|
362
|
-
merges: ArrayType(MergeRangePlanType),
|
|
363
|
-
});
|
|
364
|
-
/**
|
|
365
|
-
* The pre-`merges` plan wire shape, kept only so {@link decodePartitionPlan}
|
|
366
|
-
* can read plans a run recorded before the fan-in ran per key range.
|
|
103
|
+
* @example
|
|
104
|
+
* ```ts
|
|
105
|
+
* import { none, variant } from '@elaraai/east';
|
|
106
|
+
*
|
|
107
|
+
* const task: TaskObject = {
|
|
108
|
+
* kind: TASK_OBJECT_KIND,
|
|
109
|
+
* body: variant('east', { program: '5e7a3b...' }),
|
|
110
|
+
* runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
|
|
111
|
+
* inputs: [{ path: [variant('field', 'inputs'), variant('field', 'sales')], partition: none }],
|
|
112
|
+
* output: { path: [variant('field', 'tasks'), variant('field', 'totals'), variant('field', 'output')], kind: variant('value', null) },
|
|
113
|
+
* role: variant('data', null),
|
|
114
|
+
* environment: none,
|
|
115
|
+
* };
|
|
116
|
+
* ```
|
|
367
117
|
*/
|
|
368
|
-
const
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
118
|
+
export const TaskObjectType = StructType({
|
|
119
|
+
/** Always {@link TASK_OBJECT_KIND}. */
|
|
120
|
+
kind: StringType,
|
|
121
|
+
/** What the task runs. */
|
|
122
|
+
body: TaskBodyType,
|
|
123
|
+
/** The runtime it runs on. A `custom` runner's command is informational:
|
|
124
|
+
* the body's command IR is what spawns. */
|
|
125
|
+
runner: RunnerType,
|
|
126
|
+
/** The datasets it reads, in the order its program takes them. */
|
|
127
|
+
inputs: ArrayType(TaskInputType),
|
|
128
|
+
/** Where its output goes, and how it is made. */
|
|
129
|
+
output: TaskOutputType,
|
|
130
|
+
/** What its output is for. */
|
|
131
|
+
role: TaskRoleType,
|
|
132
|
+
/** Hash of the {@link EnvironmentSpecType} object it executes in; `none`
|
|
133
|
+
* for the stock runtime image. */
|
|
134
|
+
environment: OptionType(StringType),
|
|
373
135
|
});
|
|
374
|
-
|
|
375
|
-
export const encodePartitionPlan = encodeBeast2For(PartitionPlanType);
|
|
376
|
-
const decodeCurrentPlan = decodeBeast2For(PartitionPlanType);
|
|
377
|
-
const decodePreMergesPlan = decodeBeast2For(PreMergesPartitionPlanType);
|
|
136
|
+
const decodeCurrentTask = decodeBeast2For(TaskObjectType);
|
|
378
137
|
/**
|
|
379
|
-
* Decode a
|
|
380
|
-
* wire shape (dual-decode migration): an older plan decodes with no recorded
|
|
381
|
-
* merge ranges, so its partition slices are reused and its ranges planned.
|
|
138
|
+
* Decode a `TaskObject` from BEAST2 bytes.
|
|
382
139
|
*
|
|
383
|
-
* @param data - the
|
|
384
|
-
* @returns the
|
|
140
|
+
* @param data - the stored bytes
|
|
141
|
+
* @returns the task object
|
|
142
|
+
* @throws {Error} When the bytes are not a current task object — a package
|
|
143
|
+
* exported by an older SDK, which is re-exported with the current one.
|
|
385
144
|
*/
|
|
386
|
-
export function
|
|
145
|
+
export function decodeTaskObject(data) {
|
|
146
|
+
let task;
|
|
387
147
|
try {
|
|
388
|
-
|
|
148
|
+
task = decodeCurrentTask(data);
|
|
389
149
|
}
|
|
390
150
|
catch (err) {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
}
|
|
394
|
-
catch {
|
|
395
|
-
throw err; // no known shape — surface the current-format error
|
|
396
|
-
}
|
|
151
|
+
throw new Error(`the task object does not decode: the package was exported by an older e3 SDK — re-export it with the current one ` +
|
|
152
|
+
`(${err instanceof Error ? err.message : String(err)})`);
|
|
397
153
|
}
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
* Extracts the shape a `by` projection reads — see {@link ProjectionShape} —
|
|
401
|
-
* or `null` for any other (unaccepted) shape.
|
|
402
|
-
*
|
|
403
|
-
* @param ir - the projection's function IR, `(Key) -> Projection`
|
|
404
|
-
* @returns the shape, or `null` when the body is not an accepted projection
|
|
405
|
-
*/
|
|
406
|
-
export function partitionProjectionShape(ir) {
|
|
407
|
-
const param = ir.value.parameters[0]?.value.name;
|
|
408
|
-
if (param === undefined)
|
|
409
|
-
return null;
|
|
410
|
-
// Chase wrappers to the expression the body evaluates to.
|
|
411
|
-
const unwrap = (node) => {
|
|
412
|
-
for (;;) {
|
|
413
|
-
if (node.type === 'Block') {
|
|
414
|
-
const statements = node.value.statements;
|
|
415
|
-
if (statements.length === 0)
|
|
416
|
-
return node;
|
|
417
|
-
node = statements[statements.length - 1];
|
|
418
|
-
}
|
|
419
|
-
else if (node.type === 'Return') {
|
|
420
|
-
node = node.value.value;
|
|
421
|
-
}
|
|
422
|
-
else if (node.type === 'As' || node.type === 'WrapRecursive' || node.type === 'UnwrapRecursive') {
|
|
423
|
-
node = node.value.value;
|
|
424
|
-
}
|
|
425
|
-
else {
|
|
426
|
-
return node;
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
};
|
|
430
|
-
const isParam = (node) => node.type === 'Variable' && node.value.name === param;
|
|
431
|
-
const fieldOf = (node) => node.type === 'GetField' && isParam(unwrap(node.value.struct)) ? node.value.field : null;
|
|
432
|
-
// Walks a GetField chain down to the parameter: `key.a.b` → ['a', 'b'].
|
|
433
|
-
const pathOf = (node) => {
|
|
434
|
-
const path = [];
|
|
435
|
-
let cur = node;
|
|
436
|
-
while (cur.type === 'GetField') {
|
|
437
|
-
path.unshift(cur.value.field);
|
|
438
|
-
cur = unwrap(cur.value.struct);
|
|
439
|
-
}
|
|
440
|
-
return isParam(cur) && path.length > 0 ? path : null;
|
|
441
|
-
};
|
|
442
|
-
const body = unwrap(ir.value.body);
|
|
443
|
-
if (isParam(body))
|
|
444
|
-
return { kind: 'fields', names: [] };
|
|
445
|
-
const chain = pathOf(body);
|
|
446
|
-
if (chain !== null) {
|
|
447
|
-
// A one-step chain is a top-level field read — the `fields` family.
|
|
448
|
-
return chain.length === 1 ? { kind: 'fields', names: chain } : { kind: 'path', names: chain };
|
|
449
|
-
}
|
|
450
|
-
if (body.type === 'Struct') {
|
|
451
|
-
const fields = [];
|
|
452
|
-
for (const { value } of body.value.fields) {
|
|
453
|
-
const f = fieldOf(unwrap(value));
|
|
454
|
-
if (f === null)
|
|
455
|
-
return null;
|
|
456
|
-
fields.push(f);
|
|
457
|
-
}
|
|
458
|
-
return { kind: 'fields', names: fields };
|
|
459
|
-
}
|
|
460
|
-
return null;
|
|
461
|
-
}
|
|
462
|
-
/**
|
|
463
|
-
* The East type of a projected key — the type {@link projectKey} values have,
|
|
464
|
-
* and the one to build their comparator for: the key type itself for the
|
|
465
|
-
* identity, a struct of the named leading fields in declared order, or the
|
|
466
|
-
* type at the end of a nested first-field path.
|
|
467
|
-
*
|
|
468
|
-
* @param shape - the projection's shape
|
|
469
|
-
* @param keyType - the partitioned dataset's key type
|
|
470
|
-
* @returns the projected key type
|
|
471
|
-
* @throws {Error} When the shape reads a field the key type does not have.
|
|
472
|
-
*
|
|
473
|
-
* @example
|
|
474
|
-
* ```ts
|
|
475
|
-
* const cmp = compareFor(projectedKeyType(shape, keyTypeValue));
|
|
476
|
-
* cmp(projectKey(shape, a), projectKey(shape, b));
|
|
477
|
-
* ```
|
|
478
|
-
*/
|
|
479
|
-
export function projectedKeyType(shape, keyType) {
|
|
480
|
-
if (shape.names.length === 0)
|
|
481
|
-
return keyType;
|
|
482
|
-
const fieldType = (level, name) => {
|
|
483
|
-
const field = level.type === 'Struct' ? level.value.find((f) => f.name === name) : undefined;
|
|
484
|
-
if (field === undefined) {
|
|
485
|
-
throw new Error(`partition projection reads field '${name}' of a key level that has no such field`);
|
|
486
|
-
}
|
|
487
|
-
return field.type;
|
|
488
|
-
};
|
|
489
|
-
if (shape.kind === 'path')
|
|
490
|
-
return shape.names.reduce(fieldType, keyType);
|
|
491
|
-
return variant('Struct', shape.names.map((name) => ({ name, type: fieldType(keyType, name) })));
|
|
492
|
-
}
|
|
493
|
-
/**
|
|
494
|
-
* Projects a partition key through a `by` shape, without compiling the
|
|
495
|
-
* projection: the key itself for the identity, a struct of the named leading
|
|
496
|
-
* fields built field by field in declared order, or the value at the end of a
|
|
497
|
-
* nested first-field path.
|
|
498
|
-
*
|
|
499
|
-
* @param shape - the projection's shape
|
|
500
|
-
* @param key - a key of the partitioned dataset
|
|
501
|
-
* @returns the projected key, a value of {@link projectedKeyType}
|
|
502
|
-
*/
|
|
503
|
-
export function projectKey(shape, key) {
|
|
504
|
-
if (shape.names.length === 0)
|
|
505
|
-
return key;
|
|
506
|
-
if (shape.kind === 'path') {
|
|
507
|
-
return shape.names.reduce((level, name) => level[name], key);
|
|
154
|
+
if (task.kind !== TASK_OBJECT_KIND) {
|
|
155
|
+
throw new Error(`the object is not a task object: its kind is '${task.kind}', not '${TASK_OBJECT_KIND}'`);
|
|
508
156
|
}
|
|
509
|
-
|
|
510
|
-
for (const name of shape.names)
|
|
511
|
-
projected[name] = key[name];
|
|
512
|
-
return projected;
|
|
157
|
+
return task;
|
|
513
158
|
}
|
|
514
159
|
//# sourceMappingURL=task.js.map
|