@elaraai/e3-types 1.0.78 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/api.d.ts +569 -36
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +257 -32
- package/dist/src/api.js.map +1 -1
- package/dist/src/collection-manifest.d.ts +37 -0
- package/dist/src/collection-manifest.d.ts.map +1 -0
- package/dist/src/collection-manifest.js +37 -0
- package/dist/src/collection-manifest.js.map +1 -0
- package/dist/src/dataflow.d.ts +264 -19
- package/dist/src/dataflow.d.ts.map +1 -1
- package/dist/src/dataflow.js +156 -15
- package/dist/src/dataflow.js.map +1 -1
- package/dist/src/dataflow.spec.js +184 -27
- package/dist/src/dataflow.spec.js.map +1 -1
- package/dist/src/dataset-blob.d.ts +75 -15
- package/dist/src/dataset-blob.d.ts.map +1 -1
- package/dist/src/dataset-blob.js +95 -28
- package/dist/src/dataset-blob.js.map +1 -1
- package/dist/src/dataset-blob.spec.d.ts +6 -0
- package/dist/src/dataset-blob.spec.d.ts.map +1 -0
- package/dist/src/dataset-blob.spec.js +92 -0
- package/dist/src/dataset-blob.spec.js.map +1 -0
- package/dist/src/dataset-ref.d.ts +5 -4
- package/dist/src/dataset-ref.d.ts.map +1 -1
- package/dist/src/dataset-ref.js +5 -4
- package/dist/src/dataset-ref.js.map +1 -1
- package/dist/src/environment.d.ts +3 -2
- package/dist/src/environment.d.ts.map +1 -1
- package/dist/src/environment.js +3 -2
- package/dist/src/environment.js.map +1 -1
- package/dist/src/execution.d.ts +112 -18
- package/dist/src/execution.d.ts.map +1 -1
- package/dist/src/execution.js +174 -45
- package/dist/src/execution.js.map +1 -1
- package/dist/src/execution.spec.d.ts +6 -0
- package/dist/src/execution.spec.d.ts.map +1 -0
- package/dist/src/execution.spec.js +108 -0
- package/dist/src/execution.spec.js.map +1 -0
- package/dist/src/function.d.ts +14 -12
- package/dist/src/function.d.ts.map +1 -1
- package/dist/src/function.js +17 -32
- package/dist/src/function.js.map +1 -1
- package/dist/src/index.d.ts +16 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +25 -20
- package/dist/src/index.js.map +1 -1
- package/dist/src/lock.d.ts +68 -22
- package/dist/src/lock.d.ts.map +1 -1
- package/dist/src/lock.js +33 -22
- package/dist/src/lock.js.map +1 -1
- package/dist/src/names.d.ts +21 -0
- package/dist/src/names.d.ts.map +1 -0
- package/dist/src/names.js +31 -0
- package/dist/src/names.js.map +1 -0
- package/dist/src/package.d.ts +11 -78
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +13 -66
- package/dist/src/package.js.map +1 -1
- package/dist/src/record.d.ts +582 -18
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +405 -7
- package/dist/src/record.js.map +1 -1
- package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
- package/dist/src/record.spec.d.ts.map +1 -0
- package/dist/src/record.spec.js +143 -0
- package/dist/src/record.spec.js.map +1 -0
- package/dist/src/release.d.ts +36 -0
- package/dist/src/release.d.ts.map +1 -0
- package/dist/src/release.js +71 -0
- package/dist/src/release.js.map +1 -0
- package/dist/src/release.spec.d.ts +6 -0
- package/dist/src/release.spec.d.ts.map +1 -0
- package/dist/src/release.spec.js +42 -0
- package/dist/src/release.spec.js.map +1 -0
- package/dist/src/repository.d.ts +64 -0
- package/dist/src/repository.d.ts.map +1 -0
- package/dist/src/repository.js +49 -0
- package/dist/src/repository.js.map +1 -0
- package/dist/src/runner.d.ts +16 -62
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +16 -79
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +11 -85
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/structure.d.ts +0 -57
- package/dist/src/structure.d.ts.map +1 -1
- package/dist/src/structure.js +0 -3
- package/dist/src/structure.js.map +1 -1
- package/dist/src/task.d.ts +210 -318
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +117 -472
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +62 -154
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/transfer.d.ts +16 -22
- package/dist/src/transfer.d.ts.map +1 -1
- package/dist/src/transfer.js +16 -22
- package/dist/src/transfer.js.map +1 -1
- package/dist/src/transfer.spec.js +4 -34
- package/dist/src/transfer.spec.js.map +1 -1
- package/dist/src/unit-plan.d.ts +130 -0
- package/dist/src/unit-plan.d.ts.map +1 -0
- package/dist/src/unit-plan.js +107 -0
- package/dist/src/unit-plan.js.map +1 -0
- package/dist/src/unit-plan.spec.d.ts +6 -0
- package/dist/src/unit-plan.spec.d.ts.map +1 -0
- package/dist/src/unit-plan.spec.js +29 -0
- package/dist/src/unit-plan.spec.js.map +1 -0
- package/dist/src/workspace.d.ts +22 -11
- package/dist/src/workspace.d.ts.map +1 -1
- package/dist/src/workspace.js +10 -11
- package/dist/src/workspace.js.map +1 -1
- package/package.json +3 -3
- package/src/api.ts +276 -32
- package/src/collection-manifest.ts +53 -0
- package/src/dataflow.spec.ts +223 -31
- package/src/dataflow.ts +183 -26
- package/src/dataset-blob.spec.ts +117 -0
- package/src/dataset-blob.ts +157 -21
- package/src/dataset-ref.ts +5 -4
- package/src/environment.ts +3 -2
- package/src/execution.spec.ts +114 -0
- package/src/execution.ts +175 -51
- package/src/function.ts +19 -32
- package/src/index.ts +122 -39
- package/src/lock.ts +39 -22
- package/src/names.ts +39 -0
- package/src/package.ts +15 -71
- package/src/record.spec.ts +195 -0
- package/src/record.ts +489 -7
- package/src/release.spec.ts +47 -0
- package/src/release.ts +69 -0
- package/src/repository.ts +67 -0
- package/src/runner.spec.ts +11 -94
- package/src/runner.ts +16 -80
- package/src/structure.ts +0 -8
- package/src/task.spec.ts +67 -188
- package/src/task.ts +140 -522
- package/src/transfer.spec.ts +4 -51
- package/src/transfer.ts +16 -22
- package/src/unit-plan.spec.ts +31 -0
- package/src/unit-plan.ts +121 -0
- package/src/workspace.ts +13 -11
- package/dist/src/stream.d.ts +0 -97
- package/dist/src/stream.d.ts.map +0 -1
- package/dist/src/stream.js +0 -134
- package/dist/src/stream.js.map +0 -1
- package/dist/src/stream.spec.d.ts.map +0 -1
- package/dist/src/stream.spec.js +0 -60
- package/dist/src/stream.spec.js.map +0 -1
- package/src/stream.spec.ts +0 -88
- package/src/stream.ts +0 -170
package/dist/src/task.d.ts
CHANGED
|
@@ -5,75 +5,184 @@
|
|
|
5
5
|
/**
|
|
6
6
|
* Task object types for e3.
|
|
7
7
|
*
|
|
8
|
-
* A task object
|
|
9
|
-
*
|
|
8
|
+
* A task object says what a task runs, on which runtime, over which datasets,
|
|
9
|
+
* and what its output is: every field a mode used to be implied by is typed.
|
|
10
|
+
* Task objects are stored in the object store and referenced by packages, so
|
|
11
|
+
* they are content-addressed, which is what memoizes their executions.
|
|
10
12
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
+
* Input and output types are not stored here: they are the package structure's
|
|
14
|
+
* at the paths the task names.
|
|
15
|
+
*/
|
|
16
|
+
import { StructType, StringType, ArrayType, NullType, OptionType, VariantType, ValueTypeOf } from '@elaraai/east';
|
|
17
|
+
/** The kind tag every task object carries, which the garbage collector
|
|
18
|
+
* dispatches on. */
|
|
19
|
+
export declare const TASK_OBJECT_KIND = "$task";
|
|
20
|
+
/**
|
|
21
|
+
* What a task runs.
|
|
13
22
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
23
|
+
* - `east`: an East program, which a stock runner executes as a unit
|
|
24
|
+
* (`exec`): `program` is the hash of its IR bundle.
|
|
25
|
+
* - `command`: a command, which the custom runtime spawns. `commandIr` is the
|
|
26
|
+
* hash of the IR bundle of the East function that builds its argv,
|
|
27
|
+
* `(inputs: Array<String>, output: String) -> Array<String>`.
|
|
28
|
+
*/
|
|
29
|
+
export declare const TaskBodyType: VariantType<{
|
|
30
|
+
readonly east: StructType<{
|
|
31
|
+
readonly program: StringType;
|
|
32
|
+
}>;
|
|
33
|
+
readonly command: StructType<{
|
|
34
|
+
readonly commandIr: StringType;
|
|
35
|
+
}>;
|
|
36
|
+
}>;
|
|
37
|
+
export type TaskBodyType = typeof TaskBodyType;
|
|
38
|
+
export type TaskBody = ValueTypeOf<typeof TaskBodyType>;
|
|
39
|
+
/**
|
|
40
|
+
* An input the work may be split over: `by` names the leading key fields
|
|
41
|
+
* whose equal values are never split across pieces — a field, or a
|
|
42
|
+
* first-field path written `a.b`. Empty when any key may start a piece.
|
|
43
|
+
*/
|
|
44
|
+
export declare const TaskPartitionType: StructType<{
|
|
45
|
+
readonly by: ArrayType<StringType>;
|
|
46
|
+
}>;
|
|
47
|
+
export type TaskPartitionType = typeof TaskPartitionType;
|
|
48
|
+
export type TaskPartition = ValueTypeOf<typeof TaskPartitionType>;
|
|
49
|
+
/** A dataset a task reads, and whether the work may be split over it. */
|
|
50
|
+
export declare const TaskInputType: StructType<{
|
|
51
|
+
readonly path: ArrayType<VariantType<{
|
|
52
|
+
readonly field: StringType;
|
|
53
|
+
}>>;
|
|
54
|
+
readonly partition: OptionType<StructType<{
|
|
55
|
+
readonly by: ArrayType<StringType>;
|
|
56
|
+
}>>;
|
|
57
|
+
}>;
|
|
58
|
+
export type TaskInputType = typeof TaskInputType;
|
|
59
|
+
export type TaskInput = ValueTypeOf<typeof TaskInputType>;
|
|
60
|
+
/**
|
|
61
|
+
* How a task's output is made, which fixes how the parts of it that units
|
|
62
|
+
* write combine.
|
|
63
|
+
*
|
|
64
|
+
* - `value`: the program returns it.
|
|
65
|
+
* - `array`: emitted elements, concatenated in emission order.
|
|
66
|
+
* - `set`: emitted elements, in any order, unioned.
|
|
67
|
+
* - `dict`: emitted entries, in any order; equal keys fold with `merge`, the
|
|
68
|
+
* hash of a `(K, V, V) -> V` IR bundle, and without it are refused.
|
|
69
|
+
* - `fold`: emitted values folded with `combine`, the hash of a `(T, T) -> T`
|
|
70
|
+
* IR bundle, starting at `zero`, the hash of a stored value.
|
|
16
71
|
*/
|
|
17
|
-
|
|
18
|
-
|
|
72
|
+
export declare const TaskOutputKindType: VariantType<{
|
|
73
|
+
readonly value: NullType;
|
|
74
|
+
readonly array: NullType;
|
|
75
|
+
readonly set: NullType;
|
|
76
|
+
readonly dict: StructType<{
|
|
77
|
+
readonly merge: OptionType<StringType>;
|
|
78
|
+
}>;
|
|
79
|
+
readonly fold: StructType<{
|
|
80
|
+
readonly zero: StringType;
|
|
81
|
+
readonly combine: StringType;
|
|
82
|
+
}>;
|
|
83
|
+
}>;
|
|
84
|
+
export type TaskOutputKindType = typeof TaskOutputKindType;
|
|
85
|
+
export type TaskOutputKind = ValueTypeOf<typeof TaskOutputKindType>;
|
|
86
|
+
/** Where a task's output goes, and how it is made. */
|
|
87
|
+
export declare const TaskOutputType: StructType<{
|
|
88
|
+
readonly path: ArrayType<VariantType<{
|
|
89
|
+
readonly field: StringType;
|
|
90
|
+
}>>;
|
|
91
|
+
readonly kind: VariantType<{
|
|
92
|
+
readonly value: NullType;
|
|
93
|
+
readonly array: NullType;
|
|
94
|
+
readonly set: NullType;
|
|
95
|
+
readonly dict: StructType<{
|
|
96
|
+
readonly merge: OptionType<StringType>;
|
|
97
|
+
}>;
|
|
98
|
+
readonly fold: StructType<{
|
|
99
|
+
readonly zero: StringType;
|
|
100
|
+
readonly combine: StringType;
|
|
101
|
+
}>;
|
|
102
|
+
}>;
|
|
103
|
+
}>;
|
|
104
|
+
export type TaskOutputType = typeof TaskOutputType;
|
|
105
|
+
export type TaskOutput = ValueTypeOf<typeof TaskOutputType>;
|
|
106
|
+
/**
|
|
107
|
+
* What a ui task binds as it renders.
|
|
108
|
+
*
|
|
109
|
+
* @property paths - Dataset paths it reads or writes through `Data.bind`,
|
|
110
|
+
* including each bound record's own path, so the record's current value is
|
|
111
|
+
* preloaded and polled like any dataset.
|
|
112
|
+
* @property functions - Package functions it calls through `Func.bind`.
|
|
113
|
+
* @property records - Records it binds through `Record.bind`.
|
|
114
|
+
* @property pages - Dataset paths it reads a window at a time through
|
|
115
|
+
* `Data.bindPaged`: declared apart from `paths` because they are never
|
|
116
|
+
* preloaded or polled whole.
|
|
117
|
+
*/
|
|
118
|
+
export declare const DataManifestType: StructType<{
|
|
119
|
+
readonly paths: ArrayType<ArrayType<VariantType<{
|
|
120
|
+
readonly field: StringType;
|
|
121
|
+
}>>>;
|
|
122
|
+
readonly functions: ArrayType<StringType>;
|
|
123
|
+
readonly records: ArrayType<StringType>;
|
|
124
|
+
readonly pages: ArrayType<ArrayType<VariantType<{
|
|
125
|
+
readonly field: StringType;
|
|
126
|
+
}>>>;
|
|
127
|
+
}>;
|
|
128
|
+
export type DataManifestType = typeof DataManifestType;
|
|
129
|
+
export type DataManifest = ValueTypeOf<typeof DataManifestType>;
|
|
130
|
+
/**
|
|
131
|
+
* What a task's output is for: `data`, or a `ui` whose output is a component
|
|
132
|
+
* tree rendered against the datasets, functions and records it binds.
|
|
133
|
+
*/
|
|
134
|
+
export declare const TaskRoleType: VariantType<{
|
|
135
|
+
readonly data: NullType;
|
|
136
|
+
readonly ui: StructType<{
|
|
137
|
+
readonly paths: ArrayType<ArrayType<VariantType<{
|
|
138
|
+
readonly field: StringType;
|
|
139
|
+
}>>>;
|
|
140
|
+
readonly functions: ArrayType<StringType>;
|
|
141
|
+
readonly records: ArrayType<StringType>;
|
|
142
|
+
readonly pages: ArrayType<ArrayType<VariantType<{
|
|
143
|
+
readonly field: StringType;
|
|
144
|
+
}>>>;
|
|
145
|
+
}>;
|
|
146
|
+
}>;
|
|
147
|
+
export type TaskRoleType = typeof TaskRoleType;
|
|
148
|
+
export type TaskRole = ValueTypeOf<typeof TaskRoleType>;
|
|
19
149
|
/**
|
|
20
150
|
* Task object stored in the object store.
|
|
21
151
|
*
|
|
22
|
-
* A task is a complete executable unit that reads from input dataset paths
|
|
23
|
-
* and writes to an output dataset path. The commandIr is evaluated at runtime
|
|
24
|
-
* to produce the exec args.
|
|
25
|
-
*
|
|
26
152
|
* @remarks
|
|
27
|
-
* -
|
|
28
|
-
*
|
|
29
|
-
* - `inputs` are paths to staged input .beast2 files
|
|
30
|
-
* - `output` is the path where output should be written
|
|
31
|
-
* - Returns array of strings to exec (e.g., ["sh", "-c", "python ..."])
|
|
32
|
-
* - `inputs`: Paths to input datasets in the data tree
|
|
33
|
-
* - `output`: Path to the output dataset in the data tree
|
|
34
|
-
*
|
|
35
|
-
* Types are not stored in the task - they are inferred from the package's
|
|
36
|
-
* structure at the specified paths. This keeps tasks simple and avoids
|
|
37
|
-
* redundant type information.
|
|
153
|
+
* A package-borne wire, so it changes by hard cutover: a package exported by
|
|
154
|
+
* an older SDK is re-exported, and {@link decodeTaskObject} says so.
|
|
38
155
|
*
|
|
39
156
|
* @example
|
|
40
157
|
* ```ts
|
|
41
|
-
* import { variant } from '@elaraai/east';
|
|
158
|
+
* import { none, variant } from '@elaraai/east';
|
|
42
159
|
*
|
|
43
|
-
* // Task with command IR that generates: ["sh", "-c", "python script.py <input> <output>"]
|
|
44
160
|
* const task: TaskObject = {
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
* ],
|
|
49
|
-
* output: [variant('field', 'tasks'), variant('field', '
|
|
161
|
+
* kind: TASK_OBJECT_KIND,
|
|
162
|
+
* body: variant('east', { program: '5e7a3b...' }),
|
|
163
|
+
* runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
|
|
164
|
+
* inputs: [{ path: [variant('field', 'inputs'), variant('field', 'sales')], partition: none }],
|
|
165
|
+
* output: { path: [variant('field', 'tasks'), variant('field', 'totals'), variant('field', 'output')], kind: variant('value', null) },
|
|
166
|
+
* role: variant('data', null),
|
|
167
|
+
* environment: none,
|
|
50
168
|
* };
|
|
51
169
|
* ```
|
|
52
170
|
*/
|
|
53
171
|
export declare const TaskObjectType: StructType<{
|
|
54
|
-
/**
|
|
55
|
-
readonly
|
|
56
|
-
/**
|
|
57
|
-
readonly
|
|
58
|
-
readonly
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
readonly
|
|
68
|
-
/**
|
|
69
|
-
* The task's runner, as routing metadata (symmetric with
|
|
70
|
-
* FunctionObject.runner). For `custom` runners, `commandIr` remains
|
|
71
|
-
* authoritative for execution — the wire command is informational.
|
|
72
|
-
*
|
|
73
|
-
* NOTE: added as a hard cutover (no dual decoder) — packages exported by
|
|
74
|
-
* older SDKs must be re-exported.
|
|
75
|
-
*/
|
|
76
|
-
readonly runner: import("@elaraai/east").VariantType<{
|
|
172
|
+
/** Always {@link TASK_OBJECT_KIND}. */
|
|
173
|
+
readonly kind: StringType;
|
|
174
|
+
/** What the task runs. */
|
|
175
|
+
readonly body: VariantType<{
|
|
176
|
+
readonly east: StructType<{
|
|
177
|
+
readonly program: StringType;
|
|
178
|
+
}>;
|
|
179
|
+
readonly command: StructType<{
|
|
180
|
+
readonly commandIr: StringType;
|
|
181
|
+
}>;
|
|
182
|
+
}>;
|
|
183
|
+
/** The runtime it runs on. A `custom` runner's command is informational:
|
|
184
|
+
* the body's command IR is what spawns. */
|
|
185
|
+
readonly runner: VariantType<{
|
|
77
186
|
readonly east_node: StructType<{
|
|
78
187
|
readonly platforms: ArrayType<StringType>;
|
|
79
188
|
}>;
|
|
@@ -87,277 +196,60 @@ export declare const TaskObjectType: StructType<{
|
|
|
87
196
|
readonly command: ArrayType<StringType>;
|
|
88
197
|
}>;
|
|
89
198
|
}>;
|
|
90
|
-
/**
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
199
|
+
/** The datasets it reads, in the order its program takes them. */
|
|
200
|
+
readonly inputs: ArrayType<StructType<{
|
|
201
|
+
readonly path: ArrayType<VariantType<{
|
|
202
|
+
readonly field: StringType;
|
|
203
|
+
}>>;
|
|
204
|
+
readonly partition: OptionType<StructType<{
|
|
205
|
+
readonly by: ArrayType<StringType>;
|
|
206
|
+
}>>;
|
|
207
|
+
}>>;
|
|
208
|
+
/** Where its output goes, and how it is made. */
|
|
209
|
+
readonly output: StructType<{
|
|
210
|
+
readonly path: ArrayType<VariantType<{
|
|
211
|
+
readonly field: StringType;
|
|
212
|
+
}>>;
|
|
213
|
+
readonly kind: VariantType<{
|
|
214
|
+
readonly value: NullType;
|
|
215
|
+
readonly array: NullType;
|
|
216
|
+
readonly set: NullType;
|
|
217
|
+
readonly dict: StructType<{
|
|
218
|
+
readonly merge: OptionType<StringType>;
|
|
219
|
+
}>;
|
|
220
|
+
readonly fold: StructType<{
|
|
221
|
+
readonly zero: StringType;
|
|
222
|
+
readonly combine: StringType;
|
|
223
|
+
}>;
|
|
224
|
+
}>;
|
|
225
|
+
}>;
|
|
226
|
+
/** What its output is for. */
|
|
227
|
+
readonly role: VariantType<{
|
|
228
|
+
readonly data: NullType;
|
|
229
|
+
readonly ui: StructType<{
|
|
230
|
+
readonly paths: ArrayType<ArrayType<VariantType<{
|
|
231
|
+
readonly field: StringType;
|
|
232
|
+
}>>>;
|
|
233
|
+
readonly functions: ArrayType<StringType>;
|
|
234
|
+
readonly records: ArrayType<StringType>;
|
|
235
|
+
readonly pages: ArrayType<ArrayType<VariantType<{
|
|
236
|
+
readonly field: StringType;
|
|
237
|
+
}>>>;
|
|
238
|
+
}>;
|
|
239
|
+
}>;
|
|
240
|
+
/** Hash of the {@link EnvironmentSpecType} object it executes in; `none`
|
|
241
|
+
* for the stock runtime image. */
|
|
96
242
|
readonly environment: OptionType<StringType>;
|
|
97
243
|
}>;
|
|
98
244
|
export type TaskObjectType = typeof TaskObjectType;
|
|
99
245
|
export type TaskObject = ValueTypeOf<typeof TaskObjectType>;
|
|
100
246
|
/**
|
|
101
|
-
* Decode a `TaskObject` from BEAST2 bytes
|
|
102
|
-
* wire format (dual-decode migration, like {@link decodePackageObject}).
|
|
247
|
+
* Decode a `TaskObject` from BEAST2 bytes.
|
|
103
248
|
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
249
|
+
* @param data - the stored bytes
|
|
250
|
+
* @returns the task object
|
|
251
|
+
* @throws {Error} When the bytes are not a current task object — a package
|
|
252
|
+
* exported by an older SDK, which is re-exported with the current one.
|
|
107
253
|
*/
|
|
108
254
|
export declare function decodeTaskObject(data: Uint8Array): TaskObject;
|
|
109
|
-
/** Task kind of a partitioned task — the orchestrator carves its partitioned
|
|
110
|
-
* input(s) into key-range slices, runs each slice as an ordinary
|
|
111
|
-
* content-addressed execution, and assembles the output by splice or by
|
|
112
|
-
* combining partials. The spec rides {@link TaskObjectType}'s `metadata`
|
|
113
|
-
* slot as a beast2-encoded {@link PartitionTaskMetadataType}. */
|
|
114
|
-
export declare const TASK_KIND_PARTITION = "partition";
|
|
115
|
-
/** Task kind of a streaming task — one execution whose runner feeds the
|
|
116
|
-
* stream input lazily and writes the output incrementally through an `emit`
|
|
117
|
-
* capability. The spec rides {@link TaskObjectType}'s `metadata` slot as a
|
|
118
|
-
* beast2-encoded {@link StreamTaskMetadataType}. */
|
|
119
|
-
export declare const TASK_KIND_STREAM = "stream";
|
|
120
|
-
/** Task kind of a merge unit — the runner's `merge` command over sorted
|
|
121
|
-
* partials of a partitioned task's keyed output (issue #770). The
|
|
122
|
-
* orchestrator writes one such task per partitioned task from the
|
|
123
|
-
* package's {@link PartitionTaskMetadataType} `mergeCommand`, and runs it
|
|
124
|
-
* as an ordinary content-addressed execution per group of partials. It
|
|
125
|
-
* carries no metadata. */
|
|
126
|
-
export declare const TASK_KIND_MERGE = "merge";
|
|
127
|
-
/**
|
|
128
|
-
* Metadata of a {@link TASK_KIND_PARTITION} task.
|
|
129
|
-
*
|
|
130
|
-
* The task's wire `inputs` are laid out `[function_ir, ...partitions,
|
|
131
|
-
* ...inputs]`, so `partitions` counts how many entries after the function IR
|
|
132
|
-
* are partitioned datasets; the rest are ordinary (broadcast) inputs that
|
|
133
|
-
* hash into every partition execution's identity.
|
|
134
|
-
*
|
|
135
|
-
* `by` and `combine` are carried as `encodeEastIR` bundles (capture-free IR +
|
|
136
|
-
* its source map), not as FunctionType values or object-store hashes: IR is
|
|
137
|
-
* how executable code travels everywhere on the e3 wire (`commandIr`,
|
|
138
|
-
* `function_ir`, `bodyIr`), a FunctionType value could smuggle captures the
|
|
139
|
-
* orchestrator must not evaluate, and inline bytes stay reachable where a
|
|
140
|
-
* hash inside an opaque metadata blob would be invisible to GC.
|
|
141
|
-
*/
|
|
142
|
-
export declare const PartitionTaskMetadataType: StructType<{
|
|
143
|
-
/** Number of partitioned inputs (wire input indices `1..1+partitions`). */
|
|
144
|
-
readonly partitions: IntegerType;
|
|
145
|
-
/** `encodeEastIR` bundle of the boundary-alignment projection
|
|
146
|
-
* `(Key) -> Projection`; `none` when partitioning is free per row/segment. */
|
|
147
|
-
readonly by: OptionType<BlobType>;
|
|
148
|
-
/** `encodeEastIR` bundle of the associative fold `(Out, Out) -> Out`;
|
|
149
|
-
* `none` in splice mode (shards concatenate). */
|
|
150
|
-
readonly combine: OptionType<BlobType>;
|
|
151
|
-
/** Target carved-slice size in wire bytes — the only sizing knob. */
|
|
152
|
-
readonly targetPartitionBytes: IntegerType;
|
|
153
|
-
/**
|
|
154
|
-
* `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
|
|
155
|
-
* for a Dict output; `none` otherwise.
|
|
156
|
-
*
|
|
157
|
-
* Its presence (or {@link mergeSets}) selects the MERGE-TREE assembly:
|
|
158
|
-
* partials whose key ranges overlap are merged by the task's own runner, in
|
|
159
|
-
* a tree of merge executions ({@link mergeCommand}) folding equal keys with
|
|
160
|
-
* this function; disjoint partials are spliced; the orchestrator never
|
|
161
|
-
* decodes a partial. Appended after `targetPartitionBytes` (BEAST2 encodes
|
|
162
|
-
* struct fields positionally) with a dual decoder — see
|
|
163
|
-
* {@link decodePartitionTaskMetadata}.
|
|
164
|
-
*/
|
|
165
|
-
readonly merge: OptionType<BlobType>;
|
|
166
|
-
/** Whether a Set output assembles by the merge tree, keeping one of equal
|
|
167
|
-
* elements (`--union`). The Set twin of {@link merge}, which needs no
|
|
168
|
-
* function. */
|
|
169
|
-
readonly mergeSets: BooleanType;
|
|
170
|
-
/**
|
|
171
|
-
* `encodeEastIR` bundle of the merge command `(input_paths, output_path) ->
|
|
172
|
-
* argv` — `mergeCommandIr` over the task's runner, in `function` mode with
|
|
173
|
-
* {@link merge} and `union` mode with {@link mergeSets} — that the
|
|
174
|
-
* orchestrator's merge units execute; `none` in splice and combine modes.
|
|
175
|
-
* Built at export, so the merge unit task is a package object like any
|
|
176
|
-
* other and nothing is synthesized at run time. Appended LAST with a
|
|
177
|
-
* triple decoder — see {@link decodePartitionTaskMetadata}.
|
|
178
|
-
*/
|
|
179
|
-
readonly mergeCommand: OptionType<BlobType>;
|
|
180
|
-
}>;
|
|
181
|
-
export type PartitionTaskMetadataType = typeof PartitionTaskMetadataType;
|
|
182
|
-
export type PartitionTaskMetadata = ValueTypeOf<typeof PartitionTaskMetadataType>;
|
|
183
|
-
/** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
|
|
184
|
-
export declare const encodePartitionTaskMetadata: (value: PartitionTaskMetadata) => Uint8Array;
|
|
185
|
-
/**
|
|
186
|
-
* Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task,
|
|
187
|
-
* tolerating the two older wire formats: the v1.0.77 shape without
|
|
188
|
-
* `mergeCommand`, and the pre-`merge` shape.
|
|
189
|
-
*
|
|
190
|
-
* @param data - the metadata blob
|
|
191
|
-
* @returns the decoded metadata, with `mergeCommand` (and, for the oldest
|
|
192
|
-
* bytes, `merge`/`mergeSets`) defaulted off
|
|
193
|
-
*/
|
|
194
|
-
export declare function decodePartitionTaskMetadata(data: Uint8Array): PartitionTaskMetadata;
|
|
195
|
-
/**
|
|
196
|
-
* Metadata of a {@link TASK_KIND_STREAM} task.
|
|
197
|
-
*
|
|
198
|
-
* The task's wire `inputs` are laid out `[function_ir, merge_ir?, stream?,
|
|
199
|
-
* ...inputs]` — `merge_ir` only in `function` merge mode; the compiled body
|
|
200
|
-
* takes one trailing `emit` parameter beyond the wire inputs, and the runner
|
|
201
|
-
* writes the `-o` file from the emit sink instead of the body's (Null) return
|
|
202
|
-
* value.
|
|
203
|
-
*/
|
|
204
|
-
export declare const StreamTaskMetadataType: StructType<{
|
|
205
|
-
/** Whether the first input after the IRs is the streamed input (producer
|
|
206
|
-
* tasks have no streamed input). */
|
|
207
|
-
readonly stream: BooleanType;
|
|
208
|
-
/** The output collection kind the emit sink writes: `"array"`, `"set"`, or
|
|
209
|
-
* `"dict"` — element/key/value types come from the body IR's emit
|
|
210
|
-
* parameter. */
|
|
211
|
-
readonly emit: StringType;
|
|
212
|
-
/**
|
|
213
|
-
* How the emit sink treats equal keys, a `StreamMergeMode`: `"none"` —
|
|
214
|
-
* a duplicate key is an error; `"function"` — wire input 1 is the merge IR
|
|
215
|
-
* `(K, V, V) -> V`, folding equal Dict keys; `"union"` — equal Set elements
|
|
216
|
-
* collapse. Appended LAST (BEAST2 encodes struct fields positionally) with a
|
|
217
|
-
* dual decoder — see {@link decodeStreamTaskMetadata}.
|
|
218
|
-
*/
|
|
219
|
-
readonly merge: StringType;
|
|
220
|
-
}>;
|
|
221
|
-
export type StreamTaskMetadataType = typeof StreamTaskMetadataType;
|
|
222
|
-
export type StreamTaskMetadata = ValueTypeOf<typeof StreamTaskMetadataType>;
|
|
223
|
-
/** Encode a {@link StreamTaskMetadataType} value for `TaskObject.metadata`. */
|
|
224
|
-
export declare const encodeStreamTaskMetadata: (value: StreamTaskMetadata) => Uint8Array;
|
|
225
|
-
/**
|
|
226
|
-
* Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_STREAM} task,
|
|
227
|
-
* tolerating the pre-`merge` wire format (dual-decode migration).
|
|
228
|
-
*
|
|
229
|
-
* @param data - the metadata blob
|
|
230
|
-
* @returns the decoded metadata, with `merge` defaulted to `"none"` for older
|
|
231
|
-
* bytes
|
|
232
|
-
*/
|
|
233
|
-
export declare function decodeStreamTaskMetadata(data: Uint8Array): StreamTaskMetadata;
|
|
234
|
-
/**
|
|
235
|
-
* The plan of one merged component's fan-in (issue #770): the partials whose
|
|
236
|
-
* key ranges overlap, and the key ranges its merge units run over, as the
|
|
237
|
-
* hashes of the range blobs the units take as their input.
|
|
238
|
-
*
|
|
239
|
-
* @remarks
|
|
240
|
-
* The ranges are a pure function of the partials and the task's
|
|
241
|
-
* `targetPartitionBytes` — the boundary keys are fences of the largest
|
|
242
|
-
* partial — so a re-run whose partials are the same objects plans the same
|
|
243
|
-
* ranges; recording them lets it skip the planning probes, and because
|
|
244
|
-
* every unit's inputs are then the same objects, every merge unit
|
|
245
|
-
* cache-hits.
|
|
246
|
-
*/
|
|
247
|
-
export declare const MergeRangePlanType: StructType<{
|
|
248
|
-
/** The component's partial hashes, in partition order. */
|
|
249
|
-
readonly partials: ArrayType<StringType>;
|
|
250
|
-
/** The hashes of the component's range blobs — `Struct{from: Option<K>,
|
|
251
|
-
* to: Option<K>}` over the output's key type — in key order; one open
|
|
252
|
-
* range for a component that merges whole. */
|
|
253
|
-
readonly ranges: ArrayType<StringType>;
|
|
254
|
-
}>;
|
|
255
|
-
export type MergeRangePlanType = typeof MergeRangePlanType;
|
|
256
|
-
export type MergeRangePlan = ValueTypeOf<typeof MergeRangePlanType>;
|
|
257
|
-
/**
|
|
258
|
-
* The plan of a partitioned execution (issue #770): the partitioned inputs,
|
|
259
|
-
* where each partition starts in every one of them, the carved slices, and
|
|
260
|
-
* the ranged fan-in of each merged component.
|
|
261
|
-
*
|
|
262
|
-
* @remarks
|
|
263
|
-
* The step interpreter writes it to the object store as the run goes —
|
|
264
|
-
* after the map step with the carved slices, and again after the reduce
|
|
265
|
-
* step has planned each component's merge ranges — and points the `plan`
|
|
266
|
-
* sidecar of the execution's `(taskHash, inputsHash)` directory at it, so a
|
|
267
|
-
* re-plan or a resume that computes the same `partitions`/`boundaries`/
|
|
268
|
-
* `splits` reuses the slices instead of carving them again, partition by
|
|
269
|
-
* partition, and a component of the same partials reuses its ranges.
|
|
270
|
-
*/
|
|
271
|
-
export declare const PartitionPlanType: StructType<{
|
|
272
|
-
/** Partitioned input hashes, wire order. */
|
|
273
|
-
readonly partitions: ArrayType<StringType>;
|
|
274
|
-
/** First segment index of each partition of the primary; boundaries[0] = 0. */
|
|
275
|
-
readonly boundaries: ArrayType<IntegerType>;
|
|
276
|
-
/** Per secondary (partitions[1..]): the split point of every partition plus the end; length boundaries.length + 1. */
|
|
277
|
-
readonly splits: ArrayType<ArrayType<StructType<{
|
|
278
|
-
readonly seg: IntegerType;
|
|
279
|
-
readonly offset: IntegerType;
|
|
280
|
-
}>>>;
|
|
281
|
-
/** slices[input][partition] object hashes; empty until carved. */
|
|
282
|
-
readonly slices: ArrayType<ArrayType<StringType>>;
|
|
283
|
-
/** The ranged fan-in of each merged component, once its ranges are
|
|
284
|
-
* planned. Appended LAST (BEAST2 encodes struct fields positionally) with
|
|
285
|
-
* a dual decoder — see {@link decodePartitionPlan}. */
|
|
286
|
-
readonly merges: ArrayType<StructType<{
|
|
287
|
-
/** The component's partial hashes, in partition order. */
|
|
288
|
-
readonly partials: ArrayType<StringType>;
|
|
289
|
-
/** The hashes of the component's range blobs — `Struct{from: Option<K>,
|
|
290
|
-
* to: Option<K>}` over the output's key type — in key order; one open
|
|
291
|
-
* range for a component that merges whole. */
|
|
292
|
-
readonly ranges: ArrayType<StringType>;
|
|
293
|
-
}>>;
|
|
294
|
-
}>;
|
|
295
|
-
export type PartitionPlanType = typeof PartitionPlanType;
|
|
296
|
-
export type PartitionPlan = ValueTypeOf<typeof PartitionPlanType>;
|
|
297
|
-
/** Encode a {@link PartitionPlanType} value for the object store. */
|
|
298
|
-
export declare const encodePartitionPlan: (value: PartitionPlan) => Uint8Array;
|
|
299
|
-
/**
|
|
300
|
-
* Decode a {@link PartitionPlanType} object, tolerating the pre-`merges`
|
|
301
|
-
* wire shape (dual-decode migration): an older plan decodes with no recorded
|
|
302
|
-
* merge ranges, so its partition slices are reused and its ranges planned.
|
|
303
|
-
*
|
|
304
|
-
* @param data - the plan object's bytes
|
|
305
|
-
* @returns the decoded plan
|
|
306
|
-
*/
|
|
307
|
-
export declare function decodePartitionPlan(data: Uint8Array): PartitionPlan;
|
|
308
|
-
/**
|
|
309
|
-
* The shape a partition task's `by` projection reads, as extracted from its
|
|
310
|
-
* IR by {@link partitionProjectionShape}.
|
|
311
|
-
*
|
|
312
|
-
* - `fields`: the identity (`names: []`), or a leading prefix of the key's
|
|
313
|
-
* top-level fields — one field, or a struct literal of fields in declared
|
|
314
|
-
* order. Validated against each dataset's top-level key field order.
|
|
315
|
-
* - `path`: a nested leading-field path (`key.a.b`, two or more steps).
|
|
316
|
-
* Validated per step: each must read the FIRST field of its level's
|
|
317
|
-
* struct, which is what keeps the projection monotone in canonical key
|
|
318
|
-
* order.
|
|
319
|
-
*/
|
|
320
|
-
export interface ProjectionShape {
|
|
321
|
-
/** Top-level fields (or the identity), or a nested first-field path. */
|
|
322
|
-
kind: 'fields' | 'path';
|
|
323
|
-
/** The fields read, in order; empty for the identity. */
|
|
324
|
-
names: string[];
|
|
325
|
-
}
|
|
326
|
-
/**
|
|
327
|
-
* Extracts the shape a `by` projection reads — see {@link ProjectionShape} —
|
|
328
|
-
* or `null` for any other (unaccepted) shape.
|
|
329
|
-
*
|
|
330
|
-
* @param ir - the projection's function IR, `(Key) -> Projection`
|
|
331
|
-
* @returns the shape, or `null` when the body is not an accepted projection
|
|
332
|
-
*/
|
|
333
|
-
export declare function partitionProjectionShape(ir: FunctionIR): ProjectionShape | null;
|
|
334
|
-
/**
|
|
335
|
-
* The East type of a projected key — the type {@link projectKey} values have,
|
|
336
|
-
* and the one to build their comparator for: the key type itself for the
|
|
337
|
-
* identity, a struct of the named leading fields in declared order, or the
|
|
338
|
-
* type at the end of a nested first-field path.
|
|
339
|
-
*
|
|
340
|
-
* @param shape - the projection's shape
|
|
341
|
-
* @param keyType - the partitioned dataset's key type
|
|
342
|
-
* @returns the projected key type
|
|
343
|
-
* @throws {Error} When the shape reads a field the key type does not have.
|
|
344
|
-
*
|
|
345
|
-
* @example
|
|
346
|
-
* ```ts
|
|
347
|
-
* const cmp = compareFor(projectedKeyType(shape, keyTypeValue));
|
|
348
|
-
* cmp(projectKey(shape, a), projectKey(shape, b));
|
|
349
|
-
* ```
|
|
350
|
-
*/
|
|
351
|
-
export declare function projectedKeyType(shape: ProjectionShape, keyType: EastTypeValue): EastTypeValue;
|
|
352
|
-
/**
|
|
353
|
-
* Projects a partition key through a `by` shape, without compiling the
|
|
354
|
-
* projection: the key itself for the identity, a struct of the named leading
|
|
355
|
-
* fields built field by field in declared order, or the value at the end of a
|
|
356
|
-
* nested first-field path.
|
|
357
|
-
*
|
|
358
|
-
* @param shape - the projection's shape
|
|
359
|
-
* @param key - a key of the partitioned dataset
|
|
360
|
-
* @returns the projected key, a value of {@link projectedKeyType}
|
|
361
|
-
*/
|
|
362
|
-
export declare function projectKey(shape: ProjectionShape, key: unknown): unknown;
|
|
363
255
|
//# sourceMappingURL=task.d.ts.map
|
package/dist/src/task.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH
|
|
1
|
+
{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAmB,MAAM,eAAe,CAAC;AAInI;qBACqB;AACrB,eAAO,MAAM,gBAAgB,UAAU,CAAC;AAExC;;;;;;;;GAQG;AACH,eAAO,MAAM,YAAY;;;;;;;EAGvB,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,YAAY,CAAC;AAC/C,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,iBAAiB;;EAA4C,CAAC;AAC3E,MAAM,MAAM,iBAAiB,GAAG,OAAO,iBAAiB,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE,yEAAyE;AACzE,eAAO,MAAM,aAAa;;;;;;;EAGxB,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,aAAa,CAAC;AACjD,MAAM,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,aAAa,CAAC,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;EAM7B,CAAC;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAC3D,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEpE,sDAAsD;AACtD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;EAGzB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AACnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAE5D;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;EAK3B,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,gBAAgB,CAAC;AACvD,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEhE;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;;;EAGvB,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,YAAY,CAAC;AAC/C,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,cAAc;IACzB,uCAAuC;;IAEvC,0BAA0B;;;;;;;;;IAE1B;gDAC4C;;;;;;;;;;;;;;;IAE5C,kEAAkE;;;;;;;;;IAElE,iDAAiD;;;;;;;;;;;;;;;;;;IAEjD,8BAA8B;;;;;;;;;;;;;;IAE9B;uCACmC;;EAEnC,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAEnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAI5D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAc7D"}
|