@elaraai/e3-types 1.0.79 → 1.0.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +1 -1
  2. package/dist/src/api.d.ts +829 -36
  3. package/dist/src/api.d.ts.map +1 -1
  4. package/dist/src/api.js +301 -32
  5. package/dist/src/api.js.map +1 -1
  6. package/dist/src/collection-manifest.d.ts +37 -0
  7. package/dist/src/collection-manifest.d.ts.map +1 -0
  8. package/dist/src/collection-manifest.js +37 -0
  9. package/dist/src/collection-manifest.js.map +1 -0
  10. package/dist/src/dataflow.d.ts +264 -19
  11. package/dist/src/dataflow.d.ts.map +1 -1
  12. package/dist/src/dataflow.js +156 -15
  13. package/dist/src/dataflow.js.map +1 -1
  14. package/dist/src/dataflow.spec.js +184 -27
  15. package/dist/src/dataflow.spec.js.map +1 -1
  16. package/dist/src/dataset-blob.d.ts +75 -15
  17. package/dist/src/dataset-blob.d.ts.map +1 -1
  18. package/dist/src/dataset-blob.js +95 -28
  19. package/dist/src/dataset-blob.js.map +1 -1
  20. package/dist/src/dataset-blob.spec.d.ts +6 -0
  21. package/dist/src/dataset-blob.spec.d.ts.map +1 -0
  22. package/dist/src/dataset-blob.spec.js +92 -0
  23. package/dist/src/dataset-blob.spec.js.map +1 -0
  24. package/dist/src/dataset-ref.d.ts +5 -4
  25. package/dist/src/dataset-ref.d.ts.map +1 -1
  26. package/dist/src/dataset-ref.js +5 -4
  27. package/dist/src/dataset-ref.js.map +1 -1
  28. package/dist/src/environment.d.ts +3 -2
  29. package/dist/src/environment.d.ts.map +1 -1
  30. package/dist/src/environment.js +3 -2
  31. package/dist/src/environment.js.map +1 -1
  32. package/dist/src/execution.d.ts +112 -18
  33. package/dist/src/execution.d.ts.map +1 -1
  34. package/dist/src/execution.js +174 -45
  35. package/dist/src/execution.js.map +1 -1
  36. package/dist/src/execution.spec.d.ts +6 -0
  37. package/dist/src/execution.spec.d.ts.map +1 -0
  38. package/dist/src/execution.spec.js +108 -0
  39. package/dist/src/execution.spec.js.map +1 -0
  40. package/dist/src/function.d.ts +14 -12
  41. package/dist/src/function.d.ts.map +1 -1
  42. package/dist/src/function.js +17 -32
  43. package/dist/src/function.js.map +1 -1
  44. package/dist/src/index.d.ts +17 -12
  45. package/dist/src/index.d.ts.map +1 -1
  46. package/dist/src/index.js +28 -20
  47. package/dist/src/index.js.map +1 -1
  48. package/dist/src/intake.d.ts +75 -0
  49. package/dist/src/intake.d.ts.map +1 -0
  50. package/dist/src/intake.js +46 -0
  51. package/dist/src/intake.js.map +1 -0
  52. package/dist/src/lock.d.ts +253 -21
  53. package/dist/src/lock.d.ts.map +1 -1
  54. package/dist/src/lock.js +55 -21
  55. package/dist/src/lock.js.map +1 -1
  56. package/dist/src/names.d.ts +21 -0
  57. package/dist/src/names.d.ts.map +1 -0
  58. package/dist/src/names.js +31 -0
  59. package/dist/src/names.js.map +1 -0
  60. package/dist/src/package.d.ts +11 -78
  61. package/dist/src/package.d.ts.map +1 -1
  62. package/dist/src/package.js +13 -66
  63. package/dist/src/package.js.map +1 -1
  64. package/dist/src/record.d.ts +582 -18
  65. package/dist/src/record.d.ts.map +1 -1
  66. package/dist/src/record.js +405 -7
  67. package/dist/src/record.js.map +1 -1
  68. package/dist/src/{stream.spec.d.ts → record.spec.d.ts} +1 -1
  69. package/dist/src/record.spec.d.ts.map +1 -0
  70. package/dist/src/record.spec.js +143 -0
  71. package/dist/src/record.spec.js.map +1 -0
  72. package/dist/src/release.d.ts +36 -0
  73. package/dist/src/release.d.ts.map +1 -0
  74. package/dist/src/release.js +71 -0
  75. package/dist/src/release.js.map +1 -0
  76. package/dist/src/release.spec.d.ts +6 -0
  77. package/dist/src/release.spec.d.ts.map +1 -0
  78. package/dist/src/release.spec.js +42 -0
  79. package/dist/src/release.spec.js.map +1 -0
  80. package/dist/src/repository.d.ts +64 -0
  81. package/dist/src/repository.d.ts.map +1 -0
  82. package/dist/src/repository.js +49 -0
  83. package/dist/src/repository.js.map +1 -0
  84. package/dist/src/runner.d.ts +16 -62
  85. package/dist/src/runner.d.ts.map +1 -1
  86. package/dist/src/runner.js +16 -79
  87. package/dist/src/runner.js.map +1 -1
  88. package/dist/src/runner.spec.js +11 -85
  89. package/dist/src/runner.spec.js.map +1 -1
  90. package/dist/src/structure.d.ts +0 -57
  91. package/dist/src/structure.d.ts.map +1 -1
  92. package/dist/src/structure.js +0 -3
  93. package/dist/src/structure.js.map +1 -1
  94. package/dist/src/task.d.ts +210 -318
  95. package/dist/src/task.d.ts.map +1 -1
  96. package/dist/src/task.js +117 -472
  97. package/dist/src/task.js.map +1 -1
  98. package/dist/src/task.spec.js +62 -154
  99. package/dist/src/task.spec.js.map +1 -1
  100. package/dist/src/transfer.d.ts +38 -26
  101. package/dist/src/transfer.d.ts.map +1 -1
  102. package/dist/src/transfer.js +22 -26
  103. package/dist/src/transfer.js.map +1 -1
  104. package/dist/src/transfer.spec.js +4 -34
  105. package/dist/src/transfer.spec.js.map +1 -1
  106. package/dist/src/unit-plan.d.ts +130 -0
  107. package/dist/src/unit-plan.d.ts.map +1 -0
  108. package/dist/src/unit-plan.js +107 -0
  109. package/dist/src/unit-plan.js.map +1 -0
  110. package/dist/src/unit-plan.spec.d.ts +6 -0
  111. package/dist/src/unit-plan.spec.d.ts.map +1 -0
  112. package/dist/src/unit-plan.spec.js +29 -0
  113. package/dist/src/unit-plan.spec.js.map +1 -0
  114. package/dist/src/workspace.d.ts +22 -11
  115. package/dist/src/workspace.d.ts.map +1 -1
  116. package/dist/src/workspace.js +10 -11
  117. package/dist/src/workspace.js.map +1 -1
  118. package/package.json +3 -3
  119. package/src/api.ts +326 -32
  120. package/src/collection-manifest.ts +53 -0
  121. package/src/dataflow.spec.ts +223 -31
  122. package/src/dataflow.ts +183 -26
  123. package/src/dataset-blob.spec.ts +117 -0
  124. package/src/dataset-blob.ts +157 -21
  125. package/src/dataset-ref.ts +5 -4
  126. package/src/environment.ts +3 -2
  127. package/src/execution.spec.ts +114 -0
  128. package/src/execution.ts +175 -51
  129. package/src/function.ts +19 -32
  130. package/src/index.ts +141 -39
  131. package/src/intake.ts +53 -0
  132. package/src/lock.ts +67 -21
  133. package/src/names.ts +39 -0
  134. package/src/package.ts +15 -71
  135. package/src/record.spec.ts +195 -0
  136. package/src/record.ts +489 -7
  137. package/src/release.spec.ts +47 -0
  138. package/src/release.ts +69 -0
  139. package/src/repository.ts +67 -0
  140. package/src/runner.spec.ts +11 -94
  141. package/src/runner.ts +16 -80
  142. package/src/structure.ts +0 -8
  143. package/src/task.spec.ts +67 -188
  144. package/src/task.ts +140 -522
  145. package/src/transfer.spec.ts +4 -51
  146. package/src/transfer.ts +22 -26
  147. package/src/unit-plan.spec.ts +31 -0
  148. package/src/unit-plan.ts +121 -0
  149. package/src/workspace.ts +13 -11
  150. package/dist/src/stream.d.ts +0 -97
  151. package/dist/src/stream.d.ts.map +0 -1
  152. package/dist/src/stream.js +0 -134
  153. package/dist/src/stream.js.map +0 -1
  154. package/dist/src/stream.spec.d.ts.map +0 -1
  155. package/dist/src/stream.spec.js +0 -60
  156. package/dist/src/stream.spec.js.map +0 -1
  157. package/src/stream.spec.ts +0 -88
  158. package/src/stream.ts +0 -170
@@ -5,75 +5,184 @@
5
5
  /**
6
6
  * Task object types for e3.
7
7
  *
8
- * A task object defines a complete executable unit: the command IR that
9
- * generates the exec args, where to read inputs from, and where to write output.
8
+ * A task object says what a task runs, on which runtime, over which datasets,
9
+ * and what its output is: every field a mode used to be implied by is typed.
10
+ * Task objects are stored in the object store and referenced by packages, so
11
+ * they are content-addressed, which is what memoizes their executions.
10
12
  *
11
- * Task objects are stored in the object store and referenced by packages.
12
- * They are content-addressed, enabling deduplication and memoization.
13
+ * Input and output types are not stored here: they are the package structure's
14
+ * at the paths the task names.
15
+ */
16
+ import { StructType, StringType, ArrayType, NullType, OptionType, VariantType, ValueTypeOf } from '@elaraai/east';
17
+ /** The kind tag every task object carries, which the garbage collector
18
+ * dispatches on. */
19
+ export declare const TASK_OBJECT_KIND = "$task";
20
+ /**
21
+ * What a task runs.
13
22
  *
14
- * Input and output types are inferred from the package's structure at the
15
- * specified paths - the task just references locations, not types.
23
+ * - `east`: an East program, which a stock runner executes as a unit
24
+ * (`exec`): `program` is the hash of its IR bundle.
25
+ * - `command`: a command, which the custom runtime spawns. `commandIr` is the
26
+ * hash of the IR bundle of the East function that builds its argv,
27
+ * `(inputs: Array<String>, output: String) -> Array<String>`.
28
+ */
29
+ export declare const TaskBodyType: VariantType<{
30
+ readonly east: StructType<{
31
+ readonly program: StringType;
32
+ }>;
33
+ readonly command: StructType<{
34
+ readonly commandIr: StringType;
35
+ }>;
36
+ }>;
37
+ export type TaskBodyType = typeof TaskBodyType;
38
+ export type TaskBody = ValueTypeOf<typeof TaskBodyType>;
39
+ /**
40
+ * An input the work may be split over: `by` names the leading key fields
41
+ * whose equal values are never split across pieces — a field, or a
42
+ * first-field path written `a.b`. Empty when any key may start a piece.
43
+ */
44
+ export declare const TaskPartitionType: StructType<{
45
+ readonly by: ArrayType<StringType>;
46
+ }>;
47
+ export type TaskPartitionType = typeof TaskPartitionType;
48
+ export type TaskPartition = ValueTypeOf<typeof TaskPartitionType>;
49
+ /** A dataset a task reads, and whether the work may be split over it. */
50
+ export declare const TaskInputType: StructType<{
51
+ readonly path: ArrayType<VariantType<{
52
+ readonly field: StringType;
53
+ }>>;
54
+ readonly partition: OptionType<StructType<{
55
+ readonly by: ArrayType<StringType>;
56
+ }>>;
57
+ }>;
58
+ export type TaskInputType = typeof TaskInputType;
59
+ export type TaskInput = ValueTypeOf<typeof TaskInputType>;
60
+ /**
61
+ * How a task's output is made, which fixes how the parts of it that units
62
+ * write combine.
63
+ *
64
+ * - `value`: the program returns it.
65
+ * - `array`: emitted elements, concatenated in emission order.
66
+ * - `set`: emitted elements, in any order, unioned.
67
+ * - `dict`: emitted entries, in any order; equal keys fold with `merge`, the
68
+ * hash of a `(K, V, V) -> V` IR bundle, and without it are refused.
69
+ * - `fold`: emitted values folded with `combine`, the hash of a `(T, T) -> T`
70
+ * IR bundle, starting at `zero`, the hash of a stored value.
16
71
  */
17
- import { StructType, StringType, ArrayType, BlobType, BooleanType, IntegerType, OptionType, ValueTypeOf } from '@elaraai/east';
18
- import type { EastTypeValue, FunctionIR } from '@elaraai/east';
72
+ export declare const TaskOutputKindType: VariantType<{
73
+ readonly value: NullType;
74
+ readonly array: NullType;
75
+ readonly set: NullType;
76
+ readonly dict: StructType<{
77
+ readonly merge: OptionType<StringType>;
78
+ }>;
79
+ readonly fold: StructType<{
80
+ readonly zero: StringType;
81
+ readonly combine: StringType;
82
+ }>;
83
+ }>;
84
+ export type TaskOutputKindType = typeof TaskOutputKindType;
85
+ export type TaskOutputKind = ValueTypeOf<typeof TaskOutputKindType>;
86
+ /** Where a task's output goes, and how it is made. */
87
+ export declare const TaskOutputType: StructType<{
88
+ readonly path: ArrayType<VariantType<{
89
+ readonly field: StringType;
90
+ }>>;
91
+ readonly kind: VariantType<{
92
+ readonly value: NullType;
93
+ readonly array: NullType;
94
+ readonly set: NullType;
95
+ readonly dict: StructType<{
96
+ readonly merge: OptionType<StringType>;
97
+ }>;
98
+ readonly fold: StructType<{
99
+ readonly zero: StringType;
100
+ readonly combine: StringType;
101
+ }>;
102
+ }>;
103
+ }>;
104
+ export type TaskOutputType = typeof TaskOutputType;
105
+ export type TaskOutput = ValueTypeOf<typeof TaskOutputType>;
106
+ /**
107
+ * What a ui task binds as it renders.
108
+ *
109
+ * @property paths - Dataset paths it reads or writes through `Data.bind`,
110
+ * including each bound record's own path, so the record's current value is
111
+ * preloaded and polled like any dataset.
112
+ * @property functions - Package functions it calls through `Func.bind`.
113
+ * @property records - Records it binds through `Record.bind`.
114
+ * @property pages - Dataset paths it reads a window at a time through
115
+ * `Data.bindPaged`: declared apart from `paths` because they are never
116
+ * preloaded or polled whole.
117
+ */
118
+ export declare const DataManifestType: StructType<{
119
+ readonly paths: ArrayType<ArrayType<VariantType<{
120
+ readonly field: StringType;
121
+ }>>>;
122
+ readonly functions: ArrayType<StringType>;
123
+ readonly records: ArrayType<StringType>;
124
+ readonly pages: ArrayType<ArrayType<VariantType<{
125
+ readonly field: StringType;
126
+ }>>>;
127
+ }>;
128
+ export type DataManifestType = typeof DataManifestType;
129
+ export type DataManifest = ValueTypeOf<typeof DataManifestType>;
130
+ /**
131
+ * What a task's output is for: `data`, or a `ui` whose output is a component
132
+ * tree rendered against the datasets, functions and records it binds.
133
+ */
134
+ export declare const TaskRoleType: VariantType<{
135
+ readonly data: NullType;
136
+ readonly ui: StructType<{
137
+ readonly paths: ArrayType<ArrayType<VariantType<{
138
+ readonly field: StringType;
139
+ }>>>;
140
+ readonly functions: ArrayType<StringType>;
141
+ readonly records: ArrayType<StringType>;
142
+ readonly pages: ArrayType<ArrayType<VariantType<{
143
+ readonly field: StringType;
144
+ }>>>;
145
+ }>;
146
+ }>;
147
+ export type TaskRoleType = typeof TaskRoleType;
148
+ export type TaskRole = ValueTypeOf<typeof TaskRoleType>;
19
149
  /**
20
150
  * Task object stored in the object store.
21
151
  *
22
- * A task is a complete executable unit that reads from input dataset paths
23
- * and writes to an output dataset path. The commandIr is evaluated at runtime
24
- * to produce the exec args.
25
- *
26
152
  * @remarks
27
- * - `commandIr`: Hash of East IR object that produces exec args
28
- * - IR signature: (inputs: Array<String>, output: String) -> Array<String>
29
- * - `inputs` are paths to staged input .beast2 files
30
- * - `output` is the path where output should be written
31
- * - Returns array of strings to exec (e.g., ["sh", "-c", "python ..."])
32
- * - `inputs`: Paths to input datasets in the data tree
33
- * - `output`: Path to the output dataset in the data tree
34
- *
35
- * Types are not stored in the task - they are inferred from the package's
36
- * structure at the specified paths. This keeps tasks simple and avoids
37
- * redundant type information.
153
+ * A package-borne wire, so it changes by hard cutover: a package exported by
154
+ * an older SDK is re-exported, and {@link decodeTaskObject} says so.
38
155
  *
39
156
  * @example
40
157
  * ```ts
41
- * import { variant } from '@elaraai/east';
158
+ * import { none, variant } from '@elaraai/east';
42
159
  *
43
- * // Task with command IR that generates: ["sh", "-c", "python script.py <input> <output>"]
44
160
  * const task: TaskObject = {
45
- * commandIr: '5e7a3b...', // hash of compiled IR
46
- * inputs: [
47
- * [variant('field', 'inputs'), variant('field', 'sales')],
48
- * ],
49
- * output: [variant('field', 'tasks'), variant('field', 'train'), variant('field', 'output')],
161
+ * kind: TASK_OBJECT_KIND,
162
+ * body: variant('east', { program: '5e7a3b...' }),
163
+ * runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
164
+ * inputs: [{ path: [variant('field', 'inputs'), variant('field', 'sales')], partition: none }],
165
+ * output: { path: [variant('field', 'tasks'), variant('field', 'totals'), variant('field', 'output')], kind: variant('value', null) },
166
+ * role: variant('data', null),
167
+ * environment: none,
50
168
  * };
51
169
  * ```
52
170
  */
53
171
  export declare const TaskObjectType: StructType<{
54
- /** Hash of East IR that generates exec args: (inputs, output) -> Array<String> */
55
- readonly commandIr: StringType;
56
- /** Input paths: where to read each input dataset from the data tree */
57
- readonly inputs: ArrayType<ArrayType<import("@elaraai/east").VariantType<{
58
- readonly field: StringType;
59
- }>>>;
60
- /** Output path: where to write the output dataset in the data tree */
61
- readonly output: ArrayType<import("@elaraai/east").VariantType<{
62
- readonly field: StringType;
63
- }>>;
64
- /** Task kind: "data" (default), "ui", or future extensions. None for old packages. */
65
- readonly kind: OptionType<StringType>;
66
- /** Opaque extension metadata (beast2-encoded). Interpreted by the kind-specific consumer. */
67
- readonly metadata: OptionType<BlobType>;
68
- /**
69
- * The task's runner, as routing metadata (symmetric with
70
- * FunctionObject.runner). For `custom` runners, `commandIr` remains
71
- * authoritative for execution — the wire command is informational.
72
- *
73
- * NOTE: added as a hard cutover (no dual decoder) — packages exported by
74
- * older SDKs must be re-exported.
75
- */
76
- readonly runner: import("@elaraai/east").VariantType<{
172
+ /** Always {@link TASK_OBJECT_KIND}. */
173
+ readonly kind: StringType;
174
+ /** What the task runs. */
175
+ readonly body: VariantType<{
176
+ readonly east: StructType<{
177
+ readonly program: StringType;
178
+ }>;
179
+ readonly command: StructType<{
180
+ readonly commandIr: StringType;
181
+ }>;
182
+ }>;
183
+ /** The runtime it runs on. A `custom` runner's command is informational:
184
+ * the body's command IR is what spawns. */
185
+ readonly runner: VariantType<{
77
186
  readonly east_node: StructType<{
78
187
  readonly platforms: ArrayType<StringType>;
79
188
  }>;
@@ -87,277 +196,60 @@ export declare const TaskObjectType: StructType<{
87
196
  readonly command: ArrayType<StringType>;
88
197
  }>;
89
198
  }>;
90
- /**
91
- * Hash of an {@link EnvironmentSpecType} object the task executes in;
92
- * `none` ⇒ the stock runtime image. Appended LAST (BEAST2 encodes struct
93
- * fields positionally) with a legacy dual decoder — see
94
- * {@link decodeTaskObject}.
95
- */
199
+ /** The datasets it reads, in the order its program takes them. */
200
+ readonly inputs: ArrayType<StructType<{
201
+ readonly path: ArrayType<VariantType<{
202
+ readonly field: StringType;
203
+ }>>;
204
+ readonly partition: OptionType<StructType<{
205
+ readonly by: ArrayType<StringType>;
206
+ }>>;
207
+ }>>;
208
+ /** Where its output goes, and how it is made. */
209
+ readonly output: StructType<{
210
+ readonly path: ArrayType<VariantType<{
211
+ readonly field: StringType;
212
+ }>>;
213
+ readonly kind: VariantType<{
214
+ readonly value: NullType;
215
+ readonly array: NullType;
216
+ readonly set: NullType;
217
+ readonly dict: StructType<{
218
+ readonly merge: OptionType<StringType>;
219
+ }>;
220
+ readonly fold: StructType<{
221
+ readonly zero: StringType;
222
+ readonly combine: StringType;
223
+ }>;
224
+ }>;
225
+ }>;
226
+ /** What its output is for. */
227
+ readonly role: VariantType<{
228
+ readonly data: NullType;
229
+ readonly ui: StructType<{
230
+ readonly paths: ArrayType<ArrayType<VariantType<{
231
+ readonly field: StringType;
232
+ }>>>;
233
+ readonly functions: ArrayType<StringType>;
234
+ readonly records: ArrayType<StringType>;
235
+ readonly pages: ArrayType<ArrayType<VariantType<{
236
+ readonly field: StringType;
237
+ }>>>;
238
+ }>;
239
+ }>;
240
+ /** Hash of the {@link EnvironmentSpecType} object it executes in; `none`
241
+ * for the stock runtime image. */
96
242
  readonly environment: OptionType<StringType>;
97
243
  }>;
98
244
  export type TaskObjectType = typeof TaskObjectType;
99
245
  export type TaskObject = ValueTypeOf<typeof TaskObjectType>;
100
246
  /**
101
- * Decode a `TaskObject` from BEAST2 bytes, tolerating the pre-`environment`
102
- * wire format (dual-decode migration, like {@link decodePackageObject}).
247
+ * Decode a `TaskObject` from BEAST2 bytes.
103
248
  *
104
- * Every task-read path — local AND cloud — must use this instead of
105
- * `decodeBeast2For(TaskObjectType)` directly. Older bytes decode with
106
- * `environment` defaulted to `none`.
249
+ * @param data - the stored bytes
250
+ * @returns the task object
251
+ * @throws {Error} When the bytes are not a current task object — a package
252
+ * exported by an older SDK, which is re-exported with the current one.
107
253
  */
108
254
  export declare function decodeTaskObject(data: Uint8Array): TaskObject;
109
- /** Task kind of a partitioned task — the orchestrator carves its partitioned
110
- * input(s) into key-range slices, runs each slice as an ordinary
111
- * content-addressed execution, and assembles the output by splice or by
112
- * combining partials. The spec rides {@link TaskObjectType}'s `metadata`
113
- * slot as a beast2-encoded {@link PartitionTaskMetadataType}. */
114
- export declare const TASK_KIND_PARTITION = "partition";
115
- /** Task kind of a streaming task — one execution whose runner feeds the
116
- * stream input lazily and writes the output incrementally through an `emit`
117
- * capability. The spec rides {@link TaskObjectType}'s `metadata` slot as a
118
- * beast2-encoded {@link StreamTaskMetadataType}. */
119
- export declare const TASK_KIND_STREAM = "stream";
120
- /** Task kind of a merge unit — the runner's `merge` command over sorted
121
- * partials of a partitioned task's keyed output (issue #770). The
122
- * orchestrator writes one such task per partitioned task from the
123
- * package's {@link PartitionTaskMetadataType} `mergeCommand`, and runs it
124
- * as an ordinary content-addressed execution per group of partials. It
125
- * carries no metadata. */
126
- export declare const TASK_KIND_MERGE = "merge";
127
- /**
128
- * Metadata of a {@link TASK_KIND_PARTITION} task.
129
- *
130
- * The task's wire `inputs` are laid out `[function_ir, ...partitions,
131
- * ...inputs]`, so `partitions` counts how many entries after the function IR
132
- * are partitioned datasets; the rest are ordinary (broadcast) inputs that
133
- * hash into every partition execution's identity.
134
- *
135
- * `by` and `combine` are carried as `encodeEastIR` bundles (capture-free IR +
136
- * its source map), not as FunctionType values or object-store hashes: IR is
137
- * how executable code travels everywhere on the e3 wire (`commandIr`,
138
- * `function_ir`, `bodyIr`), a FunctionType value could smuggle captures the
139
- * orchestrator must not evaluate, and inline bytes stay reachable where a
140
- * hash inside an opaque metadata blob would be invisible to GC.
141
- */
142
- export declare const PartitionTaskMetadataType: StructType<{
143
- /** Number of partitioned inputs (wire input indices `1..1+partitions`). */
144
- readonly partitions: IntegerType;
145
- /** `encodeEastIR` bundle of the boundary-alignment projection
146
- * `(Key) -> Projection`; `none` when partitioning is free per row/segment. */
147
- readonly by: OptionType<BlobType>;
148
- /** `encodeEastIR` bundle of the associative fold `(Out, Out) -> Out`;
149
- * `none` in splice mode (shards concatenate). */
150
- readonly combine: OptionType<BlobType>;
151
- /** Target carved-slice size in wire bytes — the only sizing knob. */
152
- readonly targetPartitionBytes: IntegerType;
153
- /**
154
- * `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
155
- * for a Dict output; `none` otherwise.
156
- *
157
- * Its presence (or {@link mergeSets}) selects the MERGE-TREE assembly:
158
- * partials whose key ranges overlap are merged by the task's own runner, in
159
- * a tree of merge executions ({@link mergeCommand}) folding equal keys with
160
- * this function; disjoint partials are spliced; the orchestrator never
161
- * decodes a partial. Appended after `targetPartitionBytes` (BEAST2 encodes
162
- * struct fields positionally) with a dual decoder — see
163
- * {@link decodePartitionTaskMetadata}.
164
- */
165
- readonly merge: OptionType<BlobType>;
166
- /** Whether a Set output assembles by the merge tree, keeping one of equal
167
- * elements (`--union`). The Set twin of {@link merge}, which needs no
168
- * function. */
169
- readonly mergeSets: BooleanType;
170
- /**
171
- * `encodeEastIR` bundle of the merge command `(input_paths, output_path) ->
172
- * argv` — `mergeCommandIr` over the task's runner, in `function` mode with
173
- * {@link merge} and `union` mode with {@link mergeSets} — that the
174
- * orchestrator's merge units execute; `none` in splice and combine modes.
175
- * Built at export, so the merge unit task is a package object like any
176
- * other and nothing is synthesized at run time. Appended LAST with a
177
- * triple decoder — see {@link decodePartitionTaskMetadata}.
178
- */
179
- readonly mergeCommand: OptionType<BlobType>;
180
- }>;
181
- export type PartitionTaskMetadataType = typeof PartitionTaskMetadataType;
182
- export type PartitionTaskMetadata = ValueTypeOf<typeof PartitionTaskMetadataType>;
183
- /** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
184
- export declare const encodePartitionTaskMetadata: (value: PartitionTaskMetadata) => Uint8Array;
185
- /**
186
- * Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task,
187
- * tolerating the two older wire formats: the v1.0.77 shape without
188
- * `mergeCommand`, and the pre-`merge` shape.
189
- *
190
- * @param data - the metadata blob
191
- * @returns the decoded metadata, with `mergeCommand` (and, for the oldest
192
- * bytes, `merge`/`mergeSets`) defaulted off
193
- */
194
- export declare function decodePartitionTaskMetadata(data: Uint8Array): PartitionTaskMetadata;
195
- /**
196
- * Metadata of a {@link TASK_KIND_STREAM} task.
197
- *
198
- * The task's wire `inputs` are laid out `[function_ir, merge_ir?, stream?,
199
- * ...inputs]` — `merge_ir` only in `function` merge mode; the compiled body
200
- * takes one trailing `emit` parameter beyond the wire inputs, and the runner
201
- * writes the `-o` file from the emit sink instead of the body's (Null) return
202
- * value.
203
- */
204
- export declare const StreamTaskMetadataType: StructType<{
205
- /** Whether the first input after the IRs is the streamed input (producer
206
- * tasks have no streamed input). */
207
- readonly stream: BooleanType;
208
- /** The output collection kind the emit sink writes: `"array"`, `"set"`, or
209
- * `"dict"` — element/key/value types come from the body IR's emit
210
- * parameter. */
211
- readonly emit: StringType;
212
- /**
213
- * How the emit sink treats equal keys, a `StreamMergeMode`: `"none"` —
214
- * a duplicate key is an error; `"function"` — wire input 1 is the merge IR
215
- * `(K, V, V) -> V`, folding equal Dict keys; `"union"` — equal Set elements
216
- * collapse. Appended LAST (BEAST2 encodes struct fields positionally) with a
217
- * dual decoder — see {@link decodeStreamTaskMetadata}.
218
- */
219
- readonly merge: StringType;
220
- }>;
221
- export type StreamTaskMetadataType = typeof StreamTaskMetadataType;
222
- export type StreamTaskMetadata = ValueTypeOf<typeof StreamTaskMetadataType>;
223
- /** Encode a {@link StreamTaskMetadataType} value for `TaskObject.metadata`. */
224
- export declare const encodeStreamTaskMetadata: (value: StreamTaskMetadata) => Uint8Array;
225
- /**
226
- * Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_STREAM} task,
227
- * tolerating the pre-`merge` wire format (dual-decode migration).
228
- *
229
- * @param data - the metadata blob
230
- * @returns the decoded metadata, with `merge` defaulted to `"none"` for older
231
- * bytes
232
- */
233
- export declare function decodeStreamTaskMetadata(data: Uint8Array): StreamTaskMetadata;
234
- /**
235
- * The plan of one merged component's fan-in (issue #770): the partials whose
236
- * key ranges overlap, and the key ranges its merge units run over, as the
237
- * hashes of the range blobs the units take as their input.
238
- *
239
- * @remarks
240
- * The ranges are a pure function of the partials and the task's
241
- * `targetPartitionBytes` — the boundary keys are fences of the largest
242
- * partial — so a re-run whose partials are the same objects plans the same
243
- * ranges; recording them lets it skip the planning probes, and because
244
- * every unit's inputs are then the same objects, every merge unit
245
- * cache-hits.
246
- */
247
- export declare const MergeRangePlanType: StructType<{
248
- /** The component's partial hashes, in partition order. */
249
- readonly partials: ArrayType<StringType>;
250
- /** The hashes of the component's range blobs — `Struct{from: Option<K>,
251
- * to: Option<K>}` over the output's key type — in key order; one open
252
- * range for a component that merges whole. */
253
- readonly ranges: ArrayType<StringType>;
254
- }>;
255
- export type MergeRangePlanType = typeof MergeRangePlanType;
256
- export type MergeRangePlan = ValueTypeOf<typeof MergeRangePlanType>;
257
- /**
258
- * The plan of a partitioned execution (issue #770): the partitioned inputs,
259
- * where each partition starts in every one of them, the carved slices, and
260
- * the ranged fan-in of each merged component.
261
- *
262
- * @remarks
263
- * The step interpreter writes it to the object store as the run goes —
264
- * after the map step with the carved slices, and again after the reduce
265
- * step has planned each component's merge ranges — and points the `plan`
266
- * sidecar of the execution's `(taskHash, inputsHash)` directory at it, so a
267
- * re-plan or a resume that computes the same `partitions`/`boundaries`/
268
- * `splits` reuses the slices instead of carving them again, partition by
269
- * partition, and a component of the same partials reuses its ranges.
270
- */
271
- export declare const PartitionPlanType: StructType<{
272
- /** Partitioned input hashes, wire order. */
273
- readonly partitions: ArrayType<StringType>;
274
- /** First segment index of each partition of the primary; boundaries[0] = 0. */
275
- readonly boundaries: ArrayType<IntegerType>;
276
- /** Per secondary (partitions[1..]): the split point of every partition plus the end; length boundaries.length + 1. */
277
- readonly splits: ArrayType<ArrayType<StructType<{
278
- readonly seg: IntegerType;
279
- readonly offset: IntegerType;
280
- }>>>;
281
- /** slices[input][partition] object hashes; empty until carved. */
282
- readonly slices: ArrayType<ArrayType<StringType>>;
283
- /** The ranged fan-in of each merged component, once its ranges are
284
- * planned. Appended LAST (BEAST2 encodes struct fields positionally) with
285
- * a dual decoder — see {@link decodePartitionPlan}. */
286
- readonly merges: ArrayType<StructType<{
287
- /** The component's partial hashes, in partition order. */
288
- readonly partials: ArrayType<StringType>;
289
- /** The hashes of the component's range blobs — `Struct{from: Option<K>,
290
- * to: Option<K>}` over the output's key type — in key order; one open
291
- * range for a component that merges whole. */
292
- readonly ranges: ArrayType<StringType>;
293
- }>>;
294
- }>;
295
- export type PartitionPlanType = typeof PartitionPlanType;
296
- export type PartitionPlan = ValueTypeOf<typeof PartitionPlanType>;
297
- /** Encode a {@link PartitionPlanType} value for the object store. */
298
- export declare const encodePartitionPlan: (value: PartitionPlan) => Uint8Array;
299
- /**
300
- * Decode a {@link PartitionPlanType} object, tolerating the pre-`merges`
301
- * wire shape (dual-decode migration): an older plan decodes with no recorded
302
- * merge ranges, so its partition slices are reused and its ranges planned.
303
- *
304
- * @param data - the plan object's bytes
305
- * @returns the decoded plan
306
- */
307
- export declare function decodePartitionPlan(data: Uint8Array): PartitionPlan;
308
- /**
309
- * The shape a partition task's `by` projection reads, as extracted from its
310
- * IR by {@link partitionProjectionShape}.
311
- *
312
- * - `fields`: the identity (`names: []`), or a leading prefix of the key's
313
- * top-level fields — one field, or a struct literal of fields in declared
314
- * order. Validated against each dataset's top-level key field order.
315
- * - `path`: a nested leading-field path (`key.a.b`, two or more steps).
316
- * Validated per step: each must read the FIRST field of its level's
317
- * struct, which is what keeps the projection monotone in canonical key
318
- * order.
319
- */
320
- export interface ProjectionShape {
321
- /** Top-level fields (or the identity), or a nested first-field path. */
322
- kind: 'fields' | 'path';
323
- /** The fields read, in order; empty for the identity. */
324
- names: string[];
325
- }
326
- /**
327
- * Extracts the shape a `by` projection reads — see {@link ProjectionShape} —
328
- * or `null` for any other (unaccepted) shape.
329
- *
330
- * @param ir - the projection's function IR, `(Key) -> Projection`
331
- * @returns the shape, or `null` when the body is not an accepted projection
332
- */
333
- export declare function partitionProjectionShape(ir: FunctionIR): ProjectionShape | null;
334
- /**
335
- * The East type of a projected key — the type {@link projectKey} values have,
336
- * and the one to build their comparator for: the key type itself for the
337
- * identity, a struct of the named leading fields in declared order, or the
338
- * type at the end of a nested first-field path.
339
- *
340
- * @param shape - the projection's shape
341
- * @param keyType - the partitioned dataset's key type
342
- * @returns the projected key type
343
- * @throws {Error} When the shape reads a field the key type does not have.
344
- *
345
- * @example
346
- * ```ts
347
- * const cmp = compareFor(projectedKeyType(shape, keyTypeValue));
348
- * cmp(projectKey(shape, a), projectKey(shape, b));
349
- * ```
350
- */
351
- export declare function projectedKeyType(shape: ProjectionShape, keyType: EastTypeValue): EastTypeValue;
352
- /**
353
- * Projects a partition key through a `by` shape, without compiling the
354
- * projection: the key itself for the identity, a struct of the named leading
355
- * fields built field by field in declared order, or the value at the end of a
356
- * nested first-field path.
357
- *
358
- * @param shape - the projection's shape
359
- * @param key - a key of the partitioned dataset
360
- * @returns the projected key, a value of {@link projectedKeyType}
361
- */
362
- export declare function projectKey(shape: ProjectionShape, key: unknown): unknown;
363
255
  //# sourceMappingURL=task.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAmD,MAAM,eAAe,CAAC;AAChL,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAI/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,cAAc;IACzB,kFAAkF;;IAElF,uEAAuE;;;;IAEvE,sEAAsE;;;;IAEtE,sFAAsF;;IAEtF,6FAA6F;;IAE7F;;;;;;;OAOG;;;;;;;;;;;;;;;IAEH;;;;;OAKG;;EAEH,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAEnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAmB5D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAW7D;AAMD;;;;kEAIkE;AAClE,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAE/C;;;qDAGqD;AACrD,eAAO,MAAM,gBAAgB,WAAW,CAAC;AAEzC;;;;;2BAK2B;AAC3B,eAAO,MAAM,eAAe,UAAU,CAAC;AAEvC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,yBAAyB;IACpC,2EAA2E;;IAE3E;mFAC+E;;IAE/E;sDACkD;;IAElD,qEAAqE;;IAErE;;;;;;;;;;;OAWG;;IAEH;;oBAEgB;;IAEhB;;;;;;;;OAQG;;EAEH,CAAC;AACH,MAAM,MAAM,yBAAyB,GAAG,OAAO,yBAAyB,CAAC;AAEzE,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AA4BlF,kFAAkF;AAClF,eAAO,MAAM,2BAA2B,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,UAChC,CAAC;AAM7C;;;;;;;;GAQG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,UAAU,GAAG,qBAAqB,CAcnF;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB;IACjC;yCACqC;;IAErC;;qBAEiB;;IAEjB;;;;;;OAMG;;EAEH,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,sBAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAY5E,+EAA+E;AAC/E,eAAO,MAAM,wBAAwB,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,UAC7B,CAAC;AAK1C;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,UAAU,GAAG,kBAAkB,CAU7E;AASD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,kBAAkB;IAC7B,0DAA0D;;IAE1D;;mDAE+C;;EAE/C,CAAC;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEpE;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iBAAiB;IAC5B,4CAA4C;;IAE5C,+EAA+E;;IAE/E,sHAAsH;;;;;IAEtH,kEAAkE;;IAElE;;4DAEwD;;QApCxD,0DAA0D;;QAE1D;;uDAE+C;;;EAkC/C,CAAC;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,iBAAiB,CAAC;AAEzD,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAalE,qEAAqE;AACrE,eAAO,MAAM,mBAAmB,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,UACxB,CAAC;AAKrC;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,CAUnE;AAMD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,eAAe;IAC9B,wEAAwE;IACxE,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IACxB,yDAAyD;IACzD,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,UAAU,GAAG,eAAe,GAAG,IAAI,CAoD/E;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,aAAa,GAAG,aAAa,CAW9F;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAQxE"}
1
+ {"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAmB,MAAM,eAAe,CAAC;AAInI;qBACqB;AACrB,eAAO,MAAM,gBAAgB,UAAU,CAAC;AAExC;;;;;;;;GAQG;AACH,eAAO,MAAM,YAAY;;;;;;;EAGvB,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,YAAY,CAAC;AAC/C,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,iBAAiB;;EAA4C,CAAC;AAC3E,MAAM,MAAM,iBAAiB,GAAG,OAAO,iBAAiB,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE,yEAAyE;AACzE,eAAO,MAAM,aAAa;;;;;;;EAGxB,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,aAAa,CAAC;AACjD,MAAM,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,aAAa,CAAC,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;EAM7B,CAAC;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAC3D,MAAM,MAAM,cAAc,GAAG,WAAW,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEpE,sDAAsD;AACtD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;EAGzB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AACnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAE5D;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;EAK3B,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,gBAAgB,CAAC;AACvD,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEhE;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;;;EAGvB,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,YAAY,CAAC;AAC/C,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,cAAc;IACzB,uCAAuC;;IAEvC,0BAA0B;;;;;;;;;IAE1B;gDAC4C;;;;;;;;;;;;;;;IAE5C,kEAAkE;;;;;;;;;IAElE,iDAAiD;;;;;;;;;;;;;;;;;;IAEjD,8BAA8B;;;;;;;;;;;;;;IAE9B;uCACmC;;EAEnC,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAEnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAI5D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAc7D"}