@elaraai/e3 1.0.78 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/dist/src/browser.d.ts +5 -4
- package/dist/src/browser.d.ts.map +1 -1
- package/dist/src/browser.js +3 -2
- package/dist/src/browser.js.map +1 -1
- package/dist/src/dataset-file.d.ts +17 -19
- package/dist/src/dataset-file.d.ts.map +1 -1
- package/dist/src/dataset-file.js +26 -45
- package/dist/src/dataset-file.js.map +1 -1
- package/dist/src/export.d.ts +2 -3
- package/dist/src/export.d.ts.map +1 -1
- package/dist/src/export.js +150 -80
- package/dist/src/export.js.map +1 -1
- package/dist/src/export.spec.js +177 -57
- package/dist/src/export.spec.js.map +1 -1
- package/dist/src/function.spec.js +29 -58
- package/dist/src/function.spec.js.map +1 -1
- package/dist/src/functions-link.spec.js +19 -17
- package/dist/src/functions-link.spec.js.map +1 -1
- package/dist/src/index.d.ts +56 -8
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +11 -4
- package/dist/src/index.js.map +1 -1
- package/dist/src/input.d.ts +8 -5
- package/dist/src/input.d.ts.map +1 -1
- package/dist/src/input.js +8 -5
- package/dist/src/input.js.map +1 -1
- package/dist/src/migration.d.ts +172 -0
- package/dist/src/migration.d.ts.map +1 -0
- package/dist/src/migration.js +108 -0
- package/dist/src/migration.js.map +1 -0
- package/dist/src/migration.spec.d.ts +6 -0
- package/dist/src/migration.spec.d.ts.map +1 -0
- package/dist/src/migration.spec.js +197 -0
- package/dist/src/migration.spec.js.map +1 -0
- package/dist/src/mutation-programs.spec.d.ts +6 -0
- package/dist/src/mutation-programs.spec.d.ts.map +1 -0
- package/dist/src/mutation-programs.spec.js +310 -0
- package/dist/src/mutation-programs.spec.js.map +1 -0
- package/dist/src/mutation.d.ts +117 -9
- package/dist/src/mutation.d.ts.map +1 -1
- package/dist/src/mutation.js +104 -46
- package/dist/src/mutation.js.map +1 -1
- package/dist/src/open-beast-link.spec.js +10 -8
- package/dist/src/open-beast-link.spec.js.map +1 -1
- package/dist/src/output.d.ts +117 -0
- package/dist/src/output.d.ts.map +1 -0
- package/dist/src/output.js +119 -0
- package/dist/src/output.js.map +1 -0
- package/dist/src/package.d.ts +7 -5
- package/dist/src/package.d.ts.map +1 -1
- package/dist/src/package.js +98 -8
- package/dist/src/package.js.map +1 -1
- package/dist/src/package.spec.d.ts +6 -0
- package/dist/src/package.spec.d.ts.map +1 -0
- package/dist/src/package.spec.js +16 -0
- package/dist/src/package.spec.js.map +1 -0
- package/dist/src/record-guards.d.ts +41 -0
- package/dist/src/record-guards.d.ts.map +1 -0
- package/dist/src/record-guards.js +44 -0
- package/dist/src/record-guards.js.map +1 -0
- package/dist/src/record-index.d.ts +129 -0
- package/dist/src/record-index.d.ts.map +1 -0
- package/dist/src/record-index.js +144 -0
- package/dist/src/record-index.js.map +1 -0
- package/dist/src/record-index.spec.d.ts +6 -0
- package/dist/src/record-index.spec.d.ts.map +1 -0
- package/dist/src/record-index.spec.js +160 -0
- package/dist/src/record-index.spec.js.map +1 -0
- package/dist/src/record-programs.d.ts +128 -0
- package/dist/src/record-programs.d.ts.map +1 -0
- package/dist/src/record-programs.js +480 -0
- package/dist/src/record-programs.js.map +1 -0
- package/dist/src/record.d.ts.map +1 -1
- package/dist/src/record.js +2 -0
- package/dist/src/record.js.map +1 -1
- package/dist/src/record.spec.js +71 -13
- package/dist/src/record.spec.js.map +1 -1
- package/dist/src/runner.d.ts +6 -9
- package/dist/src/runner.d.ts.map +1 -1
- package/dist/src/runner.js +9 -25
- package/dist/src/runner.js.map +1 -1
- package/dist/src/runner.spec.js +16 -34
- package/dist/src/runner.spec.js.map +1 -1
- package/dist/src/sha256.d.ts +4 -3
- package/dist/src/sha256.d.ts.map +1 -1
- package/dist/src/sha256.js +20 -17
- package/dist/src/sha256.js.map +1 -1
- package/dist/src/sha256.spec.js +10 -0
- package/dist/src/sha256.spec.js.map +1 -1
- package/dist/src/task.d.ts +116 -251
- package/dist/src/task.d.ts.map +1 -1
- package/dist/src/task.js +169 -435
- package/dist/src/task.js.map +1 -1
- package/dist/src/task.spec.js +174 -489
- package/dist/src/task.spec.js.map +1 -1
- package/dist/src/types.d.ts +215 -23
- package/dist/src/types.d.ts.map +1 -1
- package/package.json +4 -4
package/dist/src/task.d.ts
CHANGED
|
@@ -5,13 +5,16 @@
|
|
|
5
5
|
/**
|
|
6
6
|
* Task definitions for e3 packages.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* A task reads datasets and writes one output, the dataset at
|
|
9
|
+
* `.tasks.${name}.output`: {@link task} returns it, {@link streamTask} emits
|
|
10
|
+
* it into an output kind, and {@link customTask} runs a command that writes
|
|
11
|
+
* it. The program a task runs is named by the task object export writes, not
|
|
12
|
+
* held in the data tree.
|
|
11
13
|
*/
|
|
12
|
-
import type { AsyncFunctionExpr, BlockBuilder, CallableAsyncFunctionExpr, CallableFunctionExpr,
|
|
13
|
-
import { Expr, variant, ArrayType, StringType, NullType
|
|
14
|
-
import type {
|
|
14
|
+
import type { AsyncFunctionExpr, BlockBuilder, CallableAsyncFunctionExpr, CallableFunctionExpr, EastType, ExprType, FunctionExpr } from '@elaraai/east';
|
|
15
|
+
import { Expr, variant, ArrayType, StringType, NullType } from '@elaraai/east';
|
|
16
|
+
import type { TaskRole } from '@elaraai/e3-types';
|
|
17
|
+
import type { DatasetDef, DataTreeDef, OutputDef, PartitionDef, TaskDef } from './types.js';
|
|
15
18
|
import { type Runner } from './runner.js';
|
|
16
19
|
import { type EnvironmentDecl } from './environment.js';
|
|
17
20
|
/**
|
|
@@ -21,6 +24,8 @@ import { type EnvironmentDecl } from './environment.js';
|
|
|
21
24
|
type ExtractDatasetTypes<T extends readonly DatasetDef[]> = {
|
|
22
25
|
[K in keyof T]: T[K] extends DatasetDef<infer U> ? U : never;
|
|
23
26
|
} & EastType[];
|
|
27
|
+
/** The task-output path tuple for task `Name`. */
|
|
28
|
+
type TaskOutputPath<Name extends string> = [variant<'field', 'tasks'>, variant<'field', Name>, variant<'field', 'output'>];
|
|
24
29
|
/**
|
|
25
30
|
* Singleton tree definition for `.tasks`.
|
|
26
31
|
*
|
|
@@ -31,10 +36,11 @@ export declare const tasksTree: DataTreeDef;
|
|
|
31
36
|
* Defines a task that runs an East function to produce an output dataset.
|
|
32
37
|
*
|
|
33
38
|
* Tasks read from input datasets and produce an output dataset.
|
|
34
|
-
* When input datasets change, the task re-runs automatically.
|
|
39
|
+
* When input datasets change, the task re-runs automatically. The function
|
|
40
|
+
* runs once, as one unit: a large input opens lazily, and a collection it
|
|
41
|
+
* returns is written segment by segment.
|
|
35
42
|
*
|
|
36
43
|
* Task structure:
|
|
37
|
-
* - `.tasks.${name}.function_ir` - The compiled IR (private)
|
|
38
44
|
* - `.tasks.${name}.output` - The output dataset
|
|
39
45
|
*
|
|
40
46
|
* @typeParam Name - Task name (literal type)
|
|
@@ -43,8 +49,15 @@ export declare const tasksTree: DataTreeDef;
|
|
|
43
49
|
* @param name - Task name
|
|
44
50
|
* @param inputs - Input datasets to read from
|
|
45
51
|
* @param fn - Implementation function
|
|
52
|
+
* @param config - The runner the function runs on (default
|
|
53
|
+
* {@link DEFAULT_RUNNER}), the environment it runs in, and its role: a data
|
|
54
|
+
* task (the default), or a UI task with the data its surface reads
|
|
46
55
|
* @returns A TaskDef with `.output` for chaining
|
|
56
|
+
* @throws {Error} When an input is marked with `e3.partition`, which only
|
|
57
|
+
* {@link streamTask} takes.
|
|
47
58
|
*
|
|
59
|
+
* @see {@link streamTask} for an output emitted instead of returned, and work
|
|
60
|
+
* split over an input.
|
|
48
61
|
* @see {@link customTask} for defining tasks with custom command logic (e.g. performing non-East operations).
|
|
49
62
|
*
|
|
50
63
|
* @example
|
|
@@ -67,281 +80,133 @@ export declare const tasksTree: DataTreeDef;
|
|
|
67
80
|
*/
|
|
68
81
|
export declare function task<Name extends string, Inputs extends readonly DatasetDef[], Output extends EastType>(name: Name, inputs: [...Inputs], fn: FunctionExpr<ExtractDatasetTypes<Inputs>, Output> | CallableFunctionExpr<ExtractDatasetTypes<Inputs>, Output> | AsyncFunctionExpr<ExtractDatasetTypes<Inputs>, Output> | CallableAsyncFunctionExpr<ExtractDatasetTypes<Inputs>, Output>, config?: {
|
|
69
82
|
runner?: Runner;
|
|
70
|
-
kind?: string;
|
|
71
|
-
metadata?: Uint8Array;
|
|
72
|
-
environment?: EnvironmentDecl;
|
|
73
|
-
}): TaskDef<Output, [variant<'field', 'tasks'>, variant<'field', Name>, variant<'field', 'output'>]>;
|
|
74
|
-
export declare function customTask<Name extends string, Inputs extends Array<DatasetDef>, Output extends EastType>(name: Name, inputs: Inputs, outputType: Output, command: ($: BlockBuilder<StringType>, input_paths: ExprType<ArrayType<StringType>>, output_path: ExprType<StringType>) => Expr<StringType> | void, config?: {
|
|
75
83
|
environment?: EnvironmentDecl;
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
type TaskOutputPath<Name extends string> = [variant<'field', 'tasks'>, variant<'field', Name>, variant<'field', 'output'>];
|
|
79
|
-
/** The expression parameters a body callback receives for a dataset tuple. */
|
|
80
|
-
type DatasetExprs<T extends readonly DatasetDef[]> = {
|
|
81
|
-
[K in keyof T]: T[K] extends DatasetDef<infer U> ? ExprType<U> : never;
|
|
82
|
-
};
|
|
83
|
-
/** The key type a partitioned collection is carved by: Dict keys, Set
|
|
84
|
-
* elements, or Array elements. */
|
|
85
|
-
type CollectionKeyType<T> = T extends DictType<infer K, any> ? K : T extends SetType<infer E> ? E : T extends ArrayType<infer E> ? E : never;
|
|
86
|
-
/** The per-dataset key types of a partition tuple. */
|
|
87
|
-
type PartitionKeyTypes<Ps extends readonly DatasetDef[]> = {
|
|
88
|
-
[K in keyof Ps]: Ps[K] extends DatasetDef<infer T> ? CollectionKeyType<T> : never;
|
|
89
|
-
};
|
|
90
|
-
/** Field-wise intersection of two key types: for struct keys, the fields
|
|
91
|
-
* present in both with the identical East type; otherwise the types must
|
|
92
|
-
* agree exactly. */
|
|
93
|
-
type IntersectKeys<A, B> = A extends StructType<infer FA> ? B extends StructType<infer FB> ? StructType<{
|
|
94
|
-
[F in keyof FA & keyof FB as FA[F] extends FB[F] ? (FB[F] extends FA[F] ? F : never) : never]: FA[F];
|
|
95
|
-
}> : never : A extends B ? (B extends A ? A : never) : never;
|
|
96
|
-
/** The strict field-wise intersection of every partitioned dataset's key
|
|
97
|
-
* type — what a co-partitioned `by` projection receives. */
|
|
98
|
-
type PartitionByKey<Ps extends readonly DatasetDef[]> = PartitionKeyTypes<Ps> extends readonly [infer Only] ? Only : PartitionKeyTypes<Ps> extends readonly [infer A, ...infer Rest] ? IntersectKeysAll<A, Rest> : never;
|
|
99
|
-
/** Folds {@link IntersectKeys} over the remaining key types. */
|
|
100
|
-
type IntersectKeysAll<A, Rest> = Rest extends readonly [] ? A : Rest extends readonly [infer B, ...infer More] ? IntersectKeysAll<IntersectKeys<A, B>, More> : never;
|
|
101
|
-
/** What a `by` projection may return: an expression over the key (the
|
|
102
|
-
* identity, a field read, or a nested path — leading-prefix ORDER is
|
|
103
|
-
* validated at build time, since TypeScript types carry no field order), or
|
|
104
|
-
* a struct literal drawn from the key's own fields with their own
|
|
105
|
-
* expression types. Anything else is a type error at the call site. */
|
|
106
|
-
type ByResult<K> = Expr<any> | (K extends StructType<infer F> ? {
|
|
107
|
-
[P in keyof F]?: ExprType<F[P]>;
|
|
108
|
-
} : never);
|
|
109
|
-
/** What `merge` takes for a keyed output: a Dict output folds the values of
|
|
110
|
-
* equal keys with a function of the key and two values, a Set output
|
|
111
|
-
* collapses equal elements with the literal `'union'`, and no other output
|
|
112
|
-
* takes one. */
|
|
113
|
-
type OutputMerge<Output extends EastType> = Output extends DictType<infer K, infer V> ? ($: BlockBuilder<V>, key: ExprType<K>, a: ExprType<V>, b: ExprType<V>) => SubtypeExprOrValue<V> | void : Output extends SetType<any> ? 'union' : never;
|
|
84
|
+
role?: TaskRole;
|
|
85
|
+
}): TaskDef<Output, TaskOutputPath<Name>>;
|
|
114
86
|
/**
|
|
115
|
-
*
|
|
116
|
-
* the
|
|
87
|
+
* Defines a task that runs a command rather than East: a bash script built
|
|
88
|
+
* from the staged input paths and the path the output must be written to.
|
|
117
89
|
*
|
|
118
|
-
* @typeParam
|
|
119
|
-
* @typeParam Inputs -
|
|
120
|
-
* @typeParam Output -
|
|
90
|
+
* @typeParam Name - Task name (literal type)
|
|
91
|
+
* @typeParam Inputs - Input dataset types
|
|
92
|
+
* @typeParam Output - Output type
|
|
93
|
+
* @param name - Task name
|
|
94
|
+
* @param inputs - Input datasets, staged as beast2 files
|
|
95
|
+
* @param outputType - The East type of the beast2 file the command writes
|
|
96
|
+
* @param command - Builds the bash script from the input paths and the output path
|
|
97
|
+
* @param config - The environment the command runs in
|
|
98
|
+
* @returns A TaskDef with `.output` for chaining
|
|
121
99
|
*/
|
|
122
|
-
export
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
* Set roots with a shared key space. */
|
|
126
|
-
readonly partitions: [...Partitions];
|
|
127
|
-
/** Boundary-alignment projection: rows with equal `by(key)` never split
|
|
128
|
-
* across partitions. Must read a leading prefix of every partitioned
|
|
129
|
-
* dataset's key, so aligned runs stay contiguous in canonical order.
|
|
130
|
-
* Accepted shapes (validated at build time): the key itself; a leading
|
|
131
|
-
* field (`key.f`); a nested leading-field path (`key.a.b`, each step the
|
|
132
|
-
* first field of its level); or a struct literal of leading fields in
|
|
133
|
-
* declared order (`{ f1: key.f1, f2: key.f2 }`). Any other body is
|
|
134
|
-
* rejected at build time — semantically monotone projections outside
|
|
135
|
-
* these shapes are deliberately not inferred. The return type admits
|
|
136
|
-
* only expressions over the key or a struct literal of the key's own
|
|
137
|
-
* fields ({@link ByResult}); field ORDER is the build-time half. */
|
|
138
|
-
readonly by?: ($: BlockBuilder<NeverType>, key: ExprType<PartitionByKey<Partitions>>) => ByResult<PartitionByKey<Partitions>>;
|
|
139
|
-
/** Ordinary inputs, passed whole to every partition execution (small ⇒
|
|
140
|
-
* broadcast; huge + indexed ⇒ opened lazily by the runner). Any change to
|
|
141
|
-
* them re-runs all partitions. */
|
|
142
|
-
readonly inputs?: [...Inputs];
|
|
143
|
-
/** The output dataset's East type. */
|
|
144
|
-
readonly output: Output;
|
|
145
|
-
/** Associative fold merging two partials into one. When present, each
|
|
146
|
-
* partition execution returns a partial of the output and partials fold
|
|
147
|
-
* pairwise in a fixed tree order; when absent, each execution returns its
|
|
148
|
-
* shard of the output and the shards splice in partition order. */
|
|
149
|
-
readonly combine?: ($: BlockBuilder<Output>, a: ExprType<Output>, b: ExprType<Output>) => SubtypeExprOrValue<Output> | void;
|
|
150
|
-
/**
|
|
151
|
-
* Per-key resolution for keyed partials that may collide — the third
|
|
152
|
-
* assembly mode, alongside splice and `combine`.
|
|
153
|
-
*
|
|
154
|
-
* @remarks
|
|
155
|
-
* The partials are merged by the task's runner — its `merge` command, over
|
|
156
|
-
* sorted partials, in one pass; the orchestrator never decodes them.
|
|
157
|
-
* Partials whose key ranges overlap are grouped, and each group merges
|
|
158
|
-
* through a tree of merge executions (up to 32 partials each) folding equal
|
|
159
|
-
* keys in partition order; partials no other partial reaches take no merge
|
|
160
|
-
* at all, and the results splice. Every merge unit is an ordinary cached
|
|
161
|
-
* execution: a re-run with unchanged partials merges nothing again, and the
|
|
162
|
-
* intermediate results of the tree are stored like any other execution
|
|
163
|
-
* output. The merge command is built here, at export, and carried in the
|
|
164
|
-
* package's metadata like any other command IR.
|
|
165
|
-
*
|
|
166
|
-
* A `Dict` output takes a function of the key and two of its values, over
|
|
167
|
-
* the output's own key and value types; a `Set` output takes the literal
|
|
168
|
-
* `'union'`. The function must be associative — a key's values may fold in
|
|
169
|
-
* any grouping, though never out of partition order. Mutually exclusive with
|
|
170
|
-
* {@link combine}, which stays the mode for non-collection outputs (KPIs,
|
|
171
|
-
* counts) and for authors who want a whole-value fold.
|
|
172
|
-
*
|
|
173
|
-
* Needs a stock runtime: the merge units run the runner's `merge` command,
|
|
174
|
-
* which the `custom` runtime does not have and an older runner does not
|
|
175
|
-
* ship.
|
|
176
|
-
*/
|
|
177
|
-
readonly merge?: OutputMerge<Output>;
|
|
178
|
-
/** Target carved-slice size in wire bytes. Defaults to 256 MiB. */
|
|
179
|
-
readonly targetPartitionBytes?: number;
|
|
180
|
-
/** Runtime the body runs on; defaults to {@link DEFAULT_RUNNER}. */
|
|
181
|
-
readonly runner?: Runner;
|
|
182
|
-
/** Execution environment declaration, as for {@link task}. */
|
|
183
|
-
readonly environment?: EnvironmentDecl;
|
|
184
|
-
}
|
|
100
|
+
export declare function customTask<Name extends string, Inputs extends Array<DatasetDef>, Output extends EastType>(name: Name, inputs: Inputs, outputType: Output, command: ($: BlockBuilder<StringType>, input_paths: ExprType<ArrayType<StringType>>, output_path: ExprType<StringType>) => Expr<StringType> | void, config?: {
|
|
101
|
+
environment?: EnvironmentDecl;
|
|
102
|
+
}): TaskDef<Output, TaskOutputPath<Name>>;
|
|
185
103
|
/**
|
|
186
|
-
*
|
|
187
|
-
* key-range slices, runs `fn` once per partition as an ordinary
|
|
188
|
-
* content-addressed execution (parallel, memoized per partition), and
|
|
189
|
-
* assembles the output — by splicing shards in partition order, by merging
|
|
190
|
-
* keyed partials on the task's runner when `merge` is given, or by folding
|
|
191
|
-
* partials pairwise when `combine` is given.
|
|
104
|
+
* Marks an input of a {@link streamTask} as one its work may be split over.
|
|
192
105
|
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
* contract is deliberately enforced at SPLICE TIME, not build time: whether
|
|
198
|
-
* an arbitrary body preserves key order is not decidable from types, and a
|
|
199
|
-
* static rule would false-reject permitted monotone re-keys — a violation
|
|
200
|
-
* fails the task deterministically at splice, naming the offending
|
|
201
|
-
* partitions and the remedies (`combine` / `customTask`). With `combine`
|
|
202
|
-
* each execution returns a *partial* and the partials fold.
|
|
106
|
+
* The body receives one piece of the dataset, typed as the whole: a key range
|
|
107
|
+
* of a Set or Dict, or a position range of an Array. Pieces are cut where the
|
|
108
|
+
* content says, so an insertion re-cuts only the pieces around it. Rows whose
|
|
109
|
+
* `by` fields are equal are never split across pieces.
|
|
203
110
|
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
111
|
+
* `by` names leading key fields, in order — `['account']`, or
|
|
112
|
+
* `['account', 'at.day']`, whose last entry reads the first field of `at`. Two
|
|
113
|
+
* or more partitioned inputs are cut at the same keys, so they must be Sets or
|
|
114
|
+
* Dicts whose keys, or whose `by` fields, have the same types. {@link
|
|
115
|
+
* streamTask} checks all of this against the dataset, naming the task.
|
|
207
116
|
*
|
|
208
|
-
* @typeParam
|
|
209
|
-
* @
|
|
210
|
-
* @
|
|
211
|
-
* @
|
|
212
|
-
* @param name - Task name
|
|
213
|
-
* @param spec - The declaration: partitioned inputs, alignment, output type,
|
|
214
|
-
* combine mode, sizing, runner and environment
|
|
215
|
-
* @param fn - Implementation body, run once per partition
|
|
216
|
-
* @returns A TaskDef with `.output` for chaining
|
|
217
|
-
* @throws {Error} When a partitioned dataset is not a collection, partitions
|
|
218
|
-
* mix collection kinds (or include Arrays) under co-partitioning, key
|
|
219
|
-
* types cannot align, `by` (explicit or the implicit alignment under
|
|
220
|
-
* co-partitioning) does not read a leading prefix of every partitioned
|
|
221
|
-
* dataset's key, the output is not a collection in splice mode (no
|
|
222
|
-
* `combine`), or `merge` is given alongside `combine`, on an Array output,
|
|
223
|
-
* in the wrong form for the output's kind, or with the `custom` runtime.
|
|
117
|
+
* @typeParam T - The dataset's East type
|
|
118
|
+
* @param dataset - The dataset
|
|
119
|
+
* @param config - `by`, the fields rows are grouped by
|
|
120
|
+
* @returns The marked input, to pass in a stream task's `inputs`
|
|
224
121
|
*
|
|
225
122
|
* @example
|
|
226
123
|
* ```ts
|
|
227
|
-
* const
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
124
|
+
* const totals = e3.streamTask('totals', {
|
|
125
|
+
* inputs: [e3.partition(sales, { by: ['account'] }), rates],
|
|
126
|
+
* output: e3.output.dict(StringType, FloatType, { merge: ($, account, a, b) => a.add(b) }),
|
|
127
|
+
* }, ($, sales, rates, emit) => {
|
|
128
|
+
* $.for(sales, ($, sale, key) => {
|
|
129
|
+
* $(emit(key.account, sale.amount.multiply(rates.get(sale.currency))));
|
|
130
|
+
* });
|
|
131
|
+
* });
|
|
233
132
|
* ```
|
|
234
133
|
*/
|
|
235
|
-
export declare function
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
134
|
+
export declare function partition<T extends EastType>(dataset: DatasetDef<T>, config?: {
|
|
135
|
+
by?: string[];
|
|
136
|
+
}): PartitionDef<T>;
|
|
137
|
+
/** The expression a stream task's body receives for each input. */
|
|
138
|
+
type InputExprs<T extends readonly (DatasetDef | PartitionDef)[]> = {
|
|
139
|
+
[K in keyof T]: T[K] extends PartitionDef<infer U> ? ExprType<U> : T[K] extends DatasetDef<infer U> ? ExprType<U> : never;
|
|
140
|
+
};
|
|
239
141
|
/**
|
|
240
|
-
* The declaration half of {@link streamTask}
|
|
241
|
-
*
|
|
142
|
+
* The declaration half of {@link streamTask}; the body comes last as the `fn`
|
|
143
|
+
* argument.
|
|
242
144
|
*
|
|
243
|
-
* @typeParam
|
|
244
|
-
* @typeParam
|
|
245
|
-
* @typeParam Output - The output collection's East type
|
|
145
|
+
* @typeParam Inputs - The inputs, some marked with {@link partition}
|
|
146
|
+
* @typeParam Output - The output kind
|
|
246
147
|
*/
|
|
247
|
-
export interface StreamTaskSpec<
|
|
248
|
-
/** The
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
readonly
|
|
252
|
-
/**
|
|
253
|
-
|
|
254
|
-
/** The output collection type. An Array output stores its elements in
|
|
255
|
-
* emission order. A Set or Dict output must be emitted in ascending key
|
|
256
|
-
* order (East's total order): the runner writes it in one pass, segment by
|
|
257
|
-
* segment, and an out-of-order key fails the task with the same message
|
|
258
|
-
* on every runtime. Duplicate Set/Dict keys are a runtime error unless
|
|
259
|
-
* {@link merge} folds them. */
|
|
148
|
+
export interface StreamTaskSpec<Inputs extends readonly (DatasetDef | PartitionDef)[], Output extends OutputDef> {
|
|
149
|
+
/** The inputs, in the body's parameter order. An input wrapped in
|
|
150
|
+
* {@link partition} is one the work may be split over; the others reach
|
|
151
|
+
* every piece whole, opened lazily when large. Empty for a producer. */
|
|
152
|
+
readonly inputs: [...Inputs];
|
|
153
|
+
/** The output kind — `e3.output.array`, `set`, `dict` or `fold`. It fixes
|
|
154
|
+
* `emit`'s signature and how the parts of the output combine. */
|
|
260
155
|
readonly output: Output;
|
|
261
|
-
/**
|
|
262
|
-
*
|
|
263
|
-
* together, instead of failing on the duplicate.
|
|
264
|
-
*
|
|
265
|
-
* @remarks
|
|
266
|
-
* A `Dict` output takes a function of the key and two values: adjacent
|
|
267
|
-
* equal keys fold left in emission order, `acc = merge(key, acc, value)`. A
|
|
268
|
-
* `Set` output takes the literal `'union'`: adjacent equal elements collapse
|
|
269
|
-
* to the first. The ascending contract stands — `merge` folds the equal keys
|
|
270
|
-
* of a grouped stream; keys that collide across the stream are a
|
|
271
|
-
* `partitionTask` with `merge`. The stored dataset is exactly the output the
|
|
272
|
-
* folded emissions would write. An Array output takes no `merge`.
|
|
273
|
-
*/
|
|
274
|
-
readonly merge?: OutputMerge<Output>;
|
|
275
|
-
/** Runtime the body runs on; defaults to {@link DEFAULT_RUNNER}. Every
|
|
276
|
-
* stock runtime streams the output through `emit` and feeds the `stream`
|
|
277
|
-
* input lazily (segment-fed iteration and keyed reads at O(segment)
|
|
278
|
-
* decoded memory; any other operation on it decodes the whole value
|
|
279
|
-
* once). The `custom` runtime is rejected — its argv cannot carry the
|
|
280
|
-
* streaming flags. */
|
|
156
|
+
/** Runtime the body runs on; defaults to {@link DEFAULT_RUNNER}. A stock
|
|
157
|
+
* runtime: the `custom` one runs only a program that returns its output. */
|
|
281
158
|
readonly runner?: Runner;
|
|
282
159
|
/** Execution environment declaration, as for {@link task}. */
|
|
283
160
|
readonly environment?: EnvironmentDecl;
|
|
284
161
|
}
|
|
285
|
-
/** The body parameters of a {@link streamTask}: the stream value when
|
|
286
|
-
* declared, the ordinary inputs, then the `emit` capability. */
|
|
287
|
-
type StreamTaskArgs<Stream extends DatasetDef | undefined, Inputs extends readonly DatasetDef[], Output extends EastType> = Stream extends DatasetDef<infer S> ? [ExprType<S>, ...DatasetExprs<Inputs>, ExprType<EmitOf<Output>>] : [...DatasetExprs<Inputs>, ExprType<EmitOf<Output>>];
|
|
288
162
|
/**
|
|
289
|
-
* Defines a
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
* and
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* With no `stream`, the task is a producer: its body loops over
|
|
310
|
-
* platform-function sources (paginated APIs, database cursors) and emits.
|
|
311
|
-
* A producer's real input is the outside world, so it follows the existing
|
|
312
|
-
* integration-task conventions for re-run semantics — `emit` changes nothing
|
|
313
|
-
* about that.
|
|
163
|
+
* Defines a stream task: a body that emits its output into an output kind
|
|
164
|
+
* rather than returning it, over inputs its work may be split across.
|
|
165
|
+
*
|
|
166
|
+
* `emit` is the body's trailing parameter, and the output kind fixes its
|
|
167
|
+
* signature: `emit(t)` for `array`, `set` and `fold`, `emit(k, v)` for
|
|
168
|
+
* `dict`. Emission order is free — the platform sorts sets and dicts — and
|
|
169
|
+
* the parts of the output combine as the kind says: concatenated for an
|
|
170
|
+
* array, united for a set, by key for a dict (equal keys folding with its
|
|
171
|
+
* `merge`), and folded with `combine` from `zero` for a fold.
|
|
172
|
+
*
|
|
173
|
+
* An input wrapped in {@link partition} is one the work may be split over:
|
|
174
|
+
* the body runs once per piece, each piece typed as the whole dataset, and
|
|
175
|
+
* the pieces' outputs combine by the output kind. The other inputs reach
|
|
176
|
+
* every piece whole. With no partitioned input the task is one unit, with
|
|
177
|
+
* exact left-to-right semantics; a producer has no inputs at all, and emits
|
|
178
|
+
* what it reads from platform functions.
|
|
179
|
+
*
|
|
180
|
+
* The author's contract, the only one: `merge` and `combine` are associative,
|
|
181
|
+
* `zero` is an identity of `combine`, and a partitioned body's combined
|
|
182
|
+
* result does not depend on where its input was cut.
|
|
314
183
|
*
|
|
315
184
|
* @typeParam Name - Task name (literal type)
|
|
316
|
-
* @typeParam
|
|
317
|
-
* @typeParam
|
|
318
|
-
* @typeParam Inputs - Ordinary input dataset defs
|
|
185
|
+
* @typeParam Inputs - The inputs, some marked with {@link partition}
|
|
186
|
+
* @typeParam Output - The output kind
|
|
319
187
|
* @param name - Task name
|
|
320
|
-
* @param spec - The
|
|
321
|
-
*
|
|
322
|
-
* @param fn - Implementation body; receives the stream value (when
|
|
323
|
-
* declared), the inputs, then `emit`, and returns nothing
|
|
188
|
+
* @param spec - The inputs, the output kind, the runner and the environment
|
|
189
|
+
* @param fn - The body; receives the inputs, then `emit`, and returns nothing
|
|
324
190
|
* @returns A TaskDef with `.output` for chaining
|
|
325
|
-
* @throws {Error} When the output is not
|
|
326
|
-
*
|
|
327
|
-
*
|
|
191
|
+
* @throws {Error} When the output is not an output kind, the runner is the
|
|
192
|
+
* `custom` runtime, a partitioned input is not a collection, a `by` names
|
|
193
|
+
* something other than leading key fields, or co-partitioned inputs have no
|
|
194
|
+
* common key.
|
|
328
195
|
*
|
|
329
196
|
* @example
|
|
330
197
|
* ```ts
|
|
331
|
-
* const
|
|
332
|
-
*
|
|
333
|
-
* const
|
|
334
|
-
*
|
|
335
|
-
* output:
|
|
336
|
-
* }, ($,
|
|
337
|
-
*
|
|
338
|
-
*
|
|
339
|
-
* $.assign(balance, balance.add(event.amount));
|
|
340
|
-
* $(emit({ at: event.at, balance }));
|
|
198
|
+
* const sales = e3.input('sales', DictType(SaleKeyType, SaleType));
|
|
199
|
+
*
|
|
200
|
+
* const byAccount = e3.streamTask('by_account', {
|
|
201
|
+
* inputs: [e3.partition(sales)],
|
|
202
|
+
* output: e3.output.dict(StringType, FloatType, { merge: ($, account, a, b) => a.add(b) }),
|
|
203
|
+
* }, ($, sales, emit) => {
|
|
204
|
+
* $.for(sales, ($, sale) => {
|
|
205
|
+
* $(emit(sale.account, sale.amount));
|
|
341
206
|
* });
|
|
342
207
|
* });
|
|
343
208
|
* ```
|
|
344
209
|
*/
|
|
345
|
-
export declare function streamTask<Name extends string,
|
|
210
|
+
export declare function streamTask<Name extends string, const Inputs extends readonly (DatasetDef | PartitionDef)[], Output extends OutputDef>(name: Name, spec: StreamTaskSpec<Inputs, Output>, fn: ($: BlockBuilder<NullType>, ...args: [...InputExprs<Inputs>, ExprType<Output['emit']>]) => void): TaskDef<Output['type'], TaskOutputPath<Name>>;
|
|
346
211
|
export {};
|
|
347
212
|
//# sourceMappingURL=task.d.ts.map
|
package/dist/src/task.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH
|
|
1
|
+
{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxJ,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAA8D,MAAM,eAAe,CAAC;AAC3I,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAkB,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAA2B,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEjF;;;GAGG;AACH,KAAK,mBAAmB,CAAC,CAAC,SAAS,SAAS,UAAU,EAAE,IAAI;KACzD,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK;CAC7D,GAAG,QAAQ,EAAE,CAAC;AAEf,kDAAkD;AAClD,KAAK,cAAc,CAAC,IAAI,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3H;;;;GAIG;AACH,eAAO,MAAM,SAAS,EAAE,WAKvB,CAAC;AA0EF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,wBAAgB,IAAI,CAAC,IAAI,SAAS,MAAM,EAAE,MAAM,SAAS,SAAS,UAAU,EAAE,EAAE,MAAM,SAAS,QAAQ,EACrG,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,EACnB,EAAE,EAAE,YAAY,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GACjD,oBAAoB,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GACzD,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GACtD,yBAAyB,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAClE,MAAM,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;CAAE,GAC3E,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;AAwCzC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,UAAU,CAAC,IAAI,SAAS,MAAM,EAAE,MAAM,SAAS,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,SAAS,QAAQ,EACvG,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,EAClJ,MAAM,CAAC,EAAE;IAAE,WAAW,CAAC,EAAE,eAAe,CAAA;CAAE,GACzC,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAuCvC;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE;IAAE,EAAE,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAEjH;AAED,mEAAmE;AACnE,KAAK,UAAU,CAAC,CAAC,SAAS,SAAS,CAAC,UAAU,GAAG,YAAY,CAAC,EAAE,IAAI;KACjE,CAAC,IAAI,MAAM,CAAC,GACX,CAAC,CAAC,CAAC,CAAC,SAAS,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAChD,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAC9C,KAAK;CACR,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,cAAc,CAC7B,MAAM,SAAS,SAAS,CAAC,UAAU,GAAG,YAAY,CAAC,EAAE,EACrD,MAAM,SAAS,SAAS;IAExB;;6EAEyE;IACzE,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC;IAC7B;sEACkE;IAClE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;iFAC6E;IAC7E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,8DAA8D;IAC9D,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;CACxC;AAgFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,wBAAgB,UAAU,CACxB,IAAI,SAAS,MAAM,EACnB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,UAAU,GAAG,YAAY,CAAC,EAAE,EAC3D,MAAM,SAAS,SAAS,EAExB,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,GAClG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAsC/C"}
|