@elaraai/e3 1.0.79 → 1.0.80

package/dist/src/task.js

@@ -2,9 +2,8 @@
  * Copyright (c) 2025 Elara AI Pty Ltd
  * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
  */
-import { Expr, variant, some, none, ArrayType, StringType, NullType, FunctionType, StructType, East, IRType, encodeEastIR, toEastTypeValue, isTypeValueEqual } from '@elaraai/east';
-import { TASK_KIND_PARTITION, TASK_KIND_STREAM, encodePartitionTaskMetadata, encodeStreamTaskMetadata, mergeCommandIr, partitionProjectionShape, streamCommandIr } from '@elaraai/e3-types';
-import { DEFAULT_RUNNER, runnerToCommand, runnerToVariant } from './runner.js';
+import { Expr, variant, ArrayType, StringType, NullType, East, isTypeValueEqual, printType, toEastTypeValue } from '@elaraai/east';
+import { DEFAULT_RUNNER } from './runner.js';
 import { validateEnvironmentDecl } from './environment.js';
 /**
  * Singleton tree definition for `.tasks`.
@@ -31,29 +30,6 @@ function createTaskTree(name) {
         deps: new Set([tasksTree]),
     };
 }
-/**
- * Creates a private IR dataset for a task at `.tasks.${name}.${field}`: the
- * task's `function_ir`, or a stream task's `merge_ir`.
- *
- * @param name - Task name
- * @param taskTree - The task's subtree
- * @param field - The dataset's name within the task's subtree
- * @param eastIR - The compiled function IR
- * @returns A DatasetDef for the IR (private, not typed)
- */
-function createIRDataset(name, taskTree, field, eastIR) {
-    return {
-        kind: 'dataset',
-        name: field,
-        path: [variant('field', 'tasks'), variant('field', name), variant('field', field)],
-        type: IRType,
-        // Store the full EastIR/AsyncEastIR bundle so export.ts can use encodeEastIR
-        // and preserve source_map into the beast2 blob.
-        default: eastIR,
-        writable: false,
-        deps: new Set([...taskTree.deps, taskTree]),
-    };
-}
 /**
  * Creates an output dataset for a task at `.tasks.${name}.output`.
  *
@@ -76,7 +52,7 @@ function createOutputDataset(name, taskTree, outputType) {
  * Collects all dependencies for a task.
  *
  * Walks the dependency graph to include:
- * - The task's subtree and its contents (function_ir, output)
+ * - The task's subtree and its output
  * - All input datasets and their dependencies
  */
 function collectDeps(taskTree, outputDataset, inputs) {
@@ -99,54 +75,24 @@ function collectDeps(taskTree, outputDataset, inputs) {
 export function task(name, inputs, fn, config) {
     if (config?.environment)
         validateEnvironmentDecl(config.environment, name);
-    // Keep the full EastIR bundle (IR + source_map) so we don't drop the
-    // source map before it reaches the beast2 encoder in export.ts.
-    const eastIR = fn.toIR();
-    const outputType = Expr.type(fn).output;
-    // Create the task's subtree at .tasks.${name}
+    for (const input of inputs) {
+        if (input.kind === 'partition') {
+            throw new Error(`task '${name}': input '${input.dataset.name}' is marked with e3.partition, which only e3.streamTask takes — ` +
+                `pass the dataset itself`);
+        }
+    }
     const taskTree = createTaskTree(name);
-    // Create the function_ir dataset (private, holds the IR bundle)
-    const functionIRDataset = createIRDataset(name, taskTree, 'function_ir', eastIR);
-    // The first input is the FunctionIR to execute
-    const input_datasets = [
-        functionIRDataset,
-        ...inputs
-    ];
-    // Create the output dataset
-    const output = createOutputDataset(name, taskTree, outputType);
-    // Resolve the typed Runner to argv at task-definition time. The variant is
-    // SDK-only metadata: it never reaches the IR or the wire — only the
-    // resolved string array does, baked in as an East constant below. Default
-    // is east-node + east-node-std (every e3 project has Node already).
-    const argvPrefix = runnerToCommand(config?.runner ?? DEFAULT_RUNNER);
-    // Build the command IR. At task-execution time this East function is
-    // evaluated with the staged input/output paths and returns the final argv.
-    const commandFn = East.function([ArrayType(StringType), StringType], ArrayType(StringType), ($, input_paths, output_path) => {
-        const command = $.let(argvPrefix, ArrayType(StringType));
-        // Function argument paths
-        const i = $.let(1n);
-        $.while(East.less(i, input_paths.size()), $ => {
-            $(command.pushLast("-i"));
-            $(command.pushLast(input_paths.get(i)));
-            $.assign(i, i.add(1n));
-        });
-        // Output path
-        $(command.pushLast('-o'));
-        $(command.pushLast(output_path));
-        // Function IR is the first input
-        $(command.pushLast(input_paths.get(0n)));
-        $.return(command);
-    });
+    const output = createOutputDataset(name, taskTree, Expr.type(fn).output);
     const taskDef = {
         kind: 'task',
         name,
-        // Keep the full EastIR bundle so export.ts can encode with source map.
-        command: commandFn.toIR(),
-        inputs: input_datasets,
+        // Keep the full EastIR bundle (IR + source map) so export.ts can encode
+        // it with its source map.
+        body: { kind: 'east', program: fn.toIR() },
+        inputs,
         output,
-        deps: collectDeps(taskTree, output, input_datasets),
-        taskKind: config?.kind,
-        metadata: config?.metadata,
+        role: config?.role ?? variant('data', null),
+        deps: collectDeps(taskTree, output, inputs),
         runner: config?.runner ?? DEFAULT_RUNNER,
         environment: config?.environment,
     };
@@ -154,6 +100,20 @@ export function task(name, inputs, fn, config) {
     output.deps.add(taskDef);
     return taskDef;
 }
+/**
+ * Defines a task that runs a command rather than East: a bash script built
+ * from the staged input paths and the path the output must be written to.
+ *
+ * @typeParam Name - Task name (literal type)
+ * @typeParam Inputs - Input dataset types
+ * @typeParam Output - Output type
+ * @param name - Task name
+ * @param inputs - Input datasets, staged as beast2 files
+ * @param outputType - The East type of the beast2 file the command writes
+ * @param command - Builds the bash script from the input paths and the output path
+ * @param config - The environment the command runs in
+ * @returns A TaskDef with `.output` for chaining
+ */
 export function customTask(name, inputs, outputType, command, config) {
     if (config?.environment)
         validateEnvironmentDecl(config.environment, name);
@@ -169,9 +129,10 @@ export function customTask(name, inputs, outputType, command, config) {
         kind: 'task',
         name,
         // Keep the full EastIR bundle so export.ts can encode with source map.
-        command: commandFn.toIR(),
+        body: { kind: 'command', command: commandFn.toIR() },
         inputs,
         output,
+        role: variant('data', null),
         deps: collectDeps(taskTree, output, inputs),
         environment: config?.environment,
     };
@@ -179,326 +140,156 @@ export function customTask(name, inputs, outputType, command, config) {
     output.deps.add(taskDef);
     return taskDef;
 }
-/** Default carved-slice byte target for {@link partitionTask}. */
-const DEFAULT_TARGET_PARTITION_BYTES = 256 * 1024 * 1024;
-/** The East key type a collection dataset is carved by. */
-function collectionKeyType(name, dataset) {
-    const type = dataset.type;
-    switch (type.type) {
-        case 'Dict': return type.key;
-        case 'Set': return type.key;
-        case 'Array': return type.value;
-        default:
-            throw new Error(`partitionTask '${name}': partitioned dataset '${dataset.name}' must be a collection (Array, Set or Dict), got ${type.type}`);
-    }
-}
+// =============================================================================
+// Streaming tasks
+// =============================================================================
 /**
- * Defines a partitioned task: e3 carves the huge partitioned input(s) into
- * key-range slices, runs `fn` once per partition as an ordinary
- * content-addressed execution (parallel, memoized per partition), and
- * assembles the output — by splicing shards in partition order, by merging
- * keyed partials on the task's runner when `merge` is given, or by folding
- * partials pairwise when `combine` is given.
+ * Marks an input of a {@link streamTask} as one its work may be split over.
  *
- * `fn` always returns the output type: without `combine` each execution
- * returns its *shard* of the output (Array shards concatenate freely;
- * Dict/Set shard key ranges must ascend disjointly in partition order — any
- * key-preserving or monotone re-keying transform qualifies). The shard
- * contract is deliberately enforced at SPLICE TIME, not build time: whether
- * an arbitrary body preserves key order is not decidable from types, and a
- * static rule would false-reject permitted monotone re-keys — a violation
- * fails the task deterministically at splice, naming the offending
- * partitions and the remedies (`combine` / `customTask`). With `combine`
- * each execution returns a *partial* and the partials fold.
+ * The body receives one piece of the dataset, typed as the whole: a key range
+ * of a Set or Dict, or a position range of an Array. Pieces are cut where the
+ * content says, so an insertion re-cuts only the pieces around it. Rows whose
+ * `by` fields are equal are never split across pieces.
  *
- * Parameter order is fixed and fully typed: the partition slices first (in
- * declared order, each typed as its dataset's own collection type), then the
- * ordinary `inputs` in order.
+ * `by` names leading key fields, in order — `['account']`, or
+ * `['account', 'at.day']`, whose last entry reads the first field of `at`. Two
+ * or more partitioned inputs are cut at the same keys, so they must be Sets or
+ * Dicts whose keys, or whose `by` fields, have the same types. {@link
+ * streamTask} checks all of this against the dataset, naming the task.
  *
- * @typeParam Name - Task name (literal type)
- * @typeParam Partitions - Partitioned input dataset defs
- * @typeParam Inputs - Ordinary input dataset defs
- * @typeParam Output - Output type
- * @param name - Task name
- * @param spec - The declaration: partitioned inputs, alignment, output type,
- *   combine mode, sizing, runner and environment
- * @param fn - Implementation body, run once per partition
- * @returns A TaskDef with `.output` for chaining
- * @throws {Error} When a partitioned dataset is not a collection, partitions
- *   mix collection kinds (or include Arrays) under co-partitioning, key
- *   types cannot align, `by` (explicit or the implicit alignment under
- *   co-partitioning) does not read a leading prefix of every partitioned
- *   dataset's key, the output is not a collection in splice mode (no
- *   `combine`), or `merge` is given alongside `combine`, on an Array output,
- *   in the wrong form for the output's kind, or with the `custom` runtime.
+ * @typeParam T - The dataset's East type
+ * @param dataset - The dataset
+ * @param config - `by`, the fields rows are grouped by
+ * @returns The marked input, to pass in a stream task's `inputs`
  *
  * @example
  * ```ts
- * const sales = e3.input('sales', DictType(StringType, SaleType));
- *
- * const cleaned = e3.partitionTask('cleaned', {
- *   partitions: [sales],
- *   output: DictType(StringType, SaleType),
- * }, ($, slice) => slice.filter(($, sale) => East.greater(sale.qty, 0n)));
+ * const totals = e3.streamTask('totals', {
+ *   inputs: [e3.partition(sales, { by: ['account'] }), rates],
+ *   output: e3.output.dict(StringType, FloatType, { merge: ($, account, a, b) => a.add(b) }),
+ * }, ($, sales, rates, emit) => {
+ *   $.for(sales, ($, sale, key) => {
+ *     $(emit(key.account, sale.amount.multiply(rates.get(sale.currency))));
+ *   });
+ * });
  * ```
  */
-export function partitionTask(name, spec, fn) {
-    const partitions = spec.partitions;
-    const inputs = (spec.inputs ?? []);
-    const output = spec.output;
-    // Partitioned inputs must be collections; co-partitioning is restricted to
-    // Dict/Set roots (a shared key space must exist for boundary alignment —
-    // an Array has none).
-    const keyTypes = partitions.map((p) => collectionKeyType(name, p));
-    const kinds = partitions.map((p) => p.type.type);
-    if (partitions.length > 1) {
-        const first = kinds[0];
-        if (first === 'Array' || kinds.some((k) => k !== first)) {
-            throw new Error(`partitionTask '${name}': co-partitioning is restricted to Dict/Set roots sharing a key space — ` +
-                `got [${kinds.join(', ')}]. Partition one dataset and access the others as ordinary inputs, or re-key upstream.`);
-        }
-    }
-    // The `by` parameter type: a single partition's own key, or the strict
-    // field-wise intersection of every key struct under co-partitioning.
-    let byParamType;
-    if (partitions.length === 1) {
-        byParamType = keyTypes[0];
-    }
-    else if (keyTypes.every((k) => k.type === 'Struct')) {
-        const first = keyTypes[0];
-        const shared = {};
-        for (const [field, fieldType] of Object.entries(first.fields)) {
-            const everywhere = keyTypes.every((k) => {
-                const other = k.fields[field];
-                return other !== undefined && isTypeValueEqual(toEastTypeValue(other), toEastTypeValue(fieldType));
-            });
-            if (everywhere)
-                shared[field] = fieldType;
+export function partition(dataset, config) {
+    return { kind: 'partition', dataset, by: [...(config?.by ?? [])] };
+}
+/** The key a Set or Dict is cut by; an Array, cut by position, has none. */
+function collectionKey(type) {
+    const collection = type;
+    return collection.type === 'Set' || collection.type === 'Dict' ? collection.key : undefined;
+}
+/**
+ * Checks a stream task's partitioned inputs against their datasets: each is a
+ * collection, each `by` names leading key fields, and co-partitioned inputs
+ * are Sets or Dicts cut by fields of the same types.
+ */
+function checkPartitions(name, partitions) {
+    const cuts = [];
+    for (const { dataset, by } of partitions) {
+        const where = `streamTask '${name}': partitioned input '${dataset.name}'`;
+        const type = dataset.type;
+        if (type.type !== 'Array' && type.type !== 'Set' && type.type !== 'Dict') {
+            throw new Error(`${where} must be a collection (Array, Set or Dict), got ${type.type}`);
         }
-        if (Object.keys(shared).length === 0) {
-            throw new Error(`partitionTask '${name}': co-partitioned key types share no identically-typed fields — the datasets have no common key space`);
+        const key = collectionKey(type);
+        if (key === undefined) {
+            if (by.length > 0)
+                throw new Error(`${where} is an Array, cut by position, so it has no key for \`by\` to name`);
+            continue;
         }
-        byParamType = StructType(shared);
-    }
-    else {
-        const first = toEastTypeValue(keyTypes[0]);
-        if (!keyTypes.every((k) => isTypeValueEqual(toEastTypeValue(k), first))) {
-            throw new Error(`partitionTask '${name}': co-partitioned key types must be identical (or share struct fields) for boundary alignment`);
+        if (by.length === 0) {
+            cuts.push({ input: dataset.name, types: [key] });
+            continue;
         }
-        byParamType = keyTypes[0];
-    }
-    // Reify `by` and validate the leading-prefix soundness condition per
-    // dataset: struct keys sort lexicographically by declared field order, so
-    // the projection must read a leading prefix of each dataset's OWN order.
-    let byIr;
-    let byShape = null;
-    if (spec.by !== undefined) {
-        const byFn = East.function([byParamType], undefined, spec.by);
-        const bundle = byFn.toIR();
-        const shape = partitionProjectionShape(bundle.ir);
-        if (shape === null) {
-            throw new Error(`partitionTask '${name}': \`by\` must project a leading prefix of the partition key — ` +
-                `accepted shapes: the key itself, a leading field (\`key.f\`), a nested leading-field ` +
-                `path (\`key.a.b\`, each step the first field of its level), or a struct literal of ` +
-                `leading fields in declared order (\`{ f1: key.f1, f2: key.f2 }\`)`);
+        if (key.type !== 'Struct') {
+            throw new Error(`${where} has a ${key.type} key, which has no fields for \`by\` to name`);
         }
-        if (shape.names.length > 0) {
-            for (let i = 0; i < partitions.length; i++) {
-                const keyType = keyTypes[i];
-                if (keyType.type !== 'Struct') {
-                    throw new Error(`partitionTask '${name}': \`by\` reads key fields, but partitioned dataset '${partitions[i].name}' has a non-struct key (${keyType.type}) — use the identity projection`);
-                }
-                if (shape.kind === 'fields') {
-                    const order = Object.keys(keyType.fields);
-                    const misaligned = shape.names.length > order.length || shape.names.some((f, j) => order[j] !== f);
-                    if (misaligned) {
-                        throw new Error(`partitionTask '${name}': \`by\` projects (${shape.names.join(', ')}), which is not a leading prefix of ` +
-                            `partitioned dataset '${partitions[i].name}' key field order (${order.join(', ')})`);
-                    }
-                }
-                else {
-                    // Nested path: every step must read the FIRST field of its
-                    // level's struct — struct keys sort lexicographically by declared
-                    // field order, so only first-field descent preserves it.
-                    let level = keyType;
-                    for (let j = 0; j < shape.names.length; j++) {
-                        const step = shape.names[j];
-                        if (level.type !== 'Struct') {
-                            throw new Error(`partitionTask '${name}': \`by\` path (key.${shape.names.join('.')}) descends into a ${level.type} at '${step}' — ` +
-                                `every step must read the first field of a struct level of partitioned dataset '${partitions[i].name}'`);
-                        }
-                        const levelFields = level.fields;
-                        const order = Object.keys(levelFields);
-                        if (order[0] !== step) {
-                            throw new Error(`partitionTask '${name}': \`by\` path (key.${shape.names.slice(0, j + 1).join('.')}) reads '${step}', which is not ` +
-                                `the first field of partitioned dataset '${partitions[i].name}' key level (${order.join(', ')})`);
-                        }
-                        level = levelFields[step];
-                    }
-                }
+        const fields = key.fields;
+        const order = Object.keys(fields);
+        const types = by.map((entry, i) => {
+            const [head, ...path] = entry.split('.');
+            if (order[i] !== head || (path.length > 0 && i !== by.length - 1)) {
+                throw new Error(`${where}: \`by\` (${by.join(', ')}) must name leading key fields in order — the key's fields are ` +
+                    `(${order.join(', ')}), and only the last entry may read into one, as 'at.day' does`);
             }
-        }
-        byShape = shape;
-        byIr = encodeEastIR(bundle);
-    }
-    // Under co-partitioning with non-identical key types, the EFFECTIVE
-    // boundary projection must ALSO be a leading prefix of every dataset's
-    // key order when it is implicit:
-    // - with no `by`, the runtime aligns secondaries under the PRIMARY's key
-    //   comparator — the projection is the primary's full field sequence;
-    // - with an identity `by`, it is the shared-intersection struct in its
-    //   declared field order.
-    // A key set like {sku,period} × {period,sku} passes the field-wise
-    // intersection yet sorts differently per dataset — accepted, it would
-    // silently mis-assign rows at run time with a success status.
-    if (partitions.length > 1) {
-        const firstKey = toEastTypeValue(keyTypes[0]);
-        const keysIdentical = keyTypes.every((k) => isTypeValueEqual(toEastTypeValue(k), firstKey));
-        const validatedByFields = byShape !== null && byShape.names.length > 0;
-        if (!keysIdentical && !validatedByFields) {
-            // Non-identical keys reach here only via the struct-intersection path.
-            const sourceFields = (t) => t.fields;
-            const effective = spec.by !== undefined
-                ? { label: 'the identity `by` projection reads the shared key fields', fields: sourceFields(byParamType) }
-                : { label: `with no \`by\`, co-partition boundaries align on partitioned dataset '${partitions[0].name}' key order`, fields: sourceFields(keyTypes[0]) };
-            const eff = Object.keys(effective.fields);
-            for (let i = 0; i < partitions.length; i++) {
-                const fields = sourceFields(keyTypes[i]);
-                const order = Object.keys(fields);
-                const misaligned = eff.length > order.length || eff.some((f, j) => order[j] !== f || !isTypeValueEqual(toEastTypeValue(effective.fields[f]), toEastTypeValue(fields[f])));
-                if (misaligned) {
-                    throw new Error(`partitionTask '${name}': ${effective.label} (${eff.join(', ')}), which is not a leading prefix of ` +
-                        `partitioned dataset '${partitions[i].name}' key field order (${order.join(', ')}) — ` +
-                        `declare \`by\` as a shared leading-prefix projection, or re-key upstream`);
+            let level = fields[head];
+            for (const step of path) {
+                const levelFields = level.type === 'Struct' ? Object.keys(level.fields) : [];
+                if (levelFields[0] !== step) {
+                    throw new Error(`${where}: \`by\` path '${entry}' reads '${step}', which is not the first field of ` +
+                        `${level.type === 'Struct' ? `(${levelFields.join(', ')})` : `a ${level.type}`} — rows sort by a struct's first field, so only it groups them`);
                 }
+                level = level.fields[step];
             }
-        }
-    }
-    // Splice mode (no `combine`) assembles the output from per-partition
-    // SHARDS in partition order — only a collection can splice. A
-    // non-collection output would pass definition time and fail only when the
-    // input first carves into two or more partitions.
-    if (spec.combine === undefined && output.type !== 'Array' && output.type !== 'Set' && output.type !== 'Dict') {
-        throw new Error(`partitionTask '${name}': without \`combine\`, each partition returns a shard of the output and the shards ` +
-            `splice (or, with \`merge\`, merge) in key order — the output must be a collection (Array, Set or Dict), ` +
-            `got ${output.type}. Provide \`combine\` to fold non-collection partials.`);
+            return level;
+        });
+        cuts.push({ input: dataset.name, types });
     }
-    // Reify `combine` as a free East function (Out, Out) -> Out.
-    let combineIr;
-    if (spec.combine !== undefined) {
-        const combineFn = East.function([output, output], output, spec.combine);
-        combineIr = encodeEastIR(combineFn.toIR());
+    if (partitions.length < 2)
+        return;
+    if (cuts.length < partitions.length) {
+        throw new Error(`streamTask '${name}': co-partitioned inputs are cut at the same keys, so each must be a Set or a Dict — ` +
+            `partition one input and pass the others whole`);
     }
-    // Reify `merge` as a free East function (Key, Value, Value) -> Value for a
-    // Dict output, or record the Set union flag. The task's runner merges the
-    // partials with it through its `merge` command, whose command IR is built
-    // here and carried in the metadata — the orchestrator runs it as an
-    // ordinary execution per group of partials.
-    let mergeIr;
-    let mergeSets = false;
-    let mergeCommand;
-    if (spec.merge !== undefined) {
-        if (spec.combine !== undefined) {
-            throw new Error(`partitionTask '${name}': \`merge\` and \`combine\` are two assembly modes — give one. ` +
-                `\`merge\` merges keyed partials by key; \`combine\` folds whole partials.`);
+    const [first, ...rest] = cuts;
+    for (const other of rest) {
+        const same = other.types.length === first.types.length &&
+            other.types.every((t, i) => isTypeValueEqual(toEastTypeValue(t), toEastTypeValue(first.types[i])));
+        if (!same) {
+            throw new Error(`streamTask '${name}': co-partitioned inputs '${first.input}' and '${other.input}' have no common key — ` +
+                `they are cut by (${first.types.map((t) => printType(t)).join(', ')}) and (${other.types.map((t) => printType(t)).join(', ')}); ` +
+                `give each a \`by\` naming fields of the same types`);
         }
-        if (spec.runner?.runtime === 'custom') {
-            throw new Error(`partitionTask '${name}': merge runs the fan-in on the task's runner, which must be a stock runtime ` +
-                `(east-node, east-py, east-c) — the custom runtime has no merge command`);
-        }
-        if (spec.merge === 'union') {
-            if (output.type !== 'Set') {
-                throw new Error(`partitionTask '${name}': \`merge: 'union'\` assembles a Set output, got ${output.type}` +
-                    (output.type === 'Dict' ? ' — a Dict output takes a per-key merge function' : ''));
-            }
-            mergeSets = true;
-        }
-        else {
-            if (output.type !== 'Dict') {
-                throw new Error(`partitionTask '${name}': a \`merge\` FUNCTION resolves a key present in two partials, so the output must be a Dict, ` +
-                    `got ${output.type}${output.type === 'Set' ? " — a Set output takes `merge: 'union'`" : ''}`);
-            }
-            const keyType = output.key;
-            const valueType = output.value;
-            const mergeFn = East.function([keyType, valueType, valueType], valueType, spec.merge);
-            mergeIr = encodeEastIR(mergeFn.toIR());
-        }
-        mergeCommand = encodeEastIR(mergeCommandIr(runnerToVariant(spec.runner ?? DEFAULT_RUNNER), mergeSets ? 'union' : 'function'));
-    }
-    const targetPartitionBytes = spec.targetPartitionBytes ?? DEFAULT_TARGET_PARTITION_BYTES;
-    if (!Number.isInteger(targetPartitionBytes) || targetPartitionBytes <= 0) {
-        throw new Error(`partitionTask '${name}': targetPartitionBytes must be a positive integer, got ${spec.targetPartitionBytes}`);
     }
-    const bodyFn = East.function([...partitions.map((p) => p.type), ...inputs.map((i) => i.type)], output, fn);
-    const metadata = encodePartitionTaskMetadata({
-        partitions: BigInt(partitions.length),
-        by: byIr !== undefined ? some(byIr) : none,
-        combine: combineIr !== undefined ? some(combineIr) : none,
-        targetPartitionBytes: BigInt(targetPartitionBytes),
-        merge: mergeIr !== undefined ? some(mergeIr) : none,
-        mergeSets,
-        mergeCommand: mergeCommand !== undefined ? some(mergeCommand) : none,
-    });
-    return task(name, [...partitions, ...inputs], bodyFn, {
-        kind: TASK_KIND_PARTITION,
-        metadata,
-        ...(spec.runner !== undefined && { runner: spec.runner }),
-        ...(spec.environment !== undefined && { environment: spec.environment }),
-    });
 }
 /**
- * Defines a streaming task: one execution whose runner feeds the `stream`
- * input segment-by-segment in canonical order (state is ordinary `$.let`
- * locals) and writes the output incrementally through the `emit` capability
- * — bounded memory at both ends, exact left-fold semantics, no parallelism
- * and no partial recompute.
+ * Defines a stream task: a body that emits its output into an output kind
+ * rather than returning it, over inputs its work may be split across.
  *
- * `emit` is a runner-implemented function value: `emit(key, value)` for Dict
- * outputs, `emit(element)` for Array/Set outputs. The streaming writer
- * re-batches emissions byte-adaptively and writes the output in one pass,
- * with one open batch in memory whatever the output's size. An Array output
- * takes its elements in emission order; a Set or Dict output must be emitted
- * in ascending key order, and an out-of-order key fails the task (`beast2
- * v5: Dict key emitted out of order: 1 after 2 — Set/Dict emissions must
- * ascend in East order`, the same words on every runtime). A re-key whose
- * output keys do not arrive in order is a `partitionTask` with `merge`.
- * Duplicate Set/Dict keys are a runtime error unless `merge` folds them: a
- * function for a Dict output (adjacent equal keys fold left in emission
- * order), `'union'` for a Set output. The body's return value is unused —
- * the output dataset is what `emit` wrote.
+ * `emit` is the body's trailing parameter, and the output kind fixes its
+ * signature: `emit(t)` for `array`, `set` and `fold`, `emit(k, v)` for
+ * `dict`. Emission order is free — the platform sorts sets and dicts — and
+ * the parts of the output combine as the kind says: concatenated for an
+ * array, united for a set, by key for a dict (equal keys folding with its
+ * `merge`), and folded with `combine` from `zero` for a fold.
  *
- * With no `stream`, the task is a producer: its body loops over
- * platform-function sources (paginated APIs, database cursors) and emits.
- * A producer's real input is the outside world, so it follows the existing
- * integration-task conventions for re-run semantics — `emit` changes nothing
- * about that.
+ * An input wrapped in {@link partition} is one the work may be split over:
+ * the body runs once per piece, each piece typed as the whole dataset, and
+ * the pieces' outputs combine by the output kind. The other inputs reach
+ * every piece whole. With no partitioned input the task is one unit, with
+ * exact left-to-right semantics; a producer has no inputs at all, and emits
+ * what it reads from platform functions.
+ *
+ * The author's contract, the only one: `merge` and `combine` are associative,
+ * `zero` is an identity of `combine`, and a partitioned body's combined
+ * result does not depend on where its input was cut.
  *
  * @typeParam Name - Task name (literal type)
- * @typeParam Output - Output collection type
- * @typeParam Stream - Streamed input dataset def, when present
- * @typeParam Inputs - Ordinary input dataset defs
+ * @typeParam Inputs - The inputs, some marked with {@link partition}
+ * @typeParam Output - The output kind
  * @param name - Task name
- * @param spec - The declaration: stream input, ordinary inputs, output
- *   collection type, merge, runner and environment
- * @param fn - Implementation body; receives the stream value (when
- *   declared), the inputs, then `emit`, and returns nothing
+ * @param spec - The inputs, the output kind, the runner and the environment
+ * @param fn - The body; receives the inputs, then `emit`, and returns nothing
  * @returns A TaskDef with `.output` for chaining
- * @throws {Error} When the output is not a collection type, the runner is
- *   the `custom` runtime, or `merge` is given for an Array output or in the
- *   wrong form for the output's kind.
+ * @throws {Error} When the output is not an output kind, the runner is the
+ *   `custom` runtime, a partitioned input is not a collection, a `by` names
+ *   something other than leading key fields, or co-partitioned inputs have no
+ *   common key.
  *
  * @example
  * ```ts
- * const events = e3.input('events', ArrayType(EventType));
+ * const sales = e3.input('sales', DictType(SaleKeyType, SaleType));
  *
- * const balances = e3.streamTask('balances', {
- *   stream: events,
- *   output: ArrayType(BalanceType),
- * }, ($, events, emit) => {
- *   const balance = $.let(0.0);
- *   $.for(events, ($, event) => {
- *     $.assign(balance, balance.add(event.amount));
- *     $(emit({ at: event.at, balance }));
+ * const byAccount = e3.streamTask('by_account', {
+ *   inputs: [e3.partition(sales)],
+ *   output: e3.output.dict(StringType, FloatType, { merge: ($, account, a, b) => a.add(b) }),
+ * }, ($, sales, emit) => {
+ *   $.for(sales, ($, sale) => {
+ *     $(emit(sale.account, sale.amount));
  *   });
  * });
  * ```
@@ -506,92 +297,35 @@ export function partitionTask(name, spec, fn) {
 export function streamTask(name, spec, fn) {
     if (spec.environment)
         validateEnvironmentDecl(spec.environment, name);
-    const output = spec.output;
-    const inputs = (spec.inputs ?? []);
-    let emitKind;
-    let emitType;
-    switch (output.type) {
-        case 'Dict':
-            emitKind = 'dict';
-            emitType = FunctionType([output.key, output.value], NullType);
-            break;
-        case 'Set':
-            emitKind = 'set';
-            emitType = FunctionType([output.key], NullType);
-            break;
-        case 'Array':
-            emitKind = 'array';
-            emitType = FunctionType([output.value], NullType);
-            break;
-        default:
-            throw new Error(`streamTask '${name}': output must be a collection type (Array, Set or Dict), got ${output.type} — for small results use e3.task`);
+    const outputKind = spec.output;
+    if (!['array', 'set', 'dict', 'fold'].includes(outputKind?.kind)) {
+        throw new Error(`streamTask '${name}': output is an output kind — e3.output.array, set, dict or fold`);
     }
     const runner = spec.runner ?? DEFAULT_RUNNER;
     if (runner.runtime === 'custom') {
-        throw new Error(`streamTask '${name}': the custom runtime cannot carry the streaming flags — use a stock runtime (east-node, east-py, east-c)`);
-    }
-    // `merge` folds equal keys in the runner's sink: a function for a Dict
-    // output, reified as a free East function (Key, Value, Value) -> Value;
-    // `'union'` for a Set output.
-    let mergeMode = 'none';
-    let mergeIR;
-    if (spec.merge !== undefined) {
-        if (output.type === 'Dict' && typeof spec.merge === 'function') {
-            mergeMode = 'function';
-            mergeIR = East.function([output.key, output.value, output.value], output.value, spec.merge).toIR();
-        }
-        else if (output.type === 'Set' && spec.merge === 'union') {
-            mergeMode = 'union';
-        }
-        else {
-            const given = output.type === 'Array' ? '' : typeof spec.merge === 'function' ? ' with a function' : ` with '${String(spec.merge)}'`;
-            throw new Error(`streamTask '${name}': merge applies to Dict (a function) or Set ('union') outputs, got ${output.type}${given}`);
-        }
+        throw new Error(`streamTask '${name}': the custom runtime runs only a program that returns its output — ` +
+            `use a stock runtime (east-node, east-py, east-c)`);
     }
-    const paramTypes = [
-        ...(spec.stream !== undefined ? [spec.stream.type] : []),
-        ...inputs.map((i) => i.type),
-        emitType,
-    ];
-    const bodyFn = East.function(paramTypes, NullType, fn);
-    const eastIR = bodyFn.toIR();
-    // Wire inputs: the body IR, the merge IR (function mode), the stream, then
-    // the ordinary inputs — the layout streamCommandIr's argv reads.
+    const inputs = spec.inputs;
+    checkPartitions(name, inputs.filter((i) => i.kind === 'partition'));
+    const datasets = inputs.map((i) => (i.kind === 'partition' ? i.dataset : i));
+    const parameters = [...datasets.map((d) => d.type), outputKind.emit];
+    const bodyFn = East.function(parameters, NullType, fn);
     const taskTree = createTaskTree(name);
-    const functionIRDataset = createIRDataset(name, taskTree, 'function_ir', eastIR);
-    const input_datasets = [
-        functionIRDataset,
-        ...(mergeIR !== undefined ? [createIRDataset(name, taskTree, 'merge_ir', mergeIR)] : []),
-        ...(spec.stream !== undefined ? [spec.stream] : []),
-        ...inputs,
-    ];
-    const outputDataset = createOutputDataset(name, taskTree, output);
-    // The runner constructs the emit sink for the output kind, folding equal
-    // keys as `merge` says, and feeds the stream (always the first `-i` input)
-    // lazily instead of whole-decoding it.
-    const command = streamCommandIr(runnerToVariant(runner), {
-        emit: emitKind,
-        merge: mergeMode,
-        stream: spec.stream !== undefined ? 'first' : 'none',
-    });
-    const metadata = encodeStreamTaskMetadata({
-        stream: spec.stream !== undefined,
-        emit: emitKind,
-        merge: mergeMode,
-    });
+    const output = createOutputDataset(name, taskTree, outputKind.type);
     const taskDef = {
         kind: 'task',
         name,
-        command,
-        inputs: input_datasets,
-        output: outputDataset,
-        deps: collectDeps(taskTree, outputDataset, input_datasets),
-        taskKind: TASK_KIND_STREAM,
-        metadata,
+        body: { kind: 'east', program: bodyFn.toIR() },
+        inputs,
+        output,
+        outputKind,
+        role: variant('data', null),
+        deps: collectDeps(taskTree, output, datasets),
         runner,
-        ...(spec.environment !== undefined && { environment: spec.environment }),
+        environment: spec.environment,
     };
-    outputDataset.deps.add(taskDef);
+    output.deps.add(taskDef);
     return taskDef;
 }
 //# sourceMappingURL=task.js.map