@elaraai/e3-types 1.0.75 → 1.0.77

package/dist/src/package.js

@@ -60,6 +60,24 @@ export const PackageDatasetsType = PackageDataType;
  * };
  * ```
  */
+/**
+ * Where a dataset's initial value comes from, when it is not in the package.
+ *
+ * @remarks
+ * A package carries the DESCRIPTOR, never the bytes: `e3.input('table', T,
+ * variant('file', './TABLE.beast2'))` puts `{ file: { path } }` here and leaves
+ * the ref `unassigned`, and deploy resolves it — adopting the file into the
+ * object store by hash, so a new delivery under the same path is a new hash
+ * whose consumers re-run. An inline `variant('value', v)` needs no descriptor:
+ * its bytes are an object in the bundle like any other.
+ *
+ * A variant so that further source kinds can be appended later without a wire
+ * break; `file` is the only one today.
+ */
+export const DatasetSourceWireType = VariantType({
+    /** A path on the machine that DEPLOYS the package, resolved at deploy. */
+    file: StructType({ path: StringType }),
+});
 export const PackageObjectType = StructType({
     /** Tasks defined in this package: name -> task object hash */
     tasks: DictType(StringType, StringType),
@@ -71,6 +89,20 @@ export const PackageObjectType = StructType({
      *  BEAST2 encodes struct fields positionally in declaration order, so new
      *  fields MUST be appended LAST — never inserted between existing fields. */
     records: DictType(StringType, StringType),
+    /** Unresolved dataset sources: refPath (e.g. "inputs/table") -> descriptor.
+     *  Appended LAST, per the positional rule above. */
+    sources: DictType(StringType, DatasetSourceWireType),
+});
+/**
+ * The pre-`sources` package object wire shape (tasks, data, functions,
+ * records), kept only so {@link decodePackageObject} can read packages exported
+ * before path-initialised inputs existed.
+ */
+const RecordsEraPackageObjectType = StructType({
+    tasks: DictType(StringType, StringType),
+    data: PackageDataType,
+    functions: DictType(StringType, StringType),
+    records: DictType(StringType, StringType),
 });
 /**
  * The pre-`records` package object wire shape (tasks, data, functions), kept
@@ -92,16 +124,17 @@ const LegacyPackageObjectType = StructType({
     data: PackageDataType,
 });
 const decodeCurrent = decodeBeast2For(PackageObjectType);
+const decodeRecordsEra = decodeBeast2For(RecordsEraPackageObjectType);
 const decodeFunctionsEra = decodeBeast2For(FunctionsEraPackageObjectType);
 const decodeLegacy = decodeBeast2For(LegacyPackageObjectType);
 /**
- * Decode a `PackageObject` from BEAST2 bytes, tolerating the two older wire
+ * Decode a `PackageObject` from BEAST2 bytes, tolerating the older wire
  * formats (dual-decode migration).
  *
  * Every package-read path — local AND cloud — must use this instead of
  * `decodeBeast2For(PackageObjectType)` directly, so packages exported before
- * the `records`/`functions` fields existed keep decoding. Older bytes decode
- * with the missing maps defaulted to empty.
+ * the `sources`/`records`/`functions` fields existed keep decoding. Older
+ * bytes decode with the missing maps defaulted to empty.
  */
 export function decodePackageObject(data) {
     try {
@@ -109,16 +142,22 @@ export function decodePackageObject(data) {
     }
     catch (err) {
         try {
-            const fnEra = decodeFunctionsEra(data);
-            return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map() };
+            const recEra = decodeRecordsEra(data);
+            return { ...recEra, sources: new Map() };
         }
         catch {
             try {
-                const legacy = decodeLegacy(data);
-                return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map() };
+                const fnEra = decodeFunctionsEra(data);
+                return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map(), sources: new Map() };
             }
             catch {
-                throw err; // no known shape — surface the current-format error
+                try {
+                    const legacy = decodeLegacy(data);
+                    return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map(), sources: new Map() };
+                }
+                catch {
+                    throw err; // no known shape — surface the current-format error
+                }
             }
         }
     }

package/dist/src/package.js.map

@@ -1 +1 @@
-{"version":3,"file":"package.js","sourceRoot":"","sources":["../../src/package.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAe,eAAe,EAAE,MAAM,eAAe,CAAC;AACnI,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;IACxC,gEAAgE;IAChE,SAAS,EAAE,aAAa;IACxB,uEAAuE;IACvE,IAAI,EAAE,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;CAC3C,CAAC,CAAC;AAKH,gCAAgC;AAChC,8CAA8C;AAC9C,MAAM,CAAC,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAMnD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAC;IAC1C,8DAA8D;IAC9D,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,wCAAwC;IACxC,IAAI,EAAE,eAAe;IACrB,sEAAsE;IACtE,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAC3C;;iFAE6E;IAC7E,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC1C,CAAC,CAAC;AAKH;;;;GAIG;AACH,MAAM,6BAA6B,GAAG,UAAU,CAAC;IAC/C,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,IAAI,EAAE,eAAe;IACrB,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC5C,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,UAAU,CAAC;IACzC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,IAAI,EAAE,eAAe;CACtB,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,eAAe,CAAC,iBAAiB,CAAC,CAAC;AACzD,MAAM,kBAAkB,GAAG,eAAe,CAAC,6BAA6B,CAAC,CAAC;AAC1E,MAAM,YAAY,GAAG,eAAe,CAAC,uBAAuB,CAAC,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgB;IAClD,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QAClG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBAClC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAC9F,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,GAAG,CAAC,CAAC,oDAAoD;YACjE,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,yBAAyB;AACzB,gFAAgF;AAEhF,MAAM,CAAC,MAAM,8BAA8B,GAAG,UAAU,CAAC;IACvD,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,+BAA+B,GAAG,UAAU,CAAC;IACxD,EAAE,EAAE,UAAU;IACd,SAAS,EAAE,UAAU;CACtB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,sBAAsB,GAAG,UAAU,CAAC;IAC/C,EAAE,EAAE,UAAU;CACf,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,UAAU,CAAC;IAChD,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,UAAU;IACnB,WAAW,EAAE,UAAU;IACvB,WAAW,EAAE,WAAW;CACzB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,UAAU,CAAC;IAChD,WAAW,EAAE,UAAU;IACvB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC;IACnD,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,QAAQ;IACrB,SAAS,EAAE,UAAU,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC;CACzD,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC;IACjD,UAAU,EAAE,yBAAyB;IACrC,SAAS,EAAE,uBAAuB;IAClC,MAAM,EAAE,UAAU,CAAC;QACjB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC;IACnD,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,UAAU,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC;IACxD,SAAS,EAAE,QAAQ;CACpB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC;IACjD,UAAU,EAAE,yBAAyB;IACrC,SAAS,EAAE,uBAAuB;IAClC,MAAM,EAAE,UAAU,CAAC;QACjB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC"}
+{"version":3,"file":"package.js","sourceRoot":"","sources":["../../src/package.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAe,eAAe,EAAE,MAAM,eAAe,CAAC;AACnI,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;IACxC,gEAAgE;IAChE,SAAS,EAAE,aAAa;IACxB,uEAAuE;IACvE,IAAI,EAAE,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;CAC3C,CAAC,CAAC;AAKH,gCAAgC;AAChC,8CAA8C;AAC9C,MAAM,CAAC,MAAM,mBAAmB,GAAG,eAAe,CAAC;AAMnD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC;IAC/C,0EAA0E;IAC1E,IAAI,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;CACvC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,iBAAiB,GAAG,UAAU,CAAC;IAC1C,8DAA8D;IAC9D,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,wCAAwC;IACxC,IAAI,EAAE,eAAe;IACrB,sEAAsE;IACtE,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAC3C;;iFAE6E;IAC7E,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACzC;wDACoD;IACpD,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,qBAAqB,CAAC;CACrD,CAAC,CAAC;AAKH;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,UAAU,CAAC;IAC7C,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,IAAI,EAAE,eAAe;IACrB,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAC3C,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC1C,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,6BAA6B,GAAG,UAAU,CAAC;IAC/C,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,IAAI,EAAE,eAAe;IACrB,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC5C,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,UAAU,CAAC;IACzC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IACvC,IAAI,EAAE,eAAe;CACtB,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,eAAe,CAAC,iBAAiB,CAAC,CAAC;AACzD,MAAM,gBAAgB,GAAG,eAAe,CAAC,2BAA2B,CAAC,CAAC;AACtE,MAAM,kBAAkB,GAAG,eAAe,CAAC,6BAA6B,CAAC,CAAC;AAC1E,MAAM,YAAY,GAAG,eAAe,CAAC,uBAAuB,CAAC,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgB;IAClD,IAAI,CAAC;QACH,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACtC,OAAO,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACvC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YACtH,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;oBAClC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;gBAClH,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,GAAG,CAAC,CAAC,oDAAoD;gBACjE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,yBAAyB;AACzB,gFAAgF;AAEhF,MAAM,CAAC,MAAM,8BAA8B,GAAG,UAAU,CAAC;IACvD,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,+BAA+B,GAAG,UAAU,CAAC;IACxD,EAAE,EAAE,UAAU;IACd,SAAS,EAAE,UAAU;CACtB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,sBAAsB,GAAG,UAAU,CAAC;IAC/C,EAAE,EAAE,UAAU;CACf,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,UAAU,CAAC;IAChD,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,UAAU;IACnB,WAAW,EAAE,UAAU;IACvB,WAAW,EAAE,WAAW;CACzB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,UAAU,CAAC;IAChD,WAAW,EAAE,UAAU;IACvB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC;IACnD,OAAO,EAAE,QAAQ;IACjB,WAAW,EAAE,QAAQ;IACrB,SAAS,EAAE,UAAU,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC;CACzD,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC;IACjD,UAAU,EAAE,yBAAyB;IACrC,SAAS,EAAE,uBAAuB;IAClC,MAAM,EAAE,UAAU,CAAC;QACjB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC;IACnD,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,UAAU,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC;IACxD,SAAS,EAAE,QAAQ;CACpB,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC;IACjD,UAAU,EAAE,yBAAyB;IACrC,SAAS,EAAE,uBAAuB;IAClC,MAAM,EAAE,UAAU,CAAC;QACjB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC"}

package/dist/src/task.d.ts

@@ -142,13 +142,35 @@ export declare const PartitionTaskMetadataType: StructType<{
     readonly combine: OptionType<BlobType>;
     /** Target carved-slice size in wire bytes — the only sizing knob. */
     readonly targetPartitionBytes: IntegerType;
+    /**
+     * `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
+     * for a Dict output; `none` otherwise.
+     *
+     * Its presence (or {@link mergeSets}) selects the SEGMENT-MERGE assembly:
+     * partials are walked by their segment fences in the orchestrator, disjoint
+     * segments are byte-copied exactly as a splice does, and only overlapping
+     * ones are decoded, merged and re-encoded. Appended LAST (BEAST2 encodes
+     * struct fields positionally) with a dual decoder — see
+     * {@link decodePartitionTaskMetadata}.
+     */
+    readonly merge: OptionType<BlobType>;
+    /** Whether a Set output assembles by segment merge (union). The Set twin of
+     *  {@link merge}, which needs no function. Appended LAST. */
+    readonly mergeSets: BooleanType;
 }>;
 export type PartitionTaskMetadataType = typeof PartitionTaskMetadataType;
 export type PartitionTaskMetadata = ValueTypeOf<typeof PartitionTaskMetadataType>;
 /** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
 export declare const encodePartitionTaskMetadata: (value: PartitionTaskMetadata) => Uint8Array;
-/** Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task. */
-export declare const decodePartitionTaskMetadata: (data: Uint8Array) => PartitionTaskMetadata;
+/**
+ * Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task,
+ * tolerating the pre-`merge` wire format (dual-decode migration).
+ *
+ * @param data - the metadata blob
+ * @returns the decoded metadata, with `merge`/`mergeSets` defaulted off for
+ *   older bytes
+ */
+export declare function decodePartitionTaskMetadata(data: Uint8Array): PartitionTaskMetadata;
 /**
  * Metadata of a {@link TASK_KIND_STREAM} task.
  *

package/dist/src/task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAA0C,MAAM,eAAe,CAAC;AAIvK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,cAAc;IACzB,kFAAkF;;IAElF,uEAAuE;;;;IAEvE,sEAAsE;;;;IAEtE,sFAAsF;;IAEtF,6FAA6F;;IAE7F;;;;;;;OAOG;;;;;;;;;;;;;;;IAEH;;;;;OAKG;;EAEH,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAEnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAmB5D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAW7D;AAMD;;;;kEAIkE;AAClE,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAE/C;;;qDAGqD;AACrD,eAAO,MAAM,gBAAgB,WAAW,CAAC;AAEzC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,yBAAyB;IACpC,2EAA2E;;IAE3E;mFAC+E;;IAE/E;sDACkD;;IAElD,qEAAqE;;EAErE,CAAC;AACH,MAAM,MAAM,yBAAyB,GAAG,OAAO,yBAAyB,CAAC;AAEzE,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAElF,kFAAkF;AAClF,eAAO,MAAM,2BAA2B,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,UAChC,CAAC;AAE7C,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,qBACpB,CAAC;AAE7C;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB;IACjC;8BAC0B;;IAE1B;;qBAEiB;;EAEjB,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,sBAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAE5E,+EAA+E;AAC/E,eAAO,MAAM,wBAAwB,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,UAC7B,CAAC;AAE1C,8EAA8E;AAC9E,eAAO,MAAM,wBAAwB,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,kBACpB,CAAC"}
+{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAA0C,MAAM,eAAe,CAAC;AAIvK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,cAAc;IACzB,kFAAkF;;IAElF,uEAAuE;;;;IAEvE,sEAAsE;;;;IAEtE,sFAAsF;;IAEtF,6FAA6F;;IAE7F;;;;;;;OAOG;;;;;;;;;;;;;;;IAEH;;;;;OAKG;;EAEH,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC;AAEnD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,cAAc,CAAC,CAAC;AAmB5D;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAW7D;AAMD;;;;kEAIkE;AAClE,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAE/C;;;qDAGqD;AACrD,eAAO,MAAM,gBAAgB,WAAW,CAAC;AAEzC;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,yBAAyB;IACpC,2EAA2E;;IAE3E;mFAC+E;;IAE/E;sDACkD;;IAElD,qEAAqE;;IAErE;;;;;;;;;;OAUG;;IAEH;iEAC6D;;EAE7D,CAAC;AACH,MAAM,MAAM,yBAAyB,GAAG,OAAO,yBAAyB,CAAC;AAEzE,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAclF,kFAAkF;AAClF,eAAO,MAAM,2BAA2B,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,UAChC,CAAC;AAK7C;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,UAAU,GAAG,qBAAqB,CAUnF;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB;IACjC;8BAC0B;;IAE1B;;qBAEiB;;EAEjB,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,OAAO,sBAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAE5E,+EAA+E;AAC/E,eAAO,MAAM,wBAAwB,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,UAC7B,CAAC;AAE1C,8EAA8E;AAC9E,eAAO,MAAM,wBAAwB,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,kBACpB,CAAC"}

package/dist/src/task.js

@@ -156,11 +156,58 @@ export const PartitionTaskMetadataType = StructType({
     combine: OptionType(BlobType),
     /** Target carved-slice size in wire bytes — the only sizing knob. */
     targetPartitionBytes: IntegerType,
+    /**
+     * `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
+     * for a Dict output; `none` otherwise.
+     *
+     * Its presence (or {@link mergeSets}) selects the SEGMENT-MERGE assembly:
+     * partials are walked by their segment fences in the orchestrator, disjoint
+     * segments are byte-copied exactly as a splice does, and only overlapping
+     * ones are decoded, merged and re-encoded. Appended LAST (BEAST2 encodes
+     * struct fields positionally) with a dual decoder — see
+     * {@link decodePartitionTaskMetadata}.
+     */
+    merge: OptionType(BlobType),
+    /** Whether a Set output assembles by segment merge (union). The Set twin of
+     *  {@link merge}, which needs no function. Appended LAST. */
+    mergeSets: BooleanType,
+});
+/**
+ * The pre-`merge` partition metadata wire shape, kept only so
+ * {@link decodePartitionTaskMetadata} can read tasks exported before the
+ * segment-merge assembly existed.
+ */
+const PreMergePartitionTaskMetadataType = StructType({
+    partitions: IntegerType,
+    by: OptionType(BlobType),
+    combine: OptionType(BlobType),
+    targetPartitionBytes: IntegerType,
 });
 /** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
 export const encodePartitionTaskMetadata = encodeBeast2For(PartitionTaskMetadataType);
-/** Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task. */
-export const decodePartitionTaskMetadata = decodeBeast2For(PartitionTaskMetadataType);
+const decodeCurrentPartitionMetadata = decodeBeast2For(PartitionTaskMetadataType);
+const decodePreMergePartitionMetadata = decodeBeast2For(PreMergePartitionTaskMetadataType);
+/**
+ * Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task,
+ * tolerating the pre-`merge` wire format (dual-decode migration).
+ *
+ * @param data - the metadata blob
+ * @returns the decoded metadata, with `merge`/`mergeSets` defaulted off for
+ *   older bytes
+ */
+export function decodePartitionTaskMetadata(data) {
+    try {
+        return decodeCurrentPartitionMetadata(data);
+    }
+    catch (err) {
+        try {
+            return { ...decodePreMergePartitionMetadata(data), merge: none, mergeSets: false };
+        }
+        catch {
+            throw err; // no known shape — surface the current-format error
+        }
+    }
+}
 /**
  * Metadata of a {@link TASK_KIND_STREAM} task.
  *

package/dist/src/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAe,eAAe,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACvK,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,UAAU,CAAC;IACvC,kFAAkF;IAClF,SAAS,EAAE,UAAU;IACrB,uEAAuE;IACvE,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC;IAC/B,sEAAsE;IACtE,MAAM,EAAE,YAAY;IACpB,sFAAsF;IACtF,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;IAC5B,6FAA6F;IAC7F,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC9B;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU;IAClB;;;;;OAKG;IACH,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;CACpC,CAAC,CAAC;AAKH;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,UAAU,CAAC;IAC9C,SAAS,EAAE,UAAU;IACrB,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC;IAC/B,MAAM,EAAE,YAAY;IACpB,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC9B,MAAM,EAAE,UAAU;CACnB,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;AAC1D,MAAM,wBAAwB,GAAG,eAAe,CAAC,4BAA4B,CAAC,CAAC;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,IAAI,CAAC;QACH,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAC9C,OAAO,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,CAAC,CAAC,oDAAoD;QACjE,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,gCAAgC;AAChC,gFAAgF;AAEhF;;;;kEAIkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAE/C;;;qDAGqD;AACrD,MAAM,CAAC,MAAM,gBAAgB,GAAG,QAAQ,CAAC;AAEzC;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,2EAA2E;IAC3E,UAAU,EAAE,WAAW;IACvB;mFAC+E;IAC/E,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;IACxB;sDACkD;IAClD,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC7B,qEAAqE;IACrE,oBAAoB,EAAE,WAAW;CAClC,CAAC,CAAC;AAKH,kFAAkF;AAClF,MAAM,CAAC,MAAM,2BAA2B,GACtC,eAAe,CAAC,yBAAyB,CAAC,CAAC;AAE7C,iFAAiF;AACjF,MAAM,CAAC,MAAM,2BAA2B,GACtC,eAAe,CAAC,yBAAyB,CAAC,CAAC;AAE7C;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,UAAU,CAAC;IAC/C;8BAC0B;IAC1B,MAAM,EAAE,WAAW;IACnB;;qBAEiB;IACjB,IAAI,EAAE,UAAU;CACjB,CAAC,CAAC;AAKH,+EAA+E;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GACnC,eAAe,CAAC,sBAAsB,CAAC,CAAC;AAE1C,8EAA8E;AAC9E,MAAM,CAAC,MAAM,wBAAwB,GACnC,eAAe,CAAC,sBAAsB,CAAC,CAAC"}
+{"version":3,"file":"task.js","sourceRoot":"","sources":["../../src/task.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAe,eAAe,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACvK,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,UAAU,CAAC;IACvC,kFAAkF;IAClF,SAAS,EAAE,UAAU;IACrB,uEAAuE;IACvE,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC;IAC/B,sEAAsE;IACtE,MAAM,EAAE,YAAY;IACpB,sFAAsF;IACtF,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;IAC5B,6FAA6F;IAC7F,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC9B;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU;IAClB;;;;;OAKG;IACH,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;CACpC,CAAC,CAAC;AAKH;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,UAAU,CAAC;IAC9C,SAAS,EAAE,UAAU;IACrB,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC;IAC/B,MAAM,EAAE,YAAY;IACpB,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC9B,MAAM,EAAE,UAAU;CACnB,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;AAC1D,MAAM,wBAAwB,GAAG,eAAe,CAAC,4BAA4B,CAAC,CAAC;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAgB;IAC/C,IAAI,CAAC;QACH,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAC9C,OAAO,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,CAAC,CAAC,oDAAoD;QACjE,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,gCAAgC;AAChC,gFAAgF;AAEhF;;;;kEAIkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAE/C;;;qDAGqD;AACrD,MAAM,CAAC,MAAM,gBAAgB,GAAG,QAAQ,CAAC;AAEzC;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,2EAA2E;IAC3E,UAAU,EAAE,WAAW;IACvB;mFAC+E;IAC/E,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;IACxB;sDACkD;IAClD,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC7B,qEAAqE;IACrE,oBAAoB,EAAE,WAAW;IACjC;;;;;;;;;;OAUG;IACH,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC3B;iEAC6D;IAC7D,SAAS,EAAE,WAAW;CACvB,CAAC,CAAC;AAKH;;;;GAIG;AACH,MAAM,iCAAiC,GAAG,UAAU,CAAC;IACnD,UAAU,EAAE,WAAW;IACvB,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;IACxB,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC;IAC7B,oBAAoB,EAAE,WAAW;CAClC,CAAC,CAAC;AAEH,kFAAkF;AAClF,MAAM,CAAC,MAAM,2BAA2B,GACtC,eAAe,CAAC,yBAAyB,CAAC,CAAC;AAE7C,MAAM,8BAA8B,GAAG,eAAe,CAAC,yBAAyB,CAAC,CAAC;AAClF,MAAM,+BAA+B,GAAG,eAAe,CAAC,iCAAiC,CAAC,CAAC;AAE3F;;;;;;;GAOG;AACH,MAAM,UAAU,2BAA2B,CAAC,IAAgB;IAC1D,IAAI,CAAC;QACH,OAAO,8BAA8B,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,OAAO,EAAE,GAAG,+BAA+B,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACrF,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,CAAC,CAAC,oDAAoD;QACjE,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,UAAU,CAAC;IAC/C;8BAC0B;IAC1B,MAAM,EAAE,WAAW;IACnB;;qBAEiB;IACjB,IAAI,EAAE,UAAU;CACjB,CAAC,CAAC;AAKH,+EAA+E;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GACnC,eAAe,CAAC,sBAAsB,CAAC,CAAC;AAE1C,8EAA8E;AAC9E,MAAM,CAAC,MAAM,wBAAwB,GACnC,eAAe,CAAC,sBAAsB,CAAC,CAAC"}

package/dist/src/transfer.d.ts

@@ -7,8 +7,46 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType, type ValueTypeOf } from '@elaraai/east';
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export declare const TRANSFER_PROTOCOL_VERSION = 2;
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
+ */
+export declare function transferPartCount(size: number | bigint, partBytes: number | bigint): number;
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
  */
-import { VariantType, StructType, StringType, IntegerType, NullType, type ValueTypeOf } from '@elaraai/east';
+export declare function transferPartRange(size: number | bigint, partBytes: number | bigint, part: number): {
+    start: number;
+    end: number;
+} | null;
 /**
  * Transfer upload init request.
  *
@@ -26,7 +64,15 @@ export type TransferUploadRequest = ValueTypeOf<typeof TransferUploadRequestType
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export declare const TransferUploadResponseType: VariantType<{
     readonly completed: NullType;
@@ -34,19 +80,40 @@ export declare const TransferUploadResponseType: VariantType<{
         readonly id: StringType;
         readonly uploadUrl: StringType;
     }>;
+    readonly upload_parts: StructType<{
+        readonly id: StringType;
+        readonly partBytes: IntegerType;
+    }>;
 }>;
 export type TransferUploadResponse = ValueTypeOf<typeof TransferUploadResponseType>;
 /**
- * Transfer done response.
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export declare const TransferPartResponseType: StructType<{
+    readonly url: StringType;
+    readonly headers: DictType<StringType, StringType>;
+}>;
+export type TransferPartResponse = ValueTypeOf<typeof TransferPartResponseType>;
+/**
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export declare const TransferDoneResponseType: VariantType<{
     readonly completed: NullType;
     readonly error: StructType<{
         readonly message: StringType;
     }>;
+    readonly processing: NullType;
 }>;
 export type TransferDoneResponse = ValueTypeOf<typeof TransferDoneResponseType>;
 //# sourceMappingURL=transfer.d.ts.map

package/dist/src/transfer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAE7G;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;EAGpC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAElF;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B;;;;;;EAMrC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAEpF;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB;;;;;EAKnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC"}
+{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAEvH;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAI3F;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,SAAS,EAAE,MAAM,GAAG,MAAM,EAC1B,IAAI,EAAE,MAAM,GACX;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;EAGpC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAElF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,0BAA0B;;;;;;;;;;EAUrC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAEpF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB;;;EAGnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAEhF;;;;;;;GAOG;AACH,eAAO,MAAM,wBAAwB;;;;;;EAMnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC"}

package/dist/src/transfer.js

@@ -7,8 +7,53 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType } from '@elaraai/east';
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export const TRANSFER_PROTOCOL_VERSION = 2;
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
+ */
+export function transferPartCount(size, partBytes) {
+    const total = BigInt(size);
+    const each = BigInt(partBytes);
+    return total <= each ? 1 : Number((total + each - 1n) / each);
+}
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
  */
-import { VariantType, StructType, StringType, IntegerType, NullType } from '@elaraai/east';
+export function transferPartRange(size, partBytes, part) {
+    if (!Number.isInteger(part) || part < 1 || part > transferPartCount(size, partBytes))
+        return null;
+    const start = BigInt(part - 1) * BigInt(partBytes);
+    const end = start + BigInt(partBytes) < BigInt(size) ? start + BigInt(partBytes) : BigInt(size);
+    return { start: Number(start), end: Number(end) };
+}
 /**
  * Transfer upload init request.
  *
@@ -25,7 +70,15 @@ export const TransferUploadRequestType = StructType({
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export const TransferUploadResponseType = VariantType({
     completed: NullType,
@@ -33,17 +86,37 @@ export const TransferUploadResponseType = VariantType({
         id: StringType,
         uploadUrl: StringType,
     }),
+    upload_parts: StructType({
+        id: StringType,
+        partBytes: IntegerType,
+    }),
+});
+/**
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export const TransferPartResponseType = StructType({
+    url: StringType,
+    headers: DictType(StringType, StringType),
 });
 /**
- * Transfer done response.
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export const TransferDoneResponseType = VariantType({
     completed: NullType,
     error: StructType({
         message: StringType,
     }),
+    processing: NullType,
 });
 //# sourceMappingURL=transfer.js.map

package/dist/src/transfer.js.map

@@ -1 +1 @@
-{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAoB,MAAM,eAAe,CAAC;AAE7G;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC;IACpD,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,UAAU,CAAC;QACjB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,UAAU;KACtB,CAAC;CACH,CAAC,CAAC;AAGH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;IAClD,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE,UAAU,CAAC;QAChB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC"}
+{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAoB,MAAM,eAAe,CAAC;AAEvH;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAqB,EAAE,SAA0B;IACjF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAqB,EACrB,SAA0B,EAC1B,IAAY;IAEZ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAClG,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC;IACpD,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,UAAU,CAAC;QACjB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,UAAU;KACtB,CAAC;IACF,YAAY,EAAE,UAAU,CAAC;QACvB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,WAAW;KACvB,CAAC;CACH,CAAC,CAAC;AAGH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;IACjD,GAAG,EAAE,UAAU;IACf,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC1C,CAAC,CAAC;AAGH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;IAClD,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE,UAAU,CAAC;QAChB,OAAO,EAAE,UAAU;KACpB,CAAC;IACF,UAAU,EAAE,QAAQ;CACrB,CAAC,CAAC"}

package/dist/src/transfer.spec.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+export {};
+//# sourceMappingURL=transfer.spec.d.ts.map

package/dist/src/transfer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.spec.d.ts","sourceRoot":"","sources":["../../src/transfer.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/src/transfer.spec.js

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+/**
+ * The dataset transfer protocol's two invariants that live in e3-types.
+ *
+ * Part arithmetic is shared by the client that cuts the bytes and the server
+ * that places them, so an off-by-one on either side would corrupt uploads
+ * silently until the commit's hash check. And protocol 2 only adds variant
+ * cases: a version-1 client decodes a version-1 answer from a new server, and a
+ * new client decodes an old server's answers, only while every added case sorts
+ * after the version-1 cases and so leaves their tags unchanged.
+ */
+import { describe, it } from 'node:test';
+import * as assert from 'node:assert';
+import { NullType, StringType, StructType, VariantType, decodeBeast2For, encodeBeast2For, variant } from '@elaraai/east';
+import { TransferDoneResponseType, TransferUploadResponseType, transferPartCount, transferPartRange, } from './transfer.js';
+describe('transferPartCount / transferPartRange', () => {
+    it('cuts an upload into full parts and a remainder', () => {
+        assert.strictEqual(transferPartCount(10, 4), 3);
+        assert.deepStrictEqual(transferPartRange(10, 4, 1), { start: 0, end: 4 });
+        assert.deepStrictEqual(transferPartRange(10, 4, 2), { start: 4, end: 8 });
+        assert.deepStrictEqual(transferPartRange(10, 4, 3), { start: 8, end: 10 });
+    });
+    it('gives an exact multiple no empty trailing part', () => {
+        assert.strictEqual(transferPartCount(8, 4), 2);
+        assert.deepStrictEqual(transferPartRange(8, 4, 2), { start: 4, end: 8 });
+        assert.strictEqual(transferPartRange(8, 4, 3), null);
+    });
+    it('sends an upload no larger than a part as one part', () => {
+        assert.strictEqual(transferPartCount(3, 4), 1);
+        assert.strictEqual(transferPartCount(4, 4), 1);
+        assert.deepStrictEqual(transferPartRange(3, 4, 1), { start: 0, end: 3 });
+    });
+    it('has no part 0, no fractional part and none past the end', () => {
+        assert.strictEqual(transferPartRange(10, 4, 0), null);
+        assert.strictEqual(transferPartRange(10, 4, 1.5), null);
+        assert.strictEqual(transferPartRange(10, 4, 4), null);
+    });
+    it('stays exact past 2^32 bytes, taking sizes as bigint or number', () => {
+        const size = 5n * 1024n ** 4n + 1n; // 5 TiB and a byte
+        const partBytes = 512n * 1024n ** 2n;
+        assert.strictEqual(transferPartCount(size, partBytes), 10241);
+        assert.deepStrictEqual(transferPartRange(Number(size), Number(partBytes), 10241), {
+            start: Number(size) - 1,
+            end: Number(size),
+        });
+    });
+});
+describe('transfer wire compatibility across protocol versions', () => {
+    // The version-1 shapes, as a client and server built before protocol 2 hold them.
+    const UploadResponseV1 = VariantType({
+        completed: NullType,
+        upload: StructType({ id: StringType, uploadUrl: StringType }),
+    });
+    const DoneResponseV1 = VariantType({
+        completed: NullType,
+        error: StructType({ message: StringType }),
+    });
+    it('keeps the init answers a version-1 client decodes', () => {
+        const upload = variant('upload', { id: 'abc', uploadUrl: 'http://h/api/uploads/abc' });
+        assert.deepStrictEqual(decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(upload)), upload);
+        assert.deepStrictEqual(decodeBeast2For(TransferUploadResponseType)(encodeBeast2For(UploadResponseV1)(upload)), upload);
+        const completed = variant('completed', null);
+        assert.deepStrictEqual(decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(completed)), completed);
+    });
+    it('keeps the commit answers a version-1 client decodes', () => {
+        for (const done of [variant('completed', null), variant('error', { message: 'hash mismatch' })]) {
+            assert.deepStrictEqual(decodeBeast2For(DoneResponseV1)(encodeBeast2For(TransferDoneResponseType)(done)), done);
+            assert.deepStrictEqual(decodeBeast2For(TransferDoneResponseType)(encodeBeast2For(DoneResponseV1)(done)), done);
+        }
+    });
+});
+//# sourceMappingURL=transfer.spec.js.map

package/dist/src/transfer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.spec.js","sourceRoot":"","sources":["../../src/transfer.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;GASG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACzH,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAEvB,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,IAAI,GAAG,EAAE,GAAG,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,mBAAmB;QACvD,MAAM,SAAS,GAAG,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACrC,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,EAAE;YAChF,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;YACvB,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC;SAClB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sDAAsD,EAAE,GAAG,EAAE;IACpE,kFAAkF;IAClF,MAAM,gBAAgB,GAAG,WAAW,CAAC;QACnC,SAAS,EAAE,QAAQ;QACnB,MAAM,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;KAC9D,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,WAAW,CAAC;QACjC,SAAS,EAAE,QAAQ;QACnB,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;KAC3C,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACvF,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,gBAAgB,CAAC,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,MAAM,CAAC,CAAC,EACtF,MAAM,CACP,CAAC;QACF,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,0BAA0B,CAAC,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,EACtF,MAAM,CACP,CAAC;QACF,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,gBAAgB,CAAC,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,SAAS,CAAC,CAAC,EACzF,SAAS,CACV,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAU,EAAE,CAAC;YACzG,MAAM,CAAC,eAAe,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAC/G,MAAM,CAAC,eAAe,CAAC,eAAe,CAAC,wBAAwB,CAAC,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elaraai/e3-types",
-  "version": "1.0.75",
+  "version": "1.0.77",
   "description": "Shared type definitions for e3 (East Execution Engine)",
   "type": "module",
   "main": "dist/src/index.js",
@@ -18,7 +18,7 @@
     "directory": "libs/e3/packages/e3-types"
   },
   "peerDependencies": {
-    "@elaraai/east": "1.0.75"
+    "@elaraai/east": "1.0.77"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/src/api.ts

@@ -69,6 +69,15 @@ export const DataflowErrorType = StructType({ message: StringType });
 export const PermissionDeniedErrorType = StructType({ path: StringType });
 export const InternalErrorType = StructType({ message: StringType });
 export const RepositoryNotFoundErrorType = StructType({ repo: StringType });
+/** A write refused at the door because the bytes' wire type is not the type the
+ *  dataset declares. `message` is the shared one-line rendering (declared type,
+ *  given type, first differing field) so a remote `e3 dataset set` prints the
+ *  line a local one does. */
+export const DatasetTypeMismatchErrorType = StructType({
+  workspace: StringType,
+  path: StringType,
+  message: StringType,
+});
 
 export const ErrorType = VariantType({
   repository_not_found: RepositoryNotFoundErrorType,
@@ -87,6 +96,9 @@ export const ErrorType = VariantType({
   dataflow_aborted: NullType,
   permission_denied: PermissionDeniedErrorType,
   internal: InternalErrorType,
+  // Appended last: BEAST2 encodes a variant case by index, so a case added at
+  // the end leaves every existing payload decodable.
+  dataset_type_mismatch: DatasetTypeMismatchErrorType,
 });
 
 // =============================================================================
@@ -725,6 +737,12 @@ export const DatasetStatusDetailType = StructType({
   refType: StringType,
   hash: OptionType(StringType),
   size: OptionType(IntegerType),
+  /** Segment and element counts of a stored collection, read from the blob's
+   *  trailing index — so a re-pointed input is inspectable without decoding
+   *  it. `none` for a non-collection or an unset dataset. Appended LAST, per
+   *  the positional struct rule. */
+  segments: OptionType(IntegerType),
+  rows: OptionType(IntegerType),
 });
 
 // =============================================================================