@elaraai/e3-types 1.0.76 → 1.0.78

package/dist/src/transfer.d.ts

@@ -7,8 +7,46 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType, type ValueTypeOf } from '@elaraai/east';
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export declare const TRANSFER_PROTOCOL_VERSION = 2;
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
+ */
+export declare function transferPartCount(size: number | bigint, partBytes: number | bigint): number;
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
  */
-import { VariantType, StructType, StringType, IntegerType, NullType, type ValueTypeOf } from '@elaraai/east';
+export declare function transferPartRange(size: number | bigint, partBytes: number | bigint, part: number): {
+    start: number;
+    end: number;
+} | null;
 /**
  * Transfer upload init request.
  *
@@ -26,7 +64,15 @@ export type TransferUploadRequest = ValueTypeOf<typeof TransferUploadRequestType
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export declare const TransferUploadResponseType: VariantType<{
     readonly completed: NullType;
@@ -34,19 +80,40 @@ export declare const TransferUploadResponseType: VariantType<{
         readonly id: StringType;
         readonly uploadUrl: StringType;
     }>;
+    readonly upload_parts: StructType<{
+        readonly id: StringType;
+        readonly partBytes: IntegerType;
+    }>;
 }>;
 export type TransferUploadResponse = ValueTypeOf<typeof TransferUploadResponseType>;
 /**
- * Transfer done response.
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export declare const TransferPartResponseType: StructType<{
+    readonly url: StringType;
+    readonly headers: DictType<StringType, StringType>;
+}>;
+export type TransferPartResponse = ValueTypeOf<typeof TransferPartResponseType>;
+/**
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export declare const TransferDoneResponseType: VariantType<{
     readonly completed: NullType;
     readonly error: StructType<{
         readonly message: StringType;
     }>;
+    readonly processing: NullType;
 }>;
 export type TransferDoneResponse = ValueTypeOf<typeof TransferDoneResponseType>;
 //# sourceMappingURL=transfer.d.ts.map

package/dist/src/transfer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAE7G;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;EAGpC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAElF;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B;;;;;;EAMrC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAEpF;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB;;;;;EAKnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC"}
+{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAEvH;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAI3F;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,SAAS,EAAE,MAAM,GAAG,MAAM,EAC1B,IAAI,EAAE,MAAM,GACX;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAKvC;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB;;;EAGpC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAElF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,0BAA0B;;;;;;;;;;EAUrC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAEpF;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB;;;EAGnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAEhF;;;;;;;GAOG;AACH,eAAO,MAAM,wBAAwB;;;;;;EAMnC,CAAC;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,CAAC,OAAO,wBAAwB,CAAC,CAAC"}

package/dist/src/transfer.js

@@ -7,8 +7,53 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType } from '@elaraai/east';
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export const TRANSFER_PROTOCOL_VERSION = 2;
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
+ */
+export function transferPartCount(size, partBytes) {
+    const total = BigInt(size);
+    const each = BigInt(partBytes);
+    return total <= each ? 1 : Number((total + each - 1n) / each);
+}
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
  */
-import { VariantType, StructType, StringType, IntegerType, NullType } from '@elaraai/east';
+export function transferPartRange(size, partBytes, part) {
+    if (!Number.isInteger(part) || part < 1 || part > transferPartCount(size, partBytes))
+        return null;
+    const start = BigInt(part - 1) * BigInt(partBytes);
+    const end = start + BigInt(partBytes) < BigInt(size) ? start + BigInt(partBytes) : BigInt(size);
+    return { start: Number(start), end: Number(end) };
+}
 /**
  * Transfer upload init request.
  *
@@ -25,7 +70,15 @@ export const TransferUploadRequestType = StructType({
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export const TransferUploadResponseType = VariantType({
     completed: NullType,
@@ -33,17 +86,37 @@ export const TransferUploadResponseType = VariantType({
         id: StringType,
         uploadUrl: StringType,
     }),
+    upload_parts: StructType({
+        id: StringType,
+        partBytes: IntegerType,
+    }),
+});
+/**
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export const TransferPartResponseType = StructType({
+    url: StringType,
+    headers: DictType(StringType, StringType),
 });
 /**
- * Transfer done response.
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export const TransferDoneResponseType = VariantType({
     completed: NullType,
     error: StructType({
         message: StringType,
     }),
+    processing: NullType,
 });
 //# sourceMappingURL=transfer.js.map

package/dist/src/transfer.js.map

@@ -1 +1 @@
-{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAoB,MAAM,eAAe,CAAC;AAE7G;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC;IACpD,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,UAAU,CAAC;QACjB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,UAAU;KACtB,CAAC;CACH,CAAC,CAAC;AAGH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;IAClD,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE,UAAU,CAAC;QAChB,OAAO,EAAE,UAAU;KACpB,CAAC;CACH,CAAC,CAAC"}
+{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../src/transfer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAoB,MAAM,eAAe,CAAC;AAEvH;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAqB,EAAE,SAA0B;IACjF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAqB,EACrB,SAA0B,EAC1B,IAAY;IAEZ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAClG,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,UAAU,CAAC;IAClD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;CAClB,CAAC,CAAC;AAGH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC;IACpD,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,UAAU,CAAC;QACjB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,UAAU;KACtB,CAAC;IACF,YAAY,EAAE,UAAU,CAAC;QACvB,EAAE,EAAE,UAAU;QACd,SAAS,EAAE,WAAW;KACvB,CAAC;CACH,CAAC,CAAC;AAGH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;IACjD,GAAG,EAAE,UAAU;IACf,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CAC1C,CAAC,CAAC;AAGH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC;IAClD,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE,UAAU,CAAC;QAChB,OAAO,EAAE,UAAU;KACpB,CAAC;IACF,UAAU,EAAE,QAAQ;CACrB,CAAC,CAAC"}

package/dist/src/transfer.spec.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+export {};
+//# sourceMappingURL=transfer.spec.d.ts.map

package/dist/src/transfer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.spec.d.ts","sourceRoot":"","sources":["../../src/transfer.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/src/transfer.spec.js

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+/**
+ * The dataset transfer protocol's two invariants that live in e3-types.
+ *
+ * Part arithmetic is shared by the client that cuts the bytes and the server
+ * that places them, so an off-by-one on either side would corrupt uploads
+ * silently until the commit's hash check. And protocol 2 only adds variant
+ * cases: a version-1 client decodes a version-1 answer from a new server, and a
+ * new client decodes an old server's answers, only while every added case sorts
+ * after the version-1 cases and so leaves their tags unchanged.
+ */
+import { describe, it } from 'node:test';
+import * as assert from 'node:assert';
+import { NullType, StringType, StructType, VariantType, decodeBeast2For, encodeBeast2For, variant } from '@elaraai/east';
+import { TransferDoneResponseType, TransferUploadResponseType, transferPartCount, transferPartRange, } from './transfer.js';
+describe('transferPartCount / transferPartRange', () => {
+    it('cuts an upload into full parts and a remainder', () => {
+        assert.strictEqual(transferPartCount(10, 4), 3);
+        assert.deepStrictEqual(transferPartRange(10, 4, 1), { start: 0, end: 4 });
+        assert.deepStrictEqual(transferPartRange(10, 4, 2), { start: 4, end: 8 });
+        assert.deepStrictEqual(transferPartRange(10, 4, 3), { start: 8, end: 10 });
+    });
+    it('gives an exact multiple no empty trailing part', () => {
+        assert.strictEqual(transferPartCount(8, 4), 2);
+        assert.deepStrictEqual(transferPartRange(8, 4, 2), { start: 4, end: 8 });
+        assert.strictEqual(transferPartRange(8, 4, 3), null);
+    });
+    it('sends an upload no larger than a part as one part', () => {
+        assert.strictEqual(transferPartCount(3, 4), 1);
+        assert.strictEqual(transferPartCount(4, 4), 1);
+        assert.deepStrictEqual(transferPartRange(3, 4, 1), { start: 0, end: 3 });
+    });
+    it('has no part 0, no fractional part and none past the end', () => {
+        assert.strictEqual(transferPartRange(10, 4, 0), null);
+        assert.strictEqual(transferPartRange(10, 4, 1.5), null);
+        assert.strictEqual(transferPartRange(10, 4, 4), null);
+    });
+    it('stays exact past 2^32 bytes, taking sizes as bigint or number', () => {
+        const size = 5n * 1024n ** 4n + 1n; // 5 TiB and a byte
+        const partBytes = 512n * 1024n ** 2n;
+        assert.strictEqual(transferPartCount(size, partBytes), 10241);
+        assert.deepStrictEqual(transferPartRange(Number(size), Number(partBytes), 10241), {
+            start: Number(size) - 1,
+            end: Number(size),
+        });
+    });
+});
+describe('transfer wire compatibility across protocol versions', () => {
+    // The version-1 shapes, as a client and server built before protocol 2 hold them.
+    const UploadResponseV1 = VariantType({
+        completed: NullType,
+        upload: StructType({ id: StringType, uploadUrl: StringType }),
+    });
+    const DoneResponseV1 = VariantType({
+        completed: NullType,
+        error: StructType({ message: StringType }),
+    });
+    it('keeps the init answers a version-1 client decodes', () => {
+        const upload = variant('upload', { id: 'abc', uploadUrl: 'http://h/api/uploads/abc' });
+        assert.deepStrictEqual(decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(upload)), upload);
+        assert.deepStrictEqual(decodeBeast2For(TransferUploadResponseType)(encodeBeast2For(UploadResponseV1)(upload)), upload);
+        const completed = variant('completed', null);
+        assert.deepStrictEqual(decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(completed)), completed);
+    });
+    it('keeps the commit answers a version-1 client decodes', () => {
+        for (const done of [variant('completed', null), variant('error', { message: 'hash mismatch' })]) {
+            assert.deepStrictEqual(decodeBeast2For(DoneResponseV1)(encodeBeast2For(TransferDoneResponseType)(done)), done);
+            assert.deepStrictEqual(decodeBeast2For(TransferDoneResponseType)(encodeBeast2For(DoneResponseV1)(done)), done);
+        }
+    });
+});
+//# sourceMappingURL=transfer.spec.js.map

package/dist/src/transfer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.spec.js","sourceRoot":"","sources":["../../src/transfer.spec.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;GASG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACzH,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAEvB,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,IAAI,GAAG,EAAE,GAAG,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,mBAAmB;QACvD,MAAM,SAAS,GAAG,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACrC,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,EAAE;YAChF,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;YACvB,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC;SAClB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sDAAsD,EAAE,GAAG,EAAE;IACpE,kFAAkF;IAClF,MAAM,gBAAgB,GAAG,WAAW,CAAC;QACnC,SAAS,EAAE,QAAQ;QACnB,MAAM,EAAE,UAAU,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;KAC9D,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,WAAW,CAAC;QACjC,SAAS,EAAE,QAAQ;QACnB,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;KAC3C,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;QACvF,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,gBAAgB,CAAC,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,MAAM,CAAC,CAAC,EACtF,MAAM,CACP,CAAC;QACF,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,0BAA0B,CAAC,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,EACtF,MAAM,CACP,CAAC;QACF,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,CAAC,eAAe,CACpB,eAAe,CAAC,gBAAgB,CAAC,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,SAAS,CAAC,CAAC,EACzF,SAAS,CACV,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAU,EAAE,CAAC;YACzG,MAAM,CAAC,eAAe,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAC/G,MAAM,CAAC,eAAe,CAAC,eAAe,CAAC,wBAAwB,CAAC,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elaraai/e3-types",
-  "version": "1.0.76",
+  "version": "1.0.78",
   "description": "Shared type definitions for e3 (East Execution Engine)",
   "type": "module",
   "main": "dist/src/index.js",
@@ -18,7 +18,7 @@
     "directory": "libs/e3/packages/e3-types"
   },
   "peerDependencies": {
-    "@elaraai/east": "1.0.76"
+    "@elaraai/east": "1.0.78"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/src/api.ts

@@ -69,6 +69,15 @@ export const DataflowErrorType = StructType({ message: StringType });
 export const PermissionDeniedErrorType = StructType({ path: StringType });
 export const InternalErrorType = StructType({ message: StringType });
 export const RepositoryNotFoundErrorType = StructType({ repo: StringType });
+/** A write refused at the door because the bytes' wire type is not the type the
+ *  dataset declares. `message` is the shared one-line rendering (declared type,
+ *  given type, first differing field) so a remote `e3 dataset set` prints the
+ *  line a local one does. */
+export const DatasetTypeMismatchErrorType = StructType({
+  workspace: StringType,
+  path: StringType,
+  message: StringType,
+});
 
 export const ErrorType = VariantType({
   repository_not_found: RepositoryNotFoundErrorType,
@@ -87,6 +96,9 @@ export const ErrorType = VariantType({
   dataflow_aborted: NullType,
   permission_denied: PermissionDeniedErrorType,
   internal: InternalErrorType,
+  // Appended last: BEAST2 encodes a variant case by index, so a case added at
+  // the end leaves every existing payload decodable.
+  dataset_type_mismatch: DatasetTypeMismatchErrorType,
 });
 
 // =============================================================================
@@ -725,6 +737,12 @@ export const DatasetStatusDetailType = StructType({
   refType: StringType,
   hash: OptionType(StringType),
   size: OptionType(IntegerType),
+  /** Segment and element counts of a stored collection, read from the blob's
+   *  trailing index — so a re-pointed input is inspectable without decoding
+   *  it. `none` for a non-collection or an unset dataset. Appended LAST, per
+   *  the positional struct rule. */
+  segments: OptionType(IntegerType),
+  rows: OptionType(IntegerType),
 });
 
 // =============================================================================

package/src/dataflow.ts

@@ -261,19 +261,23 @@ export type ExecutionEvent = ValueTypeOf<typeof ExecutionEventType>;
 
 /**
  * Progress notification for one unit of a partitioned task — a partition
- * slice execution or a combine step. Delivered through runner-layer
- * callbacks while the logical task runs. Deliberately NOT persisted as
- * execution events: {@link ExecutionEventType} is a frozen beast2 wire (see
- * its wire warning), and nothing consumes persisted partition progress —
- * local `[PART]`/`[MERGE]` lines come straight from this callback.
+ * slice execution, a combine step or a merge unit. Delivered through
+ * runner-layer callbacks while the logical task runs. Deliberately NOT
+ * persisted as execution events: {@link ExecutionEventType} is a frozen beast2
+ * wire (see its wire warning), and nothing consumes persisted partition
+ * progress — local `[PART]`/`[MERGE]`/`[COMBINE]` lines come straight from
+ * this callback.
  */
 export interface PartitionProgress {
   /** Which phase the unit belongs to. */
-  phase: 'partition' | 'combine';
+  phase: 'partition' | 'combine' | 'merge';
   /** Zero-based index of the unit within its phase. */
   index: number;
   /** Total units in the phase (partitions, or combine steps in the level). */
   total: number;
+  /** Units of the phase completed so far, including this one when `state`
+   *  is `completed`. */
+  completed: number;
   /** Whether the unit started or finished. */
   state: 'started' | 'completed';
   /** Whether the unit was served from the execution cache (completed only). */

package/src/dataset-type.ts

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The ONE check that a value's wire type is the type its dataset declares.
+ *
+ * No door into a dataset used to compare the two. `workspaceSetDataset`
+ * validated the path and the `writable` flag and then encoded with whatever
+ * type the caller passed; `e3 dataset set` passed a `.beast2` argument's own
+ * header type; the API `PUT` forwarded the body's header type; the transfer
+ * commit checked only that a collection blob carried an index. So a delivery
+ * whose schema had moved was accepted at the boundary and failed later, inside
+ * the consuming task — a decode error on the runner, or `cannot open a blob of
+ * type ... as ...` out of `FileSystem.openBeast` — naming neither the dataset
+ * that was set nor the field that moved.
+ *
+ * This lives in `e3-types` for the reason {@link encodeDatasetBlob} does: the
+ * rule has to hold at every door, and the doors are spread across `e3` (the
+ * export's `file` source check), `e3-core` (the set and the adopt) and
+ * `e3-api-server` (the `PUT` and the transfer commit), which share no package
+ * above this one.
+ *
+ * **Equality, not assignability.** A runner decodes the object *by the declared
+ * type* and beast2 decoding is type-directed, so a blob that is merely a
+ * subtype still decodes wrong. `isTypeValueEqual` is the relation.
+ *
+ * @packageDocumentation
+ */
+
+import {
+  diffTypeValues,
+  isTypeValueEqual,
+  isVariant,
+  printTypeValueSummary,
+  renderTypeDiff,
+  toEastTypeValue,
+  type EastType,
+  type EastTypeValue,
+  type TypeDiff,
+} from '@elaraai/east';
+
+/**
+ * A declared type and a wire type that disagree, localized to the first
+ * differing position.
+ */
+export interface DatasetTypeMismatch {
+  /** The type the dataset declares. */
+  readonly declared: EastTypeValue;
+  /** The type the bytes carry. */
+  readonly given: EastTypeValue;
+  /** Every localized difference — see `diffTypeValues`. */
+  readonly diffs: readonly TypeDiff[];
+  /** The one-line message every door reports. */
+  readonly message: string;
+}
+
+/** Normalizes either type representation to the homoiconic value form. */
+function asTypeValue(type: EastType | EastTypeValue): EastTypeValue {
+  return isVariant(type) ? (type as EastTypeValue) : toEastTypeValue(type as EastType);
+}
+
+/**
+ * Compares a dataset's declared type with the type some bytes carry.
+ *
+ * @param subject - what is being written, as it should read at the head of the
+ * message — e.g. `dataset '.inputs.table'` or `input 'table'`
+ * @param carrier - what carries the given type, e.g. `the value` or a file path
+ * @param declared - the type the dataset declares
+ * @param given - the type the bytes carry
+ * @returns `null` when the types are equal, else the localized mismatch
+ *
+ * @example
+ * ```ts
+ * const bad = checkDatasetType("dataset '.inputs.table'", 'the value', declared, header);
+ * if (bad) throw new DatasetTypeMismatchError(ws, '.inputs.table', bad);
+ * ```
+ */
+export function checkDatasetType(
+  subject: string,
+  carrier: string,
+  declared: EastType | EastTypeValue,
+  given: EastType | EastTypeValue,
+): DatasetTypeMismatch | null {
+  const declaredValue = asTypeValue(declared);
+  const givenValue = asTypeValue(given);
+  if (isTypeValueEqual(declaredValue, givenValue)) return null;
+
+  // Equality failed, so a diff normally localizes it — except where the two are
+  // assignable but unequal (a subtype, or the same struct fields in another
+  // order), which the assignability-shaped diff prunes to nothing. Say so
+  // plainly rather than printing a headline with no location under it.
+  const diffs = diffTypeValues(givenValue, declaredValue);
+  const first = diffs[0];
+  const where = first
+    ? `first difference at ${renderTypeDiff([first])}`
+    : 'the two are assignable but not identical — a dataset is decoded by its declared type, ' +
+      `so the bytes must carry exactly it${structuralHint(declaredValue, givenValue)}`;
+
+  return {
+    declared: declaredValue,
+    given: givenValue,
+    diffs,
+    message:
+      `${subject} declares ${printTypeValueSummary(declaredValue, 2)} but ${carrier} carries ` +
+      `${printTypeValueSummary(givenValue, 2)} — ${where}`,
+  };
+}
+
+/** Names the commonest assignable-but-unequal case: the same struct fields in a
+ *  different order (East struct field order is significant on the wire). */
+function structuralHint(declared: EastTypeValue, given: EastTypeValue): string {
+  if (declared.type !== 'Struct' || given.type !== 'Struct') return '';
+  const declaredOrder = (declared.value as { name: string }[]).map((f) => f.name);
+  const givenOrder = (given.value as { name: string }[]).map((f) => f.name);
+  if (declaredOrder.length !== givenOrder.length) return '';
+  const key = (names: string[]) => names.join('\\');
+  if (key([...declaredOrder].sort()) !== key([...givenOrder].sort())) return '';
+  if (key(declaredOrder) === key(givenOrder)) return '';
+  return ` (same fields, different order: expected ${declaredOrder.join(', ')}; found ${givenOrder.join(', ')})`;
+}
+
+/**
+ * Renders a dataset's tree path as it appears in messages: `.inputs.table`.
+ *
+ * Exported so every door spells the subject identically.
+ *
+ * @param segments - the path's field names, root-first
+ * @returns the dotted address, or `(root)` for an empty path
+ */
+export function datasetAddress(segments: readonly string[]): string {
+  return segments.length === 0 ? '(root)' : segments.map((s) => `.${s}`).join('');
+}

package/src/execution.ts

@@ -97,3 +97,21 @@ export const ExecutionStatusType = VariantType({
 });
 
 export type ExecutionStatus = ValueTypeOf<typeof ExecutionStatusType>;
+
+/**
+ * The orchestrator process that launched an execution (issue #770).
+ *
+ * @remarks
+ * Written beside the execution's `running` status as the `owner` sidecar. A
+ * `running` record whose runner is gone is repaired only when its owner is
+ * gone too: a live owner may be between the runner's exit and the record's
+ * write, hashing the output.
+ */
+export interface ExecutionOwner {
+  /** Process ID of the orchestrator */
+  pid: number;
+  /** Orchestrator start time in jiffies since boot (from /proc/<pid>/stat field 22) */
+  pidStartTime: number;
+  /** System boot ID (from /proc/sys/kernel/random/boot_id) */
+  bootId: string;
+}