@elaraai/e3-types 1.0.75 → 1.0.77

package/src/dataset-type.ts

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The ONE check that a value's wire type is the type its dataset declares.
+ *
+ * No door into a dataset used to compare the two. `workspaceSetDataset`
+ * validated the path and the `writable` flag and then encoded with whatever
+ * type the caller passed; `e3 dataset set` passed a `.beast2` argument's own
+ * header type; the API `PUT` forwarded the body's header type; the transfer
+ * commit checked only that a collection blob carried an index. So a delivery
+ * whose schema had moved was accepted at the boundary and failed later, inside
+ * the consuming task — a decode error on the runner, or `cannot open a blob of
+ * type ... as ...` out of `FileSystem.openBeast` — naming neither the dataset
+ * that was set nor the field that moved.
+ *
+ * This lives in `e3-types` for the reason {@link encodeDatasetBlob} does: the
+ * rule has to hold at every door, and the doors are spread across `e3` (the
+ * export's `file` source check), `e3-core` (the set and the adopt) and
+ * `e3-api-server` (the `PUT` and the transfer commit), which share no package
+ * above this one.
+ *
+ * **Equality, not assignability.** A runner decodes the object *by the declared
+ * type* and beast2 decoding is type-directed, so a blob that is merely a
+ * subtype still decodes wrong. `isTypeValueEqual` is the relation.
+ *
+ * @packageDocumentation
+ */
+
+import {
+  diffTypeValues,
+  isTypeValueEqual,
+  isVariant,
+  printTypeValueSummary,
+  renderTypeDiff,
+  toEastTypeValue,
+  type EastType,
+  type EastTypeValue,
+  type TypeDiff,
+} from '@elaraai/east';
+
+/**
+ * A declared type and a wire type that disagree, localized to the first
+ * differing position.
+ */
+export interface DatasetTypeMismatch {
+  /** The type the dataset declares. */
+  readonly declared: EastTypeValue;
+  /** The type the bytes carry. */
+  readonly given: EastTypeValue;
+  /** Every localized difference — see `diffTypeValues`. */
+  readonly diffs: readonly TypeDiff[];
+  /** The one-line message every door reports. */
+  readonly message: string;
+}
+
+/** Normalizes either type representation to the homoiconic value form. */
+function asTypeValue(type: EastType | EastTypeValue): EastTypeValue {
+  return isVariant(type) ? (type as EastTypeValue) : toEastTypeValue(type as EastType);
+}
+
+/**
+ * Compares a dataset's declared type with the type some bytes carry.
+ *
+ * @param subject - what is being written, as it should read at the head of the
+ * message — e.g. `dataset '.inputs.table'` or `input 'table'`
+ * @param carrier - what carries the given type, e.g. `the value` or a file path
+ * @param declared - the type the dataset declares
+ * @param given - the type the bytes carry
+ * @returns `null` when the types are equal, else the localized mismatch
+ *
+ * @example
+ * ```ts
+ * const bad = checkDatasetType("dataset '.inputs.table'", 'the value', declared, header);
+ * if (bad) throw new DatasetTypeMismatchError(ws, '.inputs.table', bad);
+ * ```
+ */
+export function checkDatasetType(
+  subject: string,
+  carrier: string,
+  declared: EastType | EastTypeValue,
+  given: EastType | EastTypeValue,
+): DatasetTypeMismatch | null {
+  const declaredValue = asTypeValue(declared);
+  const givenValue = asTypeValue(given);
+  if (isTypeValueEqual(declaredValue, givenValue)) return null;
+
+  // Equality failed, so a diff normally localizes it — except where the two are
+  // assignable but unequal (a subtype, or the same struct fields in another
+  // order), which the assignability-shaped diff prunes to nothing. Say so
+  // plainly rather than printing a headline with no location under it.
+  const diffs = diffTypeValues(givenValue, declaredValue);
+  const first = diffs[0];
+  const where = first
+    ? `first difference at ${renderTypeDiff([first])}`
+    : 'the two are assignable but not identical — a dataset is decoded by its declared type, ' +
+      `so the bytes must carry exactly it${structuralHint(declaredValue, givenValue)}`;
+
+  return {
+    declared: declaredValue,
+    given: givenValue,
+    diffs,
+    message:
+      `${subject} declares ${printTypeValueSummary(declaredValue, 2)} but ${carrier} carries ` +
+      `${printTypeValueSummary(givenValue, 2)} — ${where}`,
+  };
+}
+
+/** Names the commonest assignable-but-unequal case: the same struct fields in a
+ *  different order (East struct field order is significant on the wire). */
+function structuralHint(declared: EastTypeValue, given: EastTypeValue): string {
+  if (declared.type !== 'Struct' || given.type !== 'Struct') return '';
+  const declaredOrder = (declared.value as { name: string }[]).map((f) => f.name);
+  const givenOrder = (given.value as { name: string }[]).map((f) => f.name);
+  if (declaredOrder.length !== givenOrder.length) return '';
+  const key = (names: string[]) => names.join('\\');
+  if (key([...declaredOrder].sort()) !== key([...givenOrder].sort())) return '';
+  if (key(declaredOrder) === key(givenOrder)) return '';
+  return ` (same fields, different order: expected ${declaredOrder.join(', ')}; found ${givenOrder.join(', ')})`;
+}
+
+/**
+ * Renders a dataset's tree path as it appears in messages: `.inputs.table`.
+ *
+ * Exported so every door spells the subject identically.
+ *
+ * @param segments - the path's field names, root-first
+ * @returns the dotted address, or `(root)` for an empty path
+ */
+export function datasetAddress(segments: readonly string[]): string {
+  return segments.length === 0 ? '(root)' : segments.map((s) => `.${s}`).join('');
+}

package/src/index.ts

@@ -126,6 +126,8 @@ export {
   type PackageData,
   PackageObjectType,
   type PackageObject,
+  DatasetSourceWireType,
+  type DatasetSourceWire,
   decodePackageObject,
   // Backwards compatibility
   PackageDatasetsType,
@@ -175,10 +177,15 @@ export {
 
 // Dataset transfer types
 export {
+  TRANSFER_PROTOCOL_VERSION,
+  transferPartCount,
+  transferPartRange,
   TransferUploadRequestType,
   type TransferUploadRequest,
   TransferUploadResponseType,
   type TransferUploadResponse,
+  TransferPartResponseType,
+  type TransferPartResponse,
   TransferDoneResponseType,
   type TransferDoneResponse,
 } from './transfer.js';
@@ -190,6 +197,14 @@ export {
   encodeDatasetBlob,
 } from './dataset-blob.js';
 
+// The ONE declared-type-vs-wire-type check, shared by every door a value
+// enters a dataset through (the export, the set, the adopt, the API)
+export {
+  checkDatasetType,
+  datasetAddress,
+  type DatasetTypeMismatch,
+} from './dataset-type.js';
+
 // Wire format constants
 export { BEAST2_CONTENT_TYPE } from './constants.js';
 
@@ -212,6 +227,7 @@ export {
   PermissionDeniedErrorType,
   InternalErrorType,
   RepositoryNotFoundErrorType,
+  DatasetTypeMismatchErrorType,
   ErrorType,
   ResponseType,
   // Repository

package/src/package.ts

@@ -72,6 +72,27 @@ export type PackageDatasets = PackageData;
  * };
  * ```
  */
+/**
+ * Where a dataset's initial value comes from, when it is not in the package.
+ *
+ * @remarks
+ * A package carries the DESCRIPTOR, never the bytes: `e3.input('table', T,
+ * variant('file', './TABLE.beast2'))` puts `{ file: { path } }` here and leaves
+ * the ref `unassigned`, and deploy resolves it — adopting the file into the
+ * object store by hash, so a new delivery under the same path is a new hash
+ * whose consumers re-run. An inline `variant('value', v)` needs no descriptor:
+ * its bytes are an object in the bundle like any other.
+ *
+ * A variant so that further source kinds can be appended later without a wire
+ * break; `file` is the only one today.
+ */
+export const DatasetSourceWireType = VariantType({
+  /** A path on the machine that DEPLOYS the package, resolved at deploy. */
+  file: StructType({ path: StringType }),
+});
+export type DatasetSourceWireType = typeof DatasetSourceWireType;
+export type DatasetSourceWire = ValueTypeOf<typeof DatasetSourceWireType>;
+
 export const PackageObjectType = StructType({
   /** Tasks defined in this package: name -> task object hash */
   tasks: DictType(StringType, StringType),
@@ -83,11 +104,26 @@ export const PackageObjectType = StructType({
    *  BEAST2 encodes struct fields positionally in declaration order, so new
    *  fields MUST be appended LAST — never inserted between existing fields. */
   records: DictType(StringType, StringType),
+  /** Unresolved dataset sources: refPath (e.g. "inputs/table") -> descriptor.
+   *  Appended LAST, per the positional rule above. */
+  sources: DictType(StringType, DatasetSourceWireType),
 });
 export type PackageObjectType = typeof PackageObjectType;
 
 export type PackageObject = ValueTypeOf<typeof PackageObjectType>;
 
+/**
+ * The pre-`sources` package object wire shape (tasks, data, functions,
+ * records), kept only so {@link decodePackageObject} can read packages exported
+ * before path-initialised inputs existed.
+ */
+const RecordsEraPackageObjectType = StructType({
+  tasks: DictType(StringType, StringType),
+  data: PackageDataType,
+  functions: DictType(StringType, StringType),
+  records: DictType(StringType, StringType),
+});
+
 /**
  * The pre-`records` package object wire shape (tasks, data, functions), kept
  * only so {@link decodePackageObject} can read packages exported before records
@@ -110,31 +146,37 @@ const LegacyPackageObjectType = StructType({
 });
 
 const decodeCurrent = decodeBeast2For(PackageObjectType);
+const decodeRecordsEra = decodeBeast2For(RecordsEraPackageObjectType);
 const decodeFunctionsEra = decodeBeast2For(FunctionsEraPackageObjectType);
 const decodeLegacy = decodeBeast2For(LegacyPackageObjectType);
 
 /**
- * Decode a `PackageObject` from BEAST2 bytes, tolerating the two older wire
+ * Decode a `PackageObject` from BEAST2 bytes, tolerating the older wire
  * formats (dual-decode migration).
  *
  * Every package-read path — local AND cloud — must use this instead of
  * `decodeBeast2For(PackageObjectType)` directly, so packages exported before
- * the `records`/`functions` fields existed keep decoding. Older bytes decode
- * with the missing maps defaulted to empty.
+ * the `sources`/`records`/`functions` fields existed keep decoding. Older
+ * bytes decode with the missing maps defaulted to empty.
  */
 export function decodePackageObject(data: Uint8Array): PackageObject {
   try {
     return decodeCurrent(data);
   } catch (err) {
     try {
-      const fnEra = decodeFunctionsEra(data);
-      return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map() };
+      const recEra = decodeRecordsEra(data);
+      return { ...recEra, sources: new Map() };
     } catch {
       try {
-        const legacy = decodeLegacy(data);
-        return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map() };
+        const fnEra = decodeFunctionsEra(data);
+        return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map(), sources: new Map() };
       } catch {
-        throw err; // no known shape — surface the current-format error
+        try {
+          const legacy = decodeLegacy(data);
+          return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map(), sources: new Map() };
+        } catch {
+          throw err; // no known shape — surface the current-format error
+        }
       }
     }
   }

package/src/task.ts

@@ -167,18 +167,64 @@ export const PartitionTaskMetadataType = StructType({
   combine: OptionType(BlobType),
   /** Target carved-slice size in wire bytes — the only sizing knob. */
   targetPartitionBytes: IntegerType,
+  /**
+   * `encodeEastIR` bundle of the per-key merge `(Key, Value, Value) -> Value`
+   * for a Dict output; `none` otherwise.
+   *
+   * Its presence (or {@link mergeSets}) selects the SEGMENT-MERGE assembly:
+   * partials are walked by their segment fences in the orchestrator, disjoint
+   * segments are byte-copied exactly as a splice does, and only overlapping
+   * ones are decoded, merged and re-encoded. Appended LAST (BEAST2 encodes
+   * struct fields positionally) with a dual decoder — see
+   * {@link decodePartitionTaskMetadata}.
+   */
+  merge: OptionType(BlobType),
+  /** Whether a Set output assembles by segment merge (union). The Set twin of
+   *  {@link merge}, which needs no function. Appended LAST. */
+  mergeSets: BooleanType,
 });
 export type PartitionTaskMetadataType = typeof PartitionTaskMetadataType;
 
 export type PartitionTaskMetadata = ValueTypeOf<typeof PartitionTaskMetadataType>;
 
+/**
+ * The pre-`merge` partition metadata wire shape, kept only so
+ * {@link decodePartitionTaskMetadata} can read tasks exported before the
+ * segment-merge assembly existed.
+ */
+const PreMergePartitionTaskMetadataType = StructType({
+  partitions: IntegerType,
+  by: OptionType(BlobType),
+  combine: OptionType(BlobType),
+  targetPartitionBytes: IntegerType,
+});
+
 /** Encode a {@link PartitionTaskMetadataType} value for `TaskObject.metadata`. */
 export const encodePartitionTaskMetadata: (value: PartitionTaskMetadata) => Uint8Array =
   encodeBeast2For(PartitionTaskMetadataType);
 
-/** Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task. */
-export const decodePartitionTaskMetadata: (data: Uint8Array) => PartitionTaskMetadata =
-  decodeBeast2For(PartitionTaskMetadataType);
+const decodeCurrentPartitionMetadata = decodeBeast2For(PartitionTaskMetadataType);
+const decodePreMergePartitionMetadata = decodeBeast2For(PreMergePartitionTaskMetadataType);
+
+/**
+ * Decode a `TaskObject.metadata` blob of a {@link TASK_KIND_PARTITION} task,
+ * tolerating the pre-`merge` wire format (dual-decode migration).
+ *
+ * @param data - the metadata blob
+ * @returns the decoded metadata, with `merge`/`mergeSets` defaulted off for
+ *   older bytes
+ */
+export function decodePartitionTaskMetadata(data: Uint8Array): PartitionTaskMetadata {
+  try {
+    return decodeCurrentPartitionMetadata(data);
+  } catch (err) {
+    try {
+      return { ...decodePreMergePartitionMetadata(data), merge: none, mergeSets: false };
+    } catch {
+      throw err; // no known shape — surface the current-format error
+    }
+  }
+}
 
 /**
  * Metadata of a {@link TASK_KIND_STREAM} task.

package/src/transfer.spec.ts

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The dataset transfer protocol's two invariants that live in e3-types.
+ *
+ * Part arithmetic is shared by the client that cuts the bytes and the server
+ * that places them, so an off-by-one on either side would corrupt uploads
+ * silently until the commit's hash check. And protocol 2 only adds variant
+ * cases: a version-1 client decodes a version-1 answer from a new server, and a
+ * new client decodes an old server's answers, only while every added case sorts
+ * after the version-1 cases and so leaves their tags unchanged.
+ */
+
+import { describe, it } from 'node:test';
+import * as assert from 'node:assert';
+import { NullType, StringType, StructType, VariantType, decodeBeast2For, encodeBeast2For, variant } from '@elaraai/east';
+import {
+  TransferDoneResponseType,
+  TransferUploadResponseType,
+  transferPartCount,
+  transferPartRange,
+} from './transfer.js';
+
+describe('transferPartCount / transferPartRange', () => {
+  it('cuts an upload into full parts and a remainder', () => {
+    assert.strictEqual(transferPartCount(10, 4), 3);
+    assert.deepStrictEqual(transferPartRange(10, 4, 1), { start: 0, end: 4 });
+    assert.deepStrictEqual(transferPartRange(10, 4, 2), { start: 4, end: 8 });
+    assert.deepStrictEqual(transferPartRange(10, 4, 3), { start: 8, end: 10 });
+  });
+
+  it('gives an exact multiple no empty trailing part', () => {
+    assert.strictEqual(transferPartCount(8, 4), 2);
+    assert.deepStrictEqual(transferPartRange(8, 4, 2), { start: 4, end: 8 });
+    assert.strictEqual(transferPartRange(8, 4, 3), null);
+  });
+
+  it('sends an upload no larger than a part as one part', () => {
+    assert.strictEqual(transferPartCount(3, 4), 1);
+    assert.strictEqual(transferPartCount(4, 4), 1);
+    assert.deepStrictEqual(transferPartRange(3, 4, 1), { start: 0, end: 3 });
+  });
+
+  it('has no part 0, no fractional part and none past the end', () => {
+    assert.strictEqual(transferPartRange(10, 4, 0), null);
+    assert.strictEqual(transferPartRange(10, 4, 1.5), null);
+    assert.strictEqual(transferPartRange(10, 4, 4), null);
+  });
+
+  it('stays exact past 2^32 bytes, taking sizes as bigint or number', () => {
+    const size = 5n * 1024n ** 4n + 1n; // 5 TiB and a byte
+    const partBytes = 512n * 1024n ** 2n;
+    assert.strictEqual(transferPartCount(size, partBytes), 10241);
+    assert.deepStrictEqual(transferPartRange(Number(size), Number(partBytes), 10241), {
+      start: Number(size) - 1,
+      end: Number(size),
+    });
+  });
+});
+
+describe('transfer wire compatibility across protocol versions', () => {
+  // The version-1 shapes, as a client and server built before protocol 2 hold them.
+  const UploadResponseV1 = VariantType({
+    completed: NullType,
+    upload: StructType({ id: StringType, uploadUrl: StringType }),
+  });
+  const DoneResponseV1 = VariantType({
+    completed: NullType,
+    error: StructType({ message: StringType }),
+  });
+
+  it('keeps the init answers a version-1 client decodes', () => {
+    const upload = variant('upload', { id: 'abc', uploadUrl: 'http://h/api/uploads/abc' });
+    assert.deepStrictEqual(
+      decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(upload)),
+      upload,
+    );
+    assert.deepStrictEqual(
+      decodeBeast2For(TransferUploadResponseType)(encodeBeast2For(UploadResponseV1)(upload)),
+      upload,
+    );
+    const completed = variant('completed', null);
+    assert.deepStrictEqual(
+      decodeBeast2For(UploadResponseV1)(encodeBeast2For(TransferUploadResponseType)(completed)),
+      completed,
+    );
+  });
+
+  it('keeps the commit answers a version-1 client decodes', () => {
+    for (const done of [variant('completed', null), variant('error', { message: 'hash mismatch' })] as const) {
+      assert.deepStrictEqual(decodeBeast2For(DoneResponseV1)(encodeBeast2For(TransferDoneResponseType)(done)), done);
+      assert.deepStrictEqual(decodeBeast2For(TransferDoneResponseType)(encodeBeast2For(DoneResponseV1)(done)), done);
+    }
+  });
+});

package/src/transfer.ts

@@ -8,9 +8,60 @@
  *
  * Used by both the API server and client for large dataset uploads
  * that exceed inline body size limits.
+ *
+ * Two protocol versions share these types. A client names the version it speaks
+ * with `?protocol=N` on the init and commit requests (a request without it is
+ * version 1), and a server answers only in the forms that version understands —
+ * every case version 2 adds sorts after the version-1 cases, so the tags a
+ * version-1 client decodes keep their numbers.
+ */
+
+import { VariantType, StructType, StringType, IntegerType, NullType, DictType, type ValueTypeOf } from '@elaraai/east';
+
+/**
+ * The dataset transfer protocol version this build speaks.
+ *
+ * @remarks
+ * Version 2 lets the server plan an upload as parts — each sent to its own URL
+ * with the request headers the server names, so an object store can bind a
+ * checksum to a single PUT or take a multipart upload larger than one PUT
+ * allows — and lets a commit answer `processing` while the server verifies the
+ * bytes, for the client to poll.
+ */
+export const TRANSFER_PROTOCOL_VERSION = 2;
+
+/**
+ * How many parts an upload of `size` bytes has when planned with `partBytes`.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @returns The part count — at least 1
  */
+export function transferPartCount(size: number | bigint, partBytes: number | bigint): number {
+  const total = BigInt(size);
+  const each = BigInt(partBytes);
+  return total <= each ? 1 : Number((total + each - 1n) / each);
+}
 
-import { VariantType, StructType, StringType, IntegerType, NullType, type ValueTypeOf } from '@elaraai/east';
+/**
+ * The byte range part `part` of an upload covers.
+ *
+ * @param size - The upload's size in bytes
+ * @param partBytes - The size of every part but the last
+ * @param part - The part's number, from 1
+ * @returns The half-open range `[start, end)`, or `null` when the upload has no
+ *   such part
+ */
+export function transferPartRange(
+  size: number | bigint,
+  partBytes: number | bigint,
+  part: number
+): { start: number; end: number } | null {
+  if (!Number.isInteger(part) || part < 1 || part > transferPartCount(size, partBytes)) return null;
+  const start = BigInt(part - 1) * BigInt(partBytes);
+  const end = start + BigInt(partBytes) < BigInt(size) ? start + BigInt(partBytes) : BigInt(size);
+  return { start: Number(start), end: Number(end) };
+}
 
 /**
  * Transfer upload init request.
@@ -30,7 +81,15 @@ export type TransferUploadRequest = ValueTypeOf<typeof TransferUploadRequestType
  * Transfer upload init response.
  *
  * - `completed`: Object already exists (dedup), dataset ref updated
- * - `upload`: Staging slot created, client should upload then call done
+ * - `upload`: Staging slot created; the client PUTs every byte to `uploadUrl`,
+ *   then commits
+ * - `upload_parts` (protocol 2): the client sends the bytes as parts of
+ *   `partBytes` bytes each (the last part holds the remainder, and an upload no
+ *   larger than `partBytes` is one part). Part `n` (1-based) is the byte range
+ *   `[(n - 1) * partBytes, min(size, n * partBytes))`; its URL and required
+ *   headers come from `GET …/upload/<id>/parts/<n>`. Parts may be sent in any
+ *   order and concurrently, and re-sending a part replaces it. The client
+ *   commits once every part is sent.
  */
 export const TransferUploadResponseType = VariantType({
   completed: NullType,
@@ -38,19 +97,41 @@ export const TransferUploadResponseType = VariantType({
     id: StringType,
     uploadUrl: StringType,
   }),
+  upload_parts: StructType({
+    id: StringType,
+    partBytes: IntegerType,
+  }),
 });
 export type TransferUploadResponse = ValueTypeOf<typeof TransferUploadResponseType>;
 
 /**
- * Transfer done response.
+ * Where and how to send one part of a protocol-2 upload.
+ *
+ * @property url - The URL the client PUTs the part's bytes to, without an
+ *   `Authorization` header (it may be a presigned object-store URL)
+ * @property headers - Request headers the PUT must carry exactly as given — a
+ *   presigned URL can sign them, e.g. the checksum an object store verifies the
+ *   bytes against
+ */
+export const TransferPartResponseType = StructType({
+  url: StringType,
+  headers: DictType(StringType, StringType),
+});
+export type TransferPartResponse = ValueTypeOf<typeof TransferPartResponseType>;
+
+/**
+ * Transfer done response — the commit's answer, and the poll's.
  *
  * - `completed`: Hash verified, object stored, dataset ref updated
  * - `error`: Hash mismatch or other failure
+ * - `processing` (protocol 2): the server is still verifying the bytes; poll
+ *   `GET …/upload/<id>` until it answers `completed` or `error`
  */
 export const TransferDoneResponseType = VariantType({
   completed: NullType,
   error: StructType({
     message: StringType,
   }),
+  processing: NullType,
 });
 export type TransferDoneResponse = ValueTypeOf<typeof TransferDoneResponseType>;