@elaraai/e3-types 1.0.79 → 1.0.80

package/src/execution.ts

@@ -7,111 +7,235 @@
  * Execution status type definitions.
  *
  * An execution represents a single run of a task with specific inputs.
- * Executions are stored at: executions/<taskHash>/<inputsHash>/
+ * A local repository keeps each attempt at
+ * executions/<taskHash>/<inputsHash>/<executionId>/
  *
  * The status file tracks:
  * - For running: process identification for crash detection
- * - For success: output hash and timing
- * - For failed: exit code and timing
+ * - For success: output hash, timing, peak memory and a split task's last plan
+ * - For failed: exit code, timing and peak memory
  * - For error: internal error message and timing
+ * - For cancelled and interrupted: how e3, not the task, ended it
+ *
+ * Every case says whether the execution is a unit of a split task — a piece,
+ * or a merge of the pieces' outputs — which is recorded under its task's hash
+ * as the task's own executions are, but is not a run of the task.
  */
 
 import {
   VariantType,
   StructType,
   ArrayType,
+  OptionType,
   StringType,
   IntegerType,
+  BooleanType,
   DateTimeType,
   ValueTypeOf,
+  decodeBeast2For,
 } from '@elaraai/east';
 
+/** A running execution's process identification. */
+const RunningStatusType = StructType({
+  /** Unique execution ID (UUIDv7) */
+  executionId: StringType,
+  /** Input dataset hashes */
+  inputHashes: ArrayType(StringType),
+  /** When execution started */
+  startedAt: DateTimeType,
+  /** Process ID of the runner */
+  pid: IntegerType,
+  /** Process start time in jiffies since boot (from /proc/<pid>/stat field 22) */
+  pidStartTime: IntegerType,
+  /** System boot ID (from /proc/sys/kernel/random/boot_id) */
+  bootId: StringType,
+  /** Whether the execution is a unit of a split task — a piece, or a merge of
+   *  the pieces' outputs — rather than a task's own execution, which the one
+   *  unit of a task whose input closes no piece is */
+  unit: BooleanType,
+});
+
+const SuccessStatusType = StructType({
+  /** Unique execution ID (UUIDv7) */
+  executionId: StringType,
+  /** Input dataset hashes */
+  inputHashes: ArrayType(StringType),
+  /** Hash of the output dataset */
+  outputHash: StringType,
+  /** When execution started */
+  startedAt: DateTimeType,
+  /** When execution completed */
+  completedAt: DateTimeType,
+  /** The highest peak resident memory, in bytes, a runner process of the
+   *  execution reached — a split task's, the largest of its units' — when
+   *  its runners reported one */
+  peakBytes: OptionType(IntegerType),
+  /** A split task's: the `$plan` of the last stage it ran, which names the
+   *  stages before it, and so every unit its output was assembled from;
+   *  `none` for any other execution */
+  plan: OptionType(StringType),
+  /** Whether the execution is a unit of a split task — a piece, or a merge of
+   *  the pieces' outputs — rather than a task's own execution, which the one
+   *  unit of a task whose input closes no piece is */
+  unit: BooleanType,
+});
+
+const FailedStatusType = StructType({
+  /** Unique execution ID (UUIDv7) */
+  executionId: StringType,
+  /** Input dataset hashes */
+  inputHashes: ArrayType(StringType),
+  /** When execution started */
+  startedAt: DateTimeType,
+  /** When execution completed */
+  completedAt: DateTimeType,
+  /** Process exit code */
+  exitCode: IntegerType,
+  /** The highest peak resident memory, in bytes, a runner process of the
+   *  execution reached — a split task's, the largest of its units' — when
+   *  its runners reported one */
+  peakBytes: OptionType(IntegerType),
+  /** Whether the execution is a unit of a split task — a piece, or a merge of
+   *  the pieces' outputs — rather than a task's own execution, which the one
+   *  unit of a task whose input closes no piece is */
+  unit: BooleanType,
+});
+
+const ErrorStatusType = StructType({
+  /** Unique execution ID (UUIDv7) */
+  executionId: StringType,
+  /** Input dataset hashes */
+  inputHashes: ArrayType(StringType),
+  /** When execution started */
+  startedAt: DateTimeType,
+  /** When execution completed */
+  completedAt: DateTimeType,
+  /** Error message describing what went wrong */
+  message: StringType,
+  /** Whether the execution is a unit of a split task — a piece, or a merge of
+   *  the pieces' outputs — rather than a task's own execution, which the one
+   *  unit of a task whose input closes no piece is */
+  unit: BooleanType,
+});
+
 /**
- * Execution status stored in executions/<taskHash>/<inputsHash>/status.beast2
- *
- * A variant type representing the four possible states of an execution:
+ * Execution status, stored locally at
+ * executions/<taskHash>/<inputsHash>/<executionId>/status.beast2
  *
  * - `running`: Task has been launched but not yet completed
  * - `success`: Task ran and returned exit code 0
  * - `failed`: Task ran and returned non-zero exit code
  * - `error`: e3 execution engine had an internal error (runner not found, output missing, etc.)
+ * - `cancelled`: e3 stopped the execution because the run was aborted, before
+ *   its runner started or while it ran — not the task's own failure
+ * - `interrupted`: the orchestrator that owned the execution exited before it
+ *   finished, and its runner is gone too, so nothing will write its outcome
  *
  * The `running` state includes process identification fields (pid, pidStartTime, bootId)
  * to enable detection of crashed executions. See design/e3-execution.md for details.
+ *
+ * Stored state: read it with {@link decodeExecutionStatus}.
  */
 export const ExecutionStatusType = VariantType({
-  /** Task has been launched but not yet completed */
-  running: StructType({
-    /** Unique execution ID (UUIDv7) */
-    executionId: StringType,
-    /** Input dataset hashes */
-    inputHashes: ArrayType(StringType),
-    /** When execution started */
-    startedAt: DateTimeType,
-    /** Process ID of the runner */
-    pid: IntegerType,
-    /** Process start time in jiffies since boot (from /proc/<pid>/stat field 22) */
-    pidStartTime: IntegerType,
-    /** System boot ID (from /proc/sys/kernel/random/boot_id) */
-    bootId: StringType,
-  }),
-  /** Task ran and returned exit code 0 */
-  success: StructType({
-    /** Unique execution ID (UUIDv7) */
-    executionId: StringType,
-    /** Input dataset hashes */
-    inputHashes: ArrayType(StringType),
-    /** Hash of the output dataset */
-    outputHash: StringType,
-    /** When execution started */
-    startedAt: DateTimeType,
-    /** When execution completed */
-    completedAt: DateTimeType,
-  }),
-  /** Task ran and returned non-zero exit code */
-  failed: StructType({
+  running: RunningStatusType,
+  success: SuccessStatusType,
+  failed: FailedStatusType,
+  error: ErrorStatusType,
+  cancelled: StructType({
     /** Unique execution ID (UUIDv7) */
     executionId: StringType,
     /** Input dataset hashes */
     inputHashes: ArrayType(StringType),
     /** When execution started */
     startedAt: DateTimeType,
-    /** When execution completed */
+    /** When e3 stopped it */
     completedAt: DateTimeType,
-    /** Process exit code */
-    exitCode: IntegerType,
+    /** Whether the execution is a unit of a split task — a piece, or a merge of
+     *  the pieces' outputs — rather than a task's own execution, which the one
+     *  unit of a task whose input closes no piece is */
+    unit: BooleanType,
   }),
-  /** e3 execution engine had an internal error */
-  error: StructType({
+  interrupted: StructType({
     /** Unique execution ID (UUIDv7) */
     executionId: StringType,
     /** Input dataset hashes */
     inputHashes: ArrayType(StringType),
     /** When execution started */
     startedAt: DateTimeType,
-    /** When execution completed */
+    /** When the interruption was found */
     completedAt: DateTimeType,
-    /** Error message describing what went wrong */
-    message: StringType,
+    /** Process ID the runner had */
+    pid: IntegerType,
+    /** Whether the execution is a unit of a split task — a piece, or a merge of
+     *  the pieces' outputs — rather than a task's own execution, which the one
+     *  unit of a task whose input closes no piece is */
+    unit: BooleanType,
   }),
 });
 
 export type ExecutionStatus = ValueTypeOf<typeof ExecutionStatusType>;
 
 /**
- * The orchestrator process that launched an execution (issue #770).
+ * The objects an execution's record keeps from garbage collection: a
+ * success's output, and a split task's last `$plan`, which names every stage
+ * before it and so the units gc keeps beside the task.
+ *
+ * @remarks
+ * Every backend's scan of execution roots applies it, as gc does to the
+ * executions it keeps.
+ *
+ * @param status - the execution's status
+ * @returns the hashes of the objects it keeps
+ */
+export function executionStatusRoots(status: ExecutionStatus): string[] {
+  if (status.type !== 'success') return [];
+  const { outputHash, plan } = status.value;
+  return plan.type === 'some' ? [outputHash, plan.value] : [outputHash];
+}
+
+const decodeCurrentStatus = decodeBeast2For(ExecutionStatusType);
+
+/**
+ * Decode an execution status.
+ *
+ * @remarks
+ * Stored state, so it changes by hard cutover: a repository an older e3 wrote
+ * is re-created rather than read, and this says so.
+ *
+ * @param data - the stored bytes
+ * @returns the status
+ * @throws {Error} When the bytes are not a current status — one an older e3
+ *   wrote, whose repository is re-created.
+ */
+export function decodeExecutionStatus(data: Uint8Array): ExecutionStatus {
+  try {
+    return decodeCurrentStatus(data);
+  } catch (err) {
+    throw new Error(
+      `the execution status does not decode: an older e3 wrote this repository — re-create it: deploy again and import its data again ` +
+      `(${err instanceof Error ? err.message : String(err)})`,
+    );
+  }
+}
+
+/**
+ * The orchestrator process that launched an execution.
  *
  * @remarks
- * Written beside the execution's `running` status as the `owner` sidecar. A
+ * Written beside the execution's `running` status as its owner record. A
  * `running` record whose runner is gone is repaired only when its owner is
  * gone too: a live owner may be between the runner's exit and the record's
  * write, hashing the output.
+ *
+ * Stored state: a record of another shape is read as none recorded.
  */
-export interface ExecutionOwner {
+export const ExecutionOwnerType = StructType({
   /** Process ID of the orchestrator */
-  pid: number;
+  pid: IntegerType,
   /** Orchestrator start time in jiffies since boot (from /proc/<pid>/stat field 22) */
-  pidStartTime: number;
+  pidStartTime: IntegerType,
   /** System boot ID (from /proc/sys/kernel/random/boot_id) */
-  bootId: string;
-}
+  bootId: StringType,
+});
+
+export type ExecutionOwner = ValueTypeOf<typeof ExecutionOwnerType>;

package/src/function.ts

@@ -17,7 +17,7 @@
  * non-TS clients) have the types they need to encode arguments.
  */
 
-import { StructType, StringType, ArrayType, EastTypeType, OptionType, ValueTypeOf, decodeBeast2For, none } from '@elaraai/east';
+import { StructType, StringType, ArrayType, EastTypeType, OptionType, ValueTypeOf, decodeBeast2For } from '@elaraai/east';
 import { RunnerType } from './runner.js';
 
 /**
@@ -25,21 +25,19 @@ import { RunnerType } from './runner.js';
  * `PackageObject.functions`.
  */
 export const FunctionObjectType = StructType({
-  /** Hash of the encoded EastIR bundle (encodeEastIR), like a task's commandIr object. */
+  /** Hash of the encoded EastIR bundle (encodeEastIR), as a task's program is. */
   bodyIr: StringType,
   /** Positional parameter types — the IR's signature, surfaced for arity/type
    *  validation and `describe` without decoding the whole IR. */
   inputTypes: ArrayType(EastTypeType),
   /** Return type — used to decode the result `value` blob client-side. */
   outputType: EastTypeType,
-  /** Author-chosen runtime; resolved to argv by runnerToArgv. One of the known
-   *  runtime tags — there is no raw-argv runner for functions. */
+  /** Author-chosen runtime: a stock runner runs the function as a unit, and a
+   *  `custom` one runs its command with `run`'s arguments. */
   runner: RunnerType,
   /**
    * Hash of an `EnvironmentSpecType` object the function executes in;
-   * `none` ⇒ the stock runtime image. Appended LAST (BEAST2 encodes struct
-   * fields positionally) with a legacy dual decoder — see
-   * {@link decodeFunctionObject}.
+   * `none` ⇒ the stock runtime image.
    */
   environment: OptionType(StringType),
 });
@@ -47,39 +45,28 @@ export type FunctionObjectType = typeof FunctionObjectType;
 
 export type FunctionObject = ValueTypeOf<typeof FunctionObjectType>;
 
-/**
- * The pre-`environment` function object wire shape, kept only so
- * {@link decodeFunctionObject} can read functions exported before execution
- * environments existed.
- */
-const PreEnvironmentFunctionObjectType = StructType({
-  bodyIr: StringType,
-  inputTypes: ArrayType(EastTypeType),
-  outputType: EastTypeType,
-  runner: RunnerType,
-});
-
 const decodeCurrentFunction = decodeBeast2For(FunctionObjectType);
-const decodePreEnvironmentFunction = decodeBeast2For(PreEnvironmentFunctionObjectType);
 
 /**
- * Decode a `FunctionObject` from BEAST2 bytes, tolerating the
- * pre-`environment` wire format (dual-decode migration, like
- * `decodePackageObject`).
+ * Decode a `FunctionObject` from BEAST2 bytes.
+ *
+ * @remarks
+ * A package-borne wire, so it changes by hard cutover: a package exported by
+ * an older SDK is re-exported with the current one, and this says so.
  *
- * Every function-read path — local AND cloud — must use this instead of
- * `decodeBeast2For(FunctionObjectType)` directly. Older bytes decode with
- * `environment` defaulted to `none`.
+ * @param data - the stored bytes
+ * @returns the function object
+ * @throws {Error} When the bytes are not a current function object — a
+ *   package exported by an older SDK, which is re-exported with the current
+ *   one.
  */
 export function decodeFunctionObject(data: Uint8Array): FunctionObject {
   try {
     return decodeCurrentFunction(data);
   } catch (err) {
-    try {
-      const legacy = decodePreEnvironmentFunction(data);
-      return { ...legacy, environment: none };
-    } catch {
-      throw err; // no known shape — surface the current-format error
-    }
+    throw new Error(
+      `the function object does not decode: the package was exported by an older e3 SDK — re-export it with the current one ` +
+      `(${err instanceof Error ? err.message : String(err)})`,
+    );
   }
 }

package/src/index.ts

@@ -41,40 +41,26 @@ export {
 
 // Task definitions
 export {
+  TASK_OBJECT_KIND,
   TaskObjectType,
   type TaskObject,
   decodeTaskObject,
-  TASK_KIND_PARTITION,
-  TASK_KIND_STREAM,
-  TASK_KIND_MERGE,
-  PartitionTaskMetadataType,
-  type PartitionTaskMetadata,
-  encodePartitionTaskMetadata,
-  decodePartitionTaskMetadata,
-  StreamTaskMetadataType,
-  type StreamTaskMetadata,
-  encodeStreamTaskMetadata,
-  decodeStreamTaskMetadata,
-  PartitionPlanType,
-  MergeRangePlanType,
-  type PartitionPlan,
-  type MergeRangePlan,
-  encodePartitionPlan,
-  decodePartitionPlan,
-  type ProjectionShape,
-  partitionProjectionShape,
-  projectedKeyType,
-  projectKey,
+  TaskBodyType,
+  type TaskBody,
+  TaskPartitionType,
+  type TaskPartition,
+  TaskInputType,
+  type TaskInput,
+  TaskOutputKindType,
+  type TaskOutputKind,
+  TaskOutputType,
+  type TaskOutput,
+  DataManifestType,
+  type DataManifest,
+  TaskRoleType,
+  type TaskRole,
 } from './task.js';
 
-// Stream and merge commands
-export {
-  type StreamMergeMode,
-  type StreamCommandSpec,
-  streamCommandIr,
-  mergeCommandIr,
-} from './stream.js';
-
 // Execution environments
 export {
   EnvironmentSpecType,
@@ -109,17 +95,12 @@ export {
   parseDatasetPath,
   parsePackageRef,
   urlPathToTreePath,
-  // Backwards compatibility
-  DatasetSchemaType,
-  type DatasetSchema,
 } from './structure.js';
 
-// Runner wire types (functions)
+// Runner wire types
 export {
   RunnerType,
   type RunnerValue,
-  runnerToArgv,
-  withRunnerVerbose,
   withRunnerLifeline,
 } from './runner.js';
 
@@ -138,6 +119,29 @@ export {
   type MutationObject,
   RecordObjectType,
   type RecordObject,
+  decodeRecordObject,
+  RecordIndexObjectType,
+  type RecordIndexObject,
+  RECORD_STATE_KIND,
+  RecordStateType,
+  type RecordState,
+  isRecordStateType,
+  indexCollectionType,
+  indexWindowType,
+  decodeRecordCommit,
+  decodeMutationObject,
+  MutationFormType,
+  type MutationForm,
+  MigrationObjectType,
+  type MigrationObject,
+  decodeMigrationObject,
+  MigrationFormType,
+  type MigrationForm,
+  DELTA_CONFLICT,
+  patchOpsType,
+  mutationDeltaType,
+  editTypeOf,
+  type DeltaTarget,
 } from './record.js';
 
 // Package objects
@@ -149,9 +153,6 @@ export {
   DatasetSourceWireType,
   type DatasetSourceWire,
   decodePackageObject,
-  // Backwards compatibility
-  PackageDatasetsType,
-  type PackageDatasets,
   // Package transfer types
   PackageTransferInitRequestType,
   type PackageTransferInitRequest,
@@ -177,12 +178,17 @@ export {
 export {
   WorkspaceStateType,
   type WorkspaceState,
+  WorkspaceRecordType,
+  type WorkspaceRecord,
 } from './workspace.js';
 
 // Execution status
 export {
   ExecutionStatusType,
   type ExecutionStatus,
+  decodeExecutionStatus,
+  executionStatusRoots,
+  ExecutionOwnerType,
   type ExecutionOwner,
 } from './execution.js';
 
@@ -192,10 +198,30 @@ export {
   type LockOperation,
   ProcessHolderType,
   type ProcessHolder,
+  LambdaHolderType,
+  type LambdaHolder,
+  LockHolderVariantType,
+  type LockHolderVariant,
   LockStateType,
   type LockState,
 } from './lock.js';
 
+// The repository's own records
+export {
+  RepositoryRecordType,
+  type RepositoryRecord,
+  RepoStatusType,
+  type RepoStatus,
+  RepoMetadataType,
+  type RepoMetadata,
+} from './repository.js';
+
+// The release of e3 this build is, which what it keeps and ships records
+export { E3_RELEASE, compareReleases } from './release.js';
+
+// The names e3 makes paths of
+export { type NamedKind, nameProblem } from './names.js';
+
 // Dataset transfer types
 export {
   TRANSFER_PROTOCOL_VERSION,
@@ -212,12 +238,32 @@ export {
 } from './transfer.js';
 
 // Dataset blob encoding — the ONE branch deciding segmentation, shared by the
-// store path (e3-core `datasetWrite`) and the package export path (e3 `export_`)
+// store's door (e3-core `storeCollection`) and the package export (e3 `export_`)
 export {
   isCollectionRoot,
   encodeDatasetBlob,
+  writeCollectionManifest,
+  type CollectionSegmentRef,
+  type CollectionPiece,
+  type SegmentSink,
 } from './dataset-blob.js';
 
+// The segment-object layout: a collection dataset is a manifest naming
+// standalone segment objects, and the manifest is what the ref points at
+export {
+  COLLECTION_MANIFEST_KIND,
+  CollectionManifestType,
+  CollectionManifestEntryType,
+  type CollectionManifest,
+  type CollectionManifestEntry,
+  isCollectionManifestType,
+  isCollectionManifest,
+  encodeCollectionManifest,
+  decodeCollectionManifest,
+  manifestElementCount,
+  manifestByteSize,
+} from './collection-manifest.js';
+
 // The ONE declared-type-vs-wire-type check, shared by every door a value
 // enters a dataset through (the export, the set, the adopt, the API)
 export {
@@ -249,6 +295,7 @@ export {
   InternalErrorType,
   RepositoryNotFoundErrorType,
   DatasetTypeMismatchErrorType,
+  InvalidNameErrorType,
   ErrorType,
   ResponseType,
   // Repository
@@ -265,7 +312,13 @@ export {
   // Workspaces
   WorkspaceCreateRequestType,
   WorkspaceInfoType,
+  SchemaPolicyType,
   WorkspaceDeployRequestType,
+  RecordPlanType,
+  RecordIndexPlanType,
+  WorkspaceDeployResultType,
+  WorkspaceDeployProgressType,
+  WorkspaceDeployStatusType,
   WorkspaceExportRequestType,
   // Workspace Status
   DatasetStatusType,
@@ -292,6 +345,9 @@ export {
   DataflowEventType,
   ApiExecutionStatusType,
   DataflowExecutionSummaryType,
+  DataflowBudgetType,
+  UnitWaitType,
+  SplitProgressType,
   ApiDataflowExecutionStateType,
   // Task Execution History
   ExecutionHistoryStatusType,
@@ -327,6 +383,12 @@ export {
   type WorkspaceInfo,
   type WorkspaceCreateRequest,
   type WorkspaceDeployRequest,
+  type SchemaPolicy,
+  type RecordPlan,
+  type RecordIndexPlan,
+  type WorkspaceDeployResult,
+  type WorkspaceDeployProgress,
+  type WorkspaceDeployStatus,
   type DatasetStatus,
   type TaskStatus as ApiTaskStatus,
   type DatasetStatusInfo,
@@ -342,6 +404,9 @@ export {
   type DataflowEvent,
   type ApiExecutionStatus,
   type DataflowExecutionSummary,
+  type DataflowBudget,
+  type UnitWait,
+  type SplitProgress,
   type ApiDataflowExecutionState,
   type ExecutionHistoryStatus,
   type ExecutionListItem,
@@ -371,11 +436,16 @@ export {
   type DataflowGraphTask,
   DataflowGraphType,
   type DataflowGraph,
+  StageUnitType,
+  type StageUnit,
+  RequeueReasonType,
+  type RequeueReason,
   ExecutionEventType,
   type ExecutionEvent,
   type PartitionProgress,
   DataflowExecutionStateType,
   type DataflowExecutionState,
+  decodeDataflowExecutionState,
   // Dataflow run history
   DataflowRunStatusType,
   type DataflowRunStatus,
@@ -386,3 +456,16 @@ export {
   DataflowRunType,
   type DataflowRun,
 } from './dataflow.js';
+
+// A split task's unit graph, a stage at a time
+export {
+  UNIT_PLAN_KIND,
+  UnitPlanGroupType,
+  type UnitPlanGroup,
+  UnitPlanStageType,
+  type UnitPlanStage,
+  UnitPlanType,
+  type UnitPlan,
+  encodeUnitPlan,
+  decodeUnitPlan,
+} from './unit-plan.js';