@elaraai/e3-types 0.0.1-beta.0

package/src/execution.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Execution status type definitions.
+ *
+ * An execution represents a single run of a task with specific inputs.
+ * Executions are stored at: executions/<taskHash>/<inputsHash>/
+ *
+ * The status file tracks:
+ * - For running: process identification for crash detection
+ * - For success: output hash and timing
+ * - For failed: exit code and timing
+ * - For error: internal error message and timing
+ */
+
+import {
+  VariantType,
+  StructType,
+  ArrayType,
+  StringType,
+  IntegerType,
+  DateTimeType,
+  ValueTypeOf,
+} from '@elaraai/east';
+
+/**
+ * Execution status stored in executions/<taskHash>/<inputsHash>/status.beast2
+ *
+ * A variant type representing the four possible states of an execution:
+ *
+ * - `running`: Task has been launched but not yet completed
+ * - `success`: Task ran and returned exit code 0
+ * - `failed`: Task ran and returned non-zero exit code
+ * - `error`: e3 execution engine had an internal error (runner not found, output missing, etc.)
+ *
+ * The `running` state includes process identification fields (pid, pidStartTime, bootId)
+ * to enable detection of crashed executions. See design/e3-execution.md for details.
+ */
+export const ExecutionStatusType = VariantType({
+  /** Task has been launched but not yet completed */
+  running: StructType({
+    /** Input dataset hashes */
+    inputHashes: ArrayType(StringType),
+    /** When execution started */
+    startedAt: DateTimeType,
+    /** Process ID of the runner */
+    pid: IntegerType,
+    /** Process start time in jiffies since boot (from /proc/<pid>/stat field 22) */
+    pidStartTime: IntegerType,
+    /** System boot ID (from /proc/sys/kernel/random/boot_id) */
+    bootId: StringType,
+  }),
+  /** Task ran and returned exit code 0 */
+  success: StructType({
+    /** Input dataset hashes */
+    inputHashes: ArrayType(StringType),
+    /** Hash of the output dataset */
+    outputHash: StringType,
+    /** When execution started */
+    startedAt: DateTimeType,
+    /** When execution completed */
+    completedAt: DateTimeType,
+  }),
+  /** Task ran and returned non-zero exit code */
+  failed: StructType({
+    /** Input dataset hashes */
+    inputHashes: ArrayType(StringType),
+    /** When execution started */
+    startedAt: DateTimeType,
+    /** When execution completed */
+    completedAt: DateTimeType,
+    /** Process exit code */
+    exitCode: IntegerType,
+  }),
+  /** e3 execution engine had an internal error */
+  error: StructType({
+    /** Input dataset hashes */
+    inputHashes: ArrayType(StringType),
+    /** When execution started */
+    startedAt: DateTimeType,
+    /** When execution completed */
+    completedAt: DateTimeType,
+    /** Error message describing what went wrong */
+    message: StringType,
+  }),
+});
+
+export type ExecutionStatus = ValueTypeOf<typeof ExecutionStatusType>;

package/src/index.ts

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * e3-types: Shared type definitions for e3 (East Execution Engine)
+ *
+ * This package defines the East types used for serializing e3 objects:
+ * - Data references and tree structures
+ * - Task definitions (command IR, input/output paths)
+ * - Package objects
+ * - Data structure and paths
+ * - Workspace state
+ * - Execution status
+ *
+ * Terminology:
+ * - **Dataset**: A location holding a value (leaf node)
+ * - **Tree**: A location containing other locations (branch node)
+ * - **Structure**: The shape of the data tree
+ * - **Task**: A computation with command IR and input/output paths
+ * - **Path**: An address in the data tree
+ */
+
+// Data references and trees
+export {
+  DataRefType,
+  type DataRef,
+  unassignedRef,
+  nullRef,
+  DataTreeType,
+} from './dataset.js';
+
+// Task definitions
+export {
+  TaskObjectType,
+  type TaskObject,
+} from './task.js';
+
+// Data structure and paths
+export {
+  StructureType,
+  type Structure,
+  PathSegmentType,
+  type PathSegment,
+  TreePathType,
+  type TreePath,
+  type ParsePathResult,
+  treePath,
+  pathToString,
+  parsePath,
+  // Backwards compatibility
+  DatasetSchemaType,
+  type DatasetSchema,
+} from './structure.js';
+
+// Package objects
+export {
+  PackageDataType,
+  type PackageData,
+  PackageObjectType,
+  type PackageObject,
+  // Backwards compatibility
+  PackageDatasetsType,
+  type PackageDatasets,
+} from './package.js';
+
+// Workspace state
+export {
+  WorkspaceStateType,
+  type WorkspaceState,
+} from './workspace.js';
+
+// Execution status
+export {
+  ExecutionStatusType,
+  type ExecutionStatus,
+} from './execution.js';

package/src/package.ts

@@ -0,0 +1,100 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Package object types for e3.
+ *
+ * A package bundles everything needed to run computations:
+ * tasks and data structure with initial values.
+ *
+ * Terminology:
+ * - **Package**: A deployable bundle of tasks and data structure
+ * - **Structure**: The shape of the data tree
+ * - **Task**: A computation with input/output paths (stored separately)
+ */
+
+import { StructType, StringType, DictType, ValueTypeOf } from '@elaraai/east';
+import { StructureType } from './structure.js';
+
+/**
+ * Data configuration in a package.
+ *
+ * Defines the structure (which paths are datasets vs trees)
+ * and initial values (root tree hash).
+ *
+ * @remarks
+ * - `structure`: Defines which paths are datasets vs trees (recursive)
+ * - `value`: Hash of the root tree object in the object store
+ *
+ * @example
+ * ```ts
+ * const data: PackageData = {
+ *   structure: variant('struct', new Map([
+ *     ['inputs', variant('struct', new Map([
+ *       ['sales', variant('value', variant('Array', variant('Integer', null)))],
+ *     ]))],
+ *     ['tasks', variant('struct', new Map([
+ *       ['process', variant('struct', new Map([
+ *         ['function_ir', variant('value', ...)],
+ *         ['output', variant('value', variant('Integer', null))],
+ *       ]))],
+ *     ]))],
+ *   ])),
+ *   value: 'abc123...',  // Hash of initial tree
+ * };
+ * ```
+ */
+export const PackageDataType = StructType({
+  /** Structure defining tree shape (what's a group vs dataset) */
+  structure: StructureType,
+  /** Hash of the root tree object containing initial/default values */
+  value: StringType,
+});
+export type PackageDataType = typeof PackageDataType;
+
+export type PackageData = ValueTypeOf<typeof PackageDataType>;
+
+// Backwards compatibility alias
+/** @deprecated Use PackageDataType instead */
+export const PackageDatasetsType = PackageDataType;
+/** @deprecated Use PackageData instead */
+export type PackageDatasetsType = PackageDataType;
+/** @deprecated Use PackageData instead */
+export type PackageDatasets = PackageData;
+
+/**
+ * Package object stored in the object store.
+ *
+ * Packages are the unit of distribution and deployment in e3.
+ * They are immutable and content-addressed by their hash.
+ *
+ * @remarks
+ * - `tasks`: Maps task names to task object hashes. Each task object
+ *   contains runner, input paths, and output path.
+ * - `data`: The structure and initial values for the data tree.
+ *
+ * Package identity (name/version) is determined by the path in the
+ * bundle's `packages/<name>/<version>` directory structure.
+ *
+ * @example
+ * ```ts
+ * const pkg: PackageObject = {
+ *   tasks: new Map([['process', 'abc123...']]),  // hash of TaskObject
+ *   data: {
+ *     structure: variant('struct', new Map([...])),
+ *     value: 'def456...',  // hash of root tree
+ *   },
+ * };
+ * ```
+ */
+export const PackageObjectType = StructType({
+  /** Tasks defined in this package: name -> task object hash */
+  tasks: DictType(StringType, StringType),
+  /** Data structure and initial values */
+  data: PackageDataType,
+});
+export type PackageObjectType = typeof PackageObjectType;
+
+export type PackageObject = ValueTypeOf<typeof PackageObjectType>;

package/src/structure.ts

@@ -0,0 +1,270 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Data structure and path types for e3.
+ *
+ * Terminology:
+ * - **Dataset**: A location holding a value (leaf node in the data tree)
+ * - **Tree**: A location containing datasets or nested trees (branch node)
+ * - **Structure**: The shape of the data tree (what trees/datasets exist and their types)
+ * - **Path**: An address pointing to a dataset or tree
+ *
+ * Paths use East's keypath syntax:
+ * - `.field` for struct field access (backtick-quoted for special chars)
+ * - `[N]` for array index access (future)
+ * - `[key]` for dict key lookup (future)
+ *
+ * @see East serialization docs for keypath syntax details
+ */
+
+import { VariantType, StringType, ArrayType, DictType, RecursiveType, ValueTypeOf, printIdentifier, variant, EastTypeType } from '@elaraai/east';
+
+/**
+ * Structure definition for a data tree node.
+ *
+ * Defines the shape of the data tree - which paths are datasets (hold values)
+ * and which are trees (hold other nodes).
+ *
+ * @remarks
+ * - `value`: A dataset - holds a typed value. The type is an `EastTypeValue`.
+ * - `struct`: A tree - has named children, each with its own structure.
+ *
+ * MVP only supports struct trees. Future: array, dict, variant trees.
+ *
+ * @example
+ * ```ts
+ * // A dataset holding an Integer value
+ * const dataset: Structure = variant('value', variant('Integer', null));
+ *
+ * // A tree with named children
+ * const tree: Structure = variant('struct', new Map([
+ *   ['count', variant('value', variant('Integer', null))],
+ *   ['items', variant('value', variant('Array', variant('String', null)))],
+ * ]));
+ * ```
+ */
+export const StructureType = RecursiveType(self => VariantType({
+  /** Dataset: East type of the value (homoiconic EastTypeValue) */
+  value: EastTypeType,
+  /** Struct tree: named children mapping to child structures */
+  struct: DictType(StringType, self),
+}));
+export type StructureType = typeof StructureType;
+
+export type Structure = ValueTypeOf<typeof StructureType>;
+
+// Backwards compatibility alias
+/** @deprecated Use StructureType instead */
+export const DatasetSchemaType = StructureType;
+/** @deprecated Use Structure instead */
+export type DatasetSchemaType = StructureType;
+/** @deprecated Use Structure instead */
+export type DatasetSchema = Structure;
+
+/**
+ * Path segment for navigating data trees.
+ *
+ * Uses East keypath syntax for consistency:
+ * - `field`: Struct field access (rendered as `.field` or `` .`field` `` if quoted)
+ * - `index`: Array element access (rendered as `[N]`) - future
+ * - `key`: Dict key lookup (rendered as `[key]`) - future
+ *
+ * @example
+ * ```ts
+ * // Struct field access
+ * const segment: PathSegment = variant('field', 'sales');
+ *
+ * // Array index (future)
+ * const segment: PathSegment = variant('index', 0n);
+ * ```
+ */
+export const PathSegmentType = VariantType({
+  /** Struct field access by name */
+  field: StringType,
+  // Future: case: StringType for variant case identifiers
+  // Future: index: IntegerType for array access
+  // Future: key: StringType (or polymorphic) for dict access
+});
+export type PathSegmentType = typeof PathSegmentType;
+
+export type PathSegment = ValueTypeOf<typeof PathSegmentType>;
+
+/**
+ * Path: sequence of segments identifying a location in a data tree.
+ *
+ * Paths point to either a dataset (leaf) or a tree (branch).
+ * Used by tasks to specify where inputs come from and where outputs go.
+ *
+ * @example
+ * ```ts
+ * // Path to .inputs.sales.data
+ * const path: TreePath = [
+ *   variant('field', 'inputs'),
+ *   variant('field', 'sales'),
+ *   variant('field', 'data'),
+ * ];
+ * ```
+ */
+export const TreePathType = ArrayType(PathSegmentType);
+export type TreePathType = typeof TreePathType;
+
+export type TreePath = ValueTypeOf<typeof TreePathType>;
+
+/**
+ * Converts a path to East keypath string representation.
+ *
+ * Uses East's keypath syntax: `.field` for simple identifiers,
+ * `` .`field` `` for identifiers needing quoting.
+ *
+ * @param path - The path to convert
+ * @returns A keypath string (e.g., ".inputs.sales" or ".inputs.`my/field`")
+ *
+ * @example
+ * ```ts
+ * const path = treePath('inputs', 'sales');
+ * pathToString(path); // ".inputs.sales"
+ *
+ * const path2 = treePath('inputs', 'my/field');
+ * pathToString(path2); // ".inputs.`my/field`"
+ * ```
+ */
+export function pathToString(path: TreePath): string {
+  return path.map(segment => {
+    if (segment.type === 'field') {
+      return '.' + printIdentifier(segment.value);
+    } else {
+      throw new Error(`pathToString: unsupported path segment type: ${segment.type}`);
+    }
+  }).join('');
+}
+
+/**
+ * Result of parsing a path with structure validation.
+ */
+export interface ParsePathResult {
+  /** The parsed path segments */
+  path: TreePath;
+  /** The structure at the path location */
+  structure: Structure;
+}
+
+/**
+ * Parses an East keypath string into a path, validating against the structure.
+ *
+ * Supports `.field` syntax for struct field access.
+ * Backtick-quoted identifiers (`` .`field` ``) are supported for special chars.
+ *
+ * @param pathStr - A keypath string (e.g., ".inputs.sales")
+ * @param structure - The root structure to validate against
+ * @returns The parsed path and the structure at that location
+ *
+ * @throws {Error} If a field doesn't exist in the structure or path descends into a dataset
+ *
+ * @remarks
+ * - Empty string returns empty path (root) with the root structure
+ * - Path must start with `.` (no leading slash)
+ * - Backtick escaping: `` \` `` for literal backtick, `\\` for backslash
+ * - Future: will disambiguate variant cases from struct fields using structure
+ * - Future: will use structure to parse dict keys with correct type
+ *
+ * @example
+ * ```ts
+ * const structure = variant('struct', new Map([
+ *   ['inputs', variant('struct', new Map([
+ *     ['sales', variant('value', variant('Integer', null))],
+ *   ]))],
+ * ]));
+ *
+ * const { path, structure: leafStructure } = parsePath('.inputs.sales', structure);
+ * // path = [field('inputs'), field('sales')]
+ * // leafStructure = variant('value', variant('Integer', null))
+ * ```
+ */
+export function parsePath(pathStr: string, structure: Structure): ParsePathResult {
+  if (pathStr === '') return { path: [], structure };
+
+  const segments: TreePath = [];
+  let currentStructure = structure;
+  let pos = 0;
+
+  while (pos < pathStr.length) {
+    if (pathStr[pos] === '.') {
+      pos++;
+
+      // Parse identifier (TODO: export parseIdentifier from east package)
+      let fieldName: string;
+
+      // Check for backtick-quoted identifier
+      if (pos < pathStr.length && pathStr[pos] === '`') {
+        pos++;
+        fieldName = '';
+        while (pos < pathStr.length && pathStr[pos] !== '`') {
+          if (pathStr[pos] === '\\' && pos + 1 < pathStr.length) {
+            // Escape sequence
+            pos++;
+            fieldName += pathStr[pos];
+          } else {
+            fieldName += pathStr[pos];
+          }
+          pos++;
+        }
+        if (pos < pathStr.length && pathStr[pos] === '`') {
+          pos++; // consume closing backtick
+        }
+      } else {
+        // Simple identifier: [a-zA-Z_][a-zA-Z0-9_]*
+        fieldName = '';
+        while (pos < pathStr.length && /[a-zA-Z0-9_]/.test(pathStr[pos]!)) {
+          fieldName += pathStr[pos];
+          pos++;
+        }
+      }
+
+      if (fieldName.length === 0) {
+        throw new Error(`parsePath: expected identifier after '.' at position ${pos}`);
+      }
+
+      // Validate against structure
+      if (currentStructure.type === 'value') {
+        throw new Error(`parsePath: cannot descend into dataset at '${pathToString(segments)}'`);
+      }
+
+      // currentStructure.type === 'struct' (only other option after 'value' check)
+      const fields = currentStructure.value;
+      const childStructure = fields.get(fieldName);
+      if (childStructure === undefined) {
+        const available = [...fields.keys()].map(k => printIdentifier(k)).join(', ');
+        throw new Error(`parsePath: field '${fieldName}' not found at '${pathToString(segments)}'. Available: ${available}`);
+      }
+      segments.push(variant('field', fieldName));
+      currentStructure = childStructure;
+    } else {
+      throw new Error(`parsePath: unexpected character at position ${pos}: '${pathStr[pos]}'`);
+    }
+  }
+
+  return { path: segments, structure: currentStructure };
+}
+
+/**
+ * Creates a path from field names, validating against the structure.
+ *
+ * Convenience function for the common case of navigating through struct fields.
+ *
+ * @param structure - The root structure to validate against
+ * @param fields - Field names to include in the path
+ * @returns The parsed path and the structure at that location
+ *
+ * @throws {Error} If a field doesn't exist in the structure
+ *
+ * @example
+ * ```ts
+ * const { path, structure: leafStructure } = treePath(rootStructure, 'inputs', 'sales');
+ * ```
+ */
+export function treePath(structure: Structure, ...fields: string[]): ParsePathResult {
+  return parsePath('.' + fields.map(printIdentifier).join('.'), structure);
+}

package/src/task.ts

@@ -0,0 +1,66 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Task object types for e3.
+ *
+ * A task object defines a complete executable unit: the command IR that
+ * generates the exec args, where to read inputs from, and where to write output.
+ *
+ * Task objects are stored in the object store and referenced by packages.
+ * They are content-addressed, enabling deduplication and memoization.
+ *
+ * Input and output types are inferred from the package's structure at the
+ * specified paths - the task just references locations, not types.
+ */
+
+import { StructType, StringType, ArrayType, ValueTypeOf } from '@elaraai/east';
+import { TreePathType } from './structure.js';
+
+/**
+ * Task object stored in the object store.
+ *
+ * A task is a complete executable unit that reads from input dataset paths
+ * and writes to an output dataset path. The commandIr is evaluated at runtime
+ * to produce the exec args.
+ *
+ * @remarks
+ * - `commandIr`: Hash of East IR object that produces exec args
+ *   - IR signature: (inputs: Array<String>, output: String) -> Array<String>
+ *   - `inputs` are paths to staged input .beast2 files
+ *   - `output` is the path where output should be written
+ *   - Returns array of strings to exec (e.g., ["sh", "-c", "python ..."])
+ * - `inputs`: Paths to input datasets in the data tree
+ * - `output`: Path to the output dataset in the data tree
+ *
+ * Types are not stored in the task - they are inferred from the package's
+ * structure at the specified paths. This keeps tasks simple and avoids
+ * redundant type information.
+ *
+ * @example
+ * ```ts
+ * import { variant } from '@elaraai/east';
+ *
+ * // Task with command IR that generates: ["sh", "-c", "python script.py <input> <output>"]
+ * const task: TaskObject = {
+ *   commandIr: '5e7a3b...',  // hash of compiled IR
+ *   inputs: [
+ *     [variant('field', 'inputs'), variant('field', 'sales')],
+ *   ],
+ *   output: [variant('field', 'tasks'), variant('field', 'train'), variant('field', 'output')],
+ * };
+ * ```
+ */
+export const TaskObjectType = StructType({
+  /** Hash of East IR that generates exec args: (inputs, output) -> Array<String> */
+  commandIr: StringType,
+  /** Input paths: where to read each input dataset from the data tree */
+  inputs: ArrayType(TreePathType),
+  /** Output path: where to write the output dataset in the data tree */
+  output: TreePathType,
+});
+export type TaskObjectType = typeof TaskObjectType;
+
+export type TaskObject = ValueTypeOf<typeof TaskObjectType>;

package/src/workspace.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Workspace state type definitions.
+ *
+ * A workspace is a mutable working copy of a package. The state tracks:
+ * - Which package was deployed (immutable reference via hash)
+ * - When the deployment occurred
+ * - Current root data tree hash
+ * - When the root was last updated
+ *
+ * State file location: workspaces/<name>/state.beast2
+ * No state file = workspace exists but not yet deployed.
+ */
+
+import { StructType, StringType, DateTimeType, ValueTypeOf } from '@elaraai/east';
+
+/**
+ * Workspace state stored in workspaces/<name>/state.beast2
+ *
+ * Contains both deployment info and current data root in a single
+ * atomic unit to ensure consistency.
+ *
+ * Future audit trail support:
+ * When we implement full audit trail, this state will move to the object
+ * store (content-addressed) with a ref file pointing to current state hash.
+ * Additional fields for the Merkle chain:
+ *
+ *   previousStateHash: NullableType(StringType),  // null for initial deploy
+ *   message: StringType,  // "deployed package X", "user Y wrote to dataset Z"
+ *
+ * This gives a complete history of workspace changes, similar to git commits.
+ */
+export const WorkspaceStateType = StructType({
+  /** Name of the deployed package */
+  packageName: StringType,
+  /** Version of the deployed package */
+  packageVersion: StringType,
+  /** Hash of the package object at deploy time (immutable reference) */
+  packageHash: StringType,
+  /** UTC datetime when the package was deployed */
+  deployedAt: DateTimeType,
+  /** Current root data tree hash */
+  rootHash: StringType,
+  /** UTC datetime when root was last updated */
+  rootUpdatedAt: DateTimeType,
+});
+
+export type WorkspaceState = ValueTypeOf<typeof WorkspaceStateType>;

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "lib": ["ES2022"],
+    "moduleResolution": "NodeNext",
+    "outDir": "./dist",
+    "rootDir": "./",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "composite": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}