@elaraai/e3-types 0.0.2-beta.8 → 1.0.0

package/src/index.ts

@@ -31,6 +31,14 @@ export {
   DataTreeType,
 } from './dataset.js';
 
+// Per-dataset refs and version vectors (reactive dataflow)
+export {
+  VersionVectorType,
+  type VersionVector,
+  DatasetRefType,
+  type DatasetRef,
+} from './dataset-ref.js';
+
 // Task definitions
 export {
   TaskObjectType,
@@ -68,6 +76,25 @@ export {
   // Backwards compatibility
   PackageDatasetsType,
   type PackageDatasets,
+  // Package transfer types
+  PackageTransferInitRequestType,
+  type PackageTransferInitRequest,
+  PackageTransferInitResponseType,
+  type PackageTransferInitResponse,
+  PackageJobResponseType,
+  type PackageJobResponse,
+  PackageImportResultType,
+  type PackageImportResult,
+  PackageExportResultType,
+  type PackageExportResult,
+  PackageImportProgressType,
+  type PackageImportProgress,
+  PackageImportStatusType,
+  type PackageImportStatus,
+  PackageExportProgressType,
+  type PackageExportProgress,
+  PackageExportStatusType,
+  type PackageExportStatus,
 } from './package.js';
 
 // Workspace state
@@ -81,3 +108,159 @@ export {
   ExecutionStatusType,
   type ExecutionStatus,
 } from './execution.js';
+
+// Lock state
+export {
+  LockOperationType,
+  type LockOperation,
+  ProcessHolderType,
+  type ProcessHolder,
+  LockStateType,
+  type LockState,
+} from './lock.js';
+
+// Dataset transfer types
+export {
+  TransferUploadRequestType,
+  type TransferUploadRequest,
+  TransferUploadResponseType,
+  type TransferUploadResponse,
+  TransferDoneResponseType,
+  type TransferDoneResponse,
+} from './transfer.js';
+
+// Wire format constants
+export { BEAST2_CONTENT_TYPE } from './constants.js';
+
+// API wire types (shared between e3-api-client and e3-api-server)
+export {
+  // Error types
+  WorkspaceNotFoundErrorType,
+  WorkspaceNotDeployedErrorType,
+  WorkspaceExistsErrorType,
+  LockHolderType,
+  WorkspaceLockedErrorType,
+  PackageNotFoundErrorType,
+  PackageExistsErrorType,
+  PackageInvalidErrorType,
+  DatasetNotFoundErrorType,
+  TaskNotFoundErrorType,
+  ExecutionNotFoundErrorType,
+  ObjectNotFoundErrorType,
+  DataflowErrorType,
+  PermissionDeniedErrorType,
+  InternalErrorType,
+  RepositoryNotFoundErrorType,
+  ErrorType,
+  ResponseType,
+  // Repository
+  RepositoryStatusType,
+  GcRequestType,
+  GcResultType,
+  AsyncOperationStatusType,
+  GcStartResultType,
+  GcStatusResultType,
+  // Packages
+  PackageListItemType,
+  PackageInfoType,
+  PackageDetailsType,
+  // Workspaces
+  WorkspaceCreateRequestType,
+  WorkspaceInfoType,
+  WorkspaceDeployRequestType,
+  WorkspaceExportRequestType,
+  // Workspace Status
+  DatasetStatusType,
+  TaskStatusUpToDateType,
+  TaskStatusWaitingType,
+  TaskStatusInProgressType,
+  TaskStatusFailedType,
+  TaskStatusErrorType,
+  TaskStatusStaleRunningType,
+  TaskStatusType,
+  DatasetStatusInfoType,
+  TaskStatusInfoType,
+  WorkspaceStatusSummaryType,
+  WorkspaceStatusResultType,
+  // Tasks
+  TaskListItemType,
+  TaskDetailsType,
+  // Execution
+  DataflowRequestType,
+  LogChunkType,
+  TaskExecutionResultType,
+  DataflowResultType,
+  // Dataflow API polling
+  DataflowEventType,
+  ApiExecutionStatusType,
+  DataflowExecutionSummaryType,
+  ApiDataflowExecutionStateType,
+  // Task Execution History
+  ExecutionHistoryStatusType,
+  ExecutionListItemType,
+  // Dataset List
+  TreeKindType,
+  ListEntryType,
+  // Dataset Status Detail
+  DatasetStatusDetailType,
+  // Type aliases
+  type Error,
+  type RepositoryStatus,
+  type GcRequest,
+  type GcResult,
+  type AsyncOperationStatus,
+  type GcStartResult,
+  type GcStatusResult,
+  type PackageListItem,
+  type PackageInfo,
+  type PackageDetails,
+  type WorkspaceInfo,
+  type WorkspaceCreateRequest,
+  type WorkspaceDeployRequest,
+  type DatasetStatus,
+  type TaskStatus as ApiTaskStatus,
+  type DatasetStatusInfo,
+  type TaskStatusInfo,
+  type WorkspaceStatusSummary,
+  type WorkspaceStatusResult,
+  type TaskListItem,
+  type TaskDetails,
+  type DataflowRequest,
+  type LogChunk,
+  type TaskExecutionResult,
+  type DataflowResult,
+  type DataflowEvent,
+  type ApiExecutionStatus,
+  type DataflowExecutionSummary,
+  type ApiDataflowExecutionState,
+  type ExecutionHistoryStatus,
+  type ExecutionListItem,
+  type TreeKind,
+  type ListEntry,
+  type DatasetStatusDetail,
+} from './api.js';
+
+// Dataflow execution state
+export {
+  type DataflowExecutionStatus,
+  type TaskStatus,
+  TaskStateType,
+  type TaskState,
+  DataflowGraphTaskType,
+  type DataflowGraphTask,
+  DataflowGraphType,
+  type DataflowGraph,
+  ExecutionEventType,
+  type ExecutionEvent,
+  DataflowExecutionStateType,
+  type DataflowExecutionState,
+  // Dataflow run history
+  DataflowRunStatusType,
+  type DataflowRunStatus,
+  TaskExecutionRecordType,
+  type TaskExecutionRecord,
+  DataflowRunSummaryType,
+  type DataflowRunSummary,
+  DataflowRunType,
+  type DataflowRun,
+} from './dataflow.js';

package/src/lock.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Lock state type definitions.
+ *
+ * A lock provides exclusive access to a workspace. The state tracks:
+ * - What operation acquired the lock (dataflow, deployment, etc.)
+ * - Who holds the lock (East text-encoded string)
+ * - When it was acquired
+ * - Optional expiry (for cloud TTL-based locks)
+ *
+ * Lock file location: workspaces/<name>.lock
+ *
+ * The holder field is an East text-encoded string, allowing different backends
+ * to encode their own holder identification. Common patterns:
+ * - Local: `.process (pid=1234, bootId="abc-123", startTime=98765, command="e3 start")`
+ * - Cloud: `.lambda (requestId="req-123", functionName="e3-api")`
+ */
+
+import {
+  VariantType,
+  StructType,
+  StringType,
+  IntegerType,
+  DateTimeType,
+  OptionType,
+  NullType,
+  ValueTypeOf,
+} from '@elaraai/east';
+
+/**
+ * Lock operation - what acquired the lock.
+ */
+export const LockOperationType = VariantType({
+  /** Running a dataflow */
+  dataflow: NullType,
+  /** Deploying a package to the workspace */
+  deployment: NullType,
+  /** Removing the workspace */
+  removal: NullType,
+  /** Writing to a dataset */
+  dataset_write: NullType,
+  /** Exporting a workspace */
+  export: NullType,
+});
+
+export type LockOperation = ValueTypeOf<typeof LockOperationType>;
+
+/**
+ * Process holder schema - for local filesystem backends.
+ * Used to encode/decode the holder string.
+ */
+export const ProcessHolderType = StructType({
+  /** Process ID */
+  pid: IntegerType,
+  /** System boot ID (from /proc/sys/kernel/random/boot_id) */
+  bootId: StringType,
+  /** Process start time in jiffies since boot */
+  startTime: IntegerType,
+  /** Command that acquired the lock (for debugging) */
+  command: StringType,
+});
+
+export type ProcessHolder = ValueTypeOf<typeof ProcessHolderType>;
+
+/**
+ * Lock state stored in workspaces/<name>.lock
+ *
+ * Represents an advisory lock on a workspace. The actual locking mechanism
+ * is platform-specific (flock on Linux, DynamoDB conditional writes in cloud),
+ * but the lock content follows this schema.
+ *
+ * The holder field is an East text-encoded string representing a variant value.
+ * Use `printFor`/`parseInferred` from @elaraai/east to encode/decode.
+ */
+export const LockStateType = StructType({
+  /** What operation acquired the lock */
+  operation: LockOperationType,
+  /** Who holds the lock - East text-encoded variant (e.g., `.process (...)`) */
+  holder: StringType,
+  /** When the lock was acquired */
+  acquiredAt: DateTimeType,
+  /** When the lock expires (for cloud TTL-based locks) */
+  expiresAt: OptionType(DateTimeType),
+});
+
+export type LockState = ValueTypeOf<typeof LockStateType>;

package/src/package.ts

@@ -15,7 +15,8 @@
  * - **Task**: A computation with input/output paths (stored separately)
  */
 
-import { StructType, StringType, DictType, ValueTypeOf } from '@elaraai/east';
+import { StructType, StringType, IntegerType, VariantType, NullType, DictType, ValueTypeOf } from '@elaraai/east';
+import { DatasetRefType } from './dataset-ref.js';
 import { StructureType } from './structure.js';
 
 /**
@@ -26,31 +27,13 @@ import { StructureType } from './structure.js';
  *
  * @remarks
  * - `structure`: Defines which paths are datasets vs trees (recursive)
- * - `value`: Hash of the root tree object in the object store
- *
- * @example
- * ```ts
- * const data: PackageData = {
- *   structure: variant('struct', new Map([
- *     ['inputs', variant('struct', new Map([
- *       ['sales', variant('value', variant('Array', variant('Integer', null)))],
- *     ]))],
- *     ['tasks', variant('struct', new Map([
- *       ['process', variant('struct', new Map([
- *         ['function_ir', variant('value', ...)],
- *         ['output', variant('value', variant('Integer', null))],
- *       ]))],
- *     ]))],
- *   ])),
- *   value: 'abc123...',  // Hash of initial tree
- * };
- * ```
+ * - `refs`: Per-dataset refs mapping refPath to DatasetRef (replaces old root tree hash)
  */
 export const PackageDataType = StructType({
   /** Structure defining tree shape (what's a group vs dataset) */
   structure: StructureType,
-  /** Hash of the root tree object containing initial/default values */
-  value: StringType,
+  /** Per-dataset refs: refPath (e.g. "inputs/greeting") -> DatasetRef */
+  refs: DictType(StringType, DatasetRefType),
 });
 export type PackageDataType = typeof PackageDataType;
 
@@ -84,7 +67,7 @@ export type PackageDatasets = PackageData;
  *   tasks: new Map([['process', 'abc123...']]),  // hash of TaskObject
  *   data: {
  *     structure: variant('struct', new Map([...])),
- *     value: 'def456...',  // hash of root tree
+ *     refs: new Map([['inputs/sales', variant('value', { hash: 'def456...', versions: new Map() })]]),
  *   },
  * };
  * ```
@@ -98,3 +81,69 @@ export const PackageObjectType = StructType({
 export type PackageObjectType = typeof PackageObjectType;
 
 export type PackageObject = ValueTypeOf<typeof PackageObjectType>;
+
+// =============================================================================
+// Package Transfer Types
+// =============================================================================
+
+export const PackageTransferInitRequestType = StructType({
+  size: IntegerType,
+});
+export type PackageTransferInitRequest = ValueTypeOf<typeof PackageTransferInitRequestType>;
+
+export const PackageTransferInitResponseType = StructType({
+  id: StringType,
+  uploadUrl: StringType,
+});
+export type PackageTransferInitResponse = ValueTypeOf<typeof PackageTransferInitResponseType>;
+
+export const PackageJobResponseType = StructType({
+  id: StringType,
+});
+export type PackageJobResponse = ValueTypeOf<typeof PackageJobResponseType>;
+
+export const PackageImportResultType = StructType({
+  name: StringType,
+  version: StringType,
+  packageHash: StringType,
+  objectCount: IntegerType,
+});
+export type PackageImportResult = ValueTypeOf<typeof PackageImportResultType>;
+
+export const PackageExportResultType = StructType({
+  downloadUrl: StringType,
+  size: IntegerType,
+});
+export type PackageExportResult = ValueTypeOf<typeof PackageExportResultType>;
+
+export const PackageImportProgressType = VariantType({
+  pending: NullType,
+  downloading: NullType,
+  importing: StructType({ objectsProcessed: IntegerType }),
+});
+export type PackageImportProgress = ValueTypeOf<typeof PackageImportProgressType>;
+
+export const PackageImportStatusType = VariantType({
+  processing: PackageImportProgressType,
+  completed: PackageImportResultType,
+  failed: StructType({
+    message: StringType,
+  }),
+});
+export type PackageImportStatus = ValueTypeOf<typeof PackageImportStatusType>;
+
+export const PackageExportProgressType = VariantType({
+  pending: NullType,
+  exporting: StructType({ objectsProcessed: IntegerType }),
+  uploading: NullType,
+});
+export type PackageExportProgress = ValueTypeOf<typeof PackageExportProgressType>;
+
+export const PackageExportStatusType = VariantType({
+  processing: PackageExportProgressType,
+  completed: PackageExportResultType,
+  failed: StructType({
+    message: StringType,
+  }),
+});
+export type PackageExportStatus = ValueTypeOf<typeof PackageExportStatusType>;

package/src/structure.ts

@@ -20,7 +20,7 @@
  * @see East serialization docs for keypath syntax details
  */
 
-import { VariantType, StringType, ArrayType, DictType, RecursiveType, ValueTypeOf, printIdentifier, variant, EastTypeType } from '@elaraai/east';
+import { VariantType, StringType, ArrayType, DictType, RecursiveType, ValueTypeOf, printIdentifier, variant, EastTypeType, StructType, BooleanType } from '@elaraai/east';
 
 /**
  * Structure definition for a data tree node.
@@ -47,8 +47,13 @@ import { VariantType, StringType, ArrayType, DictType, RecursiveType, ValueTypeO
  * ```
  */
 export const StructureType = RecursiveType(self => VariantType({
-  /** Dataset: East type of the value (homoiconic EastTypeValue) */
-  value: EastTypeType,
+  /** Dataset: East type and writable flag */
+  value: StructType({
+    /** The East type of the dataset value (homoiconic EastTypeValue) */
+    type: EastTypeType,
+    /** Whether the dataset can be written to by users (e.g., inputs are writable, task outputs are not) */
+    writable: BooleanType,
+  }),
   /** Struct tree: named children mapping to child structures */
   struct: DictType(StringType, self),
 }));

package/src/task.ts

@@ -16,7 +16,7 @@
  * specified paths - the task just references locations, not types.
  */
 
-import { StructType, StringType, ArrayType, ValueTypeOf } from '@elaraai/east';
+import { StructType, StringType, ArrayType, BlobType, OptionType, ValueTypeOf } from '@elaraai/east';
 import { TreePathType } from './structure.js';
 
 /**
@@ -60,6 +60,10 @@ export const TaskObjectType = StructType({
   inputs: ArrayType(TreePathType),
   /** Output path: where to write the output dataset in the data tree */
   output: TreePathType,
+  /** Task kind: "data" (default), "ui", or future extensions. None for old packages. */
+  kind: OptionType(StringType),
+  /** Opaque extension metadata (beast2-encoded). Interpreted by the kind-specific consumer. */
+  metadata: OptionType(BlobType),
 });
 export type TaskObjectType = typeof TaskObjectType;
 

package/src/transfer.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Dataset transfer types for the staged upload protocol.
+ *
+ * Used by both the API server and client for large dataset uploads
+ * that exceed inline body size limits.
+ */
+
+import { VariantType, StructType, StringType, IntegerType, NullType, type ValueTypeOf } from '@elaraai/east';
+
+/**
+ * Transfer upload init request.
+ *
+ * Workspace and dataset path are now encoded in the URL.
+ *
+ * @property hash - SHA-256 hex hash of the data (computed by client)
+ * @property size - Size of the data in bytes
+ */
+export const TransferUploadRequestType = StructType({
+  hash: StringType,
+  size: IntegerType,
+});
+export type TransferUploadRequest = ValueTypeOf<typeof TransferUploadRequestType>;
+
+/**
+ * Transfer upload init response.
+ *
+ * - `completed`: Object already exists (dedup), dataset ref updated
+ * - `upload`: Staging slot created, client should upload then call done
+ */
+export const TransferUploadResponseType = VariantType({
+  completed: NullType,
+  upload: StructType({
+    id: StringType,
+    uploadUrl: StringType,
+  }),
+});
+export type TransferUploadResponse = ValueTypeOf<typeof TransferUploadResponseType>;
+
+/**
+ * Transfer done response.
+ *
+ * - `completed`: Hash verified, object stored, dataset ref updated
+ * - `error`: Hash mismatch or other failure
+ */
+export const TransferDoneResponseType = VariantType({
+  completed: NullType,
+  error: StructType({
+    message: StringType,
+  }),
+});
+export type TransferDoneResponse = ValueTypeOf<typeof TransferDoneResponseType>;

package/src/workspace.ts

@@ -16,7 +16,7 @@
  * No state file = workspace exists but not yet deployed.
  */
 
-import { StructType, StringType, DateTimeType, ValueTypeOf } from '@elaraai/east';
+import { StructType, StringType, DateTimeType, OptionType, ValueTypeOf } from '@elaraai/east';
 
 /**
  * Workspace state stored in workspaces/<name>/state.beast2
@@ -43,10 +43,8 @@ export const WorkspaceStateType = StructType({
   packageHash: StringType,
   /** UTC datetime when the package was deployed */
   deployedAt: DateTimeType,
-  /** Current root data tree hash */
-  rootHash: StringType,
-  /** UTC datetime when root was last updated */
-  rootUpdatedAt: DateTimeType,
+  /** Run ID of the latest completed dataflow run (null if never run) */
+  currentRunId: OptionType(StringType),
 });
 
 export type WorkspaceState = ValueTypeOf<typeof WorkspaceStateType>;

package/tsconfig.json

@@ -13,8 +13,7 @@
     "esModuleInterop": true,
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "composite": true
+    "resolveJsonModule": true
   },
   "include": ["src/**/*"],
   "exclude": ["node_modules", "dist"]