@elaraai/e3-types 1.0.79 → 1.0.81

package/src/intake.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * How far a file being taken into the store has got: a deploy's file source,
+ * or an upload's delivery as its commit takes it in.
+ *
+ * A file is hashed first, since its SHA-256 is how the store knows a delivery
+ * it already holds, and then taken in: a collection by intake units on the
+ * runners, a piece of its segments each, and any other value as the object the
+ * file is.
+ */
+
+import { ArrayType, IntegerType, NullType, StringType, StructType, VariantType, type ValueTypeOf } from '@elaraai/east';
+
+/**
+ * The step a file being taken in is at.
+ */
+export const IntakeStepType = VariantType({
+  /** Not started: a deploy takes a bounded number of files in at once */
+  waiting: NullType,
+  /** Being read for its SHA-256, by which the store knows a delivery it
+   *  already holds */
+  hashing: NullType,
+  /** Being taken in by intake units: its `pieces`, and how many are `done` */
+  taking_in: StructType({ pieces: IntegerType, done: IntegerType }),
+  /** In the store: one it already held; a value that is not a collection,
+   *  carried as the object the file is; or a collection, taken in by the
+   *  runners named, in the order each first took a piece in — none when every
+   *  piece was taken in before, by an intake that stopped part way */
+  done: VariantType({ known: NullType, carried: NullType, taken: ArrayType(StringType) }),
+});
+
+export type IntakeStep = ValueTypeOf<typeof IntakeStepType>;
+
+/**
+ * A file being taken into the store.
+ */
+export const IntakeFileType = StructType({
+  /** The dataset it becomes, as `inputs/<name>` */
+  path: StringType,
+  /** The step it is at */
+  step: IntakeStepType,
+  /** Bytes of the file its step has covered: read for its hash, or covered by
+   *  the pieces taken in; its size once it is done */
+  bytes: IntegerType,
+  /** The file's size */
+  total: IntegerType,
+});
+
+export type IntakeFile = ValueTypeOf<typeof IntakeFileType>;

package/src/lock.ts

@@ -6,18 +6,19 @@
 /**
  * Lock state type definitions.
  *
- * A lock provides exclusive access to a workspace. The state tracks:
- * - What operation acquired the lock (dataflow, deployment, etc.)
- * - Who holds the lock (East text-encoded string)
- * - When it was acquired
- * - Optional expiry (for cloud TTL-based locks)
+ * A lock gives exclusive or shared access to a resource: a workspace, a
+ * workspace's dataflow, the repository's tasks, or one dataset's ref. The state
+ * records:
+ * - what operation acquired the lock (dataflow, deployment, etc.);
+ * - who holds it: a local process, or a cloud function;
+ * - when it was acquired;
+ * - when it expires, for a cloud backend's lease.
  *
- * Lock file location: workspaces/<name>.lock
+ * An exclusive holder may also report how far its operation has got, for
+ * another process to read while it holds the lock: a deploy taking its file
+ * sources in and deploying its records.
  *
- * The holder field is an East text-encoded string, allowing different backends
- * to encode their own holder identification. Common patterns:
- * - Local: `.process (pid=1234, bootId="abc-123", startTime=98765, command="e3 start")`
- * - Cloud: `.lambda (requestId="req-123", functionName="e3-api")`
+ * A local repository keeps each resource's locks in `locks/<resource>/`.
  */
 
 import {
@@ -30,6 +31,7 @@ import {
   NullType,
   ValueTypeOf,
 } from '@elaraai/east';
+import { DeployProgressType } from './api.js';
 
 /**
  * Lock operation - what acquired the lock.
@@ -50,8 +52,8 @@ export const LockOperationType = VariantType({
 export type LockOperation = ValueTypeOf<typeof LockOperationType>;
 
 /**
- * Process holder schema - for local filesystem backends.
- * Used to encode/decode the holder string.
+ * A local process holding a lock: alive while a process of this pid and start
+ * time runs under this boot.
  */
 export const ProcessHolderType = StructType({
   /** Process ID */
@@ -67,20 +69,40 @@ export const ProcessHolderType = StructType({
 export type ProcessHolder = ValueTypeOf<typeof ProcessHolderType>;
 
 /**
- * Lock state stored in workspaces/<name>.lock
- *
- * Represents an advisory lock on a workspace. The actual locking mechanism
- * is platform-specific (flock on Linux, DynamoDB conditional writes in cloud),
- * but the lock content follows this schema.
+ * A cloud function holding a lock, which the lock's lease bounds.
+ */
+export const LambdaHolderType = StructType({
+  /** The invocation's request ID */
+  requestId: StringType,
+  /** The function's name */
+  functionName: StringType,
+});
+
+export type LambdaHolder = ValueTypeOf<typeof LambdaHolderType>;
+
+/**
+ * Who holds a lock.
+ */
+export const LockHolderVariantType = VariantType({
+  /** A local process */
+  process: ProcessHolderType,
+  /** A cloud function */
+  lambda: LambdaHolderType,
+});
+
+export type LockHolderVariant = ValueTypeOf<typeof LockHolderVariantType>;
+
+/**
+ * The state of a held lock.
  *
- * The holder field is an East text-encoded string representing a variant value.
- * Use `printFor`/`parseInferred` from @elaraai/east to encode/decode.
+ * The locking itself is the backend's — an atomically created file locally, a
+ * conditional write in the cloud — and every backend records this state.
  */
 export const LockStateType = StructType({
   /** What operation acquired the lock */
   operation: LockOperationType,
-  /** Who holds the lock - East text-encoded variant (e.g., `.process (...)`) */
-  holder: StringType,
+  /** Who holds the lock */
+  holder: LockHolderVariantType,
   /** When the lock was acquired */
   acquiredAt: DateTimeType,
   /** When the lock expires (for cloud TTL-based locks) */
@@ -88,3 +110,27 @@ export const LockStateType = StructType({
 });
 
 export type LockState = ValueTypeOf<typeof LockStateType>;
+
+/**
+ * How far the operation holding a lock exclusively has got, as its holder
+ * reports it for another process to read while it holds the lock.
+ */
+export const LockProgressType = VariantType({
+  /** A deploy: its file sources and its records */
+  deployment: DeployProgressType,
+});
+
+export type LockProgress = ValueTypeOf<typeof LockProgressType>;
+
+/**
+ * A resource's exclusive lock as another process reads it while it is held:
+ * what holds it, and how far that operation says it has got.
+ */
+export const LockStatusType = StructType({
+  /** The lock's state: the operation, its holder, and when it was acquired */
+  state: LockStateType,
+  /** What the holder last reported of its progress; `none` until it reports */
+  progress: OptionType(LockProgressType),
+});
+
+export type LockStatus = ValueTypeOf<typeof LockStatusType>;

package/src/names.ts

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The rule for the names e3 makes paths of: a repository's name where a server
+ * keeps several, a workspace's name, a package's name and version, and a
+ * lock's resource. One place holds it, so the SDK, the CLI, the API server and
+ * the stores refuse the same names.
+ */
+
+/** What a name names. */
+export type NamedKind = 'repository' | 'workspace' | 'package' | 'package version' | 'lock';
+
+/** What no name may hold: a path separator, a character a Windows file name
+ *  refuses, or a control character. */
+const REFUSED = /[/\\:*?"<>|\u0000-\u001f]/;
+
+/** What a workspace's name may not hold besides: the characters that join the
+ *  parts of its locks' names. */
+const REFUSED_IN_WORKSPACE = /[#~]/;
+
+/**
+ * Why a name cannot be one path segment, or `null` when it can.
+ *
+ * @param kind - What the name names
+ * @param name - The name
+ * @returns The problem, in words that follow the name, or `null`
+ */
+export function nameProblem(kind: NamedKind, name: string): string | null {
+  if (name === '') return 'is empty';
+  if (name === '.' || name === '..') return 'is a path of its own';
+  const refused = REFUSED.exec(name);
+  if (refused !== null) return `holds ${JSON.stringify(refused[0])}, which a file name cannot`;
+  const joiner = kind === 'workspace' ? REFUSED_IN_WORKSPACE.exec(name) : null;
+  if (joiner !== null) return `holds ${JSON.stringify(joiner[0])}, which joins the parts of a lock's name`;
+  return null;
+}

package/src/package.ts

@@ -39,14 +39,6 @@ export type PackageDataType = typeof PackageDataType;
 
 export type PackageData = ValueTypeOf<typeof PackageDataType>;
 
-// Backwards compatibility alias
-/** @deprecated Use PackageDataType instead */
-export const PackageDatasetsType = PackageDataType;
-/** @deprecated Use PackageData instead */
-export type PackageDatasetsType = PackageDataType;
-/** @deprecated Use PackageData instead */
-export type PackageDatasets = PackageData;
-
 /**
  * Package object stored in the object store.
  *
@@ -100,85 +92,37 @@ export const PackageObjectType = StructType({
   data: PackageDataType,
   /** Functions defined in this package: name -> FunctionObject hash. */
   functions: DictType(StringType, StringType),
-  /** Records defined in this package: name -> RecordObject hash.
-   *  BEAST2 encodes struct fields positionally in declaration order, so new
-   *  fields MUST be appended LAST — never inserted between existing fields. */
+  /** Records defined in this package: name -> RecordObject hash. */
   records: DictType(StringType, StringType),
-  /** Unresolved dataset sources: refPath (e.g. "inputs/table") -> descriptor.
-   *  Appended LAST, per the positional rule above. */
+  /** Unresolved dataset sources: refPath (e.g. "inputs/table") -> descriptor. */
   sources: DictType(StringType, DatasetSourceWireType),
 });
 export type PackageObjectType = typeof PackageObjectType;
 
 export type PackageObject = ValueTypeOf<typeof PackageObjectType>;
 
-/**
- * The pre-`sources` package object wire shape (tasks, data, functions,
- * records), kept only so {@link decodePackageObject} can read packages exported
- * before path-initialised inputs existed.
- */
-const RecordsEraPackageObjectType = StructType({
-  tasks: DictType(StringType, StringType),
-  data: PackageDataType,
-  functions: DictType(StringType, StringType),
-  records: DictType(StringType, StringType),
-});
-
-/**
- * The pre-`records` package object wire shape (tasks, data, functions), kept
- * only so {@link decodePackageObject} can read packages exported before records
- * existed.
- */
-const FunctionsEraPackageObjectType = StructType({
-  tasks: DictType(StringType, StringType),
-  data: PackageDataType,
-  functions: DictType(StringType, StringType),
-});
-
-/**
- * The pre-`functions` package object wire shape (tasks, data), kept only so
- * {@link decodePackageObject} can read packages exported before functions
- * existed.
- */
-const LegacyPackageObjectType = StructType({
-  tasks: DictType(StringType, StringType),
-  data: PackageDataType,
-});
-
 const decodeCurrent = decodeBeast2For(PackageObjectType);
-const decodeRecordsEra = decodeBeast2For(RecordsEraPackageObjectType);
-const decodeFunctionsEra = decodeBeast2For(FunctionsEraPackageObjectType);
-const decodeLegacy = decodeBeast2For(LegacyPackageObjectType);
 
 /**
- * Decode a `PackageObject` from BEAST2 bytes, tolerating the older wire
- * formats (dual-decode migration).
+ * Decode a `PackageObject` from BEAST2 bytes.
+ *
+ * @remarks
+ * A package-borne wire, so it changes by hard cutover: a package exported by
+ * an older SDK is re-exported with the current one, and this says so.
  *
- * Every package-read path — local AND cloud — must use this instead of
- * `decodeBeast2For(PackageObjectType)` directly, so packages exported before
- * the `sources`/`records`/`functions` fields existed keep decoding. Older
- * bytes decode with the missing maps defaulted to empty.
+ * @param data - the stored bytes
+ * @returns the package object
+ * @throws {Error} When the bytes are not a current package object — a package
+ *   exported by an older SDK, which is re-exported with the current one.
  */
 export function decodePackageObject(data: Uint8Array): PackageObject {
   try {
     return decodeCurrent(data);
   } catch (err) {
-    try {
-      const recEra = decodeRecordsEra(data);
-      return { ...recEra, sources: new Map() };
-    } catch {
-      try {
-        const fnEra = decodeFunctionsEra(data);
-        return { tasks: fnEra.tasks, data: fnEra.data, functions: fnEra.functions, records: new Map(), sources: new Map() };
-      } catch {
-        try {
-          const legacy = decodeLegacy(data);
-          return { tasks: legacy.tasks, data: legacy.data, functions: new Map(), records: new Map(), sources: new Map() };
-        } catch {
-          throw err; // no known shape — surface the current-format error
-        }
-      }
-    }
+    throw new Error(
+      `the package object does not decode: the package was exported by an older e3 SDK — re-export it with the current one ` +
+      `(${err instanceof Error ? err.message : String(err)})`,
+    );
   }
 }
 

package/src/record.spec.ts

@@ -0,0 +1,195 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The record wire types' decoders: each reads the current shape, and refuses
+ * an older one naming the fix — a commit an older e3 wrote, whose repository
+ * is re-created, and a mutation, migration or record object an older SDK
+ * exported, whose package is re-exported.
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { DateTimeType, DictType, IntegerType, OptionType, StringType, StructType, ArrayType, EastTypeType, encodeBeast2For, isTypeValueEqual, some, toEastTypeValue, variant } from '@elaraai/east';
+import { RunnerType } from './runner.js';
+import {
+  MigrationObjectType,
+  MutationObjectType,
+  RecordCommitType,
+  RecordObjectType,
+  decodeMigrationObject,
+  decodeMutationObject,
+  decodeRecordCommit,
+  decodeRecordObject,
+  type MigrationObject,
+  type MutationObject,
+  type RecordCommit,
+  type RecordObject,
+} from './record.js';
+
+describe('decodeRecordCommit', () => {
+  const commit: RecordCommit = {
+    parent: some('a'.repeat(64)),
+    state: 'b'.repeat(64),
+    mutation: 'increment',
+    args: some('c'.repeat(64)),
+    actor: 'cli:test',
+    at: new Date(0),
+    delta: some('d'.repeat(64)),
+  };
+
+  it('round-trips a current commit', () => {
+    assert.deepEqual(decodeRecordCommit(encodeBeast2For(RecordCommitType)(commit)), commit);
+  });
+
+  it('refuses a commit an older e3 wrote before deltas existed, naming the fix', () => {
+    const PreDeltaCommitType = StructType({
+      parent: OptionType(StringType), state: StringType, mutation: StringType,
+      args: OptionType(StringType), actor: StringType, at: DateTimeType,
+    });
+    const { delta: _delta, ...older } = commit;
+    assert.throws(
+      () => decodeRecordCommit(encodeBeast2For(PreDeltaCommitType)(older)),
+      /^Error: the record commit does not decode: an older e3 wrote this repository — re-create it: deploy again and import its data again \(/,
+    );
+  });
+
+  it('refuses bytes of no commit shape', () => {
+    assert.throws(() => decodeRecordCommit(encodeBeast2For(IntegerType)(7n)), /the record commit does not decode/);
+  });
+});
+
+describe('decodeMutationObject', () => {
+  const mutation: MutationObject = {
+    bodyIr: 'a'.repeat(64),
+    argTypes: [toEastTypeValue(IntegerType)],
+    runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
+    form: variant('edit', null),
+    programIr: 'b'.repeat(64),
+  };
+
+  /** A decoded mutation with its arg types checked off separately: a decoded
+   *  type value carries no memoized identity, so only the East comparison
+   *  says whether two of them are the same type. */
+  function read(bytes: Uint8Array): Omit<MutationObject, 'argTypes'> {
+    const decoded = decodeMutationObject(bytes);
+    assert.equal(decoded.argTypes.length, 1);
+    assert.ok(isTypeValueEqual(decoded.argTypes[0]!, toEastTypeValue(IntegerType)));
+    const { argTypes: _argTypes, ...rest } = decoded;
+    return rest;
+  }
+
+  it('round-trips a current mutation', () => {
+    const { argTypes: _argTypes, ...expected } = mutation;
+    assert.deepEqual(read(encodeBeast2For(MutationObjectType)(mutation)), expected);
+  });
+
+  it('refuses a mutation an older SDK exported before the delta, naming the fix', () => {
+    const PreDeltaMutationType = StructType({
+      bodyIr: StringType, argTypes: ArrayType(EastTypeType), runner: RunnerType,
+    });
+    const { form: _form, programIr: _programIr, argTypes: _argTypes, ...older } = mutation;
+    assert.throws(
+      () => decodeMutationObject(encodeBeast2For(PreDeltaMutationType)({ ...older, argTypes: mutation.argTypes })),
+      /^Error: the mutation object does not decode: the package was exported by an older e3 SDK — re-export it with the current one \(/,
+    );
+  });
+
+  it('refuses a mutation whose form is a String, as an older SDK wrote it, whatever the string', () => {
+    // A String form decoded as any string, and a form e3 does not know ran as
+    // whichever form its checks fell through to.
+    const StringFormMutationType = StructType({
+      bodyIr: StringType, argTypes: ArrayType(EastTypeType), runner: RunnerType, form: StringType, programIr: StringType,
+    });
+    for (const form of ['edit', 'bogus']) {
+      assert.throws(
+        () => decodeMutationObject(encodeBeast2For(StringFormMutationType)({ ...mutation, form })),
+        /^Error: the mutation object does not decode: the package was exported by an older e3 SDK — re-export it with the current one \(/,
+        form,
+      );
+    }
+  });
+});
+
+describe('decodeMigrationObject', () => {
+  const RowV1Type = StructType({ name: StringType });
+  const RowV2Type = StructType({ name: StringType, shift: IntegerType });
+  const migration: MigrationObject = {
+    form: variant('rows', null),
+    from: toEastTypeValue(DictType(StringType, RowV1Type)),
+    to: toEastTypeValue(DictType(StringType, RowV2Type)),
+    bodyIr: 'a'.repeat(64),
+    programIr: 'b'.repeat(64),
+    runner: variant('east_node', { platforms: ['@elaraai/east-node-std'] }),
+  };
+
+  it('round-trips a current migration, its types among it', () => {
+    const decoded = decodeMigrationObject(encodeBeast2For(MigrationObjectType)(migration));
+    // A decoded type value carries no memoized identity, so only the East
+    // comparison says whether two of them are the same type.
+    assert.ok(isTypeValueEqual(decoded.from, migration.from));
+    assert.ok(isTypeValueEqual(decoded.to, migration.to));
+    assert.ok(!isTypeValueEqual(decoded.from, decoded.to));
+    const { from: _from, to: _to, ...rest } = decoded;
+    const { from: _f, to: _t, ...expected } = migration;
+    assert.deepEqual(rest, expected);
+  });
+
+  it('refuses the bytes of another object, naming the re-export', () => {
+    const mutation: MutationObject = {
+      bodyIr: 'a'.repeat(64), argTypes: [], runner: migration.runner, form: variant('reduce', null), programIr: '',
+    };
+    assert.throws(
+      () => decodeMigrationObject(encodeBeast2For(MutationObjectType)(mutation)),
+      /^Error: the migration object does not decode: the package was exported by an older e3 SDK — re-export it with the current one \(/,
+    );
+  });
+
+  it('refuses a migration whose form is a String, as an older SDK wrote it, whatever the string', () => {
+    // A deploy ran a String form it did not know as a split step.
+    const StringFormMigrationType = StructType({
+      form: StringType, from: EastTypeType, to: EastTypeType, bodyIr: StringType, programIr: StringType, runner: RunnerType,
+    });
+    for (const form of ['rows', 'bogus']) {
+      assert.throws(
+        () => decodeMigrationObject(encodeBeast2For(StringFormMigrationType)({ ...migration, form })),
+        /^Error: the migration object does not decode: the package was exported by an older e3 SDK — re-export it with the current one \(/,
+        form,
+      );
+    }
+  });
+});
+
+describe('decodeRecordObject', () => {
+  const record: RecordObject = {
+    path: 'records/orders',
+    mutations: new Map([['place', 'a'.repeat(64)]]),
+    indexes: new Map([['by_status', 'b'.repeat(64)]]),
+    migrations: [{ name: 'add_shift', migration: 'c'.repeat(64) }, { name: 'widen_row', migration: 'd'.repeat(64) }],
+  };
+
+  /** A decoded record object with its dicts as entry lists: a decoded Dict is
+   *  a SortedMap, which is the same mapping and not the same container. */
+  const read = (bytes: Uint8Array) => {
+    const decoded = decodeRecordObject(bytes);
+    return { path: decoded.path, mutations: [...decoded.mutations], indexes: [...decoded.indexes], migrations: decoded.migrations };
+  };
+
+  it('round-trips a current record object, its migrations in their order', () => {
+    assert.deepEqual(read(encodeBeast2For(RecordObjectType)(record)),
+      { path: record.path, mutations: [...record.mutations], indexes: [...record.indexes], migrations: record.migrations });
+  });
+
+  it('refuses a record object an older SDK exported before migrations, naming the fix', () => {
+    const IndexesEraRecordType = StructType({
+      path: StringType, mutations: DictType(StringType, StringType), indexes: DictType(StringType, StringType),
+    });
+    const { migrations: _migrations, ...older } = record;
+    assert.throws(
+      () => decodeRecordObject(encodeBeast2For(IndexesEraRecordType)(older)),
+      /^Error: the record object does not decode: the package was exported by an older e3 SDK — re-export it with the current one \(/,
+    );
+  });
+});