@elaraai/e3-types 1.0.79 → 1.0.81

package/src/dataset-blob.spec.ts

@@ -0,0 +1,117 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * The encoder door: a collection is written as the manifest the Writer's own
+ * manifest writer writes for its value — which the conformance corpus pins in
+ * every runtime — whichever pieces it arrives in.
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  Beast2ManifestWriter,
+  DictType,
+  IntegerType,
+  SortedMap,
+  StringType,
+  StructType,
+  compareFor,
+  decodeBeast2FenceFor,
+  decodeCollectionManifest,
+  encodeBeast2For,
+  encodeBeast2PagedFor,
+  sha256Hex,
+} from '@elaraai/east';
+import { encodeDatasetBlob, writeCollectionManifest, type CollectionSegmentRef } from './dataset-blob.js';
+
+const TableType = DictType(StringType, IntegerType);
+
+describe('writeCollectionManifest', () => {
+  it("writes the manifest writer's manifest for the value, byte for byte", async () => {
+    const value = new SortedMap(
+      Array.from({ length: 20_000 }, (_, i): [string, bigint] => [`k${String(i).padStart(6, '0')}`, BigInt(i)]),
+      compareFor(StringType));
+    const objects = new Map<string, Uint8Array>();
+    const manifest = await writeCollectionManifest(TableType, [{ elements: value.entries() }], async (bytes) => {
+      const hash = sha256Hex(bytes);
+      objects.set(hash, bytes);
+      return hash;
+    });
+
+    const expectedObjects = new Map<string, Uint8Array>();
+    let expected: Uint8Array | undefined;
+    const writer = new Beast2ManifestWriter(TableType, {
+      object: (hash, bytes) => { expectedObjects.set(hash, bytes); },
+      manifest: (bytes) => { expected = bytes; },
+    });
+    for (const element of value) writer.add(element);
+    writer.finish();
+
+    assert.deepEqual(manifest, expected);
+    assert.deepEqual([...objects.keys()].sort(), [...expectedObjects.keys()].sort(), 'the same objects, header included');
+    assert.ok(decodeCollectionManifest(manifest).entries.length > 1, 'the value spans segments');
+  });
+
+  it('names the segments a piece carries with their entries again, neither reading nor writing them', async () => {
+    const value = new SortedMap(
+      Array.from({ length: 20_000 }, (_, i): [string, bigint] => [`k${String(i).padStart(6, '0')}`, BigInt(i)]),
+      compareFor(StringType));
+    const objects = new Map<string, Uint8Array>();
+    const sink = async (bytes: Uint8Array): Promise<string> => {
+      const hash = sha256Hex(bytes);
+      objects.set(hash, bytes);
+      return hash;
+    };
+    const whole = await writeCollectionManifest(TableType, [{ elements: value.entries() }], sink);
+    const entries = decodeCollectionManifest(whole).entries;
+    const half = entries.length >> 1;
+
+    // The first half as the segments they are, each with the fence that
+    // followed it; the rest as elements, from the first key of the second half.
+    const reads: string[] = [];
+    const carried: CollectionSegmentRef[] = entries.slice(0, half).map((entry, i) => ({
+      count: Number(entry.count),
+      fence: entry.fence,
+      nextFence: entries[i + 1]!.fence,
+      read: () => {
+        reads.push(entry.hash);
+        return objects.get(entry.hash)!;
+      },
+      entry,
+    }));
+    const written = new Set<string>();
+    const again = await writeCollectionManifest(TableType, [
+      { segments: carried },
+      { elements: value.entries(decodeBeast2FenceFor(StringType)(entries[half]!.fence) as string) },
+    ], async (bytes) => {
+      const hash = await sink(bytes);
+      written.add(hash);
+      return hash;
+    });
+
+    assert.deepEqual(again, whole, 'the same manifest');
+    assert.deepEqual(reads, [], 'the seam is decided by the fence that followed');
+    for (const entry of entries.slice(0, half)) assert.ok(!written.has(entry.hash), `segment ${entry.hash.slice(0, 8)} is carried`);
+  });
+});
+
+describe('encodeDatasetBlob', () => {
+  it('encodes a collection without a sink as its paged blob, and any other value whole', async () => {
+    const value = new Map([['a', 1n], ['b', 2n]]);
+    assert.deepEqual(encodeDatasetBlob(TableType, value), encodeBeast2PagedFor(TableType)(value));
+
+    const RowType = StructType({ id: IntegerType, name: StringType });
+    const row = { id: 1n, name: 'one' };
+    const whole = encodeBeast2For(RowType)(row);
+    assert.deepEqual(encodeDatasetBlob(RowType, row), whole);
+    const sunk: Uint8Array[] = [];
+    assert.deepEqual(await encodeDatasetBlob(RowType, row, async (bytes) => {
+      sunk.push(bytes);
+      return sha256Hex(bytes);
+    }), whole, 'a sink changes nothing for a value that is not a collection');
+    assert.deepEqual(sunk, []);
+  });
+});

package/src/dataset-blob.ts

@@ -7,33 +7,57 @@
  * The ONE encoder that decides how a dataset value is stored.
  *
  * Collection-rooted values (Array / Set / Dict) are ALWAYS stored segmented
- * with a trailing index, at every size: one uniform encoding per logical value,
- * so the paged read API can decode just the segments a window touches and the
- * key search can bisect the segment fences. Every other root is stored whole.
+ * under the content-defined cut rule, at every size: one uniform encoding per
+ * logical value, so the paged read API can decode just the segments a window
+ * touches and the key search can bisect the segment fences. Every other root is
+ * stored whole.
  *
- * This lives in `e3-types` — the floor both `e3` and `e3-core` stand on —
- * because the rule has to hold at EVERY door a value enters the store through,
- * and it did not. The store path (`e3-core`'s `datasetWrite`) segmented; the
- * package export path (`e3`'s `export_`) encoded flat regardless of root kind.
- * Since `workspaceDeploy` copies package refs verbatim, a freshly deployed
- * collection input pointed at an unindexed blob and could not be paged at all
- * — `dataset_not_indexed`, with no whole-decode fallback — until something
- * happened to WRITE the dataset, at which point it silently started working.
- * A demo poked by hand behaved differently from a workspace only deployed.
+ * Given a `sink` — an object store to write through — a collection goes in as
+ * **segment objects plus a {@link CollectionManifestType} naming them**, and it
+ * is the manifest the dataset ref points at. That is what makes a one-row edit
+ * cost one segment: the new manifest names the same objects as the old one bar
+ * the segment that changed, and a content-addressed store deduplicates the
+ * rest. Without a sink the value is one blob, which is what travels over the
+ * wire.
  *
- * So: one branch, one home, and both doors call it.
+ * This lives in `e3-types` — the floor both `e3` and `e3-core` stand on —
+ * because the rule has to hold at EVERY door a value enters the store through:
+ * the store's own door (`e3-core`'s `storeCollection`) and the package export
+ * both write collections through {@link writeCollectionManifest}.
  *
  * @packageDocumentation
  */
 
 import {
+  SortedMap,
+  SortedSet,
+  compareFor,
   encodeBeast2For,
   encodeBeast2PagedFor,
   isVariant,
+  recutBeast2For,
+  segmentRuleFor,
   toEastTypeValue,
+  type Beast2RecutPiece,
+  type Beast2SegmentRef,
   type EastType,
   type EastTypeValue,
 } from '@elaraai/east';
+import {
+  COLLECTION_MANIFEST_KIND,
+  encodeCollectionManifest,
+  type CollectionManifestEntry,
+} from './collection-manifest.js';
+
+/**
+ * Writes one object and answers with its content hash.
+ *
+ * @remarks
+ * The whole of the object store this module needs. Content-addressed, so
+ * writing bytes already stored is a no-op that returns the same hash — which is
+ * exactly how a new state shares its unchanged segments with the old one.
+ */
+export type SegmentSink = (bytes: Uint8Array) => Promise<string>;
 
 /**
  * Whether a dataset root type is a collection — the kinds stored segmented +
@@ -46,6 +70,89 @@ export function isCollectionRoot(typeValue: EastTypeValue): boolean {
   return typeValue.type === 'Array' || typeValue.type === 'Set' || typeValue.type === 'Dict';
 }
 
+/** The caller's type argument as an {@link EastTypeValue}. */
+function asTypeValue(type: EastType | EastTypeValue): EastTypeValue {
+  return isVariant(type) ? (type as EastTypeValue) : toEastTypeValue(type as EastType);
+}
+
+/**
+ * A segment a collection's manifest may carry over as it stands: the segment
+ * as a re-cut takes it, and the manifest entry that already names it in the
+ * store, when it is there.
+ */
+export interface CollectionSegmentRef extends Beast2SegmentRef {
+  /** The entry naming the segment in the manifest it came from. A carried
+   *  segment with one is named by it again, never read or written; one
+   *  without is read and written as it stands. */
+  readonly entry?: CollectionManifestEntry;
+}
+
+/** A piece of a collection, in order, as {@link writeCollectionManifest}
+ *  takes it: a run of segments the Writer wrote, under the canonical header
+ *  for the type, or elements. */
+export type CollectionPiece = Beast2RecutPiece<EastType, CollectionSegmentRef>;
+
+/**
+ * Write a collection as segment objects and return the manifest naming them.
+ *
+ * @remarks
+ * The pieces are re-cut into the canonical whole (`recutBeast2For`): a segment
+ * the whole shares with its piece is carried over, and everything else — the
+ * seams between pieces, and pieces given as elements — is cut by the rule and
+ * written. What comes out is what the Writer writes for the whole value,
+ * whichever pieces it came in, so two equal values have one manifest.
+ *
+ * Every segment is written under the canonical header for the type, the
+ * Writer's own: a piece's segments are carried over only when that is the
+ * header they are under, which is the caller's to establish.
+ *
+ * @param type - The collection type (Array / Set / Dict)
+ * @param pieces - The collection's pieces, in order
+ * @param sink - Writes one object and returns its hash
+ * @returns The manifest's bytes — the caller stores them, and its hash is the
+ *   dataset's content address
+ * @throws {Error} When a piece's elements do not ascend (Set / Dict), a
+ *   segment's blob is not the one segment its reference describes, or the sink
+ *   rejects.
+ */
+export async function writeCollectionManifest(
+  type: EastType | EastTypeValue,
+  pieces: Iterable<CollectionPiece> | AsyncIterable<CollectionPiece>,
+  sink: SegmentSink,
+): Promise<Uint8Array> {
+  const typeValue = asTypeValue(type);
+  const entries: CollectionManifestEntry[] = [];
+  // Frames deflate on the worker pool once the value is large enough to be
+  // worth it; the pool holds the frames in flight and nothing more.
+  const recut = recutBeast2For<EastType, CollectionSegmentRef>(typeValue, { parallel: true });
+  const stats = await recut(pieces, {
+    written: async (segment) => {
+      entries.push({
+        hash: await sink(segment.blob),
+        fence: segment.fence,
+        count: BigInt(segment.count),
+        bytes: BigInt(segment.blob.byteLength),
+      });
+    },
+    carried: async (ref) => {
+      if (ref.entry !== undefined) {
+        entries.push(ref.entry);
+        return;
+      }
+      const blob = await ref.read();
+      entries.push({ hash: await sink(blob), fence: ref.fence, count: BigInt(ref.count), bytes: BigInt(blob.byteLength) });
+    },
+  });
+  return encodeCollectionManifest({
+    kind: COLLECTION_MANIFEST_KIND,
+    level: 0n,
+    type: typeValue,
+    rule: segmentRuleFor(typeValue),
+    header: await sink(stats.header),
+    entries,
+  });
+}
+
 /**
  * Encode a dataset value for the object store.
  *
@@ -54,15 +161,44 @@ export function isCollectionRoot(typeValue: EastTypeValue): boolean {
  * encoder themselves: a value that reaches the store unsegmented is one the
  * paged endpoints refuse, and nothing on the read path can repair it.
  *
+ * With a `sink`, a collection root is written as segment objects and the
+ * returned bytes are the manifest naming them; a non-collection root ignores
+ * the sink and returns its whole blob, which is what it has always been.
+ * Without one, a collection is its canonical segmented blob — the form it
+ * travels over the wire in.
+ *
  * @param type - The dataset's declared type (an `EastType` or its homoiconic value)
  * @param value - The value to encode
- * @returns The beast2 bytes to store — segmented + indexed for a collection root
+ * @param sink - Writes one object and returns its hash
+ * @returns The beast2 bytes to store — the manifest, for a collection written
+ *   through a sink; otherwise the value's own blob
  */
-export function encodeDatasetBlob(type: EastType | EastTypeValue, value: unknown): Uint8Array {
-  const typeValue: EastTypeValue = isVariant(type)
-    ? (type as EastTypeValue)
-    : toEastTypeValue(type as EastType);
-  return isCollectionRoot(typeValue)
-    ? encodeBeast2PagedFor(typeValue)(value)
-    : encodeBeast2For(typeValue)(value);
+export function encodeDatasetBlob(type: EastType | EastTypeValue, value: unknown): Uint8Array;
+export function encodeDatasetBlob(type: EastType | EastTypeValue, value: unknown, sink: SegmentSink): Promise<Uint8Array>;
+export function encodeDatasetBlob(
+  type: EastType | EastTypeValue,
+  value: unknown,
+  sink?: SegmentSink,
+): Uint8Array | Promise<Uint8Array> {
+  const typeValue = asTypeValue(type);
+  if (!isCollectionRoot(typeValue)) {
+    const blob = encodeBeast2For(typeValue)(value);
+    return sink === undefined ? blob : Promise.resolve(blob);
+  }
+  if (sink === undefined) return encodeBeast2PagedFor(typeValue)(value);
+  return writeCollectionManifest(typeValue, [{ elements: canonicalElements(typeValue, value) }], sink);
+}
+
+/** A collection value's elements in canonical order: an Array's as they
+ *  stand, a Set's or a Dict's ascending in East order — a plain `Set` or
+ *  `Map` iterates in insertion order, and is sorted first. */
+function canonicalElements(typeValue: EastTypeValue, value: unknown): Iterable<unknown> {
+  if (typeValue.type === 'Array') return value as unknown[];
+  if (typeValue.type === 'Set') {
+    return value instanceof SortedSet ? value : [...(value as Set<unknown>)].sort(compareFor(typeValue.value as EastTypeValue));
+  }
+  const cmp = compareFor((typeValue.value as { key: EastTypeValue }).key);
+  return value instanceof SortedMap
+    ? (value as SortedMap<unknown, unknown>).entries()
+    : [...(value as Map<unknown, unknown>).entries()].sort((a, b) => cmp(a[0], b[0]));
 }

package/src/dataset-ref.ts

@@ -6,9 +6,9 @@
 /**
  * Per-dataset reference types for reactive dataflow.
  *
- * Each dataset in a workspace has its own `.ref` file instead of being
- * part of a single root tree. This enables concurrent writes and
- * reactive re-execution.
+ * Each dataset in a workspace has its own ref instead of being part of a
+ * single root tree. This enables concurrent writes and reactive
+ * re-execution.
  *
  * A DatasetRef tracks:
  * - The current value hash (or unassigned/null state)
@@ -48,7 +48,8 @@ export type VersionVector = ValueTypeOf<typeof VersionVectorType>;
 /**
  * Per-dataset reference stored in workspace data files.
  *
- * Each dataset has a `.ref` file at `workspaces/<ws>/data/<path>.ref`.
+ * A local repository keeps each dataset's ref at
+ * `workspaces/<ws>/data/<path>.beast2`, beside the revision its write minted.
  * The ref tracks the dataset's current state and version provenance.
  *
  * Variants:

package/src/environment.ts

@@ -14,8 +14,9 @@
  * implementation code (via lockfile + sdist/tarball hashes) into execution
  * cache identity.
  *
- * All fields hold object-store hashes, never inline file contents: the blobs
- * ride the existing content-addressed object store and transfer machinery.
+ * All fields hold object-store hashes, never inline file contents: each names
+ * an object holding the file's bytes as a beast2 Blob, which rides the
+ * content-addressed object store and transfer machinery.
  */
 
 import { ArrayType, OptionType, StringType, StructType, ValueTypeOf, VariantType } from '@elaraai/east';

package/src/execution.spec.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+
+/**
+ * Execution status wire: the typed outcomes with their peaks, a split task's
+ * plan and whether the execution is a unit, what each keeps from gc, and the
+ * refusal of a record an older e3 wrote before `cancelled` and `interrupted`
+ * were cases of it, before a runner's peak was recorded, before a split task's
+ * plan was, or before a record said whether it is a unit.
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { ArrayType, DateTimeType, IntegerType, OptionType, StringType, StructType, VariantType, encodeBeast2For, none, some, variant } from '@elaraai/east';
+import { ExecutionStatusType, decodeExecutionStatus, executionStatusRoots, type ExecutionStatus } from './execution.js';
+
+/** The four-case wire every record had before the typed outcomes. */
+const PreOutcomeExecutionStatusType = VariantType({
+  running: StructType({
+    executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType,
+    pid: IntegerType, pidStartTime: IntegerType, bootId: StringType,
+  }),
+  success: StructType({
+    executionId: StringType, inputHashes: ArrayType(StringType), outputHash: StringType,
+    startedAt: DateTimeType, completedAt: DateTimeType,
+  }),
+  failed: StructType({
+    executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType,
+    completedAt: DateTimeType, exitCode: IntegerType,
+  }),
+  error: StructType({
+    executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType,
+    completedAt: DateTimeType, message: StringType,
+  }),
+});
+
+describe('ExecutionStatusType', () => {
+  const stopped = {
+    executionId: '0199-a',
+    inputHashes: ['a'.repeat(64)],
+    startedAt: new Date(1000),
+    completedAt: new Date(2000),
+  };
+  const encodeOlder = encodeBeast2For(PreOutcomeExecutionStatusType);
+
+  it('round-trips the typed outcomes, with a runner\'s peak or without one, a task\'s own or a unit\'s', () => {
+    const own = { ...stopped, unit: false };
+    const statuses: ExecutionStatus[] = [
+      variant('success', { ...own, outputHash: 'b'.repeat(64), peakBytes: some(96n * 1024n ** 2n), plan: none }),
+      variant('success', { ...own, outputHash: 'b'.repeat(64), peakBytes: none, plan: some('c'.repeat(64)) }),
+      variant('success', { ...own, outputHash: 'b'.repeat(64), peakBytes: some(64n * 1024n ** 2n), plan: none, unit: true }),
+      variant('running', { executionId: '0199-b', inputHashes: ['a'.repeat(64)], startedAt: new Date(1000), pid: 4242n, pidStartTime: 7n, bootId: 'boot', unit: true }),
+      variant('failed', { ...own, exitCode: 1n, peakBytes: some(412n * 1024n ** 2n) }),
+      variant('failed', { ...own, exitCode: -1n, peakBytes: none }),
+      variant('cancelled', own),
+      variant('interrupted', { ...own, pid: 4242n }),
+      variant('error', { ...own, message: 'Failed to read output: no such file' }),
+    ];
+    const encode = encodeBeast2For(ExecutionStatusType);
+    for (const status of statuses) assert.deepEqual(decodeExecutionStatus(encode(status)), status);
+  });
+
+  it('keeps from gc a success\'s output, and a split task\'s last plan beside it', () => {
+    const success = { ...stopped, outputHash: 'b'.repeat(64), peakBytes: none, unit: false };
+    assert.deepEqual(executionStatusRoots(variant('success', { ...success, plan: none })), ['b'.repeat(64)]);
+    assert.deepEqual(executionStatusRoots(variant('success', { ...success, plan: some('c'.repeat(64)) })), ['b'.repeat(64), 'c'.repeat(64)]);
+    assert.deepEqual(executionStatusRoots(variant('failed', { ...stopped, exitCode: 1n, peakBytes: none, unit: false })), []);
+    assert.deepEqual(executionStatusRoots(variant('cancelled', { ...stopped, unit: false })), []);
+  });
+
+  it('refuses a record an older e3 wrote, naming the fix', () => {
+    const refusal = /^Error: the execution status does not decode: an older e3 wrote this repository — re-create it: deploy again and import its data again \(/;
+    const records = [
+      variant('error', { ...stopped, message: 'cancelled: e3 stopped the runner because the run was aborted' }),
+      variant('success', { ...stopped, outputHash: 'b'.repeat(64) }),
+    ] as const;
+    for (const record of records) {
+      assert.throws(() => decodeExecutionStatus(encodeOlder(record)), refusal);
+    }
+    // The typed outcomes, before a runner's peak was recorded.
+    const withoutPeaks = VariantType({
+      ...PreOutcomeExecutionStatusType.cases,
+      cancelled: StructType({ executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType, completedAt: DateTimeType }),
+      interrupted: StructType({
+        executionId: StringType, inputHashes: ArrayType(StringType), startedAt: DateTimeType, completedAt: DateTimeType, pid: IntegerType,
+      }),
+    });
+    assert.throws(() => decodeExecutionStatus(encodeBeast2For(withoutPeaks)(variant('success', { ...stopped, outputHash: 'b'.repeat(64) }))), refusal);
+    // With peaks, before a split task's plan was recorded.
+    const withoutPlans = VariantType({
+      ...ExecutionStatusType.cases,
+      success: StructType({
+        executionId: StringType, inputHashes: ArrayType(StringType), outputHash: StringType,
+        startedAt: DateTimeType, completedAt: DateTimeType, peakBytes: OptionType(IntegerType),
+      }),
+    });
+    assert.throws(() => decodeExecutionStatus(encodeBeast2For(withoutPlans)(variant('success', { ...stopped, outputHash: 'b'.repeat(64), peakBytes: none }))), refusal);
+    // With plans, before a record said whether it is a unit of a split task.
+    const withoutUnits = VariantType({
+      ...ExecutionStatusType.cases,
+      success: StructType({
+        executionId: StringType, inputHashes: ArrayType(StringType), outputHash: StringType,
+        startedAt: DateTimeType, completedAt: DateTimeType, peakBytes: OptionType(IntegerType), plan: OptionType(StringType),
+      }),
+    });
+    assert.throws(() => decodeExecutionStatus(encodeBeast2For(withoutUnits)(variant('success', { ...stopped, outputHash: 'b'.repeat(64), peakBytes: none, plan: none }))), refusal);
+  });
+
+  it('throws for bytes of no status shape', () => {
+    assert.throws(() => decodeExecutionStatus(encodeBeast2For(IntegerType)(7n)), /the execution status does not decode/);
+  });
+});