@elaraai/e3-types 1.0.84 → 1.0.86

package/src/dataflow.spec.ts

@@ -12,8 +12,8 @@
 
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
-import { ArrayType, DictType, StringType, StructType, encodeBeast2For, none, some, variant } from '@elaraai/east';
-import { decodeDataflowExecutionState } from './dataflow.js';
+import { ArrayType, DictType, StringType, StructType, encodeBeast2For, equalFor, none, printFor, some, variant } from '@elaraai/east';
+import { DataflowForceType, decodeDataflowExecutionState } from './dataflow.js';
 import { E3_RELEASE } from './release.js';
 
 /** Version 1, written by the released e3: task `double` completed, `sum`
@@ -150,8 +150,9 @@ const STATE_V5 = Buffer.from(
 );
 
 /** The version 5 state, as release 1.0.79 writes it: the release in place of
- *  the version. */
-const STATE = Buffer.from(
+ *  the version. Its `force` is a Boolean, which a repository upgrade carries
+ *  forward (`dataflow-force-tasks`). */
+const STATE_1_0_79 = Buffer.from(
   'iUVhc3QNCgUAngsoKQEFBAAIAgRub25lAwRzb21lAAoACQUEbmFtZQAEaGFzaAAGaW5wdXRzBQZvdXRwdXQACWRlcGVuZHNP' +
   'bgUKBgkBBXRhc2tzBwgCBG5vbmUDBHNvbWUICAIEbm9uZQMEc29tZQICCAIEbm9uZQMEc29tZQsIAgRub25lAwRzb21lAQkC' +
   'CmlucHV0c0hhc2gAC2V4ZWN1dGlvbklkAAgCBG5vbmUDBHNvbWUOCQsEbmFtZQAGc3RhdHVzAAZjYWNoZWQKCm91dHB1dEhh' +
@@ -180,6 +181,37 @@ const STATE = Buffer.from(
   'base64',
 );
 
+/** The release 1.0.79 state as release 1.0.85 writes it: `force` names the
+ *  tasks the run forces, `double`. */
+const STATE = Buffer.from(
+  'iUVhc3QNCgUAsgspKgEFAAoACAMDYWxsAgRub25lAgV0YXNrcwMIAgRub25lAgRzb21lAAkFBG5hbWUABGhhc2gABmlucHV0' +
+  'cwMGb3V0cHV0AAlkZXBlbmRzT24DCgYJAQV0YXNrcwcIAgRub25lAgRzb21lCAQIAgRub25lAgRzb21lCgIIAgRub25lAgRz' +
+  'b21lDAgCBG5vbmUCBHNvbWUBCQIKaW5wdXRzSGFzaAALZXhlY3V0aW9uSWQACAIEbm9uZQIEc29tZQ8JCwRuYW1lAAZzdGF0' +
+  'dXMABmNhY2hlZAsKb3V0cHV0SGFzaAUFZXJyb3IFCGV4aXRDb2RlDQlzdGFydGVkQXQOC2NvbXBsZXRlZEF0DghkdXJhdGlv' +
+  'bg0EcGxhbgUJZXhlY3V0aW9uEAsAEQsAAAsAEwkDA3NlcQwJdGltZXN0YW1wAQZyZWFzb24FCQgDc2VxDAl0aW1lc3RhbXAB' +
+  'B3N1Y2Nlc3MKCGV4ZWN1dGVkDAZjYWNoZWQMBmZhaWxlZAwHc2tpcHBlZAwIZHVyYXRpb24MCQQDc2VxDAl0aW1lc3RhbXAB' +
+  'C2V4ZWN1dGlvbklkAAp0b3RhbFRhc2tzDAkFA3NlcQwJdGltZXN0YW1wAQRwYXRoAAxwcmV2aW91c0hhc2gAB25ld0hhc2gA' +
+  'CQcDc2VxDAl0aW1lc3RhbXABBHRhc2sABmNhY2hlZAoKb3V0cHV0SGFzaAAIZHVyYXRpb24MCXBlYWtCeXRlcw0JBANzZXEM' +
+  'CXRpbWVzdGFtcAEEdGFzawAMY29uZmxpY3RQYXRoAAkGA3NlcQwJdGltZXN0YW1wAQR0YXNrAAVlcnJvcgUIZXhpdENvZGUN' +
+  'CGR1cmF0aW9uDAkEA3NlcQwJdGltZXN0YW1wAQR0YXNrAAZyZWFzb24ACQUDc2VxDAl0aW1lc3RhbXABBHRhc2sABWxldmVs' +
+  'DAZsZXZlbHMMCQYDc2VxDAl0aW1lc3RhbXABBHRhc2sABWxldmVsDAZsZXZlbHMMBXVuaXRzDAkDA3NlcQwJdGltZXN0YW1w' +
+  'AQR0YXNrAAkEA3NlcQwJdGltZXN0YW1wAQR0YXNrAAVjYXVzZQAJBANzZXEMCXRpbWVzdGFtcAEEdGFzawAGcGllY2VzDAkC' +
+  'BWxldmVsDAZsZXZlbHMMCAIEbm9uZQIEc29tZSIJAwVtZXJnZSMFaW5kZXgMBXVuaXRzDAgDBmJ1ZGdldAIDY2FwAgdtYWNo' +
+  'aW5lAgkHA3NlcQwJdGltZXN0YW1wAQR0YXNrAAR1bml0JAZyZWFzb24lBHBlYWsMCHJlc2VydmVzDAgPE2V4ZWN1dGlvbl9j' +
+  'YW5jZWxsZWQVE2V4ZWN1dGlvbl9jb21wbGV0ZWQWEWV4ZWN1dGlvbl9zdGFydGVkFw1pbnB1dF9jaGFuZ2VkGA50YXNrX2Nv' +
+  'bXBsZXRlZBkNdGFza19kZWZlcnJlZBoLdGFza19mYWlsZWQbEHRhc2tfaW52YWxpZGF0ZWQcFHRhc2tfbWVyZ2VfY29tcGxl' +
+  'dGVkHRJ0YXNrX21lcmdlX3N0YXJ0ZWQeCnRhc2tfcmVhZHkfDHRhc2tfc2tpcHBlZCAKdGFza19zcGxpdCEMdGFza19zdGFy' +
+  'dGVkHw11bml0X3JlcXVldWVkJgonCRcHcmVsZWFzZQACaWQABHJlcG8ACXdvcmtzcGFjZQAJc3RhcnRlZEF0AQVmb3JjZQQG' +
+  'ZmlsdGVyBQVncmFwaAkJZ3JhcGhIYXNoBQV0YXNrcxIIZXhlY3V0ZWQMBmNhY2hlZAwGZmFpbGVkDAdza2lwcGVkDAZzdGF0' +
+  'dXMAC2NvbXBsZXRlZEF0DgVlcnJvcgUOdmVyc2lvblZlY3RvcnMUDWlucHV0U25hcHNob3QTD3Rhc2tPdXRwdXRQYXRocwMK' +
+  'cmVleGVjdXRlZAwGZXZlbnRzKAhldmVudFNlcQwBAAHvBpkCYzPUM9CzMFUxMLQ0SDRIMtC1AAJdcwMDIAtEADEcsBSlFuQz' +
+  'lRc3PDl96n0aEwMjW0p+aVJOKgMDIwMTlO2QSCFgYOTUy8wrKC0p1qtg4NUrSSzOLtaD2cNcXJrrkEIhYGBEM5YLygUajuQn' +
+  'BrifoBRncn5uQU5qSWoKIwOjQzKFABhoFz6DwpFxQfMZECUBNDSJQkBMNBqCAhGEuTPz4guK8tOLUouLQd5l/DAJ5BAggynV' +
+  'FOh7oBB7UWleXmZeOkgaOWIc0igEQONRIwE5DoD2MjFBEhlx6ZKXBRKU0IhiYYMEKSwqKY4rCcaGhoYEXg5IAIHCjocLwWbh' +
+  '4zmwCMZhYGJhACqeBtIxjZGTb8ImmAwTExMDHwA=',
+  'base64',
+);
+
 const NEW_FORM = 'a change to the execution state\'s type is a new form: ship a repository upgrade step that carries a repository\'s states into it, move this state to the refusals, and add one the new form writes';
 
 describe('decodeDataflowExecutionState', () => {
@@ -195,7 +227,9 @@ describe('decodeDataflowExecutionState', () => {
   it('reads a state in this release\'s form', () => {
     assert.doesNotThrow(() => decodeDataflowExecutionState(STATE), NEW_FORM);
     const state = decodeDataflowExecutionState(STATE);
-    assert.equal(state.release, '1.0.79');
+    assert.equal(state.release, '1.0.85');
+    const forced = variant('tasks', ['double']);
+    assert.ok(equalFor(DataflowForceType)(state.force, forced), `it forces double, not ${printFor(DataflowForceType)(state.force)}`);
     assert.deepEqual(state.tasks.get('sum')!.plan, some('e5'));
     assert.deepEqual(state.tasks.get('sum')!.execution, none);
     assert.deepEqual(state.tasks.get('double')!.execution, some({ inputsHash: 'b'.repeat(64), executionId: '0190a0b0-8888-7000-8000-000000000001' }));
@@ -211,6 +245,13 @@ describe('decodeDataflowExecutionState', () => {
     });
   });
 
+  it('refuses a state an earlier release wrote, which a repository upgrade carries forward: release 1.0.79\'s Boolean force', () => {
+    assert.throws(
+      () => decodeDataflowExecutionState(STATE_1_0_79),
+      { message: `the execution state was written by e3 1.0.79, in a form this e3, ${E3_RELEASE}, does not read — re-create the repository: deploy again and import its data again` },
+    );
+  });
+
   it('refuses a state a newer release wrote, naming it', () => {
     const newer = encodeBeast2For(StructType({ release: StringType, id: StringType }))({ release: '999.0.0', id: '7' });
     assert.throws(
@@ -229,8 +270,9 @@ describe('decodeDataflowExecutionState', () => {
     );
   });
 
-  it('refuses data that is not an execution state', () => {
+  it('refuses data that is not an execution state: a value of another type, or bytes that are not beast2', () => {
     const other = encodeBeast2For(StructType({ id: StringType }))({ id: '7' });
-    assert.throws(() => decodeDataflowExecutionState(other), /not an execution state/);
+    assert.throws(() => decodeDataflowExecutionState(other), { message: 'the data is not an execution state: its type is not the execution state\'s' });
+    assert.throws(() => decodeDataflowExecutionState(new TextEncoder().encode('not a run')), /^Error: the data is not an execution state: it is not beast2 \(/);
   });
 });

package/src/dataflow.ts

@@ -11,7 +11,8 @@
  *
  * Key design decisions:
  * - Dates are Date objects (via DateTimeType), not strings
- * - Events stored inline as array (single file, not separate JSONL)
+ * - A run's events are kept apart from its state, so a write of the state
+ *   costs what the run holds, not what it has done
  * - Tasks stored as Dict (not Map) for beast2 compatibility
  */
 
@@ -30,8 +31,11 @@ import {
   decodeBeast2,
   decodeBeast2For,
   isTypeValueEqual,
+  lessFor,
   readBeast2Type,
   toEastTypeValue,
+  variant,
+  type EastTypeValue,
 } from '@elaraai/east';
 import { E3_RELEASE, compareReleases } from './release.js';
 
@@ -157,12 +161,13 @@ export type RequeueReason = ValueTypeOf<typeof RequeueReasonType>;
 /**
  * Execution events (VariantType for discriminated union).
  *
- * Events track the progress of a dataflow execution and are stored
- * inline in the execution state (not as a separate JSONL file).
+ * Events track the progress of a dataflow execution. A store keeps a run's
+ * events apart from its state, each numbered by its `seq`, and a poll reads
+ * those past its cursor.
  *
  * @remarks
- * Part of the execution state's wire: a reader decodes a state against the
- * whole type it was written with, so a new event is a new form of the state,
+ * A stored form, as the state is: a reader decodes events against the whole
+ * type they were written with, so a new event is a new form of a run's events,
  * which the release that makes it carries repositories into with an upgrade
  * step. Each unit's progress is a callback, {@link PartitionProgress}, and is
  * not persisted.
@@ -393,6 +398,59 @@ export interface PartitionProgress {
   duration?: number;
 }
 
+// =============================================================================
+// Forced Tasks
+// =============================================================================
+
+/**
+ * The tasks a dataflow run forces: re-executes even where the execution cache
+ * holds their results.
+ *
+ * - `none`: no task; each is served from the cache when it can be
+ * - `all`: every task the run launches — under a filter, the filter's task
+ * - `tasks`: the tasks named, and no other
+ *
+ * @remarks
+ * A forced task is forced each time the run launches it, so one invalidated
+ * mid-run and launched again is forced again. Its dependents are not forced:
+ * they re-run when its output changes. A run that names a task the graph does
+ * not have, or one its filter's run set leaves out, is refused before anything
+ * runs.
+ */
+export const DataflowForceType = VariantType({
+  none: NullType,
+  all: NullType,
+  tasks: ArrayType(StringType),
+});
+export type DataflowForce = ValueTypeOf<typeof DataflowForceType>;
+
+/**
+ * The tasks a run forces, from the option a caller passes it.
+ *
+ * @param force - `true` for every task, the names of the tasks, or `false` or
+ *   nothing for none
+ * @returns What the run forces
+ */
+export function dataflowForce(force: boolean | readonly string[] | undefined): DataflowForce {
+  if (force === undefined || force === false) return variant('none', null);
+  if (force === true) return variant('all', null);
+  return variant('tasks', [...force]);
+}
+
+/**
+ * The option a run's forced tasks are passed to an orchestrator as.
+ *
+ * @param force - What the run forces
+ * @returns `true` for every task, the names of the tasks, or `false` for none
+ */
+export function dataflowForceOption(force: DataflowForce): boolean | string[] {
+  switch (force.type) {
+    case 'none': return false;
+    case 'all': return true;
+    case 'tasks': return [...force.value];
+  }
+}
+
 // =============================================================================
 // Main Execution State
 // =============================================================================
@@ -404,7 +462,9 @@ export interface PartitionProgress {
  *
  * @remarks
  * - Tasks are stored as a Dict (serializes as object, not array of tuples)
- * - Events are stored inline (not as separate JSONL file)
+ * - A run's events are kept apart from it: `events` holds those the run has
+ *   added since its state was last written, which the write hands its store,
+ *   so a state a store holds, or reads back, holds none
  * - Dates are Date objects (via DateTimeType)
  * - Read it back with {@link decodeDataflowExecutionState}, which reads this
  *   form alone. A release that changes it ships a repository upgrade step,
@@ -426,9 +486,10 @@ export const DataflowExecutionStateType = StructType({
   startedAt: DateTimeType,
 
   // Config (immutable after initialization)
-  /** Force re-execution even if cached */
-  force: BooleanType,
-  /** Filter to run only specific task(s) by exact name */
+  /** The tasks the run re-executes even where the cache holds their results
+   *  ({@link DataflowForceType}) */
+  force: DataflowForceType,
+  /** One task's exact name: the run runs it and its dependency closure */
   filter: OptionType(StringType),
 
   // Graph (inline or by reference)
@@ -469,10 +530,13 @@ export const DataflowExecutionStateType = StructType({
   /** Number of tasks re-executed due to input changes */
   reexecuted: IntegerType,
 
-  // Events (inline array)
-  /** All events for this execution */
+  // Events, kept apart
+  /** The events the run has added since its state was last written, in
+   *  order: a write of the state hands them to its store, which keeps a run's
+   *  events apart from its state, so a stored state holds none */
   events: ArrayType(ExecutionEventType),
-  /** Sequence number for next event (auto-increment) */
+  /** The sequence number of the run's last event, which the next event's
+   *  follows: 0 while it has none */
   eventSeq: IntegerType,
 });
 export type DataflowExecutionState = ValueTypeOf<typeof DataflowExecutionStateType>;
@@ -493,10 +557,16 @@ const decodeState = decodeBeast2For(DataflowExecutionStateType);
  * @param data - the stored state
  * @returns the state
  * @throws {Error} When the state is of another form, naming the release that
- *   wrote it, or the data is not an execution state.
+ *   wrote it, or the data is not an execution state — of another type, or not
+ *   beast2 at all.
  */
 export function decodeDataflowExecutionState(data: Uint8Array): DataflowExecutionState {
-  const type = readBeast2Type(data);
+  let type: EastTypeValue;
+  try {
+    type = readBeast2Type(data);
+  } catch (err) {
+    throw new Error(`the data is not an execution state: it is not beast2 (${err instanceof Error ? err.message : `${err as string}`})`, { cause: err });
+  }
   if (isTypeValueEqual(type, STATE_TYPE)) return decodeState(data);
   const { value } = decodeBeast2(data);
   const state = typeof value === 'object' && value !== null ? value as Record<string, unknown> : null;
@@ -515,6 +585,82 @@ export function decodeDataflowExecutionState(data: Uint8Array): DataflowExecutio
   throw new Error('the data is not an execution state: its type is not the execution state\'s');
 }
 
+/**
+ * A dataflow run's state but its graph, its tasks, its reactive tracking and
+ * its events: what a poll of the run serves but the events, and the sequence
+ * number of its last event, past which a poll reads the events it has not
+ * served.
+ *
+ * @remarks
+ * An execution state store answers it for a workspace's latest run
+ * (`ExecutionStateStore.readLatestSummary`), so a poll reads it rather than
+ * the whole state, which grows with the dataflow and with every event of the
+ * run. A store may keep it beside the state, written with each change of the
+ * state; one that reads the whole state anyway derives it
+ * ({@link executionStateSummary}).
+ */
+export const ExecutionStateSummaryType = StructType({
+  /** The run's id, a UUIDv7 */
+  id: StringType,
+  /** When the run started */
+  startedAt: DateTimeType,
+  /** Number of tasks executed (not from cache) */
+  executed: IntegerType,
+  /** Number of tasks served from cache */
+  cached: IntegerType,
+  /** Number of tasks that failed */
+  failed: IntegerType,
+  /** Number of tasks skipped due to upstream failure */
+  skipped: IntegerType,
+  /** The run's status ({@link DataflowExecutionStatus}) */
+  status: StringType,
+  /** When the run ended */
+  completedAt: OptionType(DateTimeType),
+  /** Why the run failed, when it did */
+  error: OptionType(StringType),
+  /** The sequence number of the run's last event: 0 while it has none */
+  lastSeq: IntegerType,
+});
+export type ExecutionStateSummary = ValueTypeOf<typeof ExecutionStateSummaryType>;
+
+/** Whether one event sequence number comes before another. */
+const seqBefore = lessFor(IntegerType);
+
+/**
+ * The sequence number of a run's last event, as its state counts it: the
+ * state's own count, or the last event it holds, which a write has not yet
+ * handed its store, when that is later.
+ *
+ * @param state - The run's state
+ * @returns The sequence number: 0 while the run has no event
+ */
+export function lastEventSeq(state: Pick<DataflowExecutionState, 'events' | 'eventSeq'>): bigint {
+  const held = state.events.at(-1)?.value.seq;
+  return held !== undefined && seqBefore(state.eventSeq, held) ? held : state.eventSeq;
+}
+
+/**
+ * The summary of a run's state ({@link ExecutionStateSummaryType}).
+ *
+ * @param state - The run's state
+ * @returns Its summary: its last event's sequence number is the last event's
+ *   own ({@link lastEventSeq}), however the event was recorded
+ */
+export function executionStateSummary(state: DataflowExecutionState): ExecutionStateSummary {
+  return {
+    id: state.id,
+    startedAt: state.startedAt,
+    executed: state.executed,
+    cached: state.cached,
+    failed: state.failed,
+    skipped: state.skipped,
+    status: state.status,
+    completedAt: state.completedAt,
+    error: state.error,
+    lastSeq: lastEventSeq(state),
+  };
+}
+
 // =============================================================================
 // Dataflow Run History
 // =============================================================================

package/src/dataset-blob.spec.ts

@@ -29,6 +29,54 @@ import { encodeDatasetBlob, writeCollectionManifest, type CollectionSegmentRef }
 
 const TableType = DictType(StringType, IntegerType);
 
+/** `n` rows, in canonical order: 20,000 span sixteen segments. */
+function rowsOf(n: number): SortedMap<string, bigint> {
+  return new SortedMap(
+    Array.from({ length: n }, (_, i): [string, bigint] => [`k${String(i).padStart(6, '0')}`, BigInt(i)]),
+    compareFor(StringType));
+}
+
+/** A write the test ends, as the store would: stored, or failed. */
+interface HeldWrite {
+  hash: string;
+  end: (failure?: Error) => void;
+}
+
+/** A sink whose writes are held until the test ends each one: the writes in
+ *  flight, and the hash of every object it was handed, in order. */
+function heldSink(): { sink: (bytes: Uint8Array) => Promise<string>; open: HeldWrite[]; calls: string[] } {
+  const open: HeldWrite[] = [];
+  const calls: string[] = [];
+  const sink = (bytes: Uint8Array): Promise<string> => new Promise((resolve, reject) => {
+    const hash = sha256Hex(bytes);
+    calls.push(hash);
+    const write: HeldWrite = {
+      hash,
+      end: (failure) => {
+        open.splice(open.indexOf(write), 1);
+        if (failure === undefined) resolve(hash);
+        else reject(failure);
+      },
+    };
+    open.push(write);
+  });
+  return { sink, open, calls };
+}
+
+/** Lets `turns` rounds of the event loop pass. */
+async function turns(turns: number): Promise<void> {
+  for (let i = 0; i < turns; i++) await new Promise((resolve) => setImmediate(resolve));
+}
+
+/** Lets the event loop run until `done` holds. */
+async function until(done: () => boolean, what: string): Promise<void> {
+  for (let i = 0; i < 10_000; i++) {
+    if (done()) return;
+    await turns(1);
+  }
+  assert.fail(`never: ${what}`);
+}
+
 describe('writeCollectionManifest', () => {
   it("writes the manifest writer's manifest for the value, byte for byte", async () => {
     const value = new SortedMap(
@@ -96,6 +144,105 @@ describe('writeCollectionManifest', () => {
     assert.deepEqual(reads, [], 'the seam is decided by the fence that followed');
     for (const entry of entries.slice(0, half)) assert.ok(!written.has(entry.hash), `segment ${entry.hash.slice(0, 8)} is carried`);
   });
+
+  it('has the sink write the objects at once, as many as it is given, in order, and returns once every one is stored', async () => {
+    const value = rowsOf(20_000);
+    const expected = await writeCollectionManifest(TableType, [{ elements: value.entries() }], async (bytes) => sha256Hex(bytes));
+    const entries = decodeCollectionManifest(expected).entries;
+    const objects = entries.length + 1;
+    const { sink, open, calls } = heldSink();
+    let manifest: Uint8Array | undefined;
+    const writing = writeCollectionManifest(TableType, [{ elements: value.entries() }], sink, { inFlight: 3 })
+      .then((bytes) => {
+        manifest = bytes;
+      });
+
+    // Each write ends only when the test ends it, the last begun first, so the
+    // writes end in another order than they began. Between ends, the writer
+    // has as many in flight as it may: three, or the objects left.
+    while (manifest === undefined) {
+      const inFlight = (): number => Math.min(3, objects - (calls.length - open.length));
+      await until(() => manifest !== undefined || open.length === inFlight(), 'as many writes in flight as may be');
+      await turns(20);
+      if (manifest !== undefined) break;
+      assert.equal(open.length, inFlight(), `${open.length} writes in flight, of ${calls.length} begun`);
+      open.at(-1)!.end();
+    }
+    await writing;
+
+    assert.deepEqual(manifest, expected, 'the manifest a write at a time writes, each entry in its place');
+    assert.deepEqual(calls, [...entries.map((entry) => entry.hash), decodeCollectionManifest(expected).header],
+      'the segments, in order, and then the header');
+  });
+
+  /** A writer's failure, once it has failed: read through a function, which
+   *  no assertion narrows. */
+  function failureOf(writing: Promise<unknown>): () => { failure: unknown } | undefined {
+    let settled: { failure: unknown } | undefined;
+    writing.then(() => {
+      settled = { failure: new Error('the writer did not fail') };
+    }, (failure: unknown) => {
+      settled = { failure };
+    });
+    return () => settled;
+  }
+
+  it('starts no write after one fails, and throws its failure once the writes in flight have ended', async () => {
+    const { sink, open, calls } = heldSink();
+    const writing = writeCollectionManifest(TableType, [{ elements: rowsOf(20_000).entries() }], sink, { inFlight: 3 });
+    const settled = failureOf(writing);
+    await until(() => open.length === 3, 'three writes in flight');
+    const failure = new Error('the store is gone');
+    open[1]!.end(failure);
+    await turns(20);
+    assert.equal(calls.length, 3, 'nothing is begun after a write fails');
+    assert.equal(settled(), undefined, 'the writer waits for the writes still in flight');
+
+    for (const write of [...open]) write.end();
+    await writing.catch(() => {});
+    assert.equal(settled()?.failure, failure);
+  });
+
+  it('throws a failure among the last writes, which no write after it meets', async () => {
+    const value = rowsOf(20_000);
+    const { header } = decodeCollectionManifest(
+      await writeCollectionManifest(TableType, [{ elements: value.entries() }], async (bytes) => sha256Hex(bytes)));
+    const { sink, open } = heldSink();
+    const writing = writeCollectionManifest(TableType, [{ elements: value.entries() }], sink, { inFlight: 3 });
+    const settled = failureOf(writing);
+    // Every object is stored but the header, the last begun.
+    const failure = new Error('the store is gone');
+    for (;;) {
+      await until(() => settled() !== undefined || open.length > 0, 'a write in flight, or the writer\'s end');
+      if (settled() !== undefined) break;
+      const write = open[0]!;
+      write.end(write.hash === header ? failure : undefined);
+    }
+    assert.equal(settled()?.failure, failure);
+  });
+
+  it('throws a piece\'s failure once the writes in flight have ended', async () => {
+    const { sink, open } = heldSink();
+    // Rows in order, then one out of it.
+    let disordered = false;
+    function* rows(): Generator<[string, bigint]> {
+      yield* rowsOf(20_000).entries();
+      disordered = true;
+      yield ['a', 0n];
+    }
+    const writing = writeCollectionManifest(TableType, [{ elements: rows() }], sink, { inFlight: 64 });
+    const settled = failureOf(writing);
+    await until(() => disordered, 'the row out of order');
+    await turns(20);
+    assert.ok(open.length > 0, 'segments are in flight');
+    assert.equal(settled(), undefined, 'the writer waits for the writes still in flight');
+
+    for (const write of [...open]) write.end();
+    await writing.catch(() => {});
+    const failure = settled()?.failure;
+    assert.ok(failure instanceof Error, 'the piece\'s failure');
+    assert.match(failure.message, /ascend|order/);
+  });
 });
 
 describe('encodeDatasetBlob', () => {

package/src/dataset-blob.ts

@@ -59,6 +59,63 @@ import {
  */
 export type SegmentSink = (bytes: Uint8Array) => Promise<string>;
 
+/**
+ * How many objects {@link writeCollectionManifest} has a sink write at once,
+ * unless its caller says otherwise: a store whose writes are round trips is
+ * waited on once for this many segments, rather than once for each.
+ */
+export const SEGMENT_WRITES_IN_FLIGHT = 16;
+
+/**
+ * The writes of one manifest's objects: each started in the order it is
+ * handed over, at most `width` in flight, and none left running once the
+ * manifest's writer returns.
+ *
+ * @internal
+ */
+class SinkWrites {
+  private readonly inFlight = new Set<Promise<void>>();
+  private failure: { error: unknown } | undefined;
+
+  constructor(private readonly sink: SegmentSink, private readonly width: number) {}
+
+  /**
+   * Starts writing `bytes` once fewer than `width` writes are in flight.
+   *
+   * @param bytes - The object
+   * @param named - Told the object's hash once it is stored
+   * @throws The first failed write's error, once one has failed: nothing more
+   *   is started after it.
+   */
+  async start(bytes: Uint8Array, named: (hash: string) => void): Promise<void> {
+    while (this.failure === undefined && this.inFlight.size >= this.width) await Promise.race(this.inFlight);
+    if (this.failure !== undefined) throw this.failure.error;
+    const write: Promise<void> = this.sink(bytes)
+      .then(named, (error: unknown) => {
+        this.failure ??= { error };
+      })
+      .finally(() => {
+        this.inFlight.delete(write);
+      });
+    this.inFlight.add(write);
+  }
+
+  /** Waits for every write started, whatever its outcome. */
+  async settled(): Promise<void> {
+    await Promise.all(this.inFlight);
+  }
+
+  /**
+   * Waits for every write started.
+   *
+   * @throws The first failed write's error.
+   */
+  async stored(): Promise<void> {
+    await this.settled();
+    if (this.failure !== undefined) throw this.failure.error;
+  }
+}
+
 /**
  * Whether a dataset root type is a collection — the kinds stored segmented +
  * indexed so the paged read API can seek.
@@ -106,49 +163,69 @@ export type CollectionPiece = Beast2RecutPiece<EastType, CollectionSegmentRef>;
  * Writer's own: a piece's segments are carried over only when that is the
  * header they are under, which is the caller's to establish.
  *
+ * The objects are written at once, `inFlight` at a time — the segments in
+ * order, then the header — and the manifest is returned once every one is
+ * stored, so the caller stores the manifest after the objects it names. The
+ * sink is called in segment order, and each segment's blob is held until its
+ * write ends.
+ *
  * @param type - The collection type (Array / Set / Dict)
  * @param pieces - The collection's pieces, in order
  * @param sink - Writes one object and returns its hash
+ * @param options - `inFlight`, the most objects the sink writes at once
+ *   (default {@link SEGMENT_WRITES_IN_FLIGHT})
  * @returns The manifest's bytes — the caller stores them, and its hash is the
  *   dataset's content address
  * @throws {Error} When a piece's elements do not ascend (Set / Dict), a
  *   segment's blob is not the one segment its reference describes, or the sink
- *   rejects.
+ *   rejects: once the writes in flight have ended, so none runs on after the
+ *   writer returns.
  */
 export async function writeCollectionManifest(
   type: EastType | EastTypeValue,
   pieces: Iterable<CollectionPiece> | AsyncIterable<CollectionPiece>,
   sink: SegmentSink,
+  options: { inFlight?: number } = {},
 ): Promise<Uint8Array> {
   const typeValue = asTypeValue(type);
   const entries: CollectionManifestEntry[] = [];
+  const writes = new SinkWrites(sink, options.inFlight ?? SEGMENT_WRITES_IN_FLIGHT);
+  /** Writes a segment the manifest names at the place it takes now. */
+  const write = (blob: Uint8Array, fence: Uint8Array, count: number): Promise<void> => {
+    const at = entries.push({ hash: '', fence, count: BigInt(count), bytes: BigInt(blob.byteLength) }) - 1;
+    return writes.start(blob, (hash) => {
+      entries[at] = { ...entries[at]!, hash };
+    });
+  };
   // Frames deflate on the worker pool once the value is large enough to be
   // worth it; the pool holds the frames in flight and nothing more.
   const recut = recutBeast2For<EastType, CollectionSegmentRef>(typeValue, { parallel: true });
-  const stats = await recut(pieces, {
-    written: async (segment) => {
-      entries.push({
-        hash: await sink(segment.blob),
-        fence: segment.fence,
-        count: BigInt(segment.count),
-        bytes: BigInt(segment.blob.byteLength),
-      });
-    },
-    carried: async (ref) => {
-      if (ref.entry !== undefined) {
-        entries.push(ref.entry);
-        return;
-      }
-      const blob = await ref.read();
-      entries.push({ hash: await sink(blob), fence: ref.fence, count: BigInt(ref.count), bytes: BigInt(blob.byteLength) });
-    },
-  });
+  let header = '';
+  try {
+    const stats = await recut(pieces, {
+      written: (segment) => write(segment.blob, segment.fence, segment.count),
+      carried: async (ref) => {
+        if (ref.entry !== undefined) {
+          entries.push(ref.entry);
+          return;
+        }
+        await write(await ref.read(), ref.fence, ref.count);
+      },
+    });
+    await writes.start(stats.header, (hash) => {
+      header = hash;
+    });
+  } catch (error) {
+    await writes.settled();
+    throw error;
+  }
+  await writes.stored();
   return encodeCollectionManifest({
     kind: COLLECTION_MANIFEST_KIND,
     level: 0n,
     type: typeValue,
     rule: segmentRuleFor(typeValue),
-    header: await sink(stats.header),
+    header,
     entries,
   });
 }