@ultimat3/jobs 29.3.2 → 30.0.0

package/src/import-errors.ts

@@ -0,0 +1,102 @@
+// The three `X_IMPORT_*` codes an `importRows()` pass can end on. The codes, titles and retry
+// classes live in `error-codes.ts`; this file holds only the classes, for the reason
+// `export-errors.ts` and `backfill-errors.ts` are split out.
+//
+// All three are TERMINAL: the same bytes answer the same way on every attempt.
+
+import { UltimateError } from '@ultimat3/core';
+
+const listed = (names: readonly string[]): string =>
+  names.length === 0 ? 'none' : names.map((name) => JSON.stringify(name)).join(', ');
+
+/** At most this many undeclared names are shown, each cut to this many characters. */
+export const IMPORT_HEADER_SHOWN = 5;
+export const IMPORT_HEADER_NAME_CHARS = 40;
+
+/**
+ * The ONE rule for a name that is FILE text — a header cell, an undeclared ndjson key — wherever
+ * it is reported: cut to `IMPORT_HEADER_NAME_CHARS`, so a 64 KiB cell cannot become a prompt for
+ * the agent reading `x jobs show`. Every reader JSON-escapes it (`shown`, the error report's
+ * `JSON.stringify`), so a newline in it cannot forge a line either.
+ */
+export const fileText = (name: string): string =>
+  name.length > IMPORT_HEADER_NAME_CHARS ? `${name.slice(0, IMPORT_HEADER_NAME_CHARS)}…` : name;
+
+const shown = (name: string): string => JSON.stringify(fileText(name));
+
+/**
+ * A csv header that does not name exactly the declared columns. Both directions, as
+ * `encodeExportPage` refuses both: a missing column would arrive as nothing in every row, and an
+ * undeclared one is a column the app never agreed to read.
+ *
+ * `missing` is the DECLARATION's names and is printed whole. `undeclared` is the FILE's text, so
+ * it is a count plus a few escaped, truncated names, and the `fix:` carries none of it.
+ */
+export class ImportHeaderMismatchError extends UltimateError {
+  constructor(input: {
+    import: string;
+    /** The declared columns: what the file's first line must name. */
+    columns: readonly string[];
+    missing: readonly string[];
+    undeclared: readonly string[];
+  }) {
+    const names = input.undeclared.slice(0, IMPORT_HEADER_SHOWN).map(shown);
+    const more = input.undeclared.length - names.length;
+    const undeclared =
+      names.length === 0 ? 'none' : `${names.join(', ')}${more > 0 ? ` and ${more} more` : ''}`;
+    super({
+      code: 'X_IMPORT_HEADER_MISMATCH',
+      cause: `import "${input.import}" read a csv header that is missing ${listed(input.missing)} and carries ${input.undeclared.length} undeclared column(s): ${undeclared}`,
+      fix: `columns: [${input.columns.map((name) => JSON.stringify(name)).join(', ')}] on importRows("${input.import}") is what the file's first line must name — rewrite the header to it, or declare the file's columns there, then enqueue it again with a new key`,
+      meta: {
+        missing: [...input.missing],
+        undeclaredCount: input.undeclared.length,
+        undeclared: names,
+      },
+    });
+  }
+}
+
+/**
+ * A record with no end inside `maxRecordBytes`. Refused rather than read on: an unclosed quote
+ * makes every newline after it part of one cell, and buffering until it closes is how one byte
+ * turns a bounded import into the whole file in memory.
+ */
+export class ImportRecordTooLargeError extends UltimateError {
+  constructor(input: { import: string; offset: number; maxBytes: number }) {
+    super({
+      code: 'X_IMPORT_RECORD_TOO_LARGE',
+      cause: `import "${input.import}" found no record end within ${input.maxBytes} bytes of byte offset ${input.offset} — usually a quote opened and never closed`,
+      fix: `importRows({ name: "${input.import}", maxRecordBytes: ${input.maxBytes * 2}, … }) if one record really is that large (keep pageBytes above it) — otherwise repair the quote at byte offset ${input.offset} of the file`,
+      meta: { offset: input.offset, maxBytes: input.maxBytes },
+    });
+  }
+}
+
+/**
+ * More rows refused than `maxRowErrors` allows. The `batch` that crossed the ceiling is NOT handed
+ * to `handle`; every refusal so far is already written to the error parts this names.
+ */
+export class ImportTooManyErrorsError extends UltimateError {
+  constructor(input: {
+    import: string;
+    rejected: number;
+    maxRowErrors: number;
+    rows: number;
+    errorPrefix: string;
+    parts: number;
+  }) {
+    super({
+      code: 'X_IMPORT_TOO_MANY_ERRORS',
+      cause: `import "${input.import}" refused ${input.rejected} row(s) and allows ${input.maxRowErrors}; ${input.rows} row(s) were handled before it stopped`,
+      fix: `x jobs show <id> --json, then read the ${input.parts} errors-*.ndjson part(s) under "${input.errorPrefix}", repair those lines in the file and enqueue importRows("${input.import}") again with a new key`,
+      meta: {
+        rejected: input.rejected,
+        maxRowErrors: input.maxRowErrors,
+        rows: input.rows,
+        errorPrefix: input.errorPrefix,
+        parts: input.parts,
+      },
+    });
+  }
+}

package/src/import-format.ts

@@ -0,0 +1,232 @@
+// One window of an import's bytes -> its complete records and where the next window starts. Pure
+// and synchronous, the twin of `export-format.ts`, so every hazard is testable without a queue.
+//
+// Records are found on BYTES, never on decoded text: a `\n` (0x0A) and a `"` (0x22) never occur
+// inside a UTF-8 multi-byte sequence, so a boundary cut there cannot split a character, and the
+// bytes after the last complete record are simply read again by the next window. No carry.
+
+import type { ExportFormat } from './export-format';
+import { fileText, ImportHeaderMismatchError, ImportRecordTooLargeError } from './import-errors';
+
+const NEWLINE = 0x0a;
+const RETURN = 0x0d;
+const QUOTE = 0x22;
+const COMMA = 0x2c;
+const BOM = [0xef, 0xbb, 0xbf] as const;
+
+/** Why a record was refused before its schema saw it. A rule name, never the record's text. */
+export const IMPORT_REJECTIONS = [
+  'utf8.invalid',
+  'csv.quote_unclosed',
+  'csv.quote_stray',
+  'csv.field_count',
+  'json.syntax',
+  'json.not_object',
+  'record.undeclared',
+] as const;
+export type ImportRejection = (typeof IMPORT_REJECTIONS)[number];
+
+/** One complete record: the physical line it starts on (1-based) and its text, or the refusal. */
+export interface RawRecord {
+  readonly line: number;
+  /** `undefined` when the bytes are not UTF-8: refused per row, never a whole-file failure. */
+  readonly text: string | undefined;
+}
+
+export interface SplitInput {
+  readonly subject: string;
+  readonly format: ExportFormat;
+  /** The window, starting at `offset`. */
+  readonly bytes: Uint8Array;
+  readonly offset: number;
+  /** The physical line `bytes[0]` sits on, 1-based. */
+  readonly line: number;
+  /** True when the window reaches the end of the object: an unterminated tail is then a record. */
+  readonly final: boolean;
+  readonly maxRecordBytes: number;
+  /** Stop after this many records, so a page of tiny rows is still a bounded statement. */
+  readonly maxRecords: number;
+}
+
+export interface SplitWindow {
+  readonly records: readonly RawRecord[];
+  /** Absolute byte offset of the first byte not consumed — the next window's `offset`. */
+  readonly next: number;
+  readonly nextLine: number;
+}
+
+const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+
+const decode = (bytes: Uint8Array): string | undefined => {
+  try {
+    return utf8.decode(bytes);
+  } catch {
+    return undefined;
+  }
+};
+
+const hasBom = (bytes: Uint8Array): boolean => BOM.every((byte, at) => bytes[at] === byte);
+
+/**
+ * The complete records in one window. Quote-aware for csv (a newline inside quotes is data);
+ * CRLF and LF alike; a UTF-8 BOM at offset 0 is consumed and never part of the first cell; a blank
+ * line is skipped. A record longer than `maxRecordBytes` is refused, and so is a full window with
+ * no record end in it, because the next read would have to grow.
+ */
+export function splitRecords(input: SplitInput): SplitWindow {
+  const { bytes } = input;
+  const records: RawRecord[] = [];
+  let start = input.offset === 0 && hasBom(bytes) ? BOM.length : 0;
+  let line = input.line;
+  let startLine = line;
+  let quoted = false;
+  // RFC 4180 quotes a FIELD: a `"` opens one only at a field's first byte, and inside one `""` is
+  // an escaped quote (close, then re-open at once). Anywhere else it is a literal `parseCsvRecord`
+  // refuses for the row alone — flipping on every quote swallowed every later row into one record.
+  let fieldStart = true;
+  let justClosed = false;
+
+  const take = (end: number): void => {
+    if (end - start > input.maxRecordBytes) {
+      throw new ImportRecordTooLargeError({
+        import: input.subject,
+        offset: input.offset + start,
+        maxBytes: input.maxRecordBytes,
+      });
+    }
+    const last = end > start && bytes[end - 1] === RETURN ? end - 1 : end;
+    if (last > start) records.push({ line: startLine, text: decode(bytes.subarray(start, last)) });
+  };
+
+  for (let at = start; at < bytes.length && records.length < input.maxRecords; at += 1) {
+    const byte = bytes[at];
+    if (input.format === 'csv') {
+      const reopens = byte === QUOTE && !quoted && (fieldStart || justClosed);
+      justClosed = byte === QUOTE && quoted;
+      if (justClosed) quoted = false;
+      else if (reopens) quoted = true;
+      fieldStart = !quoted && byte === COMMA;
+    }
+    if (byte !== NEWLINE) continue;
+    line += 1;
+    if (quoted) continue;
+    take(at);
+    start = at + 1;
+    startLine = line;
+    fieldStart = true;
+  }
+  if (input.final && records.length < input.maxRecords && start < bytes.length) {
+    take(bytes.length);
+    start = bytes.length;
+    startLine = line;
+  }
+  if (records.length === 0 && !input.final && bytes.length - start > input.maxRecordBytes) {
+    // No record end in more than `maxRecordBytes`: reading a longer window is the buffering this
+    // refuses. The pass holds every window LONGER than `maxRecordBytes`, so a window that took no
+    // record and consumed no blank line always lands here — the loop cannot stand still.
+    throw new ImportRecordTooLargeError({
+      import: input.subject,
+      offset: input.offset + start,
+      maxBytes: input.maxRecordBytes,
+    });
+  }
+  return { records, next: input.offset + start, nextLine: startLine };
+}
+
+/** One csv field list, RFC 4180: an unquoted empty cell is `null`, a quoted `""` is `''`. */
+export function parseCsvRecord(
+  text: string,
+): { readonly cells: readonly (string | null)[] } | { readonly rejected: ImportRejection } {
+  const cells: (string | null)[] = [];
+  let at = 0;
+  for (;;) {
+    if (text[at] === '"') {
+      let value = '';
+      at += 1;
+      for (;;) {
+        const close = text.indexOf('"', at);
+        if (close < 0) return { rejected: 'csv.quote_unclosed' };
+        value += text.slice(at, close);
+        if (text[close + 1] === '"') {
+          value += '"';
+          at = close + 2;
+          continue;
+        }
+        at = close + 1;
+        break;
+      }
+      cells.push(value);
+      if (at === text.length) return { cells };
+      if (text[at] !== ',') return { rejected: 'csv.quote_stray' };
+      at += 1;
+      continue;
+    }
+    const comma = text.indexOf(',', at);
+    const value = comma < 0 ? text.slice(at) : text.slice(at, comma);
+    if (value.includes('"')) return { rejected: 'csv.quote_stray' };
+    cells.push(value === '' ? null : value);
+    if (comma < 0) return { cells };
+    at = comma + 1;
+  }
+}
+
+/**
+ * Page 0's first record against the declared columns, both directions, as `encodeExportPage`
+ * refuses both. Answers the file's column ORDER, which the rows are read by: a spreadsheet that
+ * moved a column is still the same file, and a renamed one is the refusal.
+ */
+export function readCsvHeader(
+  subject: string,
+  columns: readonly string[],
+  record: RawRecord,
+): readonly string[] {
+  const parsed = record.text === undefined ? undefined : parseCsvRecord(record.text);
+  const header = parsed !== undefined && 'cells' in parsed ? parsed.cells.map((c) => c ?? '') : [];
+  const seen = new Set<string>();
+  const undeclared = header.filter((name) => {
+    const repeated = seen.has(name);
+    seen.add(name);
+    return repeated || !columns.includes(name);
+  });
+  const missing = columns.filter((name) => !seen.has(name));
+  if (missing.length > 0 || undeclared.length > 0 || header.length === 0) {
+    throw new ImportHeaderMismatchError({ import: subject, columns, missing, undeclared });
+  }
+  return header;
+}
+
+export type ImportRead =
+  | { readonly record: Readonly<Record<string, unknown>> }
+  | { readonly rejected: ImportRejection; readonly field: string | null };
+
+/** One record's text -> a keyed object for the schema, or the format's refusal of it. */
+export function readImportRecord(
+  format: ExportFormat,
+  columns: readonly string[],
+  text: string | undefined,
+): ImportRead {
+  if (text === undefined) return { rejected: 'utf8.invalid', field: null };
+  if (format === 'csv') {
+    const parsed = parseCsvRecord(text);
+    if ('rejected' in parsed) return { rejected: parsed.rejected, field: null };
+    if (parsed.cells.length !== columns.length) return { rejected: 'csv.field_count', field: null };
+    const record: Record<string, unknown> = {};
+    columns.forEach((column, at) => {
+      record[column] = parsed.cells[at] ?? null;
+    });
+    return { record };
+  }
+  let value: unknown;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return { rejected: 'json.syntax', field: null };
+  }
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return { rejected: 'json.not_object', field: null };
+  }
+  const object = value as Record<string, unknown>;
+  const extra = Object.keys(object).find((key) => !columns.includes(key));
+  if (extra !== undefined) return { rejected: 'record.undeclared', field: fileText(extra) };
+  return { record: object };
+}

package/src/import-pass.ts

@@ -0,0 +1,255 @@
+// One `importRows()` pass: the page loop, its byte-offset checkpoints and the error report. Split
+// out of `import.ts` as `export-pass.ts` is out of `export.ts` — that file declares, this runs.
+//
+// A PAGE IS A STEP. Its output — the next byte offset, the line, two counters, the error part and
+// the csv header order — is the checkpoint, so a killed attempt resumes at the byte it stopped on.
+// Never the rows: `steps.ts` hands every completed step's output back for the whole run.
+
+import { assert, logger } from '@ultimat3/core';
+import { coerceInput, formatPath, validateAsync } from '@ultimat3/schema';
+import { JobAbortedError } from './errors';
+import { EXPORT_PART_DIGITS } from './export-sink';
+import type { ImportPlan, ImportReport } from './import';
+import { ImportHeaderMismatchError, ImportTooManyErrorsError } from './import-errors';
+import type { RawRecord } from './import-format';
+import { readCsvHeader, readImportRecord, splitRecords } from './import-format';
+import type { JobRunArgs } from './job';
+
+const STEP_PREFIX = 'page:';
+const STAT_STEP = 'stat';
+
+/** `<prefix>/errors-00000.ndjson`, keyed by PAGE index, so a replayed page overwrites its part. */
+export const importErrorKey = (prefix: string, page: number): string =>
+  `${prefix}/errors-${String(page).padStart(EXPORT_PART_DIGITS, '0')}.ndjson`;
+
+/** One refused row. The line, the field and the rule — NEVER the value the app refused. */
+export interface ImportRowError {
+  readonly line: number;
+  readonly field: string | null;
+  readonly code: string;
+}
+
+interface Checkpoint {
+  readonly offset: number;
+  readonly line: number;
+  readonly rows: number;
+  readonly rejected: number;
+  /** The page index whose error part this page wrote, or `null` for a clean page. */
+  readonly errorPart: number | null;
+  /** The csv header's column order once page 0 read it; carried, never re-read. */
+  readonly header: readonly string[] | null;
+}
+
+const fieldOf = (value: unknown, key: string): unknown =>
+  typeof value === 'object' && value !== null ? (value as Record<string, unknown>)[key] : undefined;
+
+/** `steps.ts` replays through an unchecked `as T`; an offset read wrong re-imports from zero. */
+function asCheckpoint(value: unknown, step: string): Checkpoint {
+  const offset = fieldOf(value, 'offset');
+  const line = fieldOf(value, 'line');
+  const rows = fieldOf(value, 'rows');
+  const rejected = fieldOf(value, 'rejected');
+  const errorPart = fieldOf(value, 'errorPart');
+  const header = fieldOf(value, 'header');
+  assert(
+    typeof offset === 'number' &&
+      typeof line === 'number' &&
+      typeof rows === 'number' &&
+      typeof rejected === 'number' &&
+      (errorPart === null || typeof errorPart === 'number') &&
+      (header === null || (Array.isArray(header) && header.every((h) => typeof h === 'string'))),
+    `step "${step}" replayed a value that is not an import checkpoint`,
+    `x jobs show <jobId> --json prints the run's steps — a run whose "${step}" was written by something other than this import has to be retired, not resumed`,
+  );
+  return { offset, line, rows, rejected, errorPart, header: header as readonly string[] | null };
+}
+
+interface Judged<Rec> {
+  readonly rows: Rec[];
+  readonly errors: ImportRowError[];
+  readonly rejected: number;
+}
+
+/** Every record of one page through the format and then the schema. */
+async function judge<Rec, I>(
+  plan: ImportPlan<Rec, I>,
+  columns: readonly string[],
+  records: readonly RawRecord[],
+): Promise<Judged<Rec>> {
+  const { format, schema } = plan.definition;
+  const rows: Rec[] = [];
+  const errors: ImportRowError[] = [];
+  let rejected = 0;
+  for (const raw of records) {
+    const read = readImportRecord(format, columns, raw.text);
+    if ('rejected' in read) {
+      rejected += 1;
+      errors.push({ line: raw.line, field: read.field, code: read.rejected });
+      continue;
+    }
+    // A csv cell is text, exactly the HTTP boundary's problem; ndjson carries real JSON values.
+    const value = format === 'csv' ? coerceInput(schema, read.record) : read.record;
+    const result = await validateAsync(schema, value);
+    if (result.issues === undefined) {
+      rows.push(result.value);
+      continue;
+    }
+    rejected += 1;
+    for (const issue of result.issues) {
+      const field = formatPath(issue.path);
+      errors.push({
+        line: raw.line,
+        field: field === '' ? null : field,
+        code: issue.rule ?? 'schema.invalid',
+      });
+    }
+  }
+  return { rows, errors, rejected };
+}
+
+export async function importPass<Rec, I>(
+  plan: ImportPlan<Rec, I>,
+  args: JobRunArgs<I>,
+): Promise<ImportReport> {
+  const { definition, batch, pageBytes, maxRecordBytes, maxRowErrors, pace } = plan;
+  const { step, input, ctx } = args;
+  const name = definition.name;
+  const key = definition.key(input);
+  const declaredPrefix = definition.errors.prefix(input);
+  assert(
+    declaredPrefix.length > 0 && !declaredPrefix.startsWith('/') && !declaredPrefix.includes('..'),
+    `import "${name}" errors.prefix() answered "${declaredPrefix}", which is not a key prefix`,
+    `return a relative, non-empty prefix from errors.prefix() on importRows("${name}") — "imports/<orgId>/<importId>" is the shape`,
+  );
+
+  // Under the RUN: parts are keyed by page index, so a second import of the same file under one
+  // prefix would leave the first run's later parts beside its own and read as one report.
+  const errorPrefix = `${declaredPrefix}/${args.runId}`;
+
+  // A step, so every attempt reads the file at the length the first one measured.
+  const size = await step.run(STAT_STEP, async (): Promise<number> => {
+    const object = await definition.source().stat(key);
+    assert(
+      object !== undefined,
+      `import "${name}" found no object at "${key}"`,
+      `upload the file to "${key}" before enqueueing importRows("${name}"), or fix key() on the declaration`,
+    );
+    return object.size;
+  });
+
+  let state: Checkpoint = {
+    offset: 0,
+    line: 1,
+    rows: 0,
+    rejected: 0,
+    errorPart: null,
+    header: null,
+  };
+  let rows = 0;
+  let rejected = 0;
+  let parts = 0;
+
+  for (let index = 0; state.offset < size; index += 1) {
+    const stepName = `${STEP_PREFIX}${index}`;
+    const at = state;
+    const body = async (signal: AbortSignal): Promise<Checkpoint> => {
+      await pace?.wait({ signal, step: stepName });
+      const last = Math.min(at.offset + pageBytes, size) - 1;
+      const { bytes } = await definition.source().get(key, { range: [at.offset, last] });
+      assert(
+        bytes.byteLength > 0,
+        `import "${name}" read no bytes at offset ${at.offset} of "${key}", which stat measured at ${size}`,
+        `x jobs show <jobId> --json, then re-upload "${key}" under a new key — the object changed under a running import`,
+      );
+      const final = at.offset + bytes.byteLength >= size;
+      // The WHOLE window is used, `batch` records at a time: `batch` bounds one `handle` call, never
+      // how much of a read is kept, or a file of tiny rows costs one full read per batch.
+      let cursor = at.offset;
+      let line = at.line;
+      let header = at.header;
+      let pageRows = 0;
+      let pageRejected = 0;
+      const errors: ImportRowError[] = [];
+      const report = async (): Promise<number | null> => {
+        if (errors.length === 0) return null;
+        const lines = errors.map((error) => `${JSON.stringify(error)}\n`).join('');
+        await definition.errors.sink().put(importErrorKey(errorPrefix, index), encode(lines));
+        return index;
+      };
+      while (cursor < at.offset + bytes.byteLength) {
+        const headerNeeded = definition.format === 'csv' && header === null;
+        const window = splitRecords({
+          subject: name,
+          format: definition.format,
+          bytes: bytes.subarray(cursor - at.offset),
+          offset: cursor,
+          line,
+          final,
+          maxRecordBytes,
+          maxRecords: batch + (headerNeeded ? 1 : 0),
+        });
+        if (window.next === cursor) break;
+        cursor = window.next;
+        line = window.nextLine;
+        let records = window.records;
+        if (headerNeeded && records[0] !== undefined) {
+          header = readCsvHeader(name, definition.columns, records[0]);
+          records = records.slice(1);
+        }
+        const judged = await judge(plan, header ?? definition.columns, records);
+        errors.push(...judged.errors);
+        if (rejected + pageRejected + judged.rejected > maxRowErrors) {
+          const errorPart = await report();
+          throw new ImportTooManyErrorsError({
+            import: name,
+            rejected: rejected + pageRejected + judged.rejected,
+            maxRowErrors,
+            rows: rows + pageRows,
+            errorPrefix,
+            parts: parts + (errorPart === null ? 0 : 1),
+          });
+        }
+        if (judged.rows.length > 0) {
+          // Past the deadline this attempt no longer owns the run; its successor re-runs the page.
+          if (signal.aborted) throw new JobAbortedError({ job: name, step: stepName });
+          await definition.handle({ rows: judged.rows, ctx, input });
+        }
+        pageRows += judged.rows.length;
+        pageRejected += judged.rejected;
+      }
+      assert(
+        cursor > at.offset,
+        `import "${name}" made no progress at offset ${at.offset} of "${key}"`,
+        `raise pageBytes above maxRecordBytes on importRows("${name}") — a window must be able to hold a whole record`,
+      );
+      return {
+        offset: cursor,
+        line,
+        rows: pageRows,
+        rejected: pageRejected,
+        errorPart: await report(),
+        header,
+      };
+    };
+    state = asCheckpoint(await step.run(stepName, body), stepName);
+    rows += state.rows;
+    rejected += state.rejected;
+    if (state.errorPart !== null) parts += 1;
+  }
+
+  // A file with no first line at all — empty, a BOM, blank lines — has no header to compare, and
+  // "zero rows imported" would be a success over a file that never said what it holds.
+  if (definition.format === 'csv' && state.header === null) {
+    throw new ImportHeaderMismatchError({
+      import: name,
+      columns: definition.columns,
+      missing: definition.columns,
+      undeclared: [],
+    });
+  }
+
+  logger.info('jobs.import.completed', { import: name, key, rows, rejected, parts, bytes: size });
+  return { name, key, errorPrefix, rows, rejected, parts, bytes: size };
+}
+
+const encode = (text: string): Uint8Array => new TextEncoder().encode(text);

package/src/import-source.ts

@@ -0,0 +1,49 @@
+// Where an import's bytes come from, read one window at a time.
+//
+// A SEAM and not a `@ultimat3/storage` import, for the reason `ExportSink` is one: jobs is tier 3
+// and holds no storage dependency. A `StorageDriver` satisfies it by having the two methods it
+// already has — `stat(key)` and `get(key, { range: [first, last] })` — so `disk('imports')` IS an
+// `ImportSource`; `scripts/export-sink.test.ts` compiles that assignment.
+
+/** What `stat` must know: the object's whole length, so the pass knows which window is the last. */
+export interface ImportObject {
+  readonly size: number;
+}
+
+export interface ImportSource {
+  /** `undefined` when no object is at the key — `StorageDriver.stat`'s own answer. */
+  stat(key: string): Promise<ImportObject | undefined>;
+  /**
+   * The bytes `[first, last]`, BOTH inclusive and clamped to the object, as HTTP's `Range:` and
+   * `StorageDriver.get`'s `range` read. Never called without a range: the whole object is exactly
+   * what this seam exists not to hold.
+   */
+  get(
+    key: string,
+    options: { readonly range: readonly [first: number, last: number] },
+  ): Promise<{ readonly bytes: Uint8Array }>;
+}
+
+export interface MemoryImportSource extends ImportSource {
+  /** Every range asked for, in order — what a test reads to prove the pass never read whole. */
+  reads(): readonly (readonly [number, number])[];
+}
+
+/** The dev and test source: one key, one buffer, the same clamp a real disk applies. */
+export function memoryImportSource(
+  objects: Readonly<Record<string, Uint8Array>>,
+): MemoryImportSource {
+  const reads: (readonly [number, number])[] = [];
+  return {
+    stat: (key) => {
+      const object = objects[key];
+      return Promise.resolve(object === undefined ? undefined : { size: object.byteLength });
+    },
+    get: (key, { range }) => {
+      reads.push([range[0], range[1]]);
+      const object = objects[key] ?? new Uint8Array();
+      return Promise.resolve({ bytes: object.slice(range[0], range[1] + 1) });
+    },
+    reads: () => [...reads],
+  };
+}