@littlebigbrain/create 0.3.0 → 0.4.0

package/README.md

@@ -21,9 +21,12 @@ npm test
 | `documents-map` | Maps the links of `documents-link` into the graph's own ontology, as fact proposals |
 | `review` | Writes the fact proposals at or above a gate as facts and gives the others to a person |
 | `link` | Connects the mentions of one thing across the records of a graph into one thing, and proposes records of one thing as an identity |
+| `rows` | Reads each CSV or TSV file into one instance per row, each with an edge to the file whose evidence is the row |
+| `sections` | Reads each Markdown file into its sections, each with an edge to its parent whose evidence is its lines |
 
 A template id is the id of its starter workflow, with dashes for dots:
-`documents-link` is the code of `documents.link`.
+`documents-link` is the code of `documents.link`. `link`, `rows` and
+`sections` have no LBB-run version.
 
 ## The project
 
@@ -32,7 +35,7 @@ A template id is the id of its starter workflow, with dashes for dots:
 | `src/workflow.ts` | The workflow, its settings, the triggered workflow that sends it the instances (`trigger()`) and the ontology it writes (`ontology()`) |
 | `src/*.ts` | The template's helpers and `samples.ts`, which puts the samples on a graph |
 | `src/workflow.test.ts`, `src/testing.ts` | The workflow on its samples, with mock models and no server |
-| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`) keeps them |
+| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`, `rows`, `sections`) keeps them |
 | `src/grade-run.ts` | `npm run grade`, in a template that grades a run against a gold file (`link`) |
 | `samples/` | The samples and their answer key |
 

package/lib/create.mjs

@@ -22,6 +22,8 @@ export const TEMPLATES = [
   "values-extract",
   "ontology-fit",
   "link",
+  "rows",
+  "sections",
 ];
 
 const PACKAGE = resolve(dirname(fileURLToPath(import.meta.url)), "..");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@littlebigbrain/create",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Create a project from a little big brain starter workflow template: npm create @littlebigbrain",
   "license": "Apache-2.0",
   "repository": {

package/templates/_shared/testing.ts

@@ -342,6 +342,150 @@ export class MockActivities implements LbbActivities {
   }
 }
 
+// ---- files and derivations ---------------------------------------------------------
+
+/** A file the mock holds: its bytes, name, media type and `Document` IRI. */
+export interface MockFile {
+  sha256: string;
+  name: string;
+  mediaType: string;
+  document: string;
+  bytes: Uint8Array;
+}
+
+/**
+ * The files and the derivations of a server, for the templates that read
+ * files: `files.content` and `workflows.derivations.check` / `record`, the
+ * shape of an `LbbClient`. `upload` puts the file's `Document` in `graph`
+ * with its name, SHA-256 and media type, as the upload route does.
+ */
+export class MockContent {
+  readonly held = new Map<string, MockFile>();
+  readonly recorded = new Map<string, Schemas["Derivation"]>();
+  /** Reads of a file's bytes. */
+  reads = 0;
+  /** Derivation checks. */
+  checks = 0;
+
+  constructor(readonly graph: MemoryGraph) {}
+
+  /** Hold a file and write its `Document`. */
+  upload(name: string, body: string | Uint8Array, mediaType: string): MockFile {
+    const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;
+    const sha256 = createHash("sha256").update(bytes).digest("hex");
+    const doc = this.graph.entity({ type: "Document", name, key: `file:${sha256}` });
+    doc.name = name;
+    Object.assign(doc.properties, {
+      name,
+      file_sha256: sha256,
+      content_type: mediaType,
+      file_bytes: bytes.length,
+    });
+    const file = { sha256, name, mediaType, document: doc.iri, bytes };
+    this.held.set(sha256, file);
+    return file;
+  }
+
+  /** The byte range of each read, `null` for a whole file, in order. */
+  readonly ranges: Array<{ start: number; end: number | null } | null> = [];
+  /** The bytes all reads answered. */
+  bytesRead = 0;
+
+  /**
+   * A file's bytes, or with `range` bytes `start` to `end` (both included),
+   * as `GET /v1/files/content` answers: a range past the end ends at the
+   * end, and a range from the end on (a file without bytes too) throws
+   * `LbbError` 416 `range_not_satisfiable`.
+   */
+  readonly files = {
+    content: async (
+      sha256: string,
+      options: { range?: { start: number; end?: number } } = {},
+    ): Promise<Uint8Array> => {
+      this.reads += 1;
+      const file = this.held.get(sha256);
+      if (!file)
+        throw new LbbError(404, "{}", { code: "file_not_found", message: `no file ${sha256}` });
+      const range = options.range;
+      this.ranges.push(range ? { start: range.start, end: range.end ?? null } : null);
+      if (!range) {
+        this.bytesRead += file.bytes.length;
+        return file.bytes;
+      }
+      if (range.start >= file.bytes.length)
+        throw new LbbError(416, "{}", {
+          code: "range_not_satisfiable",
+          message: `no byte of the file from ${range.start} on`,
+        });
+      const end = Math.min(range.end ?? file.bytes.length - 1, file.bytes.length - 1);
+      const bytes = file.bytes.slice(range.start, end + 1);
+      this.bytesRead += bytes.length;
+      return bytes;
+    },
+  };
+
+  readonly workflows = {
+    derivations: {
+      check: async (key: Schemas["DerivationKey"]): Promise<Schemas["DerivationCheckResponse"]> => {
+        this.checks += 1;
+        const derivation = this.recorded.get(derivationName(key));
+        return derivation ? { exists: true, derivation } : { exists: false };
+      },
+      record: async (
+        request: Schemas["DerivationRecordRequest"],
+      ): Promise<Schemas["DerivationRecordResponse"]> => {
+        const name = derivationName(request);
+        const kept = this.recorded.get(name);
+        if (kept) return { recorded: false, derivation: kept };
+        const outputs = JSON.stringify(request.outputs ?? null);
+        if (outputs.length > 16 * 1024)
+          throw new LbbError(400, "{}", {
+            code: "derivation_invalid",
+            message: "outputs over 16 KiB",
+          });
+        const derivation = {
+          sha256: request.sha256,
+          workflow_type: request.workflow_type,
+          version: request.version,
+          settings_digest: request.settings_digest,
+          run_id: request.run_id ?? null,
+          outputs: JSON.parse(outputs) as unknown,
+          recorded_at: "2026-10-15T09:00:00Z",
+        };
+        this.recorded.set(name, derivation);
+        return { recorded: true, derivation };
+      },
+    },
+  };
+}
+
+const derivationName = (key: Schemas["DerivationKey"]) =>
+  `${key.workflow_type}@${key.version}/${key.settings_digest}/${key.sha256}`;
+
+/** The message a content trigger sends for one upload: its `Document`, with origin `content`. */
+export function contentMessage(
+  workflow: string,
+  file: Pick<MockFile, "sha256" | "mediaType" | "document">,
+): WorkflowTriggerMessage {
+  return {
+    type: "instances",
+    workflow,
+    graph: "samples",
+    as_of_commit_seq: 7,
+    reason: "changed",
+    instances: [file.document],
+    removed: [],
+    origin: {
+      kind: "content",
+      trigger: workflow,
+      sha256: file.sha256,
+      media_type: file.mediaType,
+      uploaded_at_ms: 1791967651158,
+      requests: [`file-upload:${file.sha256}`],
+    },
+  };
+}
+
 function assertNever(what: string): never {
   throw new Error(`the test gives the mock no ${what} rule`);
 }
@@ -395,6 +539,27 @@ export class Crash extends Error {
 
 const STEP_KEY = /^[A-Za-z0-9_.-]{1,128}$/;
 
+/**
+ * A workflow value as JSON, as the SDK's worker writes it: a value JSON
+ * cannot hold (undefined, a function, a number that is not finite) fails
+ * the turn there, so it fails here too.
+ */
+function workflowJson(value: unknown, where: string): string {
+  return JSON.stringify(value, (key, v: unknown) => {
+    const bad =
+      v === undefined ||
+      typeof v === "function" ||
+      typeof v === "symbol" ||
+      typeof v === "bigint" ||
+      (typeof v === "number" && !Number.isFinite(v));
+    if (bad)
+      throw new Error(
+        `Workflow values must be JSON serializable (use null for no result): ${where}${key ? `.${key}` : ""} is ${typeof v}`,
+      );
+    return v;
+  });
+}
+
 /**
  * Run one turn of `definition` as the workflow engine does: each step runs
  * once and its output is saved; with `saved`, the saved steps replay without
@@ -457,7 +622,7 @@ export async function runTurn<S>(
           heartbeat: async () => {},
           progress: (phase, counts) => turn.progress.push(progressCall(key, phase, counts)),
         });
-        const json = JSON.stringify(output ?? null);
+        const json = workflowJson(output, `step "${key}"`);
         if (json.length > 64 * 1024)
           throw new Error(`step ${key} saves ${json.length} bytes, more than 64 KiB`);
         turn.bytes += json.length;
@@ -481,6 +646,8 @@ export async function runTurn<S>(
     }),
   } as unknown as WorkflowContext;
   const result = (await definition.execute(ctx, state, message)) as TurnResult<S>;
+  workflowJson(result.state, "state");
+  if (result.continuation) workflowJson(result.continuation, "continuation");
   return { result, turn };
 }
 

package/templates/rows/README.md

@@ -0,0 +1,150 @@
+# rows
+
+Reads each CSV or TSV file of a graph into one instance per row. A
+customer list gives one `Customer` per row, with the values of its
+columns. Each instance has an edge `READ_FROM` to the file's `Document`.
+The edge's evidence is the row's text, and its source is the file's IRI
+with the row: `https://littlebigbrain.com/e/41ab…#row=5`
+([evidence.md](../../../docs/reference/evidence.md#evidence-from-a-file)).
+
+The template has no LBB-run version. It is a named workflow with a
+**content trigger**: LBB sends it each upload of `text/csv` or
+`text/tab-separated-values` within seconds, without a wait for a
+publication ([workflows.md](../../../docs/reference/workflows.md#content-triggers)).
+
+## How it runs
+
+Each message holds one file. The first turn of a file:
+
+1. **Check.** Reads the file's name with a strong read, and the
+   derivation of the content for `rows@v1` and the settings
+   ([Derivations](../../../docs/reference/workflows.md#derivations)). A
+   content with a derivation is skipped, so the same bytes under a second
+   name are read once. So is a file whose name the settings' `match` does
+   not match. Other settings are another derivation, and read the file
+   again.
+2. **Plan.** Reads the header row: the first window of the file, by byte
+   range (`client.files.content(sha256, { range })`). Each column maps to
+   a property: the settings' `columns`, else each
+   column whose header is a property of the graph by its normalized name
+   (`Customer ID` and `customer_id` match). A property must be in the
+   graph's property registry with a scalar type. The key column keys each
+   instance; a column named `name`, `title` or `label` names it.
+3. **Represents.** With `represents`, when the file's name without its
+   extension is the label of exactly one instance of that class, writes
+   `Document REPRESENTS <instance>`. `q3-customers.csv` represents the
+   report `Q3 customers`: the labels compare in lower case, with `-`, `_`
+   and spaces as one space. No match, or more than one, writes nothing and
+   proposes nothing.
+
+Then each turn reads at most `rowsPerTurn` rows and writes them in owned
+writes of 400 rows. A turn reads from the byte offset where the last one
+stopped, a window of 4 MiB per read, and parses only complete rows. A row
+that a window does not end, such as a quoted field with a line break across
+the window's edge, is read on in the next window. The first turn reads on
+from the bytes of its header read, so a file is read about once in all. A
+row is one instance of the settings' class:
+
+- its key: the key column's value, else `<sha256>:<row>`. A row whose key
+  column is empty is left out;
+- its values, each in its property's type. A value that does not take the
+  type is left out and counted. An empty cell removes the property, as the
+  owned write replaces the mapped properties of the instance;
+- the edge `READ_FROM` to the `Document`, with the evidence
+  `{text: <the row's text>, source_id: "<Document IRI>#row=<n>"}`.
+
+Rows count as RFC 7111 counts them: from 1, and the header is row 1. A
+blank row counts and writes nothing. When the file has more rows, the turn
+continues (`ctx.continue`): the message's cursor keeps the byte offset of
+the next row (`byte`) and its row number (`row`), so the row numbers stay
+exact. A turn that runs again after a crash reads its rows again from the
+same offset. A cursor without `byte`, from an earlier version of the
+template, reads the file again from its start. The last turn
+records the derivation with what it wrote: the rows, the instances, the
+columns it mapped and left out, the instance the file represents, and the
+commits of its writes.
+
+The parser follows RFC 4180: fields in double quotes can hold the
+delimiter, line breaks and quotes (`""`). A row whose quoted field holds a
+line break spans two lines and keeps one row number. TSV takes the same
+quotes, with a tab as the delimiter. A byte order mark is dropped. The
+parser reads bytes: the delimiter, the quote and the line breaks are ASCII,
+so a window's edge inside a character of UTF-8 or between CR and LF changes
+no row.
+
+## Settings
+
+`defaults` in `workflow.ts`, which fit the samples:
+
+| Setting | Default | Meaning |
+| --- | --- | --- |
+| `className` | `Customer` | The class of the instances, one per data row |
+| `key` | `Customer ID` | The column whose value keys each instance. Without it, the key is `<sha256>:<row>` |
+| `columns` | none | Column to property. Without it, each column whose header is a property of the graph |
+| `represents` | `{class: "Report"}` | Write `REPRESENTS` to the one instance of this class that the file's name names |
+| `match` | `customers` | A regular expression on the file's name. Without it, every CSV and TSV file |
+| `rowsPerTurn` | 5,000 | Rows one turn reads, 1 to 20,000 |
+
+`trigger(settings)` names the workflow `rows`, with the content trigger
+`rows` for the two media types. It owns the class, `READ_FROM`,
+`REPRESENTS` when `represents` is set, and the properties of `columns`.
+`ontology(settings)` adds the class and `READ_FROM`. The properties come
+from the graph: add them with `lbb.evolve` before the first file.
+
+The settings digest is the SHA-256 of the settings' canonical JSON: a
+change of any setting reads every file again.
+
+Two content workflows own one relation when no upload can reach both:
+`rows` and `sections`, both with `represents`, go on one graph, because no
+media type of one meets a media type of the other. A second workflow that
+writes `REPRESENTS` for a type that meets one of these, such as `text/*`, is
+refused with `409 workflow_owns_conflict`
+([starter-workflows.md](../../../docs/reference/starter-workflows.md#cycles-and-conflicts)).
+
+## Commands
+
+`setup.ts`, `worker.ts` and `try.ts` are the template's own:
+`npm run setup` creates the named workflow with
+`client.workflows.named.put(trigger())`, and `npm start` serves
+`createWorkflow(lbbActivities(client), client)`: the client reads the
+files' bytes and the derivations. A fourth argument sets how the workflow
+reads: `{ window, maxRecord }`, the bytes of one read (default 4 MiB, at
+least 64) and of the longest row (default 16 MiB). They change no output,
+so they are not settings and enter no settings digest. `npm run try` creates the graph
+`try-rows`, adds the sample properties and reports, uploads the two sample
+files and creates the workflow. A content trigger reads the uploads from
+before its creation too.
+
+## Limits
+
+- A turn holds at most one window and the longest row in memory. A row
+  longer than 16 MiB, such as a quote that never closes, stops the file:
+  the turn fails with an error that names the row and its byte offset.
+- A turn that stops at `rowsPerTurn` in the middle of a window reads the
+  rest of that window again in the next turn.
+- A file deleted later keeps the instances it gave; its `READ_FROM` edges
+  go with its `Document`.
+- A message that comes again while a large file is still in progress (a
+  run now) reads the file again. The writes are the same.
+
+## Samples and test
+
+`samples/q3-customers.csv` is an invented customer list of 8 data rows. It
+holds a quoted name with a comma, a name with quotes, an address with a
+line break, a value that is not a date, a blank row, an empty cell and a
+row without a key. `samples/q3-orders.tsv` is an invented order table.
+`samples/properties.json` lists the graph properties the columns fill, and
+`samples/reports.json` three reports the file names can name.
+`samples/expected.json` gives the instances a correct run writes.
+
+`workflow.test.ts` runs the workflow without a server (`MockContent` holds
+the files and the derivations). It checks the answer key for both files,
+the evidence of each row, the skip by derivation and by name, a file of
+more rows than a turn reads, parts of 400 rows, a header-only file and an
+empty file, `represents` with zero, one and two matching instances, keys
+without a key column, the `columns` setting, the phases, and the parser
+on RFC 4180's cases. The reads by range: a file of many windows read once
+with exact row numbers, every window size from 64 to 200 bytes over
+quoted fields, letters beyond ASCII and the three line breaks, a CRLF file
+whose window ends between CR and LF, a turn that continues from its byte
+offset (also after a crash), and a row longer than the longest row.

package/templates/rows/csv.ts

@@ -0,0 +1,260 @@
+// The records of a CSV or TSV file, as RFC 4180 defines them: fields split by
+// the delimiter, records by a line break (CRLF, LF or CR), and a field in
+// double quotes may hold the delimiter, line breaks and quotes (written `""`).
+// Each record keeps its row number as RFC 7111 counts rows (from 1, the
+// header is row 1) and its text in the file, the quote of its evidence.
+//
+// The parser reads bytes. The delimiter, the quote and the line breaks are
+// ASCII, and no byte of a UTF-8 character other than ASCII is ASCII, so a
+// record boundary is never inside a character. `readRecords` reads a file
+// in windows of bytes (`client.files.content(sha256, { range })`) and keeps a
+// record that a window does not end for the next window.
+
+/** One record: its fields, its row from 1, its text without the line break, and where it starts and ends. */
+export interface CsvRecord {
+  fields: string[];
+  row: number;
+  text: string;
+  /** The byte offset of the record's first byte in the file. */
+  start: number;
+  /** The byte offset after the record's line break: where the next record starts. */
+  next: number;
+}
+
+/** Where the bytes given to `csvRecords` are in the file, and whether the file ends with them. */
+export interface CsvPart {
+  /** The byte offset of the first byte given. Default 0. */
+  base?: number;
+  /**
+   * The file ends with these bytes (default). With `false` the file goes on:
+   * a record the bytes do not end is not given, nor one that ends in a
+   * carriage return or a quote as the last byte (a line feed or a second
+   * quote can follow). The next read starts at the `next` of the last
+   * record given.
+   */
+  final?: boolean;
+}
+
+const QUOTE = 0x22;
+const CR = 0x0d;
+const LF = 0x0a;
+const decoder = new TextDecoder();
+
+/** The delimiter of a file: a tab for TSV (`text/tab-separated-values` or a `.tsv` name), else a comma. */
+export function delimiterOf(mediaType: string, name: string): string {
+  return mediaType === "text/tab-separated-values" || /\.tsv$/i.test(name) ? "\t" : ",";
+}
+
+/**
+ * The records of `input` (bytes, or text as its UTF-8 bytes) from the byte
+ * offset `at` of the file, numbered from `row`. A byte order mark at the
+ * start of the file is dropped. A blank line is a record of one empty field,
+ * so the row numbers stay those of RFC 7111. A quote inside an unquoted
+ * field, and text after a closing quote, are kept as they are. A line break
+ * at the end of the file ends the last record. `part` says where the bytes
+ * are in the file and whether the file ends with them.
+ */
+export function* csvRecords(
+  input: string | Uint8Array,
+  delimiter = ",",
+  at = 0,
+  row = 1,
+  part: CsvPart = {},
+): Generator<CsvRecord, void, undefined> {
+  const bytes = typeof input === "string" ? new TextEncoder().encode(input) : input;
+  const base = part.base ?? 0;
+  const final = part.final ?? true;
+  const split = delimiter.charCodeAt(0);
+  const n = bytes.length;
+  const text = (from: number, to: number) => decoder.decode(bytes.subarray(from, to));
+  let i = at - base;
+  if (at === 0 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) i = 3;
+  while (i < n) {
+    const start = i;
+    const fields: string[] = [];
+    let field = "";
+    let from = i;
+    let quoted = false;
+    let atFieldStart = true;
+    let end = -1;
+    let next = -1;
+    while (i < n) {
+      const c = bytes[i]!;
+      if (quoted) {
+        if (c === QUOTE) {
+          if (i + 1 >= n && !final) return;
+          if (bytes[i + 1] === QUOTE) {
+            field += `${text(from, i)}"`;
+            i += 2;
+          } else {
+            field += text(from, i);
+            quoted = false;
+            i += 1;
+          }
+          from = i;
+        } else i += 1;
+        continue;
+      }
+      if (c === QUOTE && atFieldStart) {
+        quoted = true;
+        atFieldStart = false;
+        i += 1;
+        from = i;
+        continue;
+      }
+      if (c === split) {
+        fields.push(field + text(from, i));
+        field = "";
+        atFieldStart = true;
+        i += 1;
+        from = i;
+        continue;
+      }
+      if (c === CR || c === LF) {
+        if (c === CR && i + 1 >= n && !final) return;
+        end = i;
+        next = c === CR && bytes[i + 1] === LF ? i + 2 : i + 1;
+        break;
+      }
+      atFieldStart = false;
+      i += 1;
+    }
+    if (end < 0) {
+      if (!final) return;
+      end = n;
+      next = n;
+    }
+    fields.push(field + text(from, end));
+    yield { fields, row, text: text(start, end), start: base + start, next: base + next };
+    row += 1;
+    i = next;
+  }
+}
+
+/** Bytes `start` to `end` of a file, both included: fewer at its end, none from its end on. */
+export type ReadRange = (start: number, end: number) => Promise<Uint8Array>;
+
+/** Bytes a read holds that its records did not take: they start at `base`. */
+export interface HeldBytes {
+  base: number;
+  bytes: Uint8Array;
+  /** The file ends with these bytes. */
+  eof: boolean;
+}
+
+/** What `readRecords` read. */
+export interface RecordsRead {
+  records: CsvRecord[];
+  /** The byte offset after the last record: where the next read starts. */
+  next: number;
+  /** The row number of the record at `next`. */
+  row: number;
+  /** No byte of the file is left from `next` on. */
+  end: boolean;
+  /** The bytes read from the file. */
+  bytesRead: number;
+  /** The bytes read from `next` on: a later read of the same turn starts with them. */
+  held: HeldBytes;
+}
+
+/** How `readRecords` reads. */
+export interface WindowOptions {
+  /** Bytes one read asks for. */
+  window: number;
+  /** The longest record: a record that is not complete after this many bytes stops the read with an error. */
+  maxRecord: number;
+}
+
+/** Bytes one read asks for by default: 4 MiB. */
+export const WINDOW_BYTES = 4 * 1024 * 1024;
+/** The longest record by default: 16 MiB, as a quote that never closes makes the rest of a file one record. */
+export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
+
+function joined(a: Uint8Array, b: Uint8Array): Uint8Array {
+  if (!a.length) return b;
+  if (!b.length) return a;
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a);
+  out.set(b, a.length);
+  return out;
+}
+
+/**
+ * At most `limit` complete records of a file from the byte offset `at`,
+ * numbered from `row`, read with `read` in windows of `options.window`
+ * bytes. A record that a window does not end is read on in the next
+ * window, so a quoted field can span windows and row numbers stay exact.
+ * `held` gives bytes an earlier read of the same turn holds from `at` on,
+ * which are not read again. Memory stays within the window and the longest
+ * record.
+ */
+export async function readRecords(
+  read: ReadRange,
+  delimiter: string,
+  at: number,
+  row: number,
+  limit: number,
+  options: WindowOptions,
+  held?: HeldBytes,
+): Promise<RecordsRead> {
+  const records: CsvRecord[] = [];
+  let base = at;
+  let buffer: Uint8Array = new Uint8Array(0);
+  let eof = false;
+  let bytesRead = 0;
+  if (held && held.base <= at && at <= held.base + held.bytes.length) {
+    buffer = held.bytes.subarray(at - held.base);
+    eof = held.eof;
+  }
+  let first = true;
+  while (records.length < limit) {
+    if (!eof && !(first && buffer.length)) {
+      const from = base + buffer.length;
+      const chunk = await read(from, from + options.window - 1);
+      bytesRead += chunk.length;
+      eof = chunk.length < options.window;
+      buffer = joined(buffer, chunk);
+    }
+    first = false;
+    let taken = base;
+    for (const record of csvRecords(buffer, delimiter, base, row, { base, final: eof })) {
+      records.push(record);
+      taken = record.next;
+      row = record.row + 1;
+      if (records.length >= limit) break;
+    }
+    buffer = buffer.subarray(taken - base);
+    base = taken;
+    if (eof) break;
+    if (buffer.length > options.maxRecord)
+      throw new RangeError(
+        `row ${row} is longer than ${options.maxRecord} bytes from byte ${base}: a quote that never closes?`,
+      );
+  }
+  return {
+    records,
+    next: base,
+    row,
+    end: eof && !buffer.length,
+    bytesRead,
+    held: { base, bytes: buffer, eof },
+  };
+}
+
+/** A record of empty fields only: a blank line, or a row of delimiters. */
+export function isBlank(record: Pick<CsvRecord, "fields">): boolean {
+  return record.fields.every((field) => !field.trim());
+}
+
+/**
+ * A header or property name for matching: letters and digits of any script,
+ * in lower case, without marks and without spaces or punctuation.
+ * `Customer ID`, `customer_id` and `CustomerId` are all `customerid`.
+ */
+export function normalizeName(name: string): string {
+  return name
+    .normalize("NFKD")
+    .replace(/\p{M}/gu, "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, "");
+}