@littlebigbrain/create 0.3.0 → 0.5.0

package/README.md

@@ -21,9 +21,12 @@ npm test
 | `documents-map` | Maps the links of `documents-link` into the graph's own ontology, as fact proposals |
 | `review` | Writes the fact proposals at or above a gate as facts and gives the others to a person |
 | `link` | Connects the mentions of one thing across the records of a graph into one thing, and proposes records of one thing as an identity |
+| `rows` | Reads each CSV or TSV file into one instance per row, each with an edge to the file whose evidence is the row |
+| `sections` | Reads each Markdown file into its sections, each with an edge to its parent whose evidence is its lines |
 
 A template id is the id of its starter workflow, with dashes for dots:
-`documents-link` is the code of `documents.link`.
+`documents-link` is the code of `documents.link`. `link`, `rows` and
+`sections` have no LBB-run version.
 
 ## The project
 
@@ -32,7 +35,7 @@ A template id is the id of its starter workflow, with dashes for dots:
 | `src/workflow.ts` | The workflow, its settings, the triggered workflow that sends it the instances (`trigger()`) and the ontology it writes (`ontology()`) |
 | `src/*.ts` | The template's helpers and `samples.ts`, which puts the samples on a graph |
 | `src/workflow.test.ts`, `src/testing.ts` | The workflow on its samples, with mock models and no server |
-| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`) keeps them |
+| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`, `rows`, `sections`) keeps them |
 | `src/grade-run.ts` | `npm run grade`, in a template that grades a run against a gold file (`link`) |
 | `samples/` | The samples and their answer key |
 

package/lib/create.mjs

@@ -22,6 +22,8 @@ export const TEMPLATES = [
   "values-extract",
   "ontology-fit",
   "link",
+  "rows",
+  "sections",
 ];
 
 const PACKAGE = resolve(dirname(fileURLToPath(import.meta.url)), "..");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@littlebigbrain/create",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Create a project from a little big brain starter workflow template: npm create @littlebigbrain",
   "license": "Apache-2.0",
   "repository": {

package/templates/_shared/testing.ts

@@ -342,6 +342,209 @@ export class MockActivities implements LbbActivities {
   }
 }
 
+// ---- files and derivations ---------------------------------------------------------
+
+/** A file the mock holds: its bytes, name, media type and `Document` IRI. */
+export interface MockFile {
+  sha256: string;
+  name: string;
+  mediaType: string;
+  document: string;
+  bytes: Uint8Array;
+}
+
+/**
+ * The files and the derivations of a server, for the templates that read
+ * files: `files.content` and `workflows.derivations.check` / `record`, the
+ * shape of an `LbbClient`. `upload` puts the file's `Document` in `graph`
+ * with its name, SHA-256 and media type, as the upload route does.
+ *
+ * Each upload has its `uploaded_at`, as the server's file record: an upload
+ * of a live file keeps it, an upload after `remove` gets a new one. A
+ * derivation belongs to the upload of its record. The check names the
+ * upload of a live file; a removed file has no upload and no derivation.
+ * A record that names another upload than the live one throws `LbbError`
+ * 409 `derivation_upload_changed`.
+ */
+export class MockContent {
+  readonly held = new Map<string, MockFile>();
+  readonly recorded = new Map<string, Schemas["Derivation"]>();
+  /** The upload of each content's file, live or removed. */
+  readonly uploads = new Map<string, { uploadedAt: string; live: boolean }>();
+  /** The upload each recorded derivation belongs to. */
+  private readonly stamps = new Map<string, string | undefined>();
+  /** Every derivation record request, in order. */
+  readonly records: Schemas["DerivationRecordRequest"][] = [];
+  /** Reads of a file's bytes. */
+  reads = 0;
+  /** Derivation checks. */
+  checks = 0;
+
+  constructor(readonly graph: MemoryGraph) {}
+
+  /** Hold a file and write its `Document`. */
+  upload(name: string, body: string | Uint8Array, mediaType: string): MockFile {
+    const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;
+    const sha256 = createHash("sha256").update(bytes).digest("hex");
+    if (!this.uploads.get(sha256)?.live) {
+      this.newUploads += 1;
+      const at = new Date(Date.UTC(2026, 9, 15, 9) + this.newUploads * 1_000);
+      this.uploads.set(sha256, { uploadedAt: at.toISOString(), live: true });
+    }
+    const doc = this.graph.entity({ type: "Document", name, key: `file:${sha256}` });
+    doc.name = name;
+    Object.assign(doc.properties, {
+      name,
+      file_sha256: sha256,
+      content_type: mediaType,
+      file_bytes: bytes.length,
+    });
+    const file = { sha256, name, mediaType, document: doc.iri, bytes };
+    this.held.set(sha256, file);
+    return file;
+  }
+
+  /** Uploads that gave a content a new `uploaded_at`. */
+  private newUploads = 0;
+
+  /** Delete a file, as `DELETE /v1/files`: its bytes are no longer read; its `Document` stays here. */
+  remove(sha256: string): void {
+    const upload = this.uploads.get(sha256);
+    if (!upload?.live) throw new Error(`no live file ${sha256}`);
+    this.uploads.set(sha256, { ...upload, live: false });
+    this.held.delete(sha256);
+  }
+
+  /** The `uploaded_at` of a content's file. */
+  uploadedAt(sha256: string): string | undefined {
+    return this.uploads.get(sha256)?.uploadedAt;
+  }
+
+  /** The byte range of each read, `null` for a whole file, in order. */
+  readonly ranges: Array<{ start: number; end: number | null } | null> = [];
+  /** The bytes all reads answered. */
+  bytesRead = 0;
+
+  /**
+   * A file's bytes, or with `range` bytes `start` to `end` (both included),
+   * as `GET /v1/files/content` answers: a range past the end ends at the
+   * end, and a range from the end on (a file without bytes too) throws
+   * `LbbError` 416 `range_not_satisfiable`.
+   */
+  readonly files = {
+    content: async (
+      sha256: string,
+      options: { range?: { start: number; end?: number } } = {},
+    ): Promise<Uint8Array> => {
+      this.reads += 1;
+      const file = this.held.get(sha256);
+      if (!file)
+        throw new LbbError(404, "{}", { code: "file_not_found", message: `no file ${sha256}` });
+      const range = options.range;
+      this.ranges.push(range ? { start: range.start, end: range.end ?? null } : null);
+      if (!range) {
+        this.bytesRead += file.bytes.length;
+        return file.bytes;
+      }
+      if (range.start >= file.bytes.length)
+        throw new LbbError(416, "{}", {
+          code: "range_not_satisfiable",
+          message: `no byte of the file from ${range.start} on`,
+        });
+      const end = Math.min(range.end ?? file.bytes.length - 1, file.bytes.length - 1);
+      const bytes = file.bytes.slice(range.start, end + 1);
+      this.bytesRead += bytes.length;
+      return bytes;
+    },
+  };
+
+  readonly workflows = {
+    derivations: {
+      check: async (key: Schemas["DerivationKey"]): Promise<Schemas["DerivationCheckResponse"]> => {
+        this.checks += 1;
+        const file = this.uploads.get(key.sha256);
+        const upload = file?.live ? file.uploadedAt : undefined;
+        const derivation =
+          file && !file.live ? undefined : this.current(derivationName(key), key.sha256);
+        return {
+          exists: Boolean(derivation),
+          ...(derivation ? { derivation } : {}),
+          ...(upload ? { content_uploaded_at: upload } : {}),
+        };
+      },
+      record: async (
+        request: Schemas["DerivationRecordRequest"],
+      ): Promise<Schemas["DerivationRecordResponse"]> => {
+        this.records.push(request);
+        const named = request.content_uploaded_at;
+        const upload = this.uploads.get(request.sha256);
+        if (named && !(upload?.live && upload.uploadedAt === named))
+          throw new LbbError(409, "{}", {
+            code: "derivation_upload_changed",
+            message: `the file was deleted since upload ${named}: nothing was recorded`,
+            param: "content_uploaded_at",
+          });
+        const name = derivationName(request);
+        const kept = this.current(name, request.sha256);
+        if (kept) return { recorded: false, derivation: kept };
+        const outputs = JSON.stringify(request.outputs ?? null);
+        if (outputs.length > 16 * 1024)
+          throw new LbbError(400, "{}", {
+            code: "derivation_invalid",
+            message: "outputs over 16 KiB",
+          });
+        const derivation = {
+          sha256: request.sha256,
+          workflow_type: request.workflow_type,
+          version: request.version,
+          settings_digest: request.settings_digest,
+          run_id: request.run_id ?? null,
+          outputs: JSON.parse(outputs) as unknown,
+          recorded_at: "2026-10-15T09:00:00Z",
+        };
+        this.recorded.set(name, derivation);
+        this.stamps.set(name, upload?.uploadedAt);
+        return { recorded: true, derivation };
+      },
+    },
+  };
+
+  /** The derivation `name` when it belongs to the current upload of its content. */
+  private current(name: string, sha256: string): Schemas["Derivation"] | undefined {
+    const derivation = this.recorded.get(name);
+    const upload = this.uploadedAt(sha256);
+    if (!derivation || (upload !== undefined && this.stamps.get(name) !== upload)) return undefined;
+    return derivation;
+  }
+}
+
+const derivationName = (key: Schemas["DerivationKey"]) =>
+  `${key.workflow_type}@${key.version}/${key.settings_digest}/${key.sha256}`;
+
+/** The message a content trigger sends for one upload: its `Document`, with origin `content`. */
+export function contentMessage(
+  workflow: string,
+  file: Pick<MockFile, "sha256" | "mediaType" | "document">,
+): WorkflowTriggerMessage {
+  return {
+    type: "instances",
+    workflow,
+    graph: "samples",
+    as_of_commit_seq: 7,
+    reason: "changed",
+    instances: [file.document],
+    removed: [],
+    origin: {
+      kind: "content",
+      trigger: workflow,
+      sha256: file.sha256,
+      media_type: file.mediaType,
+      uploaded_at_ms: 1791967651158,
+      requests: [`file-upload:${file.sha256}`],
+    },
+  };
+}
+
 function assertNever(what: string): never {
   throw new Error(`the test gives the mock no ${what} rule`);
 }
@@ -395,6 +598,27 @@ export class Crash extends Error {
 
 const STEP_KEY = /^[A-Za-z0-9_.-]{1,128}$/;
 
+/**
+ * A workflow value as JSON, as the SDK's worker writes it: a value JSON
+ * cannot hold (undefined, a function, a number that is not finite) fails
+ * the turn there, so it fails here too.
+ */
+function workflowJson(value: unknown, where: string): string {
+  return JSON.stringify(value, (key, v: unknown) => {
+    const bad =
+      v === undefined ||
+      typeof v === "function" ||
+      typeof v === "symbol" ||
+      typeof v === "bigint" ||
+      (typeof v === "number" && !Number.isFinite(v));
+    if (bad)
+      throw new Error(
+        `Workflow values must be JSON serializable (use null for no result): ${where}${key ? `.${key}` : ""} is ${typeof v}`,
+      );
+    return v;
+  });
+}
+
 /**
  * Run one turn of `definition` as the workflow engine does: each step runs
  * once and its output is saved; with `saved`, the saved steps replay without
@@ -457,7 +681,7 @@ export async function runTurn<S>(
           heartbeat: async () => {},
           progress: (phase, counts) => turn.progress.push(progressCall(key, phase, counts)),
         });
-        const json = JSON.stringify(output ?? null);
+        const json = workflowJson(output, `step "${key}"`);
         if (json.length > 64 * 1024)
           throw new Error(`step ${key} saves ${json.length} bytes, more than 64 KiB`);
         turn.bytes += json.length;
@@ -481,6 +705,8 @@ export async function runTurn<S>(
     }),
   } as unknown as WorkflowContext;
   const result = (await definition.execute(ctx, state, message)) as TurnResult<S>;
+  workflowJson(result.state, "state");
+  if (result.continuation) workflowJson(result.continuation, "continuation");
   return { result, turn };
 }
 

package/templates/rows/README.md

@@ -0,0 +1,157 @@
+# rows
+
+Reads each CSV or TSV file of a graph into one instance per row. A
+customer list gives one `Customer` per row, with the values of its
+columns. Each instance has an edge `READ_FROM` to the file's `Document`.
+The edge's evidence is the row's text, and its source is the file's IRI
+with the row: `https://littlebigbrain.com/e/41ab…#row=5`
+([evidence.md](../../../docs/reference/evidence.md#evidence-from-a-file)).
+
+The template has no LBB-run version. It is a named workflow with a
+**content trigger**: LBB sends it each upload of `text/csv` or
+`text/tab-separated-values` within seconds, without a wait for a
+publication ([workflows.md](../../../docs/reference/workflows.md#content-triggers)).
+
+## How it runs
+
+Each message holds one file. The first turn of a file:
+
+1. **Check.** Reads the file's name with a strong read, and the
+   derivation of the content for `rows@v1` and the settings
+   ([Derivations](../../../docs/reference/workflows.md#derivations)). A
+   content with a derivation is skipped, so the same bytes under a second
+   name are read once. So is a file whose name the settings' `match` does
+   not match. Other settings are another derivation, and read the file
+   again.
+2. **Plan.** Reads the header row: the first window of the file, by byte
+   range (`client.files.content(sha256, { range })`). Each column maps to
+   a property: the settings' `columns`, else each
+   column whose header is a property of the graph by its normalized name
+   (`Customer ID` and `customer_id` match). A property must be in the
+   graph's property registry with a scalar type. The key column keys each
+   instance; a column named `name`, `title` or `label` names it.
+3. **Represents.** With `represents`, when the file's name without its
+   extension is the label of exactly one instance of that class, writes
+   `Document REPRESENTS <instance>`. `q3-customers.csv` represents the
+   report `Q3 customers`: the labels compare in lower case, with `-`, `_`
+   and spaces as one space. No match, or more than one, writes nothing and
+   proposes nothing.
+
+Then each turn reads at most `rowsPerTurn` rows and writes them in owned
+writes of 400 rows. A turn reads from the byte offset where the last one
+stopped, a window of 4 MiB per read, and parses only complete rows. A row
+that a window does not end, such as a quoted field with a line break across
+the window's edge, is read on in the next window. The first turn reads on
+from the bytes of its header read, so a file is read about once in all. A
+row is one instance of the settings' class:
+
+- its key: the key column's value, else `<sha256>:<row>`. A row whose key
+  column is empty is left out;
+- its values, each in its property's type. A value that does not take the
+  type is left out and counted. An empty cell removes the property, as the
+  owned write replaces the mapped properties of the instance;
+- the edge `READ_FROM` to the `Document`, with the evidence
+  `{text: <the row's text>, source_id: "<Document IRI>#row=<n>"}`.
+
+Rows count as RFC 7111 counts them: from 1, and the header is row 1. A
+blank row counts and writes nothing. When the file has more rows, the turn
+continues (`ctx.continue`): the message's cursor keeps the byte offset of
+the next row (`byte`) and its row number (`row`), so the row numbers stay
+exact. A turn that runs again after a crash reads its rows again from the
+same offset. A cursor without `byte`, from an earlier version of the
+template, reads the file again from its start. The last turn
+records the derivation with what it wrote: the rows, the instances, the
+columns it mapped and left out, the instance the file represents, and the
+commits of its writes. The record names the upload the check
+answered (`content_uploaded_at`). The cursor carries it from turn to turn.
+When the file was deleted while the run read it, the record writes nothing
+(`409 derivation_upload_changed`). The run ends and counts the file as
+skipped. The run of a new upload reads the file again. Say the file was
+deleted and not uploaded again. The next read of its bytes answers
+`404 file_not_found`. The run ends there too, and counts the file as
+skipped.
+
+The parser follows RFC 4180: fields in double quotes can hold the
+delimiter, line breaks and quotes (`""`). A row whose quoted field holds a
+line break spans two lines and keeps one row number. TSV takes the same
+quotes, with a tab as the delimiter. A byte order mark is dropped. The
+parser reads bytes: the delimiter, the quote and the line breaks are ASCII,
+so a window's edge inside a character of UTF-8 or between CR and LF changes
+no row.
+
+## Settings
+
+`defaults` in `workflow.ts`, which fit the samples:
+
+| Setting | Default | Meaning |
+| --- | --- | --- |
+| `className` | `Customer` | The class of the instances, one per data row |
+| `key` | `Customer ID` | The column whose value keys each instance. Without it, the key is `<sha256>:<row>` |
+| `columns` | none | Column to property. Without it, each column whose header is a property of the graph |
+| `represents` | `{class: "Report"}` | Write `REPRESENTS` to the one instance of this class that the file's name names |
+| `match` | `customers` | A regular expression on the file's name. Without it, every CSV and TSV file |
+| `rowsPerTurn` | 5,000 | Rows one turn reads, 1 to 20,000 |
+
+`trigger(settings)` names the workflow `rows`, with the content trigger
+`rows` for the two media types. It owns the class, `READ_FROM`,
+`REPRESENTS` when `represents` is set, and the properties of `columns`.
+`ontology(settings)` adds the class and `READ_FROM`. The properties come
+from the graph: add them with `lbb.evolve` before the first file.
+
+The settings digest is the SHA-256 of the settings' canonical JSON: a
+change of any setting reads every file again.
+
+Two content workflows own one relation when no upload can reach both:
+`rows` and `sections`, both with `represents`, go on one graph, because no
+media type of one meets a media type of the other. A second workflow that
+writes `REPRESENTS` for a type that meets one of these, such as `text/*`, is
+refused with `409 workflow_owns_conflict`
+([starter-workflows.md](../../../docs/reference/starter-workflows.md#cycles-and-conflicts)).
+
+## Commands
+
+`setup.ts`, `worker.ts` and `try.ts` are the template's own:
+`npm run setup` creates the named workflow with
+`client.workflows.named.put(trigger())`, and `npm start` serves
+`createWorkflow(lbbActivities(client), client)`: the client reads the
+files' bytes and the derivations. A fourth argument sets how the workflow
+reads: `{ window, maxRecord }`, the bytes of one read (default 4 MiB, at
+least 64) and of the longest row (default 16 MiB). They change no output,
+so they are not settings and enter no settings digest. `npm run try` creates the graph
+`try-rows`, adds the sample properties and reports, uploads the two sample
+files and creates the workflow. A content trigger reads the uploads from
+before its creation too.
+
+## Limits
+
+- A turn holds at most one window and the longest row in memory. A row
+  longer than 16 MiB, such as a quote that never closes, stops the file:
+  the turn fails with an error that names the row and its byte offset.
+- A turn that stops at `rowsPerTurn` in the middle of a window reads the
+  rest of that window again in the next turn.
+- A file deleted later keeps the instances it gave; its `READ_FROM` edges
+  go with its `Document`.
+- A message that comes again while a large file is still in progress (a
+  run now) reads the file again. The writes are the same.
+
+## Samples and test
+
+`samples/q3-customers.csv` is an invented customer list of 8 data rows. It
+holds a quoted name with a comma, a name with quotes, an address with a
+line break, a value that is not a date, a blank row, an empty cell and a
+row without a key. `samples/q3-orders.tsv` is an invented order table.
+`samples/properties.json` lists the graph properties the columns fill, and
+`samples/reports.json` three reports the file names can name.
+`samples/expected.json` gives the instances a correct run writes.
+
+`workflow.test.ts` runs the workflow without a server (`MockContent` holds
+the files and the derivations). It checks the answer key for both files,
+the evidence of each row, the skip by derivation and by name, a file of
+more rows than a turn reads, parts of 400 rows, a header-only file and an
+empty file, `represents` with zero, one and two matching instances, keys
+without a key column, the `columns` setting, the phases, and the parser
+on RFC 4180's cases. The reads by range: a file of many windows read once
+with exact row numbers, every window size from 64 to 200 bytes over
+quoted fields, letters beyond ASCII and the three line breaks, a CRLF file
+whose window ends between CR and LF, a turn that continues from its byte
+offset (also after a crash), and a row longer than the longest row.