@littlebigbrain/create 0.3.0 → 0.5.0

package/templates/rows/csv.ts

@@ -0,0 +1,260 @@
+// The records of a CSV or TSV file, as RFC 4180 defines them: fields split by
+// the delimiter, records by a line break (CRLF, LF or CR), and a field in
+// double quotes may hold the delimiter, line breaks and quotes (written `""`).
+// Each record keeps its row number as RFC 7111 counts rows (from 1, the
+// header is row 1) and its text in the file, the quote of its evidence.
+//
+// The parser reads bytes. The delimiter, the quote and the line breaks are
+// ASCII, and no byte of a UTF-8 character other than ASCII is ASCII, so a
+// record boundary is never inside a character. `readRecords` reads a file
+// in windows of bytes (`client.files.content(sha256, { range })`) and keeps a
+// record that a window does not end for the next window.
+
+/** One record: its fields, its row from 1, its text without the line break, and where it starts and ends. */
+export interface CsvRecord {
+  fields: string[];
+  row: number;
+  text: string;
+  /** The byte offset of the record's first byte in the file. */
+  start: number;
+  /** The byte offset after the record's line break: where the next record starts. */
+  next: number;
+}
+
+/** Where the bytes given to `csvRecords` are in the file, and whether the file ends with them. */
+export interface CsvPart {
+  /** The byte offset of the first byte given. Default 0. */
+  base?: number;
+  /**
+   * The file ends with these bytes (default). With `false` the file goes on:
+   * a record the bytes do not end is not given, nor one that ends in a
+   * carriage return or a quote as the last byte (a line feed or a second
+   * quote can follow). The next read starts at the `next` of the last
+   * record given.
+   */
+  final?: boolean;
+}
+
+const QUOTE = 0x22;
+const CR = 0x0d;
+const LF = 0x0a;
+const decoder = new TextDecoder();
+
+/** The delimiter of a file: a tab for TSV (`text/tab-separated-values` or a `.tsv` name), else a comma. */
+export function delimiterOf(mediaType: string, name: string): string {
+  return mediaType === "text/tab-separated-values" || /\.tsv$/i.test(name) ? "\t" : ",";
+}
+
+/**
+ * The records of `input` (bytes, or text as its UTF-8 bytes) from the byte
+ * offset `at` of the file, numbered from `row`. A byte order mark at the
+ * start of the file is dropped. A blank line is a record of one empty field,
+ * so the row numbers stay those of RFC 7111. A quote inside an unquoted
+ * field, and text after a closing quote, are kept as they are. A line break
+ * at the end of the file ends the last record. `part` says where the bytes
+ * are in the file and whether the file ends with them.
+ */
+export function* csvRecords(
+  input: string | Uint8Array,
+  delimiter = ",",
+  at = 0,
+  row = 1,
+  part: CsvPart = {},
+): Generator<CsvRecord, void, undefined> {
+  const bytes = typeof input === "string" ? new TextEncoder().encode(input) : input;
+  const base = part.base ?? 0;
+  const final = part.final ?? true;
+  const split = delimiter.charCodeAt(0);
+  const n = bytes.length;
+  const text = (from: number, to: number) => decoder.decode(bytes.subarray(from, to));
+  let i = at - base;
+  if (at === 0 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) i = 3;
+  while (i < n) {
+    const start = i;
+    const fields: string[] = [];
+    let field = "";
+    let from = i;
+    let quoted = false;
+    let atFieldStart = true;
+    let end = -1;
+    let next = -1;
+    while (i < n) {
+      const c = bytes[i]!;
+      if (quoted) {
+        if (c === QUOTE) {
+          if (i + 1 >= n && !final) return;
+          if (bytes[i + 1] === QUOTE) {
+            field += `${text(from, i)}"`;
+            i += 2;
+          } else {
+            field += text(from, i);
+            quoted = false;
+            i += 1;
+          }
+          from = i;
+        } else i += 1;
+        continue;
+      }
+      if (c === QUOTE && atFieldStart) {
+        quoted = true;
+        atFieldStart = false;
+        i += 1;
+        from = i;
+        continue;
+      }
+      if (c === split) {
+        fields.push(field + text(from, i));
+        field = "";
+        atFieldStart = true;
+        i += 1;
+        from = i;
+        continue;
+      }
+      if (c === CR || c === LF) {
+        if (c === CR && i + 1 >= n && !final) return;
+        end = i;
+        next = c === CR && bytes[i + 1] === LF ? i + 2 : i + 1;
+        break;
+      }
+      atFieldStart = false;
+      i += 1;
+    }
+    if (end < 0) {
+      if (!final) return;
+      end = n;
+      next = n;
+    }
+    fields.push(field + text(from, end));
+    yield { fields, row, text: text(start, end), start: base + start, next: base + next };
+    row += 1;
+    i = next;
+  }
+}
+
+/** Bytes `start` to `end` of a file, both included: fewer at its end, none from its end on. */
+export type ReadRange = (start: number, end: number) => Promise<Uint8Array>;
+
+/** Bytes a read holds that its records did not take: they start at `base`. */
+export interface HeldBytes {
+  base: number;
+  bytes: Uint8Array;
+  /** The file ends with these bytes. */
+  eof: boolean;
+}
+
+/** What `readRecords` read. */
+export interface RecordsRead {
+  records: CsvRecord[];
+  /** The byte offset after the last record: where the next read starts. */
+  next: number;
+  /** The row number of the record at `next`. */
+  row: number;
+  /** No byte of the file is left from `next` on. */
+  end: boolean;
+  /** The bytes read from the file. */
+  bytesRead: number;
+  /** The bytes read from `next` on: a later read of the same turn starts with them. */
+  held: HeldBytes;
+}
+
+/** How `readRecords` reads. */
+export interface WindowOptions {
+  /** Bytes one read asks for. */
+  window: number;
+  /** The longest record: a record that is not complete after this many bytes stops the read with an error. */
+  maxRecord: number;
+}
+
+/** Bytes one read asks for by default: 4 MiB. */
+export const WINDOW_BYTES = 4 * 1024 * 1024;
+/** The longest record by default: 16 MiB, as a quote that never closes makes the rest of a file one record. */
+export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
+
+function joined(a: Uint8Array, b: Uint8Array): Uint8Array {
+  if (!a.length) return b;
+  if (!b.length) return a;
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a);
+  out.set(b, a.length);
+  return out;
+}
+
+/**
+ * At most `limit` complete records of a file from the byte offset `at`,
+ * numbered from `row`, read with `read` in windows of `options.window`
+ * bytes. A record that a window does not end is read on in the next
+ * window, so a quoted field can span windows and row numbers stay exact.
+ * `held` gives bytes an earlier read of the same turn holds from `at` on,
+ * which are not read again. Memory stays within the window and the longest
+ * record.
+ */
+export async function readRecords(
+  read: ReadRange,
+  delimiter: string,
+  at: number,
+  row: number,
+  limit: number,
+  options: WindowOptions,
+  held?: HeldBytes,
+): Promise<RecordsRead> {
+  const records: CsvRecord[] = [];
+  let base = at;
+  let buffer: Uint8Array = new Uint8Array(0);
+  let eof = false;
+  let bytesRead = 0;
+  if (held && held.base <= at && at <= held.base + held.bytes.length) {
+    buffer = held.bytes.subarray(at - held.base);
+    eof = held.eof;
+  }
+  let first = true;
+  while (records.length < limit) {
+    if (!eof && !(first && buffer.length)) {
+      const from = base + buffer.length;
+      const chunk = await read(from, from + options.window - 1);
+      bytesRead += chunk.length;
+      eof = chunk.length < options.window;
+      buffer = joined(buffer, chunk);
+    }
+    first = false;
+    let taken = base;
+    for (const record of csvRecords(buffer, delimiter, base, row, { base, final: eof })) {
+      records.push(record);
+      taken = record.next;
+      row = record.row + 1;
+      if (records.length >= limit) break;
+    }
+    buffer = buffer.subarray(taken - base);
+    base = taken;
+    if (eof) break;
+    if (buffer.length > options.maxRecord)
+      throw new RangeError(
+        `row ${row} is longer than ${options.maxRecord} bytes from byte ${base}: a quote that never closes?`,
+      );
+  }
+  return {
+    records,
+    next: base,
+    row,
+    end: eof && !buffer.length,
+    bytesRead,
+    held: { base, bytes: buffer, eof },
+  };
+}
+
+/** A record of empty fields only: a blank line, or a row of delimiters. */
+export function isBlank(record: Pick<CsvRecord, "fields">): boolean {
+  return record.fields.every((field) => !field.trim());
+}
+
+/**
+ * A header or property name for matching: letters and digits of any script,
+ * in lower case, without marks and without spaces or punctuation.
+ * `Customer ID`, `customer_id` and `CustomerId` are all `customerid`.
+ */
+export function normalizeName(name: string): string {
+  return name
+    .normalize("NFKD")
+    .replace(/\p{M}/gu, "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, "");
+}

package/templates/rows/samples/expected.json

@@ -0,0 +1,150 @@
+{
+  "q3-customers.csv": {
+    "settings": { "className": "Customer", "key": "Customer ID", "represents": { "class": "Report" } },
+    "rows": 8,
+    "rows_without_key": 1,
+    "values_left_out": 1,
+    "columns": [
+      "Customer ID → customer_id",
+      "Email → email",
+      "Address → address",
+      "City → city",
+      "Customer since → customer_since"
+    ],
+    "columns_left_out": ["Name", "Account manager"],
+    "represents": "Q3 customers",
+    "instances": [
+      {
+        "key": "C-1001",
+        "name": "Harbor Lane Bakery",
+        "row": 2,
+        "properties": {
+          "customer_id": "C-1001",
+          "email": "orders@harborlane.example",
+          "address": "12 Quay Street",
+          "city": "Lisbon",
+          "customer_since": "2021-03-14T00:00:00Z"
+        }
+      },
+      {
+        "key": "C-1002",
+        "name": "Okafor, Ada",
+        "row": 3,
+        "properties": {
+          "customer_id": "C-1002",
+          "email": "ada.okafor@mail.example",
+          "address": "4 Palm Avenue",
+          "city": "Lagos",
+          "customer_since": "2019-11-02T00:00:00Z"
+        }
+      },
+      {
+        "key": "C-1003",
+        "name": "Studio \"Nord\"",
+        "row": 4,
+        "properties": {
+          "customer_id": "C-1003",
+          "email": "hello@studionord.example",
+          "address": "Strandgata 9",
+          "city": "Oslo",
+          "customer_since": "2022-06-30T00:00:00Z"
+        }
+      },
+      {
+        "key": "C-1004",
+        "name": "Meridian Tutoring",
+        "row": 5,
+        "why": "The address holds a line break inside its quotes, so the row spans two lines. 'not a date' does not take the type of customer_since and is left out.",
+        "properties": {
+          "customer_id": "C-1004",
+          "email": "info@meridian.example",
+          "address": "Rua das Flores 21\nFloor 2",
+          "city": "Porto"
+        }
+      },
+      {
+        "key": "C-1005",
+        "name": "Lindqvist Maja",
+        "row": 6,
+        "properties": {
+          "customer_id": "C-1005",
+          "email": "maja@lindqvist.example",
+          "address": "Kungsgatan 3",
+          "city": "Uppsala",
+          "customer_since": "2020-01-20T00:00:00Z"
+        }
+      },
+      {
+        "key": "C-1006",
+        "name": "Calder Repair Shop",
+        "row": 8,
+        "why": "Row 7 is blank. The email cell is empty, so the instance has no email.",
+        "properties": {
+          "customer_id": "C-1006",
+          "address": "1 Mill Lane",
+          "city": "Leeds",
+          "customer_since": "2023-09-05T00:00:00Z"
+        }
+      }
+    ],
+    "leave_out": [{ "row": 9, "why": "The row has no Customer ID: no instance." }]
+  },
+  "q3-orders.tsv": {
+    "settings": { "className": "Order", "key": "Order ID", "represents": { "class": "Report" } },
+    "rows": 4,
+    "rows_without_key": 0,
+    "values_left_out": 1,
+    "represents": "Q3 orders",
+    "instances": [
+      {
+        "key": "O-5001",
+        "name": "O-5001",
+        "row": 2,
+        "properties": {
+          "order_id": "O-5001",
+          "customer_id": "C-1001",
+          "placed_on": "2026-07-03T00:00:00Z",
+          "total": 182.4,
+          "status": "paid"
+        }
+      },
+      {
+        "key": "O-5002",
+        "name": "O-5002",
+        "row": 3,
+        "why": "1250,50 has a decimal comma; the status is quoted.",
+        "properties": {
+          "order_id": "O-5002",
+          "customer_id": "C-1002",
+          "placed_on": "2026-07-19T00:00:00Z",
+          "total": 1250.5,
+          "status": "on hold"
+        }
+      },
+      {
+        "key": "O-5003",
+        "name": "O-5003",
+        "row": 4,
+        "properties": {
+          "order_id": "O-5003",
+          "customer_id": "C-1003",
+          "placed_on": "2026-08-02T00:00:00Z",
+          "total": 64,
+          "status": "paid"
+        }
+      },
+      {
+        "key": "O-5004",
+        "name": "O-5004",
+        "row": 5,
+        "why": "n/a does not take the type of total and is left out.",
+        "properties": {
+          "order_id": "O-5004",
+          "customer_id": "C-1001",
+          "placed_on": "2026-09-28T00:00:00Z",
+          "status": "cancelled"
+        }
+      }
+    ]
+  }
+}

package/templates/rows/samples/properties.json

@@ -0,0 +1,11 @@
+[
+  { "name": "customer_id", "value_type": "keyword" },
+  { "name": "email", "value_type": "keyword" },
+  { "name": "address", "value_type": "text" },
+  { "name": "city", "value_type": "keyword" },
+  { "name": "customer_since", "value_type": "date_time" },
+  { "name": "order_id", "value_type": "keyword" },
+  { "name": "placed_on", "value_type": "date_time" },
+  { "name": "total", "value_type": "f64" },
+  { "name": "status", "value_type": "keyword" }
+]

package/templates/rows/samples/q3-customers.csv

@@ -0,0 +1,10 @@
+Customer ID,Name,Email,Address,City,Customer since,Account manager
+C-1001,Harbor Lane Bakery,orders@harborlane.example,12 Quay Street,Lisbon,2021-03-14,R. Silva
+C-1002,"Okafor, Ada",ada.okafor@mail.example,4 Palm Avenue,Lagos,2019-11-02,T. Bello
+C-1003,"Studio ""Nord""",hello@studionord.example,Strandgata 9,Oslo,2022-06-30,K. Berg
+C-1004,Meridian Tutoring,info@meridian.example,"Rua das Flores 21
+Floor 2",Porto,not a date,R. Silva
+C-1005,Lindqvist Maja,maja@lindqvist.example,Kungsgatan 3,Uppsala,2020-01-20,K. Berg
+
+C-1006,Calder Repair Shop,,1 Mill Lane,Leeds,2023-09-05,T. Bello
+,Walk-in customer,walkin@shop.example,,Leeds,2024-02-11,

package/templates/rows/samples/q3-orders.tsv

@@ -0,0 +1,5 @@
+Order ID	Customer ID	Placed on	Total	Status
+O-5001	C-1001	2026-07-03	182.40	paid
+O-5002	C-1002	2026-07-19	1250,50	"on hold"
+O-5003	C-1003	2026-08-02	64	paid
+O-5004	C-1001	2026-09-28	n/a	cancelled

package/templates/rows/samples/reports.json

@@ -0,0 +1,5 @@
+[
+  { "name": "Q3 customers", "text": "The customer list at the end of the third quarter." },
+  { "name": "Q3 orders", "text": "The orders of the third quarter." },
+  { "name": "Q2 customers", "text": "The customer list at the end of the second quarter." }
+]

package/templates/rows/samples.ts

@@ -0,0 +1,56 @@
+// The samples of rows, for `npm run try`: an invented customer list
+// (samples/q3-customers.csv) and order table (samples/q3-orders.tsv), the
+// properties their columns fill (samples/properties.json), and three
+// reports (samples/reports.json) that the files' names can name.
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { LbbClient, Schemas } from "@littlebigbrain/client";
+import { defaults } from "./workflow.js";
+
+/** The sample files and their media types. */
+export const SAMPLE_FILES = [
+  { name: "q3-customers.csv", contentType: "text/csv" },
+  { name: "q3-orders.tsv", contentType: "text/tab-separated-values" },
+];
+
+/** Add the properties and the reports, then upload the sample files to the client's graph. */
+export async function loadSamples(client: LbbClient, dir: string): Promise<void> {
+  const properties = JSON.parse(await readFile(join(dir, "properties.json"), "utf8")) as {
+    name: string;
+    value_type: string;
+  }[];
+  const reports = JSON.parse(await readFile(join(dir, "reports.json"), "utf8")) as {
+    name: string;
+    text: string;
+  }[];
+  const reportClass = defaults.represents?.class ?? "Report";
+  const ops: Schemas["OntologyEvolveOp"][] = [
+    { op: "add_entity_type", name: reportClass },
+    { op: "add_property", name: "text", value_type: "text" },
+    ...properties.map((p) => ({
+      op: "add_property" as const,
+      name: p.name,
+      value_type: p.value_type,
+    })),
+  ];
+  await client.ontology.evolve({ ops });
+  await client.commit(
+    {
+      entity_properties: reports.map((report) => ({
+        type: reportClass,
+        name: report.name,
+        properties: { text: report.text },
+      })),
+    },
+    { idempotencyKey: "rows-samples" },
+  );
+  console.log(`Committed ${reports.length} ${reportClass} records`);
+  for (const file of SAMPLE_FILES) {
+    const bytes = await readFile(join(dir, file.name));
+    const uploaded = await client.files.upload(new Uint8Array(bytes), {
+      name: file.name,
+      contentType: file.contentType,
+    });
+    console.log(`Uploaded ${file.name} (${uploaded.file.sha256.slice(0, 12)})`);
+  }
+}

package/templates/rows/setup.ts

@@ -0,0 +1,18 @@
+// Create the workflow on your graph: add the class and the relation it
+// writes, then create the named workflow whose content trigger sends it each
+// CSV and TSV upload. Each call is safe to repeat.
+//
+//   LBB_URL=... LBB_API_KEY=... LBB_GRAPH=... npm run setup
+import { LbbClient, lbbActivities } from "@littlebigbrain/client";
+import { ontology, trigger } from "./workflow.js";
+
+const graph = process.env.LBB_GRAPH ?? "main";
+const client = new LbbClient({
+  baseUrl: process.env.LBB_URL ?? "http://127.0.0.1:7400",
+  apiKey: process.env.LBB_API_KEY,
+  graph,
+});
+await lbbActivities(client).evolve(ontology());
+const created = await client.workflows.named.put(trigger());
+console.log(`Created ${created.name} on the graph ${graph}.`);
+console.log(`Serve it with LBB_GRAPH=${graph} npm start.`);

package/templates/rows/try.ts

@@ -0,0 +1,32 @@
+// Try the workflow on its samples: create a graph (default try-rows), add
+// the sample properties and reports, upload the sample files, and create the
+// workflow there. Then serve it with npm start and the same LBB_GRAPH: the
+// content trigger reads the files uploaded before it too.
+//
+//   LBB_URL=... LBB_API_KEY=... LBB_GRAPH=try-rows npm run try
+import { fileURLToPath } from "node:url";
+import { LbbClient, LbbError, lbbActivities } from "@littlebigbrain/client";
+import { loadSamples } from "./samples.js";
+import { ontology, trigger } from "./workflow.js";
+
+const graph = process.env.LBB_GRAPH ?? "try-rows";
+const client = new LbbClient({
+  baseUrl: process.env.LBB_URL ?? "http://127.0.0.1:7400",
+  apiKey: process.env.LBB_API_KEY,
+  graph,
+});
+try {
+  await client.graph(graph).create();
+  console.log(`Created the graph ${graph}`);
+} catch (error) {
+  // 409: the graph exists already.
+  if (!(error instanceof LbbError && error.status === 409)) throw error;
+}
+await lbbActivities(client).evolve(ontology());
+await loadSamples(client, fileURLToPath(new URL("../samples/", import.meta.url)));
+const created = await client.workflows.named.put(trigger());
+console.log(`Created ${created.name} on the graph ${graph}.`);
+console.log(`Serve it with LBB_GRAPH=${graph} npm start, and follow it with:`);
+console.log(
+  `  curl "$LBB_URL/v1/workflows/named/queue?graph=${graph}&name=${created.name}" -H "Authorization: Bearer $LBB_API_KEY"`,
+);

package/templates/rows/worker.ts

@@ -0,0 +1,24 @@
+// Serve the workflow: the worker claims the turns of its content trigger's
+// messages and runs them, one at a time. Run more workers for more at once.
+//
+//   LBB_URL=... LBB_API_KEY=... LBB_GRAPH=... npm start
+import { LbbClient, WorkflowWorker, lbbActivities } from "@littlebigbrain/client";
+import { createWorkflow } from "./workflow.js";
+
+const graph = process.env.LBB_GRAPH ?? "main";
+const client = new LbbClient({
+  baseUrl: process.env.LBB_URL ?? "http://127.0.0.1:7400",
+  apiKey: process.env.LBB_API_KEY,
+  graph,
+});
+// The client reads the files' bytes and the derivations.
+const workflow = createWorkflow(lbbActivities(client), client);
+const stop = new AbortController();
+process.once("SIGINT", () => stop.abort());
+process.once("SIGTERM", () => stop.abort());
+console.log(
+  `Serving ${workflow.name}@${workflow.version} for the graph ${graph}. Stop with Ctrl-C.`,
+);
+await new WorkflowWorker(client, [workflow], {
+  worker: process.env.LBB_WORKER_ID ?? `${workflow.name}-1`,
+}).run({ signal: stop.signal });