@littlebigbrain/create 0.2.0 → 0.3.0

package/README.md

@@ -20,6 +20,7 @@ npm test
 | `ontology-fit` | Proposes the ontology changes the texts of a class support, as review items |
 | `documents-map` | Maps the links of `documents-link` into the graph's own ontology, as fact proposals |
 | `review` | Writes the fact proposals at or above a gate as facts and gives the others to a person |
+| `link` | Connects the mentions of one thing across the records of a graph into one thing, and proposes records of one thing as an identity |
 
 A template id is the id of its starter workflow, with dashes for dots:
 `documents-link` is the code of `documents.link`.
@@ -31,7 +32,8 @@ A template id is the id of its starter workflow, with dashes for dots:
 | `src/workflow.ts` | The workflow, its settings, the triggered workflow that sends it the instances (`trigger()`) and the ontology it writes (`ontology()`) |
 | `src/*.ts` | The template's helpers and `samples.ts`, which puts the samples on a graph |
 | `src/workflow.test.ts`, `src/testing.ts` | The workflow on its samples, with mock models and no server |
-| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try` |
+| `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`) keeps them |
+| `src/grade-run.ts` | `npm run grade`, in a template that grades a run against a gold file (`link`) |
 | `samples/` | The samples and their answer key |
 
 `npm run try` creates a graph (`LBB_GRAPH`, default `try-<template>`), puts

package/lib/create.mjs

@@ -21,6 +21,7 @@ export const TEMPLATES = [
   "review",
   "values-extract",
   "ontology-fit",
+  "link",
 ];
 
 const PACKAGE = resolve(dirname(fileURLToPath(import.meta.url)), "..");
@@ -119,15 +120,23 @@ export async function create({ template, dir, client, root = PACKAGE }) {
   const src = join(dir, "src");
   await mkdir(src, { recursive: true });
 
-  // The template's code, its test and the test support, all under src/.
-  for (const file of await readdir(from)) {
-    if (!file.endsWith(".ts")) continue;
+  // The template's code, its test and the test support, all under src/, the
+  // template's folders (`sources/`) as they are.
+  const code = await templateCode(from);
+  for (const file of code) {
     const text = await readFile(join(from, file), "utf8");
-    await writeFile(join(src, file), text.replaceAll("../_shared/testing.js", "./testing.js"));
+    const up = "../".repeat(file.split("/").length - 1);
+    await mkdir(dirname(join(src, file)), { recursive: true });
+    await writeFile(
+      join(src, file),
+      text.replaceAll(`../${up}_shared/testing.js`, `${up || "./"}testing.js`),
+    );
   }
   await cp(join(templates, "_shared", "testing.ts"), join(src, "testing.ts"));
   await cp(join(from, "samples"), join(dir, "samples"), { recursive: true });
+  // A template's own commands win: a workflow of several parts sets itself up.
   for (const file of ["worker.ts", "setup.ts", "try.ts"]) {
+    if (code.includes(file)) continue;
     const text = await readFile(join(root, "project", file), "utf8");
     await writeFile(join(src, file), text.replaceAll("__TEMPLATE__", template));
   }
@@ -144,6 +153,7 @@ export async function create({ template, dir, client, root = PACKAGE }) {
       try: "tsx src/try.ts",
       test: "tsx --test src/workflow.test.ts",
       typecheck: "tsc --noEmit",
+      ...(code.includes("grade-run.ts") ? { grade: "tsx src/grade-run.ts" } : {}),
     },
     dependencies: { "@littlebigbrain/client": client ?? `^${clientVersion(templates)}` },
     devDependencies: { "@types/node": "^22", tsx: "^4", typescript: "^5.6" },
@@ -209,6 +219,18 @@ ${projectReadme(readme).trim()}
   return listFiles(dir);
 }
 
+/**
+ * The TypeScript files of a template folder, relative, in its folders too;
+ * the samples are copied on their own.
+ * @param {string} from
+ * @returns {Promise<string[]>}
+ */
+async function templateCode(from) {
+  return (await listFiles(from)).filter(
+    (file) => file.endsWith(".ts") && !file.startsWith("samples/"),
+  );
+}
+
 /** Every file under `dir`, relative and sorted. @param {string} dir @returns {Promise<string[]>} */
 async function listFiles(dir, prefix = "") {
   /** @type {string[]} */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@littlebigbrain/create",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Create a project from a little big brain starter workflow template: npm create @littlebigbrain",
   "license": "Apache-2.0",
   "repository": {

package/templates/_shared/testing.ts

@@ -405,7 +405,14 @@ export async function runTurn<S>(
   definition: WorkflowDefinition<S, WorkflowTriggerMessage>,
   state: unknown,
   message: WorkflowTriggerMessage,
-  options: { prefix?: string; saved?: Turn; crashAfter?: number } = {},
+  options: {
+    prefix?: string;
+    saved?: Turn;
+    crashAfter?: number;
+    /** The engine identity that runs the turn (default `starter.test`), and the turn's number. */
+    workflowId?: string;
+    turn?: number;
+  } = {},
 ): Promise<{ result: TurnResult<S>; turn: Turn }> {
   const prefix = options.prefix ?? mockId(JSON.stringify(message));
   const turn: Turn = { message, keys: [], outputs: [], bytes: 0, progress: [] };
@@ -413,9 +420,9 @@ export async function runTurn<S>(
   let ran = 0;
   const ctx = {
     signal: new AbortController().signal,
-    workflowId: "starter.test",
+    workflowId: options.workflowId ?? "starter.test",
     messageId: "starter-test-1",
-    turn: 1,
+    turn: options.turn ?? 1,
     async step<T>(
       key: string,
       effect: (s: {
@@ -480,17 +487,23 @@ export async function runTurn<S>(
 /**
  * Run a message and every continuation it leads to, from the empty state the
  * server creates the instance with. Returns the final state and the turns.
+ * With `workflowId` and `turn`, the turns run as that identity, numbered
+ * from `turn` on.
  */
 export async function runMessage<S>(
   definition: WorkflowDefinition<S, WorkflowTriggerMessage>,
   message: WorkflowTriggerMessage,
   state: unknown = {},
+  options: { workflowId?: string; turn?: number } = {},
 ) {
   const turns: Turn[] = [];
   let next: WorkflowTriggerMessage | undefined = message;
   while (next) {
     if (turns.length > 100) throw new Error("more than 100 turns");
-    const ran: { result: TurnResult<S>; turn: Turn } = await runTurn(definition, state, next);
+    const ran: { result: TurnResult<S>; turn: Turn } = await runTurn(definition, state, next, {
+      ...(options.workflowId ? { workflowId: options.workflowId } : {}),
+      ...(options.turn !== undefined ? { turn: options.turn + turns.length } : {}),
+    });
     turns.push(ran.turn);
     state = ran.result.state;
     next = ran.result.continuation?.message;

package/templates/link/README.md

@@ -0,0 +1,203 @@
+# link
+
+Connects the mentions of one thing across the records of a graph. A
+contact, a lead, a deal and a note can each name one person in a different
+way: `Mira Tannenholt-Ruiz`, `Mira Ruiz`, `M. Tannenholt`, `Mimi T.`. The
+workflow groups those mentions into one `LinkedThing`. The links are
+references across the graph, and later checks compare what each source says
+about the thing.
+
+This is phase L1 of the design
+[general linking](../../../docs/architecture/general-linking.md): identities
+on records. Documents (pages) come in phase L2.
+
+## What it writes
+
+| Term | Kind | Holds |
+| --- | --- | --- |
+| `Mention` | class | `span`, `role` (`self` or `refers`), `kind`, `field`, `offset`, `length`, `block_key` (several), `window` (the sentence of a span of text); the links and constraints `linked_to`, `must_link` and `cannot_link` |
+| `LinkedThing` | class | its name (the anchor's name), `kind`, `anchor_key`, `alias` (several: the other names its members use) |
+| `MENTIONED_IN` | relation, mention to record | the record that holds the mention |
+| `SAME_THING_AS` | relation, mention to mention | a decided or accepted pair, with its confidence and its evidence (the sentence and the record's name) |
+| `REFERS_TO` | relation, mention to `LinkedThing` | the thing the mention is in |
+| `HAS_RECORD` | relation, `LinkedThing` to record | the records whose names and keys are in the thing |
+| `MERGED_INTO` | relation, `LinkedThing` to `LinkedThing` | where an old thing is now: a thing whose anchor changed, or that merged |
+| `SAME_AS` | relation, record to record | two records of one thing: a person writes it when they accept an identity |
+
+A record's own names and keys are `self` mentions. Each record anchors a
+thing. A span in a text field that names something is a `refers` mention.
+
+## How it links
+
+1. **Mentions.** A `name` or `key:*` field gives one `self` mention per
+   value. A `text` field gives `refers` mentions: first the gazetteer pass,
+   which keeps each span whose name or key a mention of the graph has; then
+   the spans written like a name or a key (`M. Tannenholt`, an email).
+2. **Block keys.** Each mention gets normal forms of its name or key as
+   graph facts (`block_key`): the sorted name, the initials forms, the name
+   parts and their sound, an email, a web address, a code. A key that more
+   than 200 mentions hold is a stop key and finds nothing.
+3. **Candidates.** The mentions that share a key, and the things of the
+   records within two hops (the deal that a note is about). At most 30, the
+   ones with a strong key first.
+4. **Decision.** See the next section.
+5. **Things.** The things identity groups the decided pairs. A person's
+   decisions are constraints. Two records are never one thing until a person
+   accepts their identity.
+
+The thing's IRI is a hash of the graph and its anchor's key. The anchor is a
+record's name first, then any other mention. So the same inputs give the
+same IRIs. A thing whose anchor changes keeps one statement, `MERGED_INTO`
+the thing that holds its members now.
+
+## Model reads of record values
+
+`modelsReadRecords` is off by default (general linking 12, decision 1).
+
+- **Off.** No model reads a record value. A strong key (an email, a web
+  address, a phone number, a handle, a code) links at confidence 1. Every
+  other candidate goes to a person, grouped by thing.
+- **On.** The decision model reads the names, keys and context fields of
+  the records. It answers a kind question and a names question per mention,
+  and a same-record question per pair of records. A link at or above `gate`
+  (0.8) is written; a link below it goes to a person. The reasoning model
+  reads only the text fields that opt in: `{ "role": "text", "extract": true }`
+  (decision 8).
+
+Every model call has an effect id made from its request. The same question
+gets its first answer from the effect cache, at no cost.
+
+## Review items
+
+| Item | One per | Accept writes |
+| --- | --- | --- |
+| Link (`starter:<name>:link:…`) | Thing | The kept `SAME_THING_AS` rows. A row left out keeps its mention apart from the thing. |
+| No link (`starter:<name>:nolink:…`) | Thing of the best candidate | The kept rows: the model chose no thing, and a person disagrees. |
+| Identity (`starter:<name>:identity:…`) | Group of 2 to 10 records | `SAME_AS` between the records, and `SAME_THING_AS` between their names. |
+
+A person always accepts an identity (decision 2). The `identityGate` (0.99)
+only orders and groups the identity items (`mapping.identity.above_gate`).
+Each item has the origin `starter:<name>` with the label `link`, so the
+inbox labels and patterns of the producer `link` read it.
+
+A person's decisions feed back. A kept row is a must-link. A row left out,
+or a dismissed item, is a cannot-link. No model answer joins what a
+cannot-link keeps apart.
+
+## Settings
+
+| Setting | Default | Meaning |
+| --- | --- | --- |
+| `name` | `link` | The named workflows are `<name>-<class>` and `<name>-reviews`; the things identity is `<name>-things` |
+| `fields` | the samples' CRM | The role of each field of each class (below) |
+| `gate` | 0.8 | A link at or above it is written, with model reads on |
+| `identityGate` | 0.99 | Orders and groups the identity items |
+| `modelsReadRecords` | `false` | Whether models may read record values |
+| `shards` | 4 | Source shards of each named workflow |
+| `sourcesPerTurn` | 16 | Records per message |
+| `textPerTurn` | 8,000 | Characters of text per turn; the rest continues in a new turn |
+
+| Field role | Gives |
+| --- | --- |
+| `name` | One `self` mention per value |
+| `name:given`, `name:family` | One `self` mention of the two joined (`first_name+last_name`) |
+| `key:email`, `key:phone`, `key:web`, `key:handle`, `key:code` | One `self` mention per value, with a strong key |
+| `text`, `{ "role": "text", "extract": true }` | `refers` mentions found in the text |
+| `context` | No mention; read for the decisions |
+
+A field named `name` (or `label`) is the record's display name, which the
+published graph holds as `rdfs:label`; the workflow watches it as `label`.
+Any other field is a property of the record.
+
+## Run it
+
+```sh
+npm create @littlebigbrain -- --template link my-links
+cd my-links && npm install
+npm test                                           # the workflow on the samples, with mock models
+LBB_GRAPH=try-link npm run try                     # the samples on a graph of your stack
+LBB_GRAPH=try-link npm start                       # serve it
+LBB_GRAPH=try-link npm run grade -- --gold samples/gold.json
+LBB_GRAPH=crm npm run setup && npm start           # your graph
+```
+
+`npm run setup` creates the classes and properties, one named workflow per
+class of the settings, the named workflow that watches accepted items, and
+the things identity. It is safe to repeat. The workflow is for a records
+graph. Do not run it on a graph where `documents.link` runs: both write
+`MENTIONED_IN` and `REFERS_TO`.
+
+## Grade a run
+
+`npm run grade` (`src/grade-run.ts`) prints the metrics of general linking
+9.2: link precision at the gate and its 95% Wilson interval, pairwise
+precision and recall over things, cluster purity and completeness
+(B-cubed), the review share, and identity precision and recall.
+
+```sh
+LBB_URL=... LBB_API_KEY=... LBB_GRAPH=crm npm run grade -- --gold .lbb-datasets/link-eval/gold.json --save run.json
+npm run grade -- --gold .lbb-datasets/link-eval/gold.json --run run.json
+```
+
+The gold file (`lbb.link.gold.v1`) lists the groups. Each group is one real
+thing, with the mentions that name it:
+
+```json
+{
+  "format": "lbb.link.gold.v1",
+  "complete": false,
+  "records": { "contact-1": "https://littlebigbrain.com/e/…" },
+  "groups": [
+    { "id": "mira", "kind": "person", "mentions": [
+      { "record": "contact-1", "field": "name", "span": "Mira Tannenholt-Ruiz" },
+      { "record": "https://littlebigbrain.com/e/…", "field": "body", "span": "Mimi T.", "occurrence": 0 }
+    ] }
+  ]
+}
+```
+
+A mention names its record (an IRI, or a name of `records`), its field
+(`first_name+last_name` for a joined name), its span as the field writes it,
+and which equal span of the field it is (`occurrence`, default 0). With
+`complete: true`, a mention that the gold does not list is a thing of its
+own. Without it, a link to such a mention counts apart as unjudged. Keep a
+gold file of real records out of git, under `.lbb-datasets/`.
+
+## The samples
+
+`samples/records.json` is an invented CRM: 7 organizations, 12 contacts, 5
+leads, 7 deals and 11 notes. Every name, company and address is invented.
+The names vary: a nickname (`Bob` for `Robert`), initials (`M. Tannenholt`,
+`F. Okafor`), accents and transliterations (`Jürgen`, `Juergen`, `Jurgen`;
+`Søren`, `Soren`; `Zoë`, `Zoe`), name order (`Wren, Tobias`), a typo
+(`Ramaswamey`), a person known only by an email, a surname that two people
+share (`Okafor`), two people of one name at two organizations (`Alex
+Brandt`), and company domains. `samples/gold.json` holds the gold groups.
+
+## How the template differs from the design
+
+- A named workflow takes one watch trigger in this release, so each class is
+  a named workflow of its own. All of them run the workflow type `link`.
+- The candidates by the name index and by meaning wait for work tokens that
+  reach those routes (N3). The keys and the neighbours find the candidates.
+- A source turn sends its decided pairs in messages of at most 24,000
+  bytes; a larger message goes in parts (`.1`, `.2`, …) until blobs (G11).
+- The things identity is an engine identity that `npm run setup` creates.
+  It reads the review items it filed to learn a person's decisions.
+
+## Files
+
+| File | Holds |
+| --- | --- |
+| `workflow.ts` | The workflow, the named workflows (`workflows()`), the ontology, the ports (`linkPorts()`) and `setup()` |
+| `settings.ts` | The settings and the names of the parts, and the decided message |
+| `names.ts` | Normal forms and block keys |
+| `candidates.ts` | The candidate reads, the scoring and the bounds |
+| `decide.ts` | The questions, the packing, the link rule and the rules without models |
+| `things.ts` | The grouping with constraints, anchors, IRIs, merges and splits |
+| `sources/records.ts` | Field roles, and the mentions of record fields |
+| `source-turn.ts`, `things-turn.ts` | The turns of a source shard and of the things identity |
+| `items.ts`, `reads.ts` | The review items, and the reads of the graph |
+| `grade.ts`, `grade-run.ts` | The grading, and its command |
+| `fixture.ts` | The test support: the graph in memory, mock models and ports, a reviewer that follows a gold file |
+| `setup.ts`, `worker.ts`, `try.ts`, `samples.ts` | The commands |