@littlebigbrain/create 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +7 -2
  2. package/lib/create.mjs +28 -4
  3. package/package.json +1 -1
  4. package/templates/_shared/testing.ts +185 -5
  5. package/templates/link/README.md +203 -0
  6. package/templates/link/candidates.ts +432 -0
  7. package/templates/link/decide.test.ts +368 -0
  8. package/templates/link/decide.ts +340 -0
  9. package/templates/link/fixture.ts +727 -0
  10. package/templates/link/grade-run.ts +53 -0
  11. package/templates/link/grade.test.ts +113 -0
  12. package/templates/link/grade.ts +335 -0
  13. package/templates/link/items.ts +333 -0
  14. package/templates/link/names.test.ts +260 -0
  15. package/templates/link/names.ts +590 -0
  16. package/templates/link/reads.ts +302 -0
  17. package/templates/link/samples/gold.json +108 -0
  18. package/templates/link/samples/records.json +48 -0
  19. package/templates/link/samples.ts +64 -0
  20. package/templates/link/settings.ts +162 -0
  21. package/templates/link/setup.ts +18 -0
  22. package/templates/link/source-turn.ts +894 -0
  23. package/templates/link/sources/records.ts +429 -0
  24. package/templates/link/things-turn.ts +984 -0
  25. package/templates/link/things.test.ts +204 -0
  26. package/templates/link/things.ts +426 -0
  27. package/templates/link/try.ts +28 -0
  28. package/templates/link/worker.ts +24 -0
  29. package/templates/link/workflow.test.ts +484 -0
  30. package/templates/link/workflow.ts +202 -0
  31. package/templates/rows/README.md +150 -0
  32. package/templates/rows/csv.ts +260 -0
  33. package/templates/rows/samples/expected.json +150 -0
  34. package/templates/rows/samples/properties.json +11 -0
  35. package/templates/rows/samples/q3-customers.csv +10 -0
  36. package/templates/rows/samples/q3-orders.tsv +5 -0
  37. package/templates/rows/samples/reports.json +5 -0
  38. package/templates/rows/samples.ts +56 -0
  39. package/templates/rows/setup.ts +18 -0
  40. package/templates/rows/try.ts +32 -0
  41. package/templates/rows/worker.ts +24 -0
  42. package/templates/rows/workflow.test.ts +541 -0
  43. package/templates/rows/workflow.ts +657 -0
  44. package/templates/sections/README.md +118 -0
  45. package/templates/sections/markdown.ts +121 -0
  46. package/templates/sections/samples/expected.json +82 -0
  47. package/templates/sections/samples/guides.json +4 -0
  48. package/templates/sections/samples/support-handbook.md +34 -0
  49. package/templates/sections/samples.ts +41 -0
  50. package/templates/sections/setup.ts +18 -0
  51. package/templates/sections/try.ts +32 -0
  52. package/templates/sections/worker.ts +24 -0
  53. package/templates/sections/workflow.test.ts +265 -0
  54. package/templates/sections/workflow.ts +433 -0
  55. package/templates/versions.json +1 -1
package/README.md CHANGED
@@ -20,9 +20,13 @@ npm test
20
20
  | `ontology-fit` | Proposes the ontology changes the texts of a class support, as review items |
21
21
  | `documents-map` | Maps the links of `documents-link` into the graph's own ontology, as fact proposals |
22
22
  | `review` | Writes the fact proposals at or above a gate as facts and gives the others to a person |
23
+ | `link` | Connects the mentions of one thing across the records of a graph into one thing, and proposes records of one thing as an identity |
24
+ | `rows` | Reads each CSV or TSV file into one instance per row, each with an edge to the file whose evidence is the row |
25
+ | `sections` | Reads each Markdown file into its sections, each with an edge to its parent whose evidence is its lines |
23
26
 
24
27
  A template id is the id of its starter workflow, with dashes for dots:
25
- `documents-link` is the code of `documents.link`.
28
+ `documents-link` is the code of `documents.link`. `link`, `rows` and
29
+ `sections` have no LBB-run version.
26
30
 
27
31
  ## The project
28
32
 
@@ -31,7 +35,8 @@ A template id is the id of its starter workflow, with dashes for dots:
31
35
  | `src/workflow.ts` | The workflow, its settings, the triggered workflow that sends it the instances (`trigger()`) and the ontology it writes (`ontology()`) |
32
36
  | `src/*.ts` | The template's helpers and `samples.ts`, which puts the samples on a graph |
33
37
  | `src/workflow.test.ts`, `src/testing.ts` | The workflow on its samples, with mock models and no server |
34
- | `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try` |
38
+ | `src/worker.ts`, `src/setup.ts`, `src/try.ts` | `npm start`, `npm run setup` and `npm run try`; a template that has its own (`link`, `rows`, `sections`) keeps them |
39
+ | `src/grade-run.ts` | `npm run grade`, in a template that grades a run against a gold file (`link`) |
35
40
  | `samples/` | The samples and their answer key |
36
41
 
37
42
  `npm run try` creates a graph (`LBB_GRAPH`, default `try-<template>`), puts
package/lib/create.mjs CHANGED
@@ -21,6 +21,9 @@ export const TEMPLATES = [
21
21
  "review",
22
22
  "values-extract",
23
23
  "ontology-fit",
24
+ "link",
25
+ "rows",
26
+ "sections",
24
27
  ];
25
28
 
26
29
  const PACKAGE = resolve(dirname(fileURLToPath(import.meta.url)), "..");
@@ -119,15 +122,23 @@ export async function create({ template, dir, client, root = PACKAGE }) {
119
122
  const src = join(dir, "src");
120
123
  await mkdir(src, { recursive: true });
121
124
 
122
- // The template's code, its test and the test support, all under src/.
123
- for (const file of await readdir(from)) {
124
- if (!file.endsWith(".ts")) continue;
125
+ // The template's code, its test and the test support, all under src/, the
126
+ // template's folders (`sources/`) as they are.
127
+ const code = await templateCode(from);
128
+ for (const file of code) {
125
129
  const text = await readFile(join(from, file), "utf8");
126
- await writeFile(join(src, file), text.replaceAll("../_shared/testing.js", "./testing.js"));
130
+ const up = "../".repeat(file.split("/").length - 1);
131
+ await mkdir(dirname(join(src, file)), { recursive: true });
132
+ await writeFile(
133
+ join(src, file),
134
+ text.replaceAll(`../${up}_shared/testing.js`, `${up || "./"}testing.js`),
135
+ );
127
136
  }
128
137
  await cp(join(templates, "_shared", "testing.ts"), join(src, "testing.ts"));
129
138
  await cp(join(from, "samples"), join(dir, "samples"), { recursive: true });
139
+ // A template's own commands win: a workflow of several parts sets itself up.
130
140
  for (const file of ["worker.ts", "setup.ts", "try.ts"]) {
141
+ if (code.includes(file)) continue;
131
142
  const text = await readFile(join(root, "project", file), "utf8");
132
143
  await writeFile(join(src, file), text.replaceAll("__TEMPLATE__", template));
133
144
  }
@@ -144,6 +155,7 @@ export async function create({ template, dir, client, root = PACKAGE }) {
144
155
  try: "tsx src/try.ts",
145
156
  test: "tsx --test src/workflow.test.ts",
146
157
  typecheck: "tsc --noEmit",
158
+ ...(code.includes("grade-run.ts") ? { grade: "tsx src/grade-run.ts" } : {}),
147
159
  },
148
160
  dependencies: { "@littlebigbrain/client": client ?? `^${clientVersion(templates)}` },
149
161
  devDependencies: { "@types/node": "^22", tsx: "^4", typescript: "^5.6" },
@@ -209,6 +221,18 @@ ${projectReadme(readme).trim()}
209
221
  return listFiles(dir);
210
222
  }
211
223
 
224
+ /**
225
+ * The TypeScript files of a template folder, relative, in its folders too;
226
+ * the samples are copied on their own.
227
+ * @param {string} from
228
+ * @returns {Promise<string[]>}
229
+ */
230
+ async function templateCode(from) {
231
+ return (await listFiles(from)).filter(
232
+ (file) => file.endsWith(".ts") && !file.startsWith("samples/"),
233
+ );
234
+ }
235
+
212
236
  /** Every file under `dir`, relative and sorted. @param {string} dir @returns {Promise<string[]>} */
213
237
  async function listFiles(dir, prefix = "") {
214
238
  /** @type {string[]} */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@littlebigbrain/create",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "description": "Create a project from a little big brain starter workflow template: npm create @littlebigbrain",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -342,6 +342,150 @@ export class MockActivities implements LbbActivities {
342
342
  }
343
343
  }
344
344
 
345
+ // ---- files and derivations ---------------------------------------------------------
346
+
347
+ /** A file the mock holds: its bytes, name, media type and `Document` IRI. */
348
+ export interface MockFile {
349
+ sha256: string;
350
+ name: string;
351
+ mediaType: string;
352
+ document: string;
353
+ bytes: Uint8Array;
354
+ }
355
+
356
+ /**
357
+ * The files and the derivations of a server, for the templates that read
358
+ * files: `files.content` and `workflows.derivations.check` / `record`, the
359
+ * shape of an `LbbClient`. `upload` puts the file's `Document` in `graph`
360
+ * with its name, SHA-256 and media type, as the upload route does.
361
+ */
362
+ export class MockContent {
363
+ readonly held = new Map<string, MockFile>();
364
+ readonly recorded = new Map<string, Schemas["Derivation"]>();
365
+ /** Reads of a file's bytes. */
366
+ reads = 0;
367
+ /** Derivation checks. */
368
+ checks = 0;
369
+
370
+ constructor(readonly graph: MemoryGraph) {}
371
+
372
+ /** Hold a file and write its `Document`. */
373
+ upload(name: string, body: string | Uint8Array, mediaType: string): MockFile {
374
+ const bytes = typeof body === "string" ? new TextEncoder().encode(body) : body;
375
+ const sha256 = createHash("sha256").update(bytes).digest("hex");
376
+ const doc = this.graph.entity({ type: "Document", name, key: `file:${sha256}` });
377
+ doc.name = name;
378
+ Object.assign(doc.properties, {
379
+ name,
380
+ file_sha256: sha256,
381
+ content_type: mediaType,
382
+ file_bytes: bytes.length,
383
+ });
384
+ const file = { sha256, name, mediaType, document: doc.iri, bytes };
385
+ this.held.set(sha256, file);
386
+ return file;
387
+ }
388
+
389
+ /** The byte range of each read, `null` for a whole file, in order. */
390
+ readonly ranges: Array<{ start: number; end: number | null } | null> = [];
391
+ /** The bytes all reads answered. */
392
+ bytesRead = 0;
393
+
394
+ /**
395
+ * A file's bytes, or with `range` bytes `start` to `end` (both included),
396
+ * as `GET /v1/files/content` answers: a range past the end ends at the
397
+ * end, and a range from the end on (a file without bytes too) throws
398
+ * `LbbError` 416 `range_not_satisfiable`.
399
+ */
400
+ readonly files = {
401
+ content: async (
402
+ sha256: string,
403
+ options: { range?: { start: number; end?: number } } = {},
404
+ ): Promise<Uint8Array> => {
405
+ this.reads += 1;
406
+ const file = this.held.get(sha256);
407
+ if (!file)
408
+ throw new LbbError(404, "{}", { code: "file_not_found", message: `no file ${sha256}` });
409
+ const range = options.range;
410
+ this.ranges.push(range ? { start: range.start, end: range.end ?? null } : null);
411
+ if (!range) {
412
+ this.bytesRead += file.bytes.length;
413
+ return file.bytes;
414
+ }
415
+ if (range.start >= file.bytes.length)
416
+ throw new LbbError(416, "{}", {
417
+ code: "range_not_satisfiable",
418
+ message: `no byte of the file from ${range.start} on`,
419
+ });
420
+ const end = Math.min(range.end ?? file.bytes.length - 1, file.bytes.length - 1);
421
+ const bytes = file.bytes.slice(range.start, end + 1);
422
+ this.bytesRead += bytes.length;
423
+ return bytes;
424
+ },
425
+ };
426
+
427
+ readonly workflows = {
428
+ derivations: {
429
+ check: async (key: Schemas["DerivationKey"]): Promise<Schemas["DerivationCheckResponse"]> => {
430
+ this.checks += 1;
431
+ const derivation = this.recorded.get(derivationName(key));
432
+ return derivation ? { exists: true, derivation } : { exists: false };
433
+ },
434
+ record: async (
435
+ request: Schemas["DerivationRecordRequest"],
436
+ ): Promise<Schemas["DerivationRecordResponse"]> => {
437
+ const name = derivationName(request);
438
+ const kept = this.recorded.get(name);
439
+ if (kept) return { recorded: false, derivation: kept };
440
+ const outputs = JSON.stringify(request.outputs ?? null);
441
+ if (outputs.length > 16 * 1024)
442
+ throw new LbbError(400, "{}", {
443
+ code: "derivation_invalid",
444
+ message: "outputs over 16 KiB",
445
+ });
446
+ const derivation = {
447
+ sha256: request.sha256,
448
+ workflow_type: request.workflow_type,
449
+ version: request.version,
450
+ settings_digest: request.settings_digest,
451
+ run_id: request.run_id ?? null,
452
+ outputs: JSON.parse(outputs) as unknown,
453
+ recorded_at: "2026-10-15T09:00:00Z",
454
+ };
455
+ this.recorded.set(name, derivation);
456
+ return { recorded: true, derivation };
457
+ },
458
+ },
459
+ };
460
+ }
461
+
462
+ const derivationName = (key: Schemas["DerivationKey"]) =>
463
+ `${key.workflow_type}@${key.version}/${key.settings_digest}/${key.sha256}`;
464
+
465
+ /** The message a content trigger sends for one upload: its `Document`, with origin `content`. */
466
+ export function contentMessage(
467
+ workflow: string,
468
+ file: Pick<MockFile, "sha256" | "mediaType" | "document">,
469
+ ): WorkflowTriggerMessage {
470
+ return {
471
+ type: "instances",
472
+ workflow,
473
+ graph: "samples",
474
+ as_of_commit_seq: 7,
475
+ reason: "changed",
476
+ instances: [file.document],
477
+ removed: [],
478
+ origin: {
479
+ kind: "content",
480
+ trigger: workflow,
481
+ sha256: file.sha256,
482
+ media_type: file.mediaType,
483
+ uploaded_at_ms: 1791967651158,
484
+ requests: [`file-upload:${file.sha256}`],
485
+ },
486
+ };
487
+ }
488
+
345
489
  function assertNever(what: string): never {
346
490
  throw new Error(`the test gives the mock no ${what} rule`);
347
491
  }
@@ -395,6 +539,27 @@ export class Crash extends Error {
395
539
 
396
540
  const STEP_KEY = /^[A-Za-z0-9_.-]{1,128}$/;
397
541
 
542
+ /**
543
+ * A workflow value as JSON, as the SDK's worker writes it: a value JSON
544
+ * cannot hold (undefined, a function, a number that is not finite) fails
545
+ * the turn there, so it fails here too.
546
+ */
547
+ function workflowJson(value: unknown, where: string): string {
548
+ return JSON.stringify(value, (key, v: unknown) => {
549
+ const bad =
550
+ v === undefined ||
551
+ typeof v === "function" ||
552
+ typeof v === "symbol" ||
553
+ typeof v === "bigint" ||
554
+ (typeof v === "number" && !Number.isFinite(v));
555
+ if (bad)
556
+ throw new Error(
557
+ `Workflow values must be JSON serializable (use null for no result): ${where}${key ? `.${key}` : ""} is ${typeof v}`,
558
+ );
559
+ return v;
560
+ });
561
+ }
562
+
398
563
  /**
399
564
  * Run one turn of `definition` as the workflow engine does: each step runs
400
565
  * once and its output is saved; with `saved`, the saved steps replay without
@@ -405,7 +570,14 @@ export async function runTurn<S>(
405
570
  definition: WorkflowDefinition<S, WorkflowTriggerMessage>,
406
571
  state: unknown,
407
572
  message: WorkflowTriggerMessage,
408
- options: { prefix?: string; saved?: Turn; crashAfter?: number } = {},
573
+ options: {
574
+ prefix?: string;
575
+ saved?: Turn;
576
+ crashAfter?: number;
577
+ /** The engine identity that runs the turn (default `starter.test`), and the turn's number. */
578
+ workflowId?: string;
579
+ turn?: number;
580
+ } = {},
409
581
  ): Promise<{ result: TurnResult<S>; turn: Turn }> {
410
582
  const prefix = options.prefix ?? mockId(JSON.stringify(message));
411
583
  const turn: Turn = { message, keys: [], outputs: [], bytes: 0, progress: [] };
@@ -413,9 +585,9 @@ export async function runTurn<S>(
413
585
  let ran = 0;
414
586
  const ctx = {
415
587
  signal: new AbortController().signal,
416
- workflowId: "starter.test",
588
+ workflowId: options.workflowId ?? "starter.test",
417
589
  messageId: "starter-test-1",
418
- turn: 1,
590
+ turn: options.turn ?? 1,
419
591
  async step<T>(
420
592
  key: string,
421
593
  effect: (s: {
@@ -450,7 +622,7 @@ export async function runTurn<S>(
450
622
  heartbeat: async () => {},
451
623
  progress: (phase, counts) => turn.progress.push(progressCall(key, phase, counts)),
452
624
  });
453
- const json = JSON.stringify(output ?? null);
625
+ const json = workflowJson(output, `step "${key}"`);
454
626
  if (json.length > 64 * 1024)
455
627
  throw new Error(`step ${key} saves ${json.length} bytes, more than 64 KiB`);
456
628
  turn.bytes += json.length;
@@ -474,23 +646,31 @@ export async function runTurn<S>(
474
646
  }),
475
647
  } as unknown as WorkflowContext;
476
648
  const result = (await definition.execute(ctx, state, message)) as TurnResult<S>;
649
+ workflowJson(result.state, "state");
650
+ if (result.continuation) workflowJson(result.continuation, "continuation");
477
651
  return { result, turn };
478
652
  }
479
653
 
480
654
  /**
481
655
  * Run a message and every continuation it leads to, from the empty state the
482
656
  * server creates the instance with. Returns the final state and the turns.
657
+ * With `workflowId` and `turn`, the turns run as that identity, numbered
658
+ * from `turn` on.
483
659
  */
484
660
  export async function runMessage<S>(
485
661
  definition: WorkflowDefinition<S, WorkflowTriggerMessage>,
486
662
  message: WorkflowTriggerMessage,
487
663
  state: unknown = {},
664
+ options: { workflowId?: string; turn?: number } = {},
488
665
  ) {
489
666
  const turns: Turn[] = [];
490
667
  let next: WorkflowTriggerMessage | undefined = message;
491
668
  while (next) {
492
669
  if (turns.length > 100) throw new Error("more than 100 turns");
493
- const ran: { result: TurnResult<S>; turn: Turn } = await runTurn(definition, state, next);
670
+ const ran: { result: TurnResult<S>; turn: Turn } = await runTurn(definition, state, next, {
671
+ ...(options.workflowId ? { workflowId: options.workflowId } : {}),
672
+ ...(options.turn !== undefined ? { turn: options.turn + turns.length } : {}),
673
+ });
494
674
  turns.push(ran.turn);
495
675
  state = ran.result.state;
496
676
  next = ran.result.continuation?.message;
@@ -0,0 +1,203 @@
1
+ # link
2
+
3
+ Connects the mentions of one thing across the records of a graph. A
4
+ contact, a lead, a deal and a note can each name one person in a different
5
+ way: `Mira Tannenholt-Ruiz`, `Mira Ruiz`, `M. Tannenholt`, `Mimi T.`. The
6
+ workflow groups those mentions into one `LinkedThing`. The links are
7
+ references across the graph, and later checks compare what each source says
8
+ about the thing.
9
+
10
+ This is phase L1 of the design
11
+ [general linking](../../../docs/architecture/general-linking.md): identities
12
+ on records. Documents (pages) come in phase L2.
13
+
14
+ ## What it writes
15
+
16
+ | Term | Kind | Holds |
17
+ | --- | --- | --- |
18
+ | `Mention` | class | `span`, `role` (`self` or `refers`), `kind`, `field`, `offset`, `length`, `block_key` (several), `window` (the sentence of a span of text); the links and constraints `linked_to`, `must_link` and `cannot_link` |
19
+ | `LinkedThing` | class | its name (the anchor's name), `kind`, `anchor_key`, `alias` (several: the other names its members use) |
20
+ | `MENTIONED_IN` | relation, mention to record | the record that holds the mention |
21
+ | `SAME_THING_AS` | relation, mention to mention | a decided or accepted pair, with its confidence and its evidence (the sentence and the record's name) |
22
+ | `REFERS_TO` | relation, mention to `LinkedThing` | the thing the mention is in |
23
+ | `HAS_RECORD` | relation, `LinkedThing` to record | the records whose names and keys are in the thing |
24
+ | `MERGED_INTO` | relation, `LinkedThing` to `LinkedThing` | where an old thing is now: a thing whose anchor changed, or that merged |
25
+ | `SAME_AS` | relation, record to record | two records of one thing: a person writes it when they accept an identity |
26
+
27
+ A record's own names and keys are `self` mentions. Each record anchors a
28
+ thing. A span in a text field that names something is a `refers` mention.
29
+
30
+ ## How it links
31
+
32
+ 1. **Mentions.** A `name` or `key:*` field gives one `self` mention per
33
+ value. A `text` field gives `refers` mentions: first the gazetteer pass,
34
+ which keeps each span whose name or key a mention of the graph has; then
35
+ the spans written like a name or a key (`M. Tannenholt`, an email).
36
+ 2. **Block keys.** Each mention gets normal forms of its name or key as
37
+ graph facts (`block_key`): the sorted name, the initials forms, the name
38
+ parts and their sound, an email, a web address, a code. A key that more
39
+ than 200 mentions hold is a stop key and finds nothing.
40
+ 3. **Candidates.** The mentions that share a key, and the things of the
41
+ records within two hops (the deal that a note is about). At most 30, the
42
+ ones with a strong key first.
43
+ 4. **Decision.** See the next section.
44
+ 5. **Things.** The things identity groups the decided pairs. A person's
45
+ decisions are constraints. Two records are never one thing until a person
46
+ accepts their identity.
47
+
48
+ The thing's IRI is a hash of the graph and its anchor's key. The anchor is a
49
+ record's name first, then any other mention. So the same inputs give the
50
+ same IRIs. A thing whose anchor changes keeps one statement, `MERGED_INTO`
51
+ the thing that holds its members now.
52
+
53
+ ## Model reads of record values
54
+
55
+ `modelsReadRecords` is off by default (general linking 12, decision 1).
56
+
57
+ - **Off.** No model reads a record value. A strong key (an email, a web
58
+ address, a phone number, a handle, a code) links at confidence 1. Every
59
+ other candidate goes to a person, grouped by thing.
60
+ - **On.** The decision model reads the names, keys and context fields of
61
+ the records. It answers a kind question and a names question per mention,
62
+ and a same-record question per pair of records. A link at or above `gate`
63
+ (0.8) is written; a link below it goes to a person. The reasoning model
64
+ reads only the text fields that opt in: `{ "role": "text", "extract": true }`
65
+ (decision 8).
66
+
67
+ Every model call has an effect id made from its request. The same question
68
+ gets its first answer from the effect cache, at no cost.
69
+
70
+ ## Review items
71
+
72
+ | Item | One per | Accept writes |
73
+ | --- | --- | --- |
74
+ | Link (`starter:<name>:link:…`) | Thing | The kept `SAME_THING_AS` rows. A row left out keeps its mention apart from the thing. |
75
+ | No link (`starter:<name>:nolink:…`) | Thing of the best candidate | The kept rows: the model chose no thing, and a person disagrees. |
76
+ | Identity (`starter:<name>:identity:…`) | Group of 2 to 10 records | `SAME_AS` between the records, and `SAME_THING_AS` between their names. |
77
+
78
+ A person always accepts an identity (decision 2). The `identityGate` (0.99)
79
+ only orders and groups the identity items (`mapping.identity.above_gate`).
80
+ Each item has the origin `starter:<name>` with the label `link`, so the
81
+ inbox labels and patterns of the producer `link` read it.
82
+
83
+ A person's decisions feed back. A kept row is a must-link. A row left out,
84
+ or a dismissed item, is a cannot-link. No model answer joins what a
85
+ cannot-link keeps apart.
86
+
87
+ ## Settings
88
+
89
+ | Setting | Default | Meaning |
90
+ | --- | --- | --- |
91
+ | `name` | `link` | The named workflows are `<name>-<class>` and `<name>-reviews`; the things identity is `<name>-things` |
92
+ | `fields` | the samples' CRM | The role of each field of each class (below) |
93
+ | `gate` | 0.8 | A link at or above it is written, with model reads on |
94
+ | `identityGate` | 0.99 | Orders and groups the identity items |
95
+ | `modelsReadRecords` | `false` | Whether models may read record values |
96
+ | `shards` | 4 | Source shards of each named workflow |
97
+ | `sourcesPerTurn` | 16 | Records per message |
98
+ | `textPerTurn` | 8,000 | Characters of text per turn; the rest continues in a new turn |
99
+
100
+ | Field role | Gives |
101
+ | --- | --- |
102
+ | `name` | One `self` mention per value |
103
+ | `name:given`, `name:family` | One `self` mention of the two joined (`first_name+last_name`) |
104
+ | `key:email`, `key:phone`, `key:web`, `key:handle`, `key:code` | One `self` mention per value, with a strong key |
105
+ | `text`, `{ "role": "text", "extract": true }` | `refers` mentions found in the text |
106
+ | `context` | No mention; read for the decisions |
107
+
108
+ A field named `name` (or `label`) is the record's display name, which the
109
+ published graph holds as `rdfs:label`; the workflow watches it as `label`.
110
+ Any other field is a property of the record.
111
+
112
+ ## Run it
113
+
114
+ ```sh
115
+ npm create @littlebigbrain -- --template link my-links
116
+ cd my-links && npm install
117
+ npm test # the workflow on the samples, with mock models
118
+ LBB_GRAPH=try-link npm run try # the samples on a graph of your stack
119
+ LBB_GRAPH=try-link npm start # serve it
120
+ LBB_GRAPH=try-link npm run grade -- --gold samples/gold.json
121
+ LBB_GRAPH=crm npm run setup && npm start # your graph
122
+ ```
123
+
124
+ `npm run setup` creates the classes and properties, one named workflow per
125
+ class of the settings, the named workflow that watches accepted items, and
126
+ the things identity. It is safe to repeat. The workflow is for a records
127
+ graph. Do not run it on a graph where `documents.link` runs: both write
128
+ `MENTIONED_IN` and `REFERS_TO`.
129
+
130
+ ## Grade a run
131
+
132
+ `npm run grade` (`src/grade-run.ts`) prints the metrics of general linking
133
+ 9.2: link precision at the gate and its 95% Wilson interval, pairwise
134
+ precision and recall over things, cluster purity and completeness
135
+ (B-cubed), the review share, and identity precision and recall.
136
+
137
+ ```sh
138
+ LBB_URL=... LBB_API_KEY=... LBB_GRAPH=crm npm run grade -- --gold .lbb-datasets/link-eval/gold.json --save run.json
139
+ npm run grade -- --gold .lbb-datasets/link-eval/gold.json --run run.json
140
+ ```
141
+
142
+ The gold file (`lbb.link.gold.v1`) lists the groups. Each group is one real
143
+ thing, with the mentions that name it:
144
+
145
+ ```json
146
+ {
147
+ "format": "lbb.link.gold.v1",
148
+ "complete": false,
149
+ "records": { "contact-1": "https://littlebigbrain.com/e/…" },
150
+ "groups": [
151
+ { "id": "mira", "kind": "person", "mentions": [
152
+ { "record": "contact-1", "field": "name", "span": "Mira Tannenholt-Ruiz" },
153
+ { "record": "https://littlebigbrain.com/e/…", "field": "body", "span": "Mimi T.", "occurrence": 0 }
154
+ ] }
155
+ ]
156
+ }
157
+ ```
158
+
159
+ A mention names its record (an IRI, or a name of `records`), its field
160
+ (`first_name+last_name` for a joined name), its span as the field writes it,
161
+ and which equal span of the field it is (`occurrence`, default 0). With
162
+ `complete: true`, a mention that the gold does not list is a thing of its
163
+ own. Without it, a link to such a mention counts apart as unjudged. Keep a
164
+ gold file of real records out of git, under `.lbb-datasets/`.
165
+
166
+ ## The samples
167
+
168
+ `samples/records.json` is an invented CRM: 7 organizations, 12 contacts, 5
169
+ leads, 7 deals and 11 notes. Every name, company and address is invented.
170
+ The names vary: a nickname (`Bob` for `Robert`), initials (`M. Tannenholt`,
171
+ `F. Okafor`), accents and transliterations (`Jürgen`, `Juergen`, `Jurgen`;
172
+ `Søren`, `Soren`; `Zoë`, `Zoe`), name order (`Wren, Tobias`), a typo
173
+ (`Ramaswamey`), a person known only by an email, a surname that two people
174
+ share (`Okafor`), two people of one name at two organizations (`Alex
175
+ Brandt`), and company domains. `samples/gold.json` holds the gold groups.
176
+
177
+ ## How the template differs from the design
178
+
179
+ - A named workflow takes one watch trigger in this release, so each class is
180
+ a named workflow of its own. All of them run the workflow type `link`.
181
+ - The candidates by the name index and by meaning wait for work tokens that
182
+ reach those routes (N3). The keys and the neighbours find the candidates.
183
+ - A source turn sends its decided pairs in messages of at most 24,000
184
+ bytes; a larger message goes in parts (`.1`, `.2`, …) until blobs (G11).
185
+ - The things identity is an engine identity that `npm run setup` creates.
186
+ It reads the review items it filed to learn a person's decisions.
187
+
188
+ ## Files
189
+
190
+ | File | Holds |
191
+ | --- | --- |
192
+ | `workflow.ts` | The workflow, the named workflows (`workflows()`), the ontology, the ports (`linkPorts()`) and `setup()` |
193
+ | `settings.ts` | The settings and the names of the parts, and the decided message |
194
+ | `names.ts` | Normal forms and block keys |
195
+ | `candidates.ts` | The candidate reads, the scoring and the bounds |
196
+ | `decide.ts` | The questions, the packing, the link rule and the rules without models |
197
+ | `things.ts` | The grouping with constraints, anchors, IRIs, merges and splits |
198
+ | `sources/records.ts` | Field roles, and the mentions of record fields |
199
+ | `source-turn.ts`, `things-turn.ts` | The turns of a source shard and of the things identity |
200
+ | `items.ts`, `reads.ts` | The review items, and the reads of the graph |
201
+ | `grade.ts`, `grade-run.ts` | The grading, and its command |
202
+ | `fixture.ts` | The test support: the graph in memory, mock models and ports, a reviewer that follows a gold file |
203
+ | `setup.ts`, `worker.ts`, `try.ts`, `samples.ts` | The commands |