@littlebigbrain/create 0.5.0 → 0.6.0

package/templates/review/workflow.ts

@@ -18,9 +18,9 @@
 // `fact_status: accepted`; a person can leave facts out of the accept, except the type of a new
 // instance. The proposal gets its `fact_status` (`written`, `in_review`, `refused`) and
 // `review_note`, owned by this workflow.
-import { createHash } from "node:crypto";
 import {
   entityRef,
+  LbbError,
   triggeredWorkflow,
   type LbbActivities,
   type Schemas,
@@ -69,15 +69,30 @@ export function ontology(settings: Settings = defaults): Schemas["OntologyEvolve
   ];
 }
 
-/** What the workflow decided, over all its turns. */
+/** What the workflow decided, over all its turns: the counters of LBB's review. */
 export interface ReviewState {
   proposals: number;
+  /** Written, or held by the graph already (`held_already`). */
   written: number;
-  held: number;
+  held_already: number;
   in_review: number;
   refused: number;
+  /** Proposals for a person that name an instance the graph does not have. */
+  new_instance: number;
+  /** Review items filed. */
+  items: number;
 }
 
+const ZERO: ReviewState = {
+  proposals: 0,
+  written: 0,
+  held_already: 0,
+  in_review: 0,
+  refused: 0,
+  new_instance: 0,
+  items: 0,
+};
+
 /** Proposals one turn decides: two steps each, and at most one review item each, in a turn of 64 steps. */
 const TURN_PROPOSALS = 20;
 const PLAIN_IRI = /^[^\s<>"{}|\\^`]+$/;
@@ -106,6 +121,10 @@ interface Outcome {
   note?: string;
   facts?: review.Facts;
   holds?: boolean;
+  /** The confidence the decision gated on (`gateValue`). */
+  gate?: number;
+  /** For a person, and it names an instance the graph does not have. */
+  newInstance?: boolean;
   /** For a person: the item's instance, its view, and what the item shows of the fact. */
   item?: {
     instance: string;
@@ -121,27 +140,46 @@ export function createWorkflow(lbb: LbbActivities, settings: Settings = defaults
   return triggeredWorkflow<ReviewState>({
     name: workflowType.workflow_type,
     version: workflowType.version,
-    initialState: { proposals: 0, written: 0, held: 0, in_review: 0, refused: 0 },
+    initialState: ZERO,
     async onInstances(ctx, message, state) {
       const batch = message.instances.slice(0, TURN_PROPOSALS);
       let cache: Promise<Read> | undefined;
       const read = () => (cache ??= reader(lbb, batch, settings));
-      const next = { ...state };
+      const next = { ...ZERO, ...state };
       const forPeople: { iri: string; outcome: Outcome }[] = [];
       for (const [i, iri] of batch.entries()) {
-        const outcome = await ctx.step(`decide-${i}`, async (s) => {
+        let outcome = await ctx.step(`decide-${i}`, async (s) => {
           s.progress("deciding proposals", { done: i, total: batch.length, unit: "proposals" });
           return decideOne(lbb, await read(), iri, settings);
         });
-        await lbb.writeOwned(ctx, `write-${i}`, async (s) => {
-          s.progress("writing facts", { done: i, total: batch.length, unit: "proposals" });
-          return statusWrite(await read(), iri, outcome, settings);
-        });
+        try {
+          await lbb.writeOwned(ctx, `write-${i}`, async (s) => {
+            s.progress("writing facts", { done: i, total: batch.length, unit: "proposals" });
+            return statusWrite(await read(), iri, outcome, settings);
+          });
+        } catch (error) {
+          // The graph refused the statement (a value it cannot hold): the proposal says why.
+          if (!(error instanceof LbbError && error.status === 400) || outcome.status !== "written")
+            throw error;
+          outcome = {
+            status: "refused",
+            note: `The graph refused the statement: ${error.message}`,
+          };
+          const refused = outcome;
+          await lbb.writeOwned(ctx, `refuse-${i}`, async () =>
+            statusWrite(await read(), iri, refused, settings),
+          );
+        }
         if (outcome.status === "decided") continue;
         next.proposals += 1;
-        if (outcome.status === "written") next[outcome.holds ? "held" : "written"] += 1;
-        else next[outcome.status] += 1;
-        if (outcome.status === "in_review") forPeople.push({ iri, outcome });
+        if (outcome.status === "written") {
+          next.written += 1;
+          if (outcome.holds) next.held_already += 1;
+        } else next[outcome.status] += 1;
+        if (outcome.status === "in_review") {
+          if (outcome.newInstance) next.new_instance += 1;
+          forPeople.push({ iri, outcome });
+        }
       }
       // One review item per instance of this turn.
       const groups = new Map<string, { iri: string; outcome: Outcome }[]>();
@@ -149,12 +187,14 @@ export function createWorkflow(lbb: LbbActivities, settings: Settings = defaults
         const instance = one.outcome.item!.instance;
         groups.set(instance, [...(groups.get(instance) ?? []), one]);
       }
-      for (const [n, members] of [...groups.values()].entries())
+      for (const [n, members] of [...groups.values()].entries()) {
         await ctx.step(`review-${n}`, async (s) => {
           s.progress("filing review items", { done: n, total: groups.size, unit: "instances" });
           await fileItem(lbb, await read(), members, message, settings);
           return true;
         });
+        next.items += 1;
+      }
       // A proposal its proposer withdrew (message.removed) keeps the fact written for it: see the README.
       const rest = message.instances.slice(TURN_PROPOSALS);
       return rest.length
@@ -295,10 +335,11 @@ async function decideOne(
     );
     edgeHolds = rows.length > 0;
   }
-  const decision = review.decide(terms, s.predicate, proposal.confidence, settings, edgeHolds);
+  const gate = review.gateValue(proposal.confidence, proposal.evidence);
+  const decision = review.decide(terms, s.predicate, gate, settings, edgeHolds);
   if (decision.status === "refused" || terms.kind === "refused")
     return { status: "refused", note: decision.status === "refused" ? decision.why : "" };
-  const facts = review.factsOf(terms, iri, proposal.confidence, proposal.evidence);
+  const facts = review.factsOf(terms, iri, gate, proposal.evidence);
   if (decision.status === "written")
     return decision.holds
       ? { status: "written", holds: true }
@@ -313,6 +354,8 @@ async function decideOne(
     status: "in_review",
     note: decision.why,
     facts,
+    gate,
+    newInstance: review.namesNewInstance(terms),
     item: {
       instance,
       ...(known.class || terms.kind === "type"
@@ -329,10 +372,13 @@ const proposalRef = (proposal: Proposal, settings: Settings) =>
   entityRef(proposal.iri, settings.className, proposal.label);
 
 /**
- * File the review item of one instance: each fact's statement and its proposal's
- * `fact_status: accepted`, in the item's order (the type, the label, values, links, the trace),
- * with one evidence sample per fact. The key carries the proposals, so other proposals file
- * another item.
+ * File the review item of one instance, as LBB's review files it: one status row per proposal
+ * (`fact_status: accepted`) in the order a person reads them (the type, the label, values,
+ * links, the trace), the type of a new instance and the trace required, one evidence sample per
+ * fact, and LBB's key, title, rationale and summary. As the design of the workflows on the
+ * fleet asks (G8), the item also holds each statement after the status rows, so its accept
+ * commits the facts: each sample names the rows and triplets of its fact (`rows`, `triplets`),
+ * and a person can leave facts out of the accept.
  */
 async function fileItem(
   lbb: LbbActivities,
@@ -341,97 +387,98 @@ async function fileItem(
   message: WorkflowTriggerMessage,
   settings: Settings,
 ) {
-  const first = members[0]!.outcome.item!;
-  const instance = first.instance;
+  const instance = members[0]!.outcome.item!.instance;
   const kind = (m: { outcome: Outcome }) => m.outcome.item!.shown.kind;
+  const text = (m: { outcome: Outcome }) => m.outcome.item!.shown.text;
   const ordered = [...members].sort(
     (a, b) =>
       review.KIND_ORDER.indexOf(kind(a)) - review.KIND_ORDER.indexOf(kind(b)) ||
-      a.outcome.item!.shown.text.localeCompare(b.outcome.item!.shown.text),
+      (text(a) < text(b) ? -1 : text(a) > text(b) ? 1 : 0) ||
+      (a.iri < b.iri ? -1 : a.iri > b.iri ? 1 : 0),
   );
-  const creates = first.isNew && ordered.some((m) => kind(m) === "type");
-  const className = ordered.map((m) => m.outcome.item!.class).find(Boolean) ?? "Instance";
-  const label =
-    ordered.map((m) => m.outcome.item!.label).find(Boolean) ??
-    instance.slice(instance.lastIndexOf("/") + 1);
+  const isNew = members.some((m) => m.outcome.item!.isNew);
+  const creates = isNew && ordered.some((m) => kind(m) === "type");
+  const className = ordered.map((m) => m.outcome.item!.class).find(Boolean);
+  const label = ordered.map((m) => m.outcome.item!.label).find(Boolean);
+  // The status rows first, one per fact, as LBB's item holds them; the statements after them.
+  const rows: Schemas["EntityPropertiesInput"][] = ordered.map(({ iri }) => ({
+    ...proposalRef(read.proposals.get(iri)!, settings),
+    properties: { fact_status: "accepted" },
+  }));
   const triplets: Schemas["TripletInput"][] = [];
-  const rows: Schemas["EntityPropertiesInput"][] = [];
-  // The rows an accept must keep (`required`, from client 0.24; an older client sends it as is).
   const required: { row: number; reason: string }[] = [];
+  const statementRequired: { row: number; reason: string }[] = [];
   const samples: Record<string, unknown>[] = [];
-  const whys = new Set<string>();
-  for (const { iri, outcome } of ordered) {
+  const whys: string[] = [];
+  const proposers = new Set<string>();
+  const parts: string[] = [];
+  let links = 0;
+  for (const [row, { iri, outcome }] of ordered.entries()) {
     const proposal = read.proposals.get(iri)!;
     const shown = outcome.item!.shown;
     const needed = shown.kind === "source" || (creates && shown.kind === "type");
-    const writeTriplets = (outcome.facts?.triplets ?? []).map((t) => triplets.push(t) - 1);
-    const writeRows = (outcome.facts?.entity_properties ?? []).map((r) => rows.push(r) - 1);
-    const row =
-      rows.push({ ...proposalRef(proposal, settings), properties: { fact_status: "accepted" } }) -
-      1;
-    if (needed)
-      for (const at of [...writeRows, row])
-        required.push({
-          row: at,
-          reason:
-            shown.kind === "type"
-              ? "a new instance needs its type"
-              : "the source of an instance goes with it",
-        });
-    if (outcome.note) whys.add(outcome.note);
+    const reason =
+      shown.kind === "type"
+        ? "a new instance needs its type"
+        : "the source of an instance goes with it";
+    const factRows = (outcome.facts?.entity_properties ?? []).map((r) => rows.push(r) - 1);
+    const factTriplets = (outcome.facts?.triplets ?? []).map((t) => triplets.push(t) - 1);
+    if (needed) {
+      required.push({ row, reason });
+      for (const at of factRows) statementRequired.push({ row: at, reason });
+    }
+    if (outcome.note && !whys.includes(outcome.note)) whys.push(outcome.note);
+    if (proposal.proposedBy) proposers.add(proposal.proposedBy);
+    if (shown.kind === "link") links += 1;
+    else if (shown.kind !== "source") parts.push(shown.text);
     samples.push({
       proposal: iri,
       row,
-      ...(writeRows.length ? { rows: writeRows } : {}),
-      ...(writeTriplets.length ? { triplets: writeTriplets } : {}),
       kind: shown.kind,
       fact: shown.text,
-      confidence: Math.round(proposal.confidence * 1000) / 1000,
+      confidence: Math.round((outcome.gate ?? proposal.confidence) * 1000) / 1000,
       ...(review.place(proposal.evidence) ?? {}),
       ...(needed ? { required: true } : {}),
+      ...(shown.waitsFor !== undefined ? { waits_for: shown.waitsFor } : {}),
+      ...(factRows.length ? { rows: factRows } : {}),
+      ...(factTriplets.length ? { triplets: factTriplets } : {}),
     });
   }
-  const proposalsHash = createHash("sha256")
-    .update(
-      ordered
-        .map((m) => m.iri)
-        .sort()
-        .join("\n"),
-    )
-    .digest("hex")
-    .slice(0, 12);
+  if (links) parts.push(links === 1 ? "1 link" : `${links} links`);
+  const tail = instance.slice(instance.lastIndexOf("/") + 1);
   const title = creates
-    ? `New ${className} ${label}`
-    : `${className} ${label}: ${members.length === 1 ? "1 fact" : `${members.length} facts`} to check`;
-  const proposers = [...new Set(ordered.map((m) => read.proposals.get(m.iri)!.proposedBy))].filter(
-    Boolean,
-  );
+    ? `New ${className ?? "Instance"} ${label ?? tail}`
+    : `${className ?? "Instance"} ${label ?? tail}: ${ordered.length === 1 ? "1 fact" : `${ordered.length} facts`} to check`;
+  const by = [...proposers].sort();
+  const rationale = `${whys.slice(0, 3).join(" ")}${by.length ? ` Proposed by ${by.join(", ")}.` : ""}`;
   await lbb.review({
     workflow: message.workflow,
-    key: `starter:${message.workflow}:instance:${instance.slice(instance.lastIndexOf("/") + 1)}:${proposalsHash}`,
-    title: review.clip(title, 200),
-    rationale: review.clip(
-      `${[...whys].slice(0, 3).join(" ")}${proposers.length ? ` Proposed by ${proposers.join(", ")}.` : ""}`,
-      500,
+    key: review.itemKey(
+      message.workflow,
+      instance,
+      ordered.map((m) => m.iri),
     ),
+    title: review.clip(title, 200),
+    rationale: review.clip(rationale.trim(), 500),
+    origin: { kind: "agent", id: `starter:${message.workflow}`, label: "review" },
     facts: {
       triplets,
       entity_properties: rows,
-      required,
-      summary: review.clip(ordered.map((m) => m.outcome.item!.shown.text).join(" · "), 500),
+      required: [...required, ...statementRequired],
+      summary: review.clip(parts.join(" · "), 500),
     } as Schemas["SuggestedFacts"],
     mapping: {
       review: {
         instance,
-        class: className,
-        label,
+        class: className ?? null,
+        label: label ?? null,
         new: creates,
-        facts: members.length,
-        shown: members.length,
+        facts: ordered.length,
+        shown: ordered.length,
         truncated: false,
       },
     },
-    evidence: { records: members.length, samples },
+    evidence: { records: ordered.length, samples },
   });
 }
 

package/templates/values-extract/README.md

@@ -11,31 +11,73 @@ The LBB-run version is the starter workflow `values.extract@v1`
 
 ## How it runs
 
+The template follows the rules of LBB's version
+(`lbb-graph-query/src/infer_llm.rs`). Its eval runs both on the same
+recorded model answers and finds the same calls and outputs
+([`../_evals/README.md`](../_evals/README.md)).
+
 The triggered workflow watches the class and its text field. For each
 instance of a trigger message, two steps:
 
-1. **Values.** The reasoning model reads the text and returns
-   `{values: [{property, value, quote}]}`; the schema allows only the
-   settings' property names. The prompt is the LBB version's: the values of
-   the instance's own item in its base version, not those of a part, an
-   accessory, an alternative or a variant under a condition; an approximate
-   value is stated. Then the checks without a model: a value takes its
-   property's type (`typed`), its quote is in the text with at least three
-   words or the value in it (`quoteHolds`), and one value per property.
+1. **Values.** The text is the field's values as LBB composes it
+   (`text: <values>`, each value's whitespace as single spaces). The
+   reasoning model reads it and returns `{values: [{property, value, unit,
+   quote}]}`; the schema allows only the settings' property names, in name
+   order, and a number property names its unit. A number comes as the text
+   states it, with its unit, and the model never converts it. The prompt is
+   the LBB version's: the values of the instance's own item in its base
+   version, not those of a part, an accessory, an alternative or a variant
+   under a condition; an approximate value is stated. Then the checks
+   without a model, in LBB's order (`check`): the property is one the
+   workflow fills (else `unknown_property`), the value is the first of its
+   property (else `duplicate`), it takes its property's type (else
+   `invalid_value`), and its quote is in the text with at least three words
+   or the value in it (else `quote_not_found`). A number of a property with
+   a unit must come in that unit, with the unit in its quote
+   (`unitProblem`); otherwise it goes to review at most, with LBB's reason.
 2. **Support.** The decision model scores how well each quote supports its
    value: `supported`, `weak` or `not_supported`. The support is the
    probability of `supported`.
 
-Then the gate: a value at or above `gate` is written, one from 0.5 to the
-gate becomes a review item with the value as its facts, one below 0.5 is
-dropped. One owned write per turn replaces the values of the turn's
-instances: a property the workflow does not write again loses its value and
-its source. An accepted review item's value is written with the others the
-next time its instance is read. An instance that leaves the class loses the
-values this workflow wrote.
-
-A turn reads at most 24 instances; the rest of a message goes to a new turn
-(`ctx.continue`).
+Then the gate: a value at or above `gate` is `written`, one from 0.5 to the
+gate goes to `review`, one below 0.5 or without an answer is
+`weak_support` and is dropped. A value in review becomes a `value` review
+item (`lbb.review({ kind: "value" })`) with the property's unit and the
+quote, one per property of an instance, under LBB's key
+`starter:<workflow>:value:<hash of instance and property>` (`valueItemKey`,
+with `stableHashHex`). One owned write per turn replaces the values of the
+turn's instances: a property the workflow does not write again loses its
+value and its source. Each value's evidence is its quote, with the
+instance's page box as its region when the instance has one (`page`, `x0`,
+`y0`, `x1`, `y1`, as a `DocEntry` has).
+
+Before it writes, the workflow reads the items (`lbb.reviewItems(keys)`): a
+person's answer wins over the model, for the evidence it was given on
+(`evidence_hash`: a hash of the value's quote and of the unit the text
+states, `evidenceOf`). With the same evidence, the value in force, or the
+value the person decided on, a value a person accepted, or corrected with
+a number in the property's unit, is written as answered. New evidence that
+reads another value asks the person again, and the answered value and its
+source link stay (the instance is written on its own, without that
+property) until a new answer is written. A value a person rejected is
+not written again, nor is one the graph refused (`effects.commit_error`; the
+item is open again for the person). An earlier version filed a value in
+review as a suggestion, one per value (`legacyValueKey`): an accepted one is
+written again, an open one stays the question, and a dismissed one is a
+reject of its value. An instance that leaves the class keeps its values:
+they go with its fields when it is retracted.
+
+The write's request id is its step's effect id, which no other turn
+repeats, so a value that changes back (A, then B, then A) is written again.
+When a model call is refused, the instance goes back to the watch
+(`lbb.requeue`, after 1, 4 and 16 minutes, at most 3 times) and keeps its
+values; the turn completes and the shard goes on. A spent budget fails the
+turn with the time to retry, and the worker waits without an attempt.
+
+The state counts what LBB's version counts: `instances`,
+`values_proposed`, one counter per outcome, `failed`, `commits` and
+`removed`. A turn reads at most 24 instances; the rest of a message goes to
+a new turn (`ctx.continue`).
 
 ## Settings
 
@@ -48,6 +90,7 @@ A turn reads at most 24 instances; the rest of a message goes to a new turn
 | `properties` | `width_cm`, `depth_cm` (`f64`), `material` (`text`), `water_supply` (`keyword`) | The properties to fill, with their types (`text`, `keyword`, `i64`, `f64`, `bool`, `date_time`) |
 | `context` | the samples' | One sentence about what the text is |
 | `gate` | 0.8 | The support a value needs to be written |
+| `units` | `width_cm`, `depth_cm`: `cm` | The unit of each number property; `ontology()` declares it on the property |
 
 `trigger(settings)` names the triggered workflow `values-extract` and
 owns the class `ValueSource`, the relation `HAS_VALUE_SOURCE` and the
@@ -57,14 +100,22 @@ relation.
 ## What the LBB-run version adds
 
 - `params.where`: a SPARQL pattern that limits the instances it reads.
-- Texts built from several watched fields, as fit sources build them, and
-  the fields chosen for you when the watch names none.
-- The instance's page box as the region of each value's evidence, for an
-  instance that has one (a `DocEntry`).
+- Texts built from several watched fields, and the fields chosen for you
+  when the watch names none.
 - A check at creation that each property is in the property registry with
   a type it can write.
 - Its own monthly budget, a preview with the values of up to 3 instances,
-  and counters per outcome in the workflow's status.
+  and its counters in the workflow's status.
+
+It differs in three places. It commits once per instance, the template once
+per turn, so its `commits` counter is larger. When the reasoning model gives
+no usable answer, it counts the instance `failed` and clears its values; the
+template sends the instance back to the watch and keeps them. And it
+converts a number the text states in another unit of the same kind (`600 mm`
+for a property in `cm`) to the unit the property declares, and refuses a
+unit it does not know or of another kind; the template sends such a number
+to review. So the question of such a number differs: LBB's item asks the
+converted value, the template's the value as stated.
 
 ## Samples and test
 
@@ -79,3 +130,21 @@ or a conditional variant (`Bei Einsatz`) does not support its value. The run
 writes the values of the answer key and none it leaves out. The test also
 checks the review path, the checks of type and quote, an instance that
 leaves the class, and a message larger than a turn.
+
+## Test on examples
+
+`npm run test:examples` runs the workflow on instances of your graph with
+the real models, and writes nothing (`test-examples.ts`). The activities go
+through `captureActivities`: the server checks each owned write with the
+commit's dry run, and keeps it. Each review item is kept too. The script
+prints, for each input, the values the workflow would write and the values
+it would give a person to review.
+
+```sh
+LBB_URL=... LBB_API_KEY=... LBB_GRAPH=tender npm run test:examples           # the first 6 instances
+LBB_URL=... LBB_API_KEY=... LBB_GRAPH=tender npm run test:examples -- <iri>  # chosen instances
+```
+
+The model calls count in your environment's budget. For the LBB-run
+version, test the workflow in the console (the workflow's Test tab) or with
+`client.workflows.tests`.

package/templates/values-extract/samples/expected.json

@@ -12,6 +12,7 @@
       "material": "text",
       "water_supply": "keyword"
     },
+    "units": { "width_cm": "cm", "depth_cm": "cm" },
     "context": "Entries of the sanitary equipment list of a school building tender, in German."
   },
   "records": [

package/templates/values-extract/test-examples.ts

@@ -0,0 +1,155 @@
+// Test the workflow on examples before it writes. `npm run test:examples`
+// runs it with the real models on chosen instances of your graph, through
+// `captureActivities`, which writes nothing: each owned write is checked
+// with the commit's dry run and kept, and each review item is kept. Then it
+// prints, per input, the values the workflow would write and the values it
+// would give a person to review.
+//
+//   LBB_URL=... LBB_API_KEY=... LBB_GRAPH=... npm run test:examples -- [<iri> …]
+//
+// Without IRIs it tests the first 6 instances of the class. The model calls
+// count in your environment's budget.
+import { pathToFileURL } from "node:url";
+import {
+  LbbClient,
+  captureActivities,
+  entityIdFromIri,
+  lbbActivities,
+  type ActivityCaptureResult,
+  type LbbActivities,
+  type WorkflowTriggerMessage,
+} from "@littlebigbrain/client";
+import { runMessage } from "../_shared/testing.js";
+import { createWorkflow, defaults, type Settings } from "./workflow.js";
+
+/** A value the workflow would write. */
+export interface ExampleValue {
+  property: string;
+  value: string;
+  quote?: string;
+  support?: number;
+}
+
+/** What the workflow would do with one input. */
+export interface ExampleResult {
+  iri: string;
+  /** Values at or above the gate, which it writes. */
+  written: ExampleValue[];
+  /** Values it gives a person to review: each item's title. */
+  review: string[];
+  /** Why the server's dry run (or the local check) refused a write. */
+  errors: string[];
+}
+
+/** The first `limit` instances of the class, and the commit they were read at. */
+export async function firstInstances(
+  client: LbbClient,
+  settings: Settings = defaults,
+  limit = 6,
+): Promise<{ iris: string[]; seq: number }> {
+  const read = await client.sparqlRows({
+    query: `SELECT ?s WHERE { ?s a <https://littlebigbrain.com/class/${encodeURIComponent(settings.className.toLowerCase())}> } LIMIT ${limit}`,
+  });
+  return {
+    iris: read.rows.map((row) => row.s!).filter(Boolean),
+    seq: read.snapshot?.served_at_seq ?? 0,
+  };
+}
+
+/** Per input, what the captured writes and review items say. */
+export function resultsOf(iris: string[], captured: ActivityCaptureResult): ExampleResult[] {
+  return iris.map((iri) => {
+    const id = entityIdFromIri(iri);
+    const mine = (key: string | null | undefined) => key?.startsWith(`value:${id}:`) ?? false;
+    const written: ExampleValue[] = [];
+    const errors: string[] = [];
+    for (const write of captured.writes) {
+      const about = (write.body.triplets ?? []).filter(
+        (edge) => edge.relation === "HAS_VALUE_SOURCE" && mine(edge.target.key),
+      );
+      if (about.length) errors.push(...write.errors);
+      for (const edge of about) {
+        const source = (write.body.entity_properties ?? []).find(
+          (entity) => entity.type === "ValueSource" && entity.key === edge.target.key,
+        );
+        const properties = (source?.properties ?? {}) as Record<string, unknown>;
+        const quote = typeof edge.evidence === "object" ? edge.evidence?.text : undefined;
+        written.push({
+          property: String(properties.property ?? edge.target.key?.split(":")[2] ?? ""),
+          value: String(properties.value ?? ""),
+          ...(quote ? { quote } : {}),
+          ...(edge.confidence != null ? { support: edge.confidence } : {}),
+        });
+      }
+    }
+    const review = captured.reviews
+      .filter((item) => item.request.key?.includes(`:value:${id}:`))
+      .map((item) => item.request.title);
+    return { iri, written, review, errors };
+  });
+}
+
+/**
+ * Run the workflow on `iris` with `activities` captured
+ * (`captureActivities`): the models run, and nothing is written. `seq` is
+ * the commit the workflow reads the texts at.
+ */
+export async function testExamples(
+  activities: LbbActivities,
+  iris: string[],
+  options: { settings?: Settings; seq?: number; graph?: string } = {},
+): Promise<ExampleResult[]> {
+  const settings = options.settings ?? defaults;
+  const capture = captureActivities(activities, { workflow: "values-extract" });
+  const message: WorkflowTriggerMessage = {
+    type: "instances",
+    workflow: "values-extract-test",
+    graph: options.graph ?? "main",
+    as_of_commit_seq: options.seq ?? 0,
+    reason: "refresh",
+    instances: iris,
+    removed: [],
+  };
+  await runMessage(createWorkflow(capture.activities, settings), message);
+  return resultsOf(iris, capture.result());
+}
+
+/** The results as lines of text. */
+export function report(results: ExampleResult[]): string[] {
+  const lines: string[] = [];
+  for (const result of results) {
+    lines.push(result.iri);
+    for (const value of result.written)
+      lines.push(
+        `  write   ${value.property} = ${value.value}${value.support != null ? ` (support ${value.support.toFixed(2)})` : ""}${value.quote ? ` "${value.quote}"` : ""}`,
+      );
+    for (const title of result.review) lines.push(`  review  ${title}`);
+    if (!result.written.length && !result.review.length) lines.push("  no value");
+    for (const error of result.errors) lines.push(`  refused ${error}`);
+  }
+  return lines;
+}
+
+async function main() {
+  const graph = process.env.LBB_GRAPH;
+  if (!graph) throw new Error("set LBB_GRAPH to the graph to test on");
+  const client = new LbbClient({
+    baseUrl: process.env.LBB_URL ?? "http://127.0.0.1:7400",
+    apiKey: process.env.LBB_API_KEY,
+    graph,
+  });
+  const named = process.argv.slice(2).filter((arg) => !arg.startsWith("-"));
+  const first = await firstInstances(client);
+  const iris = named.length ? named : first.iris;
+  if (!iris.length) throw new Error(`the graph ${graph} has no ${defaults.className} to test`);
+  const results = await testExamples(lbbActivities(client), iris, { seq: first.seq, graph });
+  for (const line of report(results)) console.log(line);
+  console.log("Nothing was written.");
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.message : error);
+    process.exitCode = 1;
+  });
+}