@littlebigbrain/create 0.5.0 → 0.6.0

package/python/versions.json

@@ -0,0 +1,3 @@
+{
+  "littlebigbrain": "0.30.0"
+}

package/templates/_shared/testing.ts

@@ -16,12 +16,15 @@ import {
   type DecideResult,
   type EmbedRequest,
   type EmbedResult,
+  type FailedInstance,
   type FilePages,
   type LbbActivities,
   type OwnedWrite,
   type ReasonRequest,
   type ReasonResult,
+  type ReviewItemRequest,
   type ReviewRequest,
+  type ReviewWaitResult,
   type Schemas,
   type SparqlResults,
   type WorkflowContext,
@@ -146,6 +149,10 @@ export interface MockRules {
   refuses?: (kind: "reason" | "decide" | "embed", request: unknown) => boolean;
   /** The status a review item gets (default `open`). */
   reviewStatus?: (request: ReviewRequest) => Schemas["OntologyChangeSuggestionStatus"];
+  /** The answer a person gave to a review item of another kind than a change (default none: open). */
+  reviewAnswer?: (request: ReviewItemRequest) => Schemas["ReviewAnswer"] | undefined;
+  /** Why the graph refused the answer's value (`effects.commit_error`), if it did. */
+  reviewRefused?: (request: ReviewItemRequest) => string | undefined;
   pages?: FilePages[];
   ontology?: Schemas["OntologyView"];
   /** What a dry-run evolve answers (default: a change that is not a no-op). */
@@ -170,7 +177,18 @@ export class MockActivities implements LbbActivities {
   readonly graph = new MemoryGraph();
   readonly calls: MockCall[] = [];
   readonly reviews = new Map<string, ReviewRequest & { status: string }>();
+  /** The review items of other kinds than a change (`value`, `choice`, …), by key. */
+  readonly questions = new Map<string, ReviewItemRequest>();
+  /** Items asked again on new evidence: the answer in force they hold until a new one. */
+  readonly previous = new Map<string, Schemas["ReviewAnswer"]>();
   readonly evolved: Schemas["OntologyEvolveOp"][][] = [];
+  /** The instances each turn sent back to the watch (`lbb.requeue`), by step effect id. */
+  readonly requeued: {
+    effectId: string;
+    workflowId: string;
+    turn: number;
+    failed: FailedInstance[];
+  }[] = [];
   private readonly kept = new Map<string, { request: string; answer: unknown }>();
   private readonly written = new Set<string>();
 
@@ -299,7 +317,23 @@ export class MockActivities implements LbbActivities {
       const { effectId } = step;
       const replayed = this.written.has(effectId);
       if (!replayed) {
-        this.graph.apply(typeof write === "function" ? await write(step) : write);
+        const body = typeof write === "function" ? await write(step) : write;
+        // As the SDK does: a write of nothing sends no commit.
+        const empty =
+          !body.subjects.length &&
+          !body.retract_entities?.length &&
+          !body.triplets?.length &&
+          !body.entity_properties?.length;
+        if (empty)
+          return {
+            commit_seq: null,
+            idempotency_key: effectId,
+            replayed: false,
+            owned: null,
+            entities_written: 0,
+            edges_written: 0,
+          };
+        this.graph.apply(body);
         this.written.add(effectId);
       }
       return {
@@ -313,7 +347,136 @@ export class MockActivities implements LbbActivities {
     });
   }
 
-  async review(request: ReviewRequest): Promise<Schemas["OntologyChangeSuggestion"]> {
+  /**
+   * `lbb.requeue` inside the step `stepKey`, as the SDK runs it: nothing
+   * without failed instances, and a repeat of the step reads its saved
+   * answer. The answer counts the attempt from 1 (the server takes it from
+   * the turn's message).
+   */
+  async requeue(
+    ctx: WorkflowContext,
+    stepKey: string,
+    failed: readonly FailedInstance[],
+  ): Promise<Schemas["WorkflowRequeueResponse"] | null> {
+    if (!failed.length) return null;
+    return ctx.step(stepKey, async (step) => {
+      this.requeued.push({
+        effectId: step.effectId,
+        workflowId: ctx.workflowId,
+        turn: ctx.turn,
+        failed: failed.map((f) => ({ ...f })),
+      });
+      return {
+        attempt: 1,
+        not_before: "2026-01-01T00:01:00Z",
+        requeued: failed.map((f) => f.instance),
+        exhausted: [],
+      };
+    });
+  }
+
+  review(request: ReviewRequest): Promise<Schemas["OntologyChangeSuggestion"]>;
+  review(request: ReviewItemRequest): Promise<Schemas["ReviewItem"]>;
+  async review(
+    request: ReviewRequest | ReviewItemRequest,
+  ): Promise<Schemas["OntologyChangeSuggestion"] | Schemas["ReviewItem"]> {
+    if ("kind" in request && request.kind !== "change") return this.question(request);
+    return this.suggestion(request as ReviewRequest);
+  }
+
+  /** The items filed under `keys`, of every status and kind. */
+  async reviewItems(keys: string[]): Promise<Schemas["ReviewItem"][]> {
+    return [...new Set(keys)].flatMap((key): Schemas["ReviewItem"][] => {
+      const request = this.questions.get(key);
+      if (request) return [this.itemOf(key, request)];
+      // A suggestion is a `change` item, answered as its status says.
+      const filed = this.reviews.get(key);
+      if (!filed) return [];
+      const decision =
+        filed.status === "accepted"
+          ? "accept"
+          : filed.status === "dismissed"
+            ? "reject"
+            : undefined;
+      return [
+        {
+          id: `rev_${key}`,
+          object: "review_item",
+          key,
+          kind: "change",
+          status: decision ? "answered" : filed.status === "superseded" ? "withdrawn" : "open",
+          title: filed.title,
+          ...(decision ? { answer: { decision } } : {}),
+        } as unknown as Schemas["ReviewItem"],
+      ];
+    });
+  }
+
+  /** A review item of another kind than a change: open, or answered by the rules. A key a
+   * suggestion holds is refused as the server refuses it (`409 review_kind_clash`). */
+  private question(request: ReviewItemRequest): Schemas["ReviewItem"] {
+    const key = request.key ?? `review-${this.questions.size}`;
+    if (this.reviews.has(key))
+      throw new LbbError(409, `key ${key} names a change item, not a ${request.kind} item`, {
+        type: "invalid_request_error",
+        code: "review_kind_clash",
+        message: `key ${key} names a change item, not a ${request.kind} item`,
+      });
+    // As the server does: an answered item stays as answered, unless new
+    // evidence brings another proposal for an answer in force, which asks
+    // again and keeps that answer as `previous_answer`.
+    const before = this.questions.get(key);
+    if (before) {
+      const was = this.itemOf(key, before);
+      const answer = was.status === "answered" ? was.answer : undefined;
+      const inForce = answer?.decision === "accept" || answer?.decision === "correct";
+      const evidenceChanged =
+        Boolean(before.evidence_hash && request.evidence_hash) &&
+        before.evidence_hash !== request.evidence_hash;
+      const held = answer?.value ?? before.question.value;
+      const asked = request.question.value;
+      if (inForce && evidenceChanged && held !== asked && before.question.value !== asked)
+        this.previous.set(key, { ...answer!, value: held });
+      else if (answer) return was;
+    }
+    this.questions.set(key, request);
+    return this.itemOf(key, request);
+  }
+
+  private itemOf(key: string, request: ReviewItemRequest): Schemas["ReviewItem"] {
+    const previous = this.previous.get(key);
+    // An item asked again waits for a new answer.
+    const answer = previous ? undefined : this.rules.reviewAnswer?.(request);
+    const refused = answer ? this.rules.reviewRefused?.(request) : undefined;
+    return {
+      id: `rev_${key}`,
+      object: "review_item",
+      key,
+      kind: request.kind,
+      // A refused answer does not stand: the item is open again, as the server keeps it.
+      status: answer && !refused ? "answered" : "open",
+      title: request.title,
+      question: request.question,
+      ...(request.evidence_hash ? { evidence_hash: request.evidence_hash } : {}),
+      ...(previous ? { previous_answer: previous } : {}),
+      ...(answer ? { answer } : {}),
+      ...(refused ? { effects: { commit_error: refused, delivery: { state: "none" } } } : {}),
+    } as unknown as Schemas["ReviewItem"];
+  }
+
+  /** Files the item; the wait returns the rules' answer, else it times out. */
+  async waitForReview(
+    ctx: WorkflowContext,
+    key: string,
+    request: ReviewItemRequest,
+  ): Promise<ReviewWaitResult> {
+    const item = await ctx.step(`${key}:ask`, async () => this.question(request));
+    return item.answer
+      ? { answered: true, answer: item.answer, item }
+      : { answered: false, answer: null, item };
+  }
+
+  private async suggestion(request: ReviewRequest): Promise<Schemas["OntologyChangeSuggestion"]> {
     const key = request.key ?? `review-${this.reviews.size}`;
     const status = this.reviews.get(key)?.status ?? this.rules.reviewStatus?.(request) ?? "open";
     this.reviews.set(key, { ...request, status });
@@ -324,6 +487,38 @@ export class MockActivities implements LbbActivities {
     } as unknown as Schemas["OntologyChangeSuggestion"];
   }
 
+  /** The review items of `reviews`, newest first, filtered as the route filters them. */
+  async listReviews(
+    options: { originId?: string; status?: string; key?: string; limit?: number } = {},
+  ): Promise<Schemas["OntologyChangeSuggestionList"]> {
+    const items = [...this.reviews.entries()]
+      .reverse()
+      .filter(
+        ([key, item]) =>
+          (options.key === undefined || key === options.key) &&
+          (options.status === undefined || item.status === options.status) &&
+          (options.originId === undefined || item.origin?.id === options.originId),
+      )
+      .slice(0, options.limit ?? 100);
+    return {
+      suggestions: items.map(([key, item]) => ({
+        suggestion_id: key,
+        key,
+        status: item.status,
+        title: item.title,
+        anchor: item.anchor ?? { kind: "ontology" },
+      })),
+    } as unknown as Schemas["OntologyChangeSuggestionList"];
+  }
+
+  /** The review item filed under `key`, or null. */
+  async reviewItem(key: string): Promise<Schemas["OntologyChangeSuggestion"] | null> {
+    const item = this.reviews.get(key);
+    return item
+      ? ({ ...item, suggestion_id: key, key } as unknown as Schemas["OntologyChangeSuggestion"])
+      : null;
+  }
+
   async evolve(ops: Schemas["OntologyEvolveOp"][], options: { dryRun?: boolean } = {}) {
     if (!options.dryRun) this.evolved.push(ops);
     const answer = this.rules.evolve?.(ops) ?? {};

package/templates/ontology-fit/README.md

@@ -8,33 +8,47 @@ evolve operations and the quotes that support it; a person accepts it, and
 the ontology changes.
 
 The LBB-run version is the starter workflow `ontology.fit@v1`, the fit
-sources of `/v1/ontology/fit-sources` (`docs/reference/fit-sources.md`).
+sources of `/v1/ontology/fit-sources` (`docs/reference/fit-sources.md`). The
+template follows its rules (`lbb-graph-query/src/fit_sources.rs`); its eval
+runs both on the same recorded model answers and finds the same calls,
+review items and counters ([`../_evals/README.md`](../_evals/README.md)).
 
 ## How it runs
 
-The triggered workflow watches the class and its text field. For each
-instance of a trigger message, one step:
+The triggered workflow watches the class and its text field. A turn first
+reads the workflow's review items (`lbb.listReviews`): the titles of the open
+ones and the names of the dismissed ones. Then, for each instance of the
+message, two steps:
 
-1. The reasoning model reads the text with the ontology
-   (`lbb.ontology()`, as classes, relations with their domain and range,
-   and properties) and the changes the earlier texts of the turn proposed,
-   and returns up to 12 proposals: `add_class`, `add_property`,
-   `add_relation` or `widen_relation`, each with a sentence of the text as
-   its quote. The prompt and schema are the LBB version's.
-2. Each proposal becomes a change (`shape`): its key, its title and its
-   evolve operations. A class the change names that the ontology lacks is
-   added first, and a new relation with the name of an existing one widens
-   it. The key follows the LBB version:
+1. **Propose.** The text is the field's values as LBB composes it
+   (`text: <values>`), read in chunks of 40,000 characters with 1,000 shared.
+   The reasoning model reads each chunk with LBB's prompt: the ontology (its
+   classes, relations with domain and range, properties, and the name style
+   it uses), and the changes already proposed and waiting for review, newest
+   support first, with the rejected names. It returns up to 12 proposals per
+   chunk: `add_class`, `add_property`, `add_relation` or `widen_relation`,
+   each with a sentence of the text as its quote. A chunk the model gives no
+   usable answer for counts as failed, and the instance goes on. Then the
+   checks without a model, in LBB's order: the shape (`shape`; a relation or
+   property name in the ontology's style, `ZUSTAENDIG` as `zustaendig`, and
+   a new relation with the name of an existing one widens it), one proposal
+   per change, the quote in the text with at least three words, and a dry run
+   (`lbb.evolve(change, { dryRun: true })`): a change the ontology has is
+   `already_covered`. The decision model scores how well each quote supports
+   its change; one below 0.5 is `weak_support`.
+2. **File.** Each kept change goes into the review item of its key:
    `fit:<workflow name>/add_class/mangel`,
    `fit:<workflow name>/add_relation/mangel/zustaendig/firma`,
-   `fit:<workflow name>/widen_relation/betrifft-raum/mangel/x`.
-3. A change is kept when its quote is in the text with at least three
-   words, and a dry run (`lbb.evolve(change, { dryRun: true })`) says the
-   ontology does not have it yet.
+   `fit:<workflow name>/widen_relation/betrifft-raum/mangel/x`. The item
+   keeps the best supported samples of every text that supports it (at most
+   20), and its rationale says how many instances support it. An item a
+   person decided is left alone. The next instance's prompt lists the
+   change.
 
-A last step files one review item per key, with the quotes of every text of
-the turn that supports it. The same key again updates the item. A turn reads
-at most 48 texts; the rest goes to a new turn.
+The state counts what LBB's fit sources count: `instances`, `chunks`,
+`failed_chunks`, `proposals`, `kept`, `filed`, `updated`, `quote_not_found`,
+`invalid`, `already_covered` and `weak_support`. A turn reads at most 24
+texts; the rest goes to a new turn.
 
 ## Settings
 
@@ -51,18 +65,13 @@ every key. The workflow writes no graph facts, so it owns nothing.
 
 ## What the LBB-run version adds
 
-- **Support scores.** The decision model scores how well each quote
-  supports its change; a change below 0.5 is dropped.
-- **Support across instances.** The instances that support each key are
-  counted over every run, and a dismissed change opens again when twice its
-  support and three more instances support it.
-- **The proposals of the source.** The open suggestions of the source and
-  the names of the dismissed ones go into every prompt, so the model reuses
-  names and does not propose a rejected change again.
-- **Name style.** A new relation or property name is written in the
-  ontology's style (`works_in` or `WORKS_IN`).
-- **Long texts.** A text past 40,000 characters is read in overlapping
-  chunks.
+- **Support across all runs.** It keeps the instances that support each key
+  beside the source, so its count stays exact past 20 samples. The template
+  counts from the item: its count, and one more for an instance the samples
+  do not hold yet.
+- **Reopening.** A dismissed change opens again when twice its support and
+  three more instances support it. The template leaves a dismissed change
+  dismissed.
 - Its own monthly budget, a preview with the proposals of a sample, and
   counters per outcome in the source's status.
 

package/templates/ontology-fit/workflow.test.ts

@@ -3,8 +3,9 @@
 // `Frist`, `Status` and `Zuständig` propose its facts, a room in the same
 // text widens `betrifft_raum`, a list of who was present proposes the
 // relation `anwesend` the ontology has, and a text about the acceptance
-// (`Abnahme`) gets a quote that is not in the text. The expected keys come
-// from samples/expected.md.
+// (`Abnahme`) gets a quote that is not in the text. The mock decision model
+// finds every quote supports its change. The expected keys come from
+// samples/expected.md.
 import { test } from "node:test";
 import assert from "node:assert/strict";
 import { readFileSync } from "node:fs";
@@ -20,7 +21,7 @@ import {
   triggerMessage,
   type MockRules,
 } from "../_shared/testing.js";
-import { createWorkflow, keyPart, quoteInText } from "./workflow.js";
+import { chunkText, createWorkflow, displayName, keyPart, quoteInText } from "./workflow.js";
 import { sampleOntology } from "./samples.js";
 
 const SAMPLES = samplesDir(import.meta.url, "ontology-fit");
@@ -183,10 +184,18 @@ function rules(more: MockRules = {}): MockRules {
     reason: (request) => ({
       proposals: propose((JSON.parse(request.prompt) as { text: string }).text),
     }),
+    decide: (request) =>
+      Object.fromEntries(
+        Object.keys(request.questions).map((id) => [
+          id,
+          { choice: "supported", confidence: 0.9, probabilities: { supported: 0.9 } },
+        ]),
+      ),
     evolve: (ops) => ({ no_op: ops.every((op) => covered(op, v)) }),
     sparql: (query) =>
       TEXTS.filter((t) => query.includes(iriOf(t.name))).map((t) => ({
         s: iriOf(t.name),
+        class: "Baustellenbericht",
         label: t.name,
         text: t.text,
       })),
@@ -211,7 +220,23 @@ test("the samples give the changes of expected.md, one review item per key", asy
   );
   assert.equal(EXPECTED_KEYS.length, 5);
   assert.deepEqual([...lbb.reviews.keys()].sort(), [...EXPECTED_KEYS].sort());
-  assert.deepEqual(state, { texts: 5, proposals: 3 + 4 + 5 + 5 + 2, filed: 5 });
+  // The counters of LBB's fit: the quote of Abnahme is not in its text, anwesend is covered
+  // (texts 1 and 2 list who was present).
+  assert.deepEqual(state, {
+    instances: 5,
+    chunks: 5,
+    failed_chunks: 0,
+    proposals: 22,
+    kept: 19,
+    filed: 5,
+    updated: 14,
+    reopened: 0,
+    quote_not_found: 1,
+    invalid: 0,
+    already_covered: 2,
+    weak_support: 0,
+    skipped: 0,
+  });
   const mangel = lbb.reviews.get("fit:berichte/add_class/mangel")!;
   assert.equal(mangel.workflow, "berichte");
   assert.deepEqual(mangel.change, [{ op: "add_entity_type", name: "Mangel" }]);
@@ -230,7 +255,18 @@ test("a change names the classes it needs, and a widening keeps the relation's n
 
   assert.deepEqual(lbb.reviews.get("fit:berichte/add_relation/mangel/zustaendig/firma")!.change, [
     { op: "add_entity_type", name: "Mangel" },
-    { op: "add_relation", name: "zustaendig", domain: ["Mangel"], range: ["Firma"] },
+    {
+      op: "add_relation",
+      name: "zustaendig",
+      domain: ["Mangel"],
+      range: ["Firma"],
+      cardinality: null,
+      temporal_semantics: null,
+      reducer: null,
+      inverse_name: null,
+      transitive: false,
+      symmetric: false,
+    },
   ]);
   assert.deepEqual(lbb.reviews.get("fit:berichte/widen_relation/betrifft-raum/mangel/x")!.change, [
     { op: "add_entity_type", name: "Mangel" },
@@ -268,3 +304,18 @@ test("keys keep lowercase ASCII letters and digits, other runs as one dash", ()
   assert.equal(keyPart("-"), "x");
   assert.equal(keyPart("zuständig"), "zust-ndig");
 });
+
+test("a long text is read in chunks as LBB reads it, and a class name in its words", () => {
+  const sentence = "Der Mangel in Raum 0.12 ist behoben. ";
+  const long = sentence.repeat(3_000);
+  const chunks = chunkText(long);
+  assert.equal(chunks.length, 3);
+  assert.ok(chunks.every((c) => [...c].length <= 40_000));
+  // Each chunk ends after a sentence and shares 1,000 characters with the next.
+  assert.ok(chunks[0]!.endsWith("behoben."));
+  assert.equal(chunks[1]!.slice(0, 1_000), chunks[0]!.slice(-1_000));
+  assert.deepEqual(chunkText("kurz"), ["kurz"]);
+  assert.equal(displayName("customerIdentityData"), "customer identity data");
+  assert.equal(displayName("depends_on"), "depends on");
+  assert.equal(displayName("baustellenbericht"), "baustellenbericht");
+});