@littlebigbrain/create 0.4.0 → 0.6.0

package/project/python/values_extract/tests/test_workflow.py

@@ -0,0 +1,150 @@
+"""The workflow on its samples, with mock models and no server: the SDK's
+``WorkflowWorker`` runs it against a fake LBB (``fake_lbb.py``).
+
+The mock reasoning model reads the first size (``60x40 cm`` or ``Breite 60
+cm``) with its unit, the first material and the water supply of each text.
+The mock decision model says a quote from a sentence about an accessory
+(``Zubehör``) or a conditional variant (``Bei Einsatz``) does not support its
+value. The values the workflow must write, and the values it must leave out,
+come from the answer key ``samples/expected.json``.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from fake_lbb import FakeLbb, iri_of, message, serve
+
+from values_extract.samples import SAMPLES
+from values_extract.workflow import DEFAULTS
+
+RECORDS: list[dict[str, str]] = json.loads((SAMPLES / "records.json").read_text(encoding="utf-8"))
+EXPECTED: dict[str, Any] = json.loads((SAMPLES / "expected.json").read_text(encoding="utf-8"))
+MATERIALS = [
+    "Sanitärkeramik",
+    "Stahl emailliert",
+    "Mineralguss",
+    "Edelstahl",
+    "Kristallglas",
+    "Keramik",
+    "Glas",
+]
+
+
+def read_values(text: str) -> list[dict[str, str]]:
+    """The mock reasoning model: the first size, material and water supply of a text."""
+    values: list[dict[str, str]] = []
+    size = re.search(r"(?:ca\. )?(\d+)x(\d+) cm", text)
+    width = re.search(r"Breite (\d+) cm", text)
+    if size:
+        values += [
+            {"property": "width_cm", "value": size[1], "unit": "cm", "quote": size[0]},
+            {"property": "depth_cm", "value": size[2], "unit": "cm", "quote": size[0]},
+        ]
+    elif width:
+        values.append({"property": "width_cm", "value": width[1], "unit": "cm", "quote": width[0]})
+    found = [
+        (m.start(), name)
+        for name in MATERIALS
+        if (m := re.search(rf"(?<![^\W\d_]){re.escape(name)}(?![^\W\d_])", text))
+    ]
+    if found:
+        material = min(found)[1]
+        values.append({"property": "material", "value": material, "unit": "", "quote": material})
+    water = re.search(r"Anschluss[^.]*?(Kalt- und Warmwasser|Kaltwasser)", text)
+    if water:
+        values.append(
+            {
+                "property": "water_supply",
+                "value": "kalt" if water[1] == "Kaltwasser" else "kalt und warm",
+                "unit": "",
+                "quote": water[0],
+            }
+        )
+    return values
+
+
+def support(text: str, instructions: str, options: list[str]) -> dict[str, Any]:
+    """The mock decision model: the support of each quote, from the sentence that holds it."""
+    found = re.search(r'Quote: "(.*)"$', instructions, re.S)
+    assert found, instructions
+    quote = found[1]
+    sentence = next((s for s in re.split(r"(?<=\.)\s+", text) if quote in s), "")
+    p = 0.05 if re.search("Zubehör|Bei Einsatz", sentence) else 0.6 if "etwa" in sentence else 0.92
+    return {
+        "choice": "supported" if p >= 0.5 else "not_supported",
+        "confidence": max(p, 1 - p),
+        "probabilities": {o: p if o == "supported" else (1 - p) / 2 for o in options},
+    }
+
+
+def fake_for(records: list[dict[str, str]]) -> FakeLbb:
+    """A fake LBB over ``records``, with the mock models."""
+
+    def rows(query: str) -> list[dict[str, str]]:
+        return [
+            {"s": iri_of(r["name"]), "label": r["name"], "text": r["text"]}
+            for r in records
+            if iri_of(r["name"]) in query
+        ]
+
+    def reason(body: dict[str, Any]) -> dict[str, Any]:
+        return {"values": read_values(json.loads(body["prompt"])["text"])}
+
+    def decide(body: dict[str, Any]) -> dict[str, Any]:
+        text = body["state"]["text"]
+        return {
+            qid: support(text, q["instructions"], list(q["criteria"]))
+            for qid, q in body["questions"].items()
+        }
+
+    return FakeLbb(rows, reason=reason, decide=decide)
+
+
+def run_samples() -> FakeLbb:
+    """The workflow on one message with every sample record."""
+    fake = fake_for(RECORDS)
+    fake.queue(message([iri_of(r["name"]) for r in RECORDS]))
+    serve(fake)
+    return fake
+
+
+def matches(got: Any, want: dict[str, Any]) -> bool:
+    """A number compares as a number; a text in lower case, trimmed, against ``accept``."""
+    if isinstance(want["value"], (int, float)):
+        return bool(got == want["value"])
+    accepted = want.get("accept") or [str(want["value"]).lower()]
+    return str(got).strip().lower() in accepted
+
+
+def test_the_samples_give_the_values_of_the_answer_key() -> None:
+    fake = run_samples()
+    for record in EXPECTED["records"]:
+        values = fake.written(record["name"])
+        for want in record["values"]:
+            assert matches(values.get(want["property"]), want), (record["name"], want)
+        for out in record["leave_out"]:
+            if out["value"] is None:
+                assert out["property"] not in values, (record["name"], out)
+            else:
+                assert not matches(values.get(out["property"]), out), (record["name"], out)
+        assert sorted(values) == sorted(v["property"] for v in record["values"]), record["name"]
+    (done,) = fake.completed
+    state = done["state"]
+    assert state["instances"] == len(RECORDS)
+    assert state["written"] == sum(len(r["values"]) for r in EXPECTED["records"])
+    assert state["failed"] == 0
+
+
+def test_each_value_has_a_source_whose_evidence_is_a_quote_of_the_text() -> None:
+    fake = run_samples()
+    texts = {r["name"]: r["text"] for r in RECORDS}
+    edges = [edge for commit in fake.commits.values() for edge in commit.get("triplets", [])]
+    assert len(edges) == sum(len(r["values"]) for r in EXPECTED["records"])
+    for edge in edges:
+        assert edge["relation"] == "HAS_VALUE_SOURCE"
+        assert edge["target"]["type"] == "ValueSource"
+        assert edge["evidence"]["text"] in texts[edge["source"]["name"]]
+        assert edge["confidence"] >= DEFAULTS.gate

package/python/values_extract/README.md

@@ -0,0 +1,52 @@
+# values-extract (Python)
+
+The Python port of the workflow template `values-extract`
+(`sdk/templates/values-extract` in the LBB repository). It fills properties of
+a class from the text of its instances. Each value comes with the words of the
+text that state it: `width_cm` 60 and `depth_cm` 40 from `ca. 60x40 cm`.
+
+The rules are those of the TypeScript template and of LBB's own
+`values.extract@v1`: the same prompts, checks, review items and counters. On
+the samples with the mock models of its test, the port writes the values of
+the answer key (`samples/expected.json`).
+
+A value between 0.5 and the gate becomes a `value` review item, one per
+property of an instance, under LBB's key (`value_item_key`). A person's
+answer wins over the model, for the evidence it was given on
+(`evidence_of`: the quote and the unit the text states). An accepted or
+corrected value is written as answered; a rejected one, or one the graph
+refused, is not written again. New evidence with another value asks again,
+and the answered value and its source link stay until a new answer. A value
+an earlier version filed as a suggestion (`legacy_value_key`) keeps working.
+
+## Run it
+
+From `sdk/python/examples`, with the Python SDK installed
+(`pip install littlebigbrain`):
+
+```sh
+LBB_URL=http://127.0.0.1:7400 LBB_API_KEY=lbb_sk_test_local python -m values_extract try
+LBB_URL=http://127.0.0.1:7400 LBB_API_KEY=lbb_sk_test_local LBB_GRAPH=try-values-extract python -m values_extract start
+```
+
+| Command | Does |
+| --- | --- |
+| `python -m values_extract try` | Creates a graph (`LBB_GRAPH`, default `try-values-extract`), puts the six sample records on it, and creates the workflow there |
+| `python -m values_extract setup` | Creates the workflow on your graph (`LBB_GRAPH`, default `main`) |
+| `python -m values_extract start` | Serves the workflow until SIGTERM or Ctrl-C. `LBB_WORKER_ID` names the worker, `LBB_CONCURRENCY` sets how many turns run at once |
+
+In the LBB repository, `npm run dev:stack -- --worker python:values-extract`
+starts a local stack with mock models and runs `try` and `start` on it.
+
+## The files
+
+| File | Holds |
+| --- | --- |
+| `workflow.py` | The settings, the trigger, the ontology, the checks and the workflow |
+| `samples.py` | The loader of the samples |
+| `samples/` | The sample records and their answer key, the template's |
+| `__main__.py` | The commands |
+
+The test of the port is `tests/test_values_extract_example.py` of the Python
+SDK. It runs the workflow with the real worker against a fake LBB, and checks
+that a worker stopped during its write replays its steps and writes once.

package/python/values_extract/__init__.py

@@ -0,0 +1,3 @@
+"""values-extract: the Python port of the workflow template
+``sdk/templates/values-extract``. Run it with ``python -m values_extract``
+from the folder that holds this package (see the README)."""

package/python/values_extract/__main__.py

@@ -0,0 +1,97 @@
+"""The commands of values-extract, as the TypeScript template project has
+them:
+
+    python -m values_extract try     # a graph with the samples and the workflow
+    python -m values_extract setup   # the workflow on your graph
+    python -m values_extract start   # serve it: claim and run its turns
+
+They read ``LBB_URL`` (your stack's endpoint, default
+``http://127.0.0.1:7400``), ``LBB_API_KEY`` (a key of the stack) and
+``LBB_GRAPH`` (``try`` uses ``try-values-extract`` by default, the others
+``main``). ``start`` also reads ``LBB_WORKER_ID`` and ``LBB_CONCURRENCY``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+from lbb import AsyncLbbClient, LbbError
+from lbb.workflows import WorkflowWorker, lbb_activities
+
+from .samples import load_samples
+from .workflow import create_workflow, ontology, trigger
+
+
+def client(default_graph: str) -> AsyncLbbClient:
+    return AsyncLbbClient(
+        os.environ.get("LBB_URL", "http://127.0.0.1:7400"),
+        api_key=os.environ.get("LBB_API_KEY"),
+        graph=os.environ.get("LBB_GRAPH", default_graph),
+    )
+
+
+async def try_samples() -> None:
+    """Create a graph, put the samples on it, and create the workflow there.
+    Then serve it with ``start`` and the same ``LBB_GRAPH``."""
+    async with client("try-values-extract") as lbb_client:
+        graph = lbb_client._graph
+        try:
+            await lbb_client.create_graph()
+            print(f"Created the graph {graph}")
+        except LbbError as error:
+            # 409: the graph exists already.
+            if error.status_code != 409:
+                raise
+        count = await load_samples(lbb_client)
+        print(f"Committed {count} sample records")
+        created = await lbb_client.use_workflow_starter(**trigger())
+        print(f"Created {created['name']} on the graph {graph}: {created['status']['state']}.")
+        print(f"Serve it with LBB_GRAPH={graph} python -m values_extract start")
+
+
+async def set_up() -> None:
+    """Add the classes and properties the workflow writes, then create the
+    triggered workflow. Each call is safe to repeat."""
+    async with client("main") as lbb_client:
+        await lbb_activities(lbb_client).evolve(ontology())
+        created = await lbb_client.use_workflow_starter(**trigger())
+        graph = lbb_client._graph
+        print(f"Created {created['name']} on the graph {graph}: {created['status']['state']}.")
+
+
+def start() -> None:
+    """Serve the workflow until SIGTERM or Ctrl-C."""
+    lbb_client = client("main")
+    workflow = create_workflow(lbb_activities(lbb_client))
+    worker = WorkflowWorker(
+        lbb_client,
+        [workflow],
+        worker=os.environ.get("LBB_WORKER_ID", f"{workflow.name}-py-1"),
+        concurrency=int(os.environ.get("LBB_CONCURRENCY", "1")),
+    )
+    print(
+        f"Serving {workflow.name}@{workflow.version} for the graph {lbb_client._graph}. "
+        "Stop with Ctrl-C.",
+        flush=True,
+    )
+    worker.run()
+
+
+def main(argv: list[str]) -> int:
+    command = argv[0] if argv else ""
+    if command == "try":
+        asyncio.run(try_samples())
+    elif command == "setup":
+        asyncio.run(set_up())
+    elif command == "start":
+        start()
+    else:
+        print(__doc__)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/python/values_extract/samples/expected.json

@@ -0,0 +1,149 @@
+{
+  "about": [
+    "The answer key for the values-extract samples: the values a correct run reads from each record, with the quote that supports each, and the values it must leave out.",
+    "All text is invented. make-samples.mjs checks that every quote is in its record's text (whitespace runs as one space) and that a number's quote holds the number.",
+    "Numbers compare as numbers. A text value compares in lower case, trimmed, against accept. A run may quote more or less of the text, as long as its quote is in the text and holds the evidence.",
+    "leave_out lists values a correct run does not write. With value null, the run writes no value for that property at all."
+  ],
+  "install": {
+    "properties": {
+      "width_cm": "f64",
+      "depth_cm": "f64",
+      "material": "text",
+      "water_supply": "keyword"
+    },
+    "units": { "width_cm": "cm", "depth_cm": "cm" },
+    "context": "Entries of the sanitary equipment list of a school building tender, in German."
+  },
+  "records": [
+    {
+      "name": "S 4.1",
+      "case": "An approximate size (ca.) is still the item's size.",
+      "values": [
+        { "property": "width_cm", "value": 60, "quote": "ca. 60x40 cm" },
+        { "property": "depth_cm", "value": 40, "quote": "ca. 60x40 cm" },
+        {
+          "property": "material",
+          "value": "Sanitärkeramik",
+          "accept": ["sanitärkeramik", "keramik", "sanitärkeramik, weiß", "sanitärkeramik weiß"],
+          "quote": "aus Sanitärkeramik, weiß"
+        },
+        {
+          "property": "water_supply",
+          "value": "kalt und warm",
+          "accept": ["kalt und warm", "warm und kalt", "kalt- und warmwasser", "kaltwasser und warmwasser"],
+          "quote": "Anschluss an Kalt- und Warmwasser"
+        }
+      ],
+      "leave_out": []
+    },
+    {
+      "name": "S 4.2",
+      "case": "An accessory of another material: the Klapprost.",
+      "values": [
+        { "property": "width_cm", "value": 50, "quote": "Stahl emailliert, 50x35 cm" },
+        { "property": "depth_cm", "value": 35, "quote": "Stahl emailliert, 50x35 cm" },
+        {
+          "property": "material",
+          "value": "Stahl emailliert",
+          "accept": ["stahl emailliert", "stahl, emailliert", "emaillierter stahl"],
+          "quote": "Ausgussbecken für Putzräume, Stahl emailliert"
+        },
+        {
+          "property": "water_supply",
+          "value": "kalt",
+          "accept": ["kalt", "kaltwasser", "nur kaltwasser", "nur kalt"],
+          "quote": "Anschluss nur an Kaltwasser"
+        }
+      ],
+      "leave_out": [
+        { "property": "material", "value": "Edelstahl", "reason": "Edelstahl is the material of the Klapprost, an accessory." }
+      ]
+    },
+    {
+      "name": "S 4.3",
+      "case": "The only size in the text belongs to an accessory, the Glasablage.",
+      "values": [
+        {
+          "property": "material",
+          "value": "Mineralguss",
+          "accept": ["mineralguss"],
+          "quote": "Waschtisch, unterfahrbar, aus Mineralguss"
+        },
+        {
+          "property": "water_supply",
+          "value": "kalt und warm",
+          "accept": ["kalt und warm", "warm und kalt", "kalt- und warmwasser", "kaltwasser und warmwasser"],
+          "quote": "Anschluss an Kalt- und Warmwasser"
+        }
+      ],
+      "leave_out": [
+        { "property": "width_cm", "value": null, "reason": "50x12 cm is the size of the Glasablage. The basin's size is left to the maker (Maße nach Hersteller)." },
+        { "property": "depth_cm", "value": null, "reason": "50x12 cm is the size of the Glasablage. The basin's size is left to the maker (Maße nach Hersteller)." },
+        { "property": "material", "value": "Glas", "reason": "Glas is the material of the Glasablage, an accessory." },
+        { "property": "material", "value": "Edelstahl", "reason": "Edelstahl is the material of the Haltegriff, an accessory." }
+      ]
+    },
+    {
+      "name": "S 4.4",
+      "case": "Every value stated once, plainly.",
+      "values": [
+        { "property": "width_cm", "value": 40, "quote": "Edelstahl, 40x30 cm" },
+        { "property": "depth_cm", "value": 30, "quote": "Edelstahl, 40x30 cm" },
+        {
+          "property": "material",
+          "value": "Edelstahl",
+          "accept": ["edelstahl"],
+          "quote": "Trinkbrunnen für den Flur, Edelstahl"
+        },
+        {
+          "property": "water_supply",
+          "value": "kalt",
+          "accept": ["kalt", "kaltwasser", "nur kaltwasser", "nur kalt"],
+          "quote": "Anschluss an Kaltwasser in Trinkwasserqualität"
+        }
+      ],
+      "leave_out": []
+    },
+    {
+      "name": "S 4.5",
+      "case": "A variant that holds only under a condition (Bei Einsatz von Geschirrspülern).",
+      "values": [
+        { "property": "width_cm", "value": 100, "quote": "Edelstahl, 100x50 cm" },
+        { "property": "depth_cm", "value": 50, "quote": "Edelstahl, 100x50 cm" },
+        {
+          "property": "material",
+          "value": "Edelstahl",
+          "accept": ["edelstahl"],
+          "quote": "Spüle für die Lehrerküche, Edelstahl"
+        },
+        {
+          "property": "water_supply",
+          "value": "kalt und warm",
+          "accept": ["kalt und warm", "warm und kalt", "kalt- und warmwasser", "kaltwasser und warmwasser"],
+          "quote": "Anschluss an Kalt- und Warmwasser"
+        }
+      ],
+      "leave_out": [
+        { "property": "width_cm", "value": 120, "reason": "The width of the variant with two basins, which holds only bei Einsatz von Geschirrspülern." }
+      ]
+    },
+    {
+      "name": "S 4.6",
+      "case": "A second size that is a height, and no water supply.",
+      "values": [
+        { "property": "width_cm", "value": 60, "quote": "Breite 60 cm" },
+        {
+          "property": "material",
+          "value": "Kristallglas",
+          "accept": ["kristallglas", "glas"],
+          "quote": "Spiegel für Waschplätze, Kristallglas"
+        }
+      ],
+      "leave_out": [
+        { "property": "depth_cm", "value": null, "reason": "45 cm is the height (Höhe). The text gives no depth." },
+        { "property": "water_supply", "value": null, "reason": "The text names no water supply." }
+      ]
+    }
+  ]
+}

package/python/values_extract/samples/records.json

@@ -0,0 +1,26 @@
+[
+  {
+    "name": "S 4.1",
+    "text": "S 4.1 Waschtisch für Klassenräume, aus Sanitärkeramik, weiß, ca. 60x40 cm, mit einem Hahnloch. Anschluss an Kalt- und Warmwasser über Einhebelmischer."
+  },
+  {
+    "name": "S 4.2",
+    "text": "S 4.2 Ausgussbecken für Putzräume, Stahl emailliert, 50x35 cm. Zubehör: Klapprost aus Edelstahl. Anschluss nur an Kaltwasser."
+  },
+  {
+    "name": "S 4.3",
+    "text": "S 4.3 Waschtisch, unterfahrbar, aus Mineralguss, Maße nach Hersteller. Zubehör: Glasablage 50x12 cm und Haltegriff aus Edelstahl. Anschluss an Kalt- und Warmwasser."
+  },
+  {
+    "name": "S 4.4",
+    "text": "S 4.4 Trinkbrunnen für den Flur, Edelstahl, 40x30 cm, mit Druckknopfventil. Anschluss an Kaltwasser in Trinkwasserqualität."
+  },
+  {
+    "name": "S 4.5",
+    "text": "S 4.5 Spüle für die Lehrerküche, Edelstahl, 100x50 cm, mit einem Becken und Abtropffläche. Anschluss an Kalt- und Warmwasser. Bei Einsatz von Geschirrspülern: Ausführung mit zwei Becken, 120x50 cm."
+  },
+  {
+    "name": "S 4.6",
+    "text": "S 4.6 Spiegel für Waschplätze, Kristallglas, Kanten poliert, Breite 60 cm, Höhe 45 cm."
+  }
+]

package/python/values_extract/samples.py

@@ -0,0 +1,37 @@
+"""The samples of values-extract, for ``python -m values_extract try``: six
+invented entries of a sanitary equipment list (``samples/records.json``),
+committed as instances of the class the settings name, each with its text.
+The same samples as the TypeScript template's."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from lbb import AsyncLbbClient
+from lbb.workflows import lbb_activities
+
+from .workflow import DEFAULTS, ontology
+
+SAMPLES = Path(__file__).resolve().parent / "samples"
+
+
+async def load_samples(client: AsyncLbbClient, folder: Path = SAMPLES) -> int:
+    """Add the class and its properties, then commit the sample records to the
+    client's graph. The commit's idempotency key makes a second load a replay."""
+    records = json.loads((folder / "records.json").read_text(encoding="utf-8"))
+    await lbb_activities(client).evolve(ontology())
+    await client.commit(
+        {
+            "entity_properties": [
+                {
+                    "type": DEFAULTS.class_name,
+                    "name": record["name"],
+                    "properties": {DEFAULTS.text_field: record["text"]},
+                }
+                for record in records
+            ]
+        },
+        idempotency_key="values-extract-samples",
+    )
+    return len(records)