@wrenautomation/birdwatch 0.0.0-stage → 0.30.0

package/dist/document/extract.js

@@ -0,0 +1,226 @@
+/**
+ * An extractor draws findings out of one document for one ask, and says what the document was
+ * worth. The claim extractor is the one in v1: the model proposes sentences with the exact words
+ * they rest on; the gate keeps a finding only when its quote is on the document, word for word,
+ * and every number in it is in the quote (a year may come from the document). What the model
+ * knows without the document never becomes a finding. The verdict is the same call's word on the
+ * document (`quality`) and on the ask (`gap`); the gate corrects a `rich` that kept nothing.
+ *
+ * No cutoff: the prompt caps neither the findings nor their length, and a document is sent
+ * whole. When the model's output cap cuts an answer (a `Cut`), the whole findings before the cut
+ * go through the gate, and the model is asked again for the text after the last quote it kept,
+ * with those findings known. A cut answer with no whole finding on the document stops the
+ * extract with an error: the one case a document stays unread.
+ */
+import { fill, numbersQuoted, pageQuoted, words } from "@wrenautomation/read/study";
+import { z } from "zod";
+import { bodyOf } from "./body.js";
+import { addUsage, isCut } from "./model.js";
+import { Quality } from "./types.js";
+const QUOTE_MIN_WORDS = 5;
+export const CLAIM_PROMPT = `You pull facts out of one document for a research ask.
+
+Return ONLY a JSON object, no prose, exactly this shape:
+{"findings": [{"text": "...", "quote": "..."}], "quality": "rich", "gap": "..."}
+
+Rules:
+- Every finding that serves the ask and is not known already, in document order.
+- text: one plain sentence a person would say out loud. Specific: a number, a named practice, a stated cause. Never "The document says" or "The page notes".
+- Say who says it, as the page shows it: "A 2025 survey of 400 firms found ...", "One commenter says ...", "A vendor of X says ...". One person is one person: never "owners say" or "commenters" for one voice.
+- Name a speaker's role, trade, size or place only when the page states it.
+- Keep the quote's hedge and scope: "probably", "sometimes", "a lot of", "in my experience" stay; never turn them into "most", "often" or a rule.
+- Advice, a question, a promise or a sales pitch is not a report of what happens. Take it only as what it is ("A seller claims ...").
+- quote: the exact words from the document that the finding rests on, copied character for character, at least 5 words, no more than the finding needs.
+- Every part of the text must be in the quote: no cause, ranking, detail or second fact the quote lacks. Two facts from two places are two findings.
+- Every number in the text must appear in the quote. A table cell without its row and column labels is not a quote.
+- A company's marketing about itself is not a fact about the market.
+- Nothing in the document serves the ask: return {"findings": [], "quality": "off", "gap": "..."}.
+- quality: "rich" when the document answers much of the ask, "thin" when it serves it a little, "off" when it is about something else.
+- gap: one short sentence on what the ask still lacks after this document, or an empty string when nothing is missing.
+
+Ask: {ask}
+{steer}Known already (look past these):
+{known}
+
+DOCUMENT: {url}{part}
+{text}
+`;
+export const ClaimAnswer = z.object({
+    findings: z.array(z.object({ text: z.string().nullable().optional(), quote: z.string().nullable().optional() })),
+    quality: Quality.nullable().optional(),
+    gap: z.string().nullable().optional(),
+});
+/** "The person says:" and their lines, or nothing: a prompt pays for the heading only when there is one. */
+export const renderSteer = (steer) => steer?.length ? `The person says:\n${steer.map((s) => `- ${s}`).join("\n")}\n` : "";
+export function claimPrompt(doc, ask, options = {}) {
+    const known = ask.known ?? [];
+    return fill(CLAIM_PROMPT, {
+        ask: ask.text,
+        steer: renderSteer(ask.steer),
+        known: known.length ? known.map((k) => `- ${k}`).join("\n") : "(nothing)",
+        url: doc.url,
+        part: options.rest ? " (the rest, after the last finding kept)" : "",
+        text: doc.text,
+    });
+}
+/**
+ * The verdict as the gate leaves it: the model's quality, unless it said `rich` and nothing
+ * survived (then `thin`); `off` when it said nothing and nothing survived; `empty` only when
+ * the document has no text. The gap is kept when it says something.
+ */
+export function verdictOf(answer, doc, kept) {
+    const gap = (answer.gap ?? "").replace(/\s+/g, " ").trim();
+    const said = answer.quality ?? (kept ? "thin" : "off");
+    const quality = !doc.text.trim()
+        ? "empty"
+        : said === "rich" && !kept
+            ? "thin"
+            : said === "empty"
+                ? kept
+                    ? "thin"
+                    : "off"
+                : said;
+    return { quality, ...(gap ? { gap } : {}) };
+}
+/** Every proposal through the gate: findings whose quote is on the document, dropped with why, the verdict. */
+export function gateFindings(answer, doc) {
+    const page = { url: doc.url, text: doc.text };
+    const findings = [];
+    const dropped = [];
+    for (const p of answer.findings) {
+        const text = (p.text ?? "").replace(/\s+/g, " ").trim();
+        if (!text)
+            continue;
+        const quote = (p.quote ?? "").trim();
+        // A short quote proves little (it is on many pages); a line break inside one folds to a space.
+        const why = !quote
+            ? "no_quote"
+            : words(quote) < QUOTE_MIN_WORDS
+                ? "quote_short"
+                : !pageQuoted(quote, [page], doc.url)
+                    ? "quote_not_found"
+                    : !numbersQuoted(text, quote, page)
+                        ? "number_not_in_quote"
+                        : null;
+        if (why)
+            dropped.push({ text, why });
+        else if (!findings.some((f) => f.proof.quote === quote))
+            findings.push({ kind: "claim", text, proof: { url: doc.url, quote } });
+    }
+    return { findings, dropped, verdict: verdictOf(answer, doc, findings.length) };
+}
+/**
+ * The whole findings in a cut answer: each complete `{...}` of the findings array, in order, up
+ * to the first that does not parse. Braces and quotes inside strings are not structure.
+ */
+export function salvageFindings(partial) {
+    const findings = [];
+    const key = partial.indexOf('"findings"');
+    const open = key >= 0 ? partial.indexOf("[", key) : -1;
+    if (open < 0)
+        return { findings };
+    let i = open + 1;
+    while (i < partial.length) {
+        const start = partial.indexOf("{", i);
+        if (start < 0)
+            break;
+        let depth = 0;
+        let inString = false;
+        let end = -1;
+        for (let j = start; j < partial.length; j++) {
+            const c = partial[j];
+            if (inString) {
+                if (c === "\\")
+                    j++;
+                else if (c === '"')
+                    inString = false;
+            }
+            else if (c === '"')
+                inString = true;
+            else if (c === "{")
+                depth++;
+            else if (c === "}" && --depth === 0) {
+                end = j;
+                break;
+            }
+            else if (c === "]" && depth === 0)
+                break;
+        }
+        if (end < 0)
+            break;
+        const one = ClaimAnswer.shape.findings.element.safeParse(JSON.parse(partial.slice(start, end + 1)));
+        if (!one.success)
+            break;
+        findings.push(one.data);
+        i = end + 1;
+    }
+    return { findings };
+}
+const RANK = { rich: 3, thin: 2, off: 1, empty: 0 };
+/** Where a quote ends in the text, at or after `from`; -1 when the text does not hold it as written. */
+function quoteEnd(text, quote, from) {
+    const q = quote.replace(/\s+/g, " ").trim();
+    let at = text.indexOf(q, from);
+    if (at < 0)
+        at = text.toLowerCase().indexOf(q.toLowerCase(), from);
+    return at < 0 ? -1 : at + q.length;
+}
+/**
+ * The one extractor of v1: claims that quote the document, and a verdict on it. A cut answer
+ * is continued from the end of the last quote it kept, with those findings known, until the
+ * model finishes an answer.
+ */
+export const claimExtractor = {
+    name: "claim",
+    async extract(whole, ask, model) {
+        // the reference list holds no finding: every part and quote comes from the body
+        const doc = { ...whole, text: bodyOf(whole.text) };
+        const findings = [];
+        const dropped = [];
+        const known = [...(ask.known ?? [])];
+        let usage;
+        let from = 0;
+        let best = null;
+        let continued = 0;
+        for (;;) {
+            const part = from ? { ...doc, text: doc.text.slice(from) } : doc;
+            let answer;
+            let cut = false;
+            try {
+                const a = await model.ask(claimPrompt(part, { ...ask, known }, { rest: from > 0 }), ClaimAnswer);
+                answer = a.value;
+                usage = addUsage(usage, a.usage);
+            }
+            catch (err) {
+                if (!isCut(err))
+                    throw err;
+                cut = true;
+                answer = salvageFindings(err.partial);
+                usage = addUsage(usage, err.usage);
+            }
+            const out = gateFindings(answer, part);
+            const fresh = out.findings.filter((f) => !findings.some((k) => k.proof.quote === f.proof.quote));
+            findings.push(...fresh);
+            dropped.push(...out.dropped);
+            if (answer.quality && (!best || RANK[answer.quality] > RANK[best]))
+                best = answer.quality;
+            if (!cut) {
+                return {
+                    findings,
+                    dropped,
+                    verdict: verdictOf({ quality: best, gap: answer.gap }, doc, findings.length),
+                    ...(usage ? { usage } : {}),
+                    ...(continued ? { continued } : {}),
+                };
+            }
+            const next = fresh.reduce((end, f) => Math.max(end, quoteEnd(doc.text, f.proof.quote ?? "", from)), -1);
+            if (next <= from)
+                throw new Error(`the answer was cut and ${fresh.length ? "no kept quote places the cut" : "nothing before the cut passed the gate"} (${doc.url}, from ${from})`);
+            for (const f of fresh)
+                known.push(f.text);
+            from = next;
+            continued++;
+        }
+    },
+};
+//# sourceMappingURL=extract.js.map

package/dist/document/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/document/extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACpF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAA0B,MAAM,YAAY,CAAC;AACrE,OAAO,EAA+B,OAAO,EAAgB,MAAM,YAAY,CAAC;AA+BhF,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,MAAM,CAAC,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0B3B,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,QAAQ,EAAE,CAAC,CAAC,KAAK,CACf,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAC9F;IACD,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAGH,4GAA4G;AAC5G,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAoC,EAAU,EAAE,CAC1E,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AAEtF,MAAM,UAAU,WAAW,CAAC,GAAa,EAAE,GAAY,EAAE,UAA8B,EAAE;IACvF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;IAC9B,OAAO,IAAI,CAAC,YAAY,EAAE;QACxB,GAAG,EAAE,GAAG,CAAC,IAAI;QACb,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC;QAC7B,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW;QACzE,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C,CAAC,CAAC,CAAC,EAAE;QACpE,IAAI,EAAE,GAAG,CAAC,IAAI;KACf,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CACvB,MAA4C,EAC5C,GAAa,EACb,IAAY;IAEZ,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACvD,MAAM,OAAO,GAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,IAAI;YACxB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,IAAI,KAAK,OAAO;gBAChB,CAAC,CAAC,IAAI;oBACJ,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,KAAK;gBACT,CAAC,CAAC,IAAI,CAAC;IACb,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED,+GAA+G;AAC/G,MAAM,UAAU,YAAY,CAAC,MAAmB,EAAE,GAAa;IAC7D,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,+FAA+F;QAC/F,MAAM,GAAG,GAAG,CAAC,KAAK;YAChB,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,eAAe;gBAC9B,CAAC,CAAC,aAAa;gBACf,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;oBACnC,CAAC,CAAC,iBAAiB;oBACnB,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC;wBACjC,CAAC,CAAC,qBAAqB;wBACvB,CAAC,CAAC,IAAI,CAAC;QACf,IAAI,GAAG;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;aAChC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;IACjB,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACtC,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM;QACrB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACrB,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,IAAI;oBAAE,CAAC,EAAE,CAAC;qBACf,IAAI,CAAC,KAAK,GAAG;oBAAE,QAAQ,GAAG,KAAK,CAAC;YACvC,CAAC;iBAAM,IAAI,CAAC,KAAK,GAAG;gBAAE,QAAQ,GAAG,IAAI,CAAC;iBACjC,IAAI,CAAC,KAAK,GAAG;gBAAE,KAAK,EAAE,CAAC;iBACvB,IAAI,CAAC,KAAK,GAAG,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpC,GAAG,GAAG,CAAC,CAAC;gBACR,MAAM;YACR,CAAC;iBAAM,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC;gBAAE,MAAM;QAC7C,CAAC;QACD,IAAI,GAAG,GAAG,CAAC;YAAE,MAAM;QACnB,MAAM,GAAG,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CACtD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAC1C,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,MAAM;QACxB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACd,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,IAAI,GAA4B,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;AAE7E,wGAAwG;AACxG,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa,EAAE,IAAY;IACzD,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,IAAI,EAAE,GAAG,CAAC;QAAE,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;IACnE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAc;IACvC,IAAI,EAAE,OAAO;IACb,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK;QAC7B,gFAAgF;QAChF,MAAM,GAAG,GAAa,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAc,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;QACrC,IAAI,KAAwB,CAAC;QAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,GAAmB,IAAI,CAAC;QAChC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,SAAS,CAAC;YACR,MAAM,IAAI,GAAa,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC3E,IAAI,MAAmB,CAAC;YACxB,IAAI,GAAG,GAAG,KAAK,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,GAAG,CACvB,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,EACxD,WAAW,CACZ,CAAC;gBACF,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC;gBACjB,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;oBAAE,MAAM,GAAG,CAAC;gBAC3B,GAAG,GAAG,IAAI,CAAC;gBACX,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC;YACD,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAC9D,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;gBAAE,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;YAC1F,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO;oBACL,QAAQ;oBACR,OAAO;oBACP,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpC,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,EACxE,CAAC,CAAC,CACH,CAAC;YACF,IAAI,IAAI,IAAI,IAAI;gBACd,MAAM,IAAI,KAAK,CACb,0BAA0B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,wCAAwC,KAAK,GAAG,CAAC,GAAG,UAAU,IAAI,GAAG,CAChJ,CAAC;YACJ,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,GAAG,IAAI,CAAC;YACZ,SAAS,EAAE,CAAC;QACd,CAAC;IACH,CAAC;CACF,CAAC"}

package/dist/document/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * The research engine's document side: `Document`, `Finding` and `Proof` (zod, so a ledger
+ * parses them back), the reader registry by media, the claim extractor, and the model seam an
+ * extractor takes. Who fetched a document and who kept a finding is the engine's and the ledger's.
+ */
+export * from "./body.js";
+export * from "./check.js";
+export * from "./extract.js";
+export * from "./model.js";
+export * from "./readers.js";
+export * from "./types.js";

package/dist/document/index.js

@@ -0,0 +1,12 @@
+/**
+ * The research engine's document side: `Document`, `Finding` and `Proof` (zod, so a ledger
+ * parses them back), the reader registry by media, the claim extractor, and the model seam an
+ * extractor takes. Who fetched a document and who kept a finding is the engine's and the ledger's.
+ */
+export * from "./body.js";
+export * from "./check.js";
+export * from "./extract.js";
+export * from "./model.js";
+export * from "./readers.js";
+export * from "./types.js";
+//# sourceMappingURL=index.js.map

package/dist/document/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/document/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC"}

package/dist/document/model.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The model seam an extractor uses: `@wrenautomation/models`' `Model`, of which an extractor
+ * calls only `ask` and reads only `id`. A model whose output cap cut the answer throws models'
+ * `Cut` with the text so far: the claim extractor keeps what is whole and asks again for the rest.
+ */
+import type { Model as FullModel } from "@wrenautomation/models";
+export type { Answer, Usage } from "@wrenautomation/models";
+export { addUsage, Cut, isCut } from "@wrenautomation/models";
+/** What an extractor needs of a model: any `@wrenautomation/models` model meets it. */
+export type Model = Pick<FullModel, "id" | "ask">;

package/dist/document/model.js

@@ -0,0 +1,2 @@
+export { addUsage, Cut, isCut } from "@wrenautomation/models";
+//# sourceMappingURL=model.js.map

package/dist/document/model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/document/model.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/document/readers.d.ts

@@ -0,0 +1,24 @@
+import { type Document, type Media } from "./types.js";
+/** What a source fetched for one URL. `text` for markup and text, `bytes` for a file. */
+export interface ReadInput {
+    url: string;
+    text?: string;
+    bytes?: Uint8Array;
+    /** What the source already knows: a hit's title, channel, date, length. */
+    title?: string;
+    channel?: string;
+    date?: string;
+    length?: number;
+}
+export interface Reader {
+    media: Media;
+    /** Null when the input cannot be read as this media (a scanned PDF, an empty page). */
+    read(input: ReadInput): Promise<Document | null>;
+}
+export declare const htmlReader: Reader;
+export declare const pdfReader: Reader;
+export declare const textReader: Reader;
+/** The readers by media. A caller adds its own (`video`, `audio`) or swaps one. */
+export declare const defaultReaders: () => Map<Media, Reader>;
+/** The media of a fetched body, from its content type first, then the URL's extension. */
+export declare function mediaOf(contentType: string | null | undefined, url: string): Media | null;

package/dist/document/readers.js

@@ -0,0 +1,87 @@
+/**
+ * A reader turns what a source fetched (markup, bytes or text) into one `Document`. The
+ * registry is keyed by media; a run picks the reader by the content type it got. Three ship
+ * here: html, pdf, text. Video and audio readers need tools and keys, so the caller registers
+ * those (the readers over yt-dlp in `@wrenautomation/read/video` return a `Document` directly).
+ */
+import { pdfText, readPage, tidy } from "@wrenautomation/read/text";
+import { Media as MediaShape } from "./types.js";
+const free = (input) => {
+    const out = {};
+    if (input.title)
+        out.title = input.title;
+    if (input.channel)
+        out.channel = input.channel;
+    if (input.date)
+        out.date = input.date;
+    if (input.length !== undefined)
+        out.length = input.length;
+    return out;
+};
+const textOf = (input) => input.text ?? (input.bytes ? new TextDecoder().decode(input.bytes) : "");
+export const htmlReader = {
+    media: "html",
+    async read(input) {
+        const page = readPage(textOf(input), input.url);
+        if (!page.text.trim())
+            return null;
+        return {
+            ...free(input),
+            ...(page.title ? { title: page.title } : {}),
+            url: input.url,
+            media: "html",
+            text: page.text,
+        };
+    },
+};
+export const pdfReader = {
+    media: "pdf",
+    async read(input) {
+        if (!input.bytes)
+            return null;
+        const text = await pdfText(input.bytes);
+        return text ? { ...free(input), url: input.url, media: "pdf", text } : null;
+    },
+};
+export const textReader = {
+    media: "text",
+    async read(input) {
+        const text = tidy(textOf(input));
+        return text ? { ...free(input), url: input.url, media: "text", text } : null;
+    },
+};
+/** The readers by media. A caller adds its own (`video`, `audio`) or swaps one. */
+export const defaultReaders = () => new Map([htmlReader, pdfReader, textReader].map((r) => [r.media, r]));
+/** The media of a fetched body, from its content type first, then the URL's extension. */
+export function mediaOf(contentType, url) {
+    const type = (contentType ?? "").split(";")[0]?.trim().toLowerCase() ?? "";
+    if (type === "application/pdf")
+        return "pdf";
+    if (type === "text/html" || type === "application/xhtml+xml")
+        return "html";
+    if (type.startsWith("text/"))
+        return "text";
+    if (type.startsWith("video/"))
+        return "video";
+    if (type.startsWith("audio/"))
+        return "audio";
+    let path = "";
+    try {
+        path = new URL(url).pathname.toLowerCase();
+    }
+    catch {
+        return null;
+    }
+    if (path.endsWith(".pdf"))
+        return "pdf";
+    if (/\.(txt|md|csv)$/.test(path))
+        return "text";
+    if (/\.(mp4|webm|mov|mkv)$/.test(path))
+        return "video";
+    if (/\.(mp3|m4a|wav|ogg)$/.test(path))
+        return "audio";
+    if (!type || type === "application/octet-stream")
+        return null;
+    return MediaShape.safeParse(type).success ? type : null;
+}
+//# sourceMappingURL=readers.js.map

package/dist/document/readers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readers.js","sourceRoot":"","sources":["../../src/document/readers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAA6B,KAAK,IAAI,UAAU,EAAE,MAAM,YAAY,CAAC;AAoB5E,MAAM,IAAI,GAAG,CAAC,KAAgB,EAAqB,EAAE;IACnD,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,KAAK;QAAE,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IACzC,IAAI,KAAK,CAAC,OAAO;QAAE,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC/C,IAAI,KAAK,CAAC,IAAI;QAAE,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1D,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAgB,EAAU,EAAE,CAC1C,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE3E,MAAM,CAAC,MAAM,UAAU,GAAW;IAChC,KAAK,EAAE,MAAM;IACb,KAAK,CAAC,IAAI,CAAC,KAAK;QACd,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO;YACL,GAAG,IAAI,CAAC,KAAK,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,MAAM;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAW;IAC/B,KAAK,EAAE,KAAK;IACZ,KAAK,CAAC,IAAI,CAAC,KAAK;QACd,IAAI,CAAC,KAAK,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAW;IAChC,KAAK,EAAE,MAAM;IACb,KAAK,CAAC,IAAI,CAAC,KAAK;QACd,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/E,CAAC;CACF,CAAC;AAEF,mFAAmF;AACnF,MAAM,CAAC,MAAM,cAAc,GAAG,GAAuB,EAAE,CACrD,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExE,0FAA0F;AAC1F,MAAM,UAAU,OAAO,CAAC,WAAsC,EAAE,GAAW;IACzE,MAAM,IAAI,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC3E,IAAI,IAAI,KAAK,iBAAiB;QAAE,OAAO,KAAK,CAAC;IAC7C,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,uBAAuB;QAAE,OAAO,MAAM,CAAC;IAC5E,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9C,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAChD,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IACvD,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IACtD,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,0BAA0B;QAAE,OAAO,IAAI,CAAC;IAC9D,OAAO,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,IAAc,CAAC,CAAC,CAAC,IAAI,CAAC;AACrE,CAAC"}

package/dist/document/types.d.ts

@@ -0,0 +1,99 @@
+/**
+ * What the research engine reads and keeps: a document (one URL read whole), a finding (one
+ * fact drawn from it) and its proof (where in the document). zod shapes, because a ledger
+ * parses them back. Who found it (`from`, `by`) is the ledger's envelope, not the finding's.
+ */
+import { z } from "zod";
+export declare const Media: z.ZodEnum<{
+    html: "html";
+    pdf: "pdf";
+    text: "text";
+    video: "video";
+    audio: "audio";
+}>;
+export type Media = z.infer<typeof Media>;
+/** One URL, read whole: a page's text, a video's markdown. The same free fields a hit shows. */
+export declare const Document: z.ZodObject<{
+    url: z.ZodString;
+    media: z.ZodEnum<{
+        html: "html";
+        pdf: "pdf";
+        text: "text";
+        video: "video";
+        audio: "audio";
+    }>;
+    title: z.ZodOptional<z.ZodString>;
+    text: z.ZodString;
+    channel: z.ZodOptional<z.ZodString>;
+    date: z.ZodOptional<z.ZodString>;
+    length: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export type Document = z.infer<typeof Document>;
+/** Where a finding rests: the exact words, a second into a recording, or a frame of it. */
+export declare const Proof: z.ZodObject<{
+    url: z.ZodString;
+    quote: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    frame: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type Proof = z.infer<typeof Proof>;
+/**
+ * One fact with its proof. `kind` names the extractor's sort of finding: `claim` in v1. Three
+ * signs this is too thin, to watch for: a finding needs two proofs, a proof needs a span inside
+ * a quote, a finding points at another finding.
+ */
+export declare const Finding: z.ZodObject<{
+    kind: z.ZodString;
+    text: z.ZodString;
+    proof: z.ZodObject<{
+        url: z.ZodString;
+        quote: z.ZodOptional<z.ZodString>;
+        timestamp: z.ZodOptional<z.ZodNumber>;
+        frame: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type Finding = z.infer<typeof Finding>;
+/** How much of the ask one document served: rich, thin, off (another subject) or empty (no text). */
+export declare const Quality: z.ZodEnum<{
+    rich: "rich";
+    thin: "thin";
+    off: "off";
+    empty: "empty";
+}>;
+export type Quality = z.infer<typeof Quality>;
+/**
+ * The extractor's word on one document after its findings: the quality, and the gap, one
+ * sentence on what the ask still lacks after this document. A filter's report to the agent,
+ * never a query: the agent reads the round's verdicts before it picks the edge.
+ */
+export declare const Verdict: z.ZodObject<{
+    quality: z.ZodEnum<{
+        rich: "rich";
+        thin: "thin";
+        off: "off";
+        empty: "empty";
+    }>;
+    gap: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type Verdict = z.infer<typeof Verdict>;
+/**
+ * What several findings say together, in the agent's words: one sentence, the findings it
+ * rests on, and how many distinct sites they come from (counted by code from the findings'
+ * proofs, never claimed). A pattern on one site is a claim; the count is what makes it a
+ * pattern. A reader judges the sentence; the count tells how far to trust it.
+ */
+export declare const Pattern: z.ZodObject<{
+    text: z.ZodString;
+    findings: z.ZodArray<z.ZodString>;
+    sources: z.ZodNumber;
+}, z.core.$strip>;
+export type Pattern = z.infer<typeof Pattern>;
+/** The host of a proof's URL, lower case, without `www.`; the URL itself when it is not one. */
+export declare const hostOf: (url: string) => string;
+/** A pattern over findings: the source count is the distinct hosts of their proofs. */
+export declare function patternOf(text: string, findings: readonly {
+    id: string;
+    proof: {
+        url: string;
+    };
+}[]): Pattern;

package/dist/document/types.js

@@ -0,0 +1,93 @@
+/**
+ * What the research engine reads and keeps: a document (one URL read whole), a finding (one
+ * fact drawn from it) and its proof (where in the document). zod shapes, because a ledger
+ * parses them back. Who found it (`from`, `by`) is the ledger's envelope, not the finding's.
+ */
+import { z } from "zod";
+export const Media = z.enum(["html", "pdf", "text", "video", "audio"]);
+/** One URL, read whole: a page's text, a video's markdown. The same free fields a hit shows. */
+export const Document = z.object({
+    url: z.string().min(1),
+    media: Media,
+    title: z.string().optional(),
+    /** All of it: never cut here. */
+    text: z.string(),
+    channel: z.string().optional(),
+    /** ISO date, when the document says one. */
+    date: z.string().optional(),
+    /** Seconds, for video and audio. */
+    length: z.number().nonnegative().optional(),
+});
+/** Where a finding rests: the exact words, a second into a recording, or a frame of it. */
+export const Proof = z
+    .object({
+    url: z.string().min(1),
+    /** Copied character for character from the document. */
+    quote: z.string().min(1).optional(),
+    /** Seconds into a video or audio. */
+    timestamp: z.number().nonnegative().optional(),
+    /** A frame's stamp (`mm:ss`) of a watched video. */
+    frame: z.string().min(1).optional(),
+})
+    .refine((p) => p.quote !== undefined || p.timestamp !== undefined || p.frame !== undefined, {
+    message: "a proof needs a quote, a timestamp or a frame",
+});
+/**
+ * One fact with its proof. `kind` names the extractor's sort of finding: `claim` in v1. Three
+ * signs this is too thin, to watch for: a finding needs two proofs, a proof needs a span inside
+ * a quote, a finding points at another finding.
+ */
+export const Finding = z.object({
+    kind: z.string().min(1),
+    /** One plain sentence. */
+    text: z.string().min(1),
+    proof: Proof,
+});
+/** How much of the ask one document served: rich, thin, off (another subject) or empty (no text). */
+export const Quality = z.enum(["rich", "thin", "off", "empty"]);
+/**
+ * The extractor's word on one document after its findings: the quality, and the gap, one
+ * sentence on what the ask still lacks after this document. A filter's report to the agent,
+ * never a query: the agent reads the round's verdicts before it picks the edge.
+ */
+export const Verdict = z.object({
+    quality: Quality,
+    gap: z.string().min(1).optional(),
+});
+/**
+ * What several findings say together, in the agent's words: one sentence, the findings it
+ * rests on, and how many distinct sites they come from (counted by code from the findings'
+ * proofs, never claimed). A pattern on one site is a claim; the count is what makes it a
+ * pattern. A reader judges the sentence; the count tells how far to trust it.
+ */
+export const Pattern = z.object({
+    /** One plain sentence. */
+    text: z.string().min(1),
+    /** Ids of the findings it rests on, in the ledger that holds them. */
+    findings: z.array(z.string().min(1)).min(1),
+    /** Distinct hosts across those findings' proofs. */
+    sources: z.number().int().positive(),
+});
+/** The host of a proof's URL, lower case, without `www.`; the URL itself when it is not one. */
+export const hostOf = (url) => {
+    try {
+        return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+    }
+    catch {
+        return url;
+    }
+};
+/** A pattern over findings: the source count is the distinct hosts of their proofs. */
+export function patternOf(text, findings) {
+    const sentence = text.replace(/\s+/g, " ").trim();
+    if (!sentence)
+        throw new Error("a pattern needs a sentence");
+    if (!findings.length)
+        throw new Error("a pattern needs at least one finding");
+    return {
+        text: sentence,
+        findings: findings.map((f) => f.id),
+        sources: new Set(findings.map((f) => hostOf(f.proof.url))).size,
+    };
+}
+//# sourceMappingURL=types.js.map

package/dist/document/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/document/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAGvE,gGAAgG;AAChG,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,iCAAiC;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,4CAA4C;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,oCAAoC;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAGH,2FAA2F;AAC3F,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC;KACnB,MAAM,CAAC;IACN,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,wDAAwD;IACxD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnC,qCAAqC;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC9C,oDAAoD;IACpD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACpC,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE;IAC1F,OAAO,EAAE,+CAA+C;CACzD,CAAC,CAAC;AAGL;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,0BAA0B;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,KAAK,EAAE,KAAK;CACb,CAAC,CAAC;AAGH,qGAAqG;AACrG,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;AAGhE;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,OAAO,EAAE,OAAO;IAChB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAGH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,0BAA0B;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,sEAAsE;IACtE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,oDAAoD;IACpD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAGH,gGAAgG;AAChG,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,GAAW,EAAU,EAAE;IAC5C,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC,CAAC;AAEF,uFAAuF;AACvF,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,QAA2D;IAE3D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC7D,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC9E,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KAChE,CAAC;AACJ,CAAC"}