@markbrutx/promptbook-core 0.1.0

package/dist/eval/assertions.js

@@ -0,0 +1,131 @@
+const EXCERPT_LIMIT = 160;
+/** A short, single-line excerpt of the output for failure messages. */
+function excerpt(text) {
+    const collapsed = text.replace(/\s+/g, " ").trim();
+    return collapsed.length > EXCERPT_LIMIT ? `${collapsed.slice(0, EXCERPT_LIMIT)}…` : collapsed;
+}
+function result(type, pass, message, output) {
+    return { type, pass, message, excerpt: excerpt(output) };
+}
+/** Require a spec field to be present, with a clear error when it is not. */
+function need(value, type, field) {
+    if (value === undefined) {
+        throw new Error(`assertion "${type}" requires a "${field}" field.`);
+    }
+    return value;
+}
+function buildRegex(assertion) {
+    const pattern = need(assertion.pattern, assertion.type, "pattern");
+    try {
+        return new RegExp(pattern, assertion.flags);
+    }
+    catch (error) {
+        throw new Error(`assertion "${assertion.type}" has an invalid pattern: ${error.message}`);
+    }
+}
+const contains = (output, a) => {
+    const value = need(a.value, "contains", "value");
+    const pass = output.includes(value);
+    return result("contains", pass, pass ? `contains "${value}"` : `missing "${value}"`, output);
+};
+const notContains = (output, a) => {
+    const value = need(a.value, "not-contains", "value");
+    const pass = !output.includes(value);
+    return result("not-contains", pass, pass ? `does not contain "${value}"` : `unexpectedly contains "${value}"`, output);
+};
+const matches = (output, a) => {
+    const re = buildRegex({ ...a, type: "matches" });
+    const pass = re.test(output);
+    return result("matches", pass, pass ? `matches /${re.source}/` : `does not match /${re.source}/`, output);
+};
+const notMatches = (output, a) => {
+    const re = buildRegex({ ...a, type: "not-matches" });
+    const pass = !re.test(output);
+    return result("not-matches", pass, pass ? `does not match /${re.source}/` : `unexpectedly matches /${re.source}/`, output);
+};
+const equals = (output, a) => {
+    const value = need(a.value, "equals", "value");
+    const pass = output.trim() === value.trim();
+    return result("equals", pass, pass ? "equals expected text" : "does not equal expected text", output);
+};
+const jsonValid = (output) => {
+    try {
+        JSON.parse(output);
+        return result("json-valid", true, "output is valid JSON", output);
+    }
+    catch (error) {
+        return result("json-valid", false, `output is not valid JSON: ${error.message}`, output);
+    }
+};
+const maxLength = (output, a) => {
+    const max = need(a.max, "max-length", "max");
+    const pass = output.length <= max;
+    return result("max-length", pass, pass ? `length ${output.length} within ${max}` : `length ${output.length} exceeds ${max}`, output);
+};
+/** Map common language tags to the script the `language` heuristic checks. */
+const SCRIPT_BY_LANGUAGE = {
+    ru: "cyrillic",
+    uk: "cyrillic",
+    be: "cyrillic",
+    bg: "cyrillic",
+    sr: "cyrillic",
+    mk: "cyrillic",
+    en: "latin",
+    fr: "latin",
+    es: "latin",
+    de: "latin",
+    it: "latin",
+    pt: "latin",
+    nl: "latin",
+    pl: "latin",
+    cs: "latin",
+    ro: "latin",
+    tr: "latin",
+    sv: "latin",
+    da: "latin",
+    no: "latin",
+    fi: "latin",
+};
+const CYRILLIC = /\p{Script=Cyrillic}/gu;
+const LATIN = /\p{Script=Latin}/gu;
+function count(text, re) {
+    return text.match(re)?.length ?? 0;
+}
+/**
+ * `language`: script-based heuristic. Resolves the spec `lang` to a script
+ * ("cyrillic"/"latin", either directly or via a language tag) and passes when
+ * that script has at least one letter and is at least as frequent as the
+ * other. Not an LLM judge — it detects script, not fluency.
+ */
+const language = (output, a) => {
+    const lang = need(a.lang, "language", "lang").toLowerCase();
+    const script = SCRIPT_BY_LANGUAGE[lang] ?? lang;
+    if (script !== "cyrillic" && script !== "latin") {
+        return result("language", false, `unsupported language "${a.lang}" (no script heuristic)`, output);
+    }
+    const cyrillic = count(output, CYRILLIC);
+    const latin = count(output, LATIN);
+    const expected = script === "cyrillic" ? cyrillic : latin;
+    const other = script === "cyrillic" ? latin : cyrillic;
+    const pass = expected > 0 && expected >= other;
+    return result("language", pass, pass
+        ? `output is predominantly ${script}`
+        : `expected predominantly ${script} (cyrillic=${cyrillic}, latin=${latin})`, output);
+};
+/**
+ * The built-in assertion registry. Pass a merged/replaced object to
+ * {@link evaluate} to customize: `{ ...defaultAssertions(), myType: fn }`.
+ */
+export function defaultAssertions() {
+    return {
+        contains,
+        "not-contains": notContains,
+        matches,
+        "not-matches": notMatches,
+        equals,
+        "json-valid": jsonValid,
+        "max-length": maxLength,
+        language,
+    };
+}
+//# sourceMappingURL=assertions.js.map

package/dist/eval/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../src/eval/assertions.ts"],"names":[],"mappings":"AAUA,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,uEAAuE;AACvE,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,OAAO,SAAS,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AAChG,CAAC;AAED,SAAS,MAAM,CAAC,IAAY,EAAE,IAAa,EAAE,OAAe,EAAE,MAAc;IAC1E,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED,6EAA6E;AAC7E,SAAS,IAAI,CAAI,KAAoB,EAAE,IAAY,EAAE,KAAa;IAChE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,iBAAiB,KAAK,UAAU,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,SAAoB;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,cAAc,SAAS,CAAC,IAAI,6BAA8B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;AACH,CAAC;AAED,MAAM,QAAQ,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,GAAG,EAAE,MAAM,CAAC,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,WAAW,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,MAAM,CACX,cAAc,EACd,IAAI,EACJ,IAAI,CAAC,CAAC,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,CAAC,0BAA0B,KAAK,GAAG,EACzE,MAAM,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,OAAO,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IACzC,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,EAAE,MAAM,CAAC,CAAC;AAC5G,CAAC,CAAC;AAEF,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC5C,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,OAAO,MAAM,CACX,aAAa,EACb,IAAI,EACJ,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,yBAAyB,EAAE,CAAC,MAAM,GAAG,EAC9E,MAAM,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,MAAM,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5C,OAAO,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;AACxG,CAAC,CAAC;AAEF,MAAM,SAAS,GAAgB,CAAC,MAAM,EAAE,EAAE;IACxC,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,CAAC,CAAC;IACpE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,6BAA8B,KAAe,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IACtG,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,SAAS,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,OAAO,MAAM,CACX,YAAY,EACZ,IAAI,EACJ,IAAI,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,EACzF,MAAM,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,kBAAkB,GAAyC;IAC/D,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACZ,CAAC;AAEF,MAAM,QAAQ,GAAG,uBAAuB,CAAC;AACzC,MAAM,KAAK,GAAG,oBAAoB,CAAC;AAEnC,SAAS,KAAK,CAAC,IAAY,EAAE,EAAU;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;;;GAKG;AACH,MAAM,QAAQ,GAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5D,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IAChD,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QAChD,OAAO,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,yBAAyB,CAAC,CAAC,IAAI,yBAAyB,EAAE,MAAM,CAAC,CAAC;IACrG,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,MAAM,KAAK,GAAG,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvD,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,IAAI,QAAQ,IAAI,KAAK,CAAC;IAC/C,OAAO,MAAM,CACX,UAAU,EACV,IAAI,EACJ,IAAI;QACF,CAAC,CAAC,2BAA2B,MAAM,EAAE;QACrC,CAAC,CAAC,0BAA0B,MAAM,cAAc,QAAQ,WAAW,KAAK,GAAG,EAC7E,MAAM,CACP,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO;QACL,QAAQ;QACR,cAAc,EAAE,WAAW;QAC3B,OAAO;QACP,aAAa,EAAE,UAAU;QACzB,MAAM;QACN,YAAY,EAAE,SAAS;QACvB,YAAY,EAAE,SAAS;QACvB,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/eval/evaluate.d.ts

@@ -0,0 +1,15 @@
+import type { EvalInput, EvalReport } from "./types.js";
+/**
+ * Run fixtures through a model adapter and report pass-rate.
+ *
+ * For each fixture: assemble the system prompt once via {@link resolveBook}
+ * (deterministic — every sample of a fixture sees the same prompt), then take
+ * N samples through `adapter.complete` and run the assertions on each. A sample
+ * passes only when all its assertions pass; `passRate = passes / samples`. A
+ * fixture meets the gate when `passRate >= passThreshold` (default 1).
+ *
+ * The engine is pure given the adapter: its control flow is deterministic and
+ * the only stochasticity/IO is `adapter.complete`. It makes no network calls.
+ */
+export declare function evaluate(input: EvalInput): Promise<EvalReport>;
+//# sourceMappingURL=evaluate.d.ts.map

package/dist/eval/evaluate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/eval/evaluate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAIV,SAAS,EACT,UAAU,EAGX,MAAM,YAAY,CAAC;AAapB;;;;;;;;;;;GAWG;AACH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAiBpE"}

package/dist/eval/evaluate.js

@@ -0,0 +1,65 @@
+import { resolveBook } from "../resolve-book.js";
+import { defaultAssertions } from "./assertions.js";
+/** Run every assertion of a fixture against one model output. */
+function runAssertions(output, asserts, registry) {
+    return asserts.map((assertion) => {
+        const fn = registry[assertion.type];
+        if (fn === undefined) {
+            return { type: assertion.type, pass: false, message: `unknown assertion type "${assertion.type}"` };
+        }
+        return fn(output, assertion);
+    });
+}
+/**
+ * Run fixtures through a model adapter and report pass-rate.
+ *
+ * For each fixture: assemble the system prompt once via {@link resolveBook}
+ * (deterministic — every sample of a fixture sees the same prompt), then take
+ * N samples through `adapter.complete` and run the assertions on each. A sample
+ * passes only when all its assertions pass; `passRate = passes / samples`. A
+ * fixture meets the gate when `passRate >= passThreshold` (default 1).
+ *
+ * The engine is pure given the adapter: its control flow is deterministic and
+ * the only stochasticity/IO is `adapter.complete`. It makes no network calls.
+ */
+export async function evaluate(input) {
+    const registry = input.assertions ?? defaultAssertions();
+    const defaultSamples = input.samples ?? 1;
+    const threshold = input.passThreshold ?? 1;
+    const results = [];
+    for (const fixture of input.fixtures) {
+        results.push(await runFixture(fixture, input, registry, defaultSamples));
+    }
+    const passed = results.filter((r) => r.passRate >= threshold).length;
+    return {
+        results,
+        passed,
+        failed: results.length - passed,
+        passRate: results.length === 0 ? 1 : passed / results.length,
+    };
+}
+async function runFixture(fixture, input, registry, defaultSamples) {
+    const { text: system } = resolveBook(input.book, fixture.prompt, fixture.context ?? {});
+    const samples = fixture.samples ?? defaultSamples;
+    let passes = 0;
+    const failures = [];
+    for (let i = 0; i < samples; i += 1) {
+        const response = await input.adapter.complete({ system, input: fixture.input });
+        const sampleResults = runAssertions(response.text, fixture.assert, registry);
+        const failed = sampleResults.filter((r) => !r.pass);
+        if (failed.length === 0) {
+            passes += 1;
+        }
+        else {
+            failures.push(...failed);
+        }
+    }
+    return {
+        name: fixture.name,
+        samples,
+        passes,
+        passRate: samples === 0 ? 0 : passes / samples,
+        failures,
+    };
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/eval/evaluate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/eval/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAWpD,iEAAiE;AACjE,SAAS,aAAa,CAAC,MAAc,EAAE,OAAoB,EAAE,QAA2B;IACtF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,2BAA2B,SAAS,CAAC,IAAI,GAAG,EAAE,CAAC;QACtG,CAAC;QACD,OAAO,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAgB;IAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,IAAI,iBAAiB,EAAE,CAAC;IACzD,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;IAE3C,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,MAAM,CAAC;IACrE,OAAO;QACL,OAAO;QACP,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;KAC7D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,OAAgB,EAChB,KAAgB,EAChB,QAA2B,EAC3B,cAAsB;IAEtB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,CAAC;IAElD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QAChF,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,CAAC;QACd,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO;QACP,MAAM;QACN,QAAQ,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO;QAC9C,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/eval/load-fixtures.d.ts

@@ -0,0 +1,12 @@
+import type { FsAdapter } from "../types.js";
+import type { Fixture } from "./types.js";
+/**
+ * Load eval fixtures from `<dir>/fixtures/*.json` into {@link Fixture}s.
+ *
+ * Mirrors {@link loadPrompts}: the same `dir` that holds `fragments/` and
+ * `rules/` also holds `fixtures/`. A missing `fixtures/` folder yields an empty
+ * list; a malformed fixture throws with its file path so the caller can report
+ * it. Files are read in sorted order for deterministic results.
+ */
+export declare function loadFixtures(dir: string, fs?: FsAdapter): Promise<Fixture[]>;
+//# sourceMappingURL=load-fixtures.d.ts.map

package/dist/eval/load-fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-fixtures.d.ts","sourceRoot":"","sources":["../../src/eval/load-fixtures.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAW,SAAS,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,KAAK,EAAa,OAAO,EAAE,MAAM,YAAY,CAAC;AA8DrD;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,GAAE,SAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAgB5F"}

package/dist/eval/load-fixtures.js

@@ -0,0 +1,87 @@
+import { nodeFs } from "../fs.js";
+import { isContextValue, isMapping } from "../guards.js";
+import { joinPath, listFiles, stripExt } from "../paths.js";
+function parseContext(raw, file) {
+    if (raw === undefined) {
+        return undefined;
+    }
+    if (!isMapping(raw)) {
+        throw new Error(`Fixture "${file}" has a "context" that is not an object.`);
+    }
+    const context = {};
+    for (const [key, value] of Object.entries(raw)) {
+        if (!isContextValue(value)) {
+            throw new Error(`Fixture "${file}" context key "${key}" must be a string, number or boolean.`);
+        }
+        context[key] = value;
+    }
+    return context;
+}
+function parseAssertions(raw, file) {
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new Error(`Fixture "${file}" must declare a non-empty "assert" array.`);
+    }
+    return raw.map((entry, index) => {
+        if (!isMapping(entry) || typeof entry.type !== "string" || entry.type === "") {
+            throw new Error(`Fixture "${file}" assertion #${index} must be an object with a string "type".`);
+        }
+        return entry;
+    });
+}
+function parseFixture(raw, file, fallbackName) {
+    if (!isMapping(raw)) {
+        throw new Error(`Fixture "${file}" is not a JSON object.`);
+    }
+    if (typeof raw.prompt !== "string" || raw.prompt === "") {
+        throw new Error(`Fixture "${file}" must declare a string "prompt".`);
+    }
+    if (typeof raw.input !== "string") {
+        throw new Error(`Fixture "${file}" must declare a string "input".`);
+    }
+    const name = typeof raw.name === "string" && raw.name !== "" ? raw.name : fallbackName;
+    const fixture = {
+        name,
+        prompt: raw.prompt,
+        input: raw.input,
+        assert: parseAssertions(raw.assert, file),
+        sourceFile: file,
+    };
+    const context = parseContext(raw.context, file);
+    if (context !== undefined) {
+        fixture.context = context;
+    }
+    if (raw.samples !== undefined) {
+        if (typeof raw.samples !== "number" || !Number.isInteger(raw.samples) || raw.samples <= 0) {
+            throw new Error(`Fixture "${file}" "samples" must be a positive integer.`);
+        }
+        fixture.samples = raw.samples;
+    }
+    return fixture;
+}
+/**
+ * Load eval fixtures from `<dir>/fixtures/*.json` into {@link Fixture}s.
+ *
+ * Mirrors {@link loadPrompts}: the same `dir` that holds `fragments/` and
+ * `rules/` also holds `fixtures/`. A missing `fixtures/` folder yields an empty
+ * list; a malformed fixture throws with its file path so the caller can report
+ * it. Files are read in sorted order for deterministic results.
+ */
+export async function loadFixtures(dir, fs = nodeFs()) {
+    const fixturesDir = joinPath(dir, "fixtures");
+    const files = await listFiles(fs, fixturesDir, [".json"]);
+    const fixtures = [];
+    for (const file of files) {
+        const full = joinPath(fixturesDir, file);
+        const raw = await fs.readFile(full);
+        let doc;
+        try {
+            doc = JSON.parse(raw);
+        }
+        catch (error) {
+            throw new Error(`Fixture "${full}" is not valid JSON: ${error.message}`);
+        }
+        fixtures.push(parseFixture(doc, full, stripExt(file)));
+    }
+    return fixtures;
+}
+//# sourceMappingURL=load-fixtures.js.map

package/dist/eval/load-fixtures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-fixtures.js","sourceRoot":"","sources":["../../src/eval/load-fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAI5D,SAAS,YAAY,CAAC,GAAY,EAAE,IAAY;IAC9C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,0CAA0C,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,OAAO,GAAY,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,kBAAkB,GAAG,wCAAwC,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,GAAY,EAAE,IAAY;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,4CAA4C,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;YAC7E,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,gBAAgB,KAAK,0CAA0C,CAAC,CAAC;QACnG,CAAC;QACD,OAAO,KAA6B,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,IAAY,EAAE,YAAoB;IACpE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,yBAAyB,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,mCAAmC,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,kCAAkC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC;IACvF,MAAM,OAAO,GAAY;QACvB,IAAI;QACJ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC;QACzC,UAAU,EAAE,IAAI;KACjB,CAAC;IACF,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAChD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;IAC5B,CAAC;IACD,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;YAC1F,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,yCAAyC,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAChC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,EAAE,GAAc,MAAM,EAAE;IACtE,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,GAAY,CAAC;QACjB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,wBAAyB,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/eval/types.d.ts

@@ -0,0 +1,123 @@
+/**
+ * Public types for the eval engine.
+ *
+ * Eval is the one place stochasticity enters the system, and it is locked
+ * behind an injectable {@link ModelAdapter}. The engine's control flow is
+ * deterministic: given the same adapter outputs it returns the same report.
+ * The core ships no adapter and makes no network calls — a concrete adapter
+ * (e.g. `@markbrutx/promptbook-openrouter`) lives in its own package.
+ */
+import type { Context, PromptBook } from "../types.js";
+/** Token accounting, when an adapter reports it. Provider-agnostic shape. */
+export interface ModelUsage {
+    promptTokens?: number;
+    completionTokens?: number;
+    totalTokens?: number;
+}
+/** One model call: the assembled system prompt plus the user input. */
+export interface ModelRequest {
+    /** The assembled system prompt from {@link resolveBook}. */
+    system: string;
+    /** The user input that accompanies the system prompt. */
+    input: string;
+    /** Optional per-request model override; adapters define the default. */
+    model?: string;
+}
+/** A model's reply. `raw` carries the provider payload for debugging. */
+export interface ModelResponse {
+    text: string;
+    usage?: ModelUsage;
+    raw?: unknown;
+}
+/**
+ * The seam stochasticity is locked behind. Implementations perform IO/network;
+ * the eval engine only ever calls `complete`, so it stays pure given a fake.
+ */
+export interface ModelAdapter {
+    complete(request: ModelRequest): Promise<ModelResponse>;
+}
+/**
+ * A single assertion spec as authored in a fixture JSON. `type` selects the
+ * checker from the assertion registry; the remaining fields are its params.
+ */
+export interface Assertion {
+    /** Registry key, e.g. "contains" or "language". */
+    type: string;
+    /** Substring (contains/not-contains) or exact text (equals). */
+    value?: string;
+    /** Regex source for matches/not-matches. */
+    pattern?: string;
+    /** Regex flags for matches/not-matches. */
+    flags?: string;
+    /** Character ceiling for max-length. */
+    max?: number;
+    /** Expected language tag or script for `language` (e.g. "ru", "latin"). */
+    lang?: string;
+}
+/** The outcome of running one assertion against one model output. */
+export interface AssertionResult {
+    /** The assertion `type` that produced this result. */
+    type: string;
+    pass: boolean;
+    /** Human-readable, domain-agnostic explanation. */
+    message: string;
+    /** A short excerpt of the output relevant to the assertion. */
+    excerpt?: string;
+}
+/** A checker for one assertion type. Pure: same output + spec, same result. */
+export type AssertionFn = (output: string, assertion: Assertion) => AssertionResult;
+/** Map of assertion `type` to its checker. Callers may supply their own. */
+export type AssertionRegistry = Record<string, AssertionFn>;
+/** A single eval test case loaded from a `fixtures/*.json` file. */
+export interface Fixture {
+    /** Lookup name; defaults to the file stem when omitted in JSON. */
+    name: string;
+    /** Composition name to assemble for this case. */
+    prompt: string;
+    /** Facts to resolve the composition under. */
+    context?: Context;
+    /** User input sent alongside the assembled system prompt. */
+    input: string;
+    /** Assertions every sample's output must satisfy. */
+    assert: Assertion[];
+    /** Per-fixture sample count; overrides the run-level default. */
+    samples?: number;
+    /** File the fixture was loaded from. */
+    sourceFile?: string;
+}
+/** Per-fixture aggregate over its samples. */
+export interface FixtureResult {
+    name: string;
+    /** Number of samples taken. */
+    samples: number;
+    /** Samples where every assertion passed. */
+    passes: number;
+    /** `passes / samples` (0 when `samples` is 0). */
+    passRate: number;
+    /** Every failing assertion across all samples, with messages/excerpts. */
+    failures: AssertionResult[];
+}
+/** The aggregated outcome of an eval run. */
+export interface EvalReport {
+    results: FixtureResult[];
+    /** Fraction of fixtures that met the threshold gate (`passed / total`). */
+    passRate: number;
+    /** Fixtures whose `passRate >= passThreshold`. */
+    passed: number;
+    /** Fixtures whose `passRate < passThreshold`. */
+    failed: number;
+}
+/** Input to {@link evaluate}. */
+export interface EvalInput {
+    book: PromptBook;
+    fixtures: Fixture[];
+    /** The injected model seam; all stochasticity/IO lives here. */
+    adapter: ModelAdapter;
+    /** Assertion checkers; defaults to {@link defaultAssertions}. */
+    assertions?: AssertionRegistry;
+    /** Default sample count when a fixture sets none. Defaults to 1. */
+    samples?: number;
+    /** A fixture passes when `passRate >= passThreshold`. Defaults to 1. */
+    passThreshold?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEvD,6EAA6E;AAC7E,MAAM,WAAW,UAAU;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,uEAAuE;AACvE,MAAM,WAAW,YAAY;IAC3B,4DAA4D;IAC5D,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,yEAAyE;AACzE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACzD;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,IAAI,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qEAAqE;AACrE,MAAM,WAAW,eAAe;IAC9B,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,+EAA+E;AAC/E,MAAM,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,KAAK,eAAe,CAAC;AAEpF,4EAA4E;AAC5E,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;AAE5D,oEAAoE;AACpE,MAAM,WAAW,OAAO;IACtB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,iEAAiE;IACjE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,0EAA0E;IAC1E,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAED,6CAA6C;AAC7C,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,iCAAiC;AACjC,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,gEAAgE;IAChE,OAAO,EAAE,YAAY,CAAC;IACtB,iEAAiE;IACjE,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,oEAAoE;IACpE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB"}

package/dist/eval/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}

package/dist/frontmatter.d.ts

@@ -0,0 +1,12 @@
+export interface ParsedFrontmatter {
+    data: Record<string, unknown>;
+    body: string;
+}
+/**
+ * Split a Markdown file into YAML frontmatter (parsed) and the body text.
+ *
+ * Intentionally dependency-light: a single YAML parser, no extra frontmatter
+ * library, so the core stays small and easy to run anywhere.
+ */
+export declare function parseFrontmatter(raw: string): ParsedFrontmatter;
+//# sourceMappingURL=frontmatter.d.ts.map

package/dist/frontmatter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../src/frontmatter.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;CACd;AAKD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,CAU/D"}

package/dist/frontmatter.js

@@ -0,0 +1,22 @@
+import { parse as parseYaml } from "yaml";
+import { isMapping } from "./guards.js";
+/** Matches a leading `---\n ... \n---` YAML frontmatter block. */
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/;
+/**
+ * Split a Markdown file into YAML frontmatter (parsed) and the body text.
+ *
+ * Intentionally dependency-light: a single YAML parser, no extra frontmatter
+ * library, so the core stays small and easy to run anywhere.
+ */
+export function parseFrontmatter(raw) {
+    const match = raw.match(FRONTMATTER_RE);
+    if (!match) {
+        return { data: {}, body: raw };
+    }
+    const yamlText = match[1] ?? "";
+    const parsed = parseYaml(yamlText);
+    const data = isMapping(parsed) ? parsed : {};
+    const body = raw.slice(match[0].length);
+    return { data, body };
+}
+//# sourceMappingURL=frontmatter.js.map

package/dist/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../src/frontmatter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAOxC,kEAAkE;AAClE,MAAM,cAAc,GAAG,mCAAmC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IACjC,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAY,CAAC;IAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC"}

package/dist/fs.d.ts

@@ -0,0 +1,11 @@
+import type { FsAdapter } from "./types.js";
+/**
+ * Default filesystem adapter backed by `node:fs/promises`.
+ *
+ * The import is lazy so that simply importing the core does not pull in any
+ * Node-only module: callers in Deno/Bun (or tests) can inject their own
+ * {@link FsAdapter} and never touch this path. The module is resolved once
+ * per adapter (when `nodeFs()` is called), not on every read.
+ */
+export declare function nodeFs(): FsAdapter;
+//# sourceMappingURL=fs.d.ts.map

package/dist/fs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;;;;GAOG;AACH,wBAAgB,MAAM,IAAI,SAAS,CAUlC"}

package/dist/fs.js

@@ -0,0 +1,20 @@
+/**
+ * Default filesystem adapter backed by `node:fs/promises`.
+ *
+ * The import is lazy so that simply importing the core does not pull in any
+ * Node-only module: callers in Deno/Bun (or tests) can inject their own
+ * {@link FsAdapter} and never touch this path. The module is resolved once
+ * per adapter (when `nodeFs()` is called), not on every read.
+ */
+export function nodeFs() {
+    const fsModule = import("node:fs/promises");
+    return {
+        async readFile(path) {
+            return (await fsModule).readFile(path, "utf8");
+        },
+        async readDir(path) {
+            return (await fsModule).readdir(path);
+        },
+    };
+}
+//# sourceMappingURL=fs.js.map

package/dist/fs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,MAAM,UAAU,MAAM;IACpB,MAAM,QAAQ,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC5C,OAAO;QACL,KAAK,CAAC,QAAQ,CAAC,IAAY;YACzB,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,IAAY;YACxB,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/guards.d.ts

@@ -0,0 +1,6 @@
+import type { ContextValue } from "./types.js";
+/** True for a plain object (a YAML/JSON mapping); excludes arrays and null. */
+export declare function isMapping(value: unknown): value is Record<string, unknown>;
+/** True for a scalar that is a valid {@link ContextValue} (string/number/boolean). */
+export declare function isContextValue(value: unknown): value is ContextValue;
+//# sourceMappingURL=guards.d.ts.map

package/dist/guards.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"guards.d.ts","sourceRoot":"","sources":["../src/guards.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,+EAA+E;AAC/E,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE1E;AAED,sFAAsF;AACtF,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAEpE"}

package/dist/guards.js

@@ -0,0 +1,9 @@
+/** True for a plain object (a YAML/JSON mapping); excludes arrays and null. */
+export function isMapping(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+/** True for a scalar that is a valid {@link ContextValue} (string/number/boolean). */
+export function isContextValue(value) {
+    return typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}
+//# sourceMappingURL=guards.js.map

package/dist/guards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"guards.js","sourceRoot":"","sources":["../src/guards.ts"],"names":[],"mappings":"AAEA,+EAA+E;AAC/E,MAAM,UAAU,SAAS,CAAC,KAAc;IACtC,OAAO,CAAC,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AACvE,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,CAAC;AAC9F,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,22 @@
+export type { Annotation, AnnotationAnchor, AnnotationStatus, AnnotationTarget, } from "./annotations.js";
+export { ANNOTATION_QUEUE_DIR, ANNOTATION_QUEUE_FILE, parseInbox, serializeAnnotationLine, serializeInbox, } from "./annotations.js";
+export type { SerializeBookOptions } from "./bundle.js";
+export { serializeBook, serializeBookExpression, serializeBookJson } from "./bundle.js";
+export { defaultAssertions } from "./eval/assertions.js";
+export { evaluate } from "./eval/evaluate.js";
+export { loadFixtures } from "./eval/load-fixtures.js";
+export type { Assertion, AssertionFn, AssertionRegistry, AssertionResult, EvalInput, EvalReport, Fixture, FixtureResult, ModelAdapter, ModelRequest, ModelResponse, ModelUsage, } from "./eval/types.js";
+export { parseFrontmatter } from "./frontmatter.js";
+export { nodeFs } from "./fs.js";
+export { interpolate } from "./interpolate.js";
+export { lint } from "./lint/lint.js";
+export type { FragmentReference } from "./lint/references.js";
+export { iterateReferences } from "./lint/references.js";
+export type { BannedTokensOptions, DanglingReferenceOptions, DeadRuleOptions, ExampleBalanceOptions, LanguageDirectivePositionOptions, TokenBudgetOptions, UnusedFragmentOptions, } from "./lint/rules/index.js";
+export { bannedTokens, type DefaultRulesOptions, danglingReference, deadRule, defaultRules, estimateTokensByChars, exampleBalance, languageDirectivePosition, tokenBudget, unusedFragment, } from "./lint/rules/index.js";
+export type { LintFinding, LintInput, LintReport, LintRule, LintScope, Severity } from "./lint/types.js";
+export { loadPrompts } from "./load.js";
+export { resolve } from "./resolve.js";
+export { resolveBook } from "./resolve-book.js";
+export type { AddTrace, CodePrompt, CodePromptSample, Composition, Context, ContextValue, ForbidTrace, Fragment, FsAdapter, PromptBook, ReplaceTrace, ResolveInput, ResolveResult, Rule, RuleAction, RuleTrace, Trace, UnmatchedAxis, When, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,UAAU,EACV,uBAAuB,EACvB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EACV,SAAS,EACT,WAAW,EACX,iBAAiB,EACjB,eAAe,EACf,SAAS,EACT,UAAU,EACV,OAAO,EACP,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,YAAY,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,YAAY,EACV,mBAAmB,EACnB,wBAAwB,EACxB,eAAe,EACf,qBAAqB,EACrB,gCAAgC,EAChC,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,YAAY,EACZ,KAAK,mBAAmB,EACxB,iBAAiB,EACjB,QAAQ,EACR,YAAY,EACZ,qBAAqB,EACrB,cAAc,EACd,yBAAyB,EACzB,WAAW,EACX,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACzG,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EACV,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,OAAO,EACP,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,EACT,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,IAAI,EACJ,UAAU,EACV,SAAS,EACT,KAAK,EACL,aAAa,EACb,IAAI,GACL,MAAM,YAAY,CAAC"}