@llm4ts/core 0.1.0

package/dist/UsageLimits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"UsageLimits.js","sourceRoot":"","sources":["../src/UsageLimits.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAiB,MAAM,aAAa,CAAA;AAE5E,MAAM,OAAO,GAAG,yCAAyC,CAAA;AACzD,MAAM,QAAQ,GAAG,wEAAwE,CAAA;AACzF,MAAM,WAAW,GAAG,wCAAwC,CAAA;AAC5D,MAAM,cAAc,GAAG,oBAAoB,CAAA;AAC3C,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;AAE7C,MAAM,kBAAkB,GAA0B;IAChD,yBAAyB;IACzB,OAAO;IACP,oBAAoB;IACpB,YAAY;IACZ,YAAY;IACZ,KAAK;CACN,CAAA;AAED,MAAM,mBAAmB,GAA0B;IACjD,GAAG,kBAAkB;IACrB,aAAa;IACb,gBAAgB;IAChB,mBAAmB;CACpB,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,OAA8B,EAAW,EAAE;IAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IACrC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;AAC9D,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;AAEhG,MAAM,cAAc,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAA;AAE3F,MAAM,aAAa,GAAG,CAAC,aAAqB,EAAqB,EAAE;IACjE,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAA;QACtC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;QAE7C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,GAAG;gBACN,YAAY,IAAI,MAAM,GAAG,UAAU,CAAA;gBACnC,MAAK;YACP,KAAK,GAAG;gBACN,YAAY,IAAI,MAAM,GAAG,SAAS,CAAA;gBAClC,MAAK;YACP,KAAK,GAAG;gBACN,YAAY,IAAI,MAAM,GAAG,MAAM,CAAA;gBAC/B,MAAK;YACP,KAAK,GAAG;gBACN,YAAY,IAAI,MAAM,GAAG,KAAK,CAAA;gBAC9B,MAAK;YACP;gBACE,YAAY,IAAI,MAAM,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;AACtC,CAAC,CAAA;AAOD,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAA6B,EAAE;IACpE,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;IACrE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;IAC7C,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,EAAE,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;QACjE,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,OAAO;QACL,IAAI,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAQD,MAAM,iBAAiB,GAAG,CAAC,QAAgB,EAAuB,EAAE,CAClE,IAAI,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;IACzC,QAAQ;IACR,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,SAAS;IAChB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,KAAK;CACjB,CAAC,CAAA;AAEJ,MAAM,WAAW,GAAG,CAClB,KAA6C,EAC7C,IAAkC,EAC1B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC,CAAA;AAE3E,MAAM,UAAU,GAAG,CAAC,WAAmB,EAAE,QAAgB,EAAE,EAAE;IAC3D,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAA;IAC9E,OAAO;QACL,IAAI,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;QAChC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC;QAClC,GAAG,EAAE,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC;QAC9B,IAAI,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC;QAChC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC;QACpC,MAAM,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC;KACrC,CAAA;AACH,CAAC,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,WAAmB,EAAE,QAAgB,EAAU,EAAE;IAC7E,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IAC/C,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,KAAK,GAAG,CAAC,EACf,KAAK,CAAC,GAAG,EACT,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,MAAM,CACb,CAAA;IACD,OAAO,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,KAAK,CAAA;AACnE,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG,CAAC,IAAkB,EAAE,IAAmB,EAAE,QAAgB,EAAU,EAAE;IAC9F,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACtF,MAAM,cAAc,GAAG,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IAC1E,OAAO,QAAQ,GAAG,oBAAoB,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAA;AAClE,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAG,CAAC,IAAkB,EAAgB,EAAE;IAC5D,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;IACxE,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE;QAC3B,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC;QAC7B,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE;KACvB,CAAA;AACH,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,CACrB,IAAY,EACZ,OAAe,EACf,GAAiB,EACjB,QAAgB,EACU,EAAE;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAA;IAC5E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;QAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;QAChD,MAAM,KAAK,GAAiB;YAC1B,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAA;QACD,MAAM,cAAc,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QAC/D,MAAM,WAAW,GACf,cAAc,GAAG,SAAS;YACxB,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QAEhE,OAAO,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAA;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,QAAgB,EAChB,IAAY,EACZ,GAAiB,EACjB,QAAgB,EACM,EAAE;IACxB,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;YAC5D,OAAO,OAAO,KAAK,SAAS;gBAC1B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;oBACnB,OAAO;oBACP,QAAQ;oBACR,OAAO,EAAE,IAAI;iBACd,CAAC,CAAA;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;YAC7D,OAAO,OAAO,KAAK,SAAS;gBAC1B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;oBACnB,OAAO;oBACP,QAAQ;oBACR,OAAO,EAAE,IAAI;iBACd,CAAC,CAAA;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,kBAAkB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACvD,IAAI,kBAAkB,KAAK,SAAS,EAAE,CAAC;gBACrC,MAAM,QAAQ,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAA;gBAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC;oBACtE,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;oBAC/C,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;wBACnB,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC;wBAC5C,QAAQ;wBACR,OAAO,EAAE,IAAI;qBACd,CAAC,CAAA;YACR,CAAC;YAED,OAAO,aAAa,CAAC,IAAI,CAAC;gBACxB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;oBACnB,QAAQ;oBACR,OAAO,EAAE,IAAI;iBACd,CAAC;gBACJ,CAAC,CAAC,SAAS,CAAA;QACf,CAAC;QACD,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,cAAc,CAAC,IAAI,CAAC;gBACzB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;oBACnB,QAAQ;oBACR,OAAO,EAAE,IAAI;iBACd,CAAC;gBACJ,CAAC,CAAC,SAAS,CAAA;QACf;YACE,OAAO,SAAS,CAAA;IACpB,CAAC;AACH,CAAC,CAAA"}

package/dist/eval/Checks.d.ts

@@ -0,0 +1,6 @@
+import { type Evaluator } from "./Evaluator.ts";
+export declare const noPii: (maxScore?: number) => Evaluator<string>;
+export declare const validJson: (maxScore?: number) => Evaluator<string>;
+export declare const matches: (regex: string, name?: string, maxScore?: number) => Evaluator<string>;
+export declare const lengthBetween: (min: number, max: number, maxScore?: number) => Evaluator<string>;
+//# sourceMappingURL=Checks.d.ts.map

package/dist/eval/Checks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Checks.d.ts","sourceRoot":"","sources":["../../src/eval/Checks.ts"],"names":[],"mappings":"AAEA,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAqB9D,eAAO,MAAM,KAAK,GAAI,iBAAY,KAAG,SAAS,CAAC,MAAM,CAWjD,CAAA;AAEJ,eAAO,MAAM,SAAS,GAAI,iBAAY,KAAG,SAAS,CAAC,MAAM,CAWrD,CAAA;AAEJ,eAAO,MAAM,OAAO,GAAI,OAAO,MAAM,EAAE,aAAe,EAAE,iBAAY,KAAG,SAAS,CAAC,MAAM,CAMnF,CAAA;AAEJ,eAAO,MAAM,aAAa,GAAI,KAAK,MAAM,EAAE,KAAK,MAAM,EAAE,iBAAY,KAAG,SAAS,CAAC,MAAM,CAUpF,CAAA"}

package/dist/eval/Checks.js

@@ -0,0 +1,39 @@
+import * as Effect from "effect/Effect";
+import { DimensionScore, EvalResult } from "./Eval.js";
+import { makeEvaluator } from "./Evaluator.js";
+const piiPatterns = [
+    ["email", /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/],
+    ["ssn", /\b\d{3}-\d{2}-\d{4}\b/],
+    ["credit-card", /\b(?:\d[ -]?){13,16}\b/],
+    ["phone", /\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b/],
+    ["ipv4", /\b(?:\d{1,3}\.){3}\d{1,3}\b/]
+];
+const one = (name, ok, maxScore, why) => EvalResult.make({
+    scores: [
+        DimensionScore.make({
+            name,
+            score: ok ? maxScore : 0,
+            reasoning: why
+        })
+    ]
+});
+export const noPii = (maxScore = 2) => makeEvaluator((input) => {
+    const hits = piiPatterns.flatMap(([label, pattern]) => (pattern.test(input) ? [label] : []));
+    return Effect.succeed(one("no-pii", hits.length === 0, maxScore, hits.length === 0 ? "no PII detected" : `matched: ${hits.join(", ")}`));
+});
+export const validJson = (maxScore = 2) => makeEvaluator((input) => {
+    let valid = true;
+    try {
+        JSON.parse(input);
+    }
+    catch {
+        valid = false;
+    }
+    return Effect.succeed(one("valid-json", valid, maxScore, valid ? "parses as JSON" : "not valid JSON"));
+});
+export const matches = (regex, name = "format", maxScore = 2) => makeEvaluator((input) => {
+    const ok = new RegExp(`^(?:${regex})$`).test(input);
+    return Effect.succeed(one(name, ok, maxScore, ok ? `matches /${regex}/` : `does not match /${regex}/`));
+});
+export const lengthBetween = (min, max, maxScore = 2) => makeEvaluator((input) => Effect.succeed(one("length", input.length >= min && input.length <= max, maxScore, `length ${input.length}; bounds [${min}, ${max}]`)));
+//# sourceMappingURL=Checks.js.map

package/dist/eval/Checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Checks.js","sourceRoot":"","sources":["../../src/eval/Checks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtD,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAA;AAE9D,MAAM,WAAW,GAA6D;IAC5E,CAAC,OAAO,EAAE,gDAAgD,CAAC;IAC3D,CAAC,KAAK,EAAE,uBAAuB,CAAC;IAChC,CAAC,aAAa,EAAE,wBAAwB,CAAC;IACzC,CAAC,OAAO,EAAE,mCAAmC,CAAC;IAC9C,CAAC,MAAM,EAAE,6BAA6B,CAAC;CACxC,CAAA;AAED,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAW,EAAE,QAAgB,EAAE,GAAW,EAAc,EAAE,CACnF,UAAU,CAAC,IAAI,CAAC;IACd,MAAM,EAAE;QACN,cAAc,CAAC,IAAI,CAAC;YAClB,IAAI;YACJ,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACxB,SAAS,EAAE,GAAG;SACf,CAAC;KACH;CACF,CAAC,CAAA;AAEJ,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAqB,EAAE,CACvD,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC5F,OAAO,MAAM,CAAC,OAAO,CACnB,GAAG,CACD,QAAQ,EACR,IAAI,CAAC,MAAM,KAAK,CAAC,EACjB,QAAQ,EACR,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtE,CACF,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAqB,EAAE,CAC3D,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,IAAI,KAAK,GAAG,IAAI,CAAA;IAChB,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,GAAG,KAAK,CAAA;IACf,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CACnB,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAChF,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,IAAI,GAAG,QAAQ,EAAE,QAAQ,GAAG,CAAC,EAAqB,EAAE,CACzF,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACnD,OAAO,MAAM,CAAC,OAAO,CACnB,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,GAAG,CAAC,CAAC,CAAC,mBAAmB,KAAK,GAAG,CAAC,CACjF,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,QAAQ,GAAG,CAAC,EAAqB,EAAE,CACzF,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CACtB,MAAM,CAAC,OAAO,CACZ,GAAG,CACD,QAAQ,EACR,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,EAC1C,QAAQ,EACR,UAAU,KAAK,CAAC,MAAM,aAAa,GAAG,KAAK,GAAG,GAAG,CAClD,CACF,CACF,CAAA"}

package/dist/eval/Eval.d.ts

@@ -0,0 +1,57 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import type { Evaluator } from "./Evaluator.ts";
+declare const Dimension_base: Schema.Class<Dimension, Schema.Struct<{
+    readonly name: Schema.String;
+    readonly rubric: Schema.String;
+    readonly maxScore: Schema.withConstructorDefault<Schema.Int>;
+}>, {}>;
+export declare class Dimension extends Dimension_base {
+}
+declare const DimensionScore_base: Schema.Class<DimensionScore, Schema.Struct<{
+    readonly name: Schema.String;
+    readonly score: Schema.Int;
+    readonly reasoning: Schema.withConstructorDefault<Schema.String>;
+}>, {}>;
+export declare class DimensionScore extends DimensionScore_base {
+}
+declare const EvalResult_base: Schema.Class<EvalResult, Schema.Struct<{
+    readonly scores: Schema.$Array<typeof DimensionScore>;
+    readonly summary: Schema.withConstructorDefault<Schema.String>;
+}>, {}>;
+export declare class EvalResult extends EvalResult_base {
+    score(name: string): number | undefined;
+    get total(): number;
+    meets(minPerDimension: number): boolean;
+}
+declare const Sample_base: Schema.Class<Sample, Schema.Struct<{
+    readonly response: Schema.String;
+    readonly query: Schema.optionalKey<Schema.String>;
+    readonly context: Schema.optionalKey<Schema.String>;
+    readonly expected: Schema.optionalKey<Schema.String>;
+}>, {}>;
+export declare class Sample extends Sample_base {
+}
+declare const DimensionStats_base: Schema.Class<DimensionStats, Schema.Struct<{
+    readonly name: Schema.String;
+    readonly median: Schema.Int;
+    readonly min: Schema.Int;
+    readonly max: Schema.Int;
+}>, {}>;
+export declare class DimensionStats extends DimensionStats_base {
+    get spread(): number;
+    isFlaky(threshold: number): boolean;
+}
+declare const RepeatedEval_base: Schema.Class<RepeatedEval, Schema.Struct<{
+    readonly runs: Schema.$Array<typeof EvalResult>;
+    readonly stats: Schema.$Array<typeof DimensionStats>;
+    readonly spreadThreshold: Schema.Int;
+}>, {}>;
+export declare class RepeatedEval extends RepeatedEval_base {
+    get flakyDimensions(): ReadonlyArray<string>;
+    get isFlaky(): boolean;
+    get aggregate(): EvalResult;
+}
+export declare const repeat: <In>(evaluator: Evaluator<In>, input: In, n?: number | undefined, spreadThreshold?: number | undefined) => Effect.Effect<RepeatedEval, import("../Errors.ts").ProviderError | import("../Errors.ts").UsageLimitError | import("../Errors.ts").AuthenticationError | import("../Errors.ts").InvalidRequestError | import("../Errors.ts").ParseError | import("../Errors.ts").ToolError | import("../Errors.ts").ConfigError | import("../Errors.ts").RateLimitError | import("../Errors.ts").TimeoutError | import("../Errors.ts").TurnLimitError, never>;
+export {};
+//# sourceMappingURL=Eval.d.ts.map

package/dist/eval/Eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Eval.d.ts","sourceRoot":"","sources":["../../src/eval/Eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;;;;;;AAE/C,qBAAa,SAAU,SAAQ,cAI7B;CAAG;;;;;;AAEL,qBAAa,cAAe,SAAQ,mBAIlC;CAAG;;;;;AAEL,qBAAa,UAAW,SAAQ,eAG9B;IACA,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIvC,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,KAAK,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;CAGxC;;;;;;;AAED,qBAAa,MAAO,SAAQ,WAK1B;CAAG;;;;;;;AAEL,qBAAa,cAAe,SAAQ,mBAKlC;IACA,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;CAGpC;;;;;;AAED,qBAAa,YAAa,SAAQ,iBAIhC;IACA,IAAI,eAAe,IAAI,aAAa,CAAC,MAAM,CAAC,CAE3C;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,SAAS,IAAI,UAAU,CAW1B;CACF;AAKD,eAAO,MAAM,MAAM,GAAyD,EAAE,qhBAyB5E,CAAA"}

package/dist/eval/Eval.js

@@ -0,0 +1,91 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+export class Dimension extends Schema.Class("Dimension")({
+    name: Schema.String,
+    rubric: Schema.String,
+    maxScore: Schema.Int.pipe(Schema.withConstructorDefault(Effect.succeed(2)))
+}) {
+}
+export class DimensionScore extends Schema.Class("DimensionScore")({
+    name: Schema.String,
+    score: Schema.Int,
+    reasoning: Schema.String.pipe(Schema.withConstructorDefault(Effect.succeed("")))
+}) {
+}
+export class EvalResult extends Schema.Class("EvalResult")({
+    scores: Schema.Array(DimensionScore),
+    summary: Schema.String.pipe(Schema.withConstructorDefault(Effect.succeed("")))
+}) {
+    score(name) {
+        return this.scores.find((item) => item.name === name)?.score;
+    }
+    get total() {
+        return this.scores.reduce((sum, item) => sum + item.score, 0);
+    }
+    meets(minPerDimension) {
+        return this.scores.every((item) => item.score >= minPerDimension);
+    }
+}
+export class Sample extends Schema.Class("Sample")({
+    response: Schema.String,
+    query: Schema.optionalKey(Schema.String),
+    context: Schema.optionalKey(Schema.String),
+    expected: Schema.optionalKey(Schema.String)
+}) {
+}
+export class DimensionStats extends Schema.Class("DimensionStats")({
+    name: Schema.String,
+    median: Schema.Int,
+    min: Schema.Int,
+    max: Schema.Int
+}) {
+    get spread() {
+        return this.max - this.min;
+    }
+    isFlaky(threshold) {
+        return this.spread > threshold;
+    }
+}
+export class RepeatedEval extends Schema.Class("RepeatedEval")({
+    runs: Schema.Array(EvalResult),
+    stats: Schema.Array(DimensionStats),
+    spreadThreshold: Schema.Int
+}) {
+    get flakyDimensions() {
+        return this.stats.filter((stat) => stat.isFlaky(this.spreadThreshold)).map((stat) => stat.name);
+    }
+    get isFlaky() {
+        return this.flakyDimensions.length > 0;
+    }
+    get aggregate() {
+        return EvalResult.make({
+            scores: this.stats.map((stat) => DimensionScore.make({
+                name: stat.name,
+                score: stat.median,
+                reasoning: `median of ${this.runs.length} runs`
+            })),
+            summary: `aggregate of ${this.runs.length} runs`
+        });
+    }
+}
+const median = (sorted) => sorted.length === 0 ? 0 : (sorted[Math.floor((sorted.length - 1) / 2)] ?? 0);
+export const repeat = Effect.fn("@llm4ts/core/eval/Eval.repeat")(function* (evaluator, input, n = 3, spreadThreshold = 1) {
+    const runs = yield* Effect.forEach(Array.from({ length: Math.max(0, n) }), () => evaluator.evaluate(input));
+    const names = [...new Set(runs.flatMap((run) => run.scores.map((score) => score.name)))];
+    const stats = names.map((name) => {
+        const values = runs
+            .flatMap((run) => {
+            const score = run.score(name);
+            return score === undefined ? [] : [score];
+        })
+            .sort((left, right) => left - right);
+        return DimensionStats.make({
+            name,
+            median: median(values),
+            min: values.at(0) ?? 0,
+            max: values.at(-1) ?? 0
+        });
+    });
+    return RepeatedEval.make({ runs, stats, spreadThreshold });
+});
+//# sourceMappingURL=Eval.js.map

package/dist/eval/Eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Eval.js","sourceRoot":"","sources":["../../src/eval/Eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAIvC,MAAM,OAAO,SAAU,SAAQ,MAAM,CAAC,KAAK,CAAY,WAAW,CAAC,CAAC;IAClE,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM;IACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;CAC5E,CAAC;CAAG;AAEL,MAAM,OAAO,cAAe,SAAQ,MAAM,CAAC,KAAK,CAAiB,gBAAgB,CAAC,CAAC;IACjF,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,KAAK,EAAE,MAAM,CAAC,GAAG;IACjB,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;CACjF,CAAC;CAAG;AAEL,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,KAAK,CAAa,YAAY,CAAC,CAAC;IACrE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;CAC/E,CAAC;IACA,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,CAAA;IAC9D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,KAAK,CAAC,eAAuB;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,eAAe,CAAC,CAAA;IACnE,CAAC;CACF;AAED,MAAM,OAAO,MAAO,SAAQ,MAAM,CAAC,KAAK,CAAS,QAAQ,CAAC,CAAC;IACzD,QAAQ,EAAE,MAAM,CAAC,MAAM;IACvB,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;CAC5C,CAAC;CAAG;AAEL,MAAM,OAAO,cAAe,SAAQ,MAAM,CAAC,KAAK,CAAiB,gBAAgB,CAAC,CAAC;IACjF,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,MAAM,EAAE,MAAM,CAAC,GAAG;IAClB,GAAG,EAAE,MAAM,CAAC,GAAG;IACf,GAAG,EAAE,MAAM,CAAC,GAAG;CAChB,CAAC;IACA,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAA;IAC5B,CAAC;IAED,OAAO,CAAC,SAAiB;QACvB,OAAO,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;IAChC,CAAC;CACF;AAED,MAAM,OAAO,YAAa,SAAQ,MAAM,CAAC,KAAK,CAAe,cAAc,CAAC,CAAC;IAC3E,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;IAC9B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;IACnC,eAAe,EAAE,MAAM,CAAC,GAAG;CAC5B,CAAC;IACA,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAA;IACxC,CAAC;IAED,IAAI,SAAS;QACX,OAAO,UAAU,CAAC,IAAI,CAAC;YACrB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC9B,cAAc,CAAC,IAAI,CAAC;gBAClB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,SAAS,EAAE,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO;aAChD,CAAC,CACH;YACD,OAAO,EAAE,gBAAgB,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO;SACjD,CAAC,CAAA;IACJ,CAAC;CACF;AAED,MAAM,MAAM,GAAG,CAAC,MAA6B,EAAU,EAAE,CACvD,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AAE9E,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,+BAA+B,CAAC,CAAC,QAAQ,CAAC,EACxE,SAAwB,EACxB,KAAS,EACT,CAAC,GAAG,CAAC,EACL,eAAe,GAAG,CAAC;IAEnB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAC9E,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC1B,CAAA;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IACxF,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/B,MAAM,MAAM,GAAG,IAAI;aAChB,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACf,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YAC7B,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAC3C,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAA;QACtC,OAAO,cAAc,CAAC,IAAI,CAAC;YACzB,IAAI;YACJ,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YACtB,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACtB,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;SACxB,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAA;AAC5D,CAAC,CAAC,CAAA"}

package/dist/eval/EvalSuite.d.ts

@@ -0,0 +1,21 @@
+import * as Effect from "effect/Effect";
+import type { EvalResult, RepeatedEval } from "./Eval.ts";
+import type { Evaluator } from "./Evaluator.ts";
+export interface EvalCase<In> {
+    readonly name: string;
+    readonly input: In;
+    readonly minPerDimension?: number;
+}
+export interface CaseReport {
+    readonly name: string;
+    readonly result: EvalResult;
+    readonly repeated: RepeatedEval | undefined;
+    readonly passed: boolean;
+}
+export interface SuiteReport {
+    readonly cases: ReadonlyArray<CaseReport>;
+    readonly passed: boolean;
+    readonly failures: ReadonlyArray<CaseReport>;
+}
+export declare const run: <In>(evaluator: Evaluator<In>, cases: readonly EvalCase<In>[], repeats?: number | undefined, spreadThreshold?: number | undefined) => Effect.Effect<SuiteReport, import("../Errors.ts").ProviderError | import("../Errors.ts").UsageLimitError | import("../Errors.ts").AuthenticationError | import("../Errors.ts").InvalidRequestError | import("../Errors.ts").ParseError | import("../Errors.ts").ToolError | import("../Errors.ts").ConfigError | import("../Errors.ts").RateLimitError | import("../Errors.ts").TimeoutError | import("../Errors.ts").TurnLimitError, never>;
+//# sourceMappingURL=EvalSuite.d.ts.map

package/dist/eval/EvalSuite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalSuite.d.ts","sourceRoot":"","sources":["../../src/eval/EvalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAE/C,MAAM,WAAW,QAAQ,CAAC,EAAE;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAA;IAClB,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,YAAY,GAAG,SAAS,CAAA;IAC3C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;CAC7C;AAED,eAAO,MAAM,GAAG,GAA2D,EAAE,+iBAqC3E,CAAA"}

package/dist/eval/EvalSuite.js

@@ -0,0 +1,30 @@
+import * as Effect from "effect/Effect";
+import { repeat } from "./Eval.js";
+export const run = Effect.fn("@llm4ts/core/eval/EvalSuite.run")(function* (evaluator, cases, repeats = 1, spreadThreshold = 1) {
+    const reports = yield* Effect.forEach(cases, (testCase) => {
+        const minimum = testCase.minPerDimension ?? 1;
+        if (repeats <= 1) {
+            return evaluator.evaluate(testCase.input).pipe(Effect.map((result) => ({
+                name: testCase.name,
+                result,
+                repeated: undefined,
+                passed: result.meets(minimum)
+            })));
+        }
+        return repeat(evaluator, testCase.input, repeats, spreadThreshold).pipe(Effect.map((repeated) => {
+            const result = repeated.aggregate;
+            return {
+                name: testCase.name,
+                result,
+                repeated,
+                passed: result.meets(minimum) && !repeated.isFlaky
+            };
+        }));
+    });
+    return {
+        cases: reports,
+        passed: reports.every((report) => report.passed),
+        failures: reports.filter((report) => !report.passed)
+    };
+});
+//# sourceMappingURL=EvalSuite.js.map

package/dist/eval/EvalSuite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalSuite.js","sourceRoot":"","sources":["../../src/eval/EvalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAuBlC,MAAM,CAAC,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,iCAAiC,CAAC,CAAC,QAAQ,CAAC,EACvE,SAAwB,EACxB,KAAkC,EAClC,OAAO,GAAG,CAAC,EACX,eAAe,GAAG,CAAC;IAEnB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;QACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,eAAe,IAAI,CAAC,CAAA;QAC7C,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YACjB,OAAO,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAC5C,MAAM,CAAC,GAAG,CACR,CAAC,MAAM,EAAc,EAAE,CAAC,CAAC;gBACvB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,MAAM;gBACN,QAAQ,EAAE,SAAS;gBACnB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;aAC9B,CAAC,CACH,CACF,CAAA;QACH,CAAC;QACD,OAAO,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC,IAAI,CACrE,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAc,EAAE;YAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAA;YACjC,OAAO;gBACL,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO;aACnD,CAAA;QACH,CAAC,CAAC,CACH,CAAA;IACH,CAAC,CAAC,CAAA;IACF,OAAO;QACL,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;KACrD,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/Evaluator.d.ts

@@ -0,0 +1,10 @@
+import * as Effect from "effect/Effect";
+import type { LlmError } from "../Errors.ts";
+import { EvalResult } from "./Eval.ts";
+export interface Evaluator<in In> {
+    readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>;
+}
+export declare const makeEvaluator: <In>(evaluate: Evaluator<In>["evaluate"]) => Evaluator<In>;
+export declare const contramap: <In, In2>(evaluator: Evaluator<In>, project: (input: In2) => In) => Evaluator<In2>;
+export declare const all: <In>(...evaluators: ReadonlyArray<Evaluator<In>>) => Evaluator<In>;
+//# sourceMappingURL=Evaluator.d.ts.map

package/dist/eval/Evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Evaluator.d.ts","sourceRoot":"","sources":["../../src/eval/Evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,MAAM,WAAW,SAAS,CAAC,EAAE,CAAC,EAAE;IAC9B,QAAQ,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;CACtE;AAED,eAAO,MAAM,aAAa,GAAI,EAAE,EAAE,UAAU,SAAS,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,KAAG,SAAS,CAAC,EAAE,CAElF,CAAA;AAEF,eAAO,MAAM,SAAS,GAAI,EAAE,EAAE,GAAG,EAC/B,WAAW,SAAS,CAAC,EAAE,CAAC,EACxB,SAAS,CAAC,KAAK,EAAE,GAAG,KAAK,EAAE,KAC1B,SAAS,CAAC,GAAG,CAAiE,CAAA;AAEjF,eAAO,MAAM,GAAG,GAAI,EAAE,EAAE,GAAG,YAAY,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,KAAG,SAAS,CAAC,EAAE,CAa/E,CAAA"}

package/dist/eval/Evaluator.js

@@ -0,0 +1,14 @@
+import * as Effect from "effect/Effect";
+import { EvalResult } from "./Eval.js";
+export const makeEvaluator = (evaluate) => ({
+    evaluate
+});
+export const contramap = (evaluator, project) => makeEvaluator((input) => evaluator.evaluate(project(input)));
+export const all = (...evaluators) => makeEvaluator((input) => Effect.forEach(evaluators, (evaluator) => evaluator.evaluate(input)).pipe(Effect.map((results) => EvalResult.make({
+    scores: results.flatMap((result) => result.scores),
+    summary: results
+        .map((result) => result.summary)
+        .filter((summary) => summary.length > 0)
+        .join("; ")
+}))));
+//# sourceMappingURL=Evaluator.js.map

package/dist/eval/Evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Evaluator.js","sourceRoot":"","sources":["../../src/eval/Evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAMtC,MAAM,CAAC,MAAM,aAAa,GAAG,CAAK,QAAmC,EAAiB,EAAE,CAAC,CAAC;IACxF,QAAQ;CACT,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,SAAwB,EACxB,OAA2B,EACX,EAAE,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAEjF,MAAM,CAAC,MAAM,GAAG,GAAG,CAAK,GAAG,UAAwC,EAAiB,EAAE,CACpF,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CACtB,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CACvE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACrB,UAAU,CAAC,IAAI,CAAC;IACd,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,OAAO;SACb,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;SAC/B,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;SACvC,IAAI,CAAC,IAAI,CAAC;CACd,CAAC,CACH,CACF,CACF,CAAA"}

package/dist/eval/Judge.d.ts

@@ -0,0 +1,15 @@
+import * as Schema from "effect/Schema";
+import type { LlmServiceShape } from "../LlmService.ts";
+import { DimensionScore } from "./Eval.ts";
+import type { Dimension, Sample } from "./Eval.ts";
+import { type Evaluator } from "./Evaluator.ts";
+declare const JudgeResponse_base: Schema.Class<JudgeResponse, Schema.Struct<{
+    readonly scores: Schema.$Array<typeof DimensionScore>;
+}>, {}>;
+export declare class JudgeResponse extends JudgeResponse_base {
+}
+export declare const defaultSystem: string;
+export declare const buildPrompt: (system: string, dimensions: ReadonlyArray<Dimension>, sample: Sample) => string;
+export declare const judge: (llm: LlmServiceShape, dimensions: ReadonlyArray<Dimension>, system?: string) => Evaluator<Sample>;
+export {};
+//# sourceMappingURL=Judge.d.ts.map

package/dist/eval/Judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Judge.d.ts","sourceRoot":"","sources":["../../src/eval/Judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAc,MAAM,WAAW,CAAA;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAiB,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAA;;;;AAE9D,qBAAa,aAAc,SAAQ,kBAEjC;CAAG;AAEL,eAAO,MAAM,aAAa,QAEW,CAAA;AAsBrC,eAAO,MAAM,WAAW,GACtB,QAAQ,MAAM,EACd,YAAY,aAAa,CAAC,SAAS,CAAC,EACpC,QAAQ,MAAM,KACb,MAoBF,CAAA;AAsBD,eAAO,MAAM,KAAK,GAChB,KAAK,eAAe,EACpB,YAAY,aAAa,CAAC,SAAS,CAAC,EACpC,eAAsB,KACrB,SAAS,CAAC,MAAM,CAKhB,CAAA"}

package/dist/eval/Judge.js

@@ -0,0 +1,73 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { DimensionScore, EvalResult } from "./Eval.js";
+import { makeEvaluator } from "./Evaluator.js";
+export class JudgeResponse extends Schema.Class("JudgeResponse")({
+    scores: Schema.Array(DimensionScore)
+}) {
+}
+export const defaultSystem = "You are an impartial evaluator. Score each dimension strictly on its rubric; when unsure, score lower. " +
+    "Return ONLY valid JSON, no prose.";
+const judgeJsonSchema = {
+    type: "object",
+    properties: {
+        scores: {
+            type: "array",
+            items: {
+                type: "object",
+                properties: {
+                    name: { type: "string" },
+                    score: { type: "integer" },
+                    reasoning: { type: "string" }
+                },
+                required: ["name", "score"]
+            }
+        }
+    },
+    required: ["scores"],
+    additionalProperties: false
+};
+export const buildPrompt = (system, dimensions, sample) => {
+    const rubric = dimensions
+        .map((dimension) => `- ${dimension.name} (0..${dimension.maxScore}): ${dimension.rubric}`)
+        .join("\n");
+    const user = [
+        sample.query === undefined ? undefined : `Query: ${sample.query}`,
+        sample.context === undefined ? undefined : `Context: ${sample.context}`,
+        `Response: ${sample.response}`,
+        sample.expected === undefined ? undefined : `Expected: ${sample.expected}`
+    ]
+        .filter((line) => line !== undefined)
+        .join("\n");
+    return `${system}
+
+Score each dimension on its 0..max scale per the rubric:
+${rubric}
+
+Return ONLY this JSON: {"scores":[{"name":"<dimension>","score":<int>,"reasoning":"<one sentence>"}]}
+
+${user}`;
+};
+const toResult = (dimensions, response) => {
+    const byName = new Map(response.scores.map((score) => [score.name, score]));
+    return EvalResult.make({
+        scores: dimensions.map((dimension) => {
+            const score = byName.get(dimension.name);
+            return score === undefined
+                ? DimensionScore.make({
+                    name: dimension.name,
+                    score: 0,
+                    reasoning: "missing"
+                })
+                : DimensionScore.make({
+                    name: dimension.name,
+                    score: Math.max(0, Math.min(dimension.maxScore, score.score)),
+                    reasoning: score.reasoning
+                });
+        })
+    });
+};
+export const judge = (llm, dimensions, system = defaultSystem) => makeEvaluator((sample) => llm
+    .executeStructured(buildPrompt(system, dimensions, sample), JudgeResponse, judgeJsonSchema)
+    .pipe(Effect.map((response) => toResult(dimensions, response))));
+//# sourceMappingURL=Judge.js.map

package/dist/eval/Judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Judge.js","sourceRoot":"","sources":["../../src/eval/Judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtD,OAAO,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAA;AAE9D,MAAM,OAAO,aAAc,SAAQ,MAAM,CAAC,KAAK,CAAgB,eAAe,CAAC,CAAC;IAC9E,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;CACrC,CAAC;CAAG;AAEL,MAAM,CAAC,MAAM,aAAa,GACxB,yGAAyG;IACzG,mCAAmC,CAAA;AAErC,MAAM,eAAe,GAAe;IAClC,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,MAAM,EAAE;YACN,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;oBAC1B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC9B;gBACD,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;aAC5B;SACF;KACF;IACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;IACpB,oBAAoB,EAAE,KAAK;CAC5B,CAAA;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,MAAc,EACd,UAAoC,EACpC,MAAc,EACN,EAAE;IACV,MAAM,MAAM,GAAG,UAAU;SACtB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,SAAS,CAAC,IAAI,QAAQ,SAAS,CAAC,QAAQ,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC;SACzF,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,MAAM,IAAI,GAAG;QACX,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,KAAK,EAAE;QACjE,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,OAAO,EAAE;QACvE,aAAa,MAAM,CAAC,QAAQ,EAAE;QAC9B,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,QAAQ,EAAE;KAC3E;SACE,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACpD,IAAI,CAAC,IAAI,CAAC,CAAA;IACb,OAAO,GAAG,MAAM;;;EAGhB,MAAM;;;;EAIN,IAAI,EAAE,CAAA;AACR,CAAC,CAAA;AAED,MAAM,QAAQ,GAAG,CAAC,UAAoC,EAAE,QAAuB,EAAc,EAAE;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3E,OAAO,UAAU,CAAC,IAAI,CAAC;QACrB,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YACxC,OAAO,KAAK,KAAK,SAAS;gBACxB,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,SAAS;iBACrB,CAAC;gBACJ,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC7D,SAAS,EAAE,KAAK,CAAC,SAAS;iBAC3B,CAAC,CAAA;QACR,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,KAAK,GAAG,CACnB,GAAoB,EACpB,UAAoC,EACpC,MAAM,GAAG,aAAa,EACH,EAAE,CACrB,aAAa,CAAC,CAAC,MAAM,EAAE,EAAE,CACvB,GAAG;KACA,iBAAiB,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,aAAa,EAAE,eAAe,CAAC;KAC1F,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,CAClE,CAAA"}

package/dist/observability/MeteredLlmService.d.ts

@@ -0,0 +1,16 @@
+import * as Schema from "effect/Schema";
+import type { LlmServiceShape } from "../LlmService.ts";
+import type { MetricsCollectorShape, RequestLabels } from "./Metrics.ts";
+declare const LlmPricing_base: Schema.Class<LlmPricing, Schema.Struct<{
+    readonly inputUsdPer1k: Schema.Number;
+    readonly outputUsdPer1k: Schema.Number;
+}>, {}>;
+export declare class LlmPricing extends LlmPricing_base {
+}
+export interface MeteredLlmOptions {
+    readonly labels: RequestLabels;
+    readonly pricing?: LlmPricing;
+}
+export declare const meterLlmService: (service: LlmServiceShape, collector: MetricsCollectorShape, options: MeteredLlmOptions) => LlmServiceShape;
+export {};
+//# sourceMappingURL=MeteredLlmService.d.ts.map

package/dist/observability/MeteredLlmService.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MeteredLlmService.d.ts","sourceRoot":"","sources":["../../src/observability/MeteredLlmService.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,kBAAkB,CAAA;AAGzE,OAAO,KAAK,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;;;;;AAExE,qBAAa,UAAW,SAAQ,eAG9B;CAAG;AAEL,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,UAAU,CAAA;CAC9B;AA+GD,eAAO,MAAM,eAAe,GAC1B,SAAS,eAAe,EACxB,WAAW,qBAAqB,EAChC,SAAS,iBAAiB,KACzB,eAqBD,CAAA"}

package/dist/observability/MeteredLlmService.js

@@ -0,0 +1,66 @@
+import * as Clock from "effect/Clock";
+import * as Effect from "effect/Effect";
+import * as Ref from "effect/Ref";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
+import { RequestMetrics } from "./Metrics.js";
+export class LlmPricing extends Schema.Class("LlmPricing")({
+    inputUsdPer1k: Schema.Number,
+    outputUsdPer1k: Schema.Number
+}) {
+}
+const estimatedCost = (usage, pricing) => usage === undefined || pricing === undefined
+    ? undefined
+    : (usage.prompt / 1_000) * pricing.inputUsdPer1k +
+        (usage.completion / 1_000) * pricing.outputUsdPer1k;
+const complete = (collector, options, startedAt, endedAt, success, errorType, usage) => {
+    const cost = estimatedCost(usage, options.pricing);
+    return collector.recordCompleted(RequestMetrics.make({
+        labels: options.labels,
+        latencyMs: Math.max(0, endedAt - startedAt),
+        success,
+        ...(errorType === undefined ? {} : { errorType }),
+        ...(usage === undefined ? {} : { tokenUsage: usage }),
+        ...(cost === undefined ? {} : { estimatedCostUsd: cost })
+    }));
+};
+const meterEffect = (effect, collector, options, usageOf) => Effect.gen(function* () {
+    const startedAt = yield* Clock.currentTimeMillis;
+    yield* collector.markRequestStarted(options.labels);
+    const result = yield* Effect.result(effect);
+    const endedAt = yield* Clock.currentTimeMillis;
+    yield* complete(collector, options, startedAt, endedAt, result._tag === "Success", result._tag === "Failure" ? result.failure._tag : undefined, result._tag === "Success" ? usageOf(result.success) : undefined);
+    return yield* Effect.fromResult(result);
+});
+const meterStream = (stream, collector, options) => Stream.unwrap(Effect.gen(function* () {
+    const startedAt = yield* Clock.currentTimeMillis;
+    yield* collector.markRequestStarted(options.labels);
+    const measurement = yield* Ref.make({
+        success: true,
+        errorType: undefined,
+        usage: undefined
+    });
+    return stream.pipe(Stream.tap((chunk) => chunk.usage === undefined
+        ? Effect.void
+        : Ref.update(measurement, (current) => ({
+            ...current,
+            usage: chunk.usage
+        }))), Stream.tapError((error) => Ref.update(measurement, (current) => ({
+        ...current,
+        success: false,
+        errorType: error._tag
+    }))), Stream.ensuring(Effect.gen(function* () {
+        const current = yield* Ref.get(measurement);
+        const endedAt = yield* Clock.currentTimeMillis;
+        yield* complete(collector, options, startedAt, endedAt, current.success, current.errorType, current.usage);
+    })));
+}));
+export const meterLlmService = (service, collector, options) => ({
+    executeStream: (prompt) => meterStream(service.executeStream(prompt), collector, options),
+    executeStreamWithHistory: (messages) => meterStream(service.executeStreamWithHistory(messages), collector, options),
+    executeWithTools: (prompt, tools) => meterEffect(service.executeWithTools(prompt, tools), collector, options, () => undefined),
+    executeStructured: (prompt, schema, jsonSchema) => meterEffect(service.executeStructured(prompt, schema, jsonSchema), collector, options, () => undefined),
+    executeStructuredWithUsage: (prompt, schema, jsonSchema) => meterEffect(service.executeStructuredWithUsage(prompt, schema, jsonSchema), collector, options, (result) => result[1]),
+    isAvailable: service.isAvailable
+});
+//# sourceMappingURL=MeteredLlmService.js.map

package/dist/observability/MeteredLlmService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MeteredLlmService.js","sourceRoot":"","sources":["../../src/observability/MeteredLlmService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,YAAY,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAIvC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAG7C,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,KAAK,CAAa,YAAY,CAAC,CAAC;IACrE,aAAa,EAAE,MAAM,CAAC,MAAM;IAC5B,cAAc,EAAE,MAAM,CAAC,MAAM;CAC9B,CAAC;CAAG;AAOL,MAAM,aAAa,GAAG,CACpB,KAA6B,EAC7B,OAA+B,EACX,EAAE,CACtB,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;IAC1C,CAAC,CAAC,SAAS;IACX,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,aAAa;QAC9C,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,cAAc,CAAA;AAEzD,MAAM,QAAQ,GAAG,CACf,SAAgC,EAChC,OAA0B,EAC1B,SAAiB,EACjB,OAAe,EACf,OAAgB,EAChB,SAA6B,EAC7B,KAA6B,EACR,EAAE;IACvB,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAClD,OAAO,SAAS,CAAC,eAAe,CAC9B,cAAc,CAAC,IAAI,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;QAC3C,OAAO;QACP,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;KAC1D,CAAC,CACH,CAAA;AACH,CAAC,CAAA;AAED,MAAM,WAAW,GAAG,CAClB,MAAqC,EACrC,SAAgC,EAChC,OAA0B,EAC1B,OAA6C,EACd,EAAE,CACjC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IAChD,KAAK,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IAC9C,KAAK,CAAC,CAAC,QAAQ,CACb,SAAS,EACT,OAAO,EACP,SAAS,EACT,OAAO,EACP,MAAM,CAAC,IAAI,KAAK,SAAS,EACzB,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAC3D,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAChE,CAAA;IACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;AACzC,CAAC,CAAC,CAAA;AAQJ,MAAM,WAAW,GAAG,CAClB,MAAyC,EACzC,SAAgC,EAChC,OAA0B,EACS,EAAE,CACrC,MAAM,CAAC,MAAM,CACX,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IAChD,KAAK,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACnD,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAoB;QACrD,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,SAAS;QACpB,KAAK,EAAE,SAAS;KACjB,CAAC,CAAA;IACF,OAAO,MAAM,CAAC,IAAI,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,MAAM,CAAC,IAAI;QACb,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,OAAO;YACV,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC,CACR,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACpC,GAAG,OAAO;QACV,OAAO,EAAE,KAAK;QACd,SAAS,EAAE,KAAK,CAAC,IAAI;KACtB,CAAC,CAAC,CACJ,EACD,MAAM,CAAC,QAAQ,CACb,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;QAC9C,KAAK,CAAC,CAAC,QAAQ,CACb,SAAS,EACT,OAAO,EACP,SAAS,EACT,OAAO,EACP,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,KAAK,CACd,CAAA;IACH,CAAC,CAAC,CACH,CACF,CAAA;AACH,CAAC,CAAC,CACH,CAAA;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,OAAwB,EACxB,SAAgC,EAChC,OAA0B,EACT,EAAE,CAAC,CAAC;IACrB,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC;IACzF,wBAAwB,EAAE,CAAC,QAAQ,EAAE,EAAE,CACrC,WAAW,CAAC,OAAO,CAAC,wBAAwB,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC;IAC7E,gBAAgB,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAClC,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC;IAC3F,iBAAiB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CAChD,WAAW,CACT,OAAO,CAAC,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EACrD,SAAS,EACT,OAAO,EACP,GAAG,EAAE,CAAC,SAAS,CAChB;IACH,0BAA0B,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CACzD,WAAW,CACT,OAAO,CAAC,0BAA0B,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAC9D,SAAS,EACT,OAAO,EACP,CAAI,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAC9C;IACH,WAAW,EAAE,OAAO,CAAC,WAAW;CACjC,CAAC,CAAA"}

package/dist/observability/Metrics.d.ts

@@ -0,0 +1,71 @@
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Schema from "effect/Schema";
+import { TokenUsage } from "../Models.ts";
+declare const RequestLabels_base: Schema.Class<RequestLabels, Schema.Struct<{
+    readonly provider: Schema.String;
+    readonly model: Schema.String;
+    readonly agent: Schema.optionalKey<Schema.String>;
+    readonly runId: Schema.optionalKey<Schema.String>;
+    readonly workflowStep: Schema.optionalKey<Schema.String>;
+}>, {}>;
+export declare class RequestLabels extends RequestLabels_base {
+}
+declare const RequestMetrics_base: Schema.Class<RequestMetrics, Schema.Struct<{
+    readonly labels: typeof RequestLabels;
+    readonly tokenUsage: Schema.optionalKey<typeof TokenUsage>;
+    readonly latencyMs: Schema.Number;
+    readonly success: Schema.Boolean;
+    readonly errorType: Schema.optionalKey<Schema.String>;
+    readonly estimatedCostUsd: Schema.optionalKey<Schema.Number>;
+}>, {}>;
+export declare class RequestMetrics extends RequestMetrics_base {
+}
+declare const ProviderHealth_base: Schema.Class<ProviderHealth, Schema.Struct<{
+    readonly provider: Schema.String;
+    readonly requestCount: Schema.Int;
+    readonly successRate: Schema.Number;
+    readonly avgLatencyMs: Schema.Number;
+    readonly p95LatencyMs: Schema.Number;
+    readonly estimatedCostUsd: Schema.Number;
+}>, {}>;
+export declare class ProviderHealth extends ProviderHealth_base {
+}
+declare const ObservabilitySnapshot_base: Schema.Class<ObservabilitySnapshot, Schema.Struct<{
+    readonly totalRequests: Schema.Int;
+    readonly totalErrors: Schema.Int;
+    readonly activeRequests: Schema.Int;
+    readonly promptTokens: Schema.Int;
+    readonly completionTokens: Schema.Int;
+    readonly totalTokens: Schema.Int;
+    readonly estimatedCostUsd: Schema.Number;
+    readonly p50LatencyMs: Schema.Number;
+    readonly p95LatencyMs: Schema.Number;
+    readonly byProvider: Schema.$Array<typeof ProviderHealth>;
+    readonly byAgentRequests: Schema.$Record<Schema.String, Schema.Int>;
+}>, {}>;
+export declare class ObservabilitySnapshot extends ObservabilitySnapshot_base {
+}
+declare const DashboardMetricsSnapshot_base: Schema.Class<DashboardMetricsSnapshot, Schema.Struct<{
+    readonly providerHealth: Schema.$Array<typeof ProviderHealth>;
+    readonly totalTokens: Schema.Int;
+    readonly totalCostUsd: Schema.Number;
+    readonly totalRequests: Schema.Int;
+    readonly errorRate: Schema.Number;
+}>, {}>;
+export declare class DashboardMetricsSnapshot extends DashboardMetricsSnapshot_base {
+}
+export interface MetricsCollectorShape {
+    readonly markRequestStarted: (labels: RequestLabels) => Effect.Effect<void>;
+    readonly recordCompleted: (metrics: RequestMetrics) => Effect.Effect<void>;
+    readonly snapshot: Effect.Effect<ObservabilitySnapshot>;
+    readonly dashboardSnapshot: Effect.Effect<DashboardMetricsSnapshot>;
+}
+declare const MetricsCollector_base: Context.ServiceClass<MetricsCollector, "@llm4ts/core/observability/MetricsCollector", MetricsCollectorShape>;
+export declare class MetricsCollector extends MetricsCollector_base {
+}
+export declare const makeMetricsCollector: () => Effect.Effect<MetricsCollectorShape, never, never>;
+export declare const MetricsCollectorLive: Layer.Layer<MetricsCollector, never, never>;
+export {};
+//# sourceMappingURL=Metrics.d.ts.map

package/dist/observability/Metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Metrics.d.ts","sourceRoot":"","sources":["../../src/observability/Metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAErC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;;;;;;;;AAEzC,qBAAa,aAAc,SAAQ,kBAMjC;CAAG;;;;;;;;;AAEL,qBAAa,cAAe,SAAQ,mBAOlC;CAAG;;;;;;;;;AAEL,qBAAa,cAAe,SAAQ,mBAOlC;CAAG;;;;;;;;;;;;;;AAEL,qBAAa,qBAAsB,SAAQ,0BAczC;CAAG;;;;;;;;AAEL,qBAAa,wBAAyB,SAAQ,6BAQ5C;CAAG;AAEL,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,kBAAkB,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC3E,QAAQ,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACvD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAA;CACpE;;AAED,qBAAa,gBAAiB,SAAQ,qBAErC;CAAG;AA0EJ,eAAO,MAAM,oBAAoB,0DAkDhC,CAAA;AAED,eAAO,MAAM,oBAAoB,6CAAyD,CAAA"}