@swarmtools/evals 0.2.1 → 0.2.3

package/dist/decision-quality.eval.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=decision-quality.eval.d.ts.map

package/dist/decision-quality.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision-quality.eval.d.ts","sourceRoot":"","sources":["../src/decision-quality.eval.ts"],"names":[],"mappings":""}

package/dist/fixtures/decision-quality-fixtures.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Test cases for decision quality evaluation
+ *
+ * Each case includes:
+ * - input: Decision trace with strategy selection and optional precedent
+ * - output: Outcome data with success/failure metrics
+ * - expected: Validation criteria for scorer thresholds
+ */
+/**
+ * Strategy selection fixture with outcome data.
+ * Used to test strategySelectionQuality scorer.
+ */
+export interface StrategySelectionFixture {
+    input: {
+        task: string;
+        strategy: string;
+        precedent_task?: string;
+        precedent_strategy?: string;
+    };
+    output: {
+        strategy: string;
+        outcome_success: boolean;
+        error_count: number;
+        duration_ms?: number;
+    };
+    expected: {
+        min_score?: number;
+        max_score?: number;
+    };
+}
+/**
+ * Precedent relevance fixture.
+ * Used to test precedentRelevance scorer (LLM-as-judge).
+ */
+export interface PrecedentRelevanceFixture {
+    input: {
+        task: string;
+        precedent_task: string;
+        precedent_strategy: string;
+    };
+    expected: {
+        min_score?: number;
+        max_score?: number;
+    };
+}
+/**
+ * Strategy selection fixtures - known good and bad outcomes.
+ */
+export declare const strategySelectionFixtures: StrategySelectionFixture[];
+/**
+ * Precedent relevance fixtures - testing LLM-as-judge similarity scoring.
+ */
+export declare const precedentRelevanceFixtures: PrecedentRelevanceFixture[];
+//# sourceMappingURL=decision-quality-fixtures.d.ts.map

package/dist/fixtures/decision-quality-fixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision-quality-fixtures.d.ts","sourceRoot":"","sources":["../../src/fixtures/decision-quality-fixtures.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,EAAE,OAAO,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,wBAAwB,EAuG/D,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,0BAA0B,EAAE,yBAAyB,EA8FjE,CAAC"}

package/dist/scorers/decision-quality-scorers.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Strategy Selection Quality Scorer
+ *
+ * Evaluates whether the chosen decomposition strategy led to successful outcomes.
+ * This is the ultimate test - did the decision actually work?
+ *
+ * Scoring:
+ * - 1.0: Success with no errors
+ * - 0.7-0.9: Success with minor errors (error_count 1-2)
+ * - 0.4-0.6: Success with significant errors (error_count 3+)
+ * - 0.0: Failure
+ *
+ * Based on calculateDecisionQuality() from swarm-mail/decision-trace-store.ts
+ */
+export declare const strategySelectionQuality: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Precedent Relevance Scorer (LLM-as-Judge)
+ *
+ * Evaluates whether cited precedent tasks are actually semantically similar
+ * to the current task. Uses Claude Haiku to judge relevance.
+ *
+ * This catches cases where coordinators cite irrelevant precedents,
+ * which can lead to poor strategy selection.
+ *
+ * Scoring:
+ * - 0.8-1.0: Highly relevant (same domain, similar requirements)
+ * - 0.5-0.7: Moderately relevant (related concepts, different scope)
+ * - 0.0-0.4: Irrelevant (different domains, no meaningful overlap)
+ */
+export declare const precedentRelevance: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=decision-quality-scorers.d.ts.map

package/dist/scorers/decision-quality-scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision-quality-scorers.d.ts","sourceRoot":"","sources":["../../src/scorers/decision-quality-scorers.ts"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,wBAAwB,6DA+CnC,CAAC;AAEH;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,kBAAkB,6DA0E7B,CAAC"}

package/dist/scorers/index.d.ts

@@ -13,6 +13,7 @@ export declare const subtaskIndependence: import("evalite").Evalite.Scorer<unkno
 export { executionSuccess, timeBalance, scopeAccuracy, scopeDrift, noRework, } from "./outcome-scorers.js";
 export { confidenceAccuracy, contextInjectionCorrectness, requiredPatternsPresent, forbiddenPatternsAbsent, compactionQuality, } from "./compaction-scorers.js";
 export { violationCount, spawnEfficiency, reviewThoroughness, timeToFirstSpawn, overallDiscipline, } from "./coordinator-discipline.js";
+export { strategySelectionQuality, precedentRelevance, } from "./decision-quality-scorers.js";
 /**
  * Checks that subtasks cover the full task scope
  *

package/dist/scorers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scorers/index.ts"],"names":[],"mappings":"AAOA;;GAEG;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,mBAAmB,6DAuC9B,CAAC;AAMH,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,aAAa,EACb,UAAU,EACV,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,6DAsD/B,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,6DAsD7B,CAAC;AAMH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,sBAAsB,6DAmFjC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scorers/index.ts"],"names":[],"mappings":"AAOA;;GAEG;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,mBAAmB,6DAuC9B,CAAC;AAMH,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,aAAa,EACb,UAAU,EACV,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAMrC,OAAO,EACL,wBAAwB,EACxB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AAEvC;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,6DAsD/B,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,6DAsD7B,CAAC;AAMH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,sBAAsB,6DAmFjC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swarmtools/evals",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "Evaluation suite for swarm-tools multi-agent coordination",
   "type": "module",
   "main": "./dist/index.js",
@@ -33,8 +33,8 @@
   "dependencies": {
     "evalite": "^1.0.0-beta.10",
     "ai": "6.0.0-beta.150",
-    "opencode-swarm-plugin": "0.44.2",
-    "swarm-mail": "1.6.0"
+    "opencode-swarm-plugin": "0.45.1",
+    "swarm-mail": "1.6.1"
   },
   "devDependencies": {
     "@types/node": "^22.19.3",