@restormel/testing-runner 0.1.5 → 0.1.6

package/dist/suite-llm-budget.js

@@ -0,0 +1,154 @@
+function hasDefinedNumericUsage(u) {
+    return ((u.promptTokens !== undefined && Number.isFinite(u.promptTokens)) ||
+        (u.completionTokens !== undefined && Number.isFinite(u.completionTokens)) ||
+        (u.totalTokens !== undefined && Number.isFinite(u.totalTokens)));
+}
+/**
+ * Enforces suite- and goal-level `llm_budget` and aggregates provider token usage for {@link RunRecord.costEstimate}.
+ */
+export class SuiteLlmBudgetTracker {
+    suiteBudget;
+    suiteStart = Date.now();
+    suiteCompletions = 0;
+    suiteAcRounds = 0;
+    goalCompletions = 0;
+    goalAcRounds = 0;
+    currentGoalId = "";
+    goalBudget;
+    promptSum = 0;
+    completionSum = 0;
+    totalReportedSum = 0;
+    /** True when at least one completion returned numeric `usage` fields (including zeros). */
+    providerReportedUsage = false;
+    estimateOnlyCalls = 0;
+    constructor(suiteBudget) {
+        this.suiteBudget = suiteBudget;
+    }
+    beginGoal(goalId, goalBudget) {
+        this.currentGoalId = goalId;
+        this.goalBudget = goalBudget;
+        this.goalCompletions = 0;
+        this.goalAcRounds = 0;
+    }
+    wallClockMs() {
+        return Date.now() - this.suiteStart;
+    }
+    /** Call before each chat completion (judge or AC agent). */
+    tryConsumeLlm(kind) {
+        const wall = this.wallClockMs();
+        if (this.suiteBudget?.maxWallClockMs !== undefined && wall >= this.suiteBudget.maxWallClockMs) {
+            return {
+                code: "SUITE_BUDGET_WALL_CLOCK",
+                summary: `Suite llm_budget: wall clock ${wall}ms >= max_wall_clock_ms ${this.suiteBudget.maxWallClockMs}`,
+            };
+        }
+        if (this.suiteBudget?.maxCompletions !== undefined && this.suiteCompletions >= this.suiteBudget.maxCompletions) {
+            return {
+                code: "SUITE_BUDGET_MAX_COMPLETIONS",
+                summary: `Suite llm_budget: ${this.suiteCompletions} LLM calls >= max_completions ${this.suiteBudget.maxCompletions}`,
+            };
+        }
+        if (kind === "ac_round" &&
+            this.suiteBudget?.maxRounds !== undefined &&
+            this.suiteAcRounds >= this.suiteBudget.maxRounds) {
+            return {
+                code: "SUITE_BUDGET_MAX_ROUNDS",
+                summary: `Suite llm_budget: ${this.suiteAcRounds} AC agent rounds >= max_rounds ${this.suiteBudget.maxRounds}`,
+            };
+        }
+        if (this.goalBudget?.maxCompletions !== undefined && this.goalCompletions >= this.goalBudget.maxCompletions) {
+            return {
+                code: "GOAL_BUDGET_MAX_COMPLETIONS",
+                summary: `Goal "${this.currentGoalId}" llm_budget: ${this.goalCompletions} LLM calls >= max_completions ${this.goalBudget.maxCompletions}`,
+            };
+        }
+        if (kind === "ac_round" &&
+            this.goalBudget?.maxRounds !== undefined &&
+            this.goalAcRounds >= this.goalBudget.maxRounds) {
+            return {
+                code: "GOAL_BUDGET_MAX_ROUNDS",
+                summary: `Goal "${this.currentGoalId}" llm_budget: ${this.goalAcRounds} AC rounds >= max_rounds ${this.goalBudget.maxRounds}`,
+            };
+        }
+        return null;
+    }
+    /** Suite wall clock or total completion cap — skip starting further goals without consuming a slot. */
+    suiteNonAcWouldBlock() {
+        const wall = this.wallClockMs();
+        if (this.suiteBudget?.maxWallClockMs !== undefined && wall >= this.suiteBudget.maxWallClockMs) {
+            return {
+                code: "SUITE_BUDGET_WALL_CLOCK",
+                summary: `Suite llm_budget: wall clock ${wall}ms >= max_wall_clock_ms ${this.suiteBudget.maxWallClockMs}`,
+            };
+        }
+        if (this.suiteBudget?.maxCompletions !== undefined && this.suiteCompletions >= this.suiteBudget.maxCompletions) {
+            return {
+                code: "SUITE_BUDGET_MAX_COMPLETIONS",
+                summary: `Suite llm_budget: ${this.suiteCompletions} LLM calls >= max_completions ${this.suiteBudget.maxCompletions}`,
+            };
+        }
+        return null;
+    }
+    /** Before an ac_sequence goal: suite AC round budget already exhausted. */
+    suiteAcBudgetWouldBlock() {
+        if (this.suiteBudget?.maxRounds !== undefined && this.suiteAcRounds >= this.suiteBudget.maxRounds) {
+            return {
+                code: "SUITE_BUDGET_MAX_ROUNDS",
+                summary: `Suite llm_budget: ${this.suiteAcRounds} AC agent rounds >= max_rounds ${this.suiteBudget.maxRounds}`,
+            };
+        }
+        return null;
+    }
+    /** Call after an LLM request returns (count attempt even if model errored after HTTP 200). */
+    recordLlmCall(kind, usage) {
+        this.suiteCompletions++;
+        this.goalCompletions++;
+        if (kind === "ac_round") {
+            this.suiteAcRounds++;
+            this.goalAcRounds++;
+        }
+        if (usage !== undefined && hasDefinedNumericUsage(usage)) {
+            this.providerReportedUsage = true;
+            this.promptSum += usage.promptTokens ?? 0;
+            this.completionSum += usage.completionTokens ?? 0;
+            const t = usage.totalTokens;
+            if (t !== undefined && Number.isFinite(t))
+                this.totalReportedSum += t;
+        }
+        else {
+            this.estimateOnlyCalls++;
+        }
+    }
+    summarizeExecution() {
+        return {
+            wallClockMs: this.wallClockMs(),
+            llmCompletions: this.suiteCompletions,
+            acAgentRounds: this.suiteAcRounds,
+        };
+    }
+    usageForCostEstimate() {
+        const hasProvider = this.providerReportedUsage;
+        if (hasProvider && this.estimateOnlyCalls > 0) {
+            return {
+                tokenUsage: {
+                    prompt: this.promptSum > 0 ? this.promptSum : undefined,
+                    completion: this.completionSum > 0 ? this.completionSum : undefined,
+                    total: this.totalReportedSum > 0 ? this.totalReportedSum : undefined,
+                },
+                usageSource: "mixed",
+            };
+        }
+        if (hasProvider) {
+            return {
+                tokenUsage: {
+                    prompt: this.promptSum > 0 ? this.promptSum : undefined,
+                    completion: this.completionSum > 0 ? this.completionSum : undefined,
+                    total: this.totalReportedSum > 0 ? this.totalReportedSum : undefined,
+                },
+                usageSource: "provider",
+            };
+        }
+        return { usageSource: "estimate" };
+    }
+}
+//# sourceMappingURL=suite-llm-budget.js.map

package/dist/suite-llm-budget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite-llm-budget.js","sourceRoot":"","sources":["../src/suite-llm-budget.ts"],"names":[],"mappings":"AAQA,SAAS,sBAAsB,CAAC,CAAgB;IAC9C,OAAO,CACL,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAChE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAeH;IAdZ,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,gBAAgB,GAAG,CAAC,CAAC;IACrB,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,GAAG,CAAC,CAAC;IACpB,YAAY,GAAG,CAAC,CAAC;IACjB,aAAa,GAAG,EAAE,CAAC;IACnB,UAAU,CAAwB;IAClC,SAAS,GAAG,CAAC,CAAC;IACd,aAAa,GAAG,CAAC,CAAC;IAClB,gBAAgB,GAAG,CAAC,CAAC;IAC7B,2FAA2F;IACnF,qBAAqB,GAAG,KAAK,CAAC;IAC9B,iBAAiB,GAAG,CAAC,CAAC;IAE9B,YAA6B,WAAkC;QAAlC,gBAAW,GAAX,WAAW,CAAuB;IAAG,CAAC;IAEnE,SAAS,CAAC,MAAc,EAAE,UAAiC;QACzD,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,4DAA4D;IAC5D,aAAa,CAAC,IAAyB;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,WAAW,EAAE,cAAc,KAAK,SAAS,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YAC9F,OAAO;gBACL,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,gCAAgC,IAAI,2BAA2B,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE;aAC1G,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YAC/G,OAAO;gBACL,IAAI,EAAE,8BAA8B;gBACpC,OAAO,EAAE,qBAAqB,IAAI,CAAC,gBAAgB,iCAAiC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE;aACtH,CAAC;QACJ,CAAC;QACD,IACE,IAAI,KAAK,UAAU;YACnB,IAAI,CAAC,WAAW,EAAE,SAAS,KAAK,SAAS;YACzC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAChD,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,qBAAqB,IAAI,CAAC,aAAa,kCAAkC,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;aAC/G,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC;YAC5G,OAAO;gBACL,IAAI,EAAE,6BAA6B;gBACnC,OAAO,EAAE,SAAS,IAAI,CAAC,aAAa,iBAAiB,IAAI,CAAC,eAAe,iCAAiC,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE;aAC3I,CAAC;QACJ,CAAC;QACD,IACE,IAAI,KAAK,UAAU;YACnB,IAAI,CAAC,UAAU,EAAE,SAAS,KAAK,SAAS;YACxC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,EAC9C,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EAAE,SAAS,IAAI,CAAC,aAAa,iBAAiB,IAAI,CAAC,YAAY,4BAA4B,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;aAC9H,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uGAAuG;IACvG,oBAAoB;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,WAAW,EAAE,cAAc,KAAK,SAAS,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YAC9F,OAAO;gBACL,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,gCAAgC,IAAI,2BAA2B,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE;aAC1G,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC;YAC/G,OAAO;gBACL,IAAI,EAAE,8BAA8B;gBACpC,OAAO,EAAE,qBAAqB,IAAI,CAAC,gBAAgB,iCAAiC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE;aACtH,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2EAA2E;IAC3E,uBAAuB;QACrB,IAAI,IAAI,CAAC,WAAW,EAAE,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;YAClG,OAAO;gBACL,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,qBAAqB,IAAI,CAAC,aAAa,kCAAkC,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;aAC/G,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8FAA8F;IAC9F,aAAa,CAAC,IAAyB,EAAE,KAAqB;QAC5D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,IAAI,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC;YACzD,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAClC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,aAAa,IAAI,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC;YAC5B,IAAI,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,kBAAkB;QAChB,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE;YAC/B,cAAc,EAAE,IAAI,CAAC,gBAAgB;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;IACJ,CAAC;IAED,oBAAoB;QAIlB,MAAM,WAAW,GAAG,IAAI,CAAC,qBAAqB,CAAC;QAC/C,IAAI,WAAW,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO;gBACL,UAAU,EAAE;oBACV,MAAM,EAAE,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACvD,UAAU,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;oBACnE,KAAK,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;iBACrE;gBACD,WAAW,EAAE,OAAO;aACrB,CAAC;QACJ,CAAC;QACD,IAAI,WAAW,EAAE,CAAC;YAChB,OAAO;gBACL,UAAU,EAAE;oBACV,MAAM,EAAE,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACvD,UAAU,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;oBACnE,KAAK,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;iBACrE;gBACD,WAAW,EAAE,UAAU;aACxB,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;IACrC,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@restormel/testing-runner",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Suite execution: browser goals, Keys-backed judges, retries, artefacts.",
   "license": "MIT",
   "repository": {
@@ -34,10 +34,10 @@
     "chrome-launcher": "^1.2.1",
     "lighthouse": "^13.1.0",
     "playwright": "^1.49.0",
-    "@restormel/testing-browser-playwright": "0.1.5",
-    "@restormel/testing-config": "0.1.5",
-    "@restormel/testing-core": "0.1.5",
-    "@restormel/testing-keys-adapter": "0.1.5"
+    "@restormel/testing-browser-playwright": "0.1.6",
+    "@restormel/testing-config": "0.1.6",
+    "@restormel/testing-keys-adapter": "0.1.6",
+    "@restormel/testing-core": "0.1.6"
   },
   "devDependencies": {
     "typescript": "^5.7.0"