@workglow/eval 0.4.9 → 0.5.0

package/dist/evals/runner.d.ts

@@ -35,8 +35,12 @@ export interface SweepOptions {
  * An unreported counter is stored as `null`, never `0` — the database has to
  * keep the distinction the in-memory type protects, or a model that never
  * mentions caching becomes indistinguishable from one that cached nothing.
+ *
+ * `at` is when the request ran, which is what a time-of-day rate is charged
+ * against. A batch that straddles the boundary of one must not have its rows
+ * priced by whenever the sweep happened to reach the write.
  */
-export declare function usageColumns(usage: Usage | undefined, pricing: ModelPricing | undefined): {
+export declare function usageColumns(usage: Usage | undefined, pricing: ModelPricing | undefined, at: Date | undefined): {
     input_tokens: number | null;
     output_tokens: number | null;
     cached_tokens: number | null;

package/dist/evals/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAEhF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAI/D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAe,MAAM,SAAS,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAChB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,KAAK,IAAI,CAAC,GAC7F,SAAS,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACpE;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;CAC9C;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,KAAK,GAAG,SAAS,EACxB,OAAO,EAAE,YAAY,GAAG,SAAS,GAChC;IACD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAWA;AAoBD;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,SAAS,gBAAgB,EAAE,EACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAmFjB"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAEhF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAI/D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAe,MAAM,SAAS,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAChB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,KAAK,IAAI,CAAC,GAC7F,SAAS,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACpE;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;CAC9C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,KAAK,GAAG,SAAS,EACxB,OAAO,EAAE,YAAY,GAAG,SAAS,EACjC,EAAE,EAAE,IAAI,GAAG,SAAS,GACnB;IACD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAWA;AAoBD;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,SAAS,gBAAgB,EAAE,EACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAoFjB"}

package/dist/report/liveTally.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"liveTally.d.ts","sourceRoot":"","sources":["../../src/report/liveTally.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AASlD;;;;;GAKG;AACH,qBAAa,SAAS;IAKlB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,YACmB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,EACrE,GAAG,EAAE,MAAM,EAGZ;IAED,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,GAAG,IAAI,CAMjE;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAevD;CACF"}
+{"version":3,"file":"liveTally.d.ts","sourceRoot":"","sources":["../../src/report/liveTally.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AASlD;;;;;GAKG;AACH,qBAAa,SAAS;IAKlB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,YACmB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,YAAY,GAAG,SAAS,EACrE,GAAG,EAAE,MAAM,EAGZ;IAED,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,GAAG,IAAI,CAMjE;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAmBvD;CACF"}

package/dist/workglow-eval.js

@@ -1056,8 +1056,8 @@ function makeSimilarityExecutor(model, options, context) {
 }
 
 // src/evals/runner.ts
-function usageColumns(usage, pricing) {
-  const estimate = usage ? estimateCost(usage, pricing) : undefined;
+function usageColumns(usage, pricing, at) {
+  const estimate = usage ? estimateCost(usage, pricing, { at }) : undefined;
   return {
     input_tokens: usage?.input ?? null,
     output_tokens: usage?.output ?? null,
@@ -1111,6 +1111,7 @@ async function runSweep(stores, rows, options) {
     }
     for (const { record, row, parseError } of parsedRows) {
       const t0 = performance.now();
+      const requestedAt = new Date;
       let outcome;
       let outcomeUsage;
       if (!executor) {
@@ -1139,7 +1140,7 @@ async function runSweep(stores, rows, options) {
         model: modelId,
         row_index: record.row_index,
         latency_ms: Math.round(latency * 100) / 100,
-        ...usageColumns(outcomeUsage, modelPricing),
+        ...usageColumns(outcomeUsage, modelPricing, requestedAt),
         ...outcome
       });
       done++;
@@ -1239,7 +1240,7 @@ class LiveTally {
     const lines = [`[${done}/${total}]`];
     for (const [model, tally] of this.byModel) {
       const tps = tally.usage?.output === undefined ? "" : ` ${Math.round(tally.usage.output / elapsedSec)} tok/s`;
-      const cost = tally.usage ? formatCost(estimateCost2(tally.usage, this.pricing(model))) : "";
+      const cost = tally.usage ? formatCost(estimateCost2(tally.usage, this.pricing(model), { at: this.startedAt })) : "";
       lines.push(`  ${model}  ${tally.ok}/${tally.rows} ok  ` + `${formatUsage(tally.usage, "detailed")}${tps}${cost ? `  ${cost}` : ""}`);
     }
     return lines.join(`
@@ -1362,7 +1363,7 @@ import { registerHuggingFaceTransformers } from "@workglow/huggingface-transform
 import { registerLlamaCpp } from "@workglow/node-llama-cpp/ai";
 import { registerOpenAiInline } from "@workglow/openai/ai-runtime";
 import { registerBaseTasks, registerBuiltInTransforms } from "@workglow/task-graph";
-import { EnvCredentialStore, setGlobalCredentialStore } from "@workglow/util";
+import { EnvCredentialStore, setGlobalCredentialStore, Worker } from "@workglow/util";
 import { registerXaiInline } from "@workglow/xai/ai-runtime";
 async function registerEvalProviders(config) {
   registerBaseTasks();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/eval",
   "type": "module",
-  "version": "0.4.9",
+  "version": "0.5.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/workglow-dev/libs.git",
@@ -33,22 +33,22 @@
     "src/**/*.md"
   ],
   "dependencies": {
-    "@workglow/ai": "0.4.9",
-    "@workglow/anthropic": "0.4.9",
-    "@workglow/cli": "0.4.9",
-    "@workglow/deepseek": "0.4.9",
-    "@workglow/google-gemini": "0.4.9",
-    "@workglow/huggingface-transformers": "0.4.9",
-    "@workglow/node-llama-cpp": "0.4.9",
-    "@workglow/openai": "0.4.9",
-    "@workglow/sqlite": "0.4.9",
-    "@workglow/storage": "0.4.9",
-    "@workglow/task-graph": "0.4.9",
-    "@workglow/util": "0.4.9",
-    "@workglow/xai": "0.4.9",
+    "@workglow/ai": "0.5.0",
+    "@workglow/anthropic": "0.5.0",
+    "@workglow/cli": "0.5.0",
+    "@workglow/deepseek": "0.5.0",
+    "@workglow/google-gemini": "0.5.0",
+    "@workglow/huggingface-transformers": "0.5.0",
+    "@workglow/node-llama-cpp": "0.5.0",
+    "@workglow/openai": "0.5.0",
+    "@workglow/sqlite": "0.5.0",
+    "@workglow/storage": "0.5.0",
+    "@workglow/task-graph": "0.5.0",
+    "@workglow/util": "0.5.0",
+    "@workglow/xai": "0.5.0",
     "@huggingface/transformers": "^4.2.0",
     "commander": "^15.0.0",
-    "hyparquet": "^1.29.2",
+    "hyparquet": "^1.30.0",
     "hyparquet-compressors": "^1.1.1"
   },
   "publishConfig": {