@smithers-orchestrator/scorers 0.24.0 → 0.25.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smithers-orchestrator/scorers",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "Smithers scorer definitions, execution, aggregation, and persistence helpers",
   "type": "module",
   "sideEffects": false,
@@ -33,12 +33,12 @@
     "drizzle-orm": "^0.45.2",
     "effect": "^3.21.1",
     "zod": "^4.3.6",
-    "@smithers-orchestrator/agents": "0.24.0",
-    "@smithers-orchestrator/errors": "0.24.0",
-    "@smithers-orchestrator/observability": "0.24.0",
-    "@smithers-orchestrator/db": "0.24.0",
-    "@smithers-orchestrator/scheduler": "0.24.0",
-    "@smithers-orchestrator/graph": "0.24.0"
+    "@smithers-orchestrator/agents": "0.25.0",
+    "@smithers-orchestrator/db": "0.25.0",
+    "@smithers-orchestrator/errors": "0.25.0",
+    "@smithers-orchestrator/observability": "0.25.0",
+    "@smithers-orchestrator/graph": "0.25.0",
+    "@smithers-orchestrator/scheduler": "0.25.0"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/__type-tests__/smithersScorers.test-d.ts

@@ -0,0 +1,49 @@
+import { smithersScorers } from "../index.js";
+
+type InsertScorer = typeof smithersScorers.$inferInsert;
+type SelectScorer = typeof smithersScorers.$inferSelect;
+
+const insertRow: InsertScorer = {
+    id: "score-1",
+    runId: "run-1",
+    nodeId: "node-1",
+    scorerId: "accuracy",
+    scorerName: "Accuracy",
+    source: "batch",
+    score: 0.95,
+    scoredAtMs: 1_700_000_000_000,
+};
+
+const selectRow: SelectScorer = {
+    id: "score-1",
+    runId: "run-1",
+    nodeId: "node-1",
+    iteration: 0,
+    attempt: 0,
+    scorerId: "accuracy",
+    scorerName: "Accuracy",
+    source: "batch",
+    score: 0.95,
+    reason: null,
+    metaJson: null,
+    inputJson: null,
+    outputJson: null,
+    groundTruthJson: null,
+    contextJson: null,
+    latencyMs: null,
+    scoredAtMs: 1_700_000_000_000,
+    durationMs: null,
+};
+
+selectRow.score satisfies number;
+selectRow.runId satisfies string;
+selectRow.reason satisfies string | null;
+
+// @ts-expect-error score must remain typed as a number, not erased to any.
+insertRow.score = "0.95";
+
+// @ts-expect-error runId must remain typed as a string, not erased to any.
+insertRow.runId = 123;
+
+// @ts-expect-error unknown columns must not be accepted.
+insertRow.unknownColumn = "nope";

package/src/aggregate.js

@@ -18,12 +18,13 @@
  */
 export async function aggregateScores(adapter, opts) {
     const conditions = [];
+    const params = [];
     if (opts?.runId)
-        conditions.push(`run_id = '${escapeSql(opts.runId)}'`);
+        addFilter(conditions, params, "run_id", opts.runId);
     if (opts?.nodeId)
-        conditions.push(`node_id = '${escapeSql(opts.nodeId)}'`);
+        addFilter(conditions, params, "node_id", opts.nodeId);
     if (opts?.scorerId)
-        conditions.push(`scorer_id = '${escapeSql(opts.scorerId)}'`);
+        addFilter(conditions, params, "scorer_id", opts.scorerId);
     const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
     // Step 1: Get aggregate stats via SQL
     const aggQuery = `
@@ -39,7 +40,7 @@ export async function aggregateScores(adapter, opts) {
     GROUP BY scorer_id, scorer_name
     ORDER BY scorer_name
   `;
-    const aggRows = (await adapter.rawQuery(aggQuery));
+    const aggRows = (await adapter.rawQuery(aggQuery, params));
     if (aggRows.length === 0)
         return [];
     // Step 2: Get all scores to compute p50 and stddev per scorer in memory
@@ -49,7 +50,7 @@ export async function aggregateScores(adapter, opts) {
     ${where}
     ORDER BY scorer_id, score
   `;
-    const allScores = (await adapter.rawQuery(scoresQuery));
+    const allScores = (await adapter.rawQuery(scoresQuery, params));
     // Group scores by scorer_id
     const scoresByScorer = new Map();
     for (const row of allScores) {
@@ -100,9 +101,12 @@ function computeStddev(values, mean) {
     return Math.sqrt(variance);
 }
 /**
+ * @param {string[]} conditions
+ * @param {string[]} params
+ * @param {string} column
  * @param {string} value
- * @returns {string}
  */
-function escapeSql(value) {
-    return value.replace(/'/g, "''");
+function addFilter(conditions, params, column, value) {
+    conditions.push(`${column} = ?`);
+    params.push(value);
 }

package/src/index.d.ts

@@ -2,6 +2,7 @@ import * as _smithers_agents_AgentLike from '@smithers-orchestrator/agents/Agent
 import { AgentLike as AgentLike$3 } from '@smithers-orchestrator/agents/AgentLike';
 import { ZodObject } from 'zod';
 import * as _smithers_db_adapter from '@smithers-orchestrator/db/adapter';
+import * as drizzle_orm_sqlite_core from 'drizzle-orm/sqlite-core';
 import * as effect_MetricState from 'effect/MetricState';
 import * as effect_MetricKeyType from 'effect/MetricKeyType';
 import { Metric } from 'effect';
@@ -74,6 +75,8 @@ type ScoreRow$1 = {
     metaJson: string | null;
     inputJson: string | null;
     outputJson: string | null;
+    groundTruthJson: string | null;
+    contextJson: string | null;
     latencyMs: number | null;
     scoredAtMs: number;
     durationMs: number | null;
@@ -97,6 +100,8 @@ type ScorerContext$2 = {
     attempt: number;
     input: unknown;
     output: unknown;
+    groundTruth?: unknown;
+    context?: unknown;
     latencyMs?: number;
     outputSchema?: ZodObject;
 };
@@ -156,7 +161,48 @@ type SmithersDb$1 = _smithers_db_adapter.SmithersDb;
  * Drizzle table definition for the `_smithers_scorers` table.
  * Stores individual scorer results for each task execution.
  */
-declare const smithersScorers: any;
+type SmithersScorerColumn<Name extends string, Data, NotNull extends boolean, HasDefault extends boolean, PrimaryKey extends boolean, ColumnType extends string, DataType extends "string" | "number"> = drizzle_orm_sqlite_core.SQLiteColumn<{
+    name: Name;
+    tableName: "_smithers_scorers";
+    dataType: DataType;
+    columnType: ColumnType;
+    data: Data;
+    driverParam: Data;
+    notNull: NotNull;
+    hasDefault: HasDefault;
+    isPrimaryKey: PrimaryKey;
+    isAutoincrement: false;
+    hasRuntimeDefault: false;
+    enumValues: DataType extends "string" ? [string, ...string[]] : undefined;
+    baseColumn: never;
+    identity: undefined;
+    generated: undefined;
+}, {}, {}>;
+declare const smithersScorers: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{
+    name: "_smithers_scorers";
+    schema: undefined;
+    columns: {
+        id: SmithersScorerColumn<"id", string, true, false, true, "SQLiteText", "string">;
+        runId: SmithersScorerColumn<"run_id", string, true, false, false, "SQLiteText", "string">;
+        nodeId: SmithersScorerColumn<"node_id", string, true, false, false, "SQLiteText", "string">;
+        iteration: SmithersScorerColumn<"iteration", number, true, true, false, "SQLiteInteger", "number">;
+        attempt: SmithersScorerColumn<"attempt", number, true, true, false, "SQLiteInteger", "number">;
+        scorerId: SmithersScorerColumn<"scorer_id", string, true, false, false, "SQLiteText", "string">;
+        scorerName: SmithersScorerColumn<"scorer_name", string, true, false, false, "SQLiteText", "string">;
+        source: SmithersScorerColumn<"source", string, true, false, false, "SQLiteText", "string">;
+        score: SmithersScorerColumn<"score", number, true, false, false, "SQLiteReal", "number">;
+        reason: SmithersScorerColumn<"reason", string, false, false, false, "SQLiteText", "string">;
+        metaJson: SmithersScorerColumn<"meta_json", string, false, false, false, "SQLiteText", "string">;
+        inputJson: SmithersScorerColumn<"input_json", string, false, false, false, "SQLiteText", "string">;
+        outputJson: SmithersScorerColumn<"output_json", string, false, false, false, "SQLiteText", "string">;
+        groundTruthJson: SmithersScorerColumn<"ground_truth_json", string, false, false, false, "SQLiteText", "string">;
+        contextJson: SmithersScorerColumn<"context_json", string, false, false, false, "SQLiteText", "string">;
+        latencyMs: SmithersScorerColumn<"latency_ms", number, false, false, false, "SQLiteReal", "number">;
+        scoredAtMs: SmithersScorerColumn<"scored_at_ms", number, true, false, false, "SQLiteInteger", "number">;
+        durationMs: SmithersScorerColumn<"duration_ms", number, false, false, false, "SQLiteReal", "number">;
+    };
+    dialect: "sqlite";
+}>;
 
 /** @typedef {import("./CreateScorerConfig.js").CreateScorerConfig} CreateScorerConfig */
 /** @typedef {import("./types.js").Scorer} Scorer */

package/src/index.js

@@ -15,9 +15,14 @@
 // @smithers-type-exports-end
 
 // Factories
-export { createScorer, llmJudge } from "./create-scorer.js";
+export { createScorer } from "./createScorer.js";
+export { llmJudge } from "./llmJudge.js";
 // Built-in scorers
-export { relevancyScorer, toxicityScorer, faithfulnessScorer, schemaAdherenceScorer, latencyScorer, } from "./builtins.js";
+export { relevancyScorer } from "./relevancyScorer.js";
+export { toxicityScorer } from "./toxicityScorer.js";
+export { faithfulnessScorer } from "./faithfulnessScorer.js";
+export { schemaAdherenceScorer } from "./schemaAdherenceScorer.js";
+export { latencyScorer } from "./latencyScorer.js";
 // Execution
 export { runScorersAsync, runScorersBatch } from "./run-scorers.js";
 // Aggregation

package/src/llmJudge.js

@@ -12,12 +12,19 @@
  * in the `reason`) do not prematurely close the object.
  *
  * @param {string} text
- * @returns {Record<string, unknown> | undefined}
+ * @returns {Record<string, unknown> | unknown[] | number | undefined}
  */
 function parseJudgeJson(text) {
     const trimmed = text.trim();
     try {
-        return JSON.parse(trimmed);
+        const parsed = JSON.parse(trimmed);
+        if (typeof parsed === "number") {
+            return parsed;
+        }
+        if (parsed && typeof parsed === "object") {
+            return parsed;
+        }
+        return undefined;
     }
     catch {
         // fall through to balanced-brace extraction
@@ -108,7 +115,16 @@ export function llmJudge(config) {
         // text, then fall back to the outermost balanced-brace object so that a
         // brace inside the judge's `reason` string does not truncate the match.
         const parsed = parseJudgeJson(text);
-        if (parsed && typeof parsed === "object") {
+        if (typeof parsed === "number") {
+            return {
+                score: Number.isFinite(parsed)
+                    ? Math.max(0, Math.min(1, parsed))
+                    : 0,
+                reason: undefined,
+                meta: { raw: text },
+            };
+        }
+        if (parsed) {
             const rawScore = Number(parsed.score);
             return {
                 score: Number.isFinite(rawScore)

package/src/types.ts

@@ -83,6 +83,8 @@ export type ScoreRow = {
   metaJson: string | null;
   inputJson: string | null;
   outputJson: string | null;
+  groundTruthJson: string | null;
+  contextJson: string | null;
   latencyMs: number | null;
   scoredAtMs: number;
   durationMs: number | null;
@@ -112,6 +114,8 @@ export type ScorerContext = {
   attempt: number;
   input: unknown;
   output: unknown;
+  groundTruth?: unknown;
+  context?: unknown;
   latencyMs?: number;
   outputSchema?: ZodObject;
 };

package/src/builtins.js

@@ -1,5 +0,0 @@
-export { relevancyScorer } from "./relevancyScorer.js";
-export { toxicityScorer } from "./toxicityScorer.js";
-export { faithfulnessScorer } from "./faithfulnessScorer.js";
-export { schemaAdherenceScorer } from "./schemaAdherenceScorer.js";
-export { latencyScorer } from "./latencyScorer.js";

package/src/create-scorer.js

@@ -1,7 +0,0 @@
-// @smithers-type-exports-begin
-/** @typedef {import("./CreateScorerConfig.js").CreateScorerConfig} CreateScorerConfig */
-/** @typedef {import("./LlmJudgeConfig.js").LlmJudgeConfig} LlmJudgeConfig */
-// @smithers-type-exports-end
-
-export { createScorer } from "./createScorer.js";
-export { llmJudge } from "./llmJudge.js";

package/src/react-types.ts

@@ -1 +0,0 @@
-export type { ScorersMap } from "@smithers-orchestrator/graph/types";