ben-ai-models-registry 1.0.49 → 1.0.50

package/index.js

@@ -1,6 +1,8 @@
 // index.js
 const JobStatus = require("./models/JobStatus");
 const Run = require("./models/Run");
+const GraphRun = require("./models/GraphRun");
+const GraphEvalResult = require("./models/GraphEvalResult");
 const Files = require("./models/Files");
 const Dataset = require("./models/Dataset");
 const Program = require("./models/Program");
@@ -18,6 +20,8 @@ const ScraperConfig = require("./models/ScraperConfig");
 module.exports = {
   JobStatus,
   Run,
+  GraphRun,
+  GraphEvalResult,
   Files,
   Dataset,
   State,

package/models/GraphEvalResult.js

@@ -0,0 +1,95 @@
+// GraphEvalResult: ONE flat doc per (question x iteration) of a GraphRun.
+// Keyed by graphRunId (indexed). Inserted independently by the graph-run-worker
+// as each eval completes — no $push into a parent, no shared-doc contention.
+// The graph results view reads these paginated, filtered by graphRunId.
+const mongoose = require("mongoose");
+const { Schema } = mongoose;
+
+// One LAAJ judge verdict. Mirrors the shape produced by ragEvaluationModel.
+const JudgeScoreSchema = new Schema(
+  {
+    score: { type: Number },
+    reasoning: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const SourceSchema = new Schema(
+  {
+    filename: { type: String },
+    url: { type: String },
+    citations: [
+      {
+        // String, not ObjectId: buildSources (eval-run route) sets this to the
+        // retrieved doc's id string. A non-ObjectId id (e.g. composite chunk id)
+        // would throw on cast; storing as String matches context[]._id too.
+        embeddingId: { type: String },
+        pageContent: { type: String },
+      },
+    ],
+  },
+  { _id: false, versionKey: false }
+);
+
+// Human evaluation (added so graph runs can be human-scored in the run view,
+// mirroring the chain Run model's humanEvaluation/humanOverallScore).
+const EvaluationQuestionSchema = new Schema(
+  {
+    question: { type: String },
+    score: { type: Number },
+    reasoning: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const HumanEvaluationSchema = new Schema(
+  {
+    criterion: { type: String },
+    questions: [EvaluationQuestionSchema],
+    reasoning: { type: String },
+    overallScore: { type: Number },
+  },
+  { _id: false, versionKey: false }
+);
+
+const GraphEvalResultSchema = new Schema(
+  {
+    graphRunId: {
+      type: Schema.Types.ObjectId,
+      ref: "GraphRun",
+      required: true,
+      index: true,
+    },
+    programId: { type: Schema.Types.ObjectId, ref: "Program" },
+    question: { type: String },
+    expectedAnswer: { type: String },
+    iteration: { type: Number },
+    generatedAnswer: { type: String },
+    // Classification of generatedAnswer: "substantive" | "refusal" |
+    // "conversational". Drives which judges apply (a refusal has no claims to
+    // ground; small talk is off the relevance axis) and the run-view rollups.
+    answerType: { type: String },
+    sources: { type: [SourceSchema], default: [] },
+    context: { type: [Schema.Types.Mixed], default: [] },
+    // The four RAG judges.
+    correctness: { type: JudgeScoreSchema },
+    relevance: { type: JudgeScoreSchema },
+    groundedness: { type: JudgeScoreSchema },
+    retrievalRelevance: { type: JudgeScoreSchema },
+    // Human evaluation (filled in from the run view; empty until a human scores it).
+    humanEvaluation: { type: [HumanEvaluationSchema], default: [] },
+    humanOverallScore: { type: String },
+    // Graph-specific signal not present in the chain path.
+    retryCount: { type: Number },
+    docsRetrieved: { type: Number },
+    latencyMs: { type: Number },
+    errorMessage: { type: String },
+    traceId: { type: String },
+    traceUrl: { type: String },
+  },
+  { timestamps: true, versionKey: false, collection: "graph_eval_results" }
+);
+
+module.exports =
+  mongoose.models.GraphEvalResult ||
+  mongoose.model("GraphEvalResult", GraphEvalResultSchema);

package/models/GraphRun.js

@@ -0,0 +1,66 @@
+// GraphRun: parent doc for a graph-engine eval run. Holds metadata + computed
+// aggregates ONLY. Per-question results live in flat GraphEvalResult docs
+// (one per question x iteration), keyed by graphRunId. This split avoids the
+// 16MB single-doc ceiling, $push races, and unqueryable nesting of the legacy
+// Run.evalResults schema. The chain (single-loop/baseline) path keeps using Run.
+const mongoose = require("mongoose");
+const { Schema } = mongoose;
+
+// Snapshot of the config the graph ran with. Empty/undefined fields mean the
+// live production default was used (no override). Present fields = a sweep.
+const ParamsSnapshotSchema = new Schema(
+  {
+    model: { type: String },
+    k: { type: Number },
+    fetchK: { type: Number },
+    instruction: { type: String },
+    embeddingCollection: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const GraphRunSchema = new Schema(
+  {
+    stateId: { type: Schema.Types.ObjectId, ref: "State" },
+    datasetId: { type: Schema.Types.ObjectId, ref: "Dataset", index: true },
+    name: { type: String },
+    description: { type: String },
+    type: { type: String, default: "graph" },
+    programId: { type: Schema.Types.ObjectId, ref: "Program" },
+    fileStore: { type: Schema.Types.ObjectId, ref: "FileStore" },
+    paramsSnapshot: { type: ParamsSnapshotSchema, default: {} },
+    evalModel: { type: String },
+    iterations: { type: Number },
+    // Aggregates computed on completion by querying GraphEvalResult.
+    // Scores are averaged ONLY over results where the metric was applicable
+    // (non-null). A metric with zero applicable results stores null, not 0.
+    llmPassRate: { type: Number },
+    relevanceScore: { type: Number },
+    groundednessScore: { type: Number },
+    retrievalRelevanceScore: { type: Number },
+    // Answer-mix rates (% of all results) so refusals/small talk are visible
+    // instead of silently inflating/deflating the scores above.
+    refusalRate: { type: Number },
+    conversationalRate: { type: Number },
+    // How many results actually contributed a score to each metric (the N
+    // behind each average — tells you how trustworthy each number is).
+    scoredCounts: {
+      correctness: { type: Number },
+      relevance: { type: Number },
+      groundedness: { type: Number },
+      retrievalRelevance: { type: Number },
+    },
+    // Set when a reviewer human-scores the run from the run view. humanPassRate
+    // is null (not 0) until at least one row is scored. humanScoredCount = how
+    // many result rows have a human score (drives the "unfinished n/N" marker).
+    humanPassRate: { type: Number },
+    humanScoredCount: { type: Number },
+    totalScore: { type: Number },
+    totalQuestions: { type: Number },
+    totalResults: { type: Number },
+  },
+  { timestamps: true, versionKey: false, collection: "graph_runs" }
+);
+
+module.exports =
+  mongoose.models.GraphRun || mongoose.model("GraphRun", GraphRunSchema);

package/models/JobStatus.js

@@ -8,7 +8,7 @@ const JobStatusSchema = new mongoose.Schema(
 		programId: { type: mongoose.Schema.Types.ObjectId, ref: "Program" },
 		jobType: {
 			type: String,
-			enum: ["run", "ingestion"],
+			enum: ["run", "ingestion", "graph-run"],
 		},
 		status: {
 			type: String,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ben-ai-models-registry",
-  "version": "1.0.49",
+  "version": "1.0.50",
   "description": "",
   "files": [
     "/models"