ben-ai-models-registry 1.0.50 → 1.0.52

package/index.js

@@ -3,6 +3,8 @@ const JobStatus = require("./models/JobStatus");
 const Run = require("./models/Run");
 const GraphRun = require("./models/GraphRun");
 const GraphEvalResult = require("./models/GraphEvalResult");
+const WikiRun = require("./models/WikiRun");
+const WikiEvalResult = require("./models/WikiEvalResult");
 const Files = require("./models/Files");
 const Dataset = require("./models/Dataset");
 const Program = require("./models/Program");
@@ -16,12 +18,20 @@ const State = require("./models/State");
 const StatePermission = require("./models/StatePermission");
 const UserChatHistory = require("./models/UserChatHistory");
 const ScraperConfig = require("./models/ScraperConfig");
+const {
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
+} = require("./models/utils/sharedSchemas/chunkingStrategies");
 
 module.exports = {
   JobStatus,
   Run,
   GraphRun,
   GraphEvalResult,
+  WikiRun,
+  WikiEvalResult,
   Files,
   Dataset,
   State,
@@ -35,4 +45,8 @@ module.exports = {
   FileStore,
   UserChatHistory,
 	ScraperConfig,
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
 };

package/models/JobStatus.js

@@ -2,35 +2,35 @@
 const mongoose = require("mongoose");
 
 const JobStatusSchema = new mongoose.Schema(
-	{
+  {
     stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
-		runId: { type: mongoose.Schema.Types.ObjectId, ref: "Run" },
-		programId: { type: mongoose.Schema.Types.ObjectId, ref: "Program" },
-		jobType: {
-			type: String,
-			enum: ["run", "ingestion", "graph-run"],
-		},
-		status: {
-			type: String,
-			enum: [
-				"pending",
-				"in_progress",
-				"completed",
-				"completed_with_errors",
-				"failed",
-			],
-			default: "pending",
-		},
-		progress: {
-			processed: { type: Number, default: 0 },
-			failed: { type: Number, default: 0 },
-			total: Number,
-		},
-		details: mongoose.Schema.Types.Mixed,
-		errorMessage: { type: String },
-	},
-	{ timestamps: true, versionKey: false }
+    runId: { type: mongoose.Schema.Types.ObjectId, ref: "Run" },
+    programId: { type: mongoose.Schema.Types.ObjectId, ref: "Program" },
+    jobType: {
+      type: String,
+      enum: ["run", "ingestion", "graph-run", "wiki-run"],
+    },
+    status: {
+      type: String,
+      enum: [
+        "pending",
+        "in_progress",
+        "completed",
+        "completed_with_errors",
+        "failed",
+      ],
+      default: "pending",
+    },
+    progress: {
+      processed: { type: Number, default: 0 },
+      failed: { type: Number, default: 0 },
+      total: Number,
+    },
+    details: mongoose.Schema.Types.Mixed,
+    errorMessage: { type: String },
+  },
+  { timestamps: true, versionKey: false },
 );
 
 module.exports =
-	mongoose.models.JobStatus || mongoose.model("JobStatus", JobStatusSchema);
+  mongoose.models.JobStatus || mongoose.model("JobStatus", JobStatusSchema);

package/models/WikiEvalResult.js

@@ -0,0 +1,99 @@
+const mongoose = require("mongoose");
+const { Schema } = mongoose;
+
+// One LAAJ judge verdict. Mirrors the shape produced by ragEvaluationModel.
+const JudgeScoreSchema = new Schema(
+  {
+    score: { type: Number },
+    reasoning: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const CitationSchema = new Schema(
+  {
+    id: { type: String },
+    text: { type: String },
+    file: { type: String },
+    pageNumber: { type: Number },
+    url: { type: String },
+    sectionTitle: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+// Human evaluation (mirrors GraphEvalResult's humanEvaluation/humanOverallScore).
+const EvaluationQuestionSchema = new Schema(
+  {
+    question: { type: String },
+    score: { type: Number },
+    reasoning: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const HumanEvaluationSchema = new Schema(
+  {
+    criterion: { type: String },
+    questions: [EvaluationQuestionSchema],
+    reasoning: { type: String },
+    overallScore: { type: Number },
+  },
+  { _id: false, versionKey: false }
+);
+
+const WikiEvalResultSchema = new Schema(
+  {
+    wikiRunId: {
+      type: Schema.Types.ObjectId,
+      ref: "WikiRun",
+      required: true,
+      index: true,
+    },
+    programId: { type: Schema.Types.ObjectId, ref: "Program" },
+    question: { type: String },
+    expectedAnswer: { type: String },
+    iteration: { type: Number },
+    // The finalAnswer variant from resolveCitations() — plain "(citation-{id})"
+    // markers, NOT displayAnswer's HTML <sup> markup. Judges must never see UI
+    // markup in their prediction input.
+    generatedAnswer: { type: String },
+    // Classification of generatedAnswer: Wiki's own real taxonomy (its
+    // AgentLoopStatusMessage values, plus "substantive" for a normal answer) —
+    // deliberately not remapped onto graph's substantive/refusal/conversational.
+    answerType: {
+      type: String,
+      enum: [
+        "substantive",
+        "refusal",
+        "insufficient_context",
+        "unrecognized_stop_reason",
+      ],
+    },
+    citations: { type: [CitationSchema], default: [] },
+    context: { type: [Schema.Types.Mixed], default: [] },
+    // The four RAG judges.
+    correctness: { type: JudgeScoreSchema },
+    relevance: { type: JudgeScoreSchema },
+    groundedness: { type: JudgeScoreSchema },
+    retrievalRelevance: { type: JudgeScoreSchema },
+    // Human evaluation (filled in from the run view; empty until a human scores it).
+    humanEvaluation: { type: [HumanEvaluationSchema], default: [] },
+    humanOverallScore: { type: String },
+    // Wiki-specific signal not present in the graph path.
+    chunksProcessed: { type: Number },
+    truncated: { type: Boolean },
+    budgetExhausted: { type: Boolean },
+    retryCount: { type: Number },
+    docsRetrieved: { type: Number },
+    latencyMs: { type: Number },
+    errorMessage: { type: String },
+    traceId: { type: String },
+    traceUrl: { type: String },
+  },
+  { timestamps: true, versionKey: false, collection: "wiki_eval_results" }
+);
+
+module.exports =
+  mongoose.models.WikiEvalResult ||
+  mongoose.model("WikiEvalResult", WikiEvalResultSchema);

package/models/WikiRun.js

@@ -0,0 +1,52 @@
+const mongoose = require("mongoose");
+const { Schema } = mongoose;
+
+const ParamsSnapshotSchema = new Schema(
+  {
+    searchMode: { type: String, enum: ["grep", "bm25"] },
+    grepMatchMode: { type: String, enum: ["word", "substring"] },
+    model: { type: String },
+    maxToolIterations: { type: Number },
+    maxInputTokens: { type: Number },
+    deadlineMs: { type: Number },
+    maxCostUsd: { type: Number },
+    instruction: { type: String },
+  },
+  { _id: false, versionKey: false }
+);
+
+const WikiRunSchema = new Schema(
+  {
+    stateId: { type: Schema.Types.ObjectId, ref: "State" },
+    datasetId: { type: Schema.Types.ObjectId, ref: "Dataset", index: true },
+    name: { type: String },
+    description: { type: String },
+    type: { type: String, default: "wiki" },
+    programId: { type: Schema.Types.ObjectId, ref: "Program" },
+    paramsSnapshot: { type: ParamsSnapshotSchema, default: {} },
+    evalModel: { type: String },
+    iterations: { type: Number },
+    llmPassRate: { type: Number },
+    relevanceScore: { type: Number },
+    groundednessScore: { type: Number },
+    retrievalRelevanceScore: { type: Number },
+    refusalRate: { type: Number },
+    insufficientContextRate: { type: Number },
+    unrecognizedStopReasonRate: { type: Number },
+    scoredCounts: {
+      correctness: { type: Number },
+      relevance: { type: Number },
+      groundedness: { type: Number },
+      retrievalRelevance: { type: Number },
+    },
+    humanPassRate: { type: Number },
+    humanScoredCount: { type: Number },
+    totalScore: { type: Number },
+    totalQuestions: { type: Number },
+    totalResults: { type: Number },
+  },
+  { timestamps: true, versionKey: false, collection: "wiki_runs" }
+);
+
+module.exports =
+  mongoose.models.WikiRun || mongoose.model("WikiRun", WikiRunSchema);

package/models/utils/sharedSchemas/ChunkingOptionsSchema.js

@@ -1,10 +1,14 @@
 const mongoose = require("mongoose");
+const {
+  CHUNKING_STRATEGY_VALUES,
+  SIMILARITY_THRESHOLD,
+} = require("./chunkingStrategies");
 
 const ChunkingOptionsSchema = new mongoose.Schema(
   {
     chunkingStrategy: {
       type: String,
-      enum: ["basic", "by_title"],
+      enum: [...CHUNKING_STRATEGY_VALUES],
     },
     maxCharacters: {
       type: Number,
@@ -18,6 +22,11 @@ const ChunkingOptionsSchema = new mongoose.Schema(
     overlapAll: {
       type: Boolean,
     },
+    similarityThreshold: {
+      type: Number,
+      min: SIMILARITY_THRESHOLD.min,
+      max: SIMILARITY_THRESHOLD.max,
+    },
   },
   { _id: false }
 );

package/models/utils/sharedSchemas/chunkingStrategies.js

@@ -0,0 +1,51 @@
+const CHUNKING_OPTION_FIELDS = Object.freeze([
+  "chunkingStrategy",
+  "maxCharacters",
+  "newAfterNChars",
+  "overlap",
+  "overlapAll",
+  "similarityThreshold",
+]);
+
+const COMMON_OPTIONS = [
+  "maxCharacters",
+  "newAfterNChars",
+  "overlap",
+  "overlapAll",
+];
+
+const CHUNKING_STRATEGIES = Object.freeze({
+  basic: Object.freeze({
+    displayName: "Basic",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_title: Object.freeze({
+    displayName: "By Title",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_page: Object.freeze({
+    displayName: "By Page",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_similarity: Object.freeze({
+    displayName: "By Similarity",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS, "similarityThreshold"]),
+  }),
+});
+
+const CHUNKING_STRATEGY_VALUES = Object.freeze(
+  Object.keys(CHUNKING_STRATEGIES)
+);
+
+const SIMILARITY_THRESHOLD = Object.freeze({
+  min: 0.01,
+  max: 0.99,
+  default: 0.5,
+});
+
+module.exports = {
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ben-ai-models-registry",
-  "version": "1.0.50",
+  "version": "1.0.52",
   "description": "",
   "files": [
     "/models"