ben-ai-models-registry 1.0.51 → 1.0.52

package/index.js

@@ -18,6 +18,12 @@ const State = require("./models/State");
 const StatePermission = require("./models/StatePermission");
 const UserChatHistory = require("./models/UserChatHistory");
 const ScraperConfig = require("./models/ScraperConfig");
+const {
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
+} = require("./models/utils/sharedSchemas/chunkingStrategies");
 
 module.exports = {
   JobStatus,
@@ -39,4 +45,8 @@ module.exports = {
   FileStore,
   UserChatHistory,
 	ScraperConfig,
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
 };

package/models/utils/sharedSchemas/ChunkingOptionsSchema.js

@@ -1,10 +1,14 @@
 const mongoose = require("mongoose");
+const {
+  CHUNKING_STRATEGY_VALUES,
+  SIMILARITY_THRESHOLD,
+} = require("./chunkingStrategies");
 
 const ChunkingOptionsSchema = new mongoose.Schema(
   {
     chunkingStrategy: {
       type: String,
-      enum: ["basic", "by_title"],
+      enum: [...CHUNKING_STRATEGY_VALUES],
     },
     maxCharacters: {
       type: Number,
@@ -18,6 +22,11 @@ const ChunkingOptionsSchema = new mongoose.Schema(
     overlapAll: {
       type: Boolean,
     },
+    similarityThreshold: {
+      type: Number,
+      min: SIMILARITY_THRESHOLD.min,
+      max: SIMILARITY_THRESHOLD.max,
+    },
   },
   { _id: false }
 );

package/models/utils/sharedSchemas/chunkingStrategies.js

@@ -0,0 +1,51 @@
+const CHUNKING_OPTION_FIELDS = Object.freeze([
+  "chunkingStrategy",
+  "maxCharacters",
+  "newAfterNChars",
+  "overlap",
+  "overlapAll",
+  "similarityThreshold",
+]);
+
+const COMMON_OPTIONS = [
+  "maxCharacters",
+  "newAfterNChars",
+  "overlap",
+  "overlapAll",
+];
+
+const CHUNKING_STRATEGIES = Object.freeze({
+  basic: Object.freeze({
+    displayName: "Basic",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_title: Object.freeze({
+    displayName: "By Title",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_page: Object.freeze({
+    displayName: "By Page",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS]),
+  }),
+  by_similarity: Object.freeze({
+    displayName: "By Similarity",
+    supportedOptions: Object.freeze([...COMMON_OPTIONS, "similarityThreshold"]),
+  }),
+});
+
+const CHUNKING_STRATEGY_VALUES = Object.freeze(
+  Object.keys(CHUNKING_STRATEGIES)
+);
+
+const SIMILARITY_THRESHOLD = Object.freeze({
+  min: 0.01,
+  max: 0.99,
+  default: 0.5,
+});
+
+module.exports = {
+  CHUNKING_STRATEGIES,
+  CHUNKING_STRATEGY_VALUES,
+  CHUNKING_OPTION_FIELDS,
+  SIMILARITY_THRESHOLD,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ben-ai-models-registry",
-  "version": "1.0.51",
+  "version": "1.0.52",
   "description": "",
   "files": [
     "/models"