ben-ai-models-registry 1.0.44 → 1.0.46

package/index.js

@@ -13,6 +13,7 @@ const FileStore = require("./models/FileStore");
 const State = require("./models/State");
 const StatePermission = require("./models/StatePermission");
 const UserChatHistory = require("./models/UserChatHistory");
+const ScraperConfig = require("./models/ScraperConfig");
 
 module.exports = {
   JobStatus,
@@ -29,4 +30,5 @@ module.exports = {
   EventLog,
   FileStore,
   UserChatHistory,
+	ScraperConfig,
 };

package/models/ChatMessages.js

@@ -60,6 +60,10 @@ const chatMessages = new Schema(
       type: [CitationSchema],
       required: false,
     },
+    traceId: {
+      type: String,
+      required: false,
+    },
   },
   { _id: false }
 );

package/models/FileStore.js

@@ -3,6 +3,22 @@ const ChunkingOptionsSchema = require("./utils/sharedSchemas/ChunkingOptionsSche
 
 const { Schema, Types } = mongoose;
 
+const FileStatusSchema = new Schema(
+  {
+    fileId: {
+      type: Schema.Types.ObjectId,
+      ref: "Files",
+      required: true,
+    },
+    ingestionStatus: {
+      type: String,
+      enum: ["not ingested", "in progress", "ingested", "skipped", "failed"],
+      default: "not ingested",
+    },
+  },
+  { _id: false }
+);
+
 const FileStoreSchema = new Schema(
   {
     stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
@@ -14,6 +30,7 @@ const FileStoreSchema = new Schema(
       required: true,
       unique: true,
     },
+    filesWithStatus: [FileStatusSchema],
     files: [{ type: Schema.Types.ObjectId, ref: "Files" }],
     openAI: {
       assistantId: {

package/models/Files.js

@@ -1,9 +1,17 @@
 const mongoose = require("mongoose");
 const { Schema } = mongoose;
 
+const ScraperMetadataSchema = new Schema({
+  scraperId: { type: Schema.Types.ObjectId, ref: "ScraperConfig" },
+  contentLength: { type: Number },
+  scrapedAt: { type: Date },
+  sourceUrl: { type: String }
+}, { _id: false });
+
 const FileSchema = new Schema(
   {
     stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
+    programId: { type: Schema.Types.ObjectId, ref: "Program", required: true },
     displayName: {
       type: String,
       required: [true, "Please provide a display name."],
@@ -23,6 +31,11 @@ const FileSchema = new Schema(
     openAIFileId: {
       type: String,
     },
+    scraperMetadata: {
+      type: ScraperMetadataSchema,
+      required: false 
+    },
+    source: { type: String, default: "scraper", required: false},
   },
   { timestamps: true, versionKey: false }
 );

package/models/ScraperConfig.js

@@ -0,0 +1,7 @@
+const mongoose = require("mongoose");
+const { Schema } = mongoose;
+
+const ScraperConfigSchema = new Schema({}, { strict: false, versionKey: false });
+
+module.exports = 
+    mongoose.models.ScraperConfig || mongoose.model("ScraperConfig", ScraperConfigSchema);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ben-ai-models-registry",
-  "version": "1.0.44",
+  "version": "1.0.46",
   "description": "",
   "files": [
     "/models"