ben-ai-models-registry 1.0.46 → 1.0.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/models/ChatMessages.js +44 -46
  2. package/models/Dataset.js +25 -25
  3. package/models/FileStore.js +2 -12
  4. package/models/Files.js +12 -12
  5. package/models/QA.js +49 -53
  6. package/models/Run.js +17 -24
  7. package/package.json +1 -1
package/models/ChatMessages.js CHANGED
@@ -24,59 +24,57 @@ const CitationSchema = new mongoose.Schema({
24
24
  },
25
25
  });
26
26
 
27
- const chatMessages = new Schema(
28
- {
29
- role: {
30
- type: String,
31
- required: true,
32
- },
33
- content: {
34
- type: String,
35
- required: true,
36
- },
37
- feedback: {
38
- like: { type: Boolean, default: false },
39
- dislike: {
40
- value: { type: Boolean, default: false },
41
- reason: {
42
- type: String,
43
- enum: [
44
- "Not helpful",
45
- "Inaccurate",
46
- "Other",
47
- "Wrong Citation",
48
- "unspecified",
49
- ],
50
- default: null,
51
- },
52
- explanation: {
53
- type: String,
54
- maxlength: 200,
55
- default: null,
27
+ const chatMessages = new Schema({
28
+ role: {
29
+ type: String,
30
+ required: true,
31
+ },
32
+ content: {
33
+ type: String,
34
+ required: true,
35
+ },
36
+ feedback: {
37
+ like: { type: Boolean, default: false },
38
+ dislike: {
39
+ value: { type: Boolean, default: false },
40
+ reason: {
41
+ type: [String],
42
+ enum: [
43
+ "Not Helpful",
44
+ "Inaccurate",
45
+ "Other",
46
+ "Wrong Citation",
47
+ "Unspecified",
48
+ ],
49
+ default: [],
50
+ },
51
+ explanation: {
52
+ type: [String],
53
+ validate: {
54
+ validator: function (arr) {
55
+ return arr.every(
56
+ (str) => typeof str === "string" && str.length <= 300,
57
+ );
58
+ },
59
+ message: "Each explanation must be a string of max 300 characters.",
56
60
  },
61
+ default: [],
57
62
  },
58
63
  },
59
- citations: {
60
- type: [CitationSchema],
61
- required: false,
62
- },
63
- traceId: {
64
- type: String,
65
- required: false,
66
- },
67
64
  },
68
- { _id: false }
69
- );
65
+ citations: {
66
+ type: [CitationSchema],
67
+ required: false,
68
+ },
69
+ traceId: {
70
+ type: String,
71
+ required: false,
72
+ },
73
+ });
70
74
 
71
75
  const ChatMessageSchema = new mongoose.Schema(
72
76
  {
73
77
  stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
74
- assistantType: {
75
- type: String,
76
- enum: {
77
- values: ["openai", "langchain"],
78
- },
79
- },
80
78
  programId: {
81
79
  type: Types.ObjectId,
82
80
  ref: "Program",
@@ -94,7 +92,7 @@ const ChatMessageSchema = new mongoose.Schema(
94
92
  {
95
93
  timestamps: true,
96
94
  versionKey: false,
97
- }
95
+ },
98
96
  );
99
97
 
100
98
  module.exports =
package/models/Dataset.js CHANGED
@@ -2,36 +2,36 @@ const mongoose = require("mongoose");
2
2
  const { Schema, Types } = mongoose;
3
3
 
4
4
  const QAPairSchema = new Schema(
5
- {
6
- question: { type: String },
7
- expectedAnswer: { type: String },
8
- },
9
- { timestamps: true, versionKey: false }
5
+ {
6
+ question: { type: String },
7
+ expectedAnswer: { type: String },
8
+ },
9
+ { timestamps: true, versionKey: false },
10
10
  );
11
11
 
12
12
  const DatasetSchema = new Schema(
13
- {
13
+ {
14
14
  stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
15
- programId: {
16
- type: Types.ObjectId,
15
+ programId: {
16
+ type: Types.ObjectId,
17
17
  ref: "Program",
18
- },
19
- name: {
20
- type: String,
21
- },
22
- description: {
23
- type: String,
24
- },
25
- qaPairs: [QAPairSchema],
26
- runs: [
27
- {
28
- type: Types.ObjectId,
29
- ref: "Run",
30
- },
31
- ],
32
- },
33
- { timestamps: true, versionKey: false }
18
+ },
19
+ name: {
20
+ type: String,
21
+ },
22
+ description: {
23
+ type: String,
24
+ },
25
+ qaPairs: [QAPairSchema],
26
+ runs: [
27
+ {
28
+ type: Types.ObjectId,
29
+ ref: "Run",
30
+ },
31
+ ],
32
+ },
33
+ { timestamps: true, versionKey: false },
34
34
  );
35
35
 
36
36
  module.exports =
37
- mongoose.models.Dataset || mongoose.model("Dataset", DatasetSchema);
37
+ mongoose.models.Dataset || mongoose.model("Dataset", DatasetSchema);
package/models/FileStore.js CHANGED
@@ -16,7 +16,7 @@ const FileStatusSchema = new Schema(
16
16
  default: "not ingested",
17
17
  },
18
18
  },
19
- { _id: false }
19
+ { _id: false },
20
20
  );
21
21
 
22
22
  const FileStoreSchema = new Schema(
@@ -32,16 +32,6 @@ const FileStoreSchema = new Schema(
32
32
  },
33
33
  filesWithStatus: [FileStatusSchema],
34
34
  files: [{ type: Schema.Types.ObjectId, ref: "Files" }],
35
- openAI: {
36
- assistantId: {
37
- type: String,
38
- },
39
- vectorStoreId: {
40
- type: String,
41
- },
42
- failedFilesOAI: [{ type: Schema.Types.ObjectId, ref: "Files" }],
43
- successfulFilesOAI: [{ type: Schema.Types.ObjectId, ref: "Files" }],
44
- },
45
35
  langchain: {
46
36
  ingestionStatus: {
47
37
  type: String,
@@ -74,7 +64,7 @@ const FileStoreSchema = new Schema(
74
64
  type: String,
75
65
  },
76
66
  },
77
- { timestamps: true, versionKey: false }
67
+ { timestamps: true, versionKey: false },
78
68
  );
79
69
 
80
70
  module.exports =
package/models/Files.js CHANGED
@@ -1,12 +1,15 @@
1
1
  const mongoose = require("mongoose");
2
2
  const { Schema } = mongoose;
3
3
 
4
- const ScraperMetadataSchema = new Schema({
5
- scraperId: { type: Schema.Types.ObjectId, ref: "ScraperConfig" },
6
- contentLength: { type: Number },
7
- scrapedAt: { type: Date },
8
- sourceUrl: { type: String }
9
- }, { _id: false });
4
+ const ScraperMetadataSchema = new Schema(
5
+ {
6
+ scraperId: { type: Schema.Types.ObjectId, ref: "ScraperConfig" },
7
+ contentLength: { type: Number },
8
+ scrapedAt: { type: Date },
9
+ sourceUrl: { type: String },
10
+ },
11
+ { _id: false },
12
+ );
10
13
 
11
14
  const FileSchema = new Schema(
12
15
  {
@@ -28,16 +31,13 @@ const FileSchema = new Schema(
28
31
  url: {
29
32
  type: String,
30
33
  },
31
- openAIFileId: {
32
- type: String,
33
- },
34
34
  scraperMetadata: {
35
35
  type: ScraperMetadataSchema,
36
- required: false
36
+ required: false,
37
37
  },
38
- source: { type: String, default: "scraper", required: false},
38
+ source: { type: String, default: "scraper", required: false },
39
39
  },
40
- { timestamps: true, versionKey: false }
40
+ { timestamps: true, versionKey: false },
41
41
  );
42
42
 
43
43
  module.exports = mongoose.models.Files || mongoose.model("Files", FileSchema);
package/models/QA.js CHANGED
@@ -2,60 +2,56 @@ const mongoose = require("mongoose");
2
2
  const { Schema, Types } = mongoose;
3
3
 
4
4
  const QASchema = new Schema(
5
- {
5
+ {
6
6
  stateId: { type: mongoose.Schema.Types.ObjectId, ref: "State" },
7
- programId: {
8
- type: Types.ObjectId,
9
- ref: "Program",
10
- },
11
- question: {
12
- type: String,
13
- },
14
- answer: {
15
- type: String,
16
- },
17
- assistantModel: {
18
- type: String,
19
- enum: ["langchain-assistant", "openAI-assistant"],
20
- },
21
- instruction: {
22
- type: String,
23
- },
24
- modelName: {
25
- type: String,
26
- enum: [
27
- "gpt-4o",
28
- "gpt-4o-mini",
29
- "gpt-4-turbo-preview",
30
- "gpt-4-turbo",
31
- "gpt-4-turbo-2024-04-09",
32
- "gpt-4-0125-preview",
33
- "gpt-4-1106-preview",
34
- "gpt-4",
35
- "gpt-4-0613",
36
- "gpt-4-32k",
37
- "gpt-4-32k-0613",
38
- "gpt-3.5-turbo-0125",
39
- "gpt-3.5-turbo",
40
- "gpt-3.5-turbo-1106",
41
- "gpt-3.5-turbo-instruct",
42
- ],
43
- },
44
- score: {
45
- type: Number,
46
- enum: [0, 1, 2, 3, 4, 5],
47
- },
48
- messageId: {
49
- type: String,
50
- },
51
- batchId: {
52
- type: String,
53
- },
54
- },
55
- {
56
- timestamps: true,
57
- versionKey: false,
58
- }
7
+ programId: {
8
+ type: Types.ObjectId,
9
+ ref: "Program",
10
+ },
11
+ question: {
12
+ type: String,
13
+ },
14
+ answer: {
15
+ type: String,
16
+ },
17
+ instruction: {
18
+ type: String,
19
+ },
20
+ modelName: {
21
+ type: String,
22
+ enum: [
23
+ "gpt-4o",
24
+ "gpt-4o-mini",
25
+ "gpt-4-turbo-preview",
26
+ "gpt-4-turbo",
27
+ "gpt-4-turbo-2024-04-09",
28
+ "gpt-4-0125-preview",
29
+ "gpt-4-1106-preview",
30
+ "gpt-4",
31
+ "gpt-4-0613",
32
+ "gpt-4-32k",
33
+ "gpt-4-32k-0613",
34
+ "gpt-3.5-turbo-0125",
35
+ "gpt-3.5-turbo",
36
+ "gpt-3.5-turbo-1106",
37
+ "gpt-3.5-turbo-instruct",
38
+ ],
39
+ },
40
+ score: {
41
+ type: Number,
42
+ enum: [0, 1, 2, 3, 4, 5],
43
+ },
44
+ messageId: {
45
+ type: String,
46
+ },
47
+ batchId: {
48
+ type: String,
49
+ },
50
+ },
51
+ {
52
+ timestamps: true,
53
+ versionKey: false,
54
+ },
59
55
  );
60
56
 
61
57
  module.exports = mongoose.models.QA || mongoose.model("QA", QASchema);
package/models/Run.js CHANGED
@@ -2,14 +2,8 @@ const mongoose = require("mongoose");
2
2
  const { Schema } = mongoose;
3
3
  const ChunkingOptionsSchema = require("./utils/sharedSchemas/ChunkingOptionsSchema");
4
4
 
5
- const AssistantModel = {
6
- Langchain: "langchain-assistant",
7
- OpenAI: "openAI-assistant",
8
- };
9
-
10
5
  const RunType = {
11
6
  SINGLE_LOOP: "single-loop",
12
- PAIR_WISE_COMPARE: "pair-wise-compare",
13
7
  BASELINE: "baseline",
14
8
  };
15
9
 
@@ -20,21 +14,20 @@ const AnswerSchema = new Schema(
20
14
  {
21
15
  filename: { type: String },
22
16
  url: { type: String },
23
- // TODO: Verify citations Schema
24
- citations: [{ type: Schema.Types.Mixed }],
17
+ citations: [
18
+ {
19
+ embeddingId: { type: Schema.Types.ObjectId, required: true },
20
+ pageContent: { type: String, required: true },
21
+ },
22
+ ],
25
23
  },
26
24
  ],
27
25
  context: {
28
26
  type: [Schema.Types.Mixed], // Allows any data type in the array
29
27
  default: [],
30
28
  },
31
- assistantModel: {
32
- type: String,
33
- enum: Object.values(AssistantModel),
34
- required: true,
35
- },
36
29
  },
37
- { versionKey: false }
30
+ { versionKey: false },
38
31
  );
39
32
 
40
33
  const EvaluationQuestionSchema = new Schema(
@@ -43,7 +36,7 @@ const EvaluationQuestionSchema = new Schema(
43
36
  score: { type: Number },
44
37
  reasoning: { type: String },
45
38
  },
46
- { versionKey: false }
39
+ { versionKey: false },
47
40
  );
48
41
 
49
42
  const EvaluationSchema = new Schema(
@@ -53,7 +46,7 @@ const EvaluationSchema = new Schema(
53
46
  reasoning: { type: String },
54
47
  overallScore: { type: Number },
55
48
  },
56
- { timestamps: true, versionKey: false }
49
+ { timestamps: true, versionKey: false },
57
50
  );
58
51
 
59
52
  const RagEvaluation = new Schema({
@@ -76,7 +69,7 @@ const EvalSchema = new Schema(
76
69
  question: { type: String },
77
70
  expectedAnswer: { type: String },
78
71
  },
79
- { versionKey: false }
72
+ { versionKey: false },
80
73
  );
81
74
 
82
75
  const RunSchema = new Schema(
@@ -104,15 +97,15 @@ const RunSchema = new Schema(
104
97
  type: String,
105
98
  },
106
99
  evalResults: { type: [EvalSchema], required: true },
107
- llmPassRate: { type: Schema.Types.Mixed },
108
- relevanceScore: { type: Schema.Types.Mixed },
109
- groundednessScore: { type: Schema.Types.Mixed },
110
- retrievalRelevanceScore: { type: Schema.Types.Mixed },
111
- humanPassRate: { type: Schema.Types.Mixed },
112
- totalScore: { type: Schema.Types.Mixed },
100
+ llmPassRate: { type: Number },
101
+ relevanceScore: { type: Number },
102
+ groundednessScore: { type: Number },
103
+ retrievalRelevanceScore: { type: Number },
104
+ humanPassRate: { type: Number },
105
+ totalScore: { type: Number },
113
106
  numberOfRetrievedDocs: { type: Number },
114
107
  },
115
- { timestamps: true, versionKey: false }
108
+ { timestamps: true, versionKey: false },
116
109
  );
117
110
 
118
111
  module.exports = mongoose.models.Run || mongoose.model("Run", RunSchema);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ben-ai-models-registry",
3
- "version": "1.0.46",
3
+ "version": "1.0.48",
4
4
  "description": "",
5
5
  "files": [
6
6
  "/models"