ben-ai-models-registry 1.0.32 → 1.0.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/.editorconfig +5 -0
  2. package/models/ChatMessages.js +61 -34
  3. package/models/Dataset.js +27 -26
  4. package/models/Files.js +28 -29
  5. package/models/LanguageObject.js +8 -8
  6. package/models/QA.js +53 -53
  7. package/models/Run.js +89 -87
  8. package/models/User.js +9 -9
  9. package/package.json +13 -13
package/.editorconfig ADDED
@@ -0,0 +1,5 @@
1
+ root = true
2
+
3
+ [*]
4
+ indent_style = space
5
+ indent_size = 2
package/models/ChatMessages.js CHANGED
@@ -1,44 +1,71 @@
1
1
  const mongoose = require("mongoose");
2
2
  const { Schema, Types } = mongoose;
3
3
 
4
+ const CitationSchema = new mongoose.Schema({
5
+ id: {
6
+ type: String,
7
+ required: true,
8
+ },
9
+ file: {
10
+ type: String,
11
+ required: true,
12
+ },
13
+ pageNumber: {
14
+ type: Number,
15
+ required: true,
16
+ },
17
+ text: {
18
+ type: String,
19
+ required: true,
20
+ },
21
+ url: {
22
+ type: String,
23
+ required: false,
24
+ },
25
+ });
26
+
4
27
  const chatMessages = new Schema(
5
- {
6
- role: {
7
- type: String,
8
- required: true,
9
- },
10
- content: {
11
- type: String,
12
- required: true,
13
- },
14
- },
15
- { _id: false }
28
+ {
29
+ role: {
30
+ type: String,
31
+ required: true,
32
+ },
33
+ content: {
34
+ type: String,
35
+ required: true,
36
+ },
37
+ citations: {
38
+ type: [CitationSchema],
39
+ required: false,
40
+ },
41
+ },
42
+ { _id: false }
16
43
  );
17
44
 
18
45
  const ChatMessageSchema = new mongoose.Schema(
19
- {
20
- assistantType: {
21
- type: String,
22
- enum: {
23
- values: ["openai", "langchain"],
24
- },
25
- },
26
- programID: {
27
- type: Types.ObjectId,
28
- ref: "Program",
29
- },
30
- userId: {
31
- type: Types.ObjectId,
32
- ref: "User",
33
- },
34
- chatMessages: [chatMessages],
35
- },
36
- {
37
- timestamps: true,
38
- versionKey: false,
39
- }
46
+ {
47
+ assistantType: {
48
+ type: String,
49
+ enum: {
50
+ values: ["openai", "langchain"],
51
+ },
52
+ },
53
+ programID: {
54
+ type: Types.ObjectId,
55
+ ref: "Program",
56
+ },
57
+ userId: {
58
+ type: Types.ObjectId,
59
+ ref: "User",
60
+ },
61
+ chatMessages: [chatMessages],
62
+ },
63
+ {
64
+ timestamps: true,
65
+ versionKey: false,
66
+ }
40
67
  );
41
68
 
42
69
  module.exports =
43
- mongoose.models.ChatMessages ||
44
- mongoose.model("ChatMessages", ChatMessageSchema);
70
+ mongoose.models.ChatMessages ||
71
+ mongoose.model("ChatMessages", ChatMessageSchema);
package/models/Dataset.js CHANGED
@@ -1,34 +1,35 @@
1
- const mongoose = require('mongoose');
1
+ const mongoose = require("mongoose");
2
2
  const { Schema, Types } = mongoose;
3
3
 
4
4
  const QAPairSchema = new Schema(
5
- {
6
- question: { type: String },
7
- expectedAnswer: { type: String },
8
- },
9
- { timestamps: true, versionKey: false }
5
+ {
6
+ question: { type: String },
7
+ expectedAnswer: { type: String },
8
+ },
9
+ { timestamps: true, versionKey: false }
10
10
  );
11
11
 
12
12
  const DatasetSchema = new Schema(
13
- {
14
- programID: {
15
- type: Types.ObjectId,
16
- },
17
- name: {
18
- type: String,
19
- },
20
- description: {
21
- type: String,
22
- },
23
- qaPairs: [QAPairSchema],
24
- runs: [
25
- {
26
- type: Types.ObjectId,
27
- ref: "Run",
28
- },
29
- ],
30
- },
31
- { timestamps: true, versionKey: false }
13
+ {
14
+ programID: {
15
+ type: Types.ObjectId,
16
+ },
17
+ name: {
18
+ type: String,
19
+ },
20
+ description: {
21
+ type: String,
22
+ },
23
+ qaPairs: [QAPairSchema],
24
+ runs: [
25
+ {
26
+ type: Types.ObjectId,
27
+ ref: "Run",
28
+ },
29
+ ],
30
+ },
31
+ { timestamps: true, versionKey: false }
32
32
  );
33
33
 
34
- module.exports = mongoose.models.Dataset || mongoose.model("Dataset", DatasetSchema);
34
+ module.exports =
35
+ mongoose.models.Dataset || mongoose.model("Dataset", DatasetSchema);
package/models/Files.js CHANGED
@@ -2,35 +2,34 @@ const mongoose = require("mongoose");
2
2
  const { Schema } = mongoose;
3
3
 
4
4
  const FileSchema = new Schema(
5
- {
6
- displayName: {
7
- type: String,
8
- required: [true, "Please provide a display name."],
9
- },
10
- refName: {
11
- type: String,
12
- },
13
- description: {
14
- type: String,
15
- },
16
- type: {
17
- type: String,
18
- },
19
- url: {
20
- type: String,
21
- },
22
- openAIFileID: {
23
- type: String,
24
- },
25
- programCount: { type: Number },
26
- ingestedByProgramId: {
27
- type: [Schema.Types.ObjectId], // Array of program IDs that have ingested the file
28
- ref: "Program",
29
- default: [],
30
- },
31
-
32
- },
33
- { timestamps: true, versionKey: false }
5
+ {
6
+ displayName: {
7
+ type: String,
8
+ required: [true, "Please provide a display name."],
9
+ },
10
+ refName: {
11
+ type: String,
12
+ },
13
+ description: {
14
+ type: String,
15
+ },
16
+ type: {
17
+ type: String,
18
+ },
19
+ url: {
20
+ type: String,
21
+ },
22
+ openAIFileID: {
23
+ type: String,
24
+ },
25
+ programCount: { type: Number },
26
+ ingestedByProgramId: {
27
+ type: [Schema.Types.ObjectId], // Array of program IDs that have ingested the file
28
+ ref: "Program",
29
+ default: [],
30
+ },
31
+ },
32
+ { timestamps: true, versionKey: false }
34
33
  );
35
34
 
36
35
  module.exports = mongoose.models.Files || mongoose.model("Files", FileSchema);
package/models/LanguageObject.js CHANGED
@@ -1,14 +1,14 @@
1
1
  const mongoose = require("mongoose");
2
2
 
3
3
  const LanguageObjectSchema = new mongoose.Schema(
4
- {},
5
- {
6
- timestamps: true,
7
- versionKey: false,
8
- strict: false
9
- }
4
+ {},
5
+ {
6
+ timestamps: true,
7
+ versionKey: false,
8
+ strict: false,
9
+ }
10
10
  );
11
11
 
12
12
  module.exports =
13
- mongoose.models.LanguageObject ||
14
- mongoose.model("LanguageObject", LanguageObjectSchema);
13
+ mongoose.models.LanguageObject ||
14
+ mongoose.model("LanguageObject", LanguageObjectSchema);
package/models/QA.js CHANGED
@@ -2,59 +2,59 @@ const mongoose = require("mongoose");
2
2
  const { Schema, Types } = mongoose;
3
3
 
4
4
  const QASchema = new Schema(
5
- {
6
- programID: {
7
- type: Types.ObjectId,
8
- ref: "Program",
9
- },
10
- question: {
11
- type: String,
12
- },
13
- answer: {
14
- type: String,
15
- },
16
- assistantModel: {
17
- type: String,
18
- enum: ["langchain-assistant", "openAI-assistant"],
19
- },
20
- instruction: {
21
- type: String,
22
- },
23
- modelName: {
24
- type: String,
25
- enum: [
26
- "gpt-4o",
27
- "gpt-4o-mini",
28
- "gpt-4-turbo-preview",
29
- "gpt-4-turbo",
30
- "gpt-4-turbo-2024-04-09",
31
- "gpt-4-0125-preview",
32
- "gpt-4-1106-preview",
33
- "gpt-4",
34
- "gpt-4-0613",
35
- "gpt-4-32k",
36
- "gpt-4-32k-0613",
37
- "gpt-3.5-turbo-0125",
38
- "gpt-3.5-turbo",
39
- "gpt-3.5-turbo-1106",
40
- "gpt-3.5-turbo-instruct",
41
- ],
42
- },
43
- score: {
44
- type: Number,
45
- enum: [0, 1, 2, 3, 4, 5],
46
- },
47
- messageId: {
48
- type: String,
49
- },
50
- batchId: {
51
- type: String,
52
- },
53
- },
54
- {
55
- timestamps: true,
56
- versionKey: false,
57
- }
5
+ {
6
+ programID: {
7
+ type: Types.ObjectId,
8
+ ref: "Program",
9
+ },
10
+ question: {
11
+ type: String,
12
+ },
13
+ answer: {
14
+ type: String,
15
+ },
16
+ assistantModel: {
17
+ type: String,
18
+ enum: ["langchain-assistant", "openAI-assistant"],
19
+ },
20
+ instruction: {
21
+ type: String,
22
+ },
23
+ modelName: {
24
+ type: String,
25
+ enum: [
26
+ "gpt-4o",
27
+ "gpt-4o-mini",
28
+ "gpt-4-turbo-preview",
29
+ "gpt-4-turbo",
30
+ "gpt-4-turbo-2024-04-09",
31
+ "gpt-4-0125-preview",
32
+ "gpt-4-1106-preview",
33
+ "gpt-4",
34
+ "gpt-4-0613",
35
+ "gpt-4-32k",
36
+ "gpt-4-32k-0613",
37
+ "gpt-3.5-turbo-0125",
38
+ "gpt-3.5-turbo",
39
+ "gpt-3.5-turbo-1106",
40
+ "gpt-3.5-turbo-instruct",
41
+ ],
42
+ },
43
+ score: {
44
+ type: Number,
45
+ enum: [0, 1, 2, 3, 4, 5],
46
+ },
47
+ messageId: {
48
+ type: String,
49
+ },
50
+ batchId: {
51
+ type: String,
52
+ },
53
+ },
54
+ {
55
+ timestamps: true,
56
+ versionKey: false,
57
+ }
58
58
  );
59
59
 
60
60
  module.exports = mongoose.models.QA || mongoose.model("QA", QASchema);
package/models/Run.js CHANGED
@@ -2,114 +2,116 @@ const mongoose = require("mongoose");
2
2
  const { Schema } = mongoose;
3
3
 
4
4
  const AssistantModel = {
5
- Langchain: "langchain-assistant",
6
- OpenAI: "openAI-assistant",
5
+ Langchain: "langchain-assistant",
6
+ OpenAI: "openAI-assistant",
7
7
  };
8
8
 
9
9
  const ModelName = {
10
- GPT4O: "gpt-4o",
11
- GPT4OMini: "gpt-4o-mini",
12
- GPT4TurboPreview: "gpt-4-turbo-preview",
13
- GPT4Turbo: "gpt-4-turbo",
14
- GPT4Turbo20240409: "gpt-4-turbo-2024-04-09",
15
- GPT40125Preview: "gpt-4-0125-preview",
16
- GPT41106Preview: "gpt-4-1106-preview",
17
- GPT4: "gpt-4",
18
- GPT40613: "gpt-4-0613",
19
- GPT432K: "gpt-4-32k",
20
- GPT432K0613: "gpt-4-32k-0613",
21
- GPT3_5Turbo0125: "gpt-3.5-turbo-0125",
22
- GPT3_5Turbo: "gpt-3.5-turbo",
23
- GPT3_5Turbo1106: "gpt-3.5-turbo-1106",
24
- GPT3_5TurboInstruct: "gpt-3.5-turbo-instruct",
10
+ GPT4O: "gpt-4o",
11
+ GPT4OMini: "gpt-4o-mini",
12
+ GPT4TurboPreview: "gpt-4-turbo-preview",
13
+ GPT4Turbo: "gpt-4-turbo",
14
+ GPT4Turbo20240409: "gpt-4-turbo-2024-04-09",
15
+ GPT40125Preview: "gpt-4-0125-preview",
16
+ GPT41106Preview: "gpt-4-1106-preview",
17
+ GPT4: "gpt-4",
18
+ GPT40613: "gpt-4-0613",
19
+ GPT432K: "gpt-4-32k",
20
+ GPT432K0613: "gpt-4-32k-0613",
21
+ GPT3_5Turbo0125: "gpt-3.5-turbo-0125",
22
+ GPT3_5Turbo: "gpt-3.5-turbo",
23
+ GPT3_5Turbo1106: "gpt-3.5-turbo-1106",
24
+ GPT3_5TurboInstruct: "gpt-3.5-turbo-instruct",
25
25
  };
26
26
 
27
27
  const RunType = {
28
- SINGLE_LOOP: "single-loop",
29
- PAIR_WISE_COMPARE: "pair-wise-compare",
28
+ SINGLE_LOOP: "single-loop",
29
+ PAIR_WISE_COMPARE: "pair-wise-compare",
30
30
  };
31
31
 
32
32
  const AnswerSchema = new Schema(
33
- {
34
- prediction: { type: String },
35
- sources: [
36
- {
37
- filename: { type: String },
38
- url: { type: String },
39
- },
40
- ],
41
- context: {
42
- type: [Schema.Types.Mixed], // Allows any data type in the array
43
- default: [],
44
- },
45
- assistantModel: {
46
- type: String,
47
- enum: Object.values(AssistantModel),
48
- required: true,
49
- },
50
- },
51
- { versionKey: false }
33
+ {
34
+ prediction: { type: String },
35
+ sources: [
36
+ {
37
+ filename: { type: String },
38
+ url: { type: String },
39
+ // TODO: Verify citations Schema
40
+ citations: [{ type: Schema.Types.Mixed }],
41
+ },
42
+ ],
43
+ context: {
44
+ type: [Schema.Types.Mixed], // Allows any data type in the array
45
+ default: [],
46
+ },
47
+ assistantModel: {
48
+ type: String,
49
+ enum: Object.values(AssistantModel),
50
+ required: true,
51
+ },
52
+ },
53
+ { versionKey: false }
52
54
  );
53
55
 
54
56
  const EvaluationQuestionSchema = new Schema(
55
- {
56
- question: { type: String },
57
- score: { type: String },
58
- reasoning: { type: String },
59
- },
60
- { versionKey: false }
57
+ {
58
+ question: { type: String },
59
+ score: { type: String },
60
+ reasoning: { type: String },
61
+ },
62
+ { versionKey: false }
61
63
  );
62
64
 
63
65
  const EvaluationSchema = new Schema(
64
- {
65
- criterion: { type: String },
66
- questions: [EvaluationQuestionSchema],
67
- reasoning: { type: String },
68
- overallScore: { type: String },
69
- },
70
- { timestamps: true, versionKey: false }
66
+ {
67
+ criterion: { type: String },
68
+ questions: [EvaluationQuestionSchema],
69
+ reasoning: { type: String },
70
+ overallScore: { type: String },
71
+ },
72
+ { timestamps: true, versionKey: false }
71
73
  );
72
74
 
73
75
  const EvalSchema = new Schema(
74
- {
75
- evalData: [
76
- {
77
- answers: { type: [AnswerSchema] },
78
- llmEvaluation: { type: EvaluationSchema },
79
- humanEvaluation: { type: [EvaluationSchema] },
80
- humanOverallScore: { type: String },
81
- },
82
- ],
83
- question: { type: String },
84
- expectedAnswer: { type: String },
85
- },
86
- { versionKey: false }
76
+ {
77
+ evalData: [
78
+ {
79
+ answers: { type: [AnswerSchema] },
80
+ llmEvaluation: { type: EvaluationSchema },
81
+ humanEvaluation: { type: [EvaluationSchema] },
82
+ humanOverallScore: { type: String },
83
+ },
84
+ ],
85
+ question: { type: String },
86
+ expectedAnswer: { type: String },
87
+ },
88
+ { versionKey: false }
87
89
  );
88
90
 
89
91
  const RunSchema = new mongoose.Schema(
90
- {
91
- name: { type: String },
92
- description: { type: String },
93
- type: {
94
- type: String,
95
- enum: Object.values(RunType),
96
- },
97
- instruction: { type: String },
98
- modelName: {
99
- type: String,
100
- enum: Object.values(ModelName),
101
- },
102
- chunkingOptions: {
103
- // TODO: import ChunkingOptionsSchema from Program model
104
- // type: [ChunkingOptionsSchema],
105
- type: Schema.Types.Mixed,
106
- },
107
- evalResults: { type: [EvalSchema], required: true },
108
- llmPassRate: { type: Schema.Types.Mixed },
109
- humanPassRate: { type: Schema.Types.Mixed },
110
- totalScore: { type: Schema.Types.Mixed },
111
- },
112
- { timestamps: true, versionKey: false }
92
+ {
93
+ name: { type: String },
94
+ description: { type: String },
95
+ type: {
96
+ type: String,
97
+ enum: Object.values(RunType),
98
+ },
99
+ instruction: { type: String },
100
+ modelName: {
101
+ type: String,
102
+ enum: Object.values(ModelName),
103
+ },
104
+ chunkingOptions: {
105
+ // TODO: import ChunkingOptionsSchema from Program model
106
+ // type: [ChunkingOptionsSchema],
107
+ type: Schema.Types.Mixed,
108
+ },
109
+ evalResults: { type: [EvalSchema], required: true },
110
+ llmPassRate: { type: Schema.Types.Mixed },
111
+ humanPassRate: { type: Schema.Types.Mixed },
112
+ totalScore: { type: Schema.Types.Mixed },
113
+ },
114
+ { timestamps: true, versionKey: false }
113
115
  );
114
116
 
115
117
  module.exports = mongoose.models.Run || mongoose.model("Run", RunSchema);
package/models/User.js CHANGED
@@ -2,15 +2,15 @@ const mongoose = require("mongoose");
2
2
  const { Schema } = mongoose;
3
3
 
4
4
  const UserSchema = new Schema(
5
- {
6
- email: { type: String, unique: true, required: true },
7
- refName: {
8
- type: String,
9
- },
10
- role: { type: String, enum: ["USER", "ADMIN"], default: "USER" },
11
- emailVerified: { type: Date },
12
- },
13
- { timestamps: true, versionKey: false }
5
+ {
6
+ email: { type: String, unique: true, required: true },
7
+ refName: {
8
+ type: String,
9
+ },
10
+ role: { type: String, enum: ["USER", "ADMIN"], default: "USER" },
11
+ emailVerified: { type: Date },
12
+ },
13
+ { timestamps: true, versionKey: false }
14
14
  );
15
15
 
16
16
  module.exports = mongoose.models.User || mongoose.model("User", UserSchema);
package/package.json CHANGED
@@ -1,15 +1,15 @@
1
1
  {
2
- "name": "ben-ai-models-registry",
3
- "version": "1.0.32",
4
- "description": "",
5
- "main": "index.js",
6
- "scripts": {
7
- "test": "echo \"Error: no test specified\" && exit 1"
8
- },
9
- "keywords": [],
10
- "author": "",
11
- "license": "ISC",
12
- "dependencies": {
13
- "mongoose": "^8.4.4"
14
- }
2
+ "name": "ben-ai-models-registry",
3
+ "version": "1.0.34",
4
+ "description": "",
5
+ "main": "index.js",
6
+ "scripts": {
7
+ "test": "echo \"Error: no test specified\" && exit 1"
8
+ },
9
+ "keywords": [],
10
+ "author": "",
11
+ "license": "ISC",
12
+ "dependencies": {
13
+ "mongoose": "^8.4.4"
14
+ }
15
15
  }