@mongoosejs/studio 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/actions/ChatThread/streamChatMessage.js +12 -12
- package/backend/chatAgent/agentSystemPrompt.js +22 -10
- package/backend/chatAgent/getAgentTools.js +19 -8
- package/backend/chatAgent/runChatAgent.js +17 -0
- package/backend/index.js +3 -0
- package/debugging.js +11 -0
- package/eval/agent/cases.js +168 -0
- package/eval/agent/fixtures/carsharing.js +147 -0
- package/eval/agent/fixtures/ecommerce.js +152 -0
- package/eval/agent/run.js +323 -0
- package/frontend/index.js +1 -2
- package/frontend/public/app.js +395 -23
- package/frontend/public/tw.css +52 -0
- package/frontend/src/_util/recent-pages-history.js +116 -0
- package/frontend/src/chat/chat-message-script/chat-message-script.html +1 -6
- package/frontend/src/chat/chat.js +2 -2
- package/frontend/src/document/document.html +2 -2
- package/frontend/src/index.js +188 -10
- package/frontend/src/modal/modal.css +26 -1
- package/frontend/src/modal/modal.html +2 -2
- package/frontend/src/modal/modal.js +5 -1
- package/frontend/src/navbar/navbar.html +77 -6
- package/frontend/src/navbar/navbar.js +65 -4
- package/index.js +2 -1
- package/package.json +3 -2
|
@@ -4,8 +4,7 @@ const Archetype = require('archetype');
|
|
|
4
4
|
const assert = require('assert');
|
|
5
5
|
const authorize = require('../../authorize');
|
|
6
6
|
const callLLM = require('../../integrations/callLLM');
|
|
7
|
-
const
|
|
8
|
-
const getAgentTools = require('../../chatAgent/getAgentTools');
|
|
7
|
+
const runChatAgent = require('../../chatAgent/runChatAgent');
|
|
9
8
|
const streamLLM = require('../../integrations/streamLLM');
|
|
10
9
|
const getModelDescriptions = require('../../helpers/getModelDescriptions');
|
|
11
10
|
const mongoose = require('mongoose');
|
|
@@ -86,14 +85,6 @@ module.exports = ({ db, studioConnection, options }) => async function* streamCh
|
|
|
86
85
|
);
|
|
87
86
|
}
|
|
88
87
|
|
|
89
|
-
const modelDescriptions = getModelDescriptions(db);
|
|
90
|
-
const system = [
|
|
91
|
-
chatThread.agentMode ? agentSystemPrompt : systemPrompt,
|
|
92
|
-
currentDateTime ? `Current date: ${currentDateTime}` : null,
|
|
93
|
-
modelDescriptions,
|
|
94
|
-
options?.context
|
|
95
|
-
].filter(Boolean).join('\n\n');
|
|
96
|
-
|
|
97
88
|
const userChatMessage = await ChatMessage.create({
|
|
98
89
|
chatThreadId,
|
|
99
90
|
role: 'user',
|
|
@@ -111,10 +102,19 @@ module.exports = ({ db, studioConnection, options }) => async function* streamCh
|
|
|
111
102
|
script: null,
|
|
112
103
|
executionResult: null
|
|
113
104
|
});
|
|
114
|
-
const llmOptions = chatThread.agentMode ? { ...options, tools: getAgentTools(db) } : options;
|
|
115
105
|
let textStream;
|
|
116
106
|
try {
|
|
117
|
-
|
|
107
|
+
if (chatThread.agentMode) {
|
|
108
|
+
textStream = runChatAgent({ db, llmMessages, currentDateTime, options });
|
|
109
|
+
} else {
|
|
110
|
+
const system = [
|
|
111
|
+
systemPrompt,
|
|
112
|
+
currentDateTime ? `Current date: ${currentDateTime}` : null,
|
|
113
|
+
getModelDescriptions(db),
|
|
114
|
+
options?.context
|
|
115
|
+
].filter(Boolean).join('\n\n');
|
|
116
|
+
textStream = streamLLM(llmMessages, system, options);
|
|
117
|
+
}
|
|
118
118
|
} catch (err) {
|
|
119
119
|
yield { message: err?.message || 'Failed to stream response' };
|
|
120
120
|
return {};
|
|
@@ -5,24 +5,30 @@ You are a data querying assistant with access to tools that can query MongoDB di
|
|
|
5
5
|
|
|
6
6
|
Your tools are for EXPLORATION ONLY — use them to understand the data before writing a script. NEVER use tool results as the data source for the final script. The final script must be fully self-contained: it must query MongoDB itself and not depend on any data you retrieved via tools.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
Always follow this process for each query (do not skip steps):
|
|
9
9
|
|
|
10
10
|
1. **Identify models**: Based on the user's question and the model descriptions below, identify which models are relevant.
|
|
11
|
-
2. **Check document counts**: Use estimatedDocumentCount on
|
|
12
|
-
3. **Test assumptions**: Use find
|
|
13
|
-
4. **
|
|
14
|
-
5. **
|
|
11
|
+
2. **Check document counts**: Use estimatedDocumentCount on each relevant model to understand data volume and choose safe query patterns.
|
|
12
|
+
3. **Test assumptions with evidence**: Use find/findOne on each relevant model to verify field names, value shapes, and relationships. Treat every unverified field name, status value, or relationship as unknown until observed.
|
|
13
|
+
4. **Draft script**: Write a self-contained script that queries MongoDB directly. Access models via \`db.models.ModelName\` (for example \`db.models.User.findOne(...)\`). Do NOT use \`mongoose.model('Name')\` — schemas are registered on the \`db\` connection, not on the global \`mongoose\` instance — and do NOT use bare \`db.ModelName\` (the model lives under \`db.models\`).
|
|
14
|
+
5. **Type-check every draft**: Run typeCheck on the script before responding.
|
|
15
|
+
6. **Fix and re-check loop**: If typeCheck reports issues, fix them and run typeCheck again. Repeat until clean.
|
|
16
|
+
7. **Return final answer**: Provide one final script by default, or multiple scripts only if the user explicitly asks for multiple scripts, and include a brief description.
|
|
17
|
+
|
|
18
|
+
Hard requirements before finalizing:
|
|
19
|
+
- You MUST run at least one assumption-checking query (find/findOne) unless the user only asks for pure counts.
|
|
20
|
+
- You MUST run typeCheck immediately before finalizing any script.
|
|
21
|
+
- If you could not run an expected step (tool failure, timeout, missing model), explicitly say what failed and provide the safest possible fallback script.
|
|
22
|
+
- Do not claim a field exists unless you observed it in exploration.
|
|
15
23
|
|
|
16
24
|
Keep scripts concise. Avoid unnecessary comments, error handling, and temporary variables.
|
|
17
25
|
|
|
18
26
|
Do not write any imports or require() statements, that will cause the script to break.
|
|
19
27
|
|
|
20
|
-
If the user approves the script, the script will run in the Node.js server in a sandboxed vm.createContext() call with the following globals: db (the Mongoose connection), mongoose, ObjectId (mongoose.Types.ObjectId), console, and MongooseStudioChartColors (an array of 8 hex color strings for chart dataset colors). The script return value will then send the response via JSON to the client. Be aware that the result of the query will be serialized to JSON before being displayed to the user. MAKE SURE TO RETURN A VALUE FROM THE SCRIPT.
|
|
28
|
+
If the user approves the script, the script will run in the Node.js server in a sandboxed vm.createContext() call with the following globals: db (the Mongoose connection; access models via \`db.models.ModelName\`), mongoose, ObjectId (mongoose.Types.ObjectId), console, and MongooseStudioChartColors (an array of 8 hex color strings for chart dataset colors). The script return value will then send the response via JSON to the client. Be aware that the result of the query will be serialized to JSON before being displayed to the user. MAKE SURE TO RETURN A VALUE FROM THE SCRIPT.
|
|
21
29
|
|
|
22
30
|
Optimize scripts for readability first, followed by reliability, followed by performance. Avoid using the aggregation framework unless explicitly requested by the user. Use indexed fields in queries where possible.
|
|
23
31
|
|
|
24
|
-
Avoid using the aggregation framework unless explicitly asked by the user.
|
|
25
|
-
|
|
26
32
|
Assume the user has pre-defined schemas and models. Do not define any new schemas or models for the user.
|
|
27
33
|
|
|
28
34
|
Use async/await where possible. Assume top-level await is allowed.
|
|
@@ -33,11 +39,17 @@ Format output as Markdown, including code fences for any scripts the user reques
|
|
|
33
39
|
|
|
34
40
|
Add a brief text description of what the script does.
|
|
35
41
|
|
|
42
|
+
Before the final script, include a brief checklist with this exact structure:
|
|
43
|
+
- Models identified: ...
|
|
44
|
+
- Counts checked: yes/no
|
|
45
|
+
- Assumptions tested: yes/no
|
|
46
|
+
- Type-check passed: yes/no
|
|
47
|
+
|
|
36
48
|
If the user's query is best answered with a chart, return a Chart.js 4 configuration as \`return { $chart: chartJSConfig };\`. Disable ChartJS animation by default unless user asks for it. Set responsive: true, maintainAspectRatio: false options unless the user explicitly asks. Use MongooseStudioChartColors for dataset backgroundColor and borderColor by default. For line/bar charts, use MongooseStudioChartColors[i] as borderColor and MongooseStudioChartColors[i] + '33' as backgroundColor for each dataset. For pie/doughnut charts, use MongooseStudioChartColors.slice(0, data.length) as backgroundColor. Only use custom colors if the user explicitly requests specific colors.
|
|
37
49
|
|
|
38
|
-
If the user's query is best answered by a map, return an object { $featureCollection } which contains a GeoJSON FeatureCollection
|
|
50
|
+
If the user's query is best answered by a map, return an object { $featureCollection } which contains a GeoJSON FeatureCollection.
|
|
39
51
|
|
|
40
|
-
If the user's query is best answered by a table, return an object { $table: { columns: string[], rows: any[][] } }
|
|
52
|
+
If the user's query is best answered by a table, return an object { $table: { columns: string[], rows: any[][] } }.
|
|
41
53
|
|
|
42
54
|
-----------
|
|
43
55
|
|
|
@@ -5,6 +5,7 @@ const ts = require('typescript');
|
|
|
5
5
|
const { tool } = require('ai');
|
|
6
6
|
const { jsonSchema } = require('@ai-sdk/provider-utils');
|
|
7
7
|
const agentToolMetadata = require('./agentToolMetadata');
|
|
8
|
+
const debugging = require('../../debugging');
|
|
8
9
|
|
|
9
10
|
module.exports = function getAgentTools(db) {
|
|
10
11
|
const modelNames = Object.keys(db.models).filter(name => !name.startsWith('__Studio'));
|
|
@@ -21,7 +22,7 @@ module.exports = function getAgentTools(db) {
|
|
|
21
22
|
required: ['modelName']
|
|
22
23
|
}),
|
|
23
24
|
execute: async({ modelName }) => {
|
|
24
|
-
|
|
25
|
+
debugLog(`estimatedDocumentCount: modelName=${modelName}`);
|
|
25
26
|
const Model = db.models[modelName];
|
|
26
27
|
if (Model == null) {
|
|
27
28
|
return { error: `Model ${modelName} not found. Available models: ${modelNames.join(', ')}` };
|
|
@@ -42,7 +43,7 @@ module.exports = function getAgentTools(db) {
|
|
|
42
43
|
required: ['modelName']
|
|
43
44
|
}),
|
|
44
45
|
execute: async({ modelName, filter = {}, limit = 10 }) => {
|
|
45
|
-
|
|
46
|
+
debugLog(`find: modelName=${modelName}, filter=${JSON.stringify(filter)}, limit=${limit}`);
|
|
46
47
|
const Model = db.models[modelName];
|
|
47
48
|
if (Model == null) {
|
|
48
49
|
return { error: `Model ${modelName} not found. Available models: ${modelNames.join(', ')}` };
|
|
@@ -62,7 +63,7 @@ module.exports = function getAgentTools(db) {
|
|
|
62
63
|
required: ['modelName']
|
|
63
64
|
}),
|
|
64
65
|
execute: async({ modelName, filter = {} }) => {
|
|
65
|
-
|
|
66
|
+
debugLog(`findOne: modelName=${modelName}, filter=${JSON.stringify(filter)}`);
|
|
66
67
|
const Model = db.models[modelName];
|
|
67
68
|
if (Model == null) {
|
|
68
69
|
return { error: `Model ${modelName} not found. Available models: ${modelNames.join(', ')}` };
|
|
@@ -72,7 +73,7 @@ module.exports = function getAgentTools(db) {
|
|
|
72
73
|
}
|
|
73
74
|
}),
|
|
74
75
|
typeCheck: tool({
|
|
75
|
-
description: toolDescriptions.typeCheck + ' The script will run in a sandbox with globals: db (mongoose.Connection), mongoose, ObjectId (mongoose.Types.ObjectId), console, and MongooseStudioChartColors (string[]). Pass the raw script body (no imports, no wrapping function). Returns any TypeScript errors or JavaScript syntax errors found. Remember that you should write JavaScript, NOT TypeScript. This tool is just to check for obvious errors.',
|
|
76
|
+
description: toolDescriptions.typeCheck + ' The script will run in a sandbox with globals: db (mongoose.Connection — access models via db.models.ModelName), mongoose, ObjectId (mongoose.Types.ObjectId), console, and MongooseStudioChartColors (string[]). Pass the raw script body (no imports, no wrapping function). Returns any TypeScript errors or JavaScript syntax errors found. Remember that you should write JavaScript, NOT TypeScript. This tool is just to check for obvious errors.',
|
|
76
77
|
inputSchema: jsonSchema({
|
|
77
78
|
type: 'object',
|
|
78
79
|
properties: {
|
|
@@ -81,7 +82,8 @@ module.exports = function getAgentTools(db) {
|
|
|
81
82
|
required: ['script']
|
|
82
83
|
}),
|
|
83
84
|
execute: async({ script }) => {
|
|
84
|
-
|
|
85
|
+
debugLog(`typeCheck: script=${script}`);
|
|
86
|
+
const wrapped = wrapScriptForTypeCheck(script, modelNames);
|
|
85
87
|
const fileName = '__check.ts';
|
|
86
88
|
const compilerOptions = {
|
|
87
89
|
noEmit: true,
|
|
@@ -118,7 +120,7 @@ module.exports = function getAgentTools(db) {
|
|
|
118
120
|
return { ok: true };
|
|
119
121
|
}
|
|
120
122
|
|
|
121
|
-
|
|
123
|
+
debugLog('Errors', errors);
|
|
122
124
|
|
|
123
125
|
return { ok: false, errors };
|
|
124
126
|
}
|
|
@@ -126,10 +128,19 @@ module.exports = function getAgentTools(db) {
|
|
|
126
128
|
};
|
|
127
129
|
};
|
|
128
130
|
|
|
129
|
-
|
|
131
|
+
function debugLog() {
|
|
132
|
+
if (debugging.isDebuggingEnabled()) {
|
|
133
|
+
console.log(...arguments);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const wrapScriptForTypeCheck = (script, modelNames) => `
|
|
130
138
|
import mongoose from 'mongoose';
|
|
131
|
-
declare const db: Omit<mongoose.Connection, 'model'> & {
|
|
139
|
+
declare const db: Omit<mongoose.Connection, 'model' | 'models'> & {
|
|
132
140
|
model(name: string): mongoose.Model<any>;
|
|
141
|
+
models: {
|
|
142
|
+
${modelNames.map(name => ` ${JSON.stringify(name)}: mongoose.Model<any>;`).join('\n')}
|
|
143
|
+
};
|
|
133
144
|
};
|
|
134
145
|
declare const ObjectId: typeof mongoose.Types.ObjectId;
|
|
135
146
|
declare const console: Console;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const agentSystemPrompt = require('./agentSystemPrompt');
|
|
4
|
+
const getAgentTools = require('./getAgentTools');
|
|
5
|
+
const streamLLM = require('../integrations/streamLLM');
|
|
6
|
+
const getModelDescriptions = require('../helpers/getModelDescriptions');
|
|
7
|
+
|
|
8
|
+
module.exports = function runChatAgent({ db, llmMessages, currentDateTime, options }) {
|
|
9
|
+
const system = [
|
|
10
|
+
agentSystemPrompt,
|
|
11
|
+
currentDateTime ? `Current date: ${currentDateTime}` : null,
|
|
12
|
+
getModelDescriptions(db),
|
|
13
|
+
options?.context
|
|
14
|
+
].filter(Boolean).join('\n\n');
|
|
15
|
+
|
|
16
|
+
return streamLLM(llmMessages, system, { ...options, tools: getAgentTools(db) });
|
|
17
|
+
};
|
package/backend/index.js
CHANGED
|
@@ -11,6 +11,9 @@ const dashboardResultSchema = require('./db/dashboardResultSchema');
|
|
|
11
11
|
|
|
12
12
|
module.exports = function backend(db, studioConnection, options) {
|
|
13
13
|
db = db || mongoose.connection;
|
|
14
|
+
if (db instanceof mongoose.Mongoose) {
|
|
15
|
+
db = db.connection;
|
|
16
|
+
}
|
|
14
17
|
|
|
15
18
|
studioConnection = studioConnection ?? db;
|
|
16
19
|
const Dashboard = studioConnection.model('__Studio_Dashboard', dashboardSchema, 'studio__dashboards');
|
package/debugging.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const assert = require('node:assert/strict');
|
|
4
|
+
|
|
5
|
+
module.exports = [
|
|
6
|
+
{
|
|
7
|
+
id: 'simple-1-user-count',
|
|
8
|
+
category: 'simple',
|
|
9
|
+
fixture: 'ecommerce',
|
|
10
|
+
prompt: 'How many users do we have in total, including deleted ones?',
|
|
11
|
+
assert(trace) {
|
|
12
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
13
|
+
const got = typeof trace.output === 'number' ? trace.output : trace.output?.count;
|
|
14
|
+
assert.equal(got, 4, `expected 4 users, got ${JSON.stringify(trace.output)}`);
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
id: 'simple-2-most-expensive-product',
|
|
19
|
+
category: 'simple',
|
|
20
|
+
fixture: 'ecommerce',
|
|
21
|
+
prompt: 'What is the single most expensive product? Return an object with its name and price.',
|
|
22
|
+
assert(trace) {
|
|
23
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
24
|
+
assert.equal(trace.output?.name, 'Laptop', `expected name=Laptop, got ${JSON.stringify(trace.output)}`);
|
|
25
|
+
assert.equal(trace.output?.price, 1200, `expected price=1200, got ${JSON.stringify(trace.output)}`);
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
id: 'simple-3-active-bookings',
|
|
30
|
+
category: 'simple',
|
|
31
|
+
fixture: 'carsharing',
|
|
32
|
+
prompt: 'How many bookings currently have status "active"?',
|
|
33
|
+
assert(trace) {
|
|
34
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
35
|
+
const got = typeof trace.output === 'number' ? trace.output : trace.output?.count;
|
|
36
|
+
assert.equal(got, 2, `expected 2 active bookings, got ${JSON.stringify(trace.output)}`);
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'simple-4-list-evs',
|
|
41
|
+
category: 'simple',
|
|
42
|
+
fixture: 'carsharing',
|
|
43
|
+
prompt: 'List every electric vehicle in the fleet, returning an array with make and model.',
|
|
44
|
+
assert(trace) {
|
|
45
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
46
|
+
const list = Array.isArray(trace.output) ? trace.output : trace.output?.vehicles;
|
|
47
|
+
assert.equal(list?.length, 2, `expected 2 EVs, got ${JSON.stringify(trace.output)}`);
|
|
48
|
+
const json = JSON.stringify(list);
|
|
49
|
+
assert.ok(json.includes('Tesla') && json.includes('Leaf'), `expected Tesla and Leaf, got ${json}`);
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
id: 'simple-5-orders-by-status',
|
|
54
|
+
category: 'simple',
|
|
55
|
+
fixture: 'ecommerce',
|
|
56
|
+
prompt: 'Show a table with the count of orders grouped by status.',
|
|
57
|
+
assert(trace) {
|
|
58
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
59
|
+
assert.ok(trace.output?.$table, `expected $table, got ${JSON.stringify(trace.output)}`);
|
|
60
|
+
assert.equal(trace.output.$table.rows.length, 5, `expected 5 status rows, got ${trace.output.$table.rows.length}`);
|
|
61
|
+
const json = JSON.stringify(trace.output.$table);
|
|
62
|
+
for (const status of ['pending', 'paid', 'shipped', 'delivered', 'cancelled']) {
|
|
63
|
+
assert.ok(json.includes(status), `expected "${status}" in table, got ${json}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
|
|
68
|
+
{
|
|
69
|
+
id: 'nested-1-orders-by-month',
|
|
70
|
+
category: 'nested',
|
|
71
|
+
fixture: 'ecommerce',
|
|
72
|
+
prompt: 'How many orders were placed each month over the past 6 months? Return a table.',
|
|
73
|
+
assert(trace) {
|
|
74
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
75
|
+
assert.ok(trace.output?.$table, `expected $table, got ${JSON.stringify(trace.output)}`);
|
|
76
|
+
assert.ok(trace.output.$table.rows.length >= 3, `expected ≥3 month rows, got ${trace.output.$table.rows.length}`);
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: 'nested-2-late-returns-unpaid',
|
|
81
|
+
category: 'nested',
|
|
82
|
+
fixture: 'carsharing',
|
|
83
|
+
prompt: 'Return an array of bookings that were returned late (returnedAt after endAt) and have at least one unpaid charge.',
|
|
84
|
+
assert(trace) {
|
|
85
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
86
|
+
assert.ok(Array.isArray(trace.output), `expected array, got ${JSON.stringify(trace.output)}`);
|
|
87
|
+
assert.equal(trace.output.length, 2, `expected 2 late+unpaid bookings, got ${trace.output.length}`);
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
id: 'nested-3-paid-orders-with-electronics',
|
|
92
|
+
category: 'nested',
|
|
93
|
+
fixture: 'ecommerce',
|
|
94
|
+
prompt: 'Return an array of paid orders that contain at least one electronics product.',
|
|
95
|
+
assert(trace) {
|
|
96
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
97
|
+
assert.ok(Array.isArray(trace.output), `expected array, got ${JSON.stringify(trace.output)}`);
|
|
98
|
+
assert.equal(trace.output.length, 2, `expected 2 paid+electronics orders, got ${trace.output.length}`);
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
|
|
102
|
+
{
|
|
103
|
+
id: 'dashboard-1-top-products',
|
|
104
|
+
category: 'dashboard',
|
|
105
|
+
fixture: 'ecommerce',
|
|
106
|
+
prompt: 'Build a dashboard showing the top 5 products by gross revenue from paid orders.',
|
|
107
|
+
assert(trace) {
|
|
108
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
109
|
+
const renderable = trace.output?.$chart || trace.output?.$table;
|
|
110
|
+
assert.ok(renderable, `expected $chart or $table, got ${JSON.stringify(trace.output)}`);
|
|
111
|
+
assert.ok(JSON.stringify(renderable).includes('Headphones'), `Headphones should be top product, got ${JSON.stringify(renderable)}`);
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
id: 'dashboard-2-bookings-per-vehicle-type',
|
|
116
|
+
category: 'dashboard',
|
|
117
|
+
fixture: 'carsharing',
|
|
118
|
+
prompt: 'Show a chart of total bookings broken down by vehicle type.',
|
|
119
|
+
assert(trace) {
|
|
120
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
121
|
+
assert.ok(trace.output?.$chart, `expected $chart, got ${JSON.stringify(trace.output)}`);
|
|
122
|
+
const json = JSON.stringify(trace.output.$chart);
|
|
123
|
+
for (const type of ['ev', 'sedan', 'suv']) {
|
|
124
|
+
assert.ok(json.includes(type), `expected "${type}" in chart, got ${json}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
id: 'dashboard-3-revenue-by-month',
|
|
130
|
+
category: 'dashboard',
|
|
131
|
+
fixture: 'ecommerce',
|
|
132
|
+
prompt: 'Chart total revenue per month from paid orders. Include only months that contain at least one paid order, in chronological order, using "YYYY-MM" format for the labels.',
|
|
133
|
+
assert(trace) {
|
|
134
|
+
assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
|
|
135
|
+
assert.ok(trace.output?.$chart, `expected $chart, got ${JSON.stringify(trace.output)}`);
|
|
136
|
+
assert.deepEqual(trace.output.$chart.data.labels, ['2026-02', '2026-03', '2026-04']);
|
|
137
|
+
assert.equal(trace.output.$chart.data.datasets.length, 1, `expected 1 dataset, got ${JSON.stringify(trace.output.$chart.data.datasets)}`);
|
|
138
|
+
assert.deepEqual(trace.output.$chart.data.datasets[0].data, [300, 60, 222]);
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
|
|
142
|
+
{
|
|
143
|
+
id: 'ambig-1-missing-user',
|
|
144
|
+
category: 'ambiguous',
|
|
145
|
+
fixture: 'carsharing',
|
|
146
|
+
prompt: 'Why does Jane Doe have a negative balance?',
|
|
147
|
+
assert(trace) {
|
|
148
|
+
assert.ok(
|
|
149
|
+
trace.toolCalls.some(t => t.toolName === 'findOne' || t.toolName === 'find'),
|
|
150
|
+
'agent should look up the user before answering'
|
|
151
|
+
);
|
|
152
|
+
const lower = trace.text.toLowerCase();
|
|
153
|
+
assert.ok(
|
|
154
|
+
lower.includes('50') || lower.includes('positive') || lower.includes('not negative'),
|
|
155
|
+
`agent should note Jane's balance is actually positive; text was: ${trace.text.slice(0, 500)}`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: 'ambig-2-investigate',
|
|
161
|
+
category: 'ambiguous',
|
|
162
|
+
fixture: 'ecommerce',
|
|
163
|
+
prompt: 'Some orders look weird. Can you investigate?',
|
|
164
|
+
assert(trace) {
|
|
165
|
+
assert.ok(trace.toolCalls.length >= 2, `expected ≥2 tool calls; saw ${trace.toolCalls.length}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
];
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const mongoose = require('mongoose');
|
|
4
|
+
|
|
5
|
+
const { Schema } = mongoose;
|
|
6
|
+
|
|
7
|
+
const userSchema = new Schema({
|
|
8
|
+
name: String,
|
|
9
|
+
email: String,
|
|
10
|
+
balance: Number,
|
|
11
|
+
joinedAt: Date
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const vehicleSchema = new Schema({
|
|
15
|
+
make: String,
|
|
16
|
+
model: String,
|
|
17
|
+
type: { type: String, enum: ['sedan', 'suv', 'ev'] },
|
|
18
|
+
licensePlate: String,
|
|
19
|
+
dailyRate: Number
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const bookingSchema = new Schema({
|
|
23
|
+
userId: { type: Schema.Types.ObjectId, ref: 'User' },
|
|
24
|
+
vehicleId: { type: Schema.Types.ObjectId, ref: 'Vehicle' },
|
|
25
|
+
startAt: Date,
|
|
26
|
+
endAt: Date,
|
|
27
|
+
returnedAt: Date,
|
|
28
|
+
status: { type: String, enum: ['active', 'completed', 'cancelled'] }
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const chargeSchema = new Schema({
|
|
32
|
+
userId: { type: Schema.Types.ObjectId, ref: 'User' },
|
|
33
|
+
bookingId: { type: Schema.Types.ObjectId, ref: 'Booking' },
|
|
34
|
+
amount: Number,
|
|
35
|
+
paid: Boolean,
|
|
36
|
+
createdAt: Date
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
module.exports = {
|
|
40
|
+
name: 'carsharing',
|
|
41
|
+
defineModels(connection) {
|
|
42
|
+
return {
|
|
43
|
+
User: connection.model('User', userSchema),
|
|
44
|
+
Vehicle: connection.model('Vehicle', vehicleSchema),
|
|
45
|
+
Booking: connection.model('Booking', bookingSchema),
|
|
46
|
+
Charge: connection.model('Charge', chargeSchema)
|
|
47
|
+
};
|
|
48
|
+
},
|
|
49
|
+
async seed({ User, Vehicle, Booking, Charge }) {
|
|
50
|
+
const users = await User.create([
|
|
51
|
+
{ name: 'Jane Doe', email: 'jane@example.com', balance: 50, joinedAt: new Date('2025-02-01') },
|
|
52
|
+
{ name: 'John Smith', email: 'john@example.com', balance: -25, joinedAt: new Date('2025-03-15') },
|
|
53
|
+
{ name: 'Alice Wonder', email: 'alice@example.com', balance: 100, joinedAt: new Date('2025-06-22') },
|
|
54
|
+
{ name: 'Bob Lee', email: 'bob@example.com', balance: 0, joinedAt: new Date('2026-01-10') }
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
const vehicles = await Vehicle.create([
|
|
58
|
+
{ make: 'Tesla', model: 'Model 3', type: 'ev', licensePlate: 'EV-001', dailyRate: 80 },
|
|
59
|
+
{ make: 'Toyota', model: 'Camry', type: 'sedan', licensePlate: 'SD-110', dailyRate: 50 },
|
|
60
|
+
{ make: 'Honda', model: 'CR-V', type: 'suv', licensePlate: 'SU-220', dailyRate: 70 },
|
|
61
|
+
{ make: 'Nissan', model: 'Leaf', type: 'ev', licensePlate: 'EV-330', dailyRate: 60 }
|
|
62
|
+
]);
|
|
63
|
+
|
|
64
|
+
const dayOffset = (n) => {
|
|
65
|
+
const d = new Date('2026-05-13T12:00:00.000Z');
|
|
66
|
+
d.setUTCDate(d.getUTCDate() + n);
|
|
67
|
+
return d;
|
|
68
|
+
};
|
|
69
|
+
const daysAgo = (n) => dayOffset(-n);
|
|
70
|
+
|
|
71
|
+
const bookings = await Booking.create([
|
|
72
|
+
{
|
|
73
|
+
userId: users[0]._id,
|
|
74
|
+
vehicleId: vehicles[0]._id,
|
|
75
|
+
startAt: daysAgo(40),
|
|
76
|
+
endAt: daysAgo(38),
|
|
77
|
+
returnedAt: daysAgo(38),
|
|
78
|
+
status: 'completed'
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
userId: users[1]._id,
|
|
82
|
+
vehicleId: vehicles[1]._id,
|
|
83
|
+
startAt: daysAgo(30),
|
|
84
|
+
endAt: daysAgo(28),
|
|
85
|
+
returnedAt: daysAgo(27),
|
|
86
|
+
status: 'completed'
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
userId: users[1]._id,
|
|
90
|
+
vehicleId: vehicles[2]._id,
|
|
91
|
+
startAt: daysAgo(20),
|
|
92
|
+
endAt: daysAgo(18),
|
|
93
|
+
returnedAt: daysAgo(15),
|
|
94
|
+
status: 'completed'
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
userId: users[2]._id,
|
|
98
|
+
vehicleId: vehicles[3]._id,
|
|
99
|
+
startAt: daysAgo(10),
|
|
100
|
+
endAt: daysAgo(8),
|
|
101
|
+
returnedAt: daysAgo(8),
|
|
102
|
+
status: 'completed'
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
userId: users[2]._id,
|
|
106
|
+
vehicleId: vehicles[0]._id,
|
|
107
|
+
startAt: daysAgo(3),
|
|
108
|
+
endAt: dayOffset(2),
|
|
109
|
+
returnedAt: null,
|
|
110
|
+
status: 'active'
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
userId: users[3]._id,
|
|
114
|
+
vehicleId: vehicles[1]._id,
|
|
115
|
+
startAt: daysAgo(1),
|
|
116
|
+
endAt: dayOffset(5),
|
|
117
|
+
returnedAt: null,
|
|
118
|
+
status: 'active'
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
userId: users[3]._id,
|
|
122
|
+
vehicleId: vehicles[3]._id,
|
|
123
|
+
startAt: daysAgo(60),
|
|
124
|
+
endAt: daysAgo(58),
|
|
125
|
+
returnedAt: null,
|
|
126
|
+
status: 'cancelled'
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
userId: users[0]._id,
|
|
130
|
+
vehicleId: vehicles[2]._id,
|
|
131
|
+
startAt: daysAgo(50),
|
|
132
|
+
endAt: daysAgo(48),
|
|
133
|
+
returnedAt: daysAgo(46),
|
|
134
|
+
status: 'completed'
|
|
135
|
+
}
|
|
136
|
+
]);
|
|
137
|
+
|
|
138
|
+
await Charge.create([
|
|
139
|
+
{ userId: users[0]._id, bookingId: bookings[0]._id, amount: 160, paid: true, createdAt: daysAgo(38) },
|
|
140
|
+
{ userId: users[1]._id, bookingId: bookings[1]._id, amount: 100, paid: true, createdAt: daysAgo(27) },
|
|
141
|
+
{ userId: users[1]._id, bookingId: bookings[2]._id, amount: 140, paid: false, createdAt: daysAgo(15) },
|
|
142
|
+
{ userId: users[1]._id, bookingId: bookings[2]._id, amount: 35, paid: false, createdAt: daysAgo(15) },
|
|
143
|
+
{ userId: users[2]._id, bookingId: bookings[3]._id, amount: 120, paid: true, createdAt: daysAgo(8) },
|
|
144
|
+
{ userId: users[0]._id, bookingId: bookings[7]._id, amount: 140, paid: false, createdAt: daysAgo(46) }
|
|
145
|
+
]);
|
|
146
|
+
}
|
|
147
|
+
};
|