@mongoosejs/studio 0.5.1 → 0.5.3

package/eval/agent/cases.js

@@ -0,0 +1,168 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+
+module.exports = [
+  {
+    id: 'simple-1-user-count',
+    category: 'simple',
+    fixture: 'ecommerce',
+    prompt: 'How many users do we have in total, including deleted ones?',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      const got = typeof trace.output === 'number' ? trace.output : trace.output?.count;
+      assert.equal(got, 4, `expected 4 users, got ${JSON.stringify(trace.output)}`);
+    }
+  },
+  {
+    id: 'simple-2-most-expensive-product',
+    category: 'simple',
+    fixture: 'ecommerce',
+    prompt: 'What is the single most expensive product? Return an object with its name and price.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.equal(trace.output?.name, 'Laptop', `expected name=Laptop, got ${JSON.stringify(trace.output)}`);
+      assert.equal(trace.output?.price, 1200, `expected price=1200, got ${JSON.stringify(trace.output)}`);
+    }
+  },
+  {
+    id: 'simple-3-active-bookings',
+    category: 'simple',
+    fixture: 'carsharing',
+    prompt: 'How many bookings currently have status "active"?',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      const got = typeof trace.output === 'number' ? trace.output : trace.output?.count;
+      assert.equal(got, 2, `expected 2 active bookings, got ${JSON.stringify(trace.output)}`);
+    }
+  },
+  {
+    id: 'simple-4-list-evs',
+    category: 'simple',
+    fixture: 'carsharing',
+    prompt: 'List every electric vehicle in the fleet, returning an array with make and model.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      const list = Array.isArray(trace.output) ? trace.output : trace.output?.vehicles;
+      assert.equal(list?.length, 2, `expected 2 EVs, got ${JSON.stringify(trace.output)}`);
+      const json = JSON.stringify(list);
+      assert.ok(json.includes('Tesla') && json.includes('Leaf'), `expected Tesla and Leaf, got ${json}`);
+    }
+  },
+  {
+    id: 'simple-5-orders-by-status',
+    category: 'simple',
+    fixture: 'ecommerce',
+    prompt: 'Show a table with the count of orders grouped by status.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(trace.output?.$table, `expected $table, got ${JSON.stringify(trace.output)}`);
+      assert.equal(trace.output.$table.rows.length, 5, `expected 5 status rows, got ${trace.output.$table.rows.length}`);
+      const json = JSON.stringify(trace.output.$table);
+      for (const status of ['pending', 'paid', 'shipped', 'delivered', 'cancelled']) {
+        assert.ok(json.includes(status), `expected "${status}" in table, got ${json}`);
+      }
+    }
+  },
+
+  {
+    id: 'nested-1-orders-by-month',
+    category: 'nested',
+    fixture: 'ecommerce',
+    prompt: 'How many orders were placed each month over the past 6 months? Return a table.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(trace.output?.$table, `expected $table, got ${JSON.stringify(trace.output)}`);
+      assert.ok(trace.output.$table.rows.length >= 3, `expected ≥3 month rows, got ${trace.output.$table.rows.length}`);
+    }
+  },
+  {
+    id: 'nested-2-late-returns-unpaid',
+    category: 'nested',
+    fixture: 'carsharing',
+    prompt: 'Return an array of bookings that were returned late (returnedAt after endAt) and have at least one unpaid charge.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(Array.isArray(trace.output), `expected array, got ${JSON.stringify(trace.output)}`);
+      assert.equal(trace.output.length, 2, `expected 2 late+unpaid bookings, got ${trace.output.length}`);
+    }
+  },
+  {
+    id: 'nested-3-paid-orders-with-electronics',
+    category: 'nested',
+    fixture: 'ecommerce',
+    prompt: 'Return an array of paid orders that contain at least one electronics product.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(Array.isArray(trace.output), `expected array, got ${JSON.stringify(trace.output)}`);
+      assert.equal(trace.output.length, 2, `expected 2 paid+electronics orders, got ${trace.output.length}`);
+    }
+  },
+
+  {
+    id: 'dashboard-1-top-products',
+    category: 'dashboard',
+    fixture: 'ecommerce',
+    prompt: 'Build a dashboard showing the top 5 products by gross revenue from paid orders.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      const renderable = trace.output?.$chart || trace.output?.$table;
+      assert.ok(renderable, `expected $chart or $table, got ${JSON.stringify(trace.output)}`);
+      assert.ok(JSON.stringify(renderable).includes('Headphones'), `Headphones should be top product, got ${JSON.stringify(renderable)}`);
+    }
+  },
+  {
+    id: 'dashboard-2-bookings-per-vehicle-type',
+    category: 'dashboard',
+    fixture: 'carsharing',
+    prompt: 'Show a chart of total bookings broken down by vehicle type.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(trace.output?.$chart, `expected $chart, got ${JSON.stringify(trace.output)}`);
+      const json = JSON.stringify(trace.output.$chart);
+      for (const type of ['ev', 'sedan', 'suv']) {
+        assert.ok(json.includes(type), `expected "${type}" in chart, got ${json}`);
+      }
+    }
+  },
+  {
+    id: 'dashboard-3-revenue-by-month',
+    category: 'dashboard',
+    fixture: 'ecommerce',
+    prompt: 'Chart total revenue per month from paid orders. Include only months that contain at least one paid order, in chronological order, using "YYYY-MM" format for the labels.',
+    assert(trace) {
+      assert.equal(trace.executionError, null, `script error: ${trace.executionError}`);
+      assert.ok(trace.output?.$chart, `expected $chart, got ${JSON.stringify(trace.output)}`);
+      assert.deepEqual(trace.output.$chart.data.labels, ['2026-02', '2026-03', '2026-04']);
+      assert.equal(trace.output.$chart.data.datasets.length, 1, `expected 1 dataset, got ${JSON.stringify(trace.output.$chart.data.datasets)}`);
+      assert.deepEqual(trace.output.$chart.data.datasets[0].data, [300, 60, 222]);
+    }
+  },
+
+  {
+    id: 'ambig-1-missing-user',
+    category: 'ambiguous',
+    fixture: 'carsharing',
+    prompt: 'Why does Jane Doe have a negative balance?',
+    assert(trace) {
+      assert.ok(
+        trace.toolCalls.some(t => t.toolName === 'findOne' || t.toolName === 'find'),
+        'agent should look up the user before answering'
+      );
+      const lower = trace.text.toLowerCase();
+      assert.ok(
+        lower.includes('50') || lower.includes('positive') || lower.includes('not negative'),
+        `agent should note Jane's balance is actually positive; text was: ${trace.text.slice(0, 500)}`
+      );
+    }
+  },
+  {
+    id: 'ambig-2-investigate',
+    category: 'ambiguous',
+    fixture: 'ecommerce',
+    prompt: 'Some orders look weird. Can you investigate?',
+    assert(trace) {
+      assert.ok(trace.toolCalls.length >= 2, `expected ≥2 tool calls; saw ${trace.toolCalls.length}`);
+    }
+  }
+];

package/eval/agent/fixtures/carsharing.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const mongoose = require('mongoose');
+
+const { Schema } = mongoose;
+
+const userSchema = new Schema({
+  name: String,
+  email: String,
+  balance: Number,
+  joinedAt: Date
+});
+
+const vehicleSchema = new Schema({
+  make: String,
+  model: String,
+  type: { type: String, enum: ['sedan', 'suv', 'ev'] },
+  licensePlate: String,
+  dailyRate: Number
+});
+
+const bookingSchema = new Schema({
+  userId: { type: Schema.Types.ObjectId, ref: 'User' },
+  vehicleId: { type: Schema.Types.ObjectId, ref: 'Vehicle' },
+  startAt: Date,
+  endAt: Date,
+  returnedAt: Date,
+  status: { type: String, enum: ['active', 'completed', 'cancelled'] }
+});
+
+const chargeSchema = new Schema({
+  userId: { type: Schema.Types.ObjectId, ref: 'User' },
+  bookingId: { type: Schema.Types.ObjectId, ref: 'Booking' },
+  amount: Number,
+  paid: Boolean,
+  createdAt: Date
+});
+
+module.exports = {
+  name: 'carsharing',
+  defineModels(connection) {
+    return {
+      User: connection.model('User', userSchema),
+      Vehicle: connection.model('Vehicle', vehicleSchema),
+      Booking: connection.model('Booking', bookingSchema),
+      Charge: connection.model('Charge', chargeSchema)
+    };
+  },
+  async seed({ User, Vehicle, Booking, Charge }) {
+    const users = await User.create([
+      { name: 'Jane Doe', email: 'jane@example.com', balance: 50, joinedAt: new Date('2025-02-01') },
+      { name: 'John Smith', email: 'john@example.com', balance: -25, joinedAt: new Date('2025-03-15') },
+      { name: 'Alice Wonder', email: 'alice@example.com', balance: 100, joinedAt: new Date('2025-06-22') },
+      { name: 'Bob Lee', email: 'bob@example.com', balance: 0, joinedAt: new Date('2026-01-10') }
+    ]);
+
+    const vehicles = await Vehicle.create([
+      { make: 'Tesla', model: 'Model 3', type: 'ev', licensePlate: 'EV-001', dailyRate: 80 },
+      { make: 'Toyota', model: 'Camry', type: 'sedan', licensePlate: 'SD-110', dailyRate: 50 },
+      { make: 'Honda', model: 'CR-V', type: 'suv', licensePlate: 'SU-220', dailyRate: 70 },
+      { make: 'Nissan', model: 'Leaf', type: 'ev', licensePlate: 'EV-330', dailyRate: 60 }
+    ]);
+
+    const dayOffset = (n) => {
+      const d = new Date('2026-05-13T12:00:00.000Z');
+      d.setUTCDate(d.getUTCDate() + n);
+      return d;
+    };
+    const daysAgo = (n) => dayOffset(-n);
+
+    const bookings = await Booking.create([
+      {
+        userId: users[0]._id,
+        vehicleId: vehicles[0]._id,
+        startAt: daysAgo(40),
+        endAt: daysAgo(38),
+        returnedAt: daysAgo(38),
+        status: 'completed'
+      },
+      {
+        userId: users[1]._id,
+        vehicleId: vehicles[1]._id,
+        startAt: daysAgo(30),
+        endAt: daysAgo(28),
+        returnedAt: daysAgo(27),
+        status: 'completed'
+      },
+      {
+        userId: users[1]._id,
+        vehicleId: vehicles[2]._id,
+        startAt: daysAgo(20),
+        endAt: daysAgo(18),
+        returnedAt: daysAgo(15),
+        status: 'completed'
+      },
+      {
+        userId: users[2]._id,
+        vehicleId: vehicles[3]._id,
+        startAt: daysAgo(10),
+        endAt: daysAgo(8),
+        returnedAt: daysAgo(8),
+        status: 'completed'
+      },
+      {
+        userId: users[2]._id,
+        vehicleId: vehicles[0]._id,
+        startAt: daysAgo(3),
+        endAt: dayOffset(2),
+        returnedAt: null,
+        status: 'active'
+      },
+      {
+        userId: users[3]._id,
+        vehicleId: vehicles[1]._id,
+        startAt: daysAgo(1),
+        endAt: dayOffset(5),
+        returnedAt: null,
+        status: 'active'
+      },
+      {
+        userId: users[3]._id,
+        vehicleId: vehicles[3]._id,
+        startAt: daysAgo(60),
+        endAt: daysAgo(58),
+        returnedAt: null,
+        status: 'cancelled'
+      },
+      {
+        userId: users[0]._id,
+        vehicleId: vehicles[2]._id,
+        startAt: daysAgo(50),
+        endAt: daysAgo(48),
+        returnedAt: daysAgo(46),
+        status: 'completed'
+      }
+    ]);
+
+    await Charge.create([
+      { userId: users[0]._id, bookingId: bookings[0]._id, amount: 160, paid: true, createdAt: daysAgo(38) },
+      { userId: users[1]._id, bookingId: bookings[1]._id, amount: 100, paid: true, createdAt: daysAgo(27) },
+      { userId: users[1]._id, bookingId: bookings[2]._id, amount: 140, paid: false, createdAt: daysAgo(15) },
+      { userId: users[1]._id, bookingId: bookings[2]._id, amount: 35, paid: false, createdAt: daysAgo(15) },
+      { userId: users[2]._id, bookingId: bookings[3]._id, amount: 120, paid: true, createdAt: daysAgo(8) },
+      { userId: users[0]._id, bookingId: bookings[7]._id, amount: 140, paid: false, createdAt: daysAgo(46) }
+    ]);
+  }
+};

package/eval/agent/fixtures/ecommerce.js

@@ -0,0 +1,152 @@
+'use strict';
+
+const mongoose = require('mongoose');
+
+const { Schema } = mongoose;
+
+const userSchema = new Schema({
+  name: String,
+  email: String,
+  signupDate: Date,
+  isDeleted: { type: Boolean, default: false }
+});
+
+const productSchema = new Schema({
+  name: String,
+  category: { type: String, enum: ['electronics', 'apparel', 'home', 'books'] },
+  price: Number,
+  inventory: Number,
+  createdAt: Date
+});
+
+const orderSchema = new Schema({
+  userId: { type: Schema.Types.ObjectId, ref: 'User' },
+  items: [{
+    productId: { type: Schema.Types.ObjectId, ref: 'Product' },
+    quantity: Number,
+    price: Number
+  }],
+  total: Number,
+  status: { type: String, enum: ['pending', 'paid', 'shipped', 'delivered', 'cancelled'] },
+  placedAt: Date,
+  paidAt: Date
+});
+
+module.exports = {
+  name: 'ecommerce',
+  defineModels(connection) {
+    return {
+      User: connection.model('User', userSchema),
+      Product: connection.model('Product', productSchema),
+      Order: connection.model('Order', orderSchema)
+    };
+  },
+  async seed({ User, Product, Order }) {
+    const users = await User.create([
+      { name: 'Alice Smith', email: 'alice@example.com', signupDate: new Date('2025-01-15'), isDeleted: false },
+      { name: 'Bob Jones', email: 'bob@example.com', signupDate: new Date('2025-04-22'), isDeleted: false },
+      { name: 'Carol Wu', email: 'carol@example.com', signupDate: new Date('2025-09-08'), isDeleted: true },
+      { name: 'David Park', email: 'david@example.com', signupDate: new Date('2025-12-03'), isDeleted: false }
+    ]);
+
+    const products = await Product.create([
+      { name: 'Laptop', category: 'electronics', price: 1200, inventory: 12, createdAt: new Date('2024-11-01') },
+      { name: 'Headphones', category: 'electronics', price: 150, inventory: 45, createdAt: new Date('2024-11-01') },
+      { name: 'T-Shirt', category: 'apparel', price: 25, inventory: 200, createdAt: new Date('2025-01-15') },
+      { name: 'Lamp', category: 'home', price: 60, inventory: 30, createdAt: new Date('2025-02-10') },
+      { name: 'Novel', category: 'books', price: 18, inventory: 100, createdAt: new Date('2025-03-04') }
+    ]);
+
+    const monthsAgo = (n) => {
+      const d = new Date('2026-05-13T12:00:00.000Z');
+      d.setUTCMonth(d.getUTCMonth() - n);
+      return d;
+    };
+
+    await Order.create([
+      {
+        userId: users[0]._id,
+        items: [{ productId: products[0]._id, quantity: 1, price: 1200 }],
+        total: 1200,
+        status: 'delivered',
+        placedAt: monthsAgo(5),
+        paidAt: monthsAgo(5)
+      },
+      {
+        userId: users[0]._id,
+        items: [{ productId: products[1]._id, quantity: 2, price: 150 }],
+        total: 300,
+        status: 'paid',
+        placedAt: monthsAgo(3),
+        paidAt: monthsAgo(3)
+      },
+      {
+        userId: users[1]._id,
+        items: [
+          { productId: products[2]._id, quantity: 3, price: 25 },
+          { productId: products[4]._id, quantity: 1, price: 18 }
+        ],
+        total: 93,
+        status: 'shipped',
+        placedAt: monthsAgo(4),
+        paidAt: monthsAgo(4)
+      },
+      {
+        userId: users[1]._id,
+        items: [{ productId: products[3]._id, quantity: 1, price: 60 }],
+        total: 60,
+        status: 'paid',
+        placedAt: monthsAgo(2),
+        paidAt: monthsAgo(2)
+      },
+      {
+        userId: users[3]._id,
+        items: [{ productId: products[0]._id, quantity: 1, price: 1200 }],
+        total: 1200,
+        status: 'cancelled',
+        placedAt: monthsAgo(2),
+        paidAt: null
+      },
+      {
+        userId: users[3]._id,
+        items: [{ productId: products[1]._id, quantity: 1, price: 150 }],
+        total: 150,
+        status: 'paid',
+        placedAt: monthsAgo(1),
+        paidAt: monthsAgo(1)
+      },
+      {
+        userId: users[1]._id,
+        items: [{ productId: products[4]._id, quantity: 4, price: 18 }],
+        total: 72,
+        status: 'paid',
+        placedAt: monthsAgo(1),
+        paidAt: monthsAgo(1)
+      },
+      {
+        userId: users[3]._id,
+        items: [{ productId: products[2]._id, quantity: 2, price: 25 }],
+        total: 50,
+        status: 'pending',
+        placedAt: monthsAgo(0),
+        paidAt: null
+      },
+      {
+        userId: users[0]._id,
+        items: [{ productId: products[3]._id, quantity: 2, price: 60 }],
+        total: 120,
+        status: 'cancelled',
+        placedAt: monthsAgo(6),
+        paidAt: null
+      },
+      {
+        userId: users[1]._id,
+        items: [{ productId: products[0]._id, quantity: 1, price: 1200 }],
+        total: 1200,
+        status: 'delivered',
+        placedAt: monthsAgo(4),
+        paidAt: monthsAgo(4)
+      }
+    ]);
+  }
+};