@resolveio/server-lib 22.3.220 → 22.3.221

package/tests/ai-assistant-snf-live-eval.ts

@@ -0,0 +1,975 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { MongoClient } from 'mongodb';
+import {
+	buildAssistantDatedPivotDisplay,
+	buildDisplayTable,
+	classifyAssistantRequestType
+} from '../src/methods/ai-terminal';
+
+type EvalRow = Record<string, any>;
+
+type EvalResult = {
+	id: string;
+	prompt: string;
+	classification: {
+		type: string;
+		dataQuestion: boolean;
+		source: string;
+		pass: boolean;
+	};
+	query: {
+		kind: 'read' | 'aggregate';
+		collection: string;
+	};
+	rowCount: number;
+	minRows: number;
+	pass: boolean;
+	notes?: string;
+	sample: EvalRow[];
+	error?: string;
+};
+
+type EvalContext = {
+	db: any;
+	now: Date;
+	startOfWeek: Date;
+	last30Days: Date;
+	last90Days: Date;
+	last2Weeks: Date;
+	last6Months: Date;
+	thisMonthStart: Date;
+	nextMonthStart: Date;
+	lastMonthStart: Date;
+	lastMonthEnd: Date;
+	targetBlendChemicalIds: Array<string>;
+};
+
+type EvalCase = {
+	id: string;
+	prompt: string;
+	minRows: number;
+	query: {
+		kind: 'read' | 'aggregate';
+		collection: string;
+	};
+	notes?: string;
+	run: (context: EvalContext) => Promise<EvalRow[]>;
+};
+
+function addDaysUtc(date: Date, days: number): Date {
+	return new Date(Date.UTC(
+		date.getUTCFullYear(),
+		date.getUTCMonth(),
+		date.getUTCDate() + days,
+		date.getUTCHours(),
+		date.getUTCMinutes(),
+		date.getUTCSeconds(),
+		date.getUTCMilliseconds()
+	));
+}
+
+function addMonthsUtc(date: Date, months: number): Date {
+	return new Date(Date.UTC(
+		date.getUTCFullYear(),
+		date.getUTCMonth() + months,
+		date.getUTCDate(),
+		date.getUTCHours(),
+		date.getUTCMinutes(),
+		date.getUTCSeconds(),
+		date.getUTCMilliseconds()
+	));
+}
+
+function startOfDayUtc(date: Date): Date {
+	return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), 0, 0, 0, 0));
+}
+
+function startOfWeekUtcMonday(date: Date): Date {
+	const day = date.getUTCDay();
+	const diff = (day + 6) % 7;
+	return startOfDayUtc(addDaysUtc(date, -diff));
+}
+
+function startOfMonthUtc(date: Date): Date {
+	return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1, 0, 0, 0, 0));
+}
+
+function startOfYearUtc(date: Date): Date {
+	return new Date(Date.UTC(date.getUTCFullYear(), 0, 1, 0, 0, 0, 0));
+}
+
+function sumChemicalQuantityExpr(quantityField: 'delivered' | 'returned' | 'shipped') {
+	return {
+		$sum: {
+			$map: {
+				input: { $ifNull: ['$chemicals', []] },
+				as: 'chem',
+				in: { $ifNull: [`$$chem.${quantityField}.quantity`, 0] }
+			}
+		}
+	};
+}
+
+async function resolveExistingCollection(db: any, candidates: string[]): Promise<string | null> {
+	const rows = await db.listCollections({}, { nameOnly: true }).toArray();
+	const names = new Set(rows.map((row: any) => String(row?.name || '').trim()));
+	for (const candidate of candidates) {
+		if (names.has(candidate)) {
+			return candidate;
+		}
+	}
+	return null;
+}
+
+const EVAL_CASES: EvalCase[] = [
+	{
+		id: 'wo_last20_this_week_group_status',
+		prompt: 'Show me the last 20 work orders created this week, grouped by status.',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					date_created: { $gte: ctx.startOfWeek, $lt: ctx.now }
+				}
+			},
+			{ $sort: { date_created: -1 } },
+			{ $limit: 20 },
+			{
+				$group: {
+					_id: { $ifNull: ['$status', 'Unknown'] },
+					work_order_count: { $sum: 1 }
+				}
+			},
+			{ $sort: { work_order_count: -1, _id: 1 } },
+			{
+				$project: {
+					_id: 0,
+					status: '$_id',
+					work_order_count: 1
+				}
+			}
+			]).toArray()
+	},
+	{
+		id: 'wo_last20_this_week_by_status_alias',
+		prompt: 'Show me the last 20 work orders created this week by status.',
+		minRows: 0,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					date_created: { $gte: ctx.startOfWeek, $lt: ctx.now }
+				}
+			},
+			{ $sort: { date_created: -1 } },
+			{ $limit: 20 },
+			{
+				$group: {
+					_id: { $ifNull: ['$status', 'Unknown'] },
+					work_order_count: { $sum: 1 }
+				}
+			},
+			{ $sort: { work_order_count: -1, _id: 1 } },
+			{
+				$project: {
+					_id: 0,
+					status: '$_id',
+					work_order_count: 1
+				}
+			}
+			]).toArray()
+	},
+	{
+		id: 'devon_jobs_chemical_sent_this_year',
+		prompt: 'Summarize Devon Energy jobs and amount of chemicals sent to them for this year',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'jobs' },
+		run: async (ctx) => {
+			const yearStart = startOfYearUtc(ctx.now);
+			const nextYearStart = new Date(Date.UTC(ctx.now.getUTCFullYear() + 1, 0, 1, 0, 0, 0, 0));
+			return await ctx.db.collection('jobs').aggregate([
+				{
+					$match: {
+						$and: [
+							{
+								$or: [
+									{ customer: { $regex: 'Devon Energy', $options: 'i' } },
+									{ 'customer_groups.customer_group': { $regex: 'Devon Energy', $options: 'i' } }
+								]
+							},
+							{
+								$or: [
+									{ date_job_created: { $gte: yearStart, $lt: nextYearStart } },
+									{ createdAt: { $gte: yearStart, $lt: nextYearStart } }
+								]
+							}
+						]
+					}
+				},
+				{ $lookup: { from: 'work-order-dynamics', localField: 'wos', foreignField: '_id', as: 'workOrders' } },
+				{ $unwind: { path: '$workOrders', preserveNullAndEmptyArrays: true } },
+				{ $unwind: { path: '$workOrders.chemicals', preserveNullAndEmptyArrays: true } },
+				{
+					$group: {
+						_id: {
+							id_job: '$_id',
+							job_number: '$job_number_string',
+							pad: '$pad',
+							status: '$status',
+							chemical: '$workOrders.chemicals.chemical',
+							unit: '$workOrders.chemicals.unit'
+						},
+						customer: {
+							$first: {
+								$ifNull: ['$customer', { $first: '$customer_groups.customer_group' }]
+							}
+						},
+						work_order_count: { $addToSet: '$workOrders._id' },
+						quantity_sent: { $sum: { $ifNull: ['$workOrders.chemicals.shipped.quantity', 0] } },
+						quantity_delivered: { $sum: { $ifNull: ['$workOrders.chemicals.delivered.quantity', 0] } }
+					}
+				},
+				{
+					$project: {
+						_id: 0,
+						job: '$_id.job_number',
+						pad: '$_id.pad',
+						customer: '$customer',
+						status: '$_id.status',
+						chemical: '$_id.chemical',
+						unit: '$_id.unit',
+						work_order_count: { $size: '$work_order_count' },
+						quantity_sent: '$quantity_sent',
+						quantity_delivered: '$quantity_delivered'
+					}
+				},
+				{ $sort: { quantity_sent: -1, pad: 1, chemical: 1 } },
+				{ $limit: 50 }
+			], { allowDiskUse: true }).toArray();
+		}
+	},
+	{
+		id: 'wo_completed_per_day_last_30d',
+		prompt: 'For the last 30 days, how many work orders were completed per day?',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					date_completed: { $gte: ctx.last30Days, $lt: ctx.now }
+				}
+			},
+			{
+				$group: {
+					_id: {
+						$dateToString: { format: '%Y-%m-%d', date: '$date_completed', timezone: 'UTC' }
+					},
+					completed_work_orders: { $sum: 1 }
+				}
+			},
+			{ $sort: { _id: 1 } },
+			{
+				$project: {
+					_id: 0,
+					day_utc: '$_id',
+					completed_work_orders: 1
+				}
+			}
+		]).toArray()
+	},
+	{
+		id: 'wo_top_customers_last_6m',
+		prompt: 'List the top 10 customers by number of work orders in the last 6 months.',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'orders' },
+		run: async (ctx) => {
+			const collection = await resolveExistingCollection(ctx.db, ['work-order-dynamics', 'maintenance-orders', 'orders']);
+			if (!collection) {
+				return [];
+			}
+			return await ctx.db.collection(collection).aggregate([
+				{
+					$addFields: {
+						effective_date: {
+							$ifNull: [
+								'$date_created',
+								{ $ifNull: ['$date_create', '$createdAt'] }
+							]
+						},
+						customer_label: {
+							$ifNull: [
+								'$customer',
+								{
+									$ifNull: [
+										'$customer_name',
+										{
+											$ifNull: [
+												'$client_name',
+												{
+													$ifNull: [
+														{ $toString: '$id_customer' },
+														{ $ifNull: ['$qb_ListID_class', 'Unknown'] }
+													]
+												}
+											]
+										}
+									]
+								}
+							]
+						}
+					}
+				},
+				{
+					$match: {
+						effective_date: { $gte: ctx.last6Months, $lt: ctx.now },
+						'deleted.date': { $exists: false }
+					}
+				},
+				{
+					$group: {
+						_id: '$customer_label',
+						work_order_count: { $sum: 1 }
+					}
+				},
+				{ $sort: { work_order_count: -1, _id: 1 } },
+				{ $limit: 10 },
+				{
+					$project: {
+						_id: 0,
+						customer: '$_id',
+						work_order_count: 1
+					}
+				}
+			]).toArray();
+		}
+	},
+	{
+		id: 'wo_open_missing_required_fields',
+		prompt: 'Show open work orders with missing required fields (customer, location, or scheduled date).',
+		minRows: 0,
+		query: { kind: 'read', collection: 'work-order-dynamics' },
+		notes: 'A zero row count can be valid here if data quality is clean.',
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').find(
+			{
+				status: { $nin: ['Closed', 'Completed', 'Canceled'] },
+				$or: [
+					{ customer: { $in: [null, '', 'N/A'] } },
+					{ location: null },
+					{ 'location.name': { $in: [null, '', 'N/A'] } },
+					{ date_appointment_time: null }
+				]
+			},
+			{
+				projection: {
+					_id: 1,
+					wo_string: 1,
+					status: 1,
+					customer: 1,
+					type: 1,
+					'location.name': 1,
+					date_created: 1,
+					date_appointment_time: 1
+				},
+				sort: { date_created: -1 },
+				limit: 50
+			}
+		).toArray()
+	},
+	{
+		id: 'blend_last10_summary',
+		prompt: 'For blending: summarize the last 10 blend tickets with product, total volume, and created date.',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'chemical-blends' },
+		run: async (ctx) => await ctx.db.collection('chemical-blends').aggregate([
+			{ $sort: { date: -1, createdAt: -1 } },
+			{ $limit: 10 },
+			{
+				$project: {
+					_id: 1,
+					product: { $ifNull: ['$blend_name', '$chemical'] },
+					total_volume: '$chemical_recipe_quantity',
+					volume_unit: { $cond: [{ $eq: ['$blend_in_pounds', true] }, 'lb', 'gal'] },
+					created_date: { $ifNull: ['$date', '$createdAt'] },
+					batch_number: 1
+				}
+			}
+		]).toArray()
+	},
+	{
+		id: 'blend_used_red_ultra_lift_last_30d',
+		prompt: 'Which blends used Chemical Red Ultra Lift in the last 30 days? Include lot/batch and volume.',
+		minRows: 0,
+		query: { kind: 'aggregate', collection: 'chemical-blends' },
+		notes: 'A zero row count is valid when this chemical is not present in the current 30-day window.',
+		run: async (ctx) => await ctx.db.collection('chemical-blends').aggregate([
+			{
+				$match: {
+					date: { $gte: ctx.last30Days, $lt: ctx.now }
+				}
+			},
+			{ $unwind: '$blend_chemicals' },
+			{
+				$match: {
+					$or: [
+						{ 'blend_chemicals.chemical': { $regex: 'Red\\s+Ultra\\s+Lift', $options: 'i' } },
+						{ 'blend_chemicals.id_chemical': { $in: ctx.targetBlendChemicalIds } }
+					]
+				}
+			},
+			{ $sort: { date: -1 } },
+			{
+				$project: {
+					_id: 1,
+					blend_name: 1,
+					batch_number: 1,
+					blend_date: '$date',
+					chemical_used: '$blend_chemicals.chemical',
+					volume: {
+						$ifNull: ['$blend_chemicals.quantity_recipe_gallons', '$blend_chemicals.quantity_recipe']
+					},
+					volume_unit: { $ifNull: ['$blend_chemicals.unit', 'gal'] }
+				}
+			},
+			{ $limit: 50 }
+		]).toArray()
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'invoices' },
+		run: async (ctx) => await ctx.db.collection('invoices').aggregate([
+			{
+				$match: {
+					date_created: { $gte: ctx.last6Months, $lt: ctx.now }
+				}
+			},
+			{
+				$group: {
+					_id: { $ifNull: ['$customer', 'Unknown'] },
+					invoice_count: { $sum: 1 },
+					invoice_total: {
+						$sum: {
+							$ifNull: ['$price', { $ifNull: ['$subtotal', 0] }]
+						}
+					}
+				}
+			},
+			{ $sort: { invoice_total: -1, _id: 1 } },
+			{ $limit: 10 },
+			{
+				$project: {
+					_id: 0,
+					customer: '$_id',
+					invoice_count: 1,
+					invoice_total: 1
+				}
+			}
+		]).toArray()
+	},
+	{
+		id: 'invoice_revenue_monthly_grid_last_6m',
+		prompt: 'Break down my total revenue over the last 6 month by month',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'invoices' },
+		run: async (ctx) => {
+			const rows = await ctx.db.collection('invoices').aggregate([
+				{
+					$match: {
+						date_created: { $gte: ctx.last6Months, $lt: ctx.now }
+					}
+				},
+				{
+					$group: {
+						_id: {
+							year: { $year: '$date_created' },
+							month: { $month: '$date_created' }
+						},
+						total_revenue: {
+							$sum: {
+								$ifNull: ['$price', { $ifNull: ['$subtotal', 0] }]
+							}
+						}
+					}
+				},
+				{
+					$project: {
+						_id: 0,
+						year: '$_id.year',
+						month: '$_id.month',
+						total_revenue: 1
+					}
+				},
+				{ $sort: { year: 1, month: 1 } }
+			]).toArray();
+			const display = buildDisplayTable(rows, { maxColumns: 40, maxRows: 500 });
+			const pivot = buildAssistantDatedPivotDisplay(display);
+			if (!pivot) {
+				throw new Error('Expected dated pivot grid for monthly revenue report.');
+			}
+			const monthColumns = pivot.columns.slice(1);
+			if (pivot.columns[0] !== 'Metric' || monthColumns.length < 2) {
+				throw new Error('Expected Metric + month columns in monthly revenue grid.');
+			}
+			return pivot.rows;
+		}
+	},
+	{
+		id: 'invoice_revenue_monthly_customer_grid_last_6m',
+		prompt: 'Break down my total revenue over the last 6 month by month by each customer',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'invoices' },
+		run: async (ctx) => {
+			const rows = await ctx.db.collection('invoices').aggregate([
+				{
+					$match: {
+						date_created: { $gte: ctx.last6Months, $lt: ctx.now }
+					}
+				},
+				{
+					$group: {
+						_id: {
+							customer: { $ifNull: ['$customer', 'Unknown'] },
+							year: { $year: '$date_created' },
+							month: { $month: '$date_created' }
+						},
+						total_revenue: {
+							$sum: {
+								$ifNull: ['$price', { $ifNull: ['$subtotal', 0] }]
+							}
+						}
+					}
+				},
+				{
+					$project: {
+						_id: 0,
+						customer: '$_id.customer',
+						year: '$_id.year',
+						month: '$_id.month',
+						total_revenue: 1
+					}
+				},
+				{ $sort: { customer: 1, year: 1, month: 1 } }
+			]).toArray();
+			const display = buildDisplayTable(rows, { maxColumns: 50, maxRows: 2000 });
+			const pivot = buildAssistantDatedPivotDisplay(display);
+			if (!pivot) {
+				throw new Error('Expected dated pivot grid for customer monthly revenue report.');
+			}
+			const firstColumn = String(pivot.columns[0] || '').toLowerCase();
+			const monthColumns = pivot.columns.slice(1);
+			if (!firstColumn.includes('customer') || monthColumns.length < 2) {
+				throw new Error('Expected Customer + month header columns in customer revenue grid.');
+			}
+			return pivot.rows;
+		}
+	},
+	{
+		id: 'invoice_missing_invoiced_date_this_month',
+		prompt: 'List invoices missing an invoiced date but created this month.',
+		minRows: 1,
+		query: { kind: 'read', collection: 'invoices' },
+		run: async (ctx) => await ctx.db.collection('invoices').find(
+			{
+				date_created: { $gte: ctx.thisMonthStart, $lt: ctx.nextMonthStart },
+				$or: [
+					{ date_invoiced: null },
+					{ date_invoiced: { $exists: false } }
+				]
+			},
+			{
+				projection: {
+					_id: 1,
+					invoice_number: 1,
+					invoice_number_string: 1,
+					customer: 1,
+					status: 1,
+					date_created: 1,
+					date_invoiced: 1,
+					price: 1
+				},
+				sort: { date_created: -1 },
+				limit: 50
+			}
+		).toArray()
+	},
+	{
+		id: 'deliveries_per_driver_last_month',
+		prompt: 'Break down the number of deliveries per driver last month',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					date_completed: { $gte: ctx.lastMonthStart, $lt: ctx.lastMonthEnd },
+					status: { $in: ['Completed', 'Closed'] }
+				}
+			},
+			{ $unwind: { path: '$drivers', preserveNullAndEmptyArrays: true } },
+			{
+				$group: {
+					_id: { $ifNull: ['$drivers.user', 'Unassigned'] },
+					delivery_count: { $sum: 1 }
+				}
+			},
+			{ $sort: { delivery_count: -1, _id: 1 } },
+			{
+				$project: {
+					_id: 0,
+					driver: '$_id',
+					delivery_count: 1
+				}
+			}
+		]).toArray()
+	},
+	{
+		id: 'last20_deliveries_product_volume_destination',
+		prompt: 'List the last 20 deliveries with product, volume, and destination.',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					status: { $in: ['Completed', 'Closed'] },
+					date_completed: { $type: 'date' }
+				}
+			},
+			{ $sort: { date_completed: -1 } },
+			{ $limit: 20 },
+			{
+				$project: {
+					_id: 1,
+					wo_string: 1,
+					customer: 1,
+					destination: '$location.name',
+					date_completed: 1,
+					products: {
+						$map: {
+							input: { $ifNull: ['$chemicals', []] },
+							as: 'chem',
+							in: '$$chem.chemical'
+						}
+					},
+					delivered_volume: sumChemicalQuantityExpr('delivered'),
+					returned_volume: sumChemicalQuantityExpr('returned'),
+					unit: {
+						$first: {
+							$map: {
+								input: { $ifNull: ['$chemicals', []] },
+								as: 'chem',
+								in: '$$chem.unit'
+							}
+						}
+					}
+				}
+			}
+		]).toArray()
+	},
+	{
+		id: 'jobs_highest_treatment_volume_last_90d',
+		prompt: 'Which jobs have the highest treatment volume in the last 90 days?',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					date_completed: { $gte: ctx.last90Days, $lt: ctx.now },
+					'job.id_job': { $exists: true, $ne: '' }
+				}
+			},
+			{ $unwind: { path: '$chemicals', preserveNullAndEmptyArrays: true } },
+			{
+				$addFields: {
+					chemical_treatment_volume: {
+						$add: [
+							{ $ifNull: ['$chemicals.delivered.quantity', 0] },
+							{ $ifNull: ['$chemicals.returned.quantity', 0] }
+						]
+					}
+				}
+			},
+			{
+				$group: {
+					_id: '$job.id_job',
+					job: { $first: '$job.job' },
+					customer: { $first: '$customer' },
+					treatment_volume: { $sum: '$chemical_treatment_volume' },
+					work_orders: { $addToSet: '$_id' }
+				}
+			},
+			{
+				$project: {
+					_id: 0,
+					job_id: '$_id',
+					job: 1,
+					customer: 1,
+					treatment_volume: 1,
+					work_order_count: { $size: '$work_orders' }
+				}
+			},
+			{ $sort: { treatment_volume: -1, work_order_count: -1 } },
+			{ $limit: 10 }
+		]).toArray()
+	},
+	{
+		id: 'last20_completed_wo_customer_product_delivered_returned',
+		prompt: 'Break down the last 20 completed work orders and tell me what customer, what product and how much we delivered or returned',
+		minRows: 1,
+		query: { kind: 'aggregate', collection: 'work-order-dynamics' },
+		run: async (ctx) => await ctx.db.collection('work-order-dynamics').aggregate([
+			{
+				$match: {
+					status: { $in: ['Completed', 'Closed'] },
+					date_completed: { $type: 'date' }
+				}
+			},
+			{ $sort: { date_completed: -1 } },
+			{ $limit: 20 },
+			{ $unwind: { path: '$chemicals', preserveNullAndEmptyArrays: true } },
+			{
+				$project: {
+					_id: 1,
+					wo_string: 1,
+					date_completed: 1,
+					customer: 1,
+					product: '$chemicals.chemical',
+					delivered_quantity: { $ifNull: ['$chemicals.delivered.quantity', 0] },
+					returned_quantity: { $ifNull: ['$chemicals.returned.quantity', 0] },
+					unit: '$chemicals.unit'
+				}
+			},
+			{ $sort: { date_completed: -1, wo_string: -1 } }
+			]).toArray()
+	},
+	{
+		id: 'support_ticket_billable_hours_per_user_per_month_last_6m',
+		prompt: 'Break up the support tickets last 6 months, give me a total per user per month of the amount of hours they are billing for',
+		minRows: 0,
+		query: { kind: 'aggregate', collection: 'support-tickets' },
+		notes: 'Zero rows are valid if support-ticket collection or ticket history is absent in this environment.',
+		run: async (ctx) => {
+			const collection = await resolveExistingCollection(ctx.db, ['support-tickets', 'supporttickets', 'support-ticket']);
+			if (!collection) {
+				return [];
+			}
+			return await ctx.db.collection(collection).aggregate([
+				{
+					$addFields: {
+						effective_date: { $ifNull: ['$date_created', '$createdAt'] },
+						effective_hours: { $ifNull: ['$billable_hours', { $ifNull: ['$estimated_billable_hours', 0] }] },
+						assigned_user_first: {
+							$let: {
+								vars: {
+									first_assigned: { $arrayElemAt: [{ $ifNull: ['$users_assigned', []] }, 0] }
+								},
+								in: {
+									$ifNull: ['$$first_assigned.user', '$$first_assigned.id_user']
+								}
+							}
+						}
+					}
+				},
+				{
+					$match: {
+						effective_date: { $gte: ctx.last6Months, $lt: ctx.now }
+					}
+				},
+				{
+					$group: {
+						_id: {
+							user: {
+								$ifNull: [
+									'$user_created',
+									{
+										$ifNull: [
+											'$client_user.user',
+											{
+												$ifNull: [
+													'$assigned_user_first',
+													{ $ifNull: [{ $toString: '$id_user_created' }, 'Unknown User'] }
+												]
+											}
+										]
+									}
+								]
+							},
+							month: { $dateToString: { format: '%Y-%m', date: '$effective_date', timezone: 'UTC' } }
+						},
+						billable_hours: { $sum: { $ifNull: ['$effective_hours', 0] } }
+					}
+				},
+				{
+					$project: {
+						_id: 0,
+						user: '$_id.user',
+						month: '$_id.month',
+						billable_hours: 1
+					}
+				},
+				{ $sort: { user: 1, month: 1 } }
+			]).toArray();
+		}
+	}
+];
+
+async function resolveTargetBlendChemicalIds(db: any): Promise<Array<string>> {
+	const rows = await db.collection('chemicals')
+		.find(
+			{ name: { $regex: 'Red\\s+Ultra\\s+Lift', $options: 'i' } },
+			{ projection: { _id: 1 }, limit: 20 }
+		)
+		.toArray();
+	return rows.map((row: any) => String(row?._id || '')).filter(Boolean);
+}
+
+async function run(): Promise<void> {
+	const mongoUrl = String(
+		process.env.AI_ASSISTANT_SNF_EVAL_MONGO_URL
+		|| process.env.MONGO_URL
+		|| 'mongodb://127.0.0.1:3001/?directConnection=true&serverSelectionTimeoutMS=5000'
+	).trim();
+	const databaseName = String(
+		process.env.AI_ASSISTANT_SNF_EVAL_DATABASE
+		|| process.env.DATABASE
+		|| 'resolveio'
+	).trim();
+	const client = new MongoClient(mongoUrl);
+	await client.connect();
+	const db = client.db(databaseName);
+	const now = new Date();
+
+	const thisMonthStart = startOfMonthUtc(now);
+	const nextMonthStart = startOfMonthUtc(addMonthsUtc(thisMonthStart, 1));
+	const lastMonthStart = startOfMonthUtc(addMonthsUtc(thisMonthStart, -1));
+	const lastMonthEnd = thisMonthStart;
+	const context: EvalContext = {
+		db,
+		now,
+		startOfWeek: startOfWeekUtcMonday(now),
+		last30Days: addDaysUtc(now, -30),
+		last90Days: addDaysUtc(now, -90),
+		last2Weeks: addDaysUtc(now, -14),
+		last6Months: addMonthsUtc(now, -6),
+		thisMonthStart,
+		nextMonthStart,
+		lastMonthStart,
+		lastMonthEnd,
+		targetBlendChemicalIds: await resolveTargetBlendChemicalIds(db)
+	};
+
+	const caseFilter = String(process.env.AI_ASSISTANT_SNF_EVAL_CASES || '')
+		.split(',')
+		.map(value => value.trim())
+		.filter(Boolean);
+	const selectedCases = caseFilter.length
+		? EVAL_CASES.filter(testCase => caseFilter.includes(testCase.id))
+		: EVAL_CASES;
+	if (!selectedCases.length) {
+		throw new Error('No matching SNF eval cases selected via AI_ASSISTANT_SNF_EVAL_CASES.');
+	}
+
+	const results: EvalResult[] = [];
+
+	for (const testCase of selectedCases) {
+		const classification = classifyAssistantRequestType(testCase.prompt);
+		const classificationPass = classification.type === 'data' && classification.dataQuestion === true;
+		try {
+			const rows = await testCase.run(context);
+			const rowCount = Array.isArray(rows) ? rows.length : 0;
+			const pass = classificationPass && rowCount >= testCase.minRows;
+			results.push({
+				id: testCase.id,
+				prompt: testCase.prompt,
+				classification: {
+					type: classification.type,
+					dataQuestion: classification.dataQuestion,
+					source: classification.source,
+					pass: classificationPass
+				},
+				query: testCase.query,
+				rowCount,
+				minRows: testCase.minRows,
+				pass,
+				notes: testCase.notes,
+				sample: rows.slice(0, 5)
+			});
+		}
+		catch (error: any) {
+			results.push({
+				id: testCase.id,
+				prompt: testCase.prompt,
+				classification: {
+					type: classification.type,
+					dataQuestion: classification.dataQuestion,
+					source: classification.source,
+					pass: classificationPass
+				},
+				query: testCase.query,
+				rowCount: 0,
+				minRows: testCase.minRows,
+				pass: false,
+				notes: testCase.notes,
+				sample: [],
+				error: String(error?.message || error || 'Unknown error')
+			});
+		}
+	}
+
+	await client.close(true);
+
+	const passCount = results.filter(result => result.pass).length;
+	const failCount = selectedCases.length - passCount;
+	const summary = {
+		total: selectedCases.length,
+		passed: passCount,
+		failed: failCount
+	};
+
+	const output = {
+		now: now.toISOString(),
+		database: databaseName,
+		window_utc: {
+			startOfWeek: context.startOfWeek.toISOString(),
+			last30Days: context.last30Days.toISOString(),
+			last90Days: context.last90Days.toISOString(),
+			last2Weeks: context.last2Weeks.toISOString(),
+			last6Months: context.last6Months.toISOString(),
+			thisMonthStart: context.thisMonthStart.toISOString(),
+			nextMonthStart: context.nextMonthStart.toISOString(),
+			lastMonthStart: context.lastMonthStart.toISOString(),
+			lastMonthEnd: context.lastMonthEnd.toISOString()
+		},
+		targetBlendChemicalIds: context.targetBlendChemicalIds,
+		summary,
+		results
+	};
+
+	const outputPath = path.join(process.cwd(), 'tmp', 'ai-assistant-snf-live-eval-output.json');
+	fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
+
+	console.log(`ai assistant snf live eval: ${passCount}/${selectedCases.length} passed`);
+	results.forEach((result) => {
+		const marker = result.pass ? 'PASS' : 'FAIL';
+		const classText = `${result.classification.type}/${result.classification.dataQuestion ? 'data' : 'non-data'}`;
+		console.log(`${marker} ${result.id} rows=${result.rowCount} class=${classText} query=${result.query.kind}:${result.query.collection}`);
+		if (result.error) {
+			console.log(`  error=${result.error}`);
+		}
+		if (result.notes) {
+			console.log(`  note=${result.notes}`);
+		}
+	});
+	console.log(`wrote ${outputPath}`);
+
+	if (failCount > 0) {
+		process.exit(1);
+	}
+	process.exit(0);
+}
+
+run().catch((error) => {
+	console.error(error);
+	process.exit(1);
+});