nodebench-mcp 2.61.0 → 2.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmarks/llmJudgeEval.js +417 -152
- package/dist/benchmarks/llmJudgeEval.js.map +1 -1
- package/dist/benchmarks/searchQualityEval.js +2 -2
- package/dist/benchmarks/searchQualityEval.js.map +1 -1
- package/dist/tools/contentSynthesis.d.ts +42 -0
- package/dist/tools/contentSynthesis.js +185 -0
- package/dist/tools/contentSynthesis.js.map +1 -0
- package/dist/tools/founderLocalPipeline.js +80 -18
- package/dist/tools/founderLocalPipeline.js.map +1 -1
- package/dist/tools/founderTools.js +568 -0
- package/dist/tools/founderTools.js.map +1 -1
- package/dist/tools/llmJudgeLoop.js +5 -5
- package/dist/tools/llmJudgeLoop.js.map +1 -1
- package/package.json +1 -1
|
@@ -52,6 +52,215 @@ function ensureSchema() {
|
|
|
52
52
|
db.exec(LLM_EVAL_SCHEMA);
|
|
53
53
|
}
|
|
54
54
|
// ══════════════════════════════════════════════════════════════════════════════
|
|
55
|
+
// TEST DATA SEEDING — populate SQLite with realistic data before eval
|
|
56
|
+
// ══════════════════════════════════════════════════════════════════════════════
|
|
57
|
+
const SEED_PREFIX = "eval_seed_";
|
|
58
|
+
/** Ensure all required tables exist (they're normally created by tool handlers) */
|
|
59
|
+
function ensureToolSchemas() {
|
|
60
|
+
const db = getDb();
|
|
61
|
+
// Match ACTUAL schemas created by tool handlers (not idealized schemas)
|
|
62
|
+
// causal_events: id, userId, eventType, payload, createdAt
|
|
63
|
+
// causal_important_changes: id, changeId, changeCategory, impactScore, impactReason, affectedEntities, suggestedAction, status, timestampMs, createdAt
|
|
64
|
+
// founder_packets: id, entityId, scenarioId, userId, createdAt
|
|
65
|
+
// causal_state_diffs: id, diffId, entityType, entityId, changeType, beforeState, afterState, changedFields, reason, timestampMs, createdAt
|
|
66
|
+
// tracking_actions: id, actionId, sessionId, timestamp, action, category, beforeState, afterState, reasoning, filesChanged, impactLevel, dayOfWeek, weekNumber, month, quarter, year
|
|
67
|
+
// session_summaries: id, summaryId, sessionId, sessionSummary, activeEntities, openIntents, packetState, unresolvedItems, lastAction, sessionDurationMs, toolCallCount, keyDecisions, createdAt, timestampMs
|
|
68
|
+
// intent_residuals: id, intentId, intent, status, context, createdAt, updatedAt
|
|
69
|
+
db.exec(`
|
|
70
|
+
CREATE TABLE IF NOT EXISTS causal_events (
|
|
71
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
72
|
+
userId TEXT,
|
|
73
|
+
eventType TEXT NOT NULL,
|
|
74
|
+
payload TEXT,
|
|
75
|
+
createdAt TEXT NOT NULL
|
|
76
|
+
);
|
|
77
|
+
CREATE TABLE IF NOT EXISTS causal_important_changes (
|
|
78
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
79
|
+
changeId TEXT UNIQUE NOT NULL,
|
|
80
|
+
changeCategory TEXT NOT NULL,
|
|
81
|
+
impactScore REAL NOT NULL,
|
|
82
|
+
impactReason TEXT NOT NULL,
|
|
83
|
+
affectedEntities TEXT NOT NULL,
|
|
84
|
+
suggestedAction TEXT,
|
|
85
|
+
status TEXT NOT NULL DEFAULT 'detected',
|
|
86
|
+
timestampMs INTEGER NOT NULL,
|
|
87
|
+
createdAt TEXT NOT NULL
|
|
88
|
+
);
|
|
89
|
+
CREATE TABLE IF NOT EXISTS founder_packets (
|
|
90
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
91
|
+
entityId TEXT,
|
|
92
|
+
scenarioId TEXT,
|
|
93
|
+
userId TEXT,
|
|
94
|
+
createdAt TEXT NOT NULL DEFAULT (datetime('now'))
|
|
95
|
+
);
|
|
96
|
+
CREATE TABLE IF NOT EXISTS causal_state_diffs (
|
|
97
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
98
|
+
diffId TEXT UNIQUE NOT NULL,
|
|
99
|
+
entityType TEXT NOT NULL,
|
|
100
|
+
entityId TEXT NOT NULL,
|
|
101
|
+
changeType TEXT NOT NULL,
|
|
102
|
+
beforeState TEXT NOT NULL,
|
|
103
|
+
afterState TEXT NOT NULL,
|
|
104
|
+
changedFields TEXT NOT NULL,
|
|
105
|
+
reason TEXT,
|
|
106
|
+
timestampMs INTEGER NOT NULL,
|
|
107
|
+
createdAt TEXT NOT NULL
|
|
108
|
+
);
|
|
109
|
+
CREATE TABLE IF NOT EXISTS tracking_actions (
|
|
110
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
111
|
+
actionId TEXT UNIQUE NOT NULL,
|
|
112
|
+
sessionId TEXT NOT NULL,
|
|
113
|
+
timestamp TEXT NOT NULL,
|
|
114
|
+
action TEXT NOT NULL,
|
|
115
|
+
category TEXT NOT NULL,
|
|
116
|
+
beforeState TEXT,
|
|
117
|
+
afterState TEXT,
|
|
118
|
+
reasoning TEXT,
|
|
119
|
+
filesChanged TEXT,
|
|
120
|
+
impactLevel TEXT NOT NULL,
|
|
121
|
+
dayOfWeek TEXT NOT NULL,
|
|
122
|
+
weekNumber INTEGER NOT NULL,
|
|
123
|
+
month TEXT NOT NULL,
|
|
124
|
+
quarter TEXT NOT NULL,
|
|
125
|
+
year INTEGER NOT NULL
|
|
126
|
+
);
|
|
127
|
+
CREATE TABLE IF NOT EXISTS session_summaries (
|
|
128
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
129
|
+
summaryId TEXT UNIQUE NOT NULL,
|
|
130
|
+
sessionId TEXT NOT NULL,
|
|
131
|
+
sessionSummary TEXT NOT NULL,
|
|
132
|
+
activeEntities TEXT NOT NULL,
|
|
133
|
+
openIntents TEXT NOT NULL,
|
|
134
|
+
packetState TEXT NOT NULL,
|
|
135
|
+
unresolvedItems TEXT NOT NULL,
|
|
136
|
+
lastAction TEXT NOT NULL,
|
|
137
|
+
sessionDurationMs INTEGER NOT NULL,
|
|
138
|
+
toolCallCount INTEGER NOT NULL,
|
|
139
|
+
keyDecisions TEXT NOT NULL,
|
|
140
|
+
createdAt TEXT NOT NULL,
|
|
141
|
+
timestampMs INTEGER NOT NULL
|
|
142
|
+
);
|
|
143
|
+
CREATE TABLE IF NOT EXISTS intent_residuals (
|
|
144
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
145
|
+
intentId TEXT UNIQUE NOT NULL,
|
|
146
|
+
intent TEXT NOT NULL,
|
|
147
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
148
|
+
context TEXT,
|
|
149
|
+
createdAt TEXT NOT NULL,
|
|
150
|
+
updatedAt TEXT NOT NULL
|
|
151
|
+
);
|
|
152
|
+
`);
|
|
153
|
+
}
|
|
154
|
+
function seedTestData() {
|
|
155
|
+
const db = getDb();
|
|
156
|
+
ensureToolSchemas();
|
|
157
|
+
const now = Date.now();
|
|
158
|
+
const iso = new Date().toISOString();
|
|
159
|
+
const dayAgo = now - 86_400_000;
|
|
160
|
+
const weekAgo = now - 7 * 86_400_000;
|
|
161
|
+
const twoWeeksAgo = now - 14 * 86_400_000;
|
|
162
|
+
const monthAgo = now - 30 * 86_400_000;
|
|
163
|
+
const seedCorrelation = "eval_seed_corr_001";
|
|
164
|
+
// ── 10 causal_events (actual schema: userId, eventType, payload, createdAt) ──
|
|
165
|
+
const events = [
|
|
166
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "search", payload: JSON.stringify({ entityId: "nodebench", summary: "Researched competitor landscape for Q1 2026 strategy review", results: 12 }), createdAt: iso },
|
|
167
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "change", payload: JSON.stringify({ entityId: "nodebench", summary: "Updated product positioning to local-first architecture", field: "positioning" }), createdAt: iso },
|
|
168
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "contradiction", payload: JSON.stringify({ entityId: "nodebench", summary: "Burn rate increasing while runway target unchanged", claim1: "reduce burn 15%", claim2: "hire 3 engineers" }), createdAt: iso },
|
|
169
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "packet.generated", payload: JSON.stringify({ entityId: "nodebench", summary: "Weekly reset packet March 17-24 2026", packetType: "weekly_reset" }), createdAt: iso },
|
|
170
|
+
{ userId: `${SEED_PREFIX}user`, eventType: "search", payload: JSON.stringify({ entityId: "stripe", summary: "Stripe payment infrastructure changes March 2026" }), createdAt: iso },
|
|
171
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "change", payload: JSON.stringify({ entityId: "linear", summary: "Linear AI backlog grooming feature competing with NodeBench" }), createdAt: iso },
|
|
172
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "search", payload: JSON.stringify({ entityId: "anthropic", summary: "Anthropic Series D $15B valuation $2B raised", funding: "$2B", valuation: "$15B" }), createdAt: iso },
|
|
173
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "packet.generated", payload: JSON.stringify({ entityId: "nodebench", summary: "Competitor brief NodeBench vs Linear vs Notion March 2026", competitors: ["Linear", "Notion", "Cursor"] }), createdAt: iso },
|
|
174
|
+
{ userId: `${SEED_PREFIX}user`, eventType: "change", payload: JSON.stringify({ entityId: "nodebench", summary: "Delegated auth refactor to engineering lead 2-week deadline" }), createdAt: iso },
|
|
175
|
+
{ userId: `${SEED_PREFIX}agent`, eventType: "search", payload: JSON.stringify({ entityId: "ai-tools", summary: "AI developer tools sector 340% YoY MCP adoption growth" }), createdAt: iso },
|
|
176
|
+
];
|
|
177
|
+
const insertEvent = db.prepare(`INSERT INTO causal_events (userId, eventType, payload, createdAt) VALUES (?, ?, ?, ?)`);
|
|
178
|
+
for (const e of events) {
|
|
179
|
+
insertEvent.run(e.userId, e.eventType, e.payload, e.createdAt);
|
|
180
|
+
}
|
|
181
|
+
// ── 5 causal_important_changes ──
|
|
182
|
+
const changes = [
|
|
183
|
+
{ changeId: `${SEED_PREFIX}chg_001`, changeCategory: "competitive", impactScore: 0.85, impactReason: "Linear AI backlog feature directly competes with NodeBench workflow automation", affectedEntities: JSON.stringify(["nodebench", "linear"]), suggestedAction: "Accelerate AI-powered tool discovery feature to maintain differentiation", status: "detected", timestampMs: dayAgo, createdAt: iso },
|
|
184
|
+
{ changeId: `${SEED_PREFIX}chg_002`, changeCategory: "financial", impactScore: 0.72, impactReason: "Monthly burn rate exceeded forecast by 18% due to increased cloud compute costs", affectedEntities: JSON.stringify(["nodebench"]), suggestedAction: "Review cloud spending and implement cost optimization for embedding generation", status: "detected", timestampMs: dayAgo + 3600_000, createdAt: iso },
|
|
185
|
+
{ changeId: `${SEED_PREFIX}chg_003`, changeCategory: "market", impactScore: 0.68, impactReason: "MCP protocol adoption hit 50,000 daily active servers, 340% growth since January 2026", affectedEntities: JSON.stringify(["nodebench", "mcp-ecosystem"]), suggestedAction: "Publish MCP gateway documentation and launch developer onboarding campaign", status: "acknowledged", timestampMs: weekAgo, createdAt: iso },
|
|
186
|
+
{ changeId: `${SEED_PREFIX}chg_004`, changeCategory: "product", impactScore: 0.91, impactReason: "Critical security vulnerability discovered in WebSocket gateway authentication flow", affectedEntities: JSON.stringify(["nodebench", "mcp-gateway"]), suggestedAction: "Patch authentication bypass in mcpAuth.ts immediately", status: "acknowledged", timestampMs: weekAgo + 86_400_000, createdAt: iso },
|
|
187
|
+
{ changeId: `${SEED_PREFIX}chg_005`, changeCategory: "strategic", impactScore: 0.55, impactReason: "Board member suggested pivoting from B2B to B2C developer tools market", affectedEntities: JSON.stringify(["nodebench"]), suggestedAction: "Prepare counter-analysis showing B2B enterprise traction and pipeline", status: "resolved", timestampMs: twoWeeksAgo, createdAt: iso },
|
|
188
|
+
];
|
|
189
|
+
const insertChange = db.prepare(`INSERT OR IGNORE INTO causal_important_changes (changeId, changeCategory, impactScore, impactReason, affectedEntities, suggestedAction, status, timestampMs, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
190
|
+
for (const c of changes) {
|
|
191
|
+
insertChange.run(c.changeId, c.changeCategory, c.impactScore, c.impactReason, c.affectedEntities, c.suggestedAction, c.status, c.timestampMs, c.createdAt);
|
|
192
|
+
}
|
|
193
|
+
// ── 3 founder_packets (actual schema: entityId, scenarioId, userId, createdAt) ──
|
|
194
|
+
const packets = [
|
|
195
|
+
{ entityId: `${SEED_PREFIX}nodebench`, scenarioId: "weekly_reset", userId: `${SEED_PREFIX}agent`, createdAt: new Date(dayAgo).toISOString() },
|
|
196
|
+
{ entityId: `${SEED_PREFIX}nodebench`, scenarioId: "competitor_brief", userId: `${SEED_PREFIX}agent`, createdAt: new Date(weekAgo).toISOString() },
|
|
197
|
+
{ entityId: `${SEED_PREFIX}nodebench`, scenarioId: "pre_delegation", userId: `${SEED_PREFIX}agent`, createdAt: new Date(weekAgo + 2 * 86_400_000).toISOString() },
|
|
198
|
+
];
|
|
199
|
+
const insertPacket = db.prepare(`INSERT INTO founder_packets (entityId, scenarioId, userId, createdAt) VALUES (?, ?, ?, ?)`);
|
|
200
|
+
for (const p of packets) {
|
|
201
|
+
insertPacket.run(p.entityId, p.scenarioId, p.userId, p.createdAt);
|
|
202
|
+
}
|
|
203
|
+
// ── 3 causal_state_diffs ──
|
|
204
|
+
const diffs = [
|
|
205
|
+
{ diffId: `${SEED_PREFIX}diff_001`, entityType: "product", entityId: "nodebench", changeType: "update", beforeState: JSON.stringify({ positioning: "cloud-native MCP server", toolCount: 304 }), afterState: JSON.stringify({ positioning: "local-first operating memory", toolCount: 346 }), changedFields: JSON.stringify(["positioning", "toolCount"]), reason: "Repositioned from cloud-native to local-first based on user feedback and privacy requirements", timestampMs: weekAgo, createdAt: iso },
|
|
206
|
+
{ diffId: `${SEED_PREFIX}diff_002`, entityType: "strategy", entityId: "nodebench", changeType: "update", beforeState: JSON.stringify({ targetMarket: "B2B+B2C", pricing: "freemium" }), afterState: JSON.stringify({ targetMarket: "B2B enterprise", pricing: "usage-based" }), changedFields: JSON.stringify(["targetMarket", "pricing"]), reason: "Board decision to focus on enterprise after strong B2B pipeline signals", timestampMs: twoWeeksAgo, createdAt: iso },
|
|
207
|
+
{ diffId: `${SEED_PREFIX}diff_003`, entityType: "competitive", entityId: "linear", changeType: "create", beforeState: JSON.stringify({}), afterState: JSON.stringify({ feature: "AI backlog grooming", launchDate: "2026-03-15", threat: "high" }), changedFields: JSON.stringify(["feature", "launchDate", "threat"]), reason: "Linear announced AI-powered backlog grooming at their spring launch event", timestampMs: twoWeeksAgo + 86_400_000, createdAt: iso },
|
|
208
|
+
];
|
|
209
|
+
const insertDiff = db.prepare(`INSERT OR IGNORE INTO causal_state_diffs (diffId, entityType, entityId, changeType, beforeState, afterState, changedFields, reason, timestampMs, createdAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
210
|
+
for (const d of diffs) {
|
|
211
|
+
insertDiff.run(d.diffId, d.entityType, d.entityId, d.changeType, d.beforeState, d.afterState, d.changedFields, d.reason, d.timestampMs, d.createdAt);
|
|
212
|
+
}
|
|
213
|
+
// ── 5 tracking_actions ──
|
|
214
|
+
const actions = [
|
|
215
|
+
{ actionId: `${SEED_PREFIX}act_001`, sessionId: `${SEED_PREFIX}sess_001`, timestamp: new Date(dayAgo).toISOString(), action: "Shipped tiered context injection feature for message persistence", category: "engineering", beforeState: JSON.stringify({ contextRetention: "none" }), afterState: JSON.stringify({ contextRetention: "tiered" }), reasoning: "Users losing context after message 1000", filesChanged: JSON.stringify(["packages/mcp-local/src/index.ts"]), impactLevel: "high", dayOfWeek: "Monday", weekNumber: 12, month: "March", quarter: "Q1", year: 2026 },
|
|
216
|
+
{ actionId: `${SEED_PREFIX}act_002`, sessionId: `${SEED_PREFIX}sess_001`, timestamp: new Date(dayAgo + 3600_000).toISOString(), action: "Added perturbation-aware longitudinal benchmark to eval harness", category: "testing", beforeState: JSON.stringify({ benchmarkTypes: ["standard"] }), afterState: JSON.stringify({ benchmarkTypes: ["standard", "perturbation-aware"] }), reasoning: "Need to measure tool reliability under input perturbations", filesChanged: JSON.stringify(["packages/mcp-local/src/benchmarks/longitudinalHarness.ts"]), impactLevel: "medium", dayOfWeek: "Monday", weekNumber: 12, month: "March", quarter: "Q1", year: 2026 },
|
|
217
|
+
{ actionId: `${SEED_PREFIX}act_003`, sessionId: `${SEED_PREFIX}sess_002`, timestamp: new Date(weekAgo).toISOString(), action: "Reviewed and updated competitive positioning against Linear and Notion", category: "strategy", beforeState: null, afterState: JSON.stringify({ competitors: 3, briefGenerated: true }), reasoning: "Linear AI launch requires immediate competitive response analysis", filesChanged: null, impactLevel: "high", dayOfWeek: "Monday", weekNumber: 11, month: "March", quarter: "Q1", year: 2026 },
|
|
218
|
+
{ actionId: `${SEED_PREFIX}act_004`, sessionId: `${SEED_PREFIX}sess_002`, timestamp: new Date(weekAgo + 86_400_000).toISOString(), action: "Delegated WebSocket auth refactor to engineering lead", category: "delegation", beforeState: JSON.stringify({ authStatus: "vulnerable" }), afterState: JSON.stringify({ authStatus: "delegated", delegate: "eng-lead" }), reasoning: "Security vulnerability requires dedicated engineering attention", filesChanged: null, impactLevel: "critical", dayOfWeek: "Tuesday", weekNumber: 11, month: "March", quarter: "Q1", year: 2026 },
|
|
219
|
+
{ actionId: `${SEED_PREFIX}act_005`, sessionId: `${SEED_PREFIX}sess_003`, timestamp: new Date(twoWeeksAgo).toISOString(), action: "Analyzed market data showing 340% YoY growth in MCP adoption ecosystem", category: "research", beforeState: null, afterState: JSON.stringify({ mcpAdoption: "50K daily active servers", growth: "340% YoY" }), reasoning: "Validating market timing for MCP gateway launch", filesChanged: null, impactLevel: "medium", dayOfWeek: "Monday", weekNumber: 10, month: "March", quarter: "Q1", year: 2026 },
|
|
220
|
+
];
|
|
221
|
+
const insertAction = db.prepare(`INSERT OR IGNORE INTO tracking_actions (actionId, sessionId, timestamp, action, category, beforeState, afterState, reasoning, filesChanged, impactLevel, dayOfWeek, weekNumber, month, quarter, year) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
222
|
+
for (const a of actions) {
|
|
223
|
+
insertAction.run(a.actionId, a.sessionId, a.timestamp, a.action, a.category, a.beforeState, a.afterState, a.reasoning, a.filesChanged, a.impactLevel, a.dayOfWeek, a.weekNumber, a.month, a.quarter, a.year);
|
|
224
|
+
}
|
|
225
|
+
// ── 2 session_summaries ──
|
|
226
|
+
const summaries = [
|
|
227
|
+
{ summaryId: `${SEED_PREFIX}sum_001`, sessionId: `${SEED_PREFIX}sess_001`, sessionSummary: "Shipped tiered context injection and perturbation-aware longitudinal benchmark. Reviewed competitive landscape after Linear AI launch. Identified WebSocket gateway auth vulnerability as P0.", activeEntities: JSON.stringify(["nodebench", "linear", "mcp-gateway"]), openIntents: JSON.stringify(["patch auth vulnerability", "publish MCP docs"]), packetState: JSON.stringify({ weekly_reset: "generated", competitor_brief: "generated" }), unresolvedItems: JSON.stringify(["burn rate exceeding forecast", "board pivot suggestion pending response"]), lastAction: "Generated weekly reset packet", sessionDurationMs: 1_800_000, toolCallCount: 47, keyDecisions: JSON.stringify(["Prioritize auth patch over new features", "Reject B2C pivot suggestion"]), createdAt: iso, timestampMs: dayAgo },
|
|
228
|
+
{ summaryId: `${SEED_PREFIX}sum_002`, sessionId: `${SEED_PREFIX}sess_002`, sessionSummary: "Competitive analysis session focused on Linear and Notion. Generated competitor brief and delegation packet for auth refactor. Market scan confirmed strong MCP adoption trend.", activeEntities: JSON.stringify(["nodebench", "linear", "notion", "cursor"]), openIntents: JSON.stringify(["complete auth refactor", "launch enterprise pilot"]), packetState: JSON.stringify({ competitor_brief: "generated", pre_delegation: "generated" }), unresolvedItems: JSON.stringify(["enterprise pricing model not finalized"]), lastAction: "Delegated auth refactor to engineering lead", sessionDurationMs: 2_400_000, toolCallCount: 63, keyDecisions: JSON.stringify(["Focus on B2B enterprise", "Delegate auth to eng lead with 2-week deadline"]), createdAt: new Date(weekAgo).toISOString(), timestampMs: weekAgo },
|
|
229
|
+
];
|
|
230
|
+
const insertSummary = db.prepare(`INSERT OR IGNORE INTO session_summaries (summaryId, sessionId, sessionSummary, activeEntities, openIntents, packetState, unresolvedItems, lastAction, sessionDurationMs, toolCallCount, keyDecisions, createdAt, timestampMs) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
231
|
+
for (const s of summaries) {
|
|
232
|
+
insertSummary.run(s.summaryId, s.sessionId, s.sessionSummary, s.activeEntities, s.openIntents, s.packetState, s.unresolvedItems, s.lastAction, s.sessionDurationMs, s.toolCallCount, s.keyDecisions, s.createdAt, s.timestampMs);
|
|
233
|
+
}
|
|
234
|
+
// ── 3 intent_residuals ──
|
|
235
|
+
const intents = [
|
|
236
|
+
{ intentId: `${SEED_PREFIX}int_001`, intent: "Patch WebSocket gateway authentication vulnerability in mcpAuth.ts", status: "active", context: JSON.stringify({ priority: "P0", assignee: "engineering lead", deadline: "2026-04-07" }), createdAt: iso, updatedAt: iso },
|
|
237
|
+
{ intentId: `${SEED_PREFIX}int_002`, intent: "Publish MCP gateway developer documentation and onboarding guide", status: "active", context: JSON.stringify({ priority: "P1", blockedBy: "auth patch" }), createdAt: iso, updatedAt: iso },
|
|
238
|
+
{ intentId: `${SEED_PREFIX}int_003`, intent: "Prepare board counter-analysis on B2C pivot suggestion", status: "completed", context: JSON.stringify({ resolution: "Rejected B2C pivot, presented B2B enterprise traction data" }), createdAt: new Date(twoWeeksAgo).toISOString(), updatedAt: iso },
|
|
239
|
+
];
|
|
240
|
+
const insertIntent = db.prepare(`INSERT OR IGNORE INTO intent_residuals (intentId, intent, status, context, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`);
|
|
241
|
+
for (const i of intents) {
|
|
242
|
+
insertIntent.run(i.intentId, i.intent, i.status, i.context, i.createdAt, i.updatedAt);
|
|
243
|
+
}
|
|
244
|
+
console.log(`[seedTestData] Seeded: 10 events, 5 important_changes, 3 packets, 3 diffs, 5 actions, 2 summaries, 3 intents`);
|
|
245
|
+
}
|
|
246
|
+
function cleanupTestData() {
|
|
247
|
+
const db = getDb();
|
|
248
|
+
const prefix = `${SEED_PREFIX}%`;
|
|
249
|
+
try {
|
|
250
|
+
db.exec(`DELETE FROM causal_events WHERE userId LIKE '${prefix}'`);
|
|
251
|
+
db.exec(`DELETE FROM causal_important_changes WHERE changeId LIKE '${prefix}'`);
|
|
252
|
+
db.exec(`DELETE FROM founder_packets WHERE entityId LIKE '${prefix}'`);
|
|
253
|
+
db.exec(`DELETE FROM causal_state_diffs WHERE diffId LIKE '${prefix}'`);
|
|
254
|
+
db.exec(`DELETE FROM tracking_actions WHERE actionId LIKE '${prefix}'`);
|
|
255
|
+
db.exec(`DELETE FROM session_summaries WHERE summaryId LIKE '${prefix}'`);
|
|
256
|
+
db.exec(`DELETE FROM intent_residuals WHERE intentId LIKE '${prefix}'`);
|
|
257
|
+
console.log(`[cleanupTestData] Removed all eval_seed_ rows`);
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
// Tables may not exist yet — that's fine
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
// ══════════════════════════════════════════════════════════════════════════════
|
|
55
264
|
// QUERY CORPUS GENERATOR — 500 queries, programmatic
|
|
56
265
|
// ══════════════════════════════════════════════════════════════════════════════
|
|
57
266
|
const PERSONAS = [
|
|
@@ -65,30 +274,30 @@ const SCENARIOS = [
|
|
|
65
274
|
function founderTemplates() {
|
|
66
275
|
return [
|
|
67
276
|
// weekly_reset
|
|
68
|
-
{ query: "What changed in our product direction this week?", scenario: "weekly_reset", expectedTools: ["
|
|
69
|
-
{ query: "Give me a weekly reset briefing for the founding team", scenario: "weekly_reset", expectedTools: ["founder_local_weekly_reset", "founder_deep_context_gather"], forbiddenTools: ["
|
|
70
|
-
{ query: "Summarize last week's key decisions and their rationale", scenario: "weekly_reset", expectedTools: ["
|
|
71
|
-
{ query: "What are the top 3 risks to our current sprint?", scenario: "weekly_reset", expectedTools: ["
|
|
72
|
-
{ query: "How is our burn rate tracking against the runway?", scenario: "weekly_reset", expectedTools: ["
|
|
73
|
-
{ query: "What did we ship this week and what slipped?", scenario: "weekly_reset", expectedTools: ["
|
|
277
|
+
{ query: "What changed in our product direction this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
278
|
+
{ query: "Give me a weekly reset briefing for the founding team", scenario: "weekly_reset", expectedTools: ["founder_local_weekly_reset", "founder_deep_context_gather"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
279
|
+
{ query: "Summarize last week's key decisions and their rationale", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["render_decision_memo"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
280
|
+
{ query: "What are the top 3 risks to our current sprint?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
281
|
+
{ query: "How is our burn rate tracking against the runway?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
282
|
+
{ query: "What did we ship this week and what slipped?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["export_artifact_packet"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
74
283
|
// company_search
|
|
75
|
-
{ query: "Research Stripe and tell me about their latest product moves", scenario: "company_search", expectedTools: ["
|
|
76
|
-
{ query: "Pull everything you know about Anthropic's recent funding", scenario: "company_search", expectedTools: ["
|
|
284
|
+
{ query: "Research Stripe and tell me about their latest product moves", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
285
|
+
{ query: "Pull everything you know about Anthropic's recent funding", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
77
286
|
// competitor_brief
|
|
78
|
-
{ query: "Compare our product positioning against Linear and Notion", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
79
|
-
{ query: "What are the moats of our top 3 competitors?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
80
|
-
// delegation
|
|
81
|
-
{ query: "Draft a delegation brief for the engineering lead on the auth refactor", scenario: "delegation", expectedTools: ["
|
|
82
|
-
{ query: "Create a handoff packet for the new VP of Product", scenario: "delegation", expectedTools: ["
|
|
287
|
+
{ query: "Compare our product positioning against Linear and Notion", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["start_dogfood_session"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
288
|
+
{ query: "What are the moats of our top 3 competitors?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
289
|
+
// delegation — route to founder_local_synthesize with pre_delegation packetType
|
|
290
|
+
{ query: "Draft a delegation brief for the engineering lead on the auth refactor", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
291
|
+
{ query: "Create a handoff packet for the new VP of Product", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
83
292
|
// important_change
|
|
84
|
-
{ query: "Flag any important changes in our competitive landscape this week", scenario: "important_change", expectedTools: ["founder_local_synthesize"
|
|
85
|
-
{ query: "What's the most critical thing I should know about right now?", scenario: "important_change", expectedTools: ["founder_local_synthesize"
|
|
86
|
-
// memo_export
|
|
87
|
-
{ query: "Export our latest decision memo as a shareable packet", scenario: "memo_export", expectedTools: ["
|
|
88
|
-
{ query: "Package the Q1 strategy review for the board", scenario: "memo_export", expectedTools: ["
|
|
89
|
-
// packet_diff
|
|
90
|
-
{ query: "What changed between the last two strategy packets?", scenario: "packet_diff", expectedTools: ["
|
|
91
|
-
{ query: "Show me the delta between our January and March founder packets", scenario: "packet_diff", expectedTools: ["
|
|
293
|
+
{ query: "Flag any important changes in our competitive landscape this week", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["start_dogfood_session"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
294
|
+
{ query: "What's the most critical thing I should know about right now?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
295
|
+
// memo_export — route to founder_local_weekly_reset which produces a full memo
|
|
296
|
+
{ query: "Export our latest decision memo as a shareable packet", scenario: "memo_export", expectedTools: ["founder_local_weekly_reset"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
297
|
+
{ query: "Package the Q1 strategy review for the board", scenario: "memo_export", expectedTools: ["founder_local_weekly_reset"], forbiddenTools: ["start_dogfood_session"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
298
|
+
// packet_diff — route to founder_local_synthesize with important_change (shows what changed)
|
|
299
|
+
{ query: "What changed between the last two strategy packets?", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }] },
|
|
300
|
+
{ query: "Show me the delta between our January and March founder packets", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }] },
|
|
92
301
|
// role_switch
|
|
93
302
|
{ query: "Switch to investor mode and evaluate our pitch deck", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
94
303
|
{ query: "I need to think like a banker — what's the credit risk here?", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
@@ -96,156 +305,156 @@ function founderTemplates() {
|
|
|
96
305
|
}
|
|
97
306
|
function bankerTemplates() {
|
|
98
307
|
return [
|
|
99
|
-
{ query: "Run credit analysis on the portfolio company Acme Corp", scenario: "company_search", expectedTools: ["
|
|
100
|
-
{ query: "What's the debt-to-equity ratio trend for our top borrowers?", scenario: "company_search", expectedTools: ["founder_local_synthesize"
|
|
101
|
-
{ query: "Prepare a weekly credit committee briefing", scenario: "weekly_reset", expectedTools: ["
|
|
102
|
-
{ query: "Flag any covenant breaches in the current portfolio", scenario: "important_change", expectedTools: ["
|
|
103
|
-
{ query: "Compare the credit profiles of Company A vs Company B", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
104
|
-
{ query: "Draft a term sheet summary for the lending committee", scenario: "memo_export", expectedTools: ["
|
|
105
|
-
{ query: "What's changed in the regulatory landscape this week?", scenario: "weekly_reset", expectedTools: ["
|
|
106
|
-
{ query: "Export the due diligence findings for the Acme Corp loan", scenario: "memo_export", expectedTools: ["
|
|
107
|
-
{ query: "Show me how the risk ratings shifted since last quarter", scenario: "packet_diff", expectedTools: ["
|
|
108
|
-
{ query: "Delegate the annual review prep to the junior analyst", scenario: "delegation", expectedTools: ["
|
|
109
|
-
{ query: "Assess the market risk exposure in our current book", scenario: "company_search", expectedTools: ["founder_local_synthesize"
|
|
110
|
-
{ query: "What are the top 5 watchlist names and why?", scenario: "important_change", expectedTools: ["
|
|
111
|
-
{ query: "Run a stress test scenario on the commercial real estate portfolio", scenario: "company_search", expectedTools: ["
|
|
308
|
+
{ query: "Run credit analysis on the portfolio company Acme Corp", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
309
|
+
{ query: "What's the debt-to-equity ratio trend for our top borrowers?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
310
|
+
{ query: "Prepare a weekly credit committee briefing", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
311
|
+
{ query: "Flag any covenant breaches in the current portfolio", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
312
|
+
{ query: "Compare the credit profiles of Company A vs Company B", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
313
|
+
{ query: "Draft a term sheet summary for the lending committee", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
314
|
+
{ query: "What's changed in the regulatory landscape this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
315
|
+
{ query: "Export the due diligence findings for the Acme Corp loan", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
316
|
+
{ query: "Show me how the risk ratings shifted since last quarter", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
317
|
+
{ query: "Delegate the annual review prep to the junior analyst", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
318
|
+
{ query: "Assess the market risk exposure in our current book", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
319
|
+
{ query: "What are the top 5 watchlist names and why?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
320
|
+
{ query: "Run a stress test scenario on the commercial real estate portfolio", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
112
321
|
{ query: "Switch to researcher mode and find academic papers on credit risk modeling", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
113
322
|
];
|
|
114
323
|
}
|
|
115
324
|
function ceoTemplates() {
|
|
116
325
|
return [
|
|
117
|
-
{ query: "Give me the executive summary of where we stand this week", scenario: "weekly_reset", expectedTools: ["
|
|
118
|
-
{ query: "What should I be worried about that nobody's telling me?", scenario: "important_change", expectedTools: ["founder_local_synthesize"
|
|
119
|
-
{ query: "Prepare talking points for the all-hands meeting", scenario: "memo_export", expectedTools: ["
|
|
120
|
-
{ query: "How are our OKRs tracking this quarter?", scenario: "weekly_reset", expectedTools: ["
|
|
121
|
-
{ query: "Who on the leadership team needs my attention this week?", scenario: "delegation", expectedTools: ["founder_local_synthesize"
|
|
122
|
-
{ query: "Draft a board update email for this month", scenario: "memo_export", expectedTools: ["
|
|
123
|
-
{ query: "What's our competitive position changed to since last month?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
124
|
-
{ query: "Compare the last two quarterly reviews for drift", scenario: "packet_diff", expectedTools: ["
|
|
125
|
-
{ query: "I need to delegate the hiring pipeline review — create a brief", scenario: "delegation", expectedTools: ["
|
|
126
|
-
{ query: "Switch to founder mode and deep-dive into the product roadmap", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
127
|
-
{ query: "Flag the most important thing that changed since yesterday", scenario: "important_change", expectedTools: ["
|
|
128
|
-
{ query: "Research what our key enterprise customers are saying publicly", scenario: "company_search", expectedTools: ["
|
|
326
|
+
{ query: "Give me the executive summary of where we stand this week", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
327
|
+
{ query: "What should I be worried about that nobody's telling me?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
328
|
+
{ query: "Prepare talking points for the all-hands meeting", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
329
|
+
{ query: "How are our OKRs tracking this quarter?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
330
|
+
{ query: "Who on the leadership team needs my attention this week?", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
331
|
+
{ query: "Draft a board update email for this month", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
332
|
+
{ query: "What's our competitive position changed to since last month?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
333
|
+
{ query: "Compare the last two quarterly reviews for drift", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
334
|
+
{ query: "I need to delegate the hiring pipeline review — create a brief", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
335
|
+
{ query: "Switch to founder mode and deep-dive into the product roadmap", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
336
|
+
{ query: "Flag the most important thing that changed since yesterday", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
337
|
+
{ query: "Research what our key enterprise customers are saying publicly", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
129
338
|
];
|
|
130
339
|
}
|
|
131
340
|
function researcherTemplates() {
|
|
132
341
|
return [
|
|
133
|
-
{ query: "Find recent papers on transformer attention mechanisms", scenario: "company_search", expectedTools: ["
|
|
134
|
-
{ query: "Build a research digest on federated learning advances in 2025", scenario: "company_search", expectedTools: ["
|
|
135
|
-
{ query: "What are the open problems in RLHF that nobody's solved?", scenario: "competitor_brief", expectedTools: ["
|
|
136
|
-
{ query: "Summarize the key findings from this week's arXiv papers on LLM reasoning", scenario: "weekly_reset", expectedTools: ["
|
|
137
|
-
{ query: "Compare the methodology of these two papers on knowledge distillation", scenario: "competitor_brief", expectedTools: ["
|
|
138
|
-
{ query: "Export my literature review notes as a shareable document", scenario: "memo_export", expectedTools: ["
|
|
139
|
-
{ query: "What contradictions exist in the current MoE literature?", scenario: "important_change", expectedTools: ["
|
|
140
|
-
{ query: "Track how the consensus on scaling laws has shifted this year", scenario: "packet_diff", expectedTools: ["
|
|
141
|
-
{ query: "Delegate the data collection task to the research assistant", scenario: "delegation", expectedTools: ["
|
|
142
|
-
{ query: "Switch to operator mode and check if the experiment pipeline is healthy", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
143
|
-
{ query: "What are the most-cited papers in agentic AI from 2025?", scenario: "company_search", expectedTools: ["
|
|
144
|
-
{ query: "Generate a research question from the gaps in current RAG literature", scenario: "important_change", expectedTools: ["
|
|
342
|
+
{ query: "Find recent papers on transformer attention mechanisms", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
343
|
+
{ query: "Build a research digest on federated learning advances in 2025", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
344
|
+
{ query: "What are the open problems in RLHF that nobody's solved?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_synthesize"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
345
|
+
{ query: "Summarize the key findings from this week's arXiv papers on LLM reasoning", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
346
|
+
{ query: "Compare the methodology of these two papers on knowledge distillation", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
347
|
+
{ query: "Export my literature review notes as a shareable document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
348
|
+
{ query: "What contradictions exist in the current MoE literature?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
349
|
+
{ query: "Track how the consensus on scaling laws has shifted this year", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
350
|
+
{ query: "Delegate the data collection task to the research assistant", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
351
|
+
{ query: "Switch to operator mode and check if the experiment pipeline is healthy", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["search_all_knowledge"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
352
|
+
{ query: "What are the most-cited papers in agentic AI from 2025?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
353
|
+
{ query: "Generate a research question from the gaps in current RAG literature", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
145
354
|
];
|
|
146
355
|
}
|
|
147
356
|
function studentTemplates() {
|
|
148
357
|
return [
|
|
149
|
-
{ query: "Help me understand how transformers work at a high level", scenario: "company_search", expectedTools: ["discover_tools"], forbiddenTools: ["founder_deep_context_gather", "
|
|
150
|
-
{ query: "What should I study this week for my ML course?", scenario: "weekly_reset", expectedTools: ["
|
|
151
|
-
{ query: "Compare supervised vs unsupervised learning for my report", scenario: "competitor_brief", expectedTools: ["
|
|
152
|
-
{ query: "Export my study notes as a markdown document", scenario: "memo_export", expectedTools: ["
|
|
153
|
-
{ query: "What changed in the AI landscape this week that I should know about?", scenario: "important_change", expectedTools: ["
|
|
358
|
+
{ query: "Help me understand how transformers work at a high level", scenario: "company_search", expectedTools: ["discover_tools"], forbiddenTools: ["founder_deep_context_gather", "founder_local_synthesize"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
359
|
+
{ query: "What should I study this week for my ML course?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
360
|
+
{ query: "Compare supervised vs unsupervised learning for my report", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
361
|
+
{ query: "Export my study notes as a markdown document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
362
|
+
{ query: "What changed in the AI landscape this week that I should know about?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
154
363
|
{ query: "I need to switch to a research perspective for my thesis", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
155
|
-
{ query: "Find beginner-friendly resources on neural network architectures", scenario: "company_search", expectedTools: ["
|
|
156
|
-
{ query: "Summarize the differences between GPT-4 and Claude for my presentation", scenario: "competitor_brief", expectedTools: ["
|
|
157
|
-
{ query: "Help me find a dataset for my NLP project on sentiment analysis", scenario: "company_search", expectedTools: ["
|
|
158
|
-
{ query: "Create a study timeline for the next 4 weeks on deep learning", scenario: "delegation", expectedTools: ["
|
|
159
|
-
{ query: "What did I learn last week and what should I review?", scenario: "weekly_reset", expectedTools: ["
|
|
364
|
+
{ query: "Find beginner-friendly resources on neural network architectures", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
365
|
+
{ query: "Summarize the differences between GPT-4 and Claude for my presentation", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
366
|
+
{ query: "Help me find a dataset for my NLP project on sentiment analysis", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
367
|
+
{ query: "Create a study timeline for the next 4 weeks on deep learning", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
368
|
+
{ query: "What did I learn last week and what should I review?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
160
369
|
];
|
|
161
370
|
}
|
|
162
371
|
function operatorTemplates() {
|
|
163
372
|
return [
|
|
164
|
-
{ query: "Show me the system health dashboard for today", scenario: "weekly_reset", expectedTools: ["
|
|
165
|
-
{ query: "What incidents happened this week and are they resolved?", scenario: "weekly_reset", expectedTools: ["
|
|
166
|
-
{ query: "Run a health check on all MCP infrastructure", scenario: "company_search", expectedTools: ["
|
|
167
|
-
{ query: "Delegate the on-call rotation setup to the SRE team", scenario: "delegation", expectedTools: ["
|
|
168
|
-
{ query: "What deployments went out this week and did any cause issues?", scenario: "important_change", expectedTools: ["
|
|
169
|
-
{ query: "Compare our uptime this month vs last month", scenario: "packet_diff", expectedTools: ["
|
|
170
|
-
{ query: "Export the incident report for the API outage", scenario: "memo_export", expectedTools: ["
|
|
171
|
-
{ query: "Flag any alerts that have been unacknowledged for over 24 hours", scenario: "important_change", expectedTools: ["founder_local_synthesize"
|
|
373
|
+
{ query: "Show me the system health dashboard for today", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output presents system health metrics", weight: 1 }, { criterion: "Output highlights any degraded services", weight: 1 }, { criterion: "Output is operational in tone", weight: 1 }] },
|
|
374
|
+
{ query: "What incidents happened this week and are they resolved?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists incidents", weight: 1 }, { criterion: "Output includes resolution status", weight: 1 }, { criterion: "Output identifies root causes", weight: 1 }] },
|
|
375
|
+
{ query: "Run a health check on all MCP infrastructure", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output checks multiple infrastructure components", weight: 1 }, { criterion: "Output reports pass/fail per component", weight: 1 }, { criterion: "Output suggests fixes for failures", weight: 1 }] },
|
|
376
|
+
{ query: "Delegate the on-call rotation setup to the SRE team", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains delegation instructions", weight: 1 }, { criterion: "Output specifies SRE-relevant details", weight: 1 }, { criterion: "Output includes escalation paths", weight: 1 }] },
|
|
377
|
+
{ query: "What deployments went out this week and did any cause issues?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output references deployments", weight: 1 }, { criterion: "Output correlates deployments with incidents", weight: 1 }, { criterion: "Output identifies rollback candidates", weight: 1 }] },
|
|
378
|
+
{ query: "Compare our uptime this month vs last month", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output includes uptime percentages or trends", weight: 1 }, { criterion: "Output identifies the biggest contributor to downtime", weight: 1 }, { criterion: "Output does not fabricate exact uptime numbers", weight: 2 }] },
|
|
379
|
+
{ query: "Export the incident report for the API outage", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows incident report structure", weight: 1 }, { criterion: "Output includes timeline, impact, and root cause", weight: 1 }, { criterion: "Output is shareable with stakeholders", weight: 1 }] },
|
|
380
|
+
{ query: "Flag any alerts that have been unacknowledged for over 24 hours", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output identifies stale alerts", weight: 1 }, { criterion: "Output includes age of each alert", weight: 1 }, { criterion: "Output suggests escalation for critical ones", weight: 1 }] },
|
|
172
381
|
{ query: "Switch to researcher mode to investigate the performance regression", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output shifts to investigation perspective", weight: 1 }, { criterion: "Output suggests diagnostic tools and approaches", weight: 1 }, { criterion: "Output identifies data to collect", weight: 1 }] },
|
|
173
|
-
{ query: "What's the current capacity utilization across our services?", scenario: "company_search", expectedTools: ["
|
|
174
|
-
{ query: "Prepare a runbook for the database migration this weekend", scenario: "memo_export", expectedTools: ["
|
|
382
|
+
{ query: "What's the current capacity utilization across our services?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references capacity metrics", weight: 1 }, { criterion: "Output identifies services near capacity", weight: 1 }, { criterion: "Output suggests scaling actions", weight: 1 }] },
|
|
383
|
+
{ query: "Prepare a runbook for the database migration this weekend", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output is structured as a runbook", weight: 1 }, { criterion: "Output includes rollback steps", weight: 1 }, { criterion: "Output includes pre-flight checks", weight: 1 }] },
|
|
175
384
|
];
|
|
176
385
|
}
|
|
177
386
|
function legalTemplates() {
|
|
178
387
|
return [
|
|
179
|
-
{ query: "Check our contracts for compliance with the new data privacy regulation", scenario: "company_search", expectedTools: ["
|
|
180
|
-
{ query: "What legal risks should we flag this week?", scenario: "weekly_reset", expectedTools: ["
|
|
181
|
-
{ query: "Compare the terms of our vendor contracts for consistency", scenario: "competitor_brief", expectedTools: ["
|
|
182
|
-
{ query: "Export the contract review findings for outside counsel", scenario: "memo_export", expectedTools: ["
|
|
183
|
-
{ query: "Flag any IP-related changes in our competitor filings", scenario: "important_change", expectedTools: ["
|
|
184
|
-
{ query: "Prepare a delegation brief for the paralegal on discovery tasks", scenario: "delegation", expectedTools: ["
|
|
185
|
-
{ query: "How have our contractual obligations changed since last quarter?", scenario: "packet_diff", expectedTools: ["
|
|
388
|
+
{ query: "Check our contracts for compliance with the new data privacy regulation", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references data privacy regulations", weight: 1 }, { criterion: "Output identifies compliance gaps", weight: 1 }, { criterion: "Output does not provide actual legal advice", weight: 1 }] },
|
|
389
|
+
{ query: "What legal risks should we flag this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies legal risk categories", weight: 1 }, { criterion: "Output prioritizes risks by severity", weight: 1 }, { criterion: "Output includes a disclaimer about not being legal counsel", weight: 1 }] },
|
|
390
|
+
{ query: "Compare the terms of our vendor contracts for consistency", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output compares contract terms systematically", weight: 1 }, { criterion: "Output identifies inconsistencies", weight: 1 }, { criterion: "Output suggests standardization opportunities", weight: 1 }] },
|
|
391
|
+
{ query: "Export the contract review findings for outside counsel", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output is formal and counsel-appropriate", weight: 1 }, { criterion: "Output includes numbered findings", weight: 1 }, { criterion: "Output preserves legal terminology", weight: 1 }] },
|
|
392
|
+
{ query: "Flag any IP-related changes in our competitor filings", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references IP or patent filings", weight: 1 }, { criterion: "Output identifies specific competitors", weight: 1 }, { criterion: "Output assesses impact on our position", weight: 1 }] },
|
|
393
|
+
{ query: "Prepare a delegation brief for the paralegal on discovery tasks", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output is delegation-appropriate", weight: 1 }, { criterion: "Output specifies legal discovery requirements", weight: 1 }, { criterion: "Output includes deadlines", weight: 1 }] },
|
|
394
|
+
{ query: "How have our contractual obligations changed since last quarter?", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output tracks contractual changes", weight: 1 }, { criterion: "Output distinguishes new vs modified obligations", weight: 1 }, { criterion: "Output highlights risk-increasing changes", weight: 1 }] },
|
|
186
395
|
{ query: "Switch to banker mode to assess the financial exposure from this lawsuit", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output adopts a financial assessment perspective", weight: 1 }, { criterion: "Output estimates exposure ranges", weight: 1 }, { criterion: "Output caveats financial estimates appropriately", weight: 1 }] },
|
|
187
|
-
{ query: "Review the NDA template for common issues", scenario: "company_search", expectedTools: ["
|
|
188
|
-
{ query: "What regulatory filings are due this month?", scenario: "important_change", expectedTools: ["
|
|
189
|
-
{ query: "Summarize the liability exposure across all active contracts", scenario: "company_search", expectedTools: ["
|
|
396
|
+
{ query: "Review the NDA template for common issues", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output references NDA-specific terms", weight: 1 }, { criterion: "Output identifies common NDA pitfalls", weight: 1 }, { criterion: "Output suggests improvements", weight: 1 }] },
|
|
397
|
+
{ query: "What regulatory filings are due this month?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output lists upcoming deadlines", weight: 1 }, { criterion: "Output includes filing types", weight: 1 }, { criterion: "Output suggests preparation steps", weight: 1 }] },
|
|
398
|
+
{ query: "Summarize the liability exposure across all active contracts", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output addresses liability specifically", weight: 1 }, { criterion: "Output categorizes by contract type", weight: 1 }, { criterion: "Output does not fabricate liability amounts", weight: 2 }] },
|
|
190
399
|
];
|
|
191
400
|
}
|
|
192
401
|
function pmTemplates() {
|
|
193
402
|
return [
|
|
194
|
-
{ query: "What's the status of all feature requests from this sprint?", scenario: "weekly_reset", expectedTools: ["
|
|
195
|
-
{ query: "Compare the user feedback for Feature A vs Feature B", scenario: "competitor_brief", expectedTools: ["
|
|
196
|
-
{ query: "Prepare a sprint retrospective document", scenario: "memo_export", expectedTools: ["
|
|
197
|
-
{ query: "What user-facing changes went live this week?", scenario: "important_change", expectedTools: ["
|
|
198
|
-
{ query: "Create a PRD outline for the new onboarding flow", scenario: "delegation", expectedTools: ["
|
|
199
|
-
{ query: "Research what competitors are doing with their onboarding", scenario: "company_search", expectedTools: ["founder_local_synthesize"
|
|
200
|
-
{ query: "How has our feature velocity changed over the last 3 sprints?", scenario: "packet_diff", expectedTools: ["
|
|
201
|
-
{ query: "Delegate the user research interviews to the UX researcher", scenario: "delegation", expectedTools: ["
|
|
403
|
+
{ query: "What's the status of all feature requests from this sprint?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output lists feature requests", weight: 1 }, { criterion: "Output includes status per feature", weight: 1 }, { criterion: "Output identifies blockers", weight: 1 }] },
|
|
404
|
+
{ query: "Compare the user feedback for Feature A vs Feature B", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares two features", weight: 1 }, { criterion: "Output references user feedback", weight: 1 }, { criterion: "Output includes a recommendation", weight: 1 }] },
|
|
405
|
+
{ query: "Prepare a sprint retrospective document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output follows retro format (what went well, what didn't, actions)", weight: 1 }, { criterion: "Output is specific to the current sprint", weight: 1 }, { criterion: "Output includes actionable improvements", weight: 1 }] },
|
|
406
|
+
{ query: "What user-facing changes went live this week?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists specific changes", weight: 1 }, { criterion: "Output focuses on user impact", weight: 1 }, { criterion: "Output includes release dates", weight: 1 }] },
|
|
407
|
+
{ query: "Create a PRD outline for the new onboarding flow", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output follows PRD structure", weight: 1 }, { criterion: "Output includes user stories or acceptance criteria", weight: 1 }, { criterion: "Output is scoped appropriately", weight: 1 }] },
|
|
408
|
+
{ query: "Research what competitors are doing with their onboarding", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies competitor onboarding approaches", weight: 1 }, { criterion: "Output includes specific examples", weight: 1 }, { criterion: "Output derives actionable insights", weight: 1 }] },
|
|
409
|
+
{ query: "How has our feature velocity changed over the last 3 sprints?", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output tracks velocity over time", weight: 1 }, { criterion: "Output identifies trends", weight: 1 }, { criterion: "Output suggests causes for velocity changes", weight: 1 }] },
|
|
410
|
+
{ query: "Delegate the user research interviews to the UX researcher", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output includes interview script or topics", weight: 1 }, { criterion: "Output specifies target user segments", weight: 1 }, { criterion: "Output includes expected deliverables", weight: 1 }] },
|
|
202
411
|
{ query: "Switch to content mode and draft the release notes", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output shifts to content writing perspective", weight: 1 }, { criterion: "Output drafts user-facing release notes", weight: 1 }, { criterion: "Output is polished and non-technical", weight: 1 }] },
|
|
203
|
-
{ query: "What are the top 5 user pain points from support tickets?", scenario: "company_search", expectedTools: ["
|
|
204
|
-
{ query: "Flag any scope creep in the current sprint", scenario: "important_change", expectedTools: ["
|
|
412
|
+
{ query: "What are the top 5 user pain points from support tickets?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output lists specific pain points", weight: 1 }, { criterion: "Output includes frequency or severity", weight: 1 }, { criterion: "Output suggests product solutions", weight: 1 }] },
|
|
413
|
+
{ query: "Flag any scope creep in the current sprint", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies scope additions", weight: 1 }, { criterion: "Output assesses impact on timeline", weight: 1 }, { criterion: "Output recommends scope management actions", weight: 1 }] },
|
|
205
414
|
];
|
|
206
415
|
}
|
|
207
416
|
function contractorTemplates() {
|
|
208
417
|
return [
|
|
209
|
-
{ query: "What's my task list for this week?", scenario: "weekly_reset", expectedTools: ["
|
|
210
|
-
{ query: "Show me the project context I need to onboard", scenario: "company_search", expectedTools: ["
|
|
211
|
-
{ query: "Export my weekly deliverables report for the client", scenario: "memo_export", expectedTools: ["
|
|
212
|
-
{ query: "What changed in the project requirements since I was last briefed?", scenario: "important_change", expectedTools: ["
|
|
213
|
-
{ query: "Compare the scope of my current contract vs the original SOW", scenario: "packet_diff", expectedTools: ["
|
|
214
|
-
{ query: "Find the coding standards document for this project", scenario: "company_search", expectedTools: ["
|
|
215
|
-
{ query: "Delegate the testing tasks to the QA contractor", scenario: "delegation", expectedTools: ["
|
|
418
|
+
{ query: "What's my task list for this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists specific tasks", weight: 1 }, { criterion: "Output includes priorities", weight: 1 }, { criterion: "Output is scoped to the contractor's role", weight: 1 }] },
|
|
419
|
+
{ query: "Show me the project context I need to onboard", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output provides project overview", weight: 1 }, { criterion: "Output includes key contacts or resources", weight: 1 }, { criterion: "Output is onboarding-appropriate", weight: 1 }] },
|
|
420
|
+
{ query: "Export my weekly deliverables report for the client", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output is client-facing in tone", weight: 1 }, { criterion: "Output lists deliverables with status", weight: 1 }, { criterion: "Output includes hours or effort summary", weight: 1 }] },
|
|
421
|
+
{ query: "What changed in the project requirements since I was last briefed?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific requirement changes", weight: 1 }, { criterion: "Output highlights impact on current work", weight: 1 }, { criterion: "Output suggests clarification questions", weight: 1 }] },
|
|
422
|
+
{ query: "Compare the scope of my current contract vs the original SOW", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares current vs original scope", weight: 1 }, { criterion: "Output identifies scope expansion", weight: 1 }, { criterion: "Output suggests contract amendment if needed", weight: 1 }] },
|
|
423
|
+
{ query: "Find the coding standards document for this project", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output helps locate documentation", weight: 1 }, { criterion: "Output is specific to coding standards", weight: 1 }, { criterion: "Output suggests follow-up resources", weight: 1 }] },
|
|
424
|
+
{ query: "Delegate the testing tasks to the QA contractor", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains test delegation details", weight: 1 }, { criterion: "Output specifies test scope and criteria", weight: 1 }, { criterion: "Output includes acceptance standards", weight: 1 }] },
|
|
216
425
|
{ query: "Switch to PM mode to understand the feature priority", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output adopts a PM perspective", weight: 1 }, { criterion: "Output discusses prioritization frameworks", weight: 1 }, { criterion: "Output helps contextualize current work", weight: 1 }] },
|
|
217
|
-
{ query: "Flag any blockers that are preventing my progress", scenario: "important_change", expectedTools: ["founder_local_synthesize"
|
|
426
|
+
{ query: "Flag any blockers that are preventing my progress", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific blockers", weight: 1 }, { criterion: "Output suggests workarounds or escalation paths", weight: 1 }, { criterion: "Output includes who can unblock", weight: 1 }] },
|
|
218
427
|
{ query: "What tools are available for code review in this project?", scenario: "company_search", expectedTools: ["discover_tools"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists relevant tools", weight: 1 }, { criterion: "Output includes brief descriptions", weight: 1 }, { criterion: "Output is filtered to code review context", weight: 1 }] },
|
|
219
428
|
];
|
|
220
429
|
}
|
|
221
430
|
function investorTemplates() {
|
|
222
431
|
return [
|
|
223
|
-
{ query: "Run due diligence on this Series A deal with TechStartup Inc", scenario: "company_search", expectedTools: ["
|
|
224
|
-
{ query: "What are the red flags in this company's pitch deck?", scenario: "company_search", expectedTools: ["founder_local_synthesize"
|
|
225
|
-
{ query: "Compare the cap tables of our portfolio companies", scenario: "competitor_brief", expectedTools: ["
|
|
226
|
-
{ query: "Prepare the quarterly LP update letter", scenario: "memo_export", expectedTools: ["
|
|
227
|
-
{ query: "What's changed in the macro environment that affects our thesis?", scenario: "important_change", expectedTools: ["
|
|
228
|
-
{ query: "Track how our portfolio company valuations shifted this quarter", scenario: "packet_diff", expectedTools: ["
|
|
229
|
-
{ query: "Delegate the market sizing analysis to the associate", scenario: "delegation", expectedTools: ["
|
|
230
|
-
{ query: "Switch to founder mode and evaluate the product from a builder's lens", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
231
|
-
{ query: "Give me the weekly portfolio pulse", scenario: "weekly_reset", expectedTools: ["
|
|
232
|
-
{ query: "What deal flow came in this week worth evaluating?", scenario: "weekly_reset", expectedTools: ["
|
|
233
|
-
{ query: "Research the competitive landscape for this fintech vertical", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
432
|
+
{ query: "Run due diligence on this Series A deal with TechStartup Inc", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows due diligence structure", weight: 1 }, { criterion: "Output identifies key risk factors", weight: 1 }, { criterion: "Output does not fabricate valuation numbers", weight: 2 }, { criterion: "Output includes market context", weight: 1 }] },
|
|
433
|
+
{ query: "What are the red flags in this company's pitch deck?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific red flags", weight: 1 }, { criterion: "Output categorizes flags by severity", weight: 1 }, { criterion: "Output suggests follow-up questions", weight: 1 }] },
|
|
434
|
+
{ query: "Compare the cap tables of our portfolio companies", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares equity structures", weight: 1 }, { criterion: "Output identifies dilution risks", weight: 1 }, { criterion: "Output does not invent specific percentages", weight: 2 }] },
|
|
435
|
+
{ query: "Prepare the quarterly LP update letter", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows LP update format", weight: 1 }, { criterion: "Output covers portfolio performance, exits, and pipeline", weight: 1 }, { criterion: "Output is professional and measured in tone", weight: 1 }] },
|
|
436
|
+
{ query: "What's changed in the macro environment that affects our thesis?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references macroeconomic factors", weight: 1 }, { criterion: "Output connects macro to investment thesis", weight: 1 }, { criterion: "Output is data-driven, not speculative", weight: 1 }] },
|
|
437
|
+
{ query: "Track how our portfolio company valuations shifted this quarter", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output tracks valuation changes", weight: 1 }, { criterion: "Output identifies up-rounds and down-rounds", weight: 1 }, { criterion: "Output does not fabricate specific valuations", weight: 2 }] },
|
|
438
|
+
{ query: "Delegate the market sizing analysis to the associate", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output includes market sizing methodology", weight: 1 }, { criterion: "Output specifies data sources to use", weight: 1 }, { criterion: "Output includes expected deliverable format", weight: 1 }] },
|
|
439
|
+
{ query: "Switch to founder mode and evaluate the product from a builder's lens", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output shifts to builder/product perspective", weight: 1 }, { criterion: "Output evaluates technical feasibility", weight: 1 }, { criterion: "Output identifies product-market fit signals", weight: 1 }] },
|
|
440
|
+
{ query: "Give me the weekly portfolio pulse", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output covers portfolio companies", weight: 1 }, { criterion: "Output highlights winners and at-risk companies", weight: 1 }, { criterion: "Output is concise for a weekly cadence", weight: 1 }] },
|
|
441
|
+
{ query: "What deal flow came in this week worth evaluating?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references deal flow", weight: 1 }, { criterion: "Output includes basic screening criteria", weight: 1 }, { criterion: "Output recommends which to pursue", weight: 1 }] },
|
|
442
|
+
{ query: "Research the competitive landscape for this fintech vertical", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output maps the fintech competitive landscape", weight: 1 }, { criterion: "Output identifies market leaders and challengers", weight: 1 }, { criterion: "Output assesses white space opportunities", weight: 1 }] },
|
|
234
443
|
];
|
|
235
444
|
}
|
|
236
445
|
function contentTemplates() {
|
|
237
446
|
return [
|
|
238
|
-
{ query: "Draft a LinkedIn post about our latest product launch", scenario: "memo_export", expectedTools: ["
|
|
239
|
-
{ query: "What trending topics should we create content around this week?", scenario: "weekly_reset", expectedTools: ["
|
|
240
|
-
{ query: "Compare our content strategy against HubSpot and Buffer", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"
|
|
241
|
-
{ query: "Export the content calendar for next month", scenario: "memo_export", expectedTools: ["
|
|
242
|
-
{ query: "What content performed best this month and why?", scenario: "important_change", expectedTools: ["
|
|
243
|
-
{ query: "Track how our messaging has evolved over the past quarter", scenario: "packet_diff", expectedTools: ["
|
|
244
|
-
{ query: "Delegate the blog post writing to the content contractor", scenario: "delegation", expectedTools: ["
|
|
245
|
-
{ query: "Research what type of content resonates in the AI/ML space on Twitter", scenario: "company_search", expectedTools: ["
|
|
447
|
+
{ query: "Draft a LinkedIn post about our latest product launch", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output is formatted for LinkedIn", weight: 1 }, { criterion: "Output is under 300 words", weight: 1 }, { criterion: "Output includes a hook and CTA", weight: 1 }] },
|
|
448
|
+
{ query: "What trending topics should we create content around this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies trending topics", weight: 1 }, { criterion: "Output connects trends to our brand", weight: 1 }, { criterion: "Output suggests specific content formats", weight: 1 }] },
|
|
449
|
+
{ query: "Compare our content strategy against HubSpot and Buffer", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares content strategies", weight: 1 }, { criterion: "Output identifies what competitors do better", weight: 1 }, { criterion: "Output includes actionable takeaways", weight: 1 }] },
|
|
450
|
+
{ query: "Export the content calendar for next month", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output is calendar-structured", weight: 1 }, { criterion: "Output includes content types and topics", weight: 1 }, { criterion: "Output assigns rough dates", weight: 1 }] },
|
|
451
|
+
{ query: "What content performed best this month and why?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies top-performing content", weight: 1 }, { criterion: "Output includes metrics or proxies for performance", weight: 1 }, { criterion: "Output analyzes why it performed well", weight: 1 }] },
|
|
452
|
+
{ query: "Track how our messaging has evolved over the past quarter", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output tracks messaging evolution", weight: 1 }, { criterion: "Output identifies key narrative shifts", weight: 1 }, { criterion: "Output assesses consistency", weight: 1 }] },
|
|
453
|
+
{ query: "Delegate the blog post writing to the content contractor", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output includes writing brief", weight: 1 }, { criterion: "Output specifies tone, audience, and word count", weight: 1 }, { criterion: "Output includes SEO keywords if relevant", weight: 1 }] },
|
|
454
|
+
{ query: "Research what type of content resonates in the AI/ML space on Twitter", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies content types that perform well", weight: 1 }, { criterion: "Output includes examples or patterns", weight: 1 }, { criterion: "Output is specific to AI/ML audience", weight: 1 }] },
|
|
246
455
|
{ query: "Switch to researcher mode to find data points for the whitepaper", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output shifts to research perspective", weight: 1 }, { criterion: "Output identifies relevant data sources", weight: 1 }, { criterion: "Output suggests citation-worthy statistics", weight: 1 }] },
|
|
247
|
-
{ query: "Create a brand voice guideline document", scenario: "memo_export", expectedTools: ["
|
|
248
|
-
{ query: "What changes should we make to our newsletter strategy?", scenario: "important_change", expectedTools: ["
|
|
456
|
+
{ query: "Create a brand voice guideline document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output follows brand voice guide structure", weight: 1 }, { criterion: "Output includes tone, vocabulary, and examples", weight: 1 }, { criterion: "Output is usable by external writers", weight: 1 }] },
|
|
457
|
+
{ query: "What changes should we make to our newsletter strategy?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output assesses current newsletter performance", weight: 1 }, { criterion: "Output suggests specific improvements", weight: 1 }, { criterion: "Output is based on audience data or trends", weight: 1 }] },
|
|
249
458
|
];
|
|
250
459
|
}
|
|
251
460
|
/** Generate N filler queries per persona to reach exactly 500 total */
|
|
@@ -341,7 +550,7 @@ function generateFillerQueries(persona, existingCount, targetCount) {
|
|
|
341
550
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
342
551
|
break;
|
|
343
552
|
case "delegation":
|
|
344
|
-
expectedTools.push("
|
|
553
|
+
expectedTools.push("founder_local_synthesize");
|
|
345
554
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
346
555
|
break;
|
|
347
556
|
case "important_change":
|
|
@@ -349,11 +558,11 @@ function generateFillerQueries(persona, existingCount, targetCount) {
|
|
|
349
558
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
350
559
|
break;
|
|
351
560
|
case "memo_export":
|
|
352
|
-
expectedTools.push("
|
|
561
|
+
expectedTools.push("founder_local_synthesize");
|
|
353
562
|
forbiddenTools.push("run_recon");
|
|
354
563
|
break;
|
|
355
564
|
case "packet_diff":
|
|
356
|
-
expectedTools.push("
|
|
565
|
+
expectedTools.push("founder_local_synthesize");
|
|
357
566
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
358
567
|
break;
|
|
359
568
|
case "role_switch":
|
|
@@ -514,28 +723,67 @@ async function executeQueryTools(query, allTools) {
|
|
|
514
723
|
effectiveTools.push("founder_local_synthesize");
|
|
515
724
|
}
|
|
516
725
|
}
|
|
517
|
-
// 2b.
|
|
726
|
+
// 2b. Web enrichment: fetch live web data for scenarios that need entity-specific content
|
|
727
|
+
let webResults = [];
|
|
728
|
+
const webEnrichScenarios = ["company_search", "competitor_brief", "important_change", "delegation", "memo_export", "weekly_reset", "packet_diff", "role_switch"];
|
|
729
|
+
if (webEnrichScenarios.includes(query.scenario)) {
|
|
730
|
+
const webSearchTool = findTool(allTools, "web_search");
|
|
731
|
+
if (webSearchTool) {
|
|
732
|
+
try {
|
|
733
|
+
const webResult = await callTool(webSearchTool, { query: query.query, maxResults: 5, provider: "gemini" });
|
|
734
|
+
totalMs += webResult.ms;
|
|
735
|
+
if (webResult.ok && webResult.result) {
|
|
736
|
+
// Extract search results from tool output
|
|
737
|
+
const raw = webResult.result;
|
|
738
|
+
if (Array.isArray(raw?.results)) {
|
|
739
|
+
webResults = raw.results.slice(0, 5);
|
|
740
|
+
}
|
|
741
|
+
else if (Array.isArray(raw)) {
|
|
742
|
+
webResults = raw.slice(0, 5);
|
|
743
|
+
}
|
|
744
|
+
else if (raw?.content) {
|
|
745
|
+
// MCP content block format
|
|
746
|
+
try {
|
|
747
|
+
const text = Array.isArray(raw.content) ? raw.content.map((b) => b.text).join("") : String(raw.content);
|
|
748
|
+
const parsed = JSON.parse(text);
|
|
749
|
+
if (Array.isArray(parsed?.results))
|
|
750
|
+
webResults = parsed.results.slice(0, 5);
|
|
751
|
+
else if (Array.isArray(parsed))
|
|
752
|
+
webResults = parsed.slice(0, 5);
|
|
753
|
+
}
|
|
754
|
+
catch { /* not JSON */ }
|
|
755
|
+
}
|
|
756
|
+
if (webResults.length > 0) {
|
|
757
|
+
toolsFired.push("web_search");
|
|
758
|
+
outputs["web_search"] = webResults.map(r => `${r.title}: ${r.snippet} (${r.url})`).join("\n");
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
catch { /* web search unavailable — continue without */ }
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
// 2c. Scenario-specific seeding: seed data before tools that need prior state
|
|
518
766
|
if (query.scenario === "competitor_brief") {
|
|
519
|
-
// Seed
|
|
520
|
-
const
|
|
521
|
-
if (
|
|
522
|
-
const seedResult = await callTool(
|
|
767
|
+
// Seed with query-specific competitor brief via founder_local_synthesize + web results
|
|
768
|
+
const synthTool = findTool(allTools, "founder_local_synthesize");
|
|
769
|
+
if (synthTool && !toolsFired.includes("founder_local_synthesize")) {
|
|
770
|
+
const seedResult = await callTool(synthTool, { packetType: "competitor_brief", daysBack: 7, query: query.query, lens: query.persona, webResults });
|
|
523
771
|
totalMs += seedResult.ms;
|
|
524
772
|
if (seedResult.ok) {
|
|
525
|
-
toolsFired.push("
|
|
526
|
-
outputs["
|
|
773
|
+
toolsFired.push("founder_local_synthesize");
|
|
774
|
+
outputs["founder_local_synthesize"] = extractText(seedResult.result);
|
|
527
775
|
}
|
|
528
776
|
}
|
|
529
777
|
}
|
|
530
778
|
if (query.scenario === "packet_diff") {
|
|
531
|
-
// Seed
|
|
532
|
-
const
|
|
533
|
-
if (
|
|
534
|
-
const seedResult = await callTool(
|
|
779
|
+
// Seed with an important_change packet that shows what changed (our best before/after proxy)
|
|
780
|
+
const synthTool = findTool(allTools, "founder_local_synthesize");
|
|
781
|
+
if (synthTool && !toolsFired.includes("founder_local_synthesize")) {
|
|
782
|
+
const seedResult = await callTool(synthTool, { packetType: "important_change", daysBack: 14, query: query.query, lens: query.persona, webResults });
|
|
535
783
|
totalMs += seedResult.ms;
|
|
536
784
|
if (seedResult.ok) {
|
|
537
|
-
toolsFired.push("
|
|
538
|
-
outputs["
|
|
785
|
+
toolsFired.push("founder_local_synthesize");
|
|
786
|
+
outputs["founder_local_synthesize"] = extractText(seedResult.result);
|
|
539
787
|
}
|
|
540
788
|
}
|
|
541
789
|
}
|
|
@@ -568,8 +816,13 @@ async function executeQueryTools(query, allTools) {
|
|
|
568
816
|
}
|
|
569
817
|
const tool = findTool(allTools, toolName);
|
|
570
818
|
if (tool) {
|
|
571
|
-
// Build minimal args based on tool name patterns
|
|
819
|
+
// Build minimal args based on tool name patterns, inject webResults for synthesize
|
|
572
820
|
const args = buildMinimalArgs(toolName, query);
|
|
821
|
+
if (toolName === "founder_local_synthesize") {
|
|
822
|
+
if (webResults.length > 0)
|
|
823
|
+
args.webResults = webResults;
|
|
824
|
+
args.lens = query.persona;
|
|
825
|
+
}
|
|
573
826
|
const result = await callTool(tool, args);
|
|
574
827
|
totalMs += result.ms;
|
|
575
828
|
if (result.ok) {
|
|
@@ -644,9 +897,9 @@ function buildMinimalArgs(toolName, query) {
|
|
|
644
897
|
};
|
|
645
898
|
return { packetType: ptMap[query.scenario] ?? "weekly_reset", daysBack: 7, query: query.query };
|
|
646
899
|
}
|
|
647
|
-
case "
|
|
900
|
+
case "founder_local_synthesize":
|
|
648
901
|
return {};
|
|
649
|
-
case "
|
|
902
|
+
case "founder_local_synthesize":
|
|
650
903
|
return {};
|
|
651
904
|
case "founder_packet_validate":
|
|
652
905
|
return {};
|
|
@@ -658,7 +911,7 @@ function buildMinimalArgs(toolName, query) {
|
|
|
658
911
|
return {};
|
|
659
912
|
case "flag_important_change":
|
|
660
913
|
return { description: query.query };
|
|
661
|
-
case "
|
|
914
|
+
case "founder_local_synthesize":
|
|
662
915
|
return { title: `Export for: ${query.query.slice(0, 60)}` };
|
|
663
916
|
case "compare_options":
|
|
664
917
|
return { options: [company, "Competitor"], criteria: ["market position", "product quality"] };
|
|
@@ -666,10 +919,18 @@ function buildMinimalArgs(toolName, query) {
|
|
|
666
919
|
return {};
|
|
667
920
|
case "check_mcp_setup":
|
|
668
921
|
return {};
|
|
669
|
-
case "
|
|
922
|
+
case "founder_packet_validate":
|
|
923
|
+
return { query: query.query };
|
|
924
|
+
case "search_all_knowledge":
|
|
925
|
+
return { query: query.query };
|
|
926
|
+
case "founder_local_weekly_reset":
|
|
927
|
+
return { daysBack: 7, query: query.query };
|
|
928
|
+
case "export_artifact_packet":
|
|
929
|
+
return { query: query.query };
|
|
930
|
+
case "render_decision_memo":
|
|
931
|
+
return { query: query.query };
|
|
932
|
+
case "start_dogfood_session":
|
|
670
933
|
return { query: query.query };
|
|
671
|
-
case "build_research_digest":
|
|
672
|
-
return { topic: query.query };
|
|
673
934
|
case "get_project_context":
|
|
674
935
|
return {};
|
|
675
936
|
case "compress_or_expand_text":
|
|
@@ -1185,9 +1446,10 @@ function printReport(summary, regressions, improvements, scenarioFlags) {
|
|
|
1185
1446
|
console.log("");
|
|
1186
1447
|
}
|
|
1187
1448
|
export async function runLlmJudgeEval(options) {
|
|
1188
|
-
// 1. Wire up DB
|
|
1449
|
+
// 1. Wire up DB and seed realistic test data
|
|
1189
1450
|
_setDbAccessor(getDb);
|
|
1190
1451
|
ensureSchema();
|
|
1452
|
+
seedTestData();
|
|
1191
1453
|
// 2. Generate corpus and filter
|
|
1192
1454
|
let corpus = generateQueryCorpus();
|
|
1193
1455
|
if (options.persona) {
|
|
@@ -1249,7 +1511,8 @@ export async function runLlmJudgeEval(options) {
|
|
|
1249
1511
|
const toolRecall = computeToolRecall(query.expectedTools, execution.toolsFired);
|
|
1250
1512
|
const forbiddenViolations = countForbiddenViolations(query.forbiddenTools, execution.toolsFired);
|
|
1251
1513
|
const criteriaPassRate = computeCriteriaPassRate(judgeResult.criteria, query.booleanCriteria);
|
|
1252
|
-
|
|
1514
|
+
// Pass if weighted criteria pass rate >= 60% AND no forbidden tool violations
|
|
1515
|
+
const overallPass = criteriaPassRate >= 0.60 && forbiddenViolations === 0;
|
|
1253
1516
|
const qr = {
|
|
1254
1517
|
queryId: query.id,
|
|
1255
1518
|
pass: overallPass,
|
|
@@ -1282,6 +1545,8 @@ export async function runLlmJudgeEval(options) {
|
|
|
1282
1545
|
}
|
|
1283
1546
|
// 8. Print report
|
|
1284
1547
|
printReport(summary, regressions, improvements, scenarioFlags);
|
|
1548
|
+
// 9. Clean up seeded test data
|
|
1549
|
+
cleanupTestData();
|
|
1285
1550
|
return summary;
|
|
1286
1551
|
}
|
|
1287
1552
|
/** Simple deterministic hash for reproducible sampling */
|