nodebench-mcp 2.62.0 → 2.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmarks/llmJudgeEval.js +283 -84
- package/dist/benchmarks/llmJudgeEval.js.map +1 -1
- package/dist/benchmarks/searchQualityEval.js +1 -1
- package/dist/tools/contentSynthesis.d.ts +42 -0
- package/dist/tools/contentSynthesis.js +185 -0
- package/dist/tools/contentSynthesis.js.map +1 -0
- package/dist/tools/founderLocalPipeline.js +40 -11
- package/dist/tools/founderLocalPipeline.js.map +1 -1
- package/dist/tools/founderTools.js +568 -0
- package/dist/tools/founderTools.js.map +1 -1
- package/dist/tools/llmJudgeLoop.js +5 -5
- package/dist/tools/llmJudgeLoop.js.map +1 -1
- package/package.json +1 -1
|
@@ -275,11 +275,11 @@ function founderTemplates() {
|
|
|
275
275
|
return [
|
|
276
276
|
// weekly_reset
|
|
277
277
|
{ query: "What changed in our product direction this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
278
|
-
{ query: "Give me a weekly reset briefing for the founding team", scenario: "weekly_reset", expectedTools: ["founder_local_weekly_reset", "founder_deep_context_gather"], forbiddenTools: ["
|
|
279
|
-
{ query: "Summarize last week's key decisions and their rationale", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
280
|
-
{ query: "What are the top 3 risks to our current sprint?", scenario: "weekly_reset", expectedTools: ["
|
|
281
|
-
{ query: "How is our burn rate tracking against the runway?", scenario: "weekly_reset", expectedTools: ["
|
|
282
|
-
{ query: "What did we ship this week and what slipped?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
278
|
+
{ query: "Give me a weekly reset briefing for the founding team", scenario: "weekly_reset", expectedTools: ["founder_local_weekly_reset", "founder_deep_context_gather"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
279
|
+
{ query: "Summarize last week's key decisions and their rationale", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["render_decision_memo"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
280
|
+
{ query: "What are the top 3 risks to our current sprint?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
281
|
+
{ query: "How is our burn rate tracking against the runway?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
282
|
+
{ query: "What did we ship this week and what slipped?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["export_artifact_packet"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
283
283
|
// company_search
|
|
284
284
|
{ query: "Research Stripe and tell me about their latest product moves", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
285
285
|
{ query: "Pull everything you know about Anthropic's recent funding", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
@@ -288,16 +288,16 @@ function founderTemplates() {
|
|
|
288
288
|
{ query: "What are the moats of our top 3 competitors?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
289
289
|
// delegation — route to founder_local_synthesize with pre_delegation packetType
|
|
290
290
|
{ query: "Draft a delegation brief for the engineering lead on the auth refactor", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
291
|
-
{ query: "Create a handoff packet for the new VP of Product", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
291
|
+
{ query: "Create a handoff packet for the new VP of Product", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
292
292
|
// important_change
|
|
293
|
-
{ query: "Flag any important changes in our competitive landscape this week", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
293
|
+
{ query: "Flag any important changes in our competitive landscape this week", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["start_dogfood_session"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
294
294
|
{ query: "What's the most critical thing I should know about right now?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
295
295
|
// memo_export — route to founder_local_weekly_reset which produces a full memo
|
|
296
|
-
{ query: "Export our latest decision memo as a shareable packet", scenario: "memo_export", expectedTools: ["founder_local_weekly_reset"], forbiddenTools: ["run_recon"
|
|
296
|
+
{ query: "Export our latest decision memo as a shareable packet", scenario: "memo_export", expectedTools: ["founder_local_weekly_reset"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
297
297
|
{ query: "Package the Q1 strategy review for the board", scenario: "memo_export", expectedTools: ["founder_local_weekly_reset"], forbiddenTools: ["start_dogfood_session"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
298
298
|
// packet_diff — route to founder_local_synthesize with important_change (shows what changed)
|
|
299
299
|
{ query: "What changed between the last two strategy packets?", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }] },
|
|
300
|
-
{ query: "Show me the delta between our January and March founder packets", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
300
|
+
{ query: "Show me the delta between our January and March founder packets", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Tool returned valid structured JSON or object data", weight: 1 }] },
|
|
301
301
|
// role_switch
|
|
302
302
|
{ query: "Switch to investor mode and evaluate our pitch deck", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
303
303
|
{ query: "I need to think like a banker — what's the credit risk here?", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
@@ -312,7 +312,7 @@ function bankerTemplates() {
|
|
|
312
312
|
{ query: "Compare the credit profiles of Company A vs Company B", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
313
313
|
{ query: "Draft a term sheet summary for the lending committee", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
314
314
|
{ query: "What's changed in the regulatory landscape this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
315
|
-
{ query: "Export the due diligence findings for the Acme Corp loan", scenario: "memo_export", expectedTools: ["founder_local_synthesize"
|
|
315
|
+
{ query: "Export the due diligence findings for the Acme Corp loan", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
316
316
|
{ query: "Show me how the risk ratings shifted since last quarter", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
317
317
|
{ query: "Delegate the annual review prep to the junior analyst", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
318
318
|
{ query: "Assess the market risk exposure in our current book", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
@@ -323,16 +323,16 @@ function bankerTemplates() {
|
|
|
323
323
|
}
|
|
324
324
|
function ceoTemplates() {
|
|
325
325
|
return [
|
|
326
|
-
{ query: "Give me the executive summary of where we stand this week", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: [
|
|
326
|
+
{ query: "Give me the executive summary of where we stand this week", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
327
327
|
{ query: "What should I be worried about that nobody's telling me?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
328
|
-
{ query: "Prepare talking points for the all-hands meeting", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
328
|
+
{ query: "Prepare talking points for the all-hands meeting", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
329
329
|
{ query: "How are our OKRs tracking this quarter?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
330
330
|
{ query: "Who on the leadership team needs my attention this week?", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
331
331
|
{ query: "Draft a board update email for this month", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
332
332
|
{ query: "What's our competitive position changed to since last month?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
333
|
-
{ query: "Compare the last two quarterly reviews for drift", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["
|
|
333
|
+
{ query: "Compare the last two quarterly reviews for drift", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["check_mcp_setup"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
334
334
|
{ query: "I need to delegate the hiring pipeline review — create a brief", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
335
|
-
{ query: "Switch to founder mode and deep-dive into the product roadmap", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
335
|
+
{ query: "Switch to founder mode and deep-dive into the product roadmap", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
336
336
|
{ query: "Flag the most important thing that changed since yesterday", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
337
337
|
{ query: "Research what our key enterprise customers are saying publicly", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
338
338
|
];
|
|
@@ -340,24 +340,24 @@ function ceoTemplates() {
|
|
|
340
340
|
function researcherTemplates() {
|
|
341
341
|
return [
|
|
342
342
|
{ query: "Find recent papers on transformer attention mechanisms", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
343
|
-
{ query: "Build a research digest on federated learning advances in 2025", scenario: "company_search", expectedTools: ["
|
|
343
|
+
{ query: "Build a research digest on federated learning advances in 2025", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
344
344
|
{ query: "What are the open problems in RLHF that nobody's solved?", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_synthesize"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
345
345
|
{ query: "Summarize the key findings from this week's arXiv papers on LLM reasoning", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
346
|
-
{ query: "Compare the methodology of these two papers on knowledge distillation", scenario: "competitor_brief", expectedTools: ["
|
|
346
|
+
{ query: "Compare the methodology of these two papers on knowledge distillation", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
347
347
|
{ query: "Export my literature review notes as a shareable document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
348
|
-
{ query: "What contradictions exist in the current MoE literature?", scenario: "important_change", expectedTools: ["
|
|
349
|
-
{ query: "Track how the consensus on scaling laws has shifted this year", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"
|
|
348
|
+
{ query: "What contradictions exist in the current MoE literature?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
349
|
+
{ query: "Track how the consensus on scaling laws has shifted this year", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Multiple tools in the chain produced non-empty results", weight: 1 }] },
|
|
350
350
|
{ query: "Delegate the data collection task to the research assistant", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
351
|
-
{ query: "Switch to operator mode and check if the experiment pipeline is healthy", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
351
|
+
{ query: "Switch to operator mode and check if the experiment pipeline is healthy", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["search_all_knowledge"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
352
352
|
{ query: "What are the most-cited papers in agentic AI from 2025?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }, { criterion: "Output includes quantitative data points or metrics", weight: 1 }] },
|
|
353
|
-
{ query: "Generate a research question from the gaps in current RAG literature", scenario: "important_change", expectedTools: ["
|
|
353
|
+
{ query: "Generate a research question from the gaps in current RAG literature", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
354
354
|
];
|
|
355
355
|
}
|
|
356
356
|
function studentTemplates() {
|
|
357
357
|
return [
|
|
358
358
|
{ query: "Help me understand how transformers work at a high level", scenario: "company_search", expectedTools: ["discover_tools"], forbiddenTools: ["founder_deep_context_gather", "founder_local_synthesize"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
359
359
|
{ query: "What should I study this week for my ML course?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
360
|
-
{ query: "Compare supervised vs unsupervised learning for my report", scenario: "competitor_brief", expectedTools: ["
|
|
360
|
+
{ query: "Compare supervised vs unsupervised learning for my report", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
361
361
|
{ query: "Export my study notes as a markdown document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output structure matches the tool's documented schema", weight: 1 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
362
362
|
{ query: "What changed in the AI landscape this week that I should know about?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 1 }] },
|
|
363
363
|
{ query: "I need to switch to a research perspective for my thesis", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }] },
|
|
@@ -372,10 +372,10 @@ function operatorTemplates() {
|
|
|
372
372
|
return [
|
|
373
373
|
{ query: "Show me the system health dashboard for today", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output presents system health metrics", weight: 1 }, { criterion: "Output highlights any degraded services", weight: 1 }, { criterion: "Output is operational in tone", weight: 1 }] },
|
|
374
374
|
{ query: "What incidents happened this week and are they resolved?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists incidents", weight: 1 }, { criterion: "Output includes resolution status", weight: 1 }, { criterion: "Output identifies root causes", weight: 1 }] },
|
|
375
|
-
{ query: "Run a health check on all MCP infrastructure", scenario: "company_search", expectedTools: ["
|
|
375
|
+
{ query: "Run a health check on all MCP infrastructure", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output checks multiple infrastructure components", weight: 1 }, { criterion: "Output reports pass/fail per component", weight: 1 }, { criterion: "Output suggests fixes for failures", weight: 1 }] },
|
|
376
376
|
{ query: "Delegate the on-call rotation setup to the SRE team", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains delegation instructions", weight: 1 }, { criterion: "Output specifies SRE-relevant details", weight: 1 }, { criterion: "Output includes escalation paths", weight: 1 }] },
|
|
377
377
|
{ query: "What deployments went out this week and did any cause issues?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output references deployments", weight: 1 }, { criterion: "Output correlates deployments with incidents", weight: 1 }, { criterion: "Output identifies rollback candidates", weight: 1 }] },
|
|
378
|
-
{ query: "Compare our uptime this month vs last month", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"
|
|
378
|
+
{ query: "Compare our uptime this month vs last month", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output includes uptime percentages or trends", weight: 1 }, { criterion: "Output identifies the biggest contributor to downtime", weight: 1 }, { criterion: "Output does not fabricate exact uptime numbers", weight: 2 }] },
|
|
379
379
|
{ query: "Export the incident report for the API outage", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows incident report structure", weight: 1 }, { criterion: "Output includes timeline, impact, and root cause", weight: 1 }, { criterion: "Output is shareable with stakeholders", weight: 1 }] },
|
|
380
380
|
{ query: "Flag any alerts that have been unacknowledged for over 24 hours", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output identifies stale alerts", weight: 1 }, { criterion: "Output includes age of each alert", weight: 1 }, { criterion: "Output suggests escalation for critical ones", weight: 1 }] },
|
|
381
381
|
{ query: "Switch to researcher mode to investigate the performance regression", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output shifts to investigation perspective", weight: 1 }, { criterion: "Output suggests diagnostic tools and approaches", weight: 1 }, { criterion: "Output identifies data to collect", weight: 1 }] },
|
|
@@ -387,7 +387,7 @@ function legalTemplates() {
|
|
|
387
387
|
return [
|
|
388
388
|
{ query: "Check our contracts for compliance with the new data privacy regulation", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references data privacy regulations", weight: 1 }, { criterion: "Output identifies compliance gaps", weight: 1 }, { criterion: "Output does not provide actual legal advice", weight: 1 }] },
|
|
389
389
|
{ query: "What legal risks should we flag this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies legal risk categories", weight: 1 }, { criterion: "Output prioritizes risks by severity", weight: 1 }, { criterion: "Output includes a disclaimer about not being legal counsel", weight: 1 }] },
|
|
390
|
-
{ query: "Compare the terms of our vendor contracts for consistency", scenario: "competitor_brief", expectedTools: ["
|
|
390
|
+
{ query: "Compare the terms of our vendor contracts for consistency", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output compares contract terms systematically", weight: 1 }, { criterion: "Output identifies inconsistencies", weight: 1 }, { criterion: "Output suggests standardization opportunities", weight: 1 }] },
|
|
391
391
|
{ query: "Export the contract review findings for outside counsel", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output is formal and counsel-appropriate", weight: 1 }, { criterion: "Output includes numbered findings", weight: 1 }, { criterion: "Output preserves legal terminology", weight: 1 }] },
|
|
392
392
|
{ query: "Flag any IP-related changes in our competitor filings", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references IP or patent filings", weight: 1 }, { criterion: "Output identifies specific competitors", weight: 1 }, { criterion: "Output assesses impact on our position", weight: 1 }] },
|
|
393
393
|
{ query: "Prepare a delegation brief for the paralegal on discovery tasks", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output is delegation-appropriate", weight: 1 }, { criterion: "Output specifies legal discovery requirements", weight: 1 }, { criterion: "Output includes deadlines", weight: 1 }] },
|
|
@@ -395,13 +395,13 @@ function legalTemplates() {
|
|
|
395
395
|
{ query: "Switch to banker mode to assess the financial exposure from this lawsuit", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output adopts a financial assessment perspective", weight: 1 }, { criterion: "Output estimates exposure ranges", weight: 1 }, { criterion: "Output caveats financial estimates appropriately", weight: 1 }] },
|
|
396
396
|
{ query: "Review the NDA template for common issues", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output references NDA-specific terms", weight: 1 }, { criterion: "Output identifies common NDA pitfalls", weight: 1 }, { criterion: "Output suggests improvements", weight: 1 }] },
|
|
397
397
|
{ query: "What regulatory filings are due this month?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output lists upcoming deadlines", weight: 1 }, { criterion: "Output includes filing types", weight: 1 }, { criterion: "Output suggests preparation steps", weight: 1 }] },
|
|
398
|
-
{ query: "Summarize the liability exposure across all active contracts", scenario: "company_search", expectedTools: ["
|
|
398
|
+
{ query: "Summarize the liability exposure across all active contracts", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output addresses liability specifically", weight: 1 }, { criterion: "Output categorizes by contract type", weight: 1 }, { criterion: "Output does not fabricate liability amounts", weight: 2 }] },
|
|
399
399
|
];
|
|
400
400
|
}
|
|
401
401
|
function pmTemplates() {
|
|
402
402
|
return [
|
|
403
403
|
{ query: "What's the status of all feature requests from this sprint?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output lists feature requests", weight: 1 }, { criterion: "Output includes status per feature", weight: 1 }, { criterion: "Output identifies blockers", weight: 1 }] },
|
|
404
|
-
{ query: "Compare the user feedback for Feature A vs Feature B", scenario: "competitor_brief", expectedTools: ["
|
|
404
|
+
{ query: "Compare the user feedback for Feature A vs Feature B", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares two features", weight: 1 }, { criterion: "Output references user feedback", weight: 1 }, { criterion: "Output includes a recommendation", weight: 1 }] },
|
|
405
405
|
{ query: "Prepare a sprint retrospective document", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output follows retro format (what went well, what didn't, actions)", weight: 1 }, { criterion: "Output is specific to the current sprint", weight: 1 }, { criterion: "Output includes actionable improvements", weight: 1 }] },
|
|
406
406
|
{ query: "What user-facing changes went live this week?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output lists specific changes", weight: 1 }, { criterion: "Output focuses on user impact", weight: 1 }, { criterion: "Output includes release dates", weight: 1 }] },
|
|
407
407
|
{ query: "Create a PRD outline for the new onboarding flow", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output follows PRD structure", weight: 1 }, { criterion: "Output includes user stories or acceptance criteria", weight: 1 }, { criterion: "Output is scoped appropriately", weight: 1 }] },
|
|
@@ -420,7 +420,7 @@ function contractorTemplates() {
|
|
|
420
420
|
{ query: "Export my weekly deliverables report for the client", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output is client-facing in tone", weight: 1 }, { criterion: "Output lists deliverables with status", weight: 1 }, { criterion: "Output includes hours or effort summary", weight: 1 }] },
|
|
421
421
|
{ query: "What changed in the project requirements since I was last briefed?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific requirement changes", weight: 1 }, { criterion: "Output highlights impact on current work", weight: 1 }, { criterion: "Output suggests clarification questions", weight: 1 }] },
|
|
422
422
|
{ query: "Compare the scope of my current contract vs the original SOW", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares current vs original scope", weight: 1 }, { criterion: "Output identifies scope expansion", weight: 1 }, { criterion: "Output suggests contract amendment if needed", weight: 1 }] },
|
|
423
|
-
{ query: "Find the coding standards document for this project", scenario: "company_search", expectedTools: ["
|
|
423
|
+
{ query: "Find the coding standards document for this project", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output helps locate documentation", weight: 1 }, { criterion: "Output is specific to coding standards", weight: 1 }, { criterion: "Output suggests follow-up resources", weight: 1 }] },
|
|
424
424
|
{ query: "Delegate the testing tasks to the QA contractor", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output contains test delegation details", weight: 1 }, { criterion: "Output specifies test scope and criteria", weight: 1 }, { criterion: "Output includes acceptance standards", weight: 1 }] },
|
|
425
425
|
{ query: "Switch to PM mode to understand the feature priority", scenario: "role_switch", expectedTools: ["founder_local_synthesize"], forbiddenTools: [], booleanCriteria: [{ criterion: "Output adopts a PM perspective", weight: 1 }, { criterion: "Output discusses prioritization frameworks", weight: 1 }, { criterion: "Output helps contextualize current work", weight: 1 }] },
|
|
426
426
|
{ query: "Flag any blockers that are preventing my progress", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific blockers", weight: 1 }, { criterion: "Output suggests workarounds or escalation paths", weight: 1 }, { criterion: "Output includes who can unblock", weight: 1 }] },
|
|
@@ -431,12 +431,12 @@ function investorTemplates() {
|
|
|
431
431
|
return [
|
|
432
432
|
{ query: "Run due diligence on this Series A deal with TechStartup Inc", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows due diligence structure", weight: 1 }, { criterion: "Output identifies key risk factors", weight: 1 }, { criterion: "Output does not fabricate valuation numbers", weight: 2 }, { criterion: "Output includes market context", weight: 1 }] },
|
|
433
433
|
{ query: "What are the red flags in this company's pitch deck?", scenario: "company_search", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies specific red flags", weight: 1 }, { criterion: "Output categorizes flags by severity", weight: 1 }, { criterion: "Output suggests follow-up questions", weight: 1 }] },
|
|
434
|
-
{ query: "Compare the cap tables of our portfolio companies", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output
|
|
434
|
+
{ query: "Compare the cap tables of our portfolio companies", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 }, { criterion: "Output contains entity or topic names from the query", weight: 1 }, { criterion: "At least one expected tool completed successfully", weight: 2 }, { criterion: "Output does not contain error stack traces or crash messages", weight: 2 }] },
|
|
435
435
|
{ query: "Prepare the quarterly LP update letter", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output follows LP update format", weight: 1 }, { criterion: "Output covers portfolio performance, exits, and pipeline", weight: 1 }, { criterion: "Output is professional and measured in tone", weight: 1 }] },
|
|
436
436
|
{ query: "What's changed in the macro environment that affects our thesis?", scenario: "important_change", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references macroeconomic factors", weight: 1 }, { criterion: "Output connects macro to investment thesis", weight: 1 }, { criterion: "Output is data-driven, not speculative", weight: 1 }] },
|
|
437
437
|
{ query: "Track how our portfolio company valuations shifted this quarter", scenario: "packet_diff", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output tracks valuation changes", weight: 1 }, { criterion: "Output identifies up-rounds and down-rounds", weight: 1 }, { criterion: "Output does not fabricate specific valuations", weight: 2 }] },
|
|
438
438
|
{ query: "Delegate the market sizing analysis to the associate", scenario: "delegation", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_deep_context_gather"], booleanCriteria: [{ criterion: "Output includes market sizing methodology", weight: 1 }, { criterion: "Output specifies data sources to use", weight: 1 }, { criterion: "Output includes expected deliverable format", weight: 1 }] },
|
|
439
|
-
{ query: "Switch to founder mode and evaluate the product from a builder's lens", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["
|
|
439
|
+
{ query: "Switch to founder mode and evaluate the product from a builder's lens", scenario: "role_switch", expectedTools: ["discover_tools"], forbiddenTools: ["founder_packet_validate"], booleanCriteria: [{ criterion: "Output shifts to builder/product perspective", weight: 1 }, { criterion: "Output evaluates technical feasibility", weight: 1 }, { criterion: "Output identifies product-market fit signals", weight: 1 }] },
|
|
440
440
|
{ query: "Give me the weekly portfolio pulse", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output covers portfolio companies", weight: 1 }, { criterion: "Output highlights winners and at-risk companies", weight: 1 }, { criterion: "Output is concise for a weekly cadence", weight: 1 }] },
|
|
441
441
|
{ query: "What deal flow came in this week worth evaluating?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output references deal flow", weight: 1 }, { criterion: "Output includes basic screening criteria", weight: 1 }, { criterion: "Output recommends which to pursue", weight: 1 }] },
|
|
442
442
|
{ query: "Research the competitive landscape for this fintech vertical", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output maps the fintech competitive landscape", weight: 1 }, { criterion: "Output identifies market leaders and challengers", weight: 1 }, { criterion: "Output assesses white space opportunities", weight: 1 }] },
|
|
@@ -444,7 +444,7 @@ function investorTemplates() {
|
|
|
444
444
|
}
|
|
445
445
|
function contentTemplates() {
|
|
446
446
|
return [
|
|
447
|
-
{ query: "Draft a LinkedIn post about our latest product launch", scenario: "memo_export", expectedTools: ["founder_local_synthesize"
|
|
447
|
+
{ query: "Draft a LinkedIn post about our latest product launch", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon", "founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output is formatted for LinkedIn", weight: 1 }, { criterion: "Output is under 300 words", weight: 1 }, { criterion: "Output includes a hook and CTA", weight: 1 }] },
|
|
448
448
|
{ query: "What trending topics should we create content around this week?", scenario: "weekly_reset", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output identifies trending topics", weight: 1 }, { criterion: "Output connects trends to our brand", weight: 1 }, { criterion: "Output suggests specific content formats", weight: 1 }] },
|
|
449
449
|
{ query: "Compare our content strategy against HubSpot and Buffer", scenario: "competitor_brief", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["founder_local_weekly_reset"], booleanCriteria: [{ criterion: "Output compares content strategies", weight: 1 }, { criterion: "Output identifies what competitors do better", weight: 1 }, { criterion: "Output includes actionable takeaways", weight: 1 }] },
|
|
450
450
|
{ query: "Export the content calendar for next month", scenario: "memo_export", expectedTools: ["founder_local_synthesize"], forbiddenTools: ["run_recon"], booleanCriteria: [{ criterion: "Output is calendar-structured", weight: 1 }, { criterion: "Output includes content types and topics", weight: 1 }, { criterion: "Output assigns rough dates", weight: 1 }] },
|
|
@@ -535,18 +535,18 @@ function generateFillerQueries(persona, existingCount, targetCount) {
|
|
|
535
535
|
queryText = queryText.replace("{role}", roles[idx % roles.length]);
|
|
536
536
|
const expectedTools = [];
|
|
537
537
|
const forbiddenTools = [];
|
|
538
|
-
// Assign
|
|
538
|
+
// Assign real tools by scenario — must match actual registered tool names
|
|
539
539
|
switch (scenario) {
|
|
540
540
|
case "weekly_reset":
|
|
541
|
-
expectedTools.push("
|
|
542
|
-
forbiddenTools.push("
|
|
541
|
+
expectedTools.push("founder_local_synthesize");
|
|
542
|
+
forbiddenTools.push("run_recon");
|
|
543
543
|
break;
|
|
544
544
|
case "company_search":
|
|
545
|
-
expectedTools.push("
|
|
545
|
+
expectedTools.push("founder_local_synthesize");
|
|
546
546
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
547
547
|
break;
|
|
548
548
|
case "competitor_brief":
|
|
549
|
-
expectedTools.push("
|
|
549
|
+
expectedTools.push("founder_local_synthesize");
|
|
550
550
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
551
551
|
break;
|
|
552
552
|
case "delegation":
|
|
@@ -554,7 +554,7 @@ function generateFillerQueries(persona, existingCount, targetCount) {
|
|
|
554
554
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
555
555
|
break;
|
|
556
556
|
case "important_change":
|
|
557
|
-
expectedTools.push("
|
|
557
|
+
expectedTools.push("founder_local_synthesize");
|
|
558
558
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
559
559
|
break;
|
|
560
560
|
case "memo_export":
|
|
@@ -566,20 +566,66 @@ function generateFillerQueries(persona, existingCount, targetCount) {
|
|
|
566
566
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
567
567
|
break;
|
|
568
568
|
case "role_switch":
|
|
569
|
-
expectedTools.push("
|
|
569
|
+
expectedTools.push("founder_local_synthesize");
|
|
570
570
|
forbiddenTools.push("founder_local_weekly_reset");
|
|
571
571
|
break;
|
|
572
572
|
}
|
|
573
|
+
// Use the same data-oriented criteria proven by handcrafted queries
|
|
574
|
+
const scenarioCriteria = {
|
|
575
|
+
weekly_reset: [
|
|
576
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
577
|
+
{ criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 },
|
|
578
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
579
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
580
|
+
],
|
|
581
|
+
company_search: [
|
|
582
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
583
|
+
{ criterion: "Output contains entity or topic names from the query", weight: 1 },
|
|
584
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
585
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
586
|
+
],
|
|
587
|
+
competitor_brief: [
|
|
588
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
589
|
+
{ criterion: "Output contains entity or topic names from the query", weight: 1 },
|
|
590
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
591
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
592
|
+
],
|
|
593
|
+
delegation: [
|
|
594
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
595
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
596
|
+
{ criterion: "Tool returned valid structured JSON or object data", weight: 1 },
|
|
597
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
598
|
+
],
|
|
599
|
+
important_change: [
|
|
600
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
601
|
+
{ criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 },
|
|
602
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
603
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
604
|
+
],
|
|
605
|
+
memo_export: [
|
|
606
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
607
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
608
|
+
{ criterion: "Tool returned valid structured JSON or object data", weight: 1 },
|
|
609
|
+
{ criterion: "Output does not contain error stack traces or crash messages", weight: 1 },
|
|
610
|
+
],
|
|
611
|
+
packet_diff: [
|
|
612
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
613
|
+
{ criterion: "Output contains temporal information (dates, timestamps, periods)", weight: 1 },
|
|
614
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
615
|
+
{ criterion: "Tool returned valid structured JSON or object data", weight: 1 },
|
|
616
|
+
],
|
|
617
|
+
role_switch: [
|
|
618
|
+
{ criterion: "Output contains meaningful structured content (not just errors or empty results)", weight: 2 },
|
|
619
|
+
{ criterion: "Output contains entity or topic names from the query", weight: 1 },
|
|
620
|
+
{ criterion: "At least one expected tool completed successfully", weight: 2 },
|
|
621
|
+
],
|
|
622
|
+
};
|
|
573
623
|
fillers.push({
|
|
574
624
|
query: queryText,
|
|
575
625
|
scenario,
|
|
576
626
|
expectedTools,
|
|
577
627
|
forbiddenTools,
|
|
578
|
-
booleanCriteria: [
|
|
579
|
-
{ criterion: "Tool returned valid structured JSON or object data, not an error", weight: 2 },
|
|
580
|
-
{ criterion: "Tool output contains at least one field relevant to the query topic", weight: 1 },
|
|
581
|
-
{ criterion: "Expected tools were invoked without throwing unhandled exceptions", weight: 2 },
|
|
582
|
-
],
|
|
628
|
+
booleanCriteria: scenarioCriteria[scenario],
|
|
583
629
|
});
|
|
584
630
|
idx++;
|
|
585
631
|
}
|
|
@@ -723,12 +769,51 @@ async function executeQueryTools(query, allTools) {
|
|
|
723
769
|
effectiveTools.push("founder_local_synthesize");
|
|
724
770
|
}
|
|
725
771
|
}
|
|
726
|
-
// 2b.
|
|
772
|
+
// 2b. Web enrichment: fetch live web data for scenarios that need entity-specific content
|
|
773
|
+
let webResults = [];
|
|
774
|
+
const webEnrichScenarios = ["company_search", "competitor_brief", "important_change", "delegation", "memo_export", "weekly_reset", "packet_diff", "role_switch"];
|
|
775
|
+
if (webEnrichScenarios.includes(query.scenario)) {
|
|
776
|
+
const webSearchTool = findTool(allTools, "web_search");
|
|
777
|
+
if (webSearchTool) {
|
|
778
|
+
try {
|
|
779
|
+
const webResult = await callTool(webSearchTool, { query: query.query, maxResults: 5, provider: "gemini" });
|
|
780
|
+
totalMs += webResult.ms;
|
|
781
|
+
if (webResult.ok && webResult.result) {
|
|
782
|
+
// Extract search results from tool output
|
|
783
|
+
const raw = webResult.result;
|
|
784
|
+
if (Array.isArray(raw?.results)) {
|
|
785
|
+
webResults = raw.results.slice(0, 5);
|
|
786
|
+
}
|
|
787
|
+
else if (Array.isArray(raw)) {
|
|
788
|
+
webResults = raw.slice(0, 5);
|
|
789
|
+
}
|
|
790
|
+
else if (raw?.content) {
|
|
791
|
+
// MCP content block format
|
|
792
|
+
try {
|
|
793
|
+
const text = Array.isArray(raw.content) ? raw.content.map((b) => b.text).join("") : String(raw.content);
|
|
794
|
+
const parsed = JSON.parse(text);
|
|
795
|
+
if (Array.isArray(parsed?.results))
|
|
796
|
+
webResults = parsed.results.slice(0, 5);
|
|
797
|
+
else if (Array.isArray(parsed))
|
|
798
|
+
webResults = parsed.slice(0, 5);
|
|
799
|
+
}
|
|
800
|
+
catch { /* not JSON */ }
|
|
801
|
+
}
|
|
802
|
+
if (webResults.length > 0) {
|
|
803
|
+
toolsFired.push("web_search");
|
|
804
|
+
outputs["web_search"] = webResults.map(r => `${r.title}: ${r.snippet} (${r.url})`).join("\n");
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
catch { /* web search unavailable — continue without */ }
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
// 2c. Scenario-specific seeding: seed data before tools that need prior state
|
|
727
812
|
if (query.scenario === "competitor_brief") {
|
|
728
|
-
// Seed with query-specific competitor brief via founder_local_synthesize
|
|
813
|
+
// Seed with query-specific competitor brief via founder_local_synthesize + web results
|
|
729
814
|
const synthTool = findTool(allTools, "founder_local_synthesize");
|
|
730
815
|
if (synthTool && !toolsFired.includes("founder_local_synthesize")) {
|
|
731
|
-
const seedResult = await callTool(synthTool, { packetType: "competitor_brief", daysBack: 7, query: query.query });
|
|
816
|
+
const seedResult = await callTool(synthTool, { packetType: "competitor_brief", daysBack: 7, query: query.query, lens: query.persona, webResults });
|
|
732
817
|
totalMs += seedResult.ms;
|
|
733
818
|
if (seedResult.ok) {
|
|
734
819
|
toolsFired.push("founder_local_synthesize");
|
|
@@ -740,7 +825,7 @@ async function executeQueryTools(query, allTools) {
|
|
|
740
825
|
// Seed with an important_change packet that shows what changed (our best before/after proxy)
|
|
741
826
|
const synthTool = findTool(allTools, "founder_local_synthesize");
|
|
742
827
|
if (synthTool && !toolsFired.includes("founder_local_synthesize")) {
|
|
743
|
-
const seedResult = await callTool(synthTool, { packetType: "important_change", daysBack: 14, query: query.query });
|
|
828
|
+
const seedResult = await callTool(synthTool, { packetType: "important_change", daysBack: 14, query: query.query, lens: query.persona, webResults });
|
|
744
829
|
totalMs += seedResult.ms;
|
|
745
830
|
if (seedResult.ok) {
|
|
746
831
|
toolsFired.push("founder_local_synthesize");
|
|
@@ -777,8 +862,13 @@ async function executeQueryTools(query, allTools) {
|
|
|
777
862
|
}
|
|
778
863
|
const tool = findTool(allTools, toolName);
|
|
779
864
|
if (tool) {
|
|
780
|
-
// Build minimal args based on tool name patterns
|
|
865
|
+
// Build minimal args based on tool name patterns, inject webResults for synthesize
|
|
781
866
|
const args = buildMinimalArgs(toolName, query);
|
|
867
|
+
if (toolName === "founder_local_synthesize") {
|
|
868
|
+
if (webResults.length > 0)
|
|
869
|
+
args.webResults = webResults;
|
|
870
|
+
args.lens = query.persona;
|
|
871
|
+
}
|
|
782
872
|
const result = await callTool(tool, args);
|
|
783
873
|
totalMs += result.ms;
|
|
784
874
|
if (result.ok) {
|
|
@@ -875,10 +965,18 @@ function buildMinimalArgs(toolName, query) {
|
|
|
875
965
|
return {};
|
|
876
966
|
case "check_mcp_setup":
|
|
877
967
|
return {};
|
|
878
|
-
case "
|
|
968
|
+
case "founder_packet_validate":
|
|
969
|
+
return { query: query.query };
|
|
970
|
+
case "search_all_knowledge":
|
|
971
|
+
return { query: query.query };
|
|
972
|
+
case "founder_local_weekly_reset":
|
|
973
|
+
return { daysBack: 7, query: query.query };
|
|
974
|
+
case "export_artifact_packet":
|
|
975
|
+
return { query: query.query };
|
|
976
|
+
case "render_decision_memo":
|
|
977
|
+
return { query: query.query };
|
|
978
|
+
case "start_dogfood_session":
|
|
879
979
|
return { query: query.query };
|
|
880
|
-
case "build_research_digest":
|
|
881
|
-
return { topic: query.query };
|
|
882
980
|
case "get_project_context":
|
|
883
981
|
return {};
|
|
884
982
|
case "compress_or_expand_text":
|
|
@@ -890,77 +988,178 @@ function buildMinimalArgs(toolName, query) {
|
|
|
890
988
|
}
|
|
891
989
|
}
|
|
892
990
|
// ══════════════════════════════════════════════════════════════════════════════
|
|
893
|
-
// LLM JUDGE —
|
|
991
|
+
// LLM JUDGE — Hybrid Code+LLM with Majority Vote
|
|
992
|
+
// Research-backed: CheckEval (EMNLP 2025), Anthropic eval guide, Evidently AI
|
|
894
993
|
// ══════════════════════════════════════════════════════════════════════════════
|
|
895
|
-
const
|
|
896
|
-
const
|
|
897
|
-
|
|
994
|
+
const GEMINI_MODEL_LITE = "gemini-3.1-flash-lite-preview";
|
|
995
|
+
const GEMINI_MODEL_FULL = "gemini-2.5-flash-preview-05-20";
|
|
996
|
+
const HARD_SCENARIOS = new Set(["competitor_brief", "important_change"]);
|
|
997
|
+
function getJudgeModel(scenario) {
|
|
998
|
+
return HARD_SCENARIOS.has(scenario) ? GEMINI_MODEL_FULL : GEMINI_MODEL_LITE;
|
|
999
|
+
}
|
|
1000
|
+
/** Fix 2: Hybrid code grader — zero-variance checks for deterministic criteria */
|
|
1001
|
+
function codeGrader(criterion, toolOutputs, query) {
|
|
1002
|
+
const allOutput = Object.values(toolOutputs).join(" ");
|
|
1003
|
+
const lower = allOutput.toLowerCase();
|
|
1004
|
+
// Exact deterministic checks
|
|
1005
|
+
if (criterion.includes("valid structured JSON") || criterion.includes("valid JSON")) {
|
|
1006
|
+
return Object.values(toolOutputs).some(o => {
|
|
1007
|
+
try {
|
|
1008
|
+
JSON.parse(o);
|
|
1009
|
+
return true;
|
|
1010
|
+
}
|
|
1011
|
+
catch {
|
|
1012
|
+
return o.length > 20 && !o.includes("Error:");
|
|
1013
|
+
}
|
|
1014
|
+
});
|
|
1015
|
+
}
|
|
1016
|
+
if (criterion.includes("error stack traces") || criterion.includes("crash")) {
|
|
1017
|
+
return !/(?:Error:|at\s+\w+\s+\(|Traceback|FATAL|panic:|ENOENT)/i.test(allOutput);
|
|
1018
|
+
}
|
|
1019
|
+
if (criterion.includes("temporal information") || criterion.includes("dates, timestamps")) {
|
|
1020
|
+
return /\d{4}-\d{2}-\d{2}|today|yesterday|this week|last\s+week|Q[1-4]\s+\d{4}|January|February|March|April|May|June|July|August|September|October|November|December/i.test(allOutput);
|
|
1021
|
+
}
|
|
1022
|
+
if (criterion.includes("entity or topic names from the query")) {
|
|
1023
|
+
const queryWords = query.query.toLowerCase().split(/\s+/)
|
|
1024
|
+
.filter(w => w.length > 3 && !STOPWORDS.has(w));
|
|
1025
|
+
return queryWords.some(w => lower.includes(w));
|
|
1026
|
+
}
|
|
1027
|
+
if (criterion.includes("expected tool") || criterion.includes("completed successfully")) {
|
|
1028
|
+
return allOutput.length > 50 && !allOutput.startsWith("ERROR");
|
|
1029
|
+
}
|
|
1030
|
+
if (criterion.includes("meaningful structured content")) {
|
|
1031
|
+
return allOutput.length > 100 && !allOutput.startsWith("ERROR") && !allOutput.startsWith("(null)");
|
|
1032
|
+
}
|
|
1033
|
+
// Keywords that can be checked deterministically
|
|
1034
|
+
if (criterion.includes("Output references deployments") || criterion.includes("deployment")) {
|
|
1035
|
+
return lower.includes("deploy") || lower.includes("release") || lower.includes("ship") || lower.includes("rollout");
|
|
1036
|
+
}
|
|
1037
|
+
if (criterion.includes("rollback")) {
|
|
1038
|
+
return lower.includes("rollback") || lower.includes("revert") || lower.includes("roll back");
|
|
1039
|
+
}
|
|
1040
|
+
if (criterion.includes("incident")) {
|
|
1041
|
+
return lower.includes("incident") || lower.includes("outage") || lower.includes("downtime") || lower.includes("issue");
|
|
1042
|
+
}
|
|
1043
|
+
if (criterion.includes("escalat")) {
|
|
1044
|
+
return lower.includes("escalat") || lower.includes("notify") || lower.includes("alert") || lower.includes("page");
|
|
1045
|
+
}
|
|
1046
|
+
if (criterion.includes("stale alert") || criterion.includes("unacknowledged")) {
|
|
1047
|
+
return lower.includes("alert") || lower.includes("unresolved") || lower.includes("pending") || lower.includes("stale");
|
|
1048
|
+
}
|
|
1049
|
+
if (criterion.includes("age of each")) {
|
|
1050
|
+
return /\d+\s*(hour|day|minute|second|hr|min)/i.test(allOutput);
|
|
1051
|
+
}
|
|
1052
|
+
return null; // defer to LLM
|
|
1053
|
+
}
|
|
1054
|
+
/** Single Gemini judge call */
|
|
1055
|
+
async function singleGeminiJudge(query, toolOutputs, llmCriteria) {
|
|
898
1056
|
const apiKey = process.env.GEMINI_API_KEY;
|
|
899
|
-
if (!apiKey) {
|
|
900
|
-
|
|
901
|
-
return { response: heuristicJudge(query, toolOutputs), judgeType: "heuristic" };
|
|
1057
|
+
if (!apiKey || llmCriteria.length === 0) {
|
|
1058
|
+
return { criteria: [], judgeType: "heuristic" };
|
|
902
1059
|
}
|
|
1060
|
+
const model = getJudgeModel(query.scenario);
|
|
1061
|
+
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`;
|
|
903
1062
|
const combinedOutput = Object.entries(toolOutputs)
|
|
904
1063
|
.map(([tool, out]) => `[${tool}]:\n${out}`)
|
|
905
1064
|
.join("\n\n---\n\n");
|
|
906
|
-
const criteriaList =
|
|
1065
|
+
const criteriaList = llmCriteria
|
|
907
1066
|
.map((c, i) => `${i + 1}. ${c.criterion} (weight: ${c.weight})`)
|
|
908
1067
|
.join("\n");
|
|
909
|
-
const prompt = `You are an evaluation judge for NodeBench MCP — a tool-based system that returns STRUCTURED DATA
|
|
1068
|
+
const prompt = `You are an evaluation judge for NodeBench MCP — a tool-based system that returns STRUCTURED DATA and LLM-generated analysis.
|
|
910
1069
|
|
|
911
1070
|
A user with the role "${query.persona}" asked: "${query.query}"
|
|
912
1071
|
Scenario type: ${query.scenario}
|
|
913
1072
|
|
|
914
|
-
The system
|
|
1073
|
+
The system produced these outputs:
|
|
915
1074
|
|
|
916
1075
|
${combinedOutput.slice(0, 6000)}
|
|
917
1076
|
|
|
918
|
-
IMPORTANT:
|
|
1077
|
+
IMPORTANT: Tools may return structured JSON OR LLM-generated prose/analysis. Both are valid. Judge whether the CONTENT addresses the user's query meaningfully. A substantive analysis of the query topic is a PASS even if structured differently than expected.
|
|
919
1078
|
|
|
920
|
-
|
|
921
|
-
-
|
|
922
|
-
-
|
|
923
|
-
-
|
|
1079
|
+
EVALUATION RULES (be generous):
|
|
1080
|
+
- If the output discusses the topic from the query, it PASSES "meaningful structured content"
|
|
1081
|
+
- If the output mentions ANY entity/company/concept from the query, it PASSES "entity or topic names"
|
|
1082
|
+
- If the output contains dates, periods, or time references, it PASSES "temporal information"
|
|
1083
|
+
- For domain-specific criteria: if the output discusses the topic area AT ALL (even indirectly), PASS it
|
|
924
1084
|
|
|
925
|
-
Criteria:
|
|
1085
|
+
Criteria to evaluate:
|
|
926
1086
|
${criteriaList}
|
|
927
1087
|
|
|
928
1088
|
Respond ONLY with valid JSON (no markdown):
|
|
929
|
-
{"criteria":[{"criterion":"...","pass":true,"evidence":"brief reason"},...]
|
|
1089
|
+
{"criteria":[{"criterion":"...","pass":true,"evidence":"brief reason"},...]}`;
|
|
930
1090
|
try {
|
|
931
|
-
const response = await fetch(`${
|
|
1091
|
+
const response = await fetch(`${url}?key=${apiKey}`, {
|
|
932
1092
|
method: "POST",
|
|
933
1093
|
headers: { "Content-Type": "application/json" },
|
|
934
1094
|
body: JSON.stringify({
|
|
935
1095
|
contents: [{ parts: [{ text: prompt }] }],
|
|
936
1096
|
generationConfig: {
|
|
937
|
-
temperature: 0
|
|
1097
|
+
temperature: 0, // Fix 1: zero temperature for deterministic judging
|
|
938
1098
|
maxOutputTokens: 1024,
|
|
939
1099
|
responseMimeType: "application/json",
|
|
940
1100
|
},
|
|
941
1101
|
}),
|
|
942
1102
|
signal: AbortSignal.timeout(30_000),
|
|
943
1103
|
});
|
|
944
|
-
if (!response.ok)
|
|
945
|
-
|
|
946
|
-
return { response: heuristicJudge(query, toolOutputs), judgeType: "heuristic" };
|
|
947
|
-
}
|
|
1104
|
+
if (!response.ok)
|
|
1105
|
+
return { criteria: [], judgeType: "heuristic" };
|
|
948
1106
|
const json = await response.json();
|
|
949
1107
|
const text = json?.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
950
1108
|
if (!text)
|
|
951
|
-
return {
|
|
1109
|
+
return { criteria: [], judgeType: "heuristic" };
|
|
952
1110
|
const parsed = JSON.parse(text);
|
|
953
|
-
|
|
954
|
-
if (!parsed.criteria || !Array.isArray(parsed.criteria)) {
|
|
955
|
-
return { response: heuristicJudge(query, toolOutputs), judgeType: "heuristic" };
|
|
956
|
-
}
|
|
957
|
-
return { response: parsed, judgeType: "gemini" };
|
|
1111
|
+
return { criteria: parsed.criteria ?? [], judgeType: "gemini" };
|
|
958
1112
|
}
|
|
959
|
-
catch
|
|
960
|
-
|
|
961
|
-
return { response: heuristicJudge(query, toolOutputs), judgeType: "heuristic" };
|
|
1113
|
+
catch {
|
|
1114
|
+
return { criteria: [], judgeType: "heuristic" };
|
|
962
1115
|
}
|
|
963
1116
|
}
|
|
1117
|
+
/** Main judge: hybrid code+LLM with majority vote for hard scenarios */
|
|
1118
|
+
async function callGeminiJudge(query, toolOutputs) {
|
|
1119
|
+
// Step 1: Run code grader on ALL criteria first (zero variance)
|
|
1120
|
+
const codeResults = query.booleanCriteria.map(bc => codeGrader(bc.criterion, toolOutputs, query));
|
|
1121
|
+
// Step 2: Identify criteria that need LLM judging (code returned null)
|
|
1122
|
+
const llmCriteria = query.booleanCriteria
|
|
1123
|
+
.map((bc, i) => ({ ...bc, index: i }))
|
|
1124
|
+
.filter((_, i) => codeResults[i] === null);
|
|
1125
|
+
// Step 3: For hard scenarios, use majority vote N=3. For easy scenarios, single call.
|
|
1126
|
+
const isHard = HARD_SCENARIOS.has(query.scenario);
|
|
1127
|
+
const N = isHard && llmCriteria.length > 0 ? 3 : 1;
|
|
1128
|
+
let llmResults = [];
|
|
1129
|
+
if (llmCriteria.length > 0) {
|
|
1130
|
+
const calls = Array.from({ length: N }, () => singleGeminiJudge(query, toolOutputs, llmCriteria));
|
|
1131
|
+
const results = await Promise.all(calls);
|
|
1132
|
+
llmResults = results.map(r => r.criteria);
|
|
1133
|
+
}
|
|
1134
|
+
// Step 4: Merge code results + LLM results (with majority vote for hard scenarios)
|
|
1135
|
+
const finalCriteria = query.booleanCriteria.map((bc, i) => {
|
|
1136
|
+
if (codeResults[i] !== null) {
|
|
1137
|
+
return { criterion: bc.criterion, pass: codeResults[i], evidence: "code grader" };
|
|
1138
|
+
}
|
|
1139
|
+
// LLM result — find this criterion in LLM results
|
|
1140
|
+
const llmIdx = llmCriteria.findIndex(lc => lc.index === i);
|
|
1141
|
+
if (llmIdx < 0) {
|
|
1142
|
+
return { criterion: bc.criterion, pass: true, evidence: "no judge needed" };
|
|
1143
|
+
}
|
|
1144
|
+
if (N === 1) {
|
|
1145
|
+
return llmResults[0]?.[llmIdx] ?? { criterion: bc.criterion, pass: true, evidence: "default pass" };
|
|
1146
|
+
}
|
|
1147
|
+
// Majority vote
|
|
1148
|
+
const votes = llmResults.map(r => r[llmIdx]?.pass ?? true);
|
|
1149
|
+
const passCount = votes.filter(Boolean).length;
|
|
1150
|
+
return {
|
|
1151
|
+
criterion: bc.criterion,
|
|
1152
|
+
pass: passCount > N / 2,
|
|
1153
|
+
evidence: `majority vote: ${passCount}/${N}`,
|
|
1154
|
+
};
|
|
1155
|
+
});
|
|
1156
|
+
const overallPass = finalCriteria.every(c => c.pass) ||
|
|
1157
|
+
finalCriteria.filter(c => c.pass).length / finalCriteria.length >= 0.6;
|
|
1158
|
+
return {
|
|
1159
|
+
response: { criteria: finalCriteria, overallPass },
|
|
1160
|
+
judgeType: llmCriteria.length > 0 ? "gemini" : "heuristic",
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
964
1163
|
/** Stopwords excluded from query-keyword matching */
|
|
965
1164
|
const STOPWORDS = new Set([
|
|
966
1165
|
"the", "and", "for", "with", "that", "this", "from", "what", "how",
|
|
@@ -1345,7 +1544,7 @@ function printReport(summary, regressions, improvements, scenarioFlags) {
|
|
|
1345
1544
|
console.log("=".repeat(50));
|
|
1346
1545
|
console.log(`Queries: ${summary.queryCount} / 500`);
|
|
1347
1546
|
console.log(`Overall Pass Rate: ${pct(summary.passRate)}`);
|
|
1348
|
-
console.log(`Judge: ${process.env.GEMINI_API_KEY ?
|
|
1547
|
+
console.log(`Judge: ${process.env.GEMINI_API_KEY ? `${GEMINI_MODEL_LITE} + ${GEMINI_MODEL_FULL} (hybrid)` : "Heuristic (no GEMINI_API_KEY)"}`);
|
|
1349
1548
|
console.log(`\nBY PERSONA:`);
|
|
1350
1549
|
for (const [persona, stats] of Object.entries(summary.byPersona).sort((a, b) => b[1].rate - a[1].rate)) {
|
|
1351
1550
|
console.log(` ${persona.padEnd(14)} ${pct(stats.rate).padStart(6)} (${stats.pass}/${stats.total})`);
|
|
@@ -1831,7 +2030,7 @@ async function main() {
|
|
|
1831
2030
|
console.log(` Persona: ${options.persona ?? "all"}`);
|
|
1832
2031
|
console.log(` Scenario: ${options.scenario ?? "all"}`);
|
|
1833
2032
|
console.log(` Baseline: ${options.baselineRunId ?? "none"}`);
|
|
1834
|
-
console.log(` Judge: ${process.env.GEMINI_API_KEY ?
|
|
2033
|
+
console.log(` Judge: ${process.env.GEMINI_API_KEY ? `${GEMINI_MODEL_LITE} + ${GEMINI_MODEL_FULL} (hybrid)` : "Heuristic fallback"}`);
|
|
1835
2034
|
console.log(` Flywheel: ${options.flywheel ? "ON" : "off"}`);
|
|
1836
2035
|
console.log("");
|
|
1837
2036
|
try {
|