nodebench-mcp 2.51.0 → 2.53.0

package/dist/benchmarks/cohorts.js

@@ -0,0 +1,143 @@
+/**
+ * cohorts.ts — Canonical cohort definitions for N=1, N=5, N=10, N=100 benchmarks.
+ *
+ * N=1   proves it can work
+ * N=5   proves it is not just for you
+ * N=10  proves it survives role and session variance
+ * N=100 proves it compounds over time
+ */
+/* ─── Canonical Users ────────────────────────────────────────────────────── */
+const USERS = {
+    founder_homen: {
+        userId: "u_founder_homen",
+        role: "founder",
+        name: "Homen (Founder)",
+        description: "Solo technical founder building NodeBench. Weekly reset, delegation, strategy.",
+        scenarios: ["weekly_reset", "pre_delegation", "important_change", "competitor_brief", "packet_diff"],
+        primaryEntity: "NodeBench",
+    },
+    banker_sarah: {
+        userId: "u_banker_sarah",
+        role: "banker",
+        name: "Sarah (Banker)",
+        description: "Investment banker evaluating AI infrastructure companies for deal flow.",
+        scenarios: ["company_search", "competitor_brief", "memo_export", "role_switch"],
+        primaryEntity: "Anthropic",
+    },
+    ceo_marcus: {
+        userId: "u_ceo_marcus",
+        role: "ceo",
+        name: "Marcus (CEO)",
+        description: "CEO of a mid-stage startup. Quarterly strategy, board narrative, resource allocation.",
+        scenarios: ["weekly_reset", "important_change", "company_search", "memo_export"],
+        primaryEntity: "Shopify",
+    },
+    researcher_lin: {
+        userId: "u_researcher_lin",
+        role: "researcher",
+        name: "Lin (Researcher)",
+        description: "AI strategy analyst. Competitor intelligence, market mapping, evidence synthesis.",
+        scenarios: ["competitor_brief", "company_search", "uploaded_notes", "html_export"],
+        primaryEntity: "Supermemory",
+    },
+    student_aisha: {
+        userId: "u_student_aisha",
+        role: "student",
+        name: "Aisha (Student)",
+        description: "MBA student studying AI commerce strategy. Needs citation-friendly study briefs.",
+        scenarios: ["company_search", "uploaded_notes", "memo_export", "role_switch"],
+        primaryEntity: "Shopify",
+    },
+    legal_david: {
+        userId: "u_legal_david",
+        role: "legal",
+        name: "David (Legal)",
+        description: "In-house counsel reviewing AI partnerships. Regulatory exposure, governance, IP.",
+        scenarios: ["company_search", "important_change", "memo_export"],
+        primaryEntity: "Anthropic",
+    },
+    pm_rachel: {
+        userId: "u_pm_rachel",
+        role: "pm",
+        name: "Rachel (PM)",
+        description: "Product manager at an AI-native company. Feature prioritization, competitor tracking.",
+        scenarios: ["weekly_reset", "competitor_brief", "important_change", "packet_diff"],
+        primaryEntity: "NodeBench",
+    },
+    contractor_kai: {
+        userId: "u_contractor_kai",
+        role: "contractor",
+        name: "Kai (Contractor)",
+        description: "Freelance developer receiving delegation packets. Needs scoped context without full history.",
+        scenarios: ["pre_delegation", "packet_diff", "uploaded_notes"],
+        primaryEntity: "NodeBench",
+    },
+    investor_priya: {
+        userId: "u_investor_priya",
+        role: "investor",
+        name: "Priya (Investor)",
+        description: "VC partner evaluating AI infrastructure deals. Pipeline tracking, comparables, diligence.",
+        scenarios: ["company_search", "competitor_brief", "memo_export", "role_switch"],
+        primaryEntity: "Anthropic",
+    },
+    content_james: {
+        userId: "u_content_james",
+        role: "content",
+        name: "James (Content)",
+        description: "Content strategist tracking AI industry trends. Post performance, audience analysis.",
+        scenarios: ["company_search", "important_change", "competitor_brief", "html_export"],
+        primaryEntity: "Shopify",
+    },
+};
+/* ─── N=1 — Golden Path ──────────────────────────────────────────────────── */
+export const COHORT_N1 = {
+    cohortId: "cohort_n1_golden_path",
+    layer: "N1",
+    users: [USERS.founder_homen],
+    sessionsPerUser: 1,
+    timeHorizons: ["same_session"],
+    description: "Single founder run: weekly reset → memo export → pre-delegation → important-change. Proves the golden path works.",
+};
+/* ─── N=5 — Role Generalization ──────────────────────────────────────────── */
+export const COHORT_N5 = {
+    cohortId: "cohort_n5_role_variance",
+    layer: "N5",
+    users: [
+        USERS.founder_homen,
+        USERS.banker_sarah,
+        USERS.ceo_marcus,
+        USERS.researcher_lin,
+        USERS.student_aisha,
+    ],
+    sessionsPerUser: 1,
+    timeHorizons: ["same_session"],
+    description: "5 users with different roles analyze the same entity. Proves role adaptation without hallucination.",
+};
+/* ─── N=10 — Repeated-Session Stability ──────────────────────────────────── */
+export const COHORT_N10 = {
+    cohortId: "cohort_n10_session_stability",
+    layer: "N10",
+    users: [
+        USERS.founder_homen,
+        USERS.banker_sarah,
+        USERS.ceo_marcus,
+        USERS.researcher_lin,
+        USERS.student_aisha,
+    ],
+    sessionsPerUser: 2,
+    timeHorizons: ["same_session", "next_day"],
+    description: "5 users × 2 sessions each. Session 2 tests: remembered context, surfaced delta, refreshed packet. Proves session continuity.",
+};
+/* ─── N=100 — Longitudinal Compounding ───────────────────────────────────── */
+export const COHORT_N100 = {
+    cohortId: "cohort_n100_longitudinal",
+    layer: "N100",
+    users: Object.values(USERS),
+    sessionsPerUser: 10,
+    timeHorizons: ["same_session", "same_day", "next_day", "weekly", "monthly", "quarterly"],
+    description: "10 users × 10 sessions across time horizons. Tests compounding memory, packet reuse, repeat-cognition avoidance, regression resistance.",
+};
+/* ─── All Cohorts ────────────────────────────────────────────────────────── */
+export const ALL_COHORTS = [COHORT_N1, COHORT_N5, COHORT_N10, COHORT_N100];
+export const ALL_USERS = USERS;
+//# sourceMappingURL=cohorts.js.map

package/dist/benchmarks/cohorts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cohorts.js","sourceRoot":"","sources":["../../src/benchmarks/cohorts.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,gFAAgF;AAEhF,MAAM,KAAK,GAAkC;IAC3C,aAAa,EAAE;QACb,MAAM,EAAE,iBAAiB;QACzB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,gFAAgF;QAC7F,SAAS,EAAE,CAAC,cAAc,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,CAAC;QACpG,aAAa,EAAE,WAAW;KAC3B;IACD,YAAY,EAAE;QACZ,MAAM,EAAE,gBAAgB;QACxB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,yEAAyE;QACtF,SAAS,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,CAAC;QAC/E,aAAa,EAAE,WAAW;KAC3B;IACD,UAAU,EAAE;QACV,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,uFAAuF;QACpG,SAAS,EAAE,CAAC,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,aAAa,CAAC;QAChF,aAAa,EAAE,SAAS;KACzB;IACD,cAAc,EAAE;QACd,MAAM,EAAE,kBAAkB;QAC1B,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,mFAAmF;QAChG,SAAS,EAAE,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,aAAa,CAAC;QAClF,aAAa,EAAE,aAAa;KAC7B;IACD,aAAa,EAAE;QACb,MAAM,EAAE,iBAAiB;QACzB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,kFAAkF;QAC/F,SAAS,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,CAAC;QAC7E,aAAa,EAAE,SAAS;KACzB;IACD,WAAW,EAAE;QACX,MAAM,EAAE,eAAe;QACvB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,kFAAkF;QAC/F,SAAS,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,CAAC;QAChE,aAAa,EAAE,WAAW;KAC3B;IACD,SAAS,EAAE;QACT,MAAM,EAAE,aAAa;QACrB,IAAI,EAAE,IAAI;QACV,IAAI,EAAE,aAAa;QACnB,WAAW,EAAE,uFAAuF;QACpG,SAAS,EAAE,CAAC,cAAc,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,CAAC;QAClF,aAAa,EAAE,WAAW;KAC3B;IACD,cAAc,EAAE;QACd,MAAM,EAAE,kBAAkB;QAC1B,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,8FAA8F;QAC3G,SAAS,EAAE,CAAC,gBAAgB,EAAE,aAAa,EAAE,gBAAgB,CAAC;QAC9D,aAAa,EAAE,WAAW;KAC3B;IACD,cAAc,EAAE;QACd,MAAM,EAAE,kBAAkB;QAC1B,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,2FAA2F;QACxG,SAAS,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,CAAC;QAC/E,aAAa,EAAE,WAAW;KAC3B;IACD,aAAa,EAAE;QACb,MAAM,EAAE,iBAAiB;QACzB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,sFAAsF;QACnG,SAAS,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,aAAa,CAAC;QACpF,aAAa,EAAE,SAAS;KACzB;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,SAAS,GAAoB;IACxC,QAAQ,EAAE,uBAAuB;IACjC,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;IAC5B,eAAe,EAAE,CAAC;IAClB,YAAY,EAAE,CAAC,cAAc,CAAC;IAC9B,WAAW,EAAE,mHAAmH;CACjI,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,SAAS,GAAoB;IACxC,QAAQ,EAAE,yBAAyB;IACnC,KAAK,EAAE,IAAI;IACX,KAAK,EAAE;QACL,KAAK,CAAC,aAAa;QACnB,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,UAAU;QAChB,KAAK,CAAC,cAAc;QACpB,KAAK,CAAC,aAAa;KACpB;IACD,eAAe,EAAE,CAAC;IAClB,YAAY,EAAE,CAAC,cAAc,CAAC;IAC9B,WAAW,EAAE,qGAAqG;CACnH,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,UAAU,GAAoB;IACzC,QAAQ,EAAE,8BAA8B;IACxC,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE;QACL,KAAK,CAAC,aAAa;QACnB,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,UAAU;QAChB,KAAK,CAAC,cAAc;QACpB,KAAK,CAAC,aAAa;KACpB;IACD,eAAe,EAAE,CAAC;IAClB,YAAY,EAAE,CAAC,cAAc,EAAE,UAAU,CAAC;IAC1C,WAAW,EAAE,8HAA8H;CAC5I,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAoB;IAC1C,QAAQ,EAAE,0BAA0B;IACpC,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;IAC3B,eAAe,EAAE,EAAE;IACnB,YAAY,EAAE,CAAC,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC;IACxF,WAAW,EAAE,yIAAyI;CACvJ,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAsB,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAC9F,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC"}

package/dist/benchmarks/longitudinalHarness.d.ts

@@ -0,0 +1,91 @@
+#!/usr/bin/env npx tsx
+/**
+ * longitudinalHarness.ts — Longitudinal dogfood benchmark harness for NodeBench MCP.
+ *
+ * Orchestrates N=1, N=5, N=10, N=100 cohort evaluations to measure whether
+ * NodeBench compounds value across users, roles, sessions, and time horizons.
+ *
+ * Core metrics:
+ *   RCA  — Repeated Cognition Avoided (% of sessions not restating prior context)
+ *   PRR  — Packet Reuse Rate (% of sessions reusing a prior packet)
+ *
+ * Usage:
+ *   cd packages/mcp-local && npx tsx src/benchmarks/longitudinalHarness.ts [n1|n5|n10|n100|all]
+ */
+export interface CohortUser {
+    userId: string;
+    role: "founder" | "banker" | "ceo" | "researcher" | "student" | "legal" | "pm" | "contractor" | "investor" | "content";
+    preset: string;
+    typicalScenarios: string[];
+}
+export interface SessionRun {
+    runId: string;
+    userId: string;
+    role: string;
+    scenarioId: string;
+    sessionIndex: number;
+    timeHorizon: "same_session" | "same_day" | "next_day" | "weekly" | "monthly" | "quarterly" | "yearly";
+    surface: "mcp" | "ai_app" | "local_dashboard" | "engine_api";
+    toolCallCount: number;
+    latencyMs: number;
+    packetGenerated: boolean;
+    packetReused: boolean;
+    repeatQuestionDetected: boolean;
+    contextRestated: boolean;
+    exportProduced: boolean;
+    judgeScore: number;
+    errors: string[];
+}
+export interface CohortReport {
+    cohortSize: number;
+    totalSessions: number;
+    rolesCovered: string[];
+    coreLoopsCovered: number;
+    repeatedCognitionAvoided: number;
+    packetReuseRate: number;
+    importantChangePrecision: number;
+    contradictionPrecision: number;
+    falseAlertRate: number;
+    exportToActionRate: number;
+    topRecurringRootCause: string;
+    topRegressionRisk: string;
+    passed: boolean;
+    passThresholds: Record<string, number>;
+}
+/**
+ * Repeated Cognition Avoided (RCA): % of sessions where the user did NOT
+ * restate context or re-ask old questions.
+ */
+export declare function computeRCA(sessions: SessionRun[]): number;
+/**
+ * Packet Reuse Rate (PRR): % of sessions where a prior packet was reused
+ * rather than regenerated from scratch.
+ */
+export declare function computePRR(sessions: SessionRun[]): number;
+/**
+ * Aggregate all metrics from a batch of sessions into a CohortReport.
+ */
+export declare function generateCohortReport(sessions: SessionRun[], cohortSize: number, layer: "n1" | "n5" | "n10" | "n100"): CohortReport;
+/**
+ * N=1: Single golden-path founder run.
+ * Validates that the core tool chain works end-to-end for one scenario.
+ */
+export declare function runN1(): Promise<{
+    session: SessionRun;
+    report: CohortReport;
+}>;
+/**
+ * N=5: 5 users (founder, banker, ceo, researcher, student), 1 session each.
+ * All against the same entity "Anthropic".
+ */
+export declare function runN5(): Promise<CohortReport>;
+/**
+ * N=10: 10 users x 1 session each OR 5 users x 2 sessions.
+ * Tests session-continuity metrics.
+ */
+export declare function runN10(): Promise<CohortReport>;
+/**
+ * N=100: 10 users x 10 sessions each (simulated across time horizons).
+ * Measures RCA + PRR compounding over time.
+ */
+export declare function runN100(): Promise<CohortReport>;