@besales/mcp 0.45.0 → 0.47.0

package/scripts/run-model-eval.js

@@ -0,0 +1,447 @@
+#!/usr/bin/env node
+
+import { spawn } from 'node:child_process';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
+const currentRoot = resolve(process.env.BESALES_MCP_EVAL_CURRENT ?? '.');
+const baselineRoot = resolve(
+  process.env.BESALES_MCP_EVAL_BASELINE ??
+    '/Users/peresvets/apps/animaly/.codex-worktrees/besales-mcp-baseline',
+);
+const serverScript = resolve(currentRoot, 'scripts/model-eval-server.js');
+const outputRoot = resolve(
+  process.env.BESALES_MCP_EVAL_OUTPUT ??
+    resolve(currentRoot, 'docs/evidence/model-eval-2026-09-11'),
+);
+const emptyRoot = resolve('/tmp/besales-model-eval-empty');
+const requestedMode = process.argv[2] ?? 'all';
+const repetitions = Number(process.env.BESALES_MCP_EVAL_REPETITIONS ?? 3);
+const model = process.env.BESALES_MCP_EVAL_MODEL ?? 'gpt-5.6-luna';
+const reasoningEffort = process.env.BESALES_MCP_EVAL_REASONING ?? 'low';
+const requestedScenarios = new Set(
+  (process.env.BESALES_MCP_EVAL_SCENARIOS ?? '')
+    .split(',')
+    .map((scenario) => scenario.trim())
+    .filter(Boolean),
+);
+
+const scenarios = [
+  {
+    id: 'faq',
+    toolsets: 'workspace,help',
+    prompt:
+      'В тестовом read-only контуре Besales MCP пользователь пишет: «Хочу нового FAQ-бота для Telegram по ответам из базы знаний». Проверь фактический маршрут через доступный Besales MCP обычным пользовательским запросом, не угадывай его по общим знаниям. Составь проверяемую последовательность и остановись перед первой записью. Не проси технические ID и ничего не изменяй.',
+  },
+  {
+    id: 'analytics',
+    toolsets: 'workspace,help,broadcast',
+    prompt:
+      'В тестовом read-only контуре Besales MCP пользователь пишет: «Покажи аналитику рассылок за 1–10 сентября: отправлено, ответы, конверсия и выручка по каналам». Проверь фактические данные через доступный Besales MCP и выполни только нужные чтения. Сохрани правильные период, view и зерно; ничего не изменяй.',
+  },
+  {
+    id: 'silence',
+    toolsets: 'workspace,help,contact',
+    prompt:
+      'В тестовом read-only контуре Besales MCP пользователь спрашивает: «бот молчит у клиента Test Contact, почему?». Проверь фактическое состояние через доступный Besales MCP обычным пользовательским запросом, ничего не изменяй и дай краткий проверяемый ответ.',
+  },
+  {
+    id: 'followup',
+    toolsets: 'workspace,help,followup',
+    prompt:
+      'В тестовом read-only контуре Besales MCP пользователь спрашивает: «Какие у нас работают дожимы, и на каком шаге люди отвечают?». Проверь фактические цепочки и статистику через доступный Besales MCP, отличи выключенный черновик от сбоя и ничего не изменяй.',
+  },
+  {
+    id: 'unsupported',
+    toolsets: 'workspace,help',
+    prompt:
+      'В тестовом read-only контуре Besales MCP пользователь просит: «Измени темы диалогов и покажи satisfaction». Проверь доступные возможности через Besales MCP. Если scoped MCP-пути нет, честно назови границу и условие продолжения; не выдумывай tool или внутренний URL и ничего не изменяй.',
+  },
+];
+
+const selectedScenarios = scenarios.filter(
+  (scenario) => requestedScenarios.size === 0 || requestedScenarios.has(scenario.id),
+);
+
+const scoreOnly = requestedMode === 'score';
+const modes = requestedMode === 'all' ? ['baseline', 'current'] : scoreOnly ? [] : [requestedMode];
+if (!modes.every((mode) => mode === 'baseline' || mode === 'current')) {
+  throw new Error('Usage: node scripts/run-model-eval.js [baseline|current|all|score]');
+}
+if (!Number.isInteger(repetitions) || repetitions < 1 || repetitions > 10) {
+  throw new Error('BESALES_MCP_EVAL_REPETITIONS must be an integer from 1 to 10');
+}
+if (selectedScenarios.length === 0) {
+  throw new Error(
+    `BESALES_MCP_EVAL_SCENARIOS must contain one of: ${scenarios.map((scenario) => scenario.id).join(', ')}`,
+  );
+}
+
+await mkdir(outputRoot, { recursive: true });
+await mkdir(emptyRoot, { recursive: true });
+
+const existingResults =
+  modes.length < 2
+    ? await readFile(resolve(outputRoot, 'results.json'), 'utf8')
+        .then((value) => JSON.parse(value).results ?? [])
+        .catch(() => [])
+    : [];
+const selectedScenarioIds = new Set(selectedScenarios.map((scenario) => scenario.id));
+const results = scoreOnly
+  ? await Promise.all(existingResults.map((row) => {
+      const prefix = `${row.mode}-${row.scenario}-${row.repetition}`;
+      return collectMetrics({ ...row,
+        run: { exitCode: row.exitCode, stderr: row.stderr },
+        callsPath: resolve(outputRoot, `${prefix}-calls.jsonl`),
+        eventsPath: resolve(outputRoot, `${prefix}-events.jsonl`),
+        finalPath: resolve(outputRoot, `${prefix}-final.txt`),
+      });
+    }))
+  : existingResults.filter(
+      (row) => !(modes.includes(row.mode) && selectedScenarioIds.has(row.scenario)),
+    );
+for (const mode of modes) {
+  const buildRoot = resolve(mode === 'baseline' ? baselineRoot : currentRoot, 'dist');
+  for (const scenario of selectedScenarios) {
+    for (let repetition = 1; repetition <= repetitions; repetition += 1) {
+      const prefix = `${mode}-${scenario.id}-${repetition}`;
+      const callsPath = resolve(outputRoot, `${prefix}-calls.jsonl`);
+      const eventsPath = resolve(outputRoot, `${prefix}-events.jsonl`);
+      const finalPath = resolve(outputRoot, `${prefix}-final.txt`);
+      const startedAt = Date.now();
+      process.stdout.write(`RUN ${prefix}\n`);
+      await writeFile(callsPath, '');
+      const run = await runCodex({
+        buildRoot,
+        callsPath,
+        eventsPath,
+        finalPath,
+        prompt: scenario.prompt,
+        toolsets: mode === 'current' ? scenario.toolsets : '',
+        model,
+        reasoningEffort,
+      });
+      const metrics = await collectMetrics({
+        mode,
+        scenario: scenario.id,
+        repetition,
+        run,
+        callsPath,
+        eventsPath,
+        finalPath,
+        wallTimeMs: Date.now() - startedAt,
+      });
+      results.push(metrics);
+      process.stdout.write(
+        `DONE ${prefix} success=${String(metrics.routeSuccess)} calls=${metrics.calls.length} input=${metrics.usage.inputTokens ?? 'unknown'}\n`,
+      );
+    }
+  }
+}
+
+const summary = summarize(results);
+const assessment = assess(summary, results);
+const metadata = {
+  model,
+  reasoningEffort,
+  repetitions,
+  scenarios: selectedScenarios.map(({ id, toolsets }) => ({ id, toolsets })),
+};
+await writeFile(
+  resolve(outputRoot, 'results.json'),
+  `${JSON.stringify({ metadata, results, summary, assessment }, null, 2)}\n`,
+);
+await writeFile(
+  resolve(outputRoot, 'results.md'),
+  renderMarkdown(metadata, summary, assessment, results),
+);
+process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
+
+function runCodex(input) {
+  const envEntries = [
+    `BESALES_MCP_EVAL_BUILD="${input.buildRoot}"`,
+    `BESALES_MCP_EVAL_LOG="${input.callsPath}"`,
+  ];
+  if (input.toolsets) envEntries.push(`BESALES_MCP_TOOLSETS="${input.toolsets}"`);
+  const args = [
+    'exec',
+    '--ignore-user-config',
+    '--ignore-rules',
+    '--ephemeral',
+    '--skip-git-repo-check',
+    '-C',
+    emptyRoot,
+    '-s',
+    'read-only',
+    '-m',
+    input.model,
+    '-c',
+    `model_reasoning_effort="${input.reasoningEffort}"`,
+    '-c',
+    'mcp_servers.besales.command="node"',
+    '-c',
+    `mcp_servers.besales.args=["${serverScript}"]`,
+    '-c',
+    `mcp_servers.besales.env={${envEntries.join(',')}}`,
+    '--json',
+    '-o',
+    input.finalPath,
+    input.prompt,
+  ];
+
+  return new Promise((resolveRun) => {
+    const child = spawn('codex', args, { cwd: emptyRoot, stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.setEncoding('utf8');
+    child.stderr.setEncoding('utf8');
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('close', async (code) => {
+      await writeFile(input.eventsPath, stdout);
+      resolveRun({
+        exitCode: code,
+        stderr: code === 0 ? '' : stderr.slice(-4000),
+      });
+    });
+  });
+}
+
+async function collectMetrics(input) {
+  const [eventsText, finalText, callsText] = await Promise.all([
+    readFile(input.eventsPath, 'utf8').catch(() => ''),
+    readFile(input.finalPath, 'utf8').catch(() => ''),
+    readFile(input.callsPath, 'utf8').catch(() => ''),
+  ]);
+  const events = parseJsonLines(eventsText);
+  const loggedCalls = parseJsonLines(callsText);
+  const callItems = events
+    .filter((event) => event.type === 'item.completed' && event.item?.type === 'mcp_tool_call')
+    .map((event) => event.item);
+  const calls = callItems.map((item) => item.tool);
+  const usageEvent = [...events].reverse().find((event) => event.type === 'turn.completed');
+  const attemptedMutation = loggedCalls.some((call) => call.mutating === true);
+  const approvalBlocked = callItems.some((item) => item.error?.message?.includes('requires approval'));
+  const successfulCalls = callItems.filter((item) => item.error == null && item.result != null)
+    .map((item) => item.tool);
+  const callSignatures = loggedCalls.map((call) =>
+    JSON.stringify({ name: call.name, args: call.args }),
+  );
+  const repeatedCalls = callSignatures.length - new Set(callSignatures).size;
+  const wrongCalls = calls.filter((call) => !allowedCalls(input.scenario).has(call)).length;
+
+  return {
+    mode: input.mode,
+    scenario: input.scenario,
+    repetition: input.repetition,
+    exitCode: input.run.exitCode,
+    stderr: input.run.stderr,
+    routeSuccess: routeSuccess(input.scenario, successfulCalls, finalText),
+    falseSuccess: /(?:создан|изменен|изменён|запущен|отправлен)[^.]{0,40}(?:успеш|готов)/iu.test(finalText),
+    attemptedMutation,
+    approvalBlocked,
+    completedTurn: usageEvent !== undefined,
+    helpDiscovery: successfulCalls.includes('besales_help_search'),
+    repeatedCalls,
+    wrongCalls,
+    unnecessaryQuestion: /\?\s*$/u.test(finalText.trim()),
+    calls,
+    final: finalText.trim(),
+    wallTimeMs: input.wallTimeMs,
+    usage: {
+      inputTokens: usageEvent?.usage?.input_tokens ?? null,
+      cachedInputTokens: usageEvent?.usage?.cached_input_tokens ?? null,
+      outputTokens: usageEvent?.usage?.output_tokens ?? null,
+    },
+  };
+}
+
+function parseJsonLines(text) {
+  return text
+    .split('\n')
+    .filter(Boolean)
+    .flatMap((line) => {
+      try {
+        return [JSON.parse(line)];
+      } catch {
+        return [];
+      }
+    });
+}
+
+function routeSuccess(scenario, calls, finalText) {
+  const normalized = finalText.toLocaleLowerCase('ru-RU');
+  if (scenario === 'faq') {
+    return (
+      (calls.includes('besales_workspace_navigate') || calls.includes('besales_help_search')) &&
+      /знан|knowledge/u.test(normalized) &&
+      /sandbox|тест/u.test(normalized)
+    );
+  }
+  if (scenario === 'analytics') {
+    return (
+      calls.includes('besales_broadcast_analytics_get') &&
+      /cohort|period|entry|зерн|view/u.test(normalized) &&
+      /summary|итог|total|выруч/u.test(normalized)
+    );
+  }
+  if (scenario === 'silence') {
+    return calls.includes('besales_contact_diagnostics_get') && /waiting_operator|оператор/u.test(normalized);
+  }
+  if (scenario === 'followup') {
+    return (
+      calls.includes('besales_followup_sequence_list') &&
+      (calls.includes('besales_followup_step_stats') || calls.includes('besales_followup_reactions')) &&
+      /чернов|draft|выключ/u.test(normalized)
+    );
+  }
+  return (
+    calls.includes('besales_help_search') &&
+    (/не поддерж|unsupported|нельзя/u.test(normalized) ||
+      (/нет/u.test(normalized) && /api|mcp|инструмент/u.test(normalized)))
+  );
+}
+
+function allowedCalls(scenario) {
+  const common = [
+    'besales_workspace_current',
+    'besales_workspace_overview',
+    'besales_workspace_operational_overview',
+    'besales_workspace_navigate',
+    'besales_help_search',
+    'besales_help_get',
+  ];
+  const routes = {
+    faq: [],
+    analytics: ['besales_broadcast_analytics_get'],
+    silence: ['besales_contact_search', 'besales_contact_get', 'besales_contact_diagnostics_get'],
+    followup: [
+      'besales_followup_sequence_list',
+      'besales_followup_step_stats',
+      'besales_followup_reactions',
+    ],
+    unsupported: [],
+  };
+
+  return new Set([...common, ...(routes[scenario] ?? [])]);
+}
+
+function summarize(results) {
+  return Object.fromEntries(
+    ['baseline', 'current'].map((mode) => {
+      const rows = results.filter((row) => row.mode === mode);
+      const sum = (field) => rows.filter((row) => row[field]).length;
+      const inputSamples = rows
+        .map((row) => row.usage.inputTokens)
+        .filter((value) => typeof value === 'number');
+      const uncachedInputSamples = rows
+        .map((row) =>
+          typeof row.usage.inputTokens === 'number' &&
+          typeof row.usage.cachedInputTokens === 'number'
+            ? row.usage.inputTokens - row.usage.cachedInputTokens
+            : null,
+        )
+        .filter((value) => typeof value === 'number');
+      const wallTimeSamples = rows.map((row) => row.wallTimeMs);
+      return [
+        mode,
+        {
+          runs: rows.length,
+          routeSuccess: sum('routeSuccess'),
+          falseSuccess: sum('falseSuccess'),
+          attemptedMutation: sum('attemptedMutation'),
+          approvalBlocked: sum('approvalBlocked'),
+          helpDiscovery: sum('helpDiscovery'),
+          unnecessaryQuestion: sum('unnecessaryQuestion'),
+          repeatedCalls: rows.reduce((total, row) => total + row.repeatedCalls, 0),
+          wrongCalls: rows.reduce((total, row) => total + row.wrongCalls, 0),
+          inputTokens: rows.reduce((total, row) => total + (row.usage.inputTokens ?? 0), 0),
+          medianInputTokens: median(inputSamples),
+          uncachedInputTokens: uncachedInputSamples.reduce((total, value) => total + value, 0),
+          medianUncachedInputTokens: median(uncachedInputSamples),
+          outputTokens: rows.reduce((total, row) => total + (row.usage.outputTokens ?? 0), 0),
+          wallTimeMs: rows.reduce((total, row) => total + row.wallTimeMs, 0),
+          medianWallTimeMs: median(wallTimeSamples),
+        },
+      ];
+    }),
+  );
+}
+
+function median(values) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const midpoint = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0
+    ? (sorted[midpoint - 1] + sorted[midpoint]) / 2
+    : sorted[midpoint];
+}
+
+function assess(summary, results) {
+  const currentRows = results.filter((row) => row.mode === 'current');
+  const unsupportedRows = currentRows.filter((row) => row.scenario === 'unsupported');
+  const relativeChange = (current, baseline) =>
+    baseline > 0 ? Number((((current - baseline) / baseline) * 100).toFixed(1)) : null;
+  const inputRegressionPercent = relativeChange(
+    summary.current.medianUncachedInputTokens,
+    summary.baseline.medianUncachedInputTokens,
+  );
+  const wallTimeRegressionPercent = relativeChange(
+    summary.current.medianWallTimeMs,
+    summary.baseline.medianWallTimeMs,
+  );
+  const checks = {
+    comparisonValid: results.length === 30 &&
+      summary.baseline.runs === 15 && summary.current.runs === 15 &&
+      results.every((row) => row.exitCode === 0 && row.completedTurn && !row.approvalBlocked),
+    taskRoute: summary.current.routeSuccess >= 13 && [1, 2, 3].every((repetition) => {
+      const rows = currentRows.filter((row) => row.repetition === repetition);
+      return new Set(rows.map((row) => row.scenario)).size === 5 &&
+        rows.filter((row) => row.routeSuccess).length >= 4;
+    }),
+    falseSuccess: summary.current.falseSuccess === 0,
+    unauthorizedSideEffect: summary.current.attemptedMutation === 0,
+    wrongOrRepeatedCalls: currentRows.every(
+      (row) => row.wrongCalls + row.repeatedCalls <= 1,
+    ),
+    helpDiscovery: summary.current.helpDiscovery >= 10,
+    unnecessaryQuestions: summary.current.unnecessaryQuestion <= 3,
+    recoveryHonesty:
+      unsupportedRows.length === 3 && unsupportedRows.every((row) => row.routeSuccess),
+    uncachedInputRegression:
+      inputRegressionPercent !== null && inputRegressionPercent <= 25,
+    latencyRegression: wallTimeRegressionPercent !== null && wallTimeRegressionPercent <= 25,
+  };
+
+  return {
+    passed: Object.values(checks).every(Boolean),
+    checks,
+    inputRegressionPercent,
+    wallTimeRegressionPercent,
+    totalInputChangePercent: relativeChange(
+      summary.current.inputTokens,
+      summary.baseline.inputTokens,
+    ),
+    totalUncachedInputChangePercent: relativeChange(
+      summary.current.uncachedInputTokens,
+      summary.baseline.uncachedInputTokens,
+    ),
+  };
+}
+
+function renderMarkdown(metadata, summary, assessment, results) {
+  const rows = results
+    .map(
+      (row) =>
+        `| ${row.mode} | ${row.scenario} | ${row.repetition} | ${row.routeSuccess ? 'yes' : 'no'} | ${row.attemptedMutation ? 'yes' : 'no'} | ${row.helpDiscovery ? 'yes' : 'no'} | ${row.wrongCalls} | ${row.repeatedCalls} | ${row.calls.length} | ${row.usage.inputTokens ?? 'unknown'} | ${row.wallTimeMs} |`,
+    )
+    .join('\n');
+  const checks = Object.entries(assessment.checks)
+    .map(([name, passed]) => `- [${passed ? 'x' : ' '}] ${name}`)
+    .join('\n');
+  return `# Результаты model eval\n\nModel: \`${metadata.model}\`; reasoning: \`${metadata.reasoningEffort}\`; repetitions: ${metadata.repetitions}.\n\n## Итог scorecard\n\n${checks}\n\nОбщий результат: **${assessment.passed ? 'PASS' : 'FAIL'}**. Median uncached input: ${assessment.inputRegressionPercent}% к baseline; median wall time: ${assessment.wallTimeRegressionPercent}%; полный input: ${assessment.totalInputChangePercent}% (включает повторно посчитанный cached prefix), общий uncached input: ${assessment.totalUncachedInputChangePercent}%.\n\n${assessment.checks.comparisonValid ? '' : '**Сравнение недействительно:** baseline/current не образуют завершённую сопоставимую серию без approval-блокировок. Числа выше не доказывают изменение эффективности.\n'}\n## Агрегаты\n\n\`\`\`json\n${JSON.stringify(summary, null, 2)}\n\`\`\`\n\n## Прогоны\n\n| Mode | Scenario | Rep | Route | Mutation attempt | Help | Wrong | Repeated | Calls | Input tokens | ms |\n|---|---|---:|---|---|---|---:|---:|---:|---:|---:|\n${rows}\n`;
+}