@librechat/agents 3.4.6 → 3.5.0

package/src/scripts/activity-labels/corpus.cjs

@@ -1,473 +0,0 @@
-/**
- * Eval corpus for activity-label prose. Two halves:
- *
- * - captured.json: the 9 real payloads from the 2026-07-29 sandbox-probe run,
- *   verbatim from Langfuse, replayed as ONE sequence so continuity variants
- *   see the same run shape production did. `productionLabel` is what shipped.
- * - synthetic: cases built for the failure modes the captured run surfaced
- *   (redundant consecutive batches, register collapse, length overflow) plus
- *   the modes it never exercised (all-failed, partial, parallel columns,
- *   truncation, entry overflow, error-shaped success).
- *
- * A case is a sequence of steps; a step is one label request. Multi-step
- * cases exist to measure cross-batch redundancy: the runner chains each
- * step's generated label into the next step's `previousLabels` for variants
- * that opt in.
- */
-const fs = require('fs');
-const path = require('path');
-
-const captured = JSON.parse(
-  fs.readFileSync(path.join(__dirname, 'captured.json'), 'utf8')
-);
-
-const capturedRun = {
-  id: 'sandbox-probe-run',
-  notes: 'the real 9-batch production run, verbatim payloads',
-  steps: captured.map((entry) => ({
-    id: entry.id,
-    verbatim: entry.prompt,
-    productionLabel: entry.productionLabel,
-  })),
-};
-
-const synthetic = [
-  {
-    id: 'all-failed',
-    notes: 'every call fails — failure register, verb-first under failure',
-    steps: [
-      {
-        payload: {
-          lastAssistantText:
-            "I'll run each of these and report exactly what happens.",
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'cat /etc/shadow' },
-              status: 'error',
-              error: 'cat: /etc/shadow: Permission denied',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'ls /nonexistent-dir' },
-              status: 'error',
-              error:
-                "ls: cannot access '/nonexistent-dir': No such file or directory",
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'curl -sS https://nope.invalid' },
-              status: 'error',
-              error: 'curl: (6) Could not resolve host: nope.invalid',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'partial-failure',
-    notes: 'mixed batch — must not read as all-success or all-failure',
-    steps: [
-      {
-        payload: {
-          thinkingExcerpts: [
-            'Three probes: create the marker dir, read the shadow file, resolve an invalid host. The first should work, the other two should fail for different reasons.',
-          ],
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'mkdir -p /tmp/probe && echo ok' },
-              toolOutput: 'stdout:\nok',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'cat /etc/shadow' },
-              status: 'error',
-              error: 'cat: /etc/shadow: Permission denied',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'getent hosts nope.invalid' },
-              status: 'error',
-              error: 'exit code 2',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'parallel-versions',
-    notes: 'one batch of parallel lookups — the groupId/parallel-columns shape',
-    steps: [
-      {
-        payload: {
-          lastAssistantText: 'Let me look up all three at once.',
-          entries: [
-            {
-              toolName: 'web_search',
-              toolInput: { query: 'Node.js latest stable version 2026' },
-              toolOutput:
-                'Node.js 24.5.0 (Current) released 2026-07-22; v24 enters LTS October 2026. nodejs.org/en/blog/release/v24.5.0',
-              status: 'success',
-            },
-            {
-              toolName: 'web_search',
-              toolInput: { query: 'Deno latest release version' },
-              toolOutput:
-                'Deno 2.4.2 released 2026-07-16 with improved node:sqlite compat. deno.com/blog/v2.4',
-              status: 'success',
-            },
-            {
-              toolName: 'web_search',
-              toolInput: { query: 'Bun latest release version' },
-              toolOutput:
-                'Bun 1.2.19 released 2026-07-25, adds --compile cross-target for linux-arm64. bun.sh/blog/bun-v1.2.19',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'fib-rapid',
-    notes: 'three near-identical consecutive batches — redundancy stress',
-    steps: [1, 2, 3].map((n) => ({
-      id: `fib-${n}`,
-      payload: {
-        entries: [
-          {
-            toolName: 'execute_code',
-            toolInput: { code: `print(fib(${n}))` },
-            toolOutput: `stdout:\n${[1, 1, 2][n - 1]}`,
-            status: 'success',
-          },
-        ],
-      },
-    })),
-  },
-  {
-    id: 'mega-batch',
-    notes:
-      'six heterogeneous probes in one batch — length-cap stress (mirrors cpu-meminfo-disk)',
-    steps: [
-      {
-        payload: {
-          thinkingExcerpts: [
-            "I'll gather the full system picture in one pass: CPU count, memory, disk, limits, user, kernel.",
-          ],
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'nproc' },
-              toolOutput: 'stdout:\n1',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'cat /proc/meminfo | head -3' },
-              toolOutput: 'stdout:\n',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'df -h / /tmp' },
-              toolOutput:
-                'stdout:\nFilesystem      Size  Used Avail Use% Mounted on\noverlay          16M   12M  4.0M  75% /\ntmpfs            20M     0   20M   0% /tmp',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'ulimit -v' },
-              toolOutput: 'stdout:\n16777216',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'whoami' },
-              toolOutput: 'stdout:\nsandbox',
-              status: 'success',
-            },
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'uname -r' },
-              toolOutput: 'stdout:\n6.1.102',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'single-trivial',
-    notes: 'one boring call — header must still say something the card cannot',
-    steps: [
-      {
-        payload: {
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'ls /mnt/data' },
-              toolOutput: 'stdout:\nnotes.md\nresults.csv\nprobe.txt',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'answer-found',
-    notes: 'the answer IS the line — a question resolved by one call',
-    steps: [
-      {
-        payload: {
-          lastAssistantText:
-            'Let me find where that 30-second timeout is actually set.',
-          entries: [
-            {
-              toolName: 'grep',
-              toolInput: {
-                pattern: 'timeout',
-                path: 'api/server/utils/streams.js',
-              },
-              toolOutput:
-                'streams.js:41:  const STREAM_TIMEOUT_MS = 30_000; // hard cap per SSE flush\nstreams.js:88:  setTimeout(() => controller.abort(), STREAM_TIMEOUT_MS);',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'bare-batch',
-    notes: 'no intent, no reasoning — minimum context',
-    steps: [
-      {
-        payload: {
-          entries: [
-            {
-              toolName: 'read_file',
-              toolInput: { path: 'package.json' },
-              toolOutput:
-                '{\n  "name": "librechat",\n  "version": "0.8.1",\n  ...',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'misleading-intent',
-    notes: 'intent asks one question, output answers it the other way',
-    steps: [
-      {
-        payload: {
-          lastAssistantText:
-            'Now checking whether response caching is enabled in this deployment.',
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'grep -A2 "cache:" config/deploy.yaml' },
-              toolOutput: 'stdout:\ncache:\n  enabled: false\n  ttl: 3600',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'truncated-output',
-    notes: 'output clipped mid-JSON by the 600-char limit',
-    steps: [
-      {
-        payload: {
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'pip list --format=json' },
-              toolOutput:
-                'stdout:\n' +
-                JSON.stringify(
-                  Array.from({ length: 60 }, (_, i) => ({
-                    name: `package-${i}`,
-                    version: `1.${i}.0`,
-                  }))
-                ),
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'silent-success',
-    notes: 'empty output — nothing came back to summarize',
-    steps: [
-      {
-        payload: {
-          lastAssistantText: "I'll write the results file now.",
-          entries: [
-            {
-              toolName: 'write_file',
-              toolInput: {
-                path: '/mnt/data/results.csv',
-                content: 'run,ms\n1,412\n2,398\n',
-              },
-              toolOutput: '',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'edit-verify',
-    notes: 'edit plus read-back in one batch — one activity, two calls',
-    steps: [
-      {
-        payload: {
-          thinkingExcerpts: [
-            'The retry cap is what causes the duplicate sends; dropping it from 5 to 1 and verifying the file took the change.',
-          ],
-          entries: [
-            {
-              toolName: 'edit_file',
-              toolInput: {
-                path: 'api/server/utils/queue.js',
-                old: 'const MAX_RETRIES = 5;',
-                new: 'const MAX_RETRIES = 1;',
-              },
-              toolOutput: 'OK',
-              status: 'success',
-            },
-            {
-              toolName: 'read_file',
-              toolInput: { path: 'api/server/utils/queue.js', range: [10, 14] },
-              toolOutput: 'const MAX_RETRIES = 1;\nconst BACKOFF_MS = 250;',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'error-shaped-success',
-    notes:
-      'tool returns an error payload with success status — must not read as success',
-    steps: [
-      {
-        payload: {
-          lastAssistantText: 'Searching for the changelog now.',
-          entries: [
-            {
-              toolName: 'web_search',
-              toolInput: { query: 'librechat 0.8.1 changelog' },
-              toolOutput:
-                '{"error":{"code":"rate_limited","message":"Search quota exceeded, retry after 3600s"}}',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'mcp-long-name',
-    notes: 'namespaced MCP tool name — echo temptation',
-    steps: [
-      {
-        payload: {
-          entries: [
-            {
-              toolName: 'mcp__github__search_repositories',
-              toolInput: { query: 'org:danny-avila librechat-agents' },
-              toolOutput:
-                '{"total_count":2,"items":[{"full_name":"danny-avila/LibreChat","stars":31200},{"full_name":"danny-avila/agents","stars":410}]}',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'dup-activity-seq',
-    notes:
-      'controlled mirror of captured steps 2/3 — same activity twice in a row',
-    steps: [
-      {
-        id: 'dup-write',
-        payload: {
-          thinkingExcerpts: [
-            'First write a marker file, then a separate call will check it survives.',
-          ],
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: {
-                code: 'echo "marker-$(date +%s)" > /tmp/persist-probe.txt && cat /tmp/persist-probe.txt',
-              },
-              toolOutput: 'stdout:\nmarker-1785932011',
-              status: 'success',
-            },
-          ],
-        },
-      },
-      {
-        id: 'dup-confirm',
-        payload: {
-          entries: [
-            {
-              toolName: 'run_tools_with_bash',
-              toolInput: { code: 'cat /tmp/persist-probe.txt' },
-              toolOutput: 'stdout:\nmarker-1785932011',
-              status: 'success',
-            },
-          ],
-        },
-      },
-    ],
-  },
-  {
-    id: 'overflow-entries',
-    notes: '14 calls — exercises the 12-entry cap and the "…and 2 more" suffix',
-    steps: [
-      {
-        payload: {
-          entries: Array.from({ length: 14 }, (_, i) => ({
-            toolName: 'run_tools_with_bash',
-            toolInput: { code: `convert page-${i + 1}.svg page-${i + 1}.png` },
-            toolOutput: '',
-            status: 'success',
-          })),
-        },
-      },
-    ],
-  },
-];
-
-/** Tool names for echo checks; captured steps bake entries into the
- *  verbatim prompt, so they are recovered from the "Tool calls:" lines. */
-function stepEntries(step) {
-  if (step.payload?.entries) {
-    return step.payload.entries;
-  }
-  return [...(step.verbatim ?? '').matchAll(/^- ([A-Za-z0-9_]+)\(/gm)].map(
-    (match) => ({
-      toolName: match[1],
-    })
-  );
-}
-
-module.exports = { cases: [capturedRun, ...synthetic], stepEntries };

package/src/scripts/activity-labels/report.cjs

@@ -1,203 +0,0 @@
-/** Aggregation + markdown rendering, shared by the live runner and the
- *  offline rescorer so metric fixes never require re-spending on the API.
- *
- *  Ported from LibreChat #14527 with three fixes pending backport: numeric
- *  sample ordering, a keyed record index for the per-case cells, and
- *  case-folded opener tallies. */
-const FLAG_TYPES = [
-  'len',
-  'punct',
-  'quote',
-  'md',
-  'opener',
-  'tool-echo',
-  'count-echo',
-  'restate',
-  'template',
-];
-
-const PRICES = { 'claude-haiku-4-5': { input: 1, output: 5 } };
-
-function flagType(flag) {
-  return flag.split(':')[0];
-}
-
-function aggregate(records, model) {
-  const byVariant = new Map();
-  for (const record of records) {
-    if (!byVariant.has(record.variant)) {
-      byVariant.set(record.variant, {
-        steps: 0,
-        errors: 0,
-        flagCounts: {},
-        /** Null prototype: labels are model prose, and an opener like
-         *  'Constructor' would otherwise hit Object.prototype.constructor
-         *  and render a garbage tally ('__proto__' would vanish). */
-        firstWords: Object.create(null),
-        totalWords: 0,
-        latencies: [],
-        inputTokens: 0,
-        outputTokens: 0,
-      });
-    }
-    const agg = byVariant.get(record.variant);
-    if (record.error) {
-      agg.errors += 1;
-      /** Some failures still billed (a 200 whose label normalized to
-       *  empty) — dropping their usage would make an all-empty variant
-       *  report $0, and dropping latency would show mean ms 0. Old
-       *  stored runs predate error latency; skip when absent. */
-      agg.inputTokens += record.inputTokens ?? 0;
-      agg.outputTokens += record.outputTokens ?? 0;
-      if (record.latencyMs != null) {
-        agg.latencies.push(record.latencyMs);
-      }
-      continue;
-    }
-    agg.steps += 1;
-    agg.totalWords += record.wordCount;
-    agg.latencies.push(record.latencyMs);
-    agg.inputTokens += record.inputTokens;
-    agg.outputTokens += record.outputTokens;
-    /** Case-folded: 'Found' and 'found' are one opener — counting them
-     *  separately would make a sentence-case violation read as MORE
-     *  register diversity, the opposite of what the tally detects. */
-    const opener =
-      typeof record.firstWord === 'string'
-        ? record.firstWord.toLowerCase()
-        : record.firstWord;
-    agg.firstWords[opener] = (agg.firstWords[opener] ?? 0) + 1;
-    for (const flag of record.flags) {
-      const type = flagType(flag);
-      agg.flagCounts[type] = (agg.flagCounts[type] ?? 0) + 1;
-    }
-  }
-  const price = PRICES[model];
-  return [...byVariant.entries()].map(([name, agg]) => {
-    const sortedFirst = Object.entries(agg.firstWords).sort(
-      (a, b) => b[1] - a[1]
-    );
-    const topOpener = sortedFirst[0] ?? ['—', 0];
-    return {
-      variant: name,
-      steps: agg.steps,
-      errors: agg.errors,
-      flagCounts: agg.flagCounts,
-      distinctOpeners: sortedFirst.length,
-      topOpener: `${topOpener[0]} ×${topOpener[1]}`,
-      avgWords: agg.steps > 0 ? (agg.totalWords / agg.steps).toFixed(1) : '—',
-      meanLatencyMs: agg.latencies.length
-        ? Math.round(
-            agg.latencies.reduce((a, b) => a + b, 0) / agg.latencies.length
-          )
-        : 0,
-      inputTokens: agg.inputTokens,
-      outputTokens: agg.outputTokens,
-      costUsd: price
-        ? (
-            (agg.inputTokens * price.input + agg.outputTokens * price.output) /
-            1e6
-          ).toFixed(4)
-        : 'n/a',
-    };
-  });
-}
-
-function markdownReport({
-  records,
-  aggregates,
-  runCases,
-  variantNames,
-  model,
-  samples,
-}) {
-  const lines = [];
-  lines.push(`# Activity-label eval — ${new Date().toISOString()}`);
-  lines.push('');
-  lines.push(
-    `model: \`${model}\` · samples: ${samples} · cases: ${runCases.length}`
-  );
-  lines.push('');
-  lines.push('## Aggregate');
-  lines.push('');
-  lines.push(
-    `| variant | steps | ${FLAG_TYPES.join(' | ')} | distinct openers | top opener | avg words | mean ms | cost |`
-  );
-  lines.push(
-    `|---|---:|${FLAG_TYPES.map(() => '---:').join('|')}|---:|---|---:|---:|---:|`
-  );
-  for (const agg of aggregates) {
-    lines.push(
-      `| ${agg.variant} | ${agg.steps}${agg.errors ? ` (+${agg.errors} err)` : ''} | ` +
-        FLAG_TYPES.map((type) => agg.flagCounts[type] ?? 0).join(' | ') +
-        /** topOpener carries a model-derived token — escape pipes like
-         *  the per-case renderer does, or a label opening with `a|b`
-         *  shifts every following aggregate column. */
-        ` | ${agg.distinctOpeners} | ${String(agg.topOpener).replace(/\|/g, '\\|')} | ${agg.avgWords} | ${agg.meanLatencyMs} | $${agg.costUsd} |`
-    );
-  }
-  lines.push('');
-  lines.push('## Per-case');
-  /** Sorted numerically — the default lexicographic sort orders samples
-   *  1, 10, 11, …, 2 once a sweep reaches ten samples. */
-  const sampleList = [...new Set(records.map((r) => r.sample))].sort(
-    (a, b) => a - b
-  );
-  /** One keyed pass instead of a records.find per table cell, which is
-   *  quadratic in paid results on large sweeps. */
-  const recordIndex = new Map(
-    records.map((record) => [
-      `${record.variant}\0${record.sample}\0${record.caseId}\0${record.stepId}`,
-      record,
-    ])
-  );
-  for (const testCase of runCases) {
-    lines.push('');
-    lines.push(`### ${testCase.id}`);
-    lines.push('');
-    lines.push(`*${testCase.notes}*`);
-    lines.push('');
-    const header = ['step'];
-    if (samples > 1) {
-      header.push('s');
-    }
-    if (testCase.steps.some((step) => step.productionLabel)) {
-      header.push('production');
-    }
-    header.push(...variantNames);
-    lines.push(`| ${header.join(' | ')} |`);
-    lines.push(`|${header.map(() => '---').join('|')}|`);
-    for (const step of testCase.steps) {
-      const stepId = step.id ?? testCase.id;
-      for (const sample of sampleList) {
-        const row = [stepId];
-        if (samples > 1) {
-          row.push(String(sample));
-        }
-        if (header.includes('production')) {
-          row.push(step.productionLabel ?? '');
-        }
-        for (const variantName of variantNames) {
-          const record = recordIndex.get(
-            `${variantName}\0${sample}\0${testCase.id}\0${stepId}`
-          );
-          if (!record) {
-            row.push('');
-          } else if (record.error) {
-            row.push(`⛔ ${record.error}`);
-          } else {
-            const flagNote =
-              record.flags.length > 0 ? ` ⚠${record.flags.join(' ⚠')}` : '';
-            row.push(`${record.label}${flagNote}`);
-          }
-        }
-        lines.push(
-          `| ${row.map((cell) => cell.replace(/\|/g, '\\|')).join(' | ')} |`
-        );
-      }
-    }
-  }
-  return lines.join('\n') + '\n';
-}
-
-module.exports = { aggregate, markdownReport, FLAG_TYPES };