@danieltmn/tamnora 0.25.1 → 0.26.0

package/harness/packages/cli/src/commands/eval.js

@@ -1,186 +0,0 @@
-import path from 'node:path';
-import {
-  loadConfig,
-  loadInstructions,
-  loadDataset,
-  runDataset,
-  summarize,
-  createJudge,
-  Store,
-} from '../lib/core.js';
-import { parse, databasePath, num } from '../lib/util.js';
-import { buildSystemPrompt, resolveCliProvider } from '../lib/runtime.js';
-import { color } from '../ui/color.js';
-
-const OPTIONS = {
-  provider: { type: 'string' },
-  model: { type: 'string', multiple: true },
-  dir: { type: 'string' },
-  trials: { type: 'string' },
-  concurrency: { type: 'string' },
-  tools: { type: 'string' },
-  'no-tools': { type: 'boolean' },
-  'judge-model': { type: 'string' },
-  yes: { type: 'boolean', short: 'y' },
-  json: { type: 'boolean' },
-  help: { type: 'boolean', short: 'h' },
-};
-
-function pct(value) {
-  return `${Math.round(value * 100)}%`;
-}
-
-export async function evalCommand(argv) {
-  const { values, positionals } = parse(argv, OPTIONS);
-  if (values.help) {
-    printHelp();
-    return 0;
-  }
-
-  const datasetFile = positionals[0];
-  if (!datasetFile) {
-    console.error(color.red('Falta el dataset. Uso: tamnora eval <dataset.jsonl> [--model <id>]'));
-    return 1;
-  }
-
-  const cwd = values.dir ? path.resolve(values.dir) : process.cwd();
-  const config = await loadConfig(cwd);
-
-  const tasks = await loadDataset(path.resolve(cwd, datasetFile));
-  if (!tasks.length) {
-    console.error(color.red('El dataset está vacío.'));
-    return 1;
-  }
-
-  const instructions = await loadInstructions(cwd, config.instructions);
-  const systemPrompt = buildSystemPrompt(instructions);
-  const trials = num(values.trials, 1);
-  const concurrency = num(values.concurrency, 1);
-
-  const store = new Store(databasePath(cwd));
-  const resolved = resolveCliProvider(store, config, { provider: values.provider, model: values.model });
-  const providerId = resolved.providerId;
-  const baseUrl = resolved.baseUrl;
-  const apiKey = resolved.apiKey;
-  const customProviders = resolved.customProviders;
-  const models = (values.model?.length ? values.model : [resolved.model]).filter(Boolean);
-  if (!models.length) {
-    console.error(color.red('Falta el modelo (usá --model, Ajustes o "model" en tamnora.json).'));
-    store.close();
-    return 1;
-  }
-
-  const judge = createJudge({
-    providerId,
-    model: values['judge-model'] || config.judgeModel,
-    baseUrl,
-    apiKey,
-    customProviders,
-  });
-
-  const enabledTools = values['no-tools']
-    ? []
-    : values.tools
-      ? String(values.tools).split(',').map((s) => s.trim()).filter(Boolean)
-      : Array.isArray(config.tools) && config.tools.length
-        ? config.tools
-        : undefined;
-
-  const runId = store.createRun({
-    dataset: datasetFile,
-    config: { providerId, models, trials, concurrency, tools: enabledTools ?? 'all' },
-  });
-  const startedAt = Date.now();
-
-  if (!values.json) {
-    process.stderr.write(
-      color.dim(
-        `run ${runId} · ${tasks.length} tareas × ${models.length} modelos × ${trials} trials · concurrencia ${concurrency}\n`
-      )
-    );
-  }
-
-  const results = await runDataset({
-    tasks,
-    models,
-    providerId,
-    baseUrl,
-    apiKey,
-    customProviders,
-    systemPrompt,
-    rootDir: cwd,
-    trials,
-    concurrency,
-    enabledTools,
-    permissions: config.permissions,
-    yes: Boolean(values.yes),
-    judge,
-    store,
-    runId,
-    onProgress(entry) {
-      if (values.json) return;
-      const mark = entry.error ? color.red('✗') : entry.passed ? color.green('✓') : color.yellow('·');
-      const scorers = (entry.trace?.scorers ?? []).map((s) => `${s.scorer}:${s.passed ? 'ok' : 'no'}`).join(' ');
-      const suffix = entry.error ? ` — ${entry.error}` : scorers ? ` [${scorers}]` : '';
-      process.stdout.write(`${mark} ${entry.taskId} · ${entry.model}${suffix}\n`);
-    },
-  });
-
-  const summary = summarize(results);
-  summary.durationMs = Date.now() - startedAt;
-  const status = summary.errors && summary.passed === 0 ? 'failed' : summary.errors ? 'done_with_errors' : 'done';
-  store.finishRun(runId, { status, summary });
-  store.close();
-
-  if (values.json) {
-    process.stdout.write(`${JSON.stringify({ runId, summary }, null, 2)}\n`);
-  } else {
-    printReport(summary, runId);
-  }
-
-  return summary.passed === 0 && summary.errors > 0 ? 1 : summary.passed < summary.total ? 1 : 0;
-}
-
-function printReport(summary, runId) {
-  const lines = [];
-  lines.push('');
-  lines.push(color.bold(`Resultados (run ${runId})`));
-  lines.push(color.dim('modelo'.padEnd(34) + 'pasó  total  pass  errores'));
-  for (const m of summary.models) {
-    lines.push(
-      `${String(m.model).slice(0, 33).padEnd(34)}${String(m.passed).padStart(4)}${String(m.total).padStart(7)}${pct(m.passRate).padStart(6)}${String(m.errors).padStart(9)}`
-    );
-  }
-  if (summary.scorers.length) {
-    lines.push('');
-    lines.push(color.dim('scorer'.padEnd(34) + 'pasó  total  pass'));
-    for (const s of summary.scorers) {
-      lines.push(
-        `${String(s.scorer).slice(0, 33).padEnd(34)}${String(s.passed).padStart(4)}${String(s.total).padStart(7)}${pct(s.passRate).padStart(6)}`
-      );
-    }
-  }
-  lines.push('');
-  lines.push(
-    color.dim(
-      `total ${summary.total} · ${summary.passed} ok · ${summary.errors} errores · ${(summary.durationMs / 1000).toFixed(1)}s`
-    )
-  );
-  process.stderr.write(`${lines.join('\n')}\n`);
-}
-
-function printHelp() {
-  console.log(`Uso: tamnora eval <dataset.jsonl> [opciones]
-
-Opciones:
-  --provider <id>      Proveedor
-  --model <id>         Modelo (repetible para comparar varios)
-  --trials <n>         Repeticiones por tarea/modelo (default: 1)
-  --concurrency <n>    Tareas en paralelo (default: 1)
-  --judge-model <id>   Modelo para el scorer llmJudge
-  --dir <path>         Directorio del proyecto
-  --tools <lista>      Tools habilitadas (coma)
-  --no-tools           Deshabilitar tools
-  -y, --yes            Aprobar automáticamente permisos "ask"
-  --json               Salida JSON`);
-}

package/harness/packages/core/src/eval/dataset.js

@@ -1,47 +0,0 @@
-import { readFile } from 'node:fs/promises';
-
-/**
- * Load an evaluation dataset. Supports:
- *  - a JSON array of tasks
- *  - a single JSON object ({ tasks: [...] } or one task)
- *  - JSONL (one task per line)
- *
- * @param {string} file
- * @returns {Promise<object[]>}
- */
-export async function loadDataset(file) {
-  const raw = await readFile(file, 'utf8');
-  const trimmed = raw.trim();
-  if (!trimmed) return [];
-
-  // JSONL: each non-empty line is a standalone JSON object
-  if (!trimmed.startsWith('[') && !trimmed.startsWith('{')) {
-    return parseJsonl(trimmed, file);
-  }
-
-  let parsed;
-  try {
-    parsed = JSON.parse(trimmed);
-  } catch {
-    // Fall back to JSONL (multi-line pretty JSON would have parsed above)
-    return parseJsonl(trimmed, file);
-  }
-
-  if (Array.isArray(parsed)) return parsed;
-  if (Array.isArray(parsed.tasks)) return parsed.tasks;
-  return [parsed];
-}
-
-function parseJsonl(text, file) {
-  const tasks = [];
-  const lines = text.split(/\r?\n/);
-  lines.forEach((line, index) => {
-    if (!line.trim()) return;
-    try {
-      tasks.push(JSON.parse(line));
-    } catch (err) {
-      throw new Error(`Línea ${index + 1} inválida en ${file}: ${err.message}`);
-    }
-  });
-  return tasks;
-}

package/harness/packages/core/src/eval/judge.js

@@ -1,39 +0,0 @@
-import { resolveProviderRuntime } from '../providers/index.js';
-import { extractJson } from './scorers.js';
-
-/**
- * Build an LLM-as-judge function. Returns null when no judge model is set.
- * The judge model receives the criteria and the candidate output and must
- * reply with JSON: { "passed": boolean, "reason": string }.
- *
- * @param {{providerId?:string, model?:string, baseUrl?:string, apiKey?:string, customProviders?:object[]}} opts
- */
-export function createJudge({ providerId, model, baseUrl, apiKey, customProviders = [] } = {}) {
-  if (!model) return null;
-  const { provider, config } = resolveProviderRuntime({ providerId, apiKey, baseUrl, customProviders });
-
-  return async ({ criteria, output, input }) => {
-    const prompt =
-      'Sos un evaluador estricto. Decidí si la RESPUESTA cumple el CRITERIO.\n\n' +
-      `CRITERIO: ${criteria}\n` +
-      `ENTRADA: ${input ?? ''}\n` +
-      `RESPUESTA: ${output ?? ''}\n\n` +
-      'Respondé SOLO un JSON: {"passed": true|false, "reason": "breve"}';
-
-    try {
-      const result = await provider.streamChat(config, {
-        model,
-        messages: [{ role: 'user', content: prompt }],
-        tools: [],
-        onEvent: () => {},
-      });
-      const { value, error } = extractJson(result.content ?? '');
-      if (error || typeof value?.passed !== 'boolean') {
-        return { passed: false, detail: `juez: respuesta inválida (${error ?? 'sin "passed"'})` };
-      }
-      return { passed: value.passed, detail: value.reason ?? 'juez' };
-    } catch (err) {
-      return { passed: false, detail: `juez error: ${err.message}` };
-    }
-  };
-}

package/harness/packages/core/src/eval/report.js

@@ -1,48 +0,0 @@
-/**
- * Aggregate evaluation results into per-model and per-scorer summaries.
- * @param {object[]} results
- */
-export function summarize(results) {
-  const byModel = new Map();
-  const byScorer = new Map();
-  let passed = 0;
-  let errors = 0;
-  let durationMs = 0;
-
-  for (const result of results) {
-    passed += result.passed ? 1 : 0;
-    errors += result.error ? 1 : 0;
-    durationMs += result.durationMs ?? 0;
-
-    const model = byModel.get(result.model) ?? {
-      model: result.model,
-      total: 0,
-      passed: 0,
-      errors: 0,
-      durationMs: 0,
-    };
-    model.total += 1;
-    model.passed += result.passed ? 1 : 0;
-    model.errors += result.error ? 1 : 0;
-    model.durationMs += result.durationMs ?? 0;
-    byModel.set(result.model, model);
-
-    for (const detail of result.trace?.scorers ?? []) {
-      const scorer = byScorer.get(detail.scorer) ?? { scorer: detail.scorer, total: 0, passed: 0 };
-      scorer.total += 1;
-      scorer.passed += detail.passed ? 1 : 0;
-      byScorer.set(detail.scorer, scorer);
-    }
-  }
-
-  const models = [...byModel.values()].map((m) => ({
-    ...m,
-    passRate: m.total ? m.passed / m.total : 0,
-  }));
-  const scorers = [...byScorer.values()].map((s) => ({
-    ...s,
-    passRate: s.total ? s.passed / s.total : 0,
-  }));
-
-  return { total: results.length, passed, errors, durationMs, models, scorers };
-}

package/harness/packages/core/src/eval/runTask.js

@@ -1,62 +0,0 @@
-import { resolveProviderRuntime } from '../providers/index.js';
-import { runAgent } from '../agent/runAgent.js';
-import { buildMcpRegistry } from '../mcp/index.js';
-import { resolveApiKey } from '../config/secrets.js';
-
-export { resolveApiKey } from '../config/secrets.js';
-
-/**
- * Run a single evaluation task headlessly.
- *
- * @param {object} task
- * @param {string} [task.provider]
- * @param {string} [task.model]
- * @param {string} [task.input]
- * @param {string} [task.systemPrompt]
- * @param {object[]} [task.tools] - explicit OpenAI-schema tool definitions
- * @param {object[]} [task.mcpServers]
- * @param {number} [task.maxRounds]
- * @param {object} [opts]
- * @param {Record<string,string>} [opts.env]
- * @param {Function} [opts.onEvent]
- * @param {AbortSignal} [opts.signal]
- * @param {Function} [opts.execute] - tool executor
- */
-export async function runTask(task, { env = {}, onEvent, signal, execute } = {}) {
-  const { provider, config } = resolveProviderRuntime({
-    providerId: task.provider,
-    apiKey: resolveApiKey(task.apiKey, env),
-    baseUrl: task.baseUrl,
-    customProviders: task.customProviders ?? [],
-  });
-
-  let tools = Array.isArray(task.tools) && task.tools[0]?.type === 'function' ? [...task.tools] : [];
-  let mcp = null;
-  if (task.mcpServers?.length) {
-    mcp = await buildMcpRegistry(task.mcpServers, { signal });
-    tools = [...tools, ...mcp.definitions];
-  }
-
-  const messages = [];
-  if (task.systemPrompt) messages.push({ role: 'system', content: task.systemPrompt });
-  messages.push({ role: 'user', content: task.input });
-
-  try {
-    const result = await runAgent({
-      provider,
-      config,
-      model: task.model,
-      messages,
-      tools,
-      execute,
-      toolContext: { ...(task.toolContext ?? {}), mcp },
-      maxRounds: task.maxRounds,
-      onEvent,
-      signal,
-    });
-
-    return { ...result, provider: provider.id, model: task.model };
-  } finally {
-    mcp?.close?.();
-  }
-}

package/harness/packages/core/src/eval/runner.js

@@ -1,153 +0,0 @@
-import { createTaskWorkspace } from './task.js';
-import { runTask } from './runTask.js';
-import { runScorers } from './scorers.js';
-import { createTools } from '../tools/index.js';
-
-function safeParse(value) {
-  if (value && typeof value === 'object') return value;
-  if (typeof value !== 'string') return {};
-  try {
-    return JSON.parse(value);
-  } catch {
-    return {};
-  }
-}
-
-export function extractToolCalls(messages = []) {
-  const calls = [];
-  for (const message of messages) {
-    if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
-      for (const call of message.tool_calls) {
-        calls.push({
-          name: call.function?.name ?? call.name,
-          args: safeParse(call.function?.arguments ?? call.arguments),
-        });
-      }
-    }
-  }
-  return calls;
-}
-
-async function pool(items, limit, worker) {
-  const results = new Array(items.length);
-  let index = 0;
-  const size = Math.max(1, Math.min(limit || 1, items.length || 1));
-  const runners = Array.from({ length: size }, async () => {
-    while (true) {
-      const i = index++;
-      if (i >= items.length) break;
-      results[i] = await worker(items[i], i);
-    }
-  });
-  await Promise.all(runners);
-  return results;
-}
-
-/**
- * Run an evaluation dataset: tasks × models × trials, each in an isolated
- * workspace, scored and (optionally) persisted.
- *
- * @param {object} opts
- * @param {object[]} opts.tasks
- * @param {string[]} opts.models
- * @param {string} opts.providerId
- * @param {string} [opts.baseUrl]
- * @param {string} [opts.apiKey]
- * @param {object[]} [opts.customProviders]
- * @param {string} [opts.systemPrompt]
- * @param {string} [opts.rootDir] - base dir for task fixtures
- * @param {number} [opts.trials]
- * @param {number} [opts.concurrency]
- * @param {string[]} [opts.enabledTools]
- * @param {object} [opts.permissions]
- * @param {boolean} [opts.yes]
- * @param {Function} [opts.judge]
- * @param {object} [opts.store]
- * @param {string} [opts.runId]
- * @param {Function} [opts.onProgress]
- * @param {AbortSignal} [opts.signal]
- */
-export async function runDataset({
-  tasks,
-  models,
-  providerId,
-  baseUrl,
-  apiKey,
-  customProviders = [],
-  systemPrompt,
-  rootDir = process.cwd(),
-  trials = 1,
-  concurrency = 1,
-  enabledTools,
-  permissions,
-  yes = false,
-  judge,
-  store,
-  runId,
-  onProgress,
-  signal,
-}) {
-  const jobs = [];
-  for (const task of tasks) {
-    for (const model of models) {
-      for (let trial = 0; trial < trials; trial++) jobs.push({ task, model, trial });
-    }
-  }
-
-  return pool(jobs, concurrency, async ({ task, model, trial }) => {
-    const workspace = await createTaskWorkspace(task, { rootDir });
-    const tools = createTools({ root: workspace.dir, enabled: enabledTools, permissions, yes });
-    const startedAt = Date.now();
-    let result = null;
-    let error = null;
-
-    try {
-      result = await runTask(
-        {
-          ...task,
-          provider: providerId,
-          model,
-          baseUrl: task.baseUrl ?? baseUrl,
-          apiKey: task.apiKey ?? apiKey,
-          customProviders: task.customProviders ?? customProviders,
-          systemPrompt: task.systemPrompt || systemPrompt,
-          tools: tools.definitions.length ? tools.definitions : undefined,
-        },
-        { env: process.env, execute: tools.execute, signal }
-      );
-    } catch (err) {
-      error = err.message;
-    }
-
-    const toolCalls = extractToolCalls(result?.messages ?? []);
-    let scoring = { passed: false, score: 0, details: [] };
-    if (!error) {
-      scoring = await runScorers(task, {
-        output: result.content,
-        toolCalls,
-        workspaceDir: workspace.dir,
-        input: task.input,
-        judge,
-      });
-    }
-    await workspace.cleanup();
-
-    const entry = {
-      runId,
-      taskId: task.id ?? String(task.input ?? '').slice(0, 40),
-      model,
-      trial,
-      output: result?.content ?? null,
-      passed: Boolean(scoring.passed),
-      score: scoring.score,
-      durationMs: Date.now() - startedAt,
-      error,
-      toolCalls,
-      trace: { rounds: result?.rounds, stopReason: result?.stopReason, scorers: scoring.details, error },
-    };
-
-    if (store) store.addRunResult(entry);
-    onProgress?.(entry);
-    return entry;
-  });
-}