@danieltmn/tamnora 0.25.0 → 0.26.0

package/harness/packages/core/src/eval/scorers.js

@@ -1,217 +0,0 @@
-import { readFile, access } from 'node:fs/promises';
-import { spawn } from 'node:child_process';
-import path from 'node:path';
-import { normalizeExpect } from './task.js';
-
-const DEFAULT_SCRIPT_TIMEOUT = 60;
-
-function outputText(output) {
-  return typeof output === 'string' ? output : output == null ? '' : JSON.stringify(output);
-}
-
-/** Extract a JSON value from text, tolerating ```json fenced blocks. */
-export function extractJson(text) {
-  const trimmed = String(text ?? '').trim();
-  if (!trimmed) return { error: 'vacío' };
-
-  try {
-    return { value: JSON.parse(trimmed) };
-  } catch {
-    // try fenced block
-  }
-
-  const match = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i);
-  if (match) {
-    try {
-      return { value: JSON.parse(match[1].trim()) };
-    } catch (err) {
-      return { error: err.message };
-    }
-  }
-
-  // inline object/array inside surrounding text
-  for (const [open, close] of [
-    ['{', '}'],
-    ['[', ']'],
-  ]) {
-    const start = trimmed.indexOf(open);
-    const end = trimmed.lastIndexOf(close);
-    if (start !== -1 && end > start) {
-      try {
-        return { value: JSON.parse(trimmed.slice(start, end + 1)) };
-      } catch {
-        // keep looking
-      }
-    }
-  }
-
-  return { error: 'no se pudo parsear JSON' };
-}
-
-/** Minimal JSON Schema validator (type, required, properties, items, enum). */
-export function validateSchema(value, schema) {
-  const errors = [];
-  const check = (val, sch, pointer) => {
-    if (!sch || typeof sch !== 'object') return;
-    if (sch.type) {
-      const types = Array.isArray(sch.type) ? sch.type : [sch.type];
-      const actual =
-        val === null ? 'null' : Array.isArray(val) ? 'array' : typeof val === 'number' && Number.isInteger(val) ? 'integer' : typeof val;
-      const ok = types.some((t) => t === actual || (t === 'number' && actual === 'integer'));
-      if (!ok) errors.push(`${pointer}: se esperaba ${types.join('|')}, llegó ${actual}`);
-    }
-    if (Array.isArray(sch.enum) && !sch.enum.includes(val)) {
-      errors.push(`${pointer}: valor no está en enum`);
-    }
-    if (sch.type === 'object' || sch.properties) {
-      for (const req of sch.required ?? []) {
-        if (val == null || val[req] === undefined) errors.push(`${pointer}: falta "${req}"`);
-      }
-      for (const [key, sub] of Object.entries(sch.properties ?? {})) {
-        if (val && val[key] !== undefined) check(val[key], sub, `${pointer}.${key}`);
-      }
-    }
-    if (sch.type === 'array' && sch.items && Array.isArray(val)) {
-      val.forEach((item, i) => check(item, sch.items, `${pointer}[${i}]`));
-    }
-  };
-  check(value, schema, '$');
-  return { valid: errors.length === 0, errors };
-}
-
-function runCommand(command, cwd, timeoutSec = DEFAULT_SCRIPT_TIMEOUT) {
-  return new Promise((resolve) => {
-    const child = spawn(command, { cwd, shell: true, windowsHide: true });
-    let output = '';
-    const append = (d) => {
-      if (output.length < 20_000) output += d.toString();
-    };
-    child.stdout?.on('data', append);
-    child.stderr?.on('data', append);
-    const timer = setTimeout(() => child.kill('SIGKILL'), timeoutSec * 1000);
-    child.on('error', (err) => {
-      clearTimeout(timer);
-      resolve({ code: -1, output: err.message });
-    });
-    child.on('close', (code) => {
-      clearTimeout(timer);
-      resolve({ code, output });
-    });
-  });
-}
-
-const SCORERS = {
-  exact(spec, ctx) {
-    const a = outputText(ctx.output).trim();
-    const b = String(spec.value ?? '').trim();
-    const match = spec.caseSensitive ? a === b : a.toLowerCase() === b.toLowerCase();
-    return { passed: match, detail: match ? 'coincide' : `esperaba "${b}"` };
-  },
-
-  contains(spec, ctx) {
-    const haystack = spec.caseSensitive ? outputText(ctx.output) : outputText(ctx.output).toLowerCase();
-    const needle = spec.caseSensitive ? String(spec.value ?? '') : String(spec.value ?? '').toLowerCase();
-    const match = needle ? haystack.includes(needle) : true;
-    return { passed: match, detail: match ? `contiene "${spec.value}"` : `no contiene "${spec.value}"` };
-  },
-
-  regex(spec, ctx) {
-    try {
-      const re = new RegExp(spec.pattern, spec.flags ?? '');
-      const match = re.test(outputText(ctx.output));
-      return { passed: match, detail: match ? 'regex ok' : `no matchea /${spec.pattern}/` };
-    } catch (err) {
-      return { passed: false, detail: `regex inválida: ${err.message}` };
-    }
-  },
-
-  jsonSchema(spec, ctx) {
-    const { value, error } = extractJson(outputText(ctx.output));
-    if (error) return { passed: false, detail: `JSON inválido: ${error}` };
-    const { valid, errors } = validateSchema(value, spec.schema ?? {});
-    return { passed: valid, detail: valid ? 'schema ok' : errors.join('; ') };
-  },
-
-  toolCalled(spec, ctx) {
-    const calls = ctx.toolCalls ?? [];
-    const match = calls.some((c) => {
-      if (c.name !== spec.name) return false;
-      if (!spec.argsContains) return true;
-      return JSON.stringify(c.args ?? {}).includes(spec.argsContains);
-    });
-    return {
-      passed: match,
-      detail: match ? `llamó ${spec.name}` : `no llamó ${spec.name}`,
-    };
-  },
-
-  async fileExists(spec, ctx) {
-    try {
-      await access(path.join(ctx.workspaceDir, spec.path));
-      return { passed: true, detail: `${spec.path} existe` };
-    } catch {
-      return { passed: false, detail: `${spec.path} no existe` };
-    }
-  },
-
-  async fileContains(spec, ctx) {
-    try {
-      const content = await readFile(path.join(ctx.workspaceDir, spec.path), 'utf8');
-      const match = content.includes(spec.value);
-      return { passed: match, detail: match ? `${spec.path} contiene "${spec.value}"` : `no contiene` };
-    } catch (err) {
-      return { passed: false, detail: `no se pudo leer ${spec.path}: ${err.message}` };
-    }
-  },
-
-  async script(spec, ctx) {
-    const { code, output } = await runCommand(spec.command, ctx.workspaceDir, spec.timeout);
-    const expected = spec.exit ?? 0;
-    return {
-      passed: code === expected,
-      detail: `exit ${code} (esperado ${expected})${output ? `: ${output.slice(0, 200)}` : ''}`,
-    };
-  },
-
-  async llmJudge(spec, ctx) {
-    if (!ctx.judge) return { passed: false, detail: 'no hay modelo juez configurado' };
-    return ctx.judge({ criteria: spec.criteria, output: outputText(ctx.output), input: ctx.input });
-  },
-};
-
-const DEFAULT_SCORER_NAMES = Object.keys(SCORERS);
-
-/**
- * Evaluate a task result against its scorers.
- * @param {object} task
- * @param {{output:any, toolCalls?:object[], workspaceDir:string, input?:string, judge?:Function}} ctx
- * @param {string[]} [scorerNames] - subset of scorers to use
- */
-export async function runScorers(task, ctx, scorerNames = DEFAULT_SCORER_NAMES) {
-  const specs = normalizeExpect(task.expect);
-  if (!specs.length) return { passed: true, score: 1, details: [] };
-
-  const details = [];
-  for (const spec of specs) {
-    if (!scorerNames.includes(spec.type) || !SCORERS[spec.type]) {
-      details.push({ scorer: spec.type, passed: false, detail: 'scorer desconocido' });
-      continue;
-    }
-    let result;
-    try {
-      result = await SCORERS[spec.type](spec, ctx);
-    } catch (err) {
-      result = { passed: false, detail: `error: ${err.message}` };
-    }
-    details.push({ scorer: spec.type, ...result });
-  }
-
-  const passedCount = details.filter((d) => d.passed).length;
-  return {
-    passed: passedCount === details.length,
-    score: details.length ? passedCount / details.length : 1,
-    details,
-  };
-}
-
-export { SCORERS, DEFAULT_SCORER_NAMES };

package/harness/packages/core/src/eval/task.js

@@ -1,59 +0,0 @@
-import { mkdtemp, cp, mkdir, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-
-/**
- * Create an isolated workspace for a task. If the task has a `fixture`
- * (a directory, relative to `rootDir`), it is copied into the workspace.
- *
- * @param {object} task
- * @param {{rootDir?: string, parentDir?: string}} [opts]
- * @returns {Promise<{dir:string, parent:string, cleanup:()=>Promise<void>}>}
- */
-export async function createTaskWorkspace(task, { rootDir = process.cwd(), parentDir } = {}) {
-  const parent = parentDir ?? (await mkdtemp(path.join(tmpdir(), 'tamnora-task-')));
-  const dir = path.join(parent, 'workspace');
-  await mkdir(dir, { recursive: true });
-
-  if (task.fixture) {
-    const source = path.isAbsolute(task.fixture)
-      ? task.fixture
-      : path.resolve(rootDir, task.fixture);
-    await cp(source, dir, { recursive: true });
-  }
-
-  const cleanup = async () => {
-    if (task.keepWorkspace) return;
-    await rm(parent, { recursive: true, force: true }).catch(() => {});
-  };
-
-  return { dir, parent, cleanup };
-}
-
-/**
- * Normalize a task's `expect` field into a list of scorer specs.
- * Accepts a string, a single object, or an array; also supports shorthand keys.
- */
-export function normalizeExpect(expect) {
-  if (expect === undefined || expect === null) return [];
-  if (typeof expect === 'string') return [{ type: 'contains', value: expect }];
-  if (Array.isArray(expect)) return expect.map(normalizeSpec);
-  return [normalizeSpec(expect)];
-}
-
-function normalizeSpec(spec) {
-  if (typeof spec === 'string') return { type: 'contains', value: spec };
-  if (spec.type) return spec;
-  if ('contains' in spec) return { type: 'contains', value: spec.contains };
-  if ('equals' in spec) return { type: 'exact', value: spec.equals };
-  if ('regex' in spec) return { type: 'regex', pattern: spec.regex };
-  if ('jsonSchema' in spec) return { type: 'jsonSchema', schema: spec.jsonSchema };
-  if ('toolCalled' in spec) return { type: 'toolCalled', name: spec.toolCalled };
-  if ('fileExists' in spec) return { type: 'fileExists', path: spec.fileExists };
-  if ('fileContains' in spec) {
-    return { type: 'fileContains', path: spec.fileContains.path, value: spec.fileContains.value };
-  }
-  if ('script' in spec) return { type: 'script', command: spec.script };
-  if ('llmJudge' in spec) return { type: 'llmJudge', criteria: spec.llmJudge };
-  return { type: 'unknown', raw: spec };
-}

package/harness/packages/core/src/procs/detect.js

@@ -1,149 +0,0 @@
-// Deteccion de como correr el proyecto en desarrollo (dev server) a partir de
-// package.json, composer/artisan y entrypoints PHP.
-
-import { existsSync, readFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import path from 'node:path';
-
-export const DEFAULT_PROCESS_ALLOW = [
-  'npm',
-  'node',
-  'npx',
-  'pnpm',
-  'yarn',
-  'bun',
-  'bunx',
-  'php',
-  'python',
-  'python3',
-  'composer',
-];
-
-const SCRIPT_ORDER = ['dev', 'start', 'serve', 'preview'];
-
-const STATIC_SERVER = fileURLToPath(new URL('../../../../scripts/static-server.mjs', import.meta.url));
-const STATIC_PORT = 8080;
-
-/** Archivo por proyecto (relativo a su raiz) que define sus procesos de dev. */
-export const PROCS_FILE = '.tamnora/procs.json';
-
-function findStaticRoot(dir) {
-  if (existsSync(path.join(dir, 'index.html'))) return '.';
-  if (existsSync(path.join(dir, 'public', 'index.html'))) return 'public';
-  return null;
-}
-
-/** True si `target` es `root` o vive dentro de el (sin salirse por `..`). */
-function isInside(root, target) {
-  const rel = path.relative(root, target);
-  return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
-}
-
-/**
- * Procesos definidos a mano por el proyecto en `<root>/.tamnora/procs.json`.
- * Formato: `{ "processes": [{ label?, dir?, command, args?, port? }] }` o un array
- * directo. `dir` es relativo a la raiz y debe quedar dentro de ella. Cada chip
- * sigue pasando por la whitelist (`processes.allow`) al arrancar.
- */
-export function loadProjectProcs(root) {
-  const file = path.join(root, PROCS_FILE);
-  if (!existsSync(file)) return [];
-  const raw = readJson(file);
-  const list = Array.isArray(raw) ? raw : Array.isArray(raw?.processes) ? raw.processes : [];
-  const chips = [];
-  for (const entry of list) {
-    if (!entry || typeof entry !== 'object') continue;
-    const command = String(entry.command ?? '').trim();
-    if (!command) continue;
-    const rel = String(entry.dir ?? '').trim().replace(/^[/\\]+/, '');
-    const cwd = path.resolve(root, rel || '.');
-    if (!isInside(root, cwd) || !existsSync(cwd)) continue;
-    const port = entry.port ? Number(entry.port) : undefined;
-    chips.push({
-      label: String(entry.label ?? command).trim() || command,
-      command,
-      args: Array.isArray(entry.args) ? entry.args.map(String) : [],
-      cwd,
-      port: Number.isInteger(port) && port > 0 ? port : undefined,
-      source: 'project',
-    });
-  }
-  return chips;
-}
-
-/** Chip para servir `dir` como sitio estático con el servidor sin dependencias. */
-export function staticServerChip(dir, { port = STATIC_PORT, label } = {}) {
-  return {
-    label: label ?? `sitio estático (${port})`,
-    command: process.execPath,
-    args: [STATIC_SERVER, dir, String(port)],
-    cwd: dir,
-    port,
-  };
-}
-
-function readJson(file) {
-  try {
-    return JSON.parse(readFileSync(file, 'utf8'));
-  } catch {
-    return null;
-  }
-}
-
-/** Package manager por `packageManager` o por lockfile (default npm). */
-export function packageManager(dir) {
-  const pkg = readJson(path.join(dir, 'package.json'));
-  if (pkg?.packageManager) return String(pkg.packageManager).split('@')[0];
-  if (existsSync(path.join(dir, 'pnpm-lock.yaml'))) return 'pnpm';
-  if (existsSync(path.join(dir, 'yarn.lock'))) return 'yarn';
-  if (existsSync(path.join(dir, 'bun.lockb')) || existsSync(path.join(dir, 'bun.lock'))) return 'bun';
-  return 'npm';
-}
-
-/** Devuelve chips `{ label, command, args, cwd, source }` para arrancar el proyecto. */
-export function detectProcs(dir) {
-  const auto = [];
-  const pkg = readJson(path.join(dir, 'package.json'));
-  if (pkg?.scripts && typeof pkg.scripts === 'object') {
-    const pm = packageManager(dir);
-    const names = SCRIPT_ORDER.filter((name) => pkg.scripts[name]);
-    if (!names.length) {
-      const first = Object.keys(pkg.scripts)[0];
-      if (first) names.push(first);
-    }
-    for (const name of names) auto.push({ label: `${name} (${pm})`, command: pm, args: ['run', name], cwd: dir });
-  }
-
-  if (existsSync(path.join(dir, 'artisan'))) {
-    auto.push({ label: 'artisan serve', command: 'php', args: ['artisan', 'serve'], cwd: dir });
-  } else if (readJson(path.join(dir, 'composer.json'))?.scripts?.dev) {
-    auto.push({ label: 'composer dev', command: 'composer', args: ['run', 'dev'], cwd: dir });
-  }
-
-  for (const rel of ['public/index.php', 'backend/public/index.php', 'index.php']) {
-    if (!existsSync(path.join(dir, rel))) continue;
-    const docroot = path.dirname(rel) === '.' ? '.' : path.dirname(rel);
-    auto.push({ label: `php -S (${rel})`, command: 'php', args: ['-S', '127.0.0.1:8000', '-t', docroot], cwd: dir });
-    break;
-  }
-
-  if (!auto.length) {
-    const staticRoot = findStaticRoot(dir);
-    if (staticRoot) {
-      const target = staticRoot === '.' ? dir : path.join(dir, staticRoot);
-      auto.push(staticServerChip(target));
-    }
-  }
-
-  const chips = [];
-  const seen = new Set();
-  const add = (chip) => {
-    const key = `${chip.command}|${(chip.args ?? []).join(' ')}|${chip.cwd ?? ''}`;
-    if (seen.has(key)) return;
-    seen.add(key);
-    chips.push(chip);
-  };
-  for (const chip of loadProjectProcs(dir)) add(chip);
-  for (const chip of auto) add({ ...chip, source: 'auto' });
-  return chips;
-}

package/harness/packages/core/test/eval.test.js

@@ -1,144 +0,0 @@
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import http from 'node:http';
-import { mkdtemp, writeFile, mkdir } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-import { normalizeExpect, createTaskWorkspace } from '../src/eval/task.js';
-import { runScorers, validateSchema } from '../src/eval/scorers.js';
-import { runDataset, extractToolCalls } from '../src/eval/runner.js';
-
-test('normalizeExpect accepts strings, objects and arrays', () => {
-  assert.deepEqual(normalizeExpect('hola'), [{ type: 'contains', value: 'hola' }]);
-  assert.deepEqual(normalizeExpect({ equals: 'x' }), [{ type: 'exact', value: 'x' }]);
-  assert.deepEqual(normalizeExpect([{ regex: 'a+' }, 'b']), [
-    { type: 'regex', pattern: 'a+' },
-    { type: 'contains', value: 'b' },
-  ]);
-  assert.deepEqual(normalizeExpect(undefined), []);
-});
-
-test('validateSchema checks types, required and properties', () => {
-  const schema = {
-    type: 'object',
-    required: ['name'],
-    properties: { name: { type: 'string' }, age: { type: 'integer' } },
-  };
-  assert.equal(validateSchema({ name: 'a', age: 3 }, schema).valid, true);
-  assert.equal(validateSchema({ age: 3 }, schema).valid, false);
-  assert.equal(validateSchema({ name: 'a', age: 'x' }, schema).valid, false);
-});
-
-test('runScorers evaluates text, tool and file scorers', async () => {
-  const dir = await mkdtemp(path.join(tmpdir(), 'tamnora-score-'));
-  await writeFile(path.join(dir, 'out.txt'), 'contenido hola');
-
-  const task = {
-    expect: [
-      { type: 'contains', value: '4' },
-      { type: 'regex', pattern: '\\d+' },
-      { type: 'toolCalled', name: 'write', argsContains: 'out.txt' },
-      { type: 'fileExists', path: 'out.txt' },
-      { type: 'fileContains', path: 'out.txt', value: 'hola' },
-      { type: 'jsonSchema', schema: { type: 'object', required: ['ok'] } },
-    ],
-  };
-
-  const scoring = await runScorers(task, {
-    output: 'El resultado es 4: {"ok": true}',
-    toolCalls: [{ name: 'write', args: { path: 'out.txt' } }],
-    workspaceDir: dir,
-  });
-
-  assert.equal(scoring.passed, true);
-  assert.equal(scoring.score, 1);
-  assert.equal(scoring.details.length, 6);
-});
-
-test('runScorers fails when a criterion is not met', async () => {
-  const dir = await mkdtemp(path.join(tmpdir(), 'tamnora-score-'));
-  const scoring = await runScorers(
-    { expect: { contains: 'gato' } },
-    { output: 'perro', workspaceDir: dir }
-  );
-  assert.equal(scoring.passed, false);
-  assert.equal(scoring.details[0].detail.includes('no contiene'), true);
-});
-
-test('createTaskWorkspace copies the fixture', async () => {
-  const root = await mkdtemp(path.join(tmpdir(), 'tamnora-fixture-'));
-  await mkdir(path.join(root, 'fixtures', 't1'), { recursive: true });
-  await writeFile(path.join(root, 'fixtures', 't1', 'seed.txt'), 'seed');
-  const ws = await createTaskWorkspace({ fixture: 'fixtures/t1' }, { rootDir: root });
-  const { readFile } = await import('node:fs/promises');
-  assert.equal(await readFile(path.join(ws.dir, 'seed.txt'), 'utf8'), 'seed');
-  await ws.cleanup();
-});
-
-test('extractToolCalls parses tool calls from messages', () => {
-  const calls = extractToolCalls([
-    { role: 'assistant', tool_calls: [{ function: { name: 'write', arguments: '{"a":1}' } }] },
-    { role: 'tool', content: 'ok' },
-  ]);
-  assert.deepEqual(calls, [{ name: 'write', args: { a: 1 } }]);
-});
-
-test('runDataset scores a task end-to-end (mock Ollama server)', async () => {
-  const server = http.createServer((req, res) => {
-    let body = '';
-    req.on('data', (c) => (body += c));
-    req.on('end', () => {
-      const parsed = JSON.parse(body);
-      const hasTool = parsed.messages.some((m) => m.role === 'tool');
-      res.setHeader('Content-Type', 'application/x-ndjson');
-      if (!hasTool) {
-        res.write(
-          `${JSON.stringify({
-            message: {
-              content: '',
-              tool_calls: [{ function: { name: 'write', arguments: { path: 'x.txt', content: 'hi' } } }],
-            },
-            done: false,
-          })}\n`
-        );
-      } else {
-        res.write(`${JSON.stringify({ message: { content: 'listo' }, done: false })}\n`);
-      }
-      res.write(`${JSON.stringify({ done: true })}\n`);
-      res.end();
-    });
-  });
-  await new Promise((r) => server.listen(0, '127.0.0.1', r));
-  const { port } = server.address();
-
-  try {
-    const results = await runDataset({
-      tasks: [
-        {
-          id: 't1',
-          input: 'creá x.txt',
-          expect: [
-            { type: 'toolCalled', name: 'write' },
-            { type: 'fileContains', path: 'x.txt', value: 'hi' },
-            { type: 'contains', value: 'listo' },
-          ],
-        },
-      ],
-      models: ['mock'],
-      providerId: 'ollama',
-      baseUrl: `http://127.0.0.1:${port}`,
-      systemPrompt: 'test',
-      enabledTools: ['write'],
-      yes: true,
-      concurrency: 1,
-      rootDir: process.cwd(),
-    });
-
-    assert.equal(results.length, 1);
-    assert.equal(results[0].passed, true);
-    assert.equal(results[0].error, null);
-    assert.equal(results[0].trace.scorers.length, 3);
-  } finally {
-    server.close();
-  }
-});