@danieltmn/tamnora 0.1.0

package/harness/packages/core/src/eval/scorers.js

@@ -0,0 +1,217 @@
+import { readFile, access } from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { normalizeExpect } from './task.js';
+
+const DEFAULT_SCRIPT_TIMEOUT = 60;
+
+function outputText(output) {
+  return typeof output === 'string' ? output : output == null ? '' : JSON.stringify(output);
+}
+
+/** Extract a JSON value from text, tolerating ```json fenced blocks. */
+export function extractJson(text) {
+  const trimmed = String(text ?? '').trim();
+  if (!trimmed) return { error: 'vacío' };
+
+  try {
+    return { value: JSON.parse(trimmed) };
+  } catch {
+    // try fenced block
+  }
+
+  const match = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (match) {
+    try {
+      return { value: JSON.parse(match[1].trim()) };
+    } catch (err) {
+      return { error: err.message };
+    }
+  }
+
+  // inline object/array inside surrounding text
+  for (const [open, close] of [
+    ['{', '}'],
+    ['[', ']'],
+  ]) {
+    const start = trimmed.indexOf(open);
+    const end = trimmed.lastIndexOf(close);
+    if (start !== -1 && end > start) {
+      try {
+        return { value: JSON.parse(trimmed.slice(start, end + 1)) };
+      } catch {
+        // keep looking
+      }
+    }
+  }
+
+  return { error: 'no se pudo parsear JSON' };
+}
+
+/** Minimal JSON Schema validator (type, required, properties, items, enum). */
+export function validateSchema(value, schema) {
+  const errors = [];
+  const check = (val, sch, pointer) => {
+    if (!sch || typeof sch !== 'object') return;
+    if (sch.type) {
+      const types = Array.isArray(sch.type) ? sch.type : [sch.type];
+      const actual =
+        val === null ? 'null' : Array.isArray(val) ? 'array' : typeof val === 'number' && Number.isInteger(val) ? 'integer' : typeof val;
+      const ok = types.some((t) => t === actual || (t === 'number' && actual === 'integer'));
+      if (!ok) errors.push(`${pointer}: se esperaba ${types.join('|')}, llegó ${actual}`);
+    }
+    if (Array.isArray(sch.enum) && !sch.enum.includes(val)) {
+      errors.push(`${pointer}: valor no está en enum`);
+    }
+    if (sch.type === 'object' || sch.properties) {
+      for (const req of sch.required ?? []) {
+        if (val == null || val[req] === undefined) errors.push(`${pointer}: falta "${req}"`);
+      }
+      for (const [key, sub] of Object.entries(sch.properties ?? {})) {
+        if (val && val[key] !== undefined) check(val[key], sub, `${pointer}.${key}`);
+      }
+    }
+    if (sch.type === 'array' && sch.items && Array.isArray(val)) {
+      val.forEach((item, i) => check(item, sch.items, `${pointer}[${i}]`));
+    }
+  };
+  check(value, schema, '$');
+  return { valid: errors.length === 0, errors };
+}
+
+function runCommand(command, cwd, timeoutSec = DEFAULT_SCRIPT_TIMEOUT) {
+  return new Promise((resolve) => {
+    const child = spawn(command, { cwd, shell: true, windowsHide: true });
+    let output = '';
+    const append = (d) => {
+      if (output.length < 20_000) output += d.toString();
+    };
+    child.stdout?.on('data', append);
+    child.stderr?.on('data', append);
+    const timer = setTimeout(() => child.kill('SIGKILL'), timeoutSec * 1000);
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      resolve({ code: -1, output: err.message });
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      resolve({ code, output });
+    });
+  });
+}
+
+const SCORERS = {
+  exact(spec, ctx) {
+    const a = outputText(ctx.output).trim();
+    const b = String(spec.value ?? '').trim();
+    const match = spec.caseSensitive ? a === b : a.toLowerCase() === b.toLowerCase();
+    return { passed: match, detail: match ? 'coincide' : `esperaba "${b}"` };
+  },
+
+  contains(spec, ctx) {
+    const haystack = spec.caseSensitive ? outputText(ctx.output) : outputText(ctx.output).toLowerCase();
+    const needle = spec.caseSensitive ? String(spec.value ?? '') : String(spec.value ?? '').toLowerCase();
+    const match = needle ? haystack.includes(needle) : true;
+    return { passed: match, detail: match ? `contiene "${spec.value}"` : `no contiene "${spec.value}"` };
+  },
+
+  regex(spec, ctx) {
+    try {
+      const re = new RegExp(spec.pattern, spec.flags ?? '');
+      const match = re.test(outputText(ctx.output));
+      return { passed: match, detail: match ? 'regex ok' : `no matchea /${spec.pattern}/` };
+    } catch (err) {
+      return { passed: false, detail: `regex inválida: ${err.message}` };
+    }
+  },
+
+  jsonSchema(spec, ctx) {
+    const { value, error } = extractJson(outputText(ctx.output));
+    if (error) return { passed: false, detail: `JSON inválido: ${error}` };
+    const { valid, errors } = validateSchema(value, spec.schema ?? {});
+    return { passed: valid, detail: valid ? 'schema ok' : errors.join('; ') };
+  },
+
+  toolCalled(spec, ctx) {
+    const calls = ctx.toolCalls ?? [];
+    const match = calls.some((c) => {
+      if (c.name !== spec.name) return false;
+      if (!spec.argsContains) return true;
+      return JSON.stringify(c.args ?? {}).includes(spec.argsContains);
+    });
+    return {
+      passed: match,
+      detail: match ? `llamó ${spec.name}` : `no llamó ${spec.name}`,
+    };
+  },
+
+  async fileExists(spec, ctx) {
+    try {
+      await access(path.join(ctx.workspaceDir, spec.path));
+      return { passed: true, detail: `${spec.path} existe` };
+    } catch {
+      return { passed: false, detail: `${spec.path} no existe` };
+    }
+  },
+
+  async fileContains(spec, ctx) {
+    try {
+      const content = await readFile(path.join(ctx.workspaceDir, spec.path), 'utf8');
+      const match = content.includes(spec.value);
+      return { passed: match, detail: match ? `${spec.path} contiene "${spec.value}"` : `no contiene` };
+    } catch (err) {
+      return { passed: false, detail: `no se pudo leer ${spec.path}: ${err.message}` };
+    }
+  },
+
+  async script(spec, ctx) {
+    const { code, output } = await runCommand(spec.command, ctx.workspaceDir, spec.timeout);
+    const expected = spec.exit ?? 0;
+    return {
+      passed: code === expected,
+      detail: `exit ${code} (esperado ${expected})${output ? `: ${output.slice(0, 200)}` : ''}`,
+    };
+  },
+
+  async llmJudge(spec, ctx) {
+    if (!ctx.judge) return { passed: false, detail: 'no hay modelo juez configurado' };
+    return ctx.judge({ criteria: spec.criteria, output: outputText(ctx.output), input: ctx.input });
+  },
+};
+
+const DEFAULT_SCORER_NAMES = Object.keys(SCORERS);
+
+/**
+ * Evaluate a task result against its scorers.
+ * @param {object} task
+ * @param {{output:any, toolCalls?:object[], workspaceDir:string, input?:string, judge?:Function}} ctx
+ * @param {string[]} [scorerNames] - subset of scorers to use
+ */
+export async function runScorers(task, ctx, scorerNames = DEFAULT_SCORER_NAMES) {
+  const specs = normalizeExpect(task.expect);
+  if (!specs.length) return { passed: true, score: 1, details: [] };
+
+  const details = [];
+  for (const spec of specs) {
+    if (!scorerNames.includes(spec.type) || !SCORERS[spec.type]) {
+      details.push({ scorer: spec.type, passed: false, detail: 'scorer desconocido' });
+      continue;
+    }
+    let result;
+    try {
+      result = await SCORERS[spec.type](spec, ctx);
+    } catch (err) {
+      result = { passed: false, detail: `error: ${err.message}` };
+    }
+    details.push({ scorer: spec.type, ...result });
+  }
+
+  const passedCount = details.filter((d) => d.passed).length;
+  return {
+    passed: passedCount === details.length,
+    score: details.length ? passedCount / details.length : 1,
+    details,
+  };
+}
+
+export { SCORERS, DEFAULT_SCORER_NAMES };

package/harness/packages/core/src/eval/task.js

@@ -0,0 +1,59 @@
+import { mkdtemp, cp, mkdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+/**
+ * Create an isolated workspace for a task. If the task has a `fixture`
+ * (a directory, relative to `rootDir`), it is copied into the workspace.
+ *
+ * @param {object} task
+ * @param {{rootDir?: string, parentDir?: string}} [opts]
+ * @returns {Promise<{dir:string, parent:string, cleanup:()=>Promise<void>}>}
+ */
+export async function createTaskWorkspace(task, { rootDir = process.cwd(), parentDir } = {}) {
+  const parent = parentDir ?? (await mkdtemp(path.join(tmpdir(), 'tamnora-task-')));
+  const dir = path.join(parent, 'workspace');
+  await mkdir(dir, { recursive: true });
+
+  if (task.fixture) {
+    const source = path.isAbsolute(task.fixture)
+      ? task.fixture
+      : path.resolve(rootDir, task.fixture);
+    await cp(source, dir, { recursive: true });
+  }
+
+  const cleanup = async () => {
+    if (task.keepWorkspace) return;
+    await rm(parent, { recursive: true, force: true }).catch(() => {});
+  };
+
+  return { dir, parent, cleanup };
+}
+
+/**
+ * Normalize a task's `expect` field into a list of scorer specs.
+ * Accepts a string, a single object, or an array; also supports shorthand keys.
+ */
+export function normalizeExpect(expect) {
+  if (expect === undefined || expect === null) return [];
+  if (typeof expect === 'string') return [{ type: 'contains', value: expect }];
+  if (Array.isArray(expect)) return expect.map(normalizeSpec);
+  return [normalizeSpec(expect)];
+}
+
+function normalizeSpec(spec) {
+  if (typeof spec === 'string') return { type: 'contains', value: spec };
+  if (spec.type) return spec;
+  if ('contains' in spec) return { type: 'contains', value: spec.contains };
+  if ('equals' in spec) return { type: 'exact', value: spec.equals };
+  if ('regex' in spec) return { type: 'regex', pattern: spec.regex };
+  if ('jsonSchema' in spec) return { type: 'jsonSchema', schema: spec.jsonSchema };
+  if ('toolCalled' in spec) return { type: 'toolCalled', name: spec.toolCalled };
+  if ('fileExists' in spec) return { type: 'fileExists', path: spec.fileExists };
+  if ('fileContains' in spec) {
+    return { type: 'fileContains', path: spec.fileContains.path, value: spec.fileContains.value };
+  }
+  if ('script' in spec) return { type: 'script', command: spec.script };
+  if ('llmJudge' in spec) return { type: 'llmJudge', criteria: spec.llmJudge };
+  return { type: 'unknown', raw: spec };
+}

package/harness/packages/core/src/index.js

@@ -0,0 +1,72 @@
+export { getEnv, envValue } from './env.js';
+export { MAX_TOOL_ROUNDS, runAgent, parseToolArguments } from './agent/runAgent.js';
+export { estimateTokens, trimContext } from './agent/context.js';
+export { diffLines } from './tools/diff.js';
+export { MODES, MODE_IDS, resolveMode } from './agent/modes.js';
+export { withRetry, isRetryableError, isAbortError } from './providers/resilience.js';
+export {
+  PROVIDERS,
+  PROVIDER_META,
+  getProvider,
+  createCustomProvider,
+  resolveProviderRuntime,
+  listProviderModels,
+  testProviderConnection,
+  streamChat,
+} from './providers/index.js';
+export {
+  connectMcpServer,
+  connectMcpHttpServer,
+  connectMcpStdioServer,
+  buildMcpRegistry,
+  mcpTransport,
+  qualifiedToolName,
+  mcpToolToDefinition,
+} from './mcp/index.js';
+export {
+  loadConfig,
+  loadInstructions,
+  resolveModel,
+  resolveProvider,
+  databasePath,
+  STATE_DIR,
+  DEFAULT_CONFIG,
+} from './config/index.js';
+export {
+  DIR_NAME,
+  setHome,
+  homeDir,
+  homePath,
+  appConfigPath,
+  databaseFile,
+  runtimePath,
+  pidPath,
+  logsDir,
+  serverLogPath,
+  bridgeLogPath,
+  bridgeStatePath,
+  ensureHome,
+  homeExists,
+  migrateLegacyDb,
+} from './config/home.js';
+export { DEFAULT_APP, readApp, writeApp, ensureApp, randomToken } from './config/app.js';
+export { hashPassword, verifyPassword, userValidName, makeUser } from './config/identity.js';
+export { composeProviderSettings } from './config/providers.js';
+export {
+  createTools,
+  createToolRegistry,
+  TOOL_DEFINITIONS,
+  TOOL_NAMES,
+  DEFAULT_PERMISSIONS,
+  evaluatePermission,
+  resolveInRoot,
+  toRel,
+} from './tools/index.js';
+export { Store, openDatabase } from './storage/sqlite.js';
+export { runTask, resolveApiKey } from './eval/runTask.js';
+export { loadDataset } from './eval/dataset.js';
+export { runDataset, extractToolCalls } from './eval/runner.js';
+export { summarize } from './eval/report.js';
+export { runScorers, validateSchema, extractJson, SCORERS } from './eval/scorers.js';
+export { createJudge } from './eval/judge.js';
+export { createTaskWorkspace, normalizeExpect } from './eval/task.js';

package/harness/packages/core/src/mcp/common.js

@@ -0,0 +1,82 @@
+// Shared MCP helpers used by both transports (HTTP and stdio).
+
+export const PROTOCOL_VERSION = '2025-06-18';
+export const CLIENT_INFO = { name: 'tamnora', version: '1.0.0' };
+
+/** Namespace an MCP tool as `mcp__<server>__<tool>`, sanitized and capped. */
+export function qualifiedToolName(serverId, toolName) {
+  const prefix = String(serverId).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 20);
+  return `mcp__${prefix}__${toolName}`.slice(0, 64);
+}
+
+export function mcpToolToDefinition(tool, server, qualifiedName) {
+  return {
+    type: 'function',
+    function: {
+      name: qualifiedName,
+      description:
+        tool.description || `MCP tool "${tool.name}" del servidor ${server.name ?? server.id}`,
+      parameters: tool.inputSchema ?? { type: 'object', properties: {} },
+    },
+  };
+}
+
+/** Flatten an MCP `tools/call` result into plain text for the model. */
+export function mcpResultToText(result) {
+  if (!result) return '';
+  if (typeof result.content === 'string') return result.content;
+  if (Array.isArray(result.content)) {
+    const text = result.content
+      .map((c) => (c.type === 'text' ? c.text : `[${c.type}]`))
+      .join('\n');
+    return result.isError ? `Error: ${text}` : text;
+  }
+  return JSON.stringify(result);
+}
+
+/** Flatten a `resources/read` result into plain text. */
+export function mcpResourceToText(result) {
+  const contents = result?.contents ?? [];
+  if (!contents.length) return '(recurso vacío)';
+  return contents
+    .map((c) => {
+      if (typeof c.text === 'string') return c.text;
+      if (c.blob) return `[binario ${c.mimeType ?? ''} ${c.blob.length}b64]`;
+      return JSON.stringify(c);
+    })
+    .join('\n');
+}
+
+/** Flatten a `prompts/get` result into plain text. */
+export function mcpPromptToText(result) {
+  if (!result) return '';
+  const parts = [];
+  if (result.description) parts.push(result.description);
+  for (const message of result.messages ?? []) {
+    const content =
+      typeof message.content === 'string'
+        ? message.content
+        : (message.content?.text ?? JSON.stringify(message.content));
+    parts.push(`[${message.role}] ${content}`);
+  }
+  return parts.join('\n') || '(prompt vacío)';
+}
+
+/** Text summary of a `resources/list` result. */
+export function formatResources(resources = []) {
+  if (!resources.length) return '(sin recursos)';
+  return resources
+    .map((r) => `${r.uri}  ${r.name ?? ''}${r.mimeType ? `  [${r.mimeType}]` : ''}`.trim())
+    .join('\n');
+}
+
+/** Text summary of a `prompts/list` result. */
+export function formatPrompts(prompts = []) {
+  if (!prompts.length) return '(sin prompts)';
+  return prompts
+    .map((p) => {
+      const args = (p.arguments ?? []).map((a) => a.name).join(', ');
+      return `${p.name}${args ? ` (${args})` : ''}  ${p.description ?? ''}`.trim();
+    })
+    .join('\n');
+}

package/harness/packages/core/src/mcp/http.js

@@ -0,0 +1,149 @@
+// MCP (Model Context Protocol) client over the Streamable HTTP transport.
+// Works in Node and the browser. The stdio transport lives in `./stdio.js`.
+
+import {
+  PROTOCOL_VERSION,
+  CLIENT_INFO,
+  mcpResultToText,
+  mcpResourceToText,
+  mcpPromptToText,
+} from './common.js';
+
+let rpcCounter = 0;
+const nextId = () => ++rpcCounter;
+
+async function rpc(url, { sessionId, method, params, id, notification, signal }) {
+  const body = notification
+    ? { jsonrpc: '2.0', method, params }
+    : { jsonrpc: '2.0', id: id ?? nextId(), method, params };
+
+  const headers = {
+    'Content-Type': 'application/json',
+    Accept: 'application/json, text/event-stream',
+  };
+  if (sessionId) headers['Mcp-Session-Id'] = sessionId;
+
+  const res = await fetch(url, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(body),
+    signal,
+  });
+  const nextSessionId = res.headers.get('Mcp-Session-Id') ?? sessionId;
+
+  if (notification) return { sessionId: nextSessionId };
+  if (!res.ok) throw new Error(`MCP HTTP ${res.status} ${res.statusText}`);
+
+  const contentType = res.headers.get('content-type') ?? '';
+  const payload = contentType.includes('text/event-stream')
+    ? parseSsePayload(await res.text())
+    : await res.json();
+
+  if (payload?.error) throw new Error(payload.error.message ?? 'MCP error');
+  return { sessionId: nextSessionId, result: payload?.result };
+}
+
+function parseSsePayload(text) {
+  for (const event of text.split(/\r?\n\r?\n/)) {
+    for (const line of event.split(/\r?\n/)) {
+      if (!line.startsWith('data:')) continue;
+      const data = line.slice(5).trim();
+      if (!data) continue;
+      try {
+        const parsed = JSON.parse(data);
+        if (parsed.result !== undefined || parsed.error !== undefined) return parsed;
+      } catch {
+        // ignore non-JSON data lines
+      }
+    }
+  }
+  throw new Error('MCP: la respuesta SSE no contiene un resultado');
+}
+
+/**
+ * Open a session against an MCP HTTP server and list its tools.
+ * @param {{id:string, name?:string, url:string}} server
+ */
+export async function connectMcpHttpServer(server, { signal } = {}) {
+  const { url } = server;
+
+  const init = await rpc(url, {
+    method: 'initialize',
+    id: 1,
+    signal,
+    params: { protocolVersion: PROTOCOL_VERSION, capabilities: {}, clientInfo: CLIENT_INFO },
+  });
+
+  const sessionId = init.sessionId;
+  await rpc(url, { sessionId, method: 'notifications/initialized', notification: true, signal });
+
+  const capabilities = init.result?.capabilities ?? {};
+  const listed = await rpc(url, { sessionId, method: 'tools/list', params: {}, id: 2, signal });
+  const tools = listed.result?.tools ?? [];
+
+  let resources = [];
+  if (capabilities.resources) {
+    try {
+      const res = await rpc(url, { sessionId, method: 'resources/list', params: {}, id: 3, signal });
+      resources = res.result?.resources ?? [];
+    } catch {
+      resources = [];
+    }
+  }
+  let prompts = [];
+  if (capabilities.prompts) {
+    try {
+      const res = await rpc(url, { sessionId, method: 'prompts/list', params: {}, id: 4, signal });
+      prompts = res.result?.prompts ?? [];
+    } catch {
+      prompts = [];
+    }
+  }
+
+  return {
+    server,
+    transport: 'http',
+    sessionId,
+    capabilities,
+    tools,
+    resources,
+    prompts,
+    alive: () => true,
+    async callTool(name, args) {
+      const res = await rpc(url, {
+        sessionId,
+        method: 'tools/call',
+        id: 5,
+        params: { name, arguments: args ?? {} },
+        signal,
+      });
+      return mcpResultToText(res.result);
+    },
+    async refreshTools() {
+      const res = await rpc(url, { sessionId, method: 'tools/list', params: {}, signal });
+      return res.result?.tools ?? tools;
+    },
+    async listResources() {
+      const res = await rpc(url, { sessionId, method: 'resources/list', params: {}, signal });
+      return res.result?.resources ?? resources;
+    },
+    async readResource(uri) {
+      const res = await rpc(url, { sessionId, method: 'resources/read', params: { uri }, signal });
+      return mcpResourceToText(res.result);
+    },
+    async listPrompts() {
+      const res = await rpc(url, { sessionId, method: 'prompts/list', params: {}, signal });
+      return res.result?.prompts ?? prompts;
+    },
+    async getPrompt(name, args) {
+      const res = await rpc(url, {
+        sessionId,
+        method: 'prompts/get',
+        params: { name, arguments: args ?? {} },
+        signal,
+      });
+      return mcpPromptToText(res.result);
+    },
+    close() {},
+  };
+}