@danieltmn/tamnora 0.1.0

package/harness/packages/core/src/config/app.js

@@ -0,0 +1,49 @@
+import { randomBytes } from 'node:crypto';
+import { readFileSync, writeFileSync, renameSync } from 'node:fs';
+import { appConfigPath, ensureHome } from './home.js';
+
+/**
+ * Configuracion de la app del server (no del proyecto): puerto, tunel y, mas
+ * adelante, el usuario y los secretos de autenticacion.
+ */
+export const DEFAULT_APP = {
+  port: 4517,
+  host: '127.0.0.1',
+  tunnel: { provider: 'tunnelmole', domain: '' },
+  authSecret: '',
+  user: null,
+};
+
+export function randomToken(bytes = 32) {
+  return randomBytes(bytes).toString('base64url');
+}
+
+export function readApp() {
+  let data = {};
+  try {
+    data = JSON.parse(readFileSync(appConfigPath(), 'utf8'));
+  } catch {
+    data = {};
+  }
+  const app = { ...DEFAULT_APP, ...(data && typeof data === 'object' ? data : {}) };
+  app.tunnel = { ...DEFAULT_APP.tunnel, ...(app.tunnel || {}) };
+  return app;
+}
+
+export function writeApp(app) {
+  ensureHome();
+  const file = appConfigPath();
+  const tmp = `${file}.${process.pid}.tmp`;
+  writeFileSync(tmp, `${JSON.stringify(app, null, 2)}\n`);
+  renameSync(tmp, file);
+}
+
+/** Crea `app.json` con un `authSecret` si falta. Devuelve la app efectiva. */
+export function ensureApp() {
+  const app = readApp();
+  if (!app.authSecret) {
+    app.authSecret = randomToken(32);
+    writeApp(app);
+  }
+  return app;
+}

package/harness/packages/core/src/config/home.js

@@ -0,0 +1,98 @@
+import { existsSync, mkdirSync, copyFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+/**
+ * Casa de Tamnora: directorio global donde viven la configuracion de la app
+ * (app.json), la base SQLite, los logs y los archivos de runtime del daemon.
+ *
+ * Orden de resolucion: override programatico (`setHome`) > `TAMNORA_HOME` >
+ * directorio de datos del sistema. La base por defecto es por usuario (no por
+ * proyecto) para que proveedores y claves sean unicos entre proyectos.
+ */
+
+export const DIR_NAME = '.tamnora';
+
+let overrideHome = null;
+
+export function setHome(dir) {
+  overrideHome = dir ? path.resolve(dir) : null;
+}
+
+function appDataDir() {
+  if (process.platform === 'win32') {
+    return process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming');
+  }
+  if (process.platform === 'darwin') {
+    return path.join(os.homedir(), 'Library', 'Application Support');
+  }
+  return process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config');
+}
+
+export function homeDir() {
+  if (overrideHome) return overrideHome;
+  if (process.env.TAMNORA_HOME) return path.resolve(process.env.TAMNORA_HOME);
+  return path.join(appDataDir(), 'tamnora');
+}
+
+export function homePath(...parts) {
+  return path.join(homeDir(), ...parts);
+}
+
+export function appConfigPath() {
+  return homePath('app.json');
+}
+export function databaseFile() {
+  return homePath('tamnora.db');
+}
+export function runtimePath() {
+  return homePath('runtime.json');
+}
+export function pidPath() {
+  return homePath('.tamnora.pid');
+}
+export function logsDir() {
+  return homePath('logs');
+}
+export function serverLogPath() {
+  return homePath('logs', 'server.log');
+}
+export function bridgeLogPath() {
+  return homePath('logs', 'bridge.log');
+}
+export function bridgeStatePath() {
+  return homePath('bridge.json');
+}
+
+export function ensureHome() {
+  for (const dir of [homeDir(), logsDir()]) {
+    try {
+      mkdirSync(dir, { recursive: true });
+    } catch {
+      /* ya existe */
+    }
+  }
+}
+
+export function homeExists() {
+  return existsSync(appConfigPath());
+}
+
+/**
+ * Migra una base local legacy (`<cwd>/.tamnora/tamnora.db`) a la casa global,
+ * una sola vez. Devuelve la ruta del archivo global.
+ */
+export function migrateLegacyDb(cwd = process.cwd()) {
+  const target = databaseFile();
+  if (existsSync(target)) return target;
+  const legacy = path.join(cwd, DIR_NAME, 'tamnora.db');
+  if (existsSync(legacy)) {
+    try {
+      ensureHome();
+      copyFileSync(legacy, target);
+    } catch {
+      /* si falla, se arranca con una base nueva */
+    }
+  }
+  return target;
+}

package/harness/packages/core/src/config/identity.js

@@ -0,0 +1,41 @@
+import { randomBytes, scryptSync, timingSafeEqual } from 'node:crypto';
+
+/**
+ * Identidad local (v1: un solo usuario). La contrasena se guarda como hash
+ * scrypt con salt; `pv` (password version) invalida las sesiones al cambiarla.
+ */
+
+export function hashPassword(password) {
+  const salt = randomBytes(16).toString('hex');
+  const keylen = 64;
+  const hash = scryptSync(String(password), salt, keylen).toString('hex');
+  return { algo: 'scrypt', salt, hash, keylen };
+}
+
+export function verifyPassword(password, stored) {
+  if (!stored || stored.algo !== 'scrypt' || !stored.salt || !stored.hash) return false;
+  const keylen = parseInt(stored.keylen, 10) || 64;
+  let calc;
+  try {
+    calc = scryptSync(String(password), stored.salt, keylen);
+  } catch {
+    return false;
+  }
+  const expected = Buffer.from(stored.hash, 'hex');
+  if (calc.length !== expected.length) return false;
+  return timingSafeEqual(calc, expected);
+}
+
+export function userValidName(name) {
+  return typeof name === 'string' && /^[A-Za-z0-9._-]{1,32}$/.test(name);
+}
+
+export function makeUser(name, password, prev = null) {
+  return {
+    name: String(name),
+    password: hashPassword(password),
+    pv: (parseInt(prev?.pv, 10) || 0) + 1,
+    created: prev?.created || new Date().toISOString(),
+    updated: new Date().toISOString(),
+  };
+}

package/harness/packages/core/src/config/index.js

@@ -0,0 +1,69 @@
+import { readFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { migrateLegacyDb } from './home.js';
+
+export const DEFAULT_CONFIG = {
+  provider: 'openrouter',
+  model: '',
+  instructions: ['AGENTS.md'],
+  maxRounds: 8,
+  tools: [],
+  permissions: {},
+  mcp: [],
+  server: { host: '127.0.0.1', port: 4517 },
+};
+
+/** Directory (relative to the project) where harness state lived historically. */
+export const STATE_DIR = '.tamnora';
+
+/**
+ * Absolute path to the SQLite database. Es **global** (casa de Tamnora) para
+ * que proveedores y claves no se dupliquen entre proyectos. `TAMNORA_DB` lo
+ * sobrescribe; si no existe la base global se migra la legacy del `cwd`.
+ */
+export function databasePath(cwd = process.cwd()) {
+  return process.env.TAMNORA_DB || migrateLegacyDb(cwd);
+}
+
+/**
+ * Load `tamnora.json` from `cwd` (if present) merged over defaults.
+ * @param {string} [cwd]
+ */
+export async function loadConfig(cwd = process.cwd()) {
+  const file = path.join(cwd, 'tamnora.json');
+  let user = {};
+  if (existsSync(file)) {
+    try {
+      user = JSON.parse(await readFile(file, 'utf8'));
+    } catch (err) {
+      throw new Error(`tamnora.json inválido: ${err.message}`);
+    }
+  }
+  return { ...DEFAULT_CONFIG, ...user, cwd };
+}
+
+/**
+ * Read instruction files (AGENTS.md by default) and concatenate them.
+ * @param {string} cwd
+ * @param {string[]} files
+ */
+export async function loadInstructions(cwd = process.cwd(), files = DEFAULT_CONFIG.instructions) {
+  const parts = [];
+  for (const relative of files) {
+    const file = path.join(cwd, relative);
+    if (existsSync(file)) {
+      parts.push((await readFile(file, 'utf8')).trim());
+    }
+  }
+  return parts.join('\n\n---\n\n');
+}
+
+/** Resolve the effective model string from config/env, honoring overrides. */
+export function resolveModel(config, override) {
+  return override || config.model || '';
+}
+
+export function resolveProvider(config, override) {
+  return override || config.provider || 'openrouter';
+}

package/harness/packages/core/src/config/providers.js

@@ -0,0 +1,24 @@
+// Compose the settings object that `resolveProviderRuntime` / `listProviderModels`
+// consume, merging a stored provider record over the project config.
+
+/**
+ * @param {object} opts
+ * @param {string} opts.providerId
+ * @param {object} [opts.stored] - provider row from the Store ({ apiKey, baseUrl, models, custom, label })
+ * @param {object} [opts.config] - project config (tamnora.json)
+ * @param {object[]} [opts.customProviders] - custom provider definitions from config
+ */
+export function composeProviderSettings({ providerId, stored = {}, config = {}, customProviders = [] } = {}) {
+  const models = Array.isArray(stored.models) ? stored.models : [];
+  const custom = stored.custom
+    ? [{ id: providerId, label: stored.label || providerId, baseUrl: stored.baseUrl || '', models }]
+    : customProviders;
+
+  return {
+    providerId,
+    apiKey: stored.apiKey || config.apiKey || '',
+    baseUrl: stored.baseUrl || config.baseUrl || undefined,
+    customProviders: custom,
+    providerModels: models.length ? { [providerId]: models } : config.providerModels ?? {},
+  };
+}

package/harness/packages/core/src/env.js

@@ -0,0 +1,19 @@
+// Environment access that works in Node and (defensively) in the browser.
+// The Node CLI/harness reads keys from process.env (e.g. OPENROUTER_API_KEY
+// or the VITE_* variants used by the web app).
+
+export function getEnv() {
+  if (typeof process !== 'undefined' && process.env) return process.env;
+  return {};
+}
+
+/** First defined value among the given env names. */
+export function envValue(names, fallback = '') {
+  const env = getEnv();
+  const list = Array.isArray(names) ? names : [names];
+  for (const name of list) {
+    const value = env[name];
+    if (value !== undefined && value !== '') return value;
+  }
+  return fallback;
+}

package/harness/packages/core/src/eval/dataset.js

@@ -0,0 +1,47 @@
+import { readFile } from 'node:fs/promises';
+
+/**
+ * Load an evaluation dataset. Supports:
+ *  - a JSON array of tasks
+ *  - a single JSON object ({ tasks: [...] } or one task)
+ *  - JSONL (one task per line)
+ *
+ * @param {string} file
+ * @returns {Promise<object[]>}
+ */
+export async function loadDataset(file) {
+  const raw = await readFile(file, 'utf8');
+  const trimmed = raw.trim();
+  if (!trimmed) return [];
+
+  // JSONL: each non-empty line is a standalone JSON object
+  if (!trimmed.startsWith('[') && !trimmed.startsWith('{')) {
+    return parseJsonl(trimmed, file);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch {
+    // Fall back to JSONL (multi-line pretty JSON would have parsed above)
+    return parseJsonl(trimmed, file);
+  }
+
+  if (Array.isArray(parsed)) return parsed;
+  if (Array.isArray(parsed.tasks)) return parsed.tasks;
+  return [parsed];
+}
+
+function parseJsonl(text, file) {
+  const tasks = [];
+  const lines = text.split(/\r?\n/);
+  lines.forEach((line, index) => {
+    if (!line.trim()) return;
+    try {
+      tasks.push(JSON.parse(line));
+    } catch (err) {
+      throw new Error(`Línea ${index + 1} inválida en ${file}: ${err.message}`);
+    }
+  });
+  return tasks;
+}

package/harness/packages/core/src/eval/judge.js

@@ -0,0 +1,39 @@
+import { resolveProviderRuntime } from '../providers/index.js';
+import { extractJson } from './scorers.js';
+
+/**
+ * Build an LLM-as-judge function. Returns null when no judge model is set.
+ * The judge model receives the criteria and the candidate output and must
+ * reply with JSON: { "passed": boolean, "reason": string }.
+ *
+ * @param {{providerId?:string, model?:string, baseUrl?:string, apiKey?:string}} opts
+ */
+export function createJudge({ providerId, model, baseUrl, apiKey } = {}) {
+  if (!model) return null;
+  const { provider, config } = resolveProviderRuntime({ providerId, apiKey, baseUrl });
+
+  return async ({ criteria, output, input }) => {
+    const prompt =
+      'Sos un evaluador estricto. Decidí si la RESPUESTA cumple el CRITERIO.\n\n' +
+      `CRITERIO: ${criteria}\n` +
+      `ENTRADA: ${input ?? ''}\n` +
+      `RESPUESTA: ${output ?? ''}\n\n` +
+      'Respondé SOLO un JSON: {"passed": true|false, "reason": "breve"}';
+
+    try {
+      const result = await provider.streamChat(config, {
+        model,
+        messages: [{ role: 'user', content: prompt }],
+        tools: [],
+        onEvent: () => {},
+      });
+      const { value, error } = extractJson(result.content ?? '');
+      if (error || typeof value?.passed !== 'boolean') {
+        return { passed: false, detail: `juez: respuesta inválida (${error ?? 'sin "passed"'})` };
+      }
+      return { passed: value.passed, detail: value.reason ?? 'juez' };
+    } catch (err) {
+      return { passed: false, detail: `juez error: ${err.message}` };
+    }
+  };
+}

package/harness/packages/core/src/eval/report.js

@@ -0,0 +1,48 @@
+/**
+ * Aggregate evaluation results into per-model and per-scorer summaries.
+ * @param {object[]} results
+ */
+export function summarize(results) {
+  const byModel = new Map();
+  const byScorer = new Map();
+  let passed = 0;
+  let errors = 0;
+  let durationMs = 0;
+
+  for (const result of results) {
+    passed += result.passed ? 1 : 0;
+    errors += result.error ? 1 : 0;
+    durationMs += result.durationMs ?? 0;
+
+    const model = byModel.get(result.model) ?? {
+      model: result.model,
+      total: 0,
+      passed: 0,
+      errors: 0,
+      durationMs: 0,
+    };
+    model.total += 1;
+    model.passed += result.passed ? 1 : 0;
+    model.errors += result.error ? 1 : 0;
+    model.durationMs += result.durationMs ?? 0;
+    byModel.set(result.model, model);
+
+    for (const detail of result.trace?.scorers ?? []) {
+      const scorer = byScorer.get(detail.scorer) ?? { scorer: detail.scorer, total: 0, passed: 0 };
+      scorer.total += 1;
+      scorer.passed += detail.passed ? 1 : 0;
+      byScorer.set(detail.scorer, scorer);
+    }
+  }
+
+  const models = [...byModel.values()].map((m) => ({
+    ...m,
+    passRate: m.total ? m.passed / m.total : 0,
+  }));
+  const scorers = [...byScorer.values()].map((s) => ({
+    ...s,
+    passRate: s.total ? s.passed / s.total : 0,
+  }));
+
+  return { total: results.length, passed, errors, durationMs, models, scorers };
+}

package/harness/packages/core/src/eval/runTask.js

@@ -0,0 +1,71 @@
+import { resolveProviderRuntime } from '../providers/index.js';
+import { runAgent } from '../agent/runAgent.js';
+import { buildMcpRegistry } from '../mcp/index.js';
+
+/**
+ * Resolve `env:VAR` references in a task's apiKey field.
+ * @param {string} value
+ * @param {Record<string,string>} env
+ */
+export function resolveApiKey(value, env = {}) {
+  if (typeof value === 'string' && value.startsWith('env:')) {
+    return env[value.slice(4)] ?? '';
+  }
+  return value ?? '';
+}
+
+/**
+ * Run a single evaluation task headlessly.
+ *
+ * @param {object} task
+ * @param {string} [task.provider]
+ * @param {string} [task.model]
+ * @param {string} [task.input]
+ * @param {string} [task.systemPrompt]
+ * @param {object[]} [task.tools] - explicit OpenAI-schema tool definitions
+ * @param {object[]} [task.mcpServers]
+ * @param {number} [task.maxRounds]
+ * @param {object} [opts]
+ * @param {Record<string,string>} [opts.env]
+ * @param {Function} [opts.onEvent]
+ * @param {AbortSignal} [opts.signal]
+ * @param {Function} [opts.execute] - tool executor
+ */
+export async function runTask(task, { env = {}, onEvent, signal, execute } = {}) {
+  const { provider, config } = resolveProviderRuntime({
+    providerId: task.provider,
+    apiKey: resolveApiKey(task.apiKey, env),
+    baseUrl: task.baseUrl,
+    customProviders: task.customProviders ?? [],
+  });
+
+  let tools = Array.isArray(task.tools) && task.tools[0]?.type === 'function' ? [...task.tools] : [];
+  let mcp = null;
+  if (task.mcpServers?.length) {
+    mcp = await buildMcpRegistry(task.mcpServers, { signal });
+    tools = [...tools, ...mcp.definitions];
+  }
+
+  const messages = [];
+  if (task.systemPrompt) messages.push({ role: 'system', content: task.systemPrompt });
+  messages.push({ role: 'user', content: task.input });
+
+  try {
+    const result = await runAgent({
+      provider,
+      config,
+      model: task.model,
+      messages,
+      tools,
+      execute,
+      toolContext: { ...(task.toolContext ?? {}), mcp },
+      maxRounds: task.maxRounds,
+      onEvent,
+      signal,
+    });
+
+    return { ...result, provider: provider.id, model: task.model };
+  } finally {
+    mcp?.close?.();
+  }
+}

package/harness/packages/core/src/eval/runner.js

@@ -0,0 +1,150 @@
+import { createTaskWorkspace } from './task.js';
+import { runTask } from './runTask.js';
+import { runScorers } from './scorers.js';
+import { createTools } from '../tools/index.js';
+
+function safeParse(value) {
+  if (value && typeof value === 'object') return value;
+  if (typeof value !== 'string') return {};
+  try {
+    return JSON.parse(value);
+  } catch {
+    return {};
+  }
+}
+
+export function extractToolCalls(messages = []) {
+  const calls = [];
+  for (const message of messages) {
+    if (message.role === 'assistant' && Array.isArray(message.tool_calls)) {
+      for (const call of message.tool_calls) {
+        calls.push({
+          name: call.function?.name ?? call.name,
+          args: safeParse(call.function?.arguments ?? call.arguments),
+        });
+      }
+    }
+  }
+  return calls;
+}
+
+async function pool(items, limit, worker) {
+  const results = new Array(items.length);
+  let index = 0;
+  const size = Math.max(1, Math.min(limit || 1, items.length || 1));
+  const runners = Array.from({ length: size }, async () => {
+    while (true) {
+      const i = index++;
+      if (i >= items.length) break;
+      results[i] = await worker(items[i], i);
+    }
+  });
+  await Promise.all(runners);
+  return results;
+}
+
+/**
+ * Run an evaluation dataset: tasks × models × trials, each in an isolated
+ * workspace, scored and (optionally) persisted.
+ *
+ * @param {object} opts
+ * @param {object[]} opts.tasks
+ * @param {string[]} opts.models
+ * @param {string} opts.providerId
+ * @param {string} [opts.baseUrl]
+ * @param {string} [opts.apiKey]
+ * @param {string} [opts.systemPrompt]
+ * @param {string} [opts.rootDir] - base dir for task fixtures
+ * @param {number} [opts.trials]
+ * @param {number} [opts.concurrency]
+ * @param {string[]} [opts.enabledTools]
+ * @param {object} [opts.permissions]
+ * @param {boolean} [opts.yes]
+ * @param {Function} [opts.judge]
+ * @param {object} [opts.store]
+ * @param {string} [opts.runId]
+ * @param {Function} [opts.onProgress]
+ * @param {AbortSignal} [opts.signal]
+ */
+export async function runDataset({
+  tasks,
+  models,
+  providerId,
+  baseUrl,
+  apiKey,
+  systemPrompt,
+  rootDir = process.cwd(),
+  trials = 1,
+  concurrency = 1,
+  enabledTools,
+  permissions,
+  yes = false,
+  judge,
+  store,
+  runId,
+  onProgress,
+  signal,
+}) {
+  const jobs = [];
+  for (const task of tasks) {
+    for (const model of models) {
+      for (let trial = 0; trial < trials; trial++) jobs.push({ task, model, trial });
+    }
+  }
+
+  return pool(jobs, concurrency, async ({ task, model, trial }) => {
+    const workspace = await createTaskWorkspace(task, { rootDir });
+    const tools = createTools({ root: workspace.dir, enabled: enabledTools, permissions, yes });
+    const startedAt = Date.now();
+    let result = null;
+    let error = null;
+
+    try {
+      result = await runTask(
+        {
+          ...task,
+          provider: providerId,
+          model,
+          baseUrl: task.baseUrl ?? baseUrl,
+          apiKey: task.apiKey ?? apiKey,
+          systemPrompt: task.systemPrompt || systemPrompt,
+          tools: tools.definitions.length ? tools.definitions : undefined,
+        },
+        { env: process.env, execute: tools.execute, signal }
+      );
+    } catch (err) {
+      error = err.message;
+    }
+
+    const toolCalls = extractToolCalls(result?.messages ?? []);
+    let scoring = { passed: false, score: 0, details: [] };
+    if (!error) {
+      scoring = await runScorers(task, {
+        output: result.content,
+        toolCalls,
+        workspaceDir: workspace.dir,
+        input: task.input,
+        judge,
+      });
+    }
+    await workspace.cleanup();
+
+    const entry = {
+      runId,
+      taskId: task.id ?? String(task.input ?? '').slice(0, 40),
+      model,
+      trial,
+      output: result?.content ?? null,
+      passed: Boolean(scoring.passed),
+      score: scoring.score,
+      durationMs: Date.now() - startedAt,
+      error,
+      toolCalls,
+      trace: { rounds: result?.rounds, stopReason: result?.stopReason, scorers: scoring.details, error },
+    };
+
+    if (store) store.addRunResult(entry);
+    onProgress?.(entry);
+    return entry;
+  });
+}