evaldog 0.1.0

package/README.md

@@ -0,0 +1,46 @@
+# evaldog
+
+Run LLM / prompt evals locally — **deterministic grading, zero LLM tokens.**
+
+Built for CI gates and AI agents that need a fast pass/fail on prompt/RAG outputs
+without burning context window.
+
+```bash
+npx evaldog run cases.csv
+npx evaldog run cases.csv --min 80     # exit 1 if score < 80 (CI gate)
+npx evaldog run cases.csv --json       # machine-readable (for agents)
+```
+
+## File formats
+
+**CSV** — `name,output,expected,assert`
+```csv
+name,output,expected,assert
+Password reset,Click the reset link.,reset link,contains
+JSON shape,"{""ok"":true}",,is-json
+```
+
+**JSON / YAML**
+```yaml
+cases:
+  - name: greeting
+    output: "Sure! Happy to help."
+    assert:
+      - { type: not-empty }
+```
+
+Assertions: `contains · icontains · equals · regex · is-json · not-empty`
+(default `contains` when an `expected` value is present, else `not-empty`).
+
+## Why agents like it
+
+An agent can grade 200 outputs with one shell call and read back a single number
++ exit code — instead of streaming every case through the model.
+
+```bash
+evaldog run outputs.csv --json --min 90 || echo "regression!"
+```
+
+Hosted dashboard + scheduled drift alerts: **https://evaldog.com**
+
+MIT © The Testing Academy

package/index.js

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * EvalDog CLI — run LLM/prompt evals locally with deterministic grading.
+ * Zero LLM tokens: it grades outputs you already have against simple assertions.
+ * Perfect for CI gates and AI agents that need a pass/fail without burning context.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+let yaml = null;
+let Papa = null;
+try {
+  yaml = require('js-yaml');
+} catch (_) {
+  /* optional until a yaml file is used */
+}
+try {
+  Papa = require('papaparse');
+} catch (_) {
+  /* optional until a csv file is used */
+}
+
+// ---------------------------------------------------------------- grading ----
+const ALIASES = {
+  contains: 'contains',
+  icontains: 'icontains',
+  'i-contains': 'icontains',
+  equals: 'equals',
+  equal: 'equals',
+  exact: 'equals',
+  regex: 'regex',
+  matches: 'regex',
+  'is-json': 'is-json',
+  json: 'is-json',
+  'not-empty': 'not-empty',
+  notempty: 'not-empty',
+};
+const normType = (t) => ALIASES[String(t).toLowerCase().trim()] || 'contains';
+const truncate = (s, n) => {
+  s = String(s);
+  return s.length > n ? s.slice(0, n) + '…' : s;
+};
+
+function runAssert(output, a, expected) {
+  const out = output == null ? '' : String(output);
+  const val = String(a.value != null ? a.value : expected != null ? expected : '');
+  switch (a.type) {
+    case 'equals':
+      return { ok: out.trim() === val.trim(), label: `equals "${truncate(val, 24)}"` };
+    case 'contains':
+      return { ok: out.includes(val), label: `contains "${truncate(val, 24)}"` };
+    case 'icontains':
+      return { ok: out.toLowerCase().includes(val.toLowerCase()), label: `icontains "${truncate(val, 24)}"` };
+    case 'regex':
+      try {
+        return { ok: new RegExp(val).test(out), label: `matches /${truncate(val, 20)}/` };
+      } catch (_) {
+        return { ok: false, label: 'invalid regex' };
+      }
+    case 'is-json':
+      try {
+        JSON.parse(out);
+        return { ok: true, label: 'valid JSON' };
+      } catch (_) {
+        return { ok: false, label: 'valid JSON' };
+      }
+    case 'not-empty':
+      return { ok: out.trim().length > 0, label: 'not empty' };
+    default:
+      return { ok: false, label: 'unknown assert' };
+  }
+}
+
+function grade(cases) {
+  const results = cases.map((c, i) => {
+    const asserts = c.asserts && c.asserts.length ? c.asserts : [{ type: 'not-empty' }];
+    const checks = asserts.map((a) => runAssert(c.output, a, c.expected));
+    return { name: c.name || `Case ${i + 1}`, output: c.output || '', passed: checks.every((x) => x.ok), checks };
+  });
+  const passed = results.filter((r) => r.passed).length;
+  const total = results.length;
+  return { score: total ? Math.round((passed / total) * 100) : 0, total, passed, failed: total - passed, cases: results };
+}
+
+// ---------------------------------------------------------------- parsing ----
+function parseCsv(text) {
+  if (!Papa) throw new Error('Install "papaparse" to read CSV files');
+  const { data } = Papa.parse(text, { header: true, skipEmptyLines: true });
+  return data.map((row, i) => {
+    const low = {};
+    for (const k of Object.keys(row)) low[k.toLowerCase().trim()] = row[k];
+    const output = low.output != null ? low.output : low.actual != null ? low.actual : low.response || '';
+    const expected = low.expected != null ? low.expected : low.expected_output != null ? low.expected_output : low.gold;
+    const at = low.assert ? normType(low.assert) : expected ? 'contains' : 'not-empty';
+    return { name: low.name || low.id || low.test || `Case ${i + 1}`, output, expected, asserts: [{ type: at, value: expected }] };
+  });
+}
+
+function parseStructured(obj) {
+  const arr = Array.isArray(obj) ? obj : (obj && (obj.cases || obj.tests)) || [];
+  return arr.map((t, i) => {
+    const vars = t.vars || {};
+    const output = t.output != null ? t.output : t.actual != null ? t.actual : t.response != null ? t.response : vars.output || '';
+    const expected = t.expected != null ? t.expected : t.gold;
+    let asserts = [];
+    const raw = t.assert || t.asserts;
+    if (Array.isArray(raw)) asserts = raw.filter((a) => a && a.type).map((a) => ({ type: normType(a.type), value: a.value != null ? a.value : a.expected }));
+    if (!asserts.length) asserts = [{ type: expected ? 'contains' : 'not-empty', value: expected }];
+    return { name: t.name || t.description || t.id || `Case ${i + 1}`, output, expected, asserts };
+  });
+}
+
+function parseFile(file) {
+  const text = fs.readFileSync(file, 'utf8');
+  const ext = path.extname(file).toLowerCase().replace('.', '');
+  if (ext === 'csv') return parseCsv(text);
+  if (ext === 'json') return parseStructured(JSON.parse(text));
+  if (ext === 'yaml' || ext === 'yml') {
+    if (!yaml) throw new Error('Install "js-yaml" to read YAML files');
+    return parseStructured(yaml.load(text));
+  }
+  try {
+    return parseStructured(JSON.parse(text));
+  } catch (_) {
+    /* not json */
+  }
+  if (yaml) {
+    try {
+      const y = yaml.load(text);
+      if (y && typeof y === 'object') return parseStructured(y);
+    } catch (_) {
+      /* not yaml */
+    }
+  }
+  return parseCsv(text);
+}
+
+// -------------------------------------------------------------------- cli ----
+const C = { g: '\x1b[32m', r: '\x1b[31m', y: '\x1b[33m', d: '\x1b[2m', b: '\x1b[1m', x: '\x1b[0m' };
+const color = (s, c) => (process.stdout.isTTY ? c + s + C.x : s);
+
+const args = process.argv.slice(2);
+const has = (f) => args.includes(f);
+function flagVal(name, def) {
+  const i = args.indexOf(name);
+  if (i >= 0 && args[i + 1] && !args[i + 1].startsWith('--')) return args[i + 1];
+  return def;
+}
+
+function help() {
+  console.log(`
+${color('EvalDog', C.b)} — run LLM/prompt evals locally. Deterministic. Zero tokens.
+
+Usage:
+  npx evaldog run <file>             Grade a .csv / .json / .yaml eval file
+  npx evaldog run <file> --min 80    Exit 1 if score < 80  (CI / agent gate)
+  npx evaldog run <file> --json      Machine-readable output (for AI agents)
+  npx evaldog run <file> --quiet     Print only the summary line
+
+Assertions:  contains | icontains | equals | regex | is-json | not-empty
+CSV columns: name,output,expected,assert
+Docs:        https://evaldog.com/quickstart
+`);
+}
+
+const cmd = args[0];
+if (!cmd || cmd === '-h' || cmd === '--help' || cmd === 'help') {
+  help();
+  process.exit(0);
+}
+if (cmd !== 'run') {
+  console.error('Unknown command: ' + cmd);
+  help();
+  process.exit(2);
+}
+
+const file = args[1];
+if (!file || file.startsWith('--')) {
+  console.error('Provide a file:  evaldog run <file>');
+  process.exit(2);
+}
+if (!fs.existsSync(file)) {
+  console.error('File not found: ' + file);
+  process.exit(2);
+}
+
+const min = parseInt(flagVal('--min', '0'), 10) || 0;
+let res;
+try {
+  const cases = parseFile(file);
+  if (!cases.length) {
+    console.error('No test cases found in ' + file);
+    process.exit(2);
+  }
+  res = grade(cases);
+} catch (e) {
+  console.error('Error: ' + e.message);
+  process.exit(2);
+}
+
+if (has('--json')) {
+  console.log(JSON.stringify(res, null, 2));
+  process.exit(res.score >= min ? 0 : 1);
+}
+
+if (!has('--quiet')) {
+  for (const c of res.cases) {
+    console.log(`${c.passed ? color('✓', C.g) : color('✗', C.r)} ${c.name}`);
+    if (!c.passed) for (const ck of c.checks) if (!ck.ok) console.log(color(`    ✗ ${ck.label}`, C.d));
+  }
+  console.log('');
+}
+
+const col = res.score >= 80 ? C.g : res.score >= 50 ? C.y : C.r;
+console.log(`${color(res.score + '%', col + C.b)}  ${res.passed}/${res.total} passed` + (min ? `  (gate ${min}%)` : ''));
+process.exit(res.score >= min ? 0 : 1);

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "evaldog",
+  "version": "0.1.0",
+  "description": "Run LLM/prompt evals locally — deterministic grading for CI and AI agents. Zero tokens.",
+  "bin": { "evaldog": "index.js" },
+  "type": "commonjs",
+  "files": ["index.js", "README.md"],
+  "keywords": ["llm", "eval", "evaluation", "prompt", "testing", "rag", "ci", "ai-agent", "llmops"],
+  "homepage": "https://evaldog.com",
+  "license": "MIT",
+  "author": "The Testing Academy",
+  "engines": { "node": ">=18" },
+  "dependencies": {
+    "js-yaml": "^4.1.0",
+    "papaparse": "^5.5.2"
+  }
+}