@kodevibe/harness 0.11.0 → 0.11.2

package/src/llm-bench.js

@@ -0,0 +1,323 @@
+'use strict';
+
+const crypto = require('crypto');
+
+const DEFAULT_THRESHOLDS = {
+  overall: 0.7,
+  decompositionPrecision: 0.6,
+  passAt1: 0.6,
+  instructionCompliance: 0.8,
+};
+
+const DEFAULT_WEIGHTS = {
+  decompositionPrecision: 0.4,
+  passAt1: 0.4,
+  instructionCompliance: 0.2,
+};
+
+const DEFAULT_REQUIREMENTS = {
+  minModels: 1,
+  minRunsPerModel: 1,
+  requiredScenarios: [],
+  allowExampleIds: true,
+  requireRunSeals: false,
+  requireScenarioManifest: false,
+  requireScenarioPromptHash: false,
+};
+
+function clamp01(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+function normalizeText(value) {
+  return String(value || '')
+    .toLowerCase()
+    .replace(/[`*_()[\]{}:;,.!?]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function taskText(task) {
+  if (typeof task === 'string') return task;
+  if (task && typeof task === 'object') return task.id || task.title || task.text || task.name || '';
+  return '';
+}
+
+function sha256Hex(value) {
+  return crypto.createHash('sha256').update(String(value || ''), 'utf8').digest('hex');
+}
+
+function scenarioPromptHash(scenario = {}) {
+  if (scenario.promptHash) return String(scenario.promptHash);
+  if (typeof scenario.prompt === 'string' && scenario.prompt.length > 0) return sha256Hex(scenario.prompt);
+  return '';
+}
+
+function normalizeScenarioManifest(manifest = []) {
+  const scenarios = Array.isArray(manifest)
+    ? manifest
+    : manifest.scenarios || manifest.requiredScenarios || [];
+
+  return scenarios
+    .filter((scenario) => scenario && (scenario.id || scenario.scenario))
+    .map((scenario) => ({
+      id: scenario.id || scenario.scenario,
+      title: scenario.title || scenario.name || '',
+      prompt: scenario.prompt || '',
+      promptHash: scenarioPromptHash(scenario),
+      expectedTasks: Array.isArray(scenario.expectedTasks) ? scenario.expectedTasks : [],
+      requiredInstructionChecks: Array.isArray(scenario.requiredInstructionChecks)
+        ? scenario.requiredInstructionChecks
+        : Array.isArray(scenario.instructionChecks)
+          ? scenario.instructionChecks.map((check) => (typeof check === 'string' ? check : check.id)).filter(Boolean)
+          : [],
+    }));
+}
+
+function sealRun({
+  scenario,
+  prompt,
+  output,
+  capturedAt = new Date().toISOString(),
+  expectedTasks = [],
+  actualTasks = [],
+  passed = false,
+  instructionChecks = [],
+} = {}) {
+  return {
+    scenario,
+    capturedAt,
+    promptHash: sha256Hex(prompt),
+    outputHash: sha256Hex(output),
+    expectedTasks,
+    actualTasks,
+    passed,
+    instructionChecks,
+  };
+}
+
+function scoreDecomposition(expectedTasks = [], actualTasks = []) {
+  const expected = expectedTasks.map(taskText).map(normalizeText).filter(Boolean);
+  const actual = actualTasks.map(taskText).map(normalizeText).filter(Boolean);
+  if (expected.length === 0) return 1;
+
+  let matched = 0;
+  for (const exp of expected) {
+    if (actual.some((act) => act === exp || act.includes(exp) || exp.includes(act))) matched++;
+  }
+  return clamp01(matched / expected.length);
+}
+
+function scoreInstructionChecks(checks = []) {
+  if (!Array.isArray(checks) || checks.length === 0) return 1;
+  const passed = checks.filter((check) => check && check.passed === true).length;
+  return clamp01(passed / checks.length);
+}
+
+function weightedOverall(metrics, weights = DEFAULT_WEIGHTS) {
+  const weightSum = Object.values(weights).reduce((sum, weight) => sum + weight, 0) || 1;
+  const score = Object.entries(weights).reduce((sum, [key, weight]) => {
+    return sum + clamp01(metrics[key]) * weight;
+  }, 0);
+  return clamp01(score / weightSum);
+}
+
+function scoreRun(run = {}, weights = DEFAULT_WEIGHTS) {
+  const metrics = {
+    decompositionPrecision: scoreDecomposition(run.expectedTasks || run.requiredTasks || [], run.actualTasks || run.producedTasks || []),
+    passAt1: run.passed === true ? 1 : 0,
+    instructionCompliance: scoreInstructionChecks(run.instructionChecks || run.checks || []),
+  };
+  return {
+    scenario: run.scenario || run.id || '(unnamed)',
+    metrics,
+    overall: weightedOverall(metrics, weights),
+  };
+}
+
+function average(values) {
+  if (!values.length) return 0;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function scoreModel(model = {}, weights = DEFAULT_WEIGHTS) {
+  const runs = Array.isArray(model.runs) ? model.runs.map((run) => scoreRun(run, weights)) : [];
+  const metrics = {
+    decompositionPrecision: average(runs.map((run) => run.metrics.decompositionPrecision)),
+    passAt1: average(runs.map((run) => run.metrics.passAt1)),
+    instructionCompliance: average(runs.map((run) => run.metrics.instructionCompliance)),
+  };
+  return {
+    id: model.id || model.name || '(unnamed-model)',
+    tier: model.tier || 'unknown',
+    runs,
+    metrics,
+    overall: weightedOverall(metrics, weights),
+  };
+}
+
+function validateThresholds(summary, thresholds = DEFAULT_THRESHOLDS) {
+  const violations = [];
+  for (const model of summary.models) {
+    if (model.overall < thresholds.overall) {
+      violations.push(`${model.id}: overall ${model.overall.toFixed(2)} < ${thresholds.overall}`);
+    }
+    for (const key of ['decompositionPrecision', 'passAt1', 'instructionCompliance']) {
+      const min = thresholds[key];
+      if (typeof min === 'number' && model.metrics[key] < min) {
+        violations.push(`${model.id}: ${key} ${model.metrics[key].toFixed(2)} < ${min}`);
+      }
+    }
+  }
+  return violations;
+}
+
+function validateRequirements(input = {}, summary = {}, requirements = DEFAULT_REQUIREMENTS) {
+  const violations = [];
+  const models = Array.isArray(input.models) ? input.models : [];
+  const scenarioSpecs = normalizeScenarioManifest(requirements.scenarios || requirements.scenarioManifest || []);
+  const scenarioSpecMap = new Map(scenarioSpecs.map((scenario) => [scenario.id, scenario]));
+
+  if (models.length < requirements.minModels) {
+    violations.push(`bench requires at least ${requirements.minModels} model(s), found ${models.length}`);
+  }
+
+  const requiredScenarios = requirements.requiredScenarios || [];
+  if (requirements.requireScenarioManifest) {
+    for (const scenario of requiredScenarios) {
+      if (!scenarioSpecMap.has(scenario)) {
+        violations.push(`bench scenario manifest missing required scenario "${scenario}"`);
+      }
+    }
+  }
+
+  for (const model of models) {
+    const id = model.id || model.name || '(unnamed-model)';
+    const runs = Array.isArray(model.runs) ? model.runs : [];
+    if (runs.length < requirements.minRunsPerModel) {
+      violations.push(`${id}: bench requires at least ${requirements.minRunsPerModel} run(s), found ${runs.length}`);
+    }
+    if (!requirements.allowExampleIds && /\bexample|sample|placeholder|dummy\b/i.test(id)) {
+      violations.push(`${id}: example/sample model id is not valid release evidence`);
+    }
+    const scenarioSet = new Set(runs.map((run) => run.scenario || run.id).filter(Boolean));
+    for (const scenario of requiredScenarios) {
+      if (!scenarioSet.has(scenario)) {
+        violations.push(`${id}: missing required scenario "${scenario}"`);
+      }
+    }
+    if (requirements.requireRunSeals) {
+      for (const run of runs) {
+        const scenario = run.scenario || run.id || '(unnamed-scenario)';
+        if (!/^\d{4}-\d{2}-\d{2}T/.test(String(run.capturedAt || ''))) {
+          violations.push(`${id}/${scenario}: missing capturedAt ISO timestamp`);
+        }
+        if (!/^[a-f0-9]{8,}$/i.test(String(run.promptHash || ''))) {
+          violations.push(`${id}/${scenario}: missing promptHash`);
+        }
+        if (!/^[a-f0-9]{8,}$/i.test(String(run.outputHash || run.transcriptHash || ''))) {
+          violations.push(`${id}/${scenario}: missing outputHash or transcriptHash`);
+        }
+      }
+    }
+    if (requirements.requireScenarioManifest) {
+      for (const run of runs) {
+        const scenario = run.scenario || run.id || '(unnamed-scenario)';
+        const spec = scenarioSpecMap.get(scenario);
+        if (!spec) {
+          violations.push(`${id}/${scenario}: scenario is not listed in the bench manifest`);
+          continue;
+        }
+
+        for (const expectedTask of spec.expectedTasks) {
+          if (scoreDecomposition([expectedTask], run.expectedTasks || run.requiredTasks || []) < 1) {
+            violations.push(`${id}/${scenario}: expectedTasks missing manifest task "${expectedTask}"`);
+          }
+        }
+
+        const checks = run.instructionChecks || run.checks || [];
+        const checkMap = new Map(checks
+          .filter((check) => check && check.id)
+          .map((check) => [check.id, check]));
+        for (const checkId of spec.requiredInstructionChecks) {
+          const check = checkMap.get(checkId);
+          if (!check) {
+            violations.push(`${id}/${scenario}: instructionChecks missing manifest check "${checkId}"`);
+          } else if (check.passed !== true) {
+            violations.push(`${id}/${scenario}: manifest check "${checkId}" did not pass`);
+          }
+        }
+
+        if (requirements.requireScenarioPromptHash) {
+          const runPromptHash = String(run.promptHash || '');
+          if (!spec.promptHash) {
+            violations.push(`${id}/${scenario}: manifest promptHash is missing`);
+          } else if (!runPromptHash) {
+            if (!requirements.requireRunSeals) {
+              violations.push(`${id}/${scenario}: missing promptHash for manifest comparison`);
+            }
+          } else if (runPromptHash !== spec.promptHash) {
+            violations.push(`${id}/${scenario}: promptHash does not match manifest prompt`);
+          }
+        }
+      }
+    }
+  }
+
+  if (!summary.models || summary.models.length === 0) {
+    violations.push('bench has no scored model results');
+  }
+  return violations;
+}
+
+function scoreBench(input = {}, options = {}) {
+  const weights = { ...DEFAULT_WEIGHTS, ...(input.weights || {}) };
+  const thresholds = { ...DEFAULT_THRESHOLDS, ...(input.thresholds || {}) };
+  const requirements = {
+    ...DEFAULT_REQUIREMENTS,
+    ...(input.requirements || {}),
+    ...(options.requirements || {}),
+  };
+  const models = Array.isArray(input.models) ? input.models.map((model) => scoreModel(model, weights)) : [];
+  const metrics = {
+    decompositionPrecision: average(models.map((model) => model.metrics.decompositionPrecision)),
+    passAt1: average(models.map((model) => model.metrics.passAt1)),
+    instructionCompliance: average(models.map((model) => model.metrics.instructionCompliance)),
+  };
+  const summary = {
+    suite: input.suite || 'kode:harness-r10',
+    models,
+    metrics,
+    overall: weightedOverall(metrics, weights),
+    thresholds,
+    requirements,
+  };
+  const violations = [
+    ...validateThresholds(summary, thresholds),
+    ...validateRequirements(input, summary, requirements),
+  ];
+  return {
+    ...summary,
+    ok: violations.length === 0,
+    violations,
+  };
+}
+
+module.exports = {
+  DEFAULT_THRESHOLDS,
+  DEFAULT_WEIGHTS,
+  DEFAULT_REQUIREMENTS,
+  sha256Hex,
+  scenarioPromptHash,
+  normalizeScenarioManifest,
+  sealRun,
+  normalizeText,
+  scoreDecomposition,
+  scoreInstructionChecks,
+  scoreRun,
+  scoreModel,
+  scoreBench,
+  validateThresholds,
+  validateRequirements,
+};

package/src/pack-check.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { runGuard } = require('./guard');
+
+function parsePackJson(input) {
+  const parsed = typeof input === 'string' ? JSON.parse(input) : input;
+  const pack = Array.isArray(parsed) ? parsed[0] : parsed;
+  if (!pack || !Array.isArray(pack.files)) {
+    throw new Error('npm pack JSON must include a files array');
+  }
+  return pack;
+}
+
+function normalizePackPath(filePath) {
+  return String(filePath || '')
+    .replace(/\\/g, '/')
+    .replace(/^package\//, '')
+    .replace(/^\.?\//, '');
+}
+
+function packFilePaths(input) {
+  const pack = parsePackJson(input);
+  return pack.files
+    .map((file) => normalizePackPath(file.path))
+    .filter(Boolean);
+}
+
+function checkPublicPack({ packJson, cwd = process.cwd() }) {
+  const files = packFilePaths(packJson)
+    .filter((file) => {
+      const abs = path.join(cwd, file);
+      return fs.existsSync(abs) && fs.statSync(abs).isFile();
+    });
+  return {
+    files,
+    ...runGuard({ files, cwd }),
+  };
+}
+
+module.exports = {
+  parsePackJson,
+  normalizePackPath,
+  packFilePaths,
+  checkPublicPack,
+};