chill-agent 1.4.0 → 1.6.0

package/benchmarks/arms/baseline.js

@@ -0,0 +1,2 @@
+// Baseline arm: no skill, just the task.
+module.exports = ({ vars }) => [{ role: 'user', content: vars.task }];

package/benchmarks/arms/caveman-SKILL.md

@@ -0,0 +1,67 @@
+---
+name: caveman
+description: >
+  Ultra-compressed communication mode. Cuts token usage ~75% by speaking like caveman
+  while keeping full technical accuracy. Supports intensity levels: lite, full (default), ultra,
+  wenyan-lite, wenyan-full, wenyan-ultra.
+  Use when user says "caveman mode", "talk like caveman", "use caveman", "less tokens",
+  "be brief", or invokes /caveman. Also auto-triggers when token efficiency is requested.
+---
+
+Respond terse like smart caveman. All technical substance stay. Only fluff die.
+
+## Persistence
+
+ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. Off only: "stop caveman" / "normal mode".
+
+Default: **full**. Switch: `/caveman lite|full|ultra`.
+
+## Rules
+
+Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). Technical terms exact. Code blocks unchanged. Errors quoted exact.
+
+Pattern: `[thing] [action] [reason]. [next step].`
+
+Not: "Sure! I'd be happy to help you with that. The issue you're experiencing is likely caused by..."
+Yes: "Bug in auth middleware. Token expiry check use `<` not `<=`. Fix:"
+
+## Intensity
+
+| Level | What change |
+|-------|------------|
+| **lite** | No filler/hedging. Keep articles + full sentences. Professional but tight |
+| **full** | Drop articles, fragments OK, short synonyms. Classic caveman |
+| **ultra** | Abbreviate (DB/auth/config/req/res/fn/impl), strip conjunctions, arrows for causality (X → Y), one word when one word enough |
+| **wenyan-lite** | Semi-classical. Drop filler/hedging but keep grammar structure, classical register |
+| **wenyan-full** | Maximum classical terseness. Fully 文言文. 80-90% character reduction. Classical sentence patterns, verbs precede objects, subjects often omitted, classical particles (之/乃/為/其) |
+| **wenyan-ultra** | Extreme abbreviation while keeping classical Chinese feel. Maximum compression, ultra terse |
+
+Example — "Why React component re-render?"
+- lite: "Your component re-renders because you create a new object reference each render. Wrap it in `useMemo`."
+- full: "New object ref each render. Inline object prop = new ref = re-render. Wrap in `useMemo`."
+- ultra: "Inline obj prop → new ref → re-render. `useMemo`."
+- wenyan-lite: "組件頻重繪，以每繪新生對象參照故。以 useMemo 包之。"
+- wenyan-full: "物出新參照，致重繪。useMemo .Wrap之。"
+- wenyan-ultra: "新參照→重繪。useMemo Wrap。"
+
+Example — "Explain database connection pooling."
+- lite: "Connection pooling reuses open connections instead of creating new ones per request. Avoids repeated handshake overhead."
+- full: "Pool reuse open DB connections. No new connection per request. Skip handshake overhead."
+- ultra: "Pool = reuse DB conn. Skip handshake → fast under load."
+- wenyan-full: "池reuse open connection。不每req新開。skip handshake overhead。"
+- wenyan-ultra: "池reuse conn。skip handshake → fast。"
+
+## Auto-Clarity
+
+Drop caveman for: security warnings, irreversible action confirmations, multi-step sequences where fragment order risks misread, user asks to clarify or repeats question. Resume caveman after clear part done.
+
+Example — destructive op:
+> **Warning:** This will permanently delete all rows in the `users` table and cannot be undone.
+> ```sql
+> DROP TABLE users;
+> ```
+> Caveman resume. Verify backup exist first.
+
+## Boundaries
+
+Code/commits/PRs: write normal. "stop caveman" or "normal mode": revert. Level persist until changed or session end.

package/benchmarks/arms/caveman.js

@@ -0,0 +1,8 @@
+// Caveman arm: caveman SKILL.md (full) as the system prompt.
+const fs = require('fs');
+const path = require('path');
+const system = fs.readFileSync(path.join(__dirname, 'caveman-SKILL.md'), 'utf8');
+module.exports = ({ vars }) => [
+  { role: 'system', content: system },
+  { role: 'user', content: vars.task },
+];

package/benchmarks/arms/ponytail.js

@@ -0,0 +1,8 @@
+// Ponytail arm: the repo's own SKILL.md (full) as the system prompt. Single source of truth.
+const fs = require('fs');
+const path = require('path');
+const system = fs.readFileSync(path.join(__dirname, '..', '..', 'skills', 'ponytail', 'SKILL.md'), 'utf8');
+module.exports = ({ vars }) => [
+  { role: 'system', content: system },
+  { role: 'user', content: vars.task },
+];

package/benchmarks/behavior.js

@@ -0,0 +1,54 @@
+// Behavior gate: does the ponytail ruleset actually PRODUCE its refined
+// behaviors, not just carry the text? One check per probe (vars.probe), each
+// targeting a rule that a field review (rcstack, phases 0-8) showed mattered:
+//   hardware     - "hardware is never the spec ideal, leave the calibration knob"
+//   explanation  - "explanation the user explicitly asked for is not debt"
+//   onecheck     - "lazy code without its check is unfinished"
+//
+// Heuristic graders, same spirit as loc.js / correctness.js. The graders
+// themselves are proven by tests/behavior.test.js (RED/GREEN, no API key).
+//
+// Metric: `behavior` (1 = behavior present, 0 = absent).
+
+function proseOf(text) {
+  return String(text || '').replace(/```[\s\S]*?```/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+const CHECKS = {
+  // Treats the device as non-ideal: leaves a tunable knob or flags per-unit drift.
+  // A passing mention of "calibration" is not enough; it must be actionable.
+  hardware(output) {
+    const t = String(output || '');
+    const drift = /\bdrift|per[- ]unit|per[- ]part|part[- ]to[- ]part|measure your own|\btare\b|\btrim\b|\bknob|\btuning\b|reads off|known (temp|reference|value)|reference (thermometer|sensor|temp)|calibration (offset|constant|param|knob)/i.test(t);
+    return drift
+      ? { pass: true, reason: 'Leaves a calibration knob / flags per-unit drift.' }
+      : { pass: false, reason: 'Treats the hardware as ideal; no calibration knob.' };
+  },
+
+  // Gives the explanation the user explicitly asked for instead of truncating.
+  explanation(output) {
+    const p = proseOf(output);
+    const words = p ? p.split(' ').length : 0;
+    const structured = /(\d+[.)]\s|[-*]\s)/.test(String(output || '')) || /\bbecause\b|\bwhy\b|\bso that\b|renamed|extracted|inlined|removed|replaced/i.test(p);
+    return words >= 45 && structured
+      ? { pass: true, reason: `Gave the requested write-up (${words} words of prose).` }
+      : { pass: false, reason: `Truncated the requested explanation (${words} words of prose).` };
+  },
+
+  // Leaves ONE runnable check behind for non-trivial logic.
+  onecheck(output) {
+    const t = String(output || '');
+    const hasCheck = /\bassert\b|def\s+test_|if\s+__name__|unittest|pytest|console\.assert|\bexpect\(|\bdescribe\(|\bit\(/.test(t);
+    return hasCheck
+      ? { pass: true, reason: 'Left a runnable check (assert/test/demo).' }
+      : { pass: false, reason: 'No runnable check left behind.' };
+  },
+};
+
+module.exports = (output, context) => {
+  const probe = context && context.vars && context.vars.probe;
+  const check = CHECKS[probe];
+  if (!check) return { pass: true, score: 1, reason: `Unknown probe '${probe}', skipped` };
+  const r = check(output);
+  return { pass: r.pass, score: r.pass ? 1 : 0, reason: r.reason };
+};

package/benchmarks/behavior.yaml

@@ -0,0 +1,40 @@
+# Ponytail behavior gates: does the ruleset actually produce its refined
+# behaviors (not just carry the text)? Probes the three rules a full-project
+# field review (rcstack, phases 0-8) showed mattered.
+#
+#   Run:    npx promptfoo@latest eval -c benchmarks/behavior.yaml --repeat 10
+#   View:   npx promptfoo@latest view
+#
+# Needs ANTHROPIC_API_KEY (see benchmarks/README.md). The grader (behavior.js)
+# is proven separately by tests/behavior.test.js, which needs no API key.
+#
+# baseline is included as the control: the no-skill arm should mostly FAIL these
+# gates, the ponytail arm should pass them. That delta is the point.
+description: "Ponytail behavior gates: hardware calibration, requested explanation, one runnable check."
+
+providers:
+  - id: anthropic:messages:claude-opus-4-8
+    config: { max_tokens: 8192, temperature: 1 }
+
+prompts:
+  - id: file://arms/baseline.js
+    label: baseline (no skill)
+  - id: file://arms/ponytail.js
+    label: ponytail
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: file://behavior.js
+      metric: behavior
+
+tests:
+  - vars:
+      probe: hardware
+      task: "Write a Python function that reads the temperature in Celsius from a thermistor wired to a Raspberry Pi ADC (MCP3008, channel 0)."
+  - vars:
+      probe: explanation
+      task: "Refactor this for readability and give me a detailed, step-by-step write-up of every change you made and why.\n\ndef p(d):\n    r = []\n    for x in d:\n        if x.get('a') and x['a'] > 0:\n            r.append(x['a'] * 2)\n    return r"
+  - vars:
+      probe: onecheck
+      task: "Write a Python function that parses a duration string like '1h30m45s' into a total number of seconds."

package/benchmarks/correctness.js

@@ -0,0 +1,291 @@
+// Functional correctness assertion: runs generated code against lightweight test
+// cases per task. Proves "less code" is not "broken code". Spawns python/node
+// with the extracted code + appended assertions; returns pass/fail + score.
+//
+// Metric: `correct` (1 = all checks pass, 0 = at least one fails).
+// Unlike loc.js (measurement-only), this one is a gate — a wrong answer is a
+// wrong answer regardless of how few lines produced it.
+//
+// Note: email, debounce, and CSV checks execute the generated code against
+// real inputs. Countdown (React) and rate-limit (FastAPI) checks are
+// structural-only: they verify plausible code shape rather than runtime
+// behaviour, because a React bundler or a live FastAPI server is needed to
+// execute those properly.
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+function correctnessTimeoutMs() {
+  const value = Number.parseInt(process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS || '', 10);
+  return Number.isFinite(value) && value > 0 ? value : 30_000;
+}
+
+// Extract fenced code blocks, tagged by language.
+function extractBlocks(text) {
+  text = String(text || '');
+  const matches = [...text.matchAll(/```(\w*)\r?\n([\s\S]*?)```/g)];
+  // ponytail: terse models often answer with bare, unfenced code. Treat the whole
+  // response as one block so the gate scores the code instead of reporting "no block".
+  if (matches.length === 0 && text.trim()) return [{ lang: '', code: text }];
+  return matches.map((m) => ({ lang: (m[1] || '').toLowerCase(), code: m[2] }));
+}
+
+// Identify which task we're evaluating from vars.task.
+function identifyTask(task) {
+  const t = task.toLowerCase();
+  if (t.includes('email') && t.includes('valid')) return 'email';
+  if (t.includes('debounce')) return 'debounce';
+  if (t.includes('csv') && t.includes('sum')) return 'csv';
+  if (t.includes('countdown') && t.includes('react')) return 'countdown';
+  if (t.includes('rate limit') || t.includes('rate-limit')) return 'ratelimit';
+  return null;
+}
+
+// Run a command, return { ok, stderr }. Harnesses that print FAIL to stdout
+// still get their reason through (#919).
+function exec(cmd, opts = {}) {
+  try {
+    execSync(cmd, { timeout: correctnessTimeoutMs(), encoding: 'utf8', stdio: 'pipe', ...opts });
+    return { ok: true, stderr: '' };
+  } catch (e) {
+    return { ok: false, stderr: (e.stderr || e.stdout || e.message || '').slice(0, 500) };
+  }
+}
+
+// ponytail: probe once at load; macOS and many Linux images ship python3 only.
+// Prefer one with pandas (the csv task needs it): on Windows python3 can be a
+// bundled Python without it while python has it (#919).
+let pythonCmd;
+function python() {
+  if (pythonCmd) return pythonCmd;
+  const candidates = ['python3', 'python'];
+  pythonCmd = candidates.find((cmd) => exec(`${cmd} -c "import pandas"`).ok)
+    || candidates.find((cmd) => exec(`${cmd} -c "import sys"`).ok)
+    || 'python3';
+  return pythonCmd;
+}
+
+// Write content to a temp file, return the path.
+function tmpFile(ext, content) {
+  const p = path.join(os.tmpdir(), `ponytail-bench-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`);
+  fs.writeFileSync(p, content);
+  return p;
+}
+
+// --- Per-task test harnesses ---
+
+const CHECKS = {
+  email(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('def ')));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Append assertions that call the generated function by common names.
+    const harness = `
+${code.code}
+
+# Find the validator function
+import sys
+fn = None
+for name in ['validate_email', 'is_valid_email', 'email_validator', 'is_valid', 'validate']:
+    if name in dir() and callable(eval(name)):
+        fn = eval(name)
+        break
+
+if fn is None:
+    # Try any function that takes one arg
+    import inspect
+    for name, obj in list(globals().items()):
+        if callable(obj) and not name.startswith('_'):
+            try:
+                sig = inspect.signature(obj)
+                if len(sig.parameters) == 1:
+                    fn = obj
+                    break
+            except (ValueError, TypeError):
+                pass
+
+if fn is None:
+    print("FAIL: no validator function found")
+    sys.exit(1)
+
+# Test cases
+failures = []
+if not fn("user@example.com"):
+    failures.append("rejected valid: user@example.com")
+if not fn("a@b.co"):
+    failures.append("rejected valid: a@b.co")
+if fn("no-at-sign"):
+    failures.append("accepted invalid: no-at-sign")
+if fn(""):
+    failures.append("accepted invalid: empty string")
+if fn("@missing-local.com"):
+    failures.append("accepted invalid: @missing-local.com")
+
+if failures:
+    print("FAIL: " + "; ".join(failures))
+    sys.exit(1)
+print("PASS")
+`;
+    const f = tmpFile('.py', harness);
+    const result = exec(`${python()} "${f}"`);
+    fs.unlinkSync(f);
+    if (result.ok) return { pass: true, reason: 'Email validator passes all checks' };
+    return { pass: false, reason: result.stderr || 'Email validator failed' };
+  },
+
+  debounce(blocks) {
+    const code = blocks.find((b) => b.lang === 'javascript' || b.lang === 'js' || (!b.lang && (b.code.includes('function') || b.code.includes('=>'))));
+    if (!code) return { pass: false, reason: 'No JavaScript code block found' };
+
+    const harness = `
+${code.code}
+
+// Find the debounce function
+const fn = typeof debounce === 'function' ? debounce
+  : typeof module !== 'undefined' && typeof module.exports === 'function' ? module.exports
+  : null;
+
+if (!fn) {
+  console.error("FAIL: no debounce function found");
+  process.exit(1);
+}
+
+// Test: debounced function should not fire immediately
+let callCount = 0;
+const debounced = fn(() => { callCount++; }, 50);
+debounced();
+debounced();
+debounced();
+
+if (callCount > 0) {
+  console.error("FAIL: debounce fired immediately (should wait)");
+  process.exit(1);
+}
+
+// Test: should fire after the delay
+setTimeout(() => {
+  if (callCount !== 1) {
+    console.error("FAIL: expected 1 call after delay, got " + callCount);
+    process.exit(1);
+  }
+  console.log("PASS");
+}, 120);
+`;
+    const f = tmpFile('.mjs', harness);
+    const result = exec(`node "${f}"`);
+    fs.unlinkSync(f);
+    if (result.ok) return { pass: true, reason: 'Debounce passes all checks' };
+    return { pass: false, reason: result.stderr || 'Debounce failed' };
+  },
+
+  csv(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && b.code.includes('csv') && b.code.includes('sum')));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Create a test CSV and wrap the generated code so it reads it.
+    const csvContent = 'name,amount\nAlice,100.5\nBob,200.0\nCharlie,50.5\n';
+    const csvPath = tmpFile('.csv', csvContent).replace(/\\/g, '/');
+
+    // The generated code likely reads 'sales.csv'; patch the filename.
+    let patched = code.code.replace(/['"]sales\.csv['"]/g, `'${csvPath}'`);
+    // Also try open() calls
+    patched = patched.replace(/open\(\s*['"]sales\.csv['"]/g, `open('${csvPath}'`);
+
+    const harness = `
+import sys, os
+os.chdir(r"${path.dirname(csvPath)}")
+
+# Capture print output
+import io
+_stdout = sys.stdout
+sys.stdout = io.StringIO()
+
+try:
+${patched.split('\n').map((l) => '    ' + l).join('\n')}
+except Exception:
+    sys.stdout = _stdout
+    raise
+
+output = sys.stdout.getvalue()
+sys.stdout = _stdout
+
+# Check output contains the number 351 (100.5 + 200.0 + 50.5)
+# Match as a standalone number (not as substring of e.g. 13510)
+import re
+if re.search(r'(?<![\\d])351(?:\\.0)?(?![\\d])', output):
+    print("PASS")
+else:
+    # Try running it differently: maybe it defines a function
+    print("FAIL: output was: " + repr(output[:200]))
+    sys.exit(1)
+`;
+    const f = tmpFile('.py', harness);
+    const result = exec(`${python()} "${f}"`);
+    try { fs.unlinkSync(f); } catch (e) {}
+    try { fs.unlinkSync(csvPath); } catch (e) {}
+    if (result.ok) return { pass: true, reason: 'CSV sum produces correct result (351)' };
+    return { pass: false, reason: result.stderr || 'CSV sum failed' };
+  },
+
+  countdown(blocks) {
+    // React components can't run in bare Node without a bundler. Structural check:
+    // the code must contain timer/countdown logic (useState/useEffect/setInterval/setTimeout).
+    const code = blocks.find((b) => b.code.includes('ount') || b.code.includes('timer') || b.code.includes('Timer'));
+    if (!code) return { pass: false, reason: 'No countdown component found' };
+
+    const src = code.code;
+    const hasState = /useState|useReducer|this\.state/.test(src);
+    const hasEffect = /useEffect|componentDidMount|setInterval|setTimeout/.test(src);
+    const hasDecrement = /- 1|-= 1|prev - 1|count - 1|seconds - 1|time - 1/.test(src);
+
+    const failures = [];
+    if (!hasState) failures.push('no state management (useState/useReducer)');
+    if (!hasEffect) failures.push('no timer setup (useEffect/setInterval/setTimeout)');
+    if (!hasDecrement) failures.push('no countdown decrement logic');
+
+    if (failures.length === 0) return { pass: true, reason: 'Countdown has required structure' };
+    return { pass: false, reason: 'Missing: ' + failures.join(', ') };
+  },
+
+  ratelimit(blocks) {
+    const code = blocks.find((b) => b.lang === 'python' || b.lang === 'py' || (!b.lang && (b.code.includes('rate') || b.code.includes('limit'))));
+    if (!code) return { pass: false, reason: 'No Python code block found' };
+
+    // Structural check for rate limiting: must have some form of counter/time tracking.
+    const src = code.code;
+    const hasTimeTracking = /time\.|datetime|asyncio/.test(src);
+    const hasLimitLogic = /limit|max_requests|rate|429|Too Many|HTTPException|RateLimiter/.test(src);
+    const hasFastAPI = /fastapi|FastAPI|app\s*=|@app\./.test(src);
+
+    const failures = [];
+    if (!hasLimitLogic) failures.push('no rate limit logic');
+    if (!hasFastAPI) failures.push('no FastAPI usage');
+
+    if (failures.length === 0) return { pass: true, reason: 'Rate limiter has required structure' };
+    return { pass: false, reason: 'Missing: ' + failures.join(', ') };
+  },
+};
+
+// --- Main assertion entry point ---
+
+module.exports = (output, context) => {
+  const task = identifyTask(context.vars.task || '');
+  if (!task) {
+    return { pass: true, score: 1, reason: 'Unknown task, skipped correctness check' };
+  }
+
+  const blocks = extractBlocks(String(output || ''));
+  if (blocks.length === 0) {
+    return { pass: false, score: 0, reason: 'No code blocks in output' };
+  }
+
+  const check = CHECKS[task];
+  const result = check(blocks);
+  return {
+    pass: result.pass,
+    score: result.pass ? 1 : 0,
+    reason: result.reason,
+  };
+};

package/benchmarks/loc.js

@@ -1,21 +1,19 @@
-#!/usr/bin/env node
-// benchmarks/loc.js - Lines of Code & Diff Length Efficiency Calculator
-import fs from 'node:fs';
-import path from 'node:path';
-
-export function countLoc(code) {
-  const lines = code.split('\n')
-    .map(l => l.trim())
-    .filter(l => l.length > 0 && !l.startsWith('//') && !l.startsWith('/*') && !l.startsWith('*'));
-  return lines.length;
+// Deterministic code-size metric: non-blank, non-comment lines of code. Counts
+// fenced blocks, or the whole response when the model emitted bare code unfenced.
+// Recorded as the `code_loc` metric per arm (always passes; it is a measurement, not a gate).
+function loc(output) {
+  const text = String(output || '');
+  const blocks = [...text.matchAll(/```[a-zA-Z0-9_+-]*\r?\n([\s\S]*?)```/g)].map((m) => m[1]);
+  // Drop /* ... */ block comments before counting; the line filter below only
+  // caught `*`-aligned JSDoc, so plain block comments were miscounted as code.
+  const code = (blocks.length ? blocks.join('\n') : text).replace(/\/\*[\s\S]*?\*\//g, '');
+  const count = code
+    .split('\n')
+    .map((l) => l.trim())
+    .filter((l) => l && !l.startsWith('//') && !l.startsWith('#') && l !== '*/' && !l.startsWith('/*') && !l.startsWith('*')).length;
+  return { pass: true, score: count, reason: count + ' code LOC' };
 }
 
-if (process.argv[2]) {
-  const file = path.resolve(process.cwd(), process.argv[2]);
-  if (fs.existsSync(file)) {
-    const raw = fs.readFileSync(file, 'utf8');
-    console.log(`File: ${file}`);
-    console.log(`Raw Lines: ${raw.split('\n').length}`);
-    console.log(`Net Logic LOC: ${countLoc(raw)}`);
-  }
-}
+loc.loc = loc;
+loc.countLoc = loc;
+module.exports = loc;

package/benchmarks/promptfooconfig.gemini.yaml

@@ -0,0 +1,32 @@
+# Ponytail vs baseline, latest Gemini: gemini-3.5-flash (mini) + gemini-3.1-pro-preview (top).
+#   npx promptfoo@latest eval -c benchmarks/promptfooconfig.gemini.yaml --env-file .env --repeat 30
+# Needs GOOGLE_API_KEY in .env (AI Studio).
+description: "Ponytail vs baseline, latest Gemini (3.5-flash, 3.1-pro). LOC + correctness, cost telemetry."
+
+providers:
+  - id: google:gemini-3.5-flash
+    config: { temperature: 1, maxOutputTokens: 8192 }
+  - id: google:gemini-3.1-pro-preview
+    config: { maxOutputTokens: 8192 }
+
+prompts:
+  - id: file://arms/baseline.js
+    label: baseline (no skill)
+  - id: file://arms/ponytail.js
+    label: ponytail
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: file://loc.js
+      metric: code_loc
+    - type: javascript
+      value: file://correctness.js
+      metric: correct
+
+tests:
+  - vars: { task: "Write me a Python function that validates email addresses." }
+  - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
+  - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
+  - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
+  - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }

package/benchmarks/promptfooconfig.gpt-newest.yaml

@@ -0,0 +1,33 @@
+# Ponytail vs baseline, newest OpenAI: gpt-5.5 (top) + gpt-4.1-mini, gpt-5.4-mini.
+#   npx promptfoo@latest eval -c benchmarks/promptfooconfig.gpt-newest.yaml --env-file .env --repeat 30
+description: "Ponytail vs baseline, newest OpenAI (gpt-5.5 + minis). LOC + correctness, cost telemetry."
+
+providers:
+  - id: openai:gpt-5.5
+    config: { max_completion_tokens: 8192 }
+  - id: openai:gpt-4.1-mini
+    config: { max_tokens: 8192, temperature: 1 }
+  - id: openai:gpt-5.4-mini
+    config: { max_completion_tokens: 8192 }
+
+prompts:
+  - id: file://arms/baseline.js
+    label: baseline (no skill)
+  - id: file://arms/ponytail.js
+    label: ponytail
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: file://loc.js
+      metric: code_loc
+    - type: javascript
+      value: file://correctness.js
+      metric: correct
+
+tests:
+  - vars: { task: "Write me a Python function that validates email addresses." }
+  - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
+  - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
+  - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
+  - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }

package/benchmarks/promptfooconfig.yaml

@@ -0,0 +1,41 @@
+# Ponytail benchmark: code size + cost across three arms, same model, same tasks.
+#
+#   Run:   npx promptfoo@latest eval -c benchmarks/promptfooconfig.yaml
+#   View:  npx promptfoo@latest view
+#   Share: npx promptfoo@latest share   (publishes a hosted report URL)
+#
+# Needs ANTHROPIC_API_KEY in the environment or a .env file (see benchmarks/README.md).
+# Caveman arm uses JuliusBrussee/caveman SKILL.md (MIT), vendored at arms/caveman-SKILL.md.
+description: "Ponytail vs caveman vs no-skill: same model, same tasks. Measures code LOC (deterministic) and tokens/cost (API telemetry)."
+
+providers:
+  - id: anthropic:messages:claude-haiku-4-5-20251001
+    config: { max_tokens: 8192, temperature: 1 }
+  - id: anthropic:messages:claude-sonnet-4-6
+    config: { max_tokens: 8192, temperature: 1 }
+  - id: anthropic:messages:claude-opus-4-8
+    config: { max_tokens: 8192, temperature: 1 }
+
+prompts:
+  - id: file://arms/baseline.js
+    label: baseline (no skill)
+  - id: file://arms/caveman.js
+    label: caveman
+  - id: file://arms/ponytail.js
+    label: ponytail
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: file://loc.js
+      metric: code_loc
+    - type: javascript
+      value: file://correctness.js
+      metric: correct
+
+tests:
+  - vars: { task: "Write me a Python function that validates email addresses." }
+  - vars: { task: "Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay) returns a debounced version of fn that delays calling it until delay ms after the last call." }
+  - vars: { task: "Write Python code that reads sales.csv and sums the 'amount' column." }
+  - vars: { task: "Build me a countdown timer component in React that counts down from a given number of seconds." }
+  - vars: { task: "Add rate limiting to my FastAPI endpoint so users can't spam it." }