chill-agent 1.4.0 → 1.6.0

package/tests/commands.test.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+// Every ponytail command the pi extension registers must also ship as a
+// file-based command for the hosts that need one: Claude Code (commands/*.toml,
+// which Gemini CLI reuses) and OpenCode (.opencode/command/*.md). /ponytail-help
+// was advertised in the README and the help card but missing both files; this
+// guards that drift -- a registered command with no adapter file fails here.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const root = path.join(__dirname, '..');
+
+// pi-extension registers the canonical command set.
+const piSource = fs.readFileSync(path.join(root, 'pi-extension', 'index.js'), 'utf8');
+const commands = [...piSource.matchAll(/registerCommand\(["']([\w-]+)["']/g)].map((m) => m[1]);
+
+test('pi registers at least the base command', () => {
+  assert.ok(commands.includes('ponytail'), 'expected pi to register a ponytail command');
+});
+
+test('every registered command ships a Claude commands/*.toml', () => {
+  for (const name of commands) {
+    assert.ok(
+      fs.existsSync(path.join(root, 'commands', `${name}.toml`)),
+      `missing commands/${name}.toml`,
+    );
+  }
+});
+
+test('every registered command ships an OpenCode .opencode/command/*.md', () => {
+  for (const name of commands) {
+    assert.ok(
+      fs.existsSync(path.join(root, '.opencode', 'command', `${name}.md`)),
+      `missing .opencode/command/${name}.md`,
+    );
+  }
+});
+
+// /ponytail-debt documents one grep for the whole tree, shipped in four copies.
+// It missed C-style /* ponytail: */ block comments (#810), so the ledger silently
+// skipped every C/Java/CSS marker. Feed each shipped expression to real grep
+// rather than restating it here: a copied regex only tests the copy.
+const debtCopies = [
+  'skills/ponytail-debt/SKILL.md',
+  '.openclaw/skills/ponytail-debt/SKILL.md',
+  'commands/ponytail-debt.toml',
+  '.opencode/command/ponytail-debt.md',
+];
+const debtSamples = [
+  ['# ponytail: hash marker', true],
+  ['#ponytail: hash marker, no space', true],
+  ['// ponytail: slash marker', true],
+  ['//ponytail: slash marker, no space', true],
+  ['/* ponytail: block marker, closed here */', true],
+  ['/*ponytail: block marker, no space */', true],
+  ['/* ponytail: block opener, closed some lines below', true],
+  ['prose that merely mentions ponytail: markers stays out of the ledger', false],
+];
+
+test('every ponytail-debt copy greps #, // and /* markers but not prose', (t) => {
+  const input = debtSamples.map(([line]) => line).join('\n') + '\n';
+  const expected = debtSamples.flatMap(([line, hit], i) => (hit ? [`${i + 1}:${line}`] : []));
+
+  for (const rel of debtCopies) {
+    const documented = fs.readFileSync(path.join(root, rel), 'utf8').match(/git grep [^']*-nE '([^']+)'/);
+    assert.ok(documented, `${rel} no longer documents a git grep ... -nE '<pattern>'`);
+
+    const result = spawnSync('grep', ['-nE', documented[1]], { input, encoding: 'utf8', timeout: 5000 });
+    if (result.error) {
+      if (result.error.code === 'ENOENT') return t.skip('grep not available on this platform');
+      throw result.error;
+    }
+    assert.equal(result.status, 0, `grep failed for ${rel}: ${result.stderr}`);
+    assert.deepEqual(result.stdout.split('\n').filter(Boolean), expected, `${rel} misses or over-matches markers`);
+  }
+});
+
+// The full documented command must skip .git and whatever .gitignore ignores, in any stack (#948, #1100).
+test('the ponytail-debt scan skips .git and ignored dirs like node_modules, dist, build and .next', (t) => {
+  const cmd = fs.readFileSync(path.join(root, 'skills/ponytail-debt/SKILL.md'), 'utf8').match(/`(git grep [^`]+)`/)[1];
+  const dir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'ponytail-debt-'));
+  for (const sub of ['src', '.git', 'node_modules/pkg', 'dist', 'build', '.next']) {
+    fs.mkdirSync(path.join(dir, sub), { recursive: true });
+    fs.writeFileSync(path.join(dir, sub, 'a.js'), '// ponytail: marker\n');
+  }
+  fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\ndist/\nbuild/\n.next/\n');
+  const result = spawnSync('sh', ['-c', cmd], { cwd: dir, encoding: 'utf8', timeout: 5000 });
+  fs.rmSync(dir, { recursive: true, force: true });
+  if (result.error || result.status === 127) return t.skip(`sh/git not available: ${result.error ? result.error.code : result.stderr}`);
+  assert.deepEqual(result.stdout.split('\n').filter(Boolean), ['src/a.js:1:// ponytail: marker']);
+});

package/tests/copilot-plugin.test.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+// Smoke test for the Copilot plugin adapter: keep command wiring minimal and
+// ensure the debt command is part of the shared command surface.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('fs');
+const path = require('path');
+
+const root = path.join(__dirname, '..');
+const REQUIRED_COMMAND_FILES = [
+  'ponytail.toml',
+  'ponytail-review.toml',
+  'ponytail-audit.toml',
+  'ponytail-debt.toml',
+  'ponytail-gain.toml',
+  'ponytail-help.toml',
+];
+
+function readJSON(relPath) {
+  return JSON.parse(fs.readFileSync(path.join(root, relPath), 'utf8'));
+}
+
+test('copilot plugin command directory includes ponytail-debt', () => {
+  const manifest = readJSON('.github/plugin/plugin.json');
+  assert.equal(manifest.name, 'ponytail');
+  assert.equal(manifest.commands, 'commands/');
+
+  for (const file of REQUIRED_COMMAND_FILES) {
+    assert.ok(
+      fs.existsSync(path.join(root, manifest.commands, file)),
+      `missing command file: ${manifest.commands}${file}`,
+    );
+  }
+});

package/tests/correctness.test.js

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+// Unit test for the correctness benchmark assertion. Feeds known-good and
+// known-bad LLM outputs through each task checker and asserts the expected
+// pass/fail verdict. Runs without promptfoo — just node:test + the module.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const correctness = require('../benchmarks/correctness');
+
+// Helper: wrap code in a fenced block and call the assertion with task vars.
+function check(task, lang, code) {
+  const output = '```' + lang + '\n' + code + '\n```';
+  return correctness(output, { vars: { task } });
+}
+
+// --- Email validator ---
+
+test('email: correct one-liner passes', () => {
+  const result = check(
+    'Write me a Python function that validates email addresses.',
+    'python',
+    'def validate_email(email):\n    return "@" in email and "." in email.split("@")[-1] and email.split("@")[0] != ""',
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('email: always-true validator fails', () => {
+  const result = check(
+    'Write me a Python function that validates email addresses.',
+    'python',
+    'def validate_email(email):\n    return True',
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('email: no code block fails', () => {
+  const result = correctness('Here is my answer: just use regex.', {
+    vars: { task: 'Write me a Python function that validates email addresses.' },
+  });
+  assert.equal(result.pass, false);
+});
+
+test('email: bare unfenced code is still scored (#65)', () => {
+  const result = correctness('import re\ndef validate_email(e):\n    return bool(re.match(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$", e))', {
+    vars: { task: 'Write me a Python function that validates email addresses.' },
+  });
+  assert.equal(result.pass, true);
+});
+
+// --- Debounce ---
+
+test('debounce: correct implementation passes', () => {
+  const result = check(
+    'Add debounce to a search input in vanilla JavaScript.',
+    'javascript',
+    `function debounce(fn, delay) {
+  let timer;
+  return function(...args) {
+    clearTimeout(timer);
+    timer = setTimeout(() => fn.apply(this, args), delay);
+  };
+}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('debounce: immediate-call implementation fails', () => {
+  const result = check(
+    'Add debounce to a search input in vanilla JavaScript.',
+    'javascript',
+    `function debounce(fn, delay) {
+  return function(...args) { fn.apply(this, args); };
+}`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('debounce: bare unfenced arrow function is still scored (#65)', () => {
+  const result = correctness('const debounce = (fn, delay) => {\n  let t;\n  return (...a) => { clearTimeout(t); t = setTimeout(() => fn(...a), delay); };\n};', {
+    vars: { task: 'Write a reusable debounce function in vanilla JavaScript: debounce(fn, delay).' },
+  });
+  assert.equal(result.pass, true);
+});
+
+// --- CSV sum ---
+
+test('csv: correct pandas one-liner passes', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `import pandas as pd
+df = pd.read_csv('sales.csv')
+print(df['amount'].sum())`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('csv: an exception in the code is reported, not swallowed (#919)', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `raise RuntimeError("boom")`,
+  );
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /boom/);
+});
+
+test('csv: code that prints wrong value fails', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `print(999)`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('csv: value containing 351 as substring fails (e.g. 13510)', () => {
+  const result = check(
+    "Write Python code that reads sales.csv and sums the 'amount' column.",
+    'python',
+    `print(13510)`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+test('csv: timeout can be raised for slow pandas startup', () => {
+  const previous = process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS;
+  try {
+    process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = '1';
+    const timedOut = check(
+      "Write Python code that reads sales.csv and sums the 'amount' column.",
+      'python',
+      `import time
+time.sleep(0.05)
+print(351)`,
+    );
+    assert.equal(timedOut.pass, false);
+    assert.match(timedOut.reason, /ETIMEDOUT|timed out/i);
+
+    process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = '1000';
+    const completed = check(
+      "Write Python code that reads sales.csv and sums the 'amount' column.",
+      'python',
+      `import time
+time.sleep(0.05)
+print(351)`,
+    );
+    assert.equal(completed.pass, true);
+    assert.equal(completed.score, 1);
+  } finally {
+    if (previous === undefined) delete process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS;
+    else process.env.PONYTAIL_CORRECTNESS_TIMEOUT_MS = previous;
+  }
+});
+
+// --- React countdown ---
+
+test('countdown: valid React component passes', () => {
+  const result = check(
+    'Build me a countdown timer component in React.',
+    'javascript',
+    `import { useState, useEffect } from 'react';
+export default function Countdown({ seconds }) {
+  const [count, setCount] = useState(seconds);
+  useEffect(() => {
+    if (count <= 0) return;
+    const id = setInterval(() => setCount(prev => prev - 1), 1000);
+    return () => clearInterval(id);
+  }, [count]);
+  return <div>{count}</div>;
+}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('countdown: static div without state fails', () => {
+  const result = check(
+    'Build me a countdown timer component in React.',
+    'javascript',
+    `export default function Countdown() { return <div>10</div>; }`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- Rate limiter ---
+
+test('ratelimit: FastAPI with limit logic passes', () => {
+  const result = check(
+    'Add rate limiting to my FastAPI endpoint so users can\'t spam it.',
+    'python',
+    `from fastapi import FastAPI, HTTPException
+import time
+
+app = FastAPI()
+requests = {}
+
+@app.get("/api")
+def endpoint(user: str = "anon"):
+    now = time.time()
+    window = requests.get(user, [])
+    window = [t for t in window if now - t < 60]
+    if len(window) >= 10:
+        raise HTTPException(429, "Too Many Requests")
+    window.append(now)
+    requests[user] = window
+    return {"ok": True}`,
+  );
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+});
+
+test('ratelimit: plain endpoint without limiting fails', () => {
+  const result = check(
+    'Add rate limiting to my FastAPI endpoint.',
+    'python',
+    `from fastapi import FastAPI
+app = FastAPI()
+
+@app.get("/api")
+def endpoint():
+    return {"ok": True}`,
+  );
+  assert.equal(result.pass, false);
+  assert.equal(result.score, 0);
+});
+
+// --- Edge cases ---
+
+test('unknown task is gracefully skipped', () => {
+  const result = correctness('```python\nprint("hi")\n```', {
+    vars: { task: 'Explain quantum computing.' },
+  });
+  assert.equal(result.pass, true);
+  assert.equal(result.score, 1);
+  assert.match(result.reason, /unknown task/i);
+});