codehere 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/package.json +1 -1
- package/src/brain.js +2 -2
- package/src/commands.js +10 -7
- package/src/gemma.js +17 -3
- package/src/hooks.js +25 -13
- package/src/signals.js +52 -13
package/README.md
CHANGED
|
@@ -65,7 +65,7 @@ codehere ✕ npm test · 2 failed · agent sent back · 1 of 3
|
|
|
65
65
|
codehere ✓ npm test · 48 passed · re-run here · 6.2 s
|
|
66
66
|
```
|
|
67
67
|
|
|
68
|
-
✓ marks only a test run Codehere just made on your machine. A teammate's "verified" is plain text: it was true on their machine, and anyone with push access can edit a lesson file. The agent is told whether the files the fix changed have changed since.
|
|
68
|
+
✓ marks only a test run Codehere just made on your machine. A run whose own summary counts a failed test is a failure, even when the command exits 0. A teammate's "verified" is plain text: it was true on their machine, and anyone with push access can edit a lesson file. A tick written into a rule or a lesson reaches the agent as `[tick]`. The agent is told whether the files the fix changed have changed since.
|
|
69
69
|
|
|
70
70
|
Don't want to commit help cards? Copy the printed text into WhatsApp, Discord or Slack instead.
|
|
71
71
|
|
|
@@ -126,7 +126,7 @@ Everything is plain markdown, so you can read, edit, review and delete it like a
|
|
|
126
126
|
- **It only reacts to this session's work.** Changes you had before the session are ignored. If the agent changed nothing, Codehere does nothing.
|
|
127
127
|
- **It never traps the agent.** At most 3 "not done yet" messages per session, then it lets go.
|
|
128
128
|
- **It fails open.** If anything breaks, the hook exits quietly and the agent carries on.
|
|
129
|
-
- **Small model, small job.** Gemma 4 (via Ollama) is optional and only writes summaries for briefs and help cards, which you read before using. Turn it on with `--gemma`.
|
|
129
|
+
- **Small model, small job.** Gemma 4 (via Ollama) is optional and only writes summaries for briefs and help cards, which you read before using. Turn it on with `--gemma`. The text goes only to a model on this machine: Codehere refuses an `OLLAMA_HOST` on another machine and Ollama cloud models, and uses the plain text instead.
|
|
130
130
|
- **Only human-confirmed context at session start.** Rules are written by people. Lessons appear only when a matching error shows up, because auto-generated context files can make agents worse.
|
|
131
131
|
|
|
132
132
|
## Try the demo (no Claude account needed)
|
|
@@ -140,7 +140,7 @@ npm test # end-to-end checks
|
|
|
140
140
|
## Known limits
|
|
141
141
|
|
|
142
142
|
- Claude Code, Codex CLI and Copilot CLI. OpenCode is next. Copilot in VS Code reads the same hook files but has not been checked, and Copilot's cloud agent is not supported. See [Which agents](#which-agents) for what works on each.
|
|
143
|
-
- Lessons match by error text.
|
|
143
|
+
- Lessons match by error text: at least 0.45 of the words in the two error lines must be shared (words in both, divided by words in either). When both the failure and the lesson name a failing test, the names must be the same, so a lesson for a failed `assert.equal` reaches only a failing test with the same name. Two tests with the same name in different files or suites still share lessons. A generic error outside a test can bring up a lesson that doesn't fit, which is why the agent gets it as reference, not as an instruction.
|
|
144
144
|
- Redaction catches common keys, tokens and emails. Always read a help card before sharing it.
|
|
145
145
|
- The "done" check needs a test command. Without one, only the tampering check runs.
|
|
146
146
|
- Approvals are per folder. Each git worktree or extra clone needs its own `codehere approve`.
|
|
@@ -153,6 +153,6 @@ npm test # end-to-end checks
|
|
|
153
153
|
- Only the test command needs approval. A teammate who sets `verifyOnStop` to `false` in `.codehere/config.json` turns the finish check off for everyone; `codehere status` shows it as off.
|
|
154
154
|
- Sharing is through git. There is no server, so teammates see updates after `git pull`.
|
|
155
155
|
- Prompt checks are plain word lists (English and Indonesian). They can miss a short prompt in another language.
|
|
156
|
-
- The weakened-test scan catches skipped tests, deleted test files
|
|
156
|
+
- The weakened-test scan catches skipped and focused tests (`.skip`, `xit`, `{ skip: true }`, `.only`, `fit`), deleted test files, an assertion commented out with `//` or `#`, assertions that cannot fail (`assert.equal(1, 1)`), at least two more assertion lines removed than added, and a `package.json` test script changed to always pass (`|| true`). It does not catch one removed assertion, assertions wrapped in `/* */`, options spread over several lines, an edited expected value, or a `return` above an assertion; see `docs/overlap-with-check.md`.
|
|
157
157
|
|
|
158
158
|
MIT licensed.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codehere",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.3",
|
|
4
4
|
"description": "A git-native team brain for vibe coders. Silent fixes for AI coding agents: breaks doom loops, verifies 'done', blocks test tampering, and shares what your team learned.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
package/src/brain.js
CHANGED
|
@@ -323,8 +323,8 @@ export function gitHashes(root, files) {
|
|
|
323
323
|
|
|
324
324
|
// A tick is Codehere's mark for a run it made on this machine (DESIGN.md §1). Text from the
|
|
325
325
|
// repo or from a teammate never brings its own.
|
|
326
|
-
export function noTick(text) {
|
|
327
|
-
return String(text).replace(/[
|
|
326
|
+
export function noTick(text, mark = '?') {
|
|
327
|
+
return String(text).replace(/[✓✔☑✅√🗸🗹]/gu, mark);
|
|
328
328
|
}
|
|
329
329
|
|
|
330
330
|
// A commit id read back from state.json, or null. The state file can arrive through git,
|
package/src/commands.js
CHANGED
|
@@ -440,7 +440,10 @@ function repeatedErrors(s) {
|
|
|
440
440
|
return Object.entries(s.errors || {})
|
|
441
441
|
.sort((a, b) => b[1] - a[1])
|
|
442
442
|
.slice(0, 3)
|
|
443
|
-
.map(([sig, n]) =>
|
|
443
|
+
.map(([sig, n]) => {
|
|
444
|
+
const test = s.tests && s.tests[sig];
|
|
445
|
+
return { sig, n, raw: (s.rawErrors || {})[sig] || [], test: typeof test === 'string' ? test : '' };
|
|
446
|
+
});
|
|
444
447
|
}
|
|
445
448
|
|
|
446
449
|
function gitSummary(root, base) {
|
|
@@ -493,8 +496,8 @@ export async function brief(flags) {
|
|
|
493
496
|
`Rewrite this into one clear, complete instruction for a coding agent. Keep every requirement and constraint. Plain English, max 120 words.\n\n${text}`,
|
|
494
497
|
cfg.gemmaModel
|
|
495
498
|
);
|
|
496
|
-
if (summary) text = text.replace('## Goal', `## In one paragraph (written by ${summary.model} on this machine; an opinion, check it)\n${summary.text}\n\n## Goal`);
|
|
497
|
-
else console.error('(Gemma not reachable through Ollama; using the plain brief.)');
|
|
499
|
+
if (summary?.text) text = text.replace('## Goal', `## In one paragraph (written by ${summary.model} on this machine; an opinion, check it)\n${summary.text}\n\n## Goal`);
|
|
500
|
+
else console.error(summary?.error || '(Gemma not reachable through Ollama; using the plain brief.)');
|
|
498
501
|
}
|
|
499
502
|
|
|
500
503
|
const file = path.join(paths(root).briefs, `${stamp()}.md`);
|
|
@@ -543,8 +546,8 @@ export async function help(question, flags) {
|
|
|
543
546
|
`Summarise this help request for a busy teammate in 2 sentences: what is wrong and what you need from them.\n\n${text}`,
|
|
544
547
|
cfg.gemmaModel
|
|
545
548
|
);
|
|
546
|
-
if (summary) text = `## In short (written by ${summary.model} on this machine; an opinion, check it)\n${summary.text}\n\n${text}`;
|
|
547
|
-
else console.error('(Gemma not reachable through Ollama; using the plain help card.)');
|
|
549
|
+
if (summary?.text) text = `## In short (written by ${summary.model} on this machine; an opinion, check it)\n${summary.text}\n\n${text}`;
|
|
550
|
+
else console.error(summary?.error || '(Gemma not reachable through Ollama; using the plain help card.)');
|
|
548
551
|
}
|
|
549
552
|
|
|
550
553
|
const id = `${stamp()}-${slug(ask, 30)}`;
|
|
@@ -552,7 +555,7 @@ export async function help(question, flags) {
|
|
|
552
555
|
fs.writeFileSync(
|
|
553
556
|
file,
|
|
554
557
|
renderDoc(
|
|
555
|
-
{ id, author: me(root), date: today(), status: 'open', signature: errs[0] ? errs[0].sig : '' },
|
|
558
|
+
{ id, author: me(root), date: today(), status: 'open', signature: errs[0] ? errs[0].sig : '', ...(errs[0]?.test ? { test: redact(errs[0].test).slice(0, 120) } : {}) },
|
|
556
559
|
text
|
|
557
560
|
)
|
|
558
561
|
);
|
|
@@ -614,7 +617,7 @@ export function reply(id, text, flags) {
|
|
|
614
617
|
fs.writeFileSync(
|
|
615
618
|
lessonFile,
|
|
616
619
|
renderDoc(
|
|
617
|
-
{ title: q, signature: meta.signature || q, author: who, date: today(), from: `help/${d.id}` },
|
|
620
|
+
{ title: q, signature: meta.signature || q, ...(meta.test ? { test: meta.test } : {}), author: who, date: today(), from: `help/${d.id}` },
|
|
618
621
|
`## Fix\n- ${text}\n`
|
|
619
622
|
)
|
|
620
623
|
);
|
package/src/gemma.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
const url = () => {
|
|
6
6
|
const host = process.env.OLLAMA_HOST || '127.0.0.1:11434';
|
|
7
|
-
return /^https
|
|
7
|
+
return /^https?:\/\//i.test(host) ? host : `http://${host}`; // Ollama allows OLLAMA_HOST without a scheme
|
|
8
8
|
};
|
|
9
9
|
|
|
10
10
|
async function post(path, body, ms) {
|
|
@@ -36,10 +36,24 @@ export function pickModel(installed, wanted) {
|
|
|
36
36
|
return null;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
//
|
|
39
|
+
// Text goes only to a model on this machine. Ollama forwards a cloud model (`gpt-oss:120b-cloud`,
|
|
40
|
+
// `glm-4.6:cloud`) to its own servers, and OLLAMA_HOST can name another machine.
|
|
41
|
+
const LOCAL_HOST = /^(localhost\.?|127(\.\d{1,3}){3}|0\.0\.0\.0|\[::1?\])$/i;
|
|
42
|
+
const isCloud = (name) => /[:-]cloud$/i.test(name || '');
|
|
43
|
+
|
|
44
|
+
// Returns { text, model }, { error } when Codehere refuses to send the text, or null.
|
|
40
45
|
export async function summarize(prompt, wanted = 'gemma4:e4b') {
|
|
46
|
+
let host = '';
|
|
47
|
+
try {
|
|
48
|
+
host = new URL(url()).hostname;
|
|
49
|
+
} catch {}
|
|
50
|
+
if (!LOCAL_HOST.test(host)) return { error: `(OLLAMA_HOST is ${process.env.OLLAMA_HOST}, which is not a loopback address. Codehere sends text only to a local model; using the plain text.)` };
|
|
51
|
+
if (isCloud(wanted)) return { error: `(${wanted} is an Ollama cloud model. Codehere sends text only to a local model; using the plain text.)` };
|
|
41
52
|
const tags = await post('/api/tags', null, 3000);
|
|
42
|
-
const
|
|
53
|
+
const all = (tags && tags.models) || [];
|
|
54
|
+
const local = all.filter((m) => !isCloud(m.name) && !m.remote_host);
|
|
55
|
+
const model = pickModel(local.map((m) => m.name), wanted);
|
|
56
|
+
if (!model && local.length < all.length) return { error: '(No local Gemma model, and Ollama cloud models are not used. Codehere sends text only to a local model; using the plain text.)' };
|
|
43
57
|
if (!model) return null;
|
|
44
58
|
// Thinking off: a summary does not need it, and it is several times slower.
|
|
45
59
|
const data = await post('/api/generate', { model, prompt, stream: false, think: false, options: { temperature: 0.2 } }, 60000);
|
package/src/hooks.js
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
removedAt,
|
|
12
12
|
} from './brain.js';
|
|
13
13
|
import {
|
|
14
|
-
responseText, signature, errorLines, similarity, tamperFindings, failingTest, lowInfoPrompt, vagueFeature, testCounts,
|
|
14
|
+
responseText, signature, errorLines, similarity, tamperFindings, failingTest, lowInfoPrompt, vagueFeature, testCounts, redact,
|
|
15
15
|
} from './signals.js';
|
|
16
16
|
|
|
17
17
|
function readStdin() {
|
|
@@ -145,9 +145,11 @@ function sessionStart(root, input, agent) {
|
|
|
145
145
|
}
|
|
146
146
|
|
|
147
147
|
writeState(root, state);
|
|
148
|
+
// Rules, briefs and help cards come from the repo: none of it may show the agent Codehere's tick.
|
|
149
|
+
const text = noTick(out.join('\n'), '[tick]'); // "[tick]" keeps the meaning of a spec that asks for one
|
|
148
150
|
// Copilot reads only JSON here; Claude Code and Codex take the plain text
|
|
149
|
-
if (out.length && agent === 'copilot') emitJson({ additionalContext:
|
|
150
|
-
if (out.length) process.stdout.write(
|
|
151
|
+
if (out.length && agent === 'copilot') emitJson({ additionalContext: text });
|
|
152
|
+
if (out.length) process.stdout.write(text + '\n');
|
|
151
153
|
quiet();
|
|
152
154
|
}
|
|
153
155
|
|
|
@@ -251,10 +253,12 @@ export function readOnly(command) {
|
|
|
251
253
|
return bare.split(/&&|\|\||\|&|[;|\n]/).every((part) => !part.trim() || READ_ONLY.test(part));
|
|
252
254
|
}
|
|
253
255
|
|
|
254
|
-
function matchLesson(root, sig, shown) {
|
|
256
|
+
function matchLesson(root, sig, shown, test) {
|
|
255
257
|
let best = null;
|
|
256
258
|
for (const doc of listDocs(paths(root).lessons)) {
|
|
257
259
|
if (shown.includes(doc.id) || doc.body.includes('- Fix: <one line>')) continue; // not filled in yet
|
|
260
|
+
// Every failed assert.equal shares one first line, so when both name the failing test, it must be the same test.
|
|
261
|
+
if (test && doc.meta.test && doc.meta.test !== test) continue;
|
|
258
262
|
const score = Math.max(
|
|
259
263
|
similarity(sig, doc.meta.signature || ''),
|
|
260
264
|
similarity(sig, doc.meta.title || '') * 0.8
|
|
@@ -318,7 +322,7 @@ function lessonText(root, doc) {
|
|
|
318
322
|
: 'Proof none recorded.'
|
|
319
323
|
);
|
|
320
324
|
out.push('Check whether it fits before trying something new.');
|
|
321
|
-
return out.join('\n');
|
|
325
|
+
return noTick(out.join('\n'), '[tick]'); // a teammate's text cannot show the agent Codehere's tick
|
|
322
326
|
}
|
|
323
327
|
|
|
324
328
|
function lessonLine(doc) {
|
|
@@ -328,13 +332,13 @@ function lessonLine(doc) {
|
|
|
328
332
|
}
|
|
329
333
|
|
|
330
334
|
// Codehere writes the proof fields itself; the agent fills in the title, cause and fix.
|
|
331
|
-
function writeLesson(root, sig, proofCommand, changed) {
|
|
335
|
+
function writeLesson(root, sig, proofCommand, changed, test) {
|
|
332
336
|
const dir = paths(root).lessons;
|
|
333
337
|
fs.mkdirSync(dir, { recursive: true });
|
|
334
338
|
const name = `${today()}-${slug(sig, 30)}`;
|
|
335
339
|
let file = path.join(dir, `${name}.md`);
|
|
336
340
|
for (let i = 2; fs.existsSync(file); i++) file = path.join(dir, `${name}-${i}.md`);
|
|
337
|
-
const meta = { title: PLACEHOLDERS[0], signature: sig, author: me(root), date: today(), state: proofCommand ? 'verified' : 'unverified' };
|
|
341
|
+
const meta = { title: PLACEHOLDERS[0], signature: sig, ...(test ? { test: redact(test).slice(0, 120) } : {}), author: me(root), date: today(), state: proofCommand ? 'verified' : 'unverified' };
|
|
338
342
|
if (proofCommand) {
|
|
339
343
|
Object.assign(meta, {
|
|
340
344
|
proof_command: proofCommand,
|
|
@@ -371,10 +375,14 @@ function postTool(root, input, agent) {
|
|
|
371
375
|
const s = session(state, input.session_id);
|
|
372
376
|
s.errors[sig] = (s.errors[sig] || 0) + 1;
|
|
373
377
|
s.rawErrors[sig] = errorLines(text, 8);
|
|
374
|
-
|
|
378
|
+
const test = failingTest(text);
|
|
379
|
+
// A lesson names the failing test only if every failure on this line was that test.
|
|
380
|
+
const tests = s.tests && typeof s.tests === 'object' ? s.tests : {};
|
|
381
|
+
s.tests = { ...tests, [sig]: Object.hasOwn(tests, sig) && tests[sig] !== test ? '' : test || '' };
|
|
382
|
+
s.lastFailure = { sig, command, test, lines: errorLines(text, 3), prompt: s.prompts || 0 };
|
|
375
383
|
const count = s.errors[sig];
|
|
376
384
|
|
|
377
|
-
const lesson = matchLesson(root, sig, s.shownLessons);
|
|
385
|
+
const lesson = matchLesson(root, sig, s.shownLessons, test);
|
|
378
386
|
if (lesson) s.shownLessons.push(lesson.id);
|
|
379
387
|
writeState(root, state);
|
|
380
388
|
|
|
@@ -543,15 +551,17 @@ function runTests(root, cfg) {
|
|
|
543
551
|
});
|
|
544
552
|
const output = `${r.stdout || ''}\n${r.stderr || ''}`.trim();
|
|
545
553
|
const timedOut = r.error && r.error.code === 'ETIMEDOUT';
|
|
546
|
-
|
|
554
|
+
// The runner's own count of failures wins over exit 0: `|| true` on a test script is one edit.
|
|
555
|
+
const c = testCounts(output);
|
|
556
|
+
return { ok: r.status === 0 && !timedOut && !(c && c.failed), output, timedOut, code: r.status, seconds: (Date.now() - started) / 1000 };
|
|
547
557
|
}
|
|
548
558
|
|
|
549
559
|
// "48 passed" or "2 failed" when the runner's summary can be read, else the exit code
|
|
550
560
|
export function runResult(t) {
|
|
551
561
|
if (t.timedOut) return 'timed out';
|
|
552
562
|
const c = testCounts(t.output);
|
|
553
|
-
if (t.ok) return c && !c.failed ? `${c.passed} passed` : 'exit 0'; //
|
|
554
|
-
return c && c.failed ? `${c.failed} failed` : `exit ${t.code}`;
|
|
563
|
+
if (t.ok) return c && !c.failed ? `${c.passed} passed` : 'exit 0'; // no summary to read: say only the exit code
|
|
564
|
+
return c && c.failed ? `${c.failed} failed${t.code === 0 ? ' · exit 0' : ''}` : `exit ${t.code}`;
|
|
555
565
|
}
|
|
556
566
|
|
|
557
567
|
const removedLine = (at) => `codehere ✕ codehere uninstall ran at ${new Date(at).toTimeString().slice(0, 5)} · this repo's checks are removed`;
|
|
@@ -659,6 +669,7 @@ function stop(root, input) {
|
|
|
659
669
|
block(
|
|
660
670
|
[
|
|
661
671
|
`Codehere ran \`${cmd}\` and it ${t.timedOut ? 'timed out' : 'failed'}. The task is not done yet.`,
|
|
672
|
+
...(t.code === 0 && !t.timedOut ? ['Its own summary counts failed tests, though the command exited 0.'] : []),
|
|
662
673
|
'Last lines of output:',
|
|
663
674
|
tail,
|
|
664
675
|
'Fix the cause, then finish. Do not change or skip tests to make them pass.',
|
|
@@ -685,7 +696,8 @@ function stop(root, input) {
|
|
|
685
696
|
if (solved.length && !s.lessonAsked) {
|
|
686
697
|
s.lessonAsked = true;
|
|
687
698
|
const [sig] = solved.sort((a, b) => b[1] - a[1])[0];
|
|
688
|
-
const
|
|
699
|
+
const named = s.tests && s.tests[sig];
|
|
700
|
+
const file = writeLesson(root, sig, proven ? cfg.testCommand : null, changed, typeof named === 'string' ? named : '');
|
|
689
701
|
if (mayBlock) {
|
|
690
702
|
s.stopBlocks++;
|
|
691
703
|
writeState(root, state);
|
package/src/signals.js
CHANGED
|
@@ -57,7 +57,7 @@ export function failingTest(text) {
|
|
|
57
57
|
const t = stripAnsi(text);
|
|
58
58
|
const m =
|
|
59
59
|
t.match(/^\s*(?:✖|×|✗|not ok \d+ -)\s+(?!failing tests)(.+?)(?:\s+\(?\d[\d.]*\s?m?s\)?)?\s*$/m) ||
|
|
60
|
-
t.match(/^\s*●\s+(.+?)\s*$/m) ||
|
|
60
|
+
t.match(/^\s*●\s+(?!Console\s*$)(.+?)\s*$/m) ||
|
|
61
61
|
t.match(/^FAILED\s+(\S+)/m) ||
|
|
62
62
|
t.match(/^--- FAIL:\s+(\S+)/m);
|
|
63
63
|
return m ? m[1].trim().slice(0, 120) : null;
|
|
@@ -82,7 +82,7 @@ export function lowInfoPrompt(p) {
|
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
const BUILD_START =
|
|
85
|
-
/^\s*(please\s+|tolong\s+|can you\s+|could you\s+)?(build|create|make|add|implement|develop|design|set up|setup|buat|buatkan|bikin|bikinin|tambah|tambahkan|bangun)\b/i;
|
|
85
|
+
/^\s*(please\s+|tolong\s+|can you\s+|could you\s+)?(build|create|make|add|implement|develop|design|set up|setup|buat|buatkan|buatin|bikin|bikinin|tambah|tambahkan|tambahin|bangun)\b/i;
|
|
86
86
|
const BIG = /\b(app|application|feature|system|platform|website|site|dashboard|service|bot|game|aplikasi|fitur|sistem|situs|web)\b/i;
|
|
87
87
|
const SMALL = /\b(make sure|pastikan|instead|still|again|fix|bug|error|rename|change|ubah|ganti|refactor|button|color|colour|font|style)\b/i;
|
|
88
88
|
const CHECKABLE =
|
|
@@ -133,6 +133,25 @@ const TAMPER_ADDED = [
|
|
|
133
133
|
{ re: /expect\([^)]*\)\.(toBe|toEqual)\(\s*(true|1)\s*\)\s*;?\s*\/\/\s*(todo|temp|hack)/i, why: 'an assertion was replaced with a placeholder' },
|
|
134
134
|
];
|
|
135
135
|
|
|
136
|
+
// Only in test files: elsewhere `{ skip: true }` or `assert.equal(1, 1)` can be ordinary code.
|
|
137
|
+
const LITERAL = String.raw`(\d+(?:\.\d+)?|true|false|null|'[^']*'|"[^"]*")`;
|
|
138
|
+
const TAMPER_ADDED_IN_TESTS = [
|
|
139
|
+
// test('name', { skip: true }, ...): the name cannot run past its closing quote, so the options are this test's
|
|
140
|
+
{ re: /(?<![.\w])(it|test|describe|suite)\s*\(\s*(['"`])(?:(?!\2)[^\\]|\\.)*\2\s*,\s*\{[^}]*\b(skip|todo)\s*:\s*(true\b|['"`])/, why: 'a test was skipped ({ skip: true })' },
|
|
141
|
+
// fit/fdescribe only in jest's form, starting a statement with a quoted name then a callback:
|
|
142
|
+
// `fit(` alone is a common function name
|
|
143
|
+
{ re: /(?<![.\w])(it|test|describe|context|suite)\.only\s*\(|^[+-]\s*(fit|fdescribe)\s*\(\s*(['"`])(?:(?!\3)[^\\]|\\.)*\3\s*,\s*(async\s+)?(\(|function\b|\w+\s*=>)/, why: 'other tests were turned off (.only)' },
|
|
144
|
+
{ re: /\bassert(\.ok)?\(\s*true\s*\)|^\+\s*assert\s+True\s*$/, why: 'an assertion that cannot fail was added' },
|
|
145
|
+
{ re: new RegExp(String.raw`\bassert\.\w+\(\s*${LITERAL}\s*,\s*\1\s*\)`), why: 'an assertion that cannot fail was added' },
|
|
146
|
+
{ re: new RegExp(String.raw`\bexpect\(\s*${LITERAL}\s*\)\.(toBe|toEqual|toStrictEqual)\(\s*\1\s*\)`), why: 'an assertion that cannot fail was added' },
|
|
147
|
+
];
|
|
148
|
+
// assert(...), assert.equal(...), expect(...), self.assertEqual(...), and pytest's bare `assert x`;
|
|
149
|
+
// not expect.objectContaining(...) and the other matchers inside one assertion
|
|
150
|
+
const ASSERT = /\b(expect\s*\(|assert[\w.]*\s*\()|^[+-]\s*assert\s/;
|
|
151
|
+
// package.json's test script made to pass whatever the tests do: `|| true`, `|| :`, `|| echo`, `; exit 0`
|
|
152
|
+
const ALWAYS_PASSES = /"test"\s*:\s*"[^"]*(\|\|\s*(true|:|exit\s+0|echo\b[^"&|;]*)|;\s*(true|exit\s+0))\s*"/;
|
|
153
|
+
const COMMENT = /^[+-]\s*(\/\/|#|\/\*|\*)\s*/;
|
|
154
|
+
|
|
136
155
|
export function tamperFindings(nameStatus, diff) {
|
|
137
156
|
const found = [];
|
|
138
157
|
for (const line of nameStatus.split('\n')) {
|
|
@@ -142,17 +161,32 @@ export function tamperFindings(nameStatus, diff) {
|
|
|
142
161
|
let current = '';
|
|
143
162
|
let removedAsserts = 0;
|
|
144
163
|
let addedAsserts = 0;
|
|
164
|
+
let removed = new Set(); // this file's removed assertions, to spot one coming back as a comment
|
|
165
|
+
let hadFallback = false; // the test script already ended in `|| true` before this session
|
|
145
166
|
for (const line of diff.split('\n')) {
|
|
146
167
|
if (line.startsWith('+++ ')) {
|
|
147
168
|
current = line.replace(/^\+\+\+ (b\/)?/, '');
|
|
169
|
+
removed = new Set();
|
|
170
|
+
hadFallback = false;
|
|
148
171
|
continue;
|
|
149
172
|
}
|
|
150
173
|
if (line.startsWith('--- ')) continue;
|
|
174
|
+
const inTest = TEST_FILE.test(current);
|
|
175
|
+
const probe = line.slice(0, 10000); // the patterns can take quadratic time on a long line
|
|
176
|
+
if (/(^|\/)package\.json$/.test(current) && ALWAYS_PASSES.test(probe)) {
|
|
177
|
+
if (line.startsWith('-')) hadFallback = true;
|
|
178
|
+
else if (!hadFallback) found.push(`the test script was changed to always exit 0 in ${current}`);
|
|
179
|
+
}
|
|
151
180
|
if (line.startsWith('+')) {
|
|
152
|
-
for (const t of TAMPER_ADDED) if (t.re.test(
|
|
153
|
-
if (
|
|
154
|
-
|
|
155
|
-
if (
|
|
181
|
+
for (const t of TAMPER_ADDED) if (t.re.test(probe)) found.push(`${t.why} in ${current}`);
|
|
182
|
+
if (!inTest) continue;
|
|
183
|
+
for (const t of TAMPER_ADDED_IN_TESTS) if (t.re.test(probe)) found.push(`${t.why} in ${current}`);
|
|
184
|
+
if (COMMENT.test(line)) {
|
|
185
|
+
if (removed.has(line.replace(COMMENT, '').trim())) found.push(`an assertion was commented out in ${current}`);
|
|
186
|
+
} else if (ASSERT.test(line)) addedAsserts++;
|
|
187
|
+
} else if (line.startsWith('-') && inTest && !COMMENT.test(line) && ASSERT.test(line)) {
|
|
188
|
+
removedAsserts++;
|
|
189
|
+
removed.add(line.slice(1).trim());
|
|
156
190
|
}
|
|
157
191
|
}
|
|
158
192
|
if (removedAsserts - addedAsserts >= 2) {
|
|
@@ -192,6 +226,7 @@ const num = (s, re) => {
|
|
|
192
226
|
const m = s.match(re);
|
|
193
227
|
return m ? Number(m[1]) : null;
|
|
194
228
|
};
|
|
229
|
+
const after = (t, line) => t.slice(t.indexOf(line) + line.length);
|
|
195
230
|
const counts = (passed, failed) => (passed == null && failed == null ? null : { passed: passed || 0, failed: failed || 0 });
|
|
196
231
|
const SUMMARIES = [
|
|
197
232
|
// node --test, spec and TAP reporters: "ℹ tests 3", "ℹ pass 2", "ℹ fail 1"
|
|
@@ -203,15 +238,19 @@ const SUMMARIES = [
|
|
|
203
238
|
re: /^[=\s]*(?:\d+ \w+(?:, )?)+ in [\d.]+s\b.*$/gm,
|
|
204
239
|
read: (t, line) => counts(num(line, /(\d+) passed/), (num(line, /(\d+) failed/) || 0) + (num(line, /(\d+) errors?/) || 0)),
|
|
205
240
|
},
|
|
206
|
-
// mocha "46 passing" and "2 failing"
|
|
207
|
-
{
|
|
208
|
-
|
|
241
|
+
// mocha "46 passing", then "2 pending" and "2 failing" on the next lines (a log line elsewhere is not the count)
|
|
242
|
+
{
|
|
243
|
+
re: /^\s*\d+ passing\b.*$/gm,
|
|
244
|
+
read: (t, line) => counts(num(line, /(\d+) passing/), num(after(t, line).split('\n').slice(1, 3).join('\n'), /^\s*(\d+) failing\b/m)),
|
|
245
|
+
},
|
|
246
|
+
// python unittest "Ran 5 tests in 0.001s", then the status line: "OK" or "FAILED (failures=2, errors=1)"
|
|
209
247
|
{
|
|
210
248
|
re: /^Ran \d+ tests? in [\d.]+s$/gm,
|
|
211
|
-
read: (t) => {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
249
|
+
read: (t, line) => {
|
|
250
|
+
const status = (after(t, line).trimStart().match(/^(OK|FAILED)\b.*/) || [''])[0];
|
|
251
|
+
if (!status || /expected failures|unexpected successes/.test(status)) return null; // neither passed nor failed
|
|
252
|
+
const failed = (num(status, /failures=(\d+)/) || 0) + (num(status, /errors=(\d+)/) || 0);
|
|
253
|
+
return counts(num(line, /^Ran (\d+)/) - failed - (num(status, /skipped=(\d+)/) || 0), failed);
|
|
215
254
|
},
|
|
216
255
|
},
|
|
217
256
|
// cargo "test result: ok. 4 passed; 0 failed", one line per test binary
|