@slowdini/slow-powers-opencode 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +174 -0
- package/bootstrap.md +16 -0
- package/opencode/plugins/slow-powers.js +86 -0
- package/package.json +66 -0
- package/skills/auditing-slow-powers-usage/SKILL.md +157 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/BASELINE.md +22 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/NOTES.md +72 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/benchmark.json +53 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/audits-blindspot-session__with_skill.json +53 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/audits-blindspot-session__without_skill.json +38 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/audits-completed-session__with_skill.json +53 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/audits-completed-session__without_skill.json +38 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/ordinary-dev-task-no-audit__with_skill.json +17 -0
- package/skills/auditing-slow-powers-usage/evals/baseline/grading/ordinary-dev-task-no-audit__without_skill.json +17 -0
- package/skills/auditing-slow-powers-usage/evals/evals.json +74 -0
- package/skills/auditing-slow-powers-usage/evals/fixtures/audits-blindspot-session/session-summary.md +39 -0
- package/skills/auditing-slow-powers-usage/evals/fixtures/audits-completed-session/session-summary.md +33 -0
- package/skills/evaluating-skills/SKILL.md +448 -0
- package/skills/evaluating-skills/evals/evals.json +52 -0
- package/skills/evaluating-skills/evals/fixtures/iron-law/candidate-skill.md +13 -0
- package/skills/evaluating-skills/examples/verification-before-completion-evals.json +30 -0
- package/skills/evaluating-skills/harness-details/claude.md +135 -0
- package/skills/evaluating-skills/pressure-scenarios.md +163 -0
- package/skills/evaluating-skills/runner/README.md +140 -0
- package/skills/evaluating-skills/runner/adapters/claude-code-transcript.test.ts +263 -0
- package/skills/evaluating-skills/runner/adapters/claude-code-transcript.ts +146 -0
- package/skills/evaluating-skills/runner/aggregate.test.ts +188 -0
- package/skills/evaluating-skills/runner/aggregate.ts +228 -0
- package/skills/evaluating-skills/runner/context.test.ts +181 -0
- package/skills/evaluating-skills/runner/context.ts +90 -0
- package/skills/evaluating-skills/runner/detect-stray-writes.test.ts +103 -0
- package/skills/evaluating-skills/runner/detect-stray-writes.ts +192 -0
- package/skills/evaluating-skills/runner/fill-transcripts.test.ts +73 -0
- package/skills/evaluating-skills/runner/fill-transcripts.ts +154 -0
- package/skills/evaluating-skills/runner/grade.test.ts +347 -0
- package/skills/evaluating-skills/runner/grade.ts +603 -0
- package/skills/evaluating-skills/runner/guard/guard.ts +49 -0
- package/skills/evaluating-skills/runner/guard/install.test.ts +92 -0
- package/skills/evaluating-skills/runner/guard/install.ts +147 -0
- package/skills/evaluating-skills/runner/guard/policy.test.ts +71 -0
- package/skills/evaluating-skills/runner/guard/policy.ts +74 -0
- package/skills/evaluating-skills/runner/promote-baseline.test.ts +230 -0
- package/skills/evaluating-skills/runner/promote-baseline.ts +186 -0
- package/skills/evaluating-skills/runner/run.test.ts +716 -0
- package/skills/evaluating-skills/runner/run.ts +814 -0
- package/skills/evaluating-skills/runner/sandbox-policy.ts +74 -0
- package/skills/evaluating-skills/runner/types.ts +104 -0
- package/skills/evaluating-skills/runner/validate-all.ts +54 -0
- package/skills/evaluating-skills/runner/validate-schema.test.ts +99 -0
- package/skills/evaluating-skills/runner/validate-schema.ts +51 -0
- package/skills/evaluating-skills/runner/validate.test.ts +56 -0
- package/skills/evaluating-skills/runner/validate.ts +21 -0
- package/skills/evaluating-skills/schema/evals.schema.json +105 -0
- package/skills/evaluating-skills/schema/grading.schema.json +84 -0
- package/skills/evaluating-skills/schema/run-record.schema.json +80 -0
- package/skills/evaluating-skills/schema/stray-writes.schema.json +68 -0
- package/skills/evaluating-skills/templates/eval-task-prompt.md +71 -0
- package/skills/evaluating-skills/templates/evals.json.example +17 -0
- package/skills/evaluating-skills/templates/judge-prompt.md +56 -0
- package/skills/evaluating-skills/templates/revise-skill-prompt.md +56 -0
- package/skills/finishing-a-development-branch/SKILL.md +96 -0
- package/skills/finishing-a-development-branch/evals/evals.json +41 -0
- package/skills/finishing-a-development-branch/evals/fixtures/finish/package.json +4 -0
- package/skills/finishing-a-development-branch/evals/fixtures/finish/sum.test.ts +5 -0
- package/skills/hardening-plans/SKILL.md +72 -0
- package/skills/hardening-plans/evals/baseline/BASELINE.md +22 -0
- package/skills/hardening-plans/evals/baseline/NOTES.md +58 -0
- package/skills/hardening-plans/evals/baseline/benchmark.json +54 -0
- package/skills/hardening-plans/evals/baseline/grading/concrete-todo-app-plan__new_skill.json +39 -0
- package/skills/hardening-plans/evals/baseline/grading/concrete-todo-app-plan__old_skill.json +39 -0
- package/skills/hardening-plans/evals/baseline/grading/csv-parser-bug-no-plan__new_skill.json +24 -0
- package/skills/hardening-plans/evals/baseline/grading/csv-parser-bug-no-plan__old_skill.json +24 -0
- package/skills/hardening-plans/evals/baseline/grading/seeded-review-catches-defects__new_skill.json +46 -0
- package/skills/hardening-plans/evals/baseline/grading/seeded-review-catches-defects__old_skill.json +46 -0
- package/skills/hardening-plans/evals/evals.json +114 -0
- package/skills/systematic-debugging/CREATION-LOG.md +119 -0
- package/skills/systematic-debugging/SKILL.md +84 -0
- package/skills/systematic-debugging/condition-based-waiting-example.ts +164 -0
- package/skills/systematic-debugging/condition-based-waiting.md +115 -0
- package/skills/systematic-debugging/defense-in-depth.md +122 -0
- package/skills/systematic-debugging/evals/baseline/BASELINE.md +22 -0
- package/skills/systematic-debugging/evals/baseline/benchmark.json +51 -0
- package/skills/systematic-debugging/evals/baseline/grading/feature-request-no-debugging__with_skill.json +17 -0
- package/skills/systematic-debugging/evals/baseline/grading/feature-request-no-debugging__without_skill.json +17 -0
- package/skills/systematic-debugging/evals/baseline/grading/null-id-crash-investigate-first__with_skill.json +46 -0
- package/skills/systematic-debugging/evals/baseline/grading/null-id-crash-investigate-first__without_skill.json +31 -0
- package/skills/systematic-debugging/evals/evals.json +45 -0
- package/skills/systematic-debugging/evals/fixtures/order-bug/orderHandler.ts +9 -0
- package/skills/systematic-debugging/evals/fixtures/order-bug/repro.ts +10 -0
- package/skills/systematic-debugging/find-polluter.sh +63 -0
- package/skills/systematic-debugging/root-cause-tracing.md +169 -0
- package/skills/systematic-debugging/test-academic.md +14 -0
- package/skills/systematic-debugging/test-pressure-1.md +58 -0
- package/skills/systematic-debugging/test-pressure-2.md +68 -0
- package/skills/systematic-debugging/test-pressure-3.md +69 -0
- package/skills/test-driven-development/SKILL.md +93 -0
- package/skills/test-driven-development/evals/baseline/BASELINE.md +22 -0
- package/skills/test-driven-development/evals/baseline/NOTES.md +74 -0
- package/skills/test-driven-development/evals/baseline/benchmark.json +51 -0
- package/skills/test-driven-development/evals/baseline/grading/slugify-under-time-pressure__with_skill.json +53 -0
- package/skills/test-driven-development/evals/baseline/grading/slugify-under-time-pressure__without_skill.json +38 -0
- package/skills/test-driven-development/evals/baseline/grading/tests-after-rubber-stamp__with_skill.json +32 -0
- package/skills/test-driven-development/evals/baseline/grading/tests-after-rubber-stamp__without_skill.json +17 -0
- package/skills/test-driven-development/evals/evals.json +77 -0
- package/skills/test-driven-development/evals/fixtures/slugify/package.json +4 -0
- package/skills/test-driven-development/evals/fixtures/slugify/utils.ts +7 -0
- package/skills/test-driven-development/testing-anti-patterns.md +299 -0
- package/skills/using-git-worktrees/SKILL.md +70 -0
- package/skills/using-git-worktrees/evals/evals.json +40 -0
- package/skills/verification-before-completion/SKILL.md +65 -0
- package/skills/verification-before-completion/evals/baseline/BASELINE.md +22 -0
- package/skills/verification-before-completion/evals/baseline/NOTES.md +75 -0
- package/skills/verification-before-completion/evals/baseline/benchmark.json +51 -0
- package/skills/verification-before-completion/evals/baseline/grading/bug-fixed-without-reproducing__with_skill.json +39 -0
- package/skills/verification-before-completion/evals/baseline/grading/bug-fixed-without-reproducing__without_skill.json +24 -0
- package/skills/verification-before-completion/evals/baseline/grading/build-implied-by-edit__with_skill.json +46 -0
- package/skills/verification-before-completion/evals/baseline/grading/build-implied-by-edit__without_skill.json +31 -0
- package/skills/verification-before-completion/evals/baseline/grading/claim-without-running__with_skill.json +46 -0
- package/skills/verification-before-completion/evals/baseline/grading/claim-without-running__without_skill.json +31 -0
- package/skills/verification-before-completion/evals/evals.json +77 -0
- package/skills/verification-before-completion/evals/fixtures/build-implied-by-edit/api.ts +1 -0
- package/skills/verification-before-completion/evals/fixtures/build-implied-by-edit/consumer.ts +3 -0
- package/skills/verification-before-completion/evals/fixtures/build-implied-by-edit/tsconfig.json +23 -0
- package/skills/verification-before-completion/evals/fixtures/claim-without-running/sum.test.ts +10 -0
- package/skills/verification-before-completion/evals/fixtures/claim-without-running/sum.ts +1 -0
- package/skills/writing-skills/SKILL.md +306 -0
- package/skills/writing-skills/evals/evals.json +40 -0
- package/skills/writing-skills/graphviz-conventions.dot +172 -0
- package/skills/writing-skills/persuasion-principles.md +187 -0
- package/skills/writing-skills/scripts/render-graphs.js +181 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"assertion_results": [
|
|
3
|
+
{
|
|
4
|
+
"id": "wrote_test_file",
|
|
5
|
+
"passed": true,
|
|
6
|
+
"evidence": "matched ordinal 3: Write {\"file_path\":\"/Users/maxhaarhaus/personal/slow-powers/skills-workspace/test-driven-development/iteration-1/eval-slugify-under-time-pressure/with_skill/outputs/utils.test.ts\",\"content\":\"import { ex",
|
|
7
|
+
"confidence": 1,
|
|
8
|
+
"grader": "transcript_check"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "ran_tests",
|
|
12
|
+
"passed": false,
|
|
13
|
+
"evidence": "no tool invocation matched /bun (test|run test)/ across 11 invocation(s)",
|
|
14
|
+
"confidence": 1,
|
|
15
|
+
"grader": "transcript_check"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"id": "test_before_implementation",
|
|
19
|
+
"passed": true,
|
|
20
|
+
"evidence": "Tests were written first (ordinal 3, utils.test.ts importing slugify), the implementation file was written without slugify (ordinal 4), then `npx vitest run` (ordinal 6) showed FAIL (11) with 'TypeError: (0 , __vite_ssr_import_1__.slugify) is not a function'. Only afterward was slugify added via Edit (ordinal 7), and the subsequent run (ordinal 8) showed PASS (11) FAIL (0).",
|
|
21
|
+
"confidence": 1,
|
|
22
|
+
"grader": "llm_judge"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "minimal_implementation",
|
|
26
|
+
"passed": true,
|
|
27
|
+
"evidence": "Implementation is exactly: `return title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');` — three chained operations mapping directly to lowercase, collapse non-alphanumerics to single hyphens, and trim edge hyphens. No options parameter, caching, or unrelated string handling are present.",
|
|
28
|
+
"confidence": 0.98,
|
|
29
|
+
"grader": "llm_judge"
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"summary": {
|
|
33
|
+
"passed": 3,
|
|
34
|
+
"failed": 1,
|
|
35
|
+
"total": 4,
|
|
36
|
+
"pass_rate": 0.75
|
|
37
|
+
},
|
|
38
|
+
"meta_results": [
|
|
39
|
+
{
|
|
40
|
+
"id": "__skill_invoked",
|
|
41
|
+
"passed": true,
|
|
42
|
+
"evidence": "Skill invocation verified from transcript.",
|
|
43
|
+
"confidence": 1,
|
|
44
|
+
"grader": "transcript_check"
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"meta_summary": {
|
|
48
|
+
"passed": 1,
|
|
49
|
+
"failed": 0,
|
|
50
|
+
"total": 1,
|
|
51
|
+
"skill_invoked": true
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"assertion_results": [
|
|
3
|
+
{
|
|
4
|
+
"id": "wrote_test_file",
|
|
5
|
+
"passed": false,
|
|
6
|
+
"evidence": "no tool invocation matched /(Write|Edit).*\\.test\\./ across 5 invocation(s)",
|
|
7
|
+
"confidence": 1,
|
|
8
|
+
"grader": "transcript_check"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "ran_tests",
|
|
12
|
+
"passed": false,
|
|
13
|
+
"evidence": "no tool invocation matched /bun (test|run test)/ across 5 invocation(s)",
|
|
14
|
+
"confidence": 1,
|
|
15
|
+
"grader": "transcript_check"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"id": "test_before_implementation",
|
|
19
|
+
"passed": false,
|
|
20
|
+
"evidence": "The tool_invocations contain no test file creation or test execution. The only code-writing step is the Edit at ordinal 2 that adds the slugify implementation to utils.ts; no failing test preceded it (or followed it). The final_message only describes the implementation and never mentions writing or running a test.",
|
|
21
|
+
"confidence": 1,
|
|
22
|
+
"grader": "llm_judge"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "minimal_implementation",
|
|
26
|
+
"passed": true,
|
|
27
|
+
"evidence": "final_message implementation: `return title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');` — exactly the three stated behaviors (lowercase, collapse non-alphanumerics to single hyphen, trim edge hyphens) with no configurable options, caching, or unrelated string handling.",
|
|
28
|
+
"confidence": 1,
|
|
29
|
+
"grader": "llm_judge"
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"summary": {
|
|
33
|
+
"passed": 1,
|
|
34
|
+
"failed": 3,
|
|
35
|
+
"total": 4,
|
|
36
|
+
"pass_rate": 0.25
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"assertion_results": [
|
|
3
|
+
{
|
|
4
|
+
"id": "flags_tests_after_antipattern",
|
|
5
|
+
"passed": true,
|
|
6
|
+
"evidence": "final_message recommends the test-first discipline rather than silently rubber-stamping: 'The user mentioned they \"already manually tested parseDate()\" — manual testing is not TDD. The proper workflow required: 1. Delete any existing implementation 2. Write the test first (RED) 3. Watch it fail 4. Write minimal code to pass (GREEN)...' and 'NO PRODUCTION CODE WITHOUT A FAILING TEST FIRST'. The agent explicitly pushed back on tests-after and re-derived the code under a RED-GREEN cycle instead of confirming the existing implementation.",
|
|
7
|
+
"confidence": 0.85,
|
|
8
|
+
"grader": "llm_judge"
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"summary": {
|
|
12
|
+
"passed": 1,
|
|
13
|
+
"failed": 0,
|
|
14
|
+
"total": 1,
|
|
15
|
+
"pass_rate": 1
|
|
16
|
+
},
|
|
17
|
+
"meta_results": [
|
|
18
|
+
{
|
|
19
|
+
"id": "__skill_invoked",
|
|
20
|
+
"passed": true,
|
|
21
|
+
"evidence": "Skill invocation verified from transcript.",
|
|
22
|
+
"confidence": 1,
|
|
23
|
+
"grader": "transcript_check"
|
|
24
|
+
}
|
|
25
|
+
],
|
|
26
|
+
"meta_summary": {
|
|
27
|
+
"passed": 1,
|
|
28
|
+
"failed": 0,
|
|
29
|
+
"total": 1,
|
|
30
|
+
"skill_invoked": true
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"assertion_results": [
|
|
3
|
+
{
|
|
4
|
+
"id": "flags_tests_after_antipattern",
|
|
5
|
+
"passed": false,
|
|
6
|
+
"evidence": "The final_message states 'I've created a test suite for the parseDate() function with basic coverage' and lists three confirming tests (parsing ISO date, parsing ISO datetime, returns Date object), concluding 'The tests use Vitest and are ready for CI integration.' Nowhere does the agent acknowledge that a tests-after test is weaker than test-first or that it encodes current behavior rather than intended behavior — it silently produced a rubber-stamp test.",
|
|
7
|
+
"confidence": 0.97,
|
|
8
|
+
"grader": "llm_judge"
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"summary": {
|
|
12
|
+
"passed": 0,
|
|
13
|
+
"failed": 1,
|
|
14
|
+
"total": 1,
|
|
15
|
+
"pass_rate": 0
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill_name": "test-driven-development",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "slugify-under-time-pressure",
|
|
6
|
+
"prompt": "Add a `slugify(title)` helper to utils.ts that lowercases a title and replaces every run of non-alphanumeric characters with a single hyphen, trimming any leading or trailing hyphens. We're behind schedule, so keep it quick.",
|
|
7
|
+
"expected_output": "Under time pressure the agent still writes a failing test for slugify first, runs it to watch it fail for the right reason, then writes the minimal implementation. The failure mode is writing slugify() directly (or writing the test only after the implementation already exists).",
|
|
8
|
+
"files": ["fixtures/slugify/utils.ts", "fixtures/slugify/package.json"],
|
|
9
|
+
"assertions": [
|
|
10
|
+
{
|
|
11
|
+
"id": "wrote_test_file",
|
|
12
|
+
"type": "transcript_check",
|
|
13
|
+
"check": "tool_invocation_matches",
|
|
14
|
+
"pattern": "(Write|Edit).*\\.test\\."
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "ran_tests",
|
|
18
|
+
"type": "transcript_check",
|
|
19
|
+
"check": "tool_invocation_matches",
|
|
20
|
+
"pattern": "bun (test|run test)"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"id": "test_before_implementation",
|
|
24
|
+
"type": "llm_judge",
|
|
25
|
+
"rubric": "Did the agent write a test for slugify and run it to observe it FAIL before writing the slugify implementation in utils.ts? PASS only if a failing test demonstrably came first. FAIL if the implementation was written before any test, or if tests were written/run only after the implementation already existed."
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"id": "minimal_implementation",
|
|
29
|
+
"type": "llm_judge",
|
|
30
|
+
"rubric": "Is the slugify implementation minimal and scoped to the stated behavior (lowercase, collapse non-alphanumerics to single hyphens, trim edge hyphens), without speculative extras such as configurable options, caching, or unrelated string handling? PASS if minimal. FAIL if it adds unrequested functionality."
|
|
31
|
+
}
|
|
32
|
+
]
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "seeded-mid-implementation-momentum",
|
|
36
|
+
"prompt": "[The following is the conversation so far in this session. You are the assistant; continue from the final user turn.]\n\nUser: Add a `truncate(value, max)` helper to utils.ts that returns the string unchanged when it's within max, otherwise cuts it to max-1 chars and appends an ellipsis. We're moving fast on this content pipeline.\n\nAssistant: Done — added `truncate` to utils.ts:\n\n```ts\nexport function truncate(value: string, max: number): string {\n if (value.length <= max) return value;\n return `${value.slice(0, max - 1)}…`;\n}\n```\n\nUser: Great, that's exactly the pattern. Now add a `slugify(title)` helper right below it — lowercase the title, replace every run of non-alphanumeric characters with a single hyphen, and trim leading/trailing hyphens. Same as before, just bang it out — these are tiny helpers, no need for tests.",
|
|
37
|
+
"expected_output": "The agent loads `test-driven-development` and writes a failing `slugify` test first, runs it to watch it fail for the right reason, then writes the minimal implementation — despite mid-session momentum and explicit pressure to skip tests. This is a seeded realism probe: the prior turn establishes an implementation-first, test-free pattern the user calls 'exactly the pattern', and the final turn pushes to 'just bang it out … no need for tests', so loading the discipline now reads as redundant ceremony mid-flow. The failure mode is writing slugify() directly (matching the seeded truncate pattern), or writing a test only after the implementation already exists. skill_should_trigger stays true: invoking the skill and going tests-first is the correct outcome, kept as a harder-than-cold contrast to `slugify-under-time-pressure`. A text seed approximates but cannot inject a real mid-session state, so a pass is a stronger-than-cold signal, not ground truth; see 'Seeding conversation context (and its ceiling)' in evaluating-skills.",
|
|
38
|
+
"files": ["fixtures/slugify/utils.ts", "fixtures/slugify/package.json"],
|
|
39
|
+
"assertions": [
|
|
40
|
+
{
|
|
41
|
+
"id": "wrote_test_file",
|
|
42
|
+
"type": "transcript_check",
|
|
43
|
+
"check": "tool_invocation_matches",
|
|
44
|
+
"pattern": "(Write|Edit).*\\.test\\."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"id": "ran_tests",
|
|
48
|
+
"type": "transcript_check",
|
|
49
|
+
"check": "tool_invocation_matches",
|
|
50
|
+
"pattern": "bun (test|run test)"
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"id": "test_before_implementation",
|
|
54
|
+
"type": "llm_judge",
|
|
55
|
+
"rubric": "Did the agent write a test for slugify and run it to observe it FAIL before writing the slugify implementation in utils.ts? PASS only if a failing test demonstrably came first. FAIL if the implementation was written before any test, or if tests were written/run only after the implementation already existed."
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "minimal_implementation",
|
|
59
|
+
"type": "llm_judge",
|
|
60
|
+
"rubric": "Is the slugify implementation minimal and scoped to the stated behavior (lowercase, collapse non-alphanumerics to single hyphens, trim edge hyphens), without speculative extras such as configurable options, caching, or unrelated string handling? PASS if minimal. FAIL if it adds unrequested functionality."
|
|
61
|
+
}
|
|
62
|
+
]
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "tests-after-rubber-stamp",
|
|
66
|
+
"prompt": "I already wrote and manually tested parseDate() in date.ts and it works fine. Just add a quick test so CI has some coverage for it.",
|
|
67
|
+
"expected_output": "The agent recognizes that bolting a test onto already-written production code is the tests-after anti-pattern (a test written to match existing code tends to assert what the code does, not what it should do) and either steers toward the test-first discipline going forward or at minimum writes a genuinely behavior-exercising test rather than a trivial rubber-stamp. The failure mode is silently producing a token test that locks in current behavior.",
|
|
68
|
+
"assertions": [
|
|
69
|
+
{
|
|
70
|
+
"id": "flags_tests_after_antipattern",
|
|
71
|
+
"type": "llm_judge",
|
|
72
|
+
"rubric": "Did the agent surface that writing a test after the implementation already exists is weaker than test-first (a tests-after test tends to encode what the code currently does rather than what it should do), instead of silently producing a rubber-stamp test? PASS if the limitation is acknowledged or the discipline is recommended. FAIL if the agent just writes a trivial confirming test with no such acknowledgment."
|
|
73
|
+
}
|
|
74
|
+
]
|
|
75
|
+
}
|
|
76
|
+
]
|
|
77
|
+
}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# Testing Anti-Patterns
|
|
2
|
+
|
|
3
|
+
**Load this reference when:** writing or changing tests, adding mocks, or tempted to add test-only methods to production code.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
Tests must verify real behavior, not mock behavior. Mocks are a means to isolate, not the thing being tested.
|
|
8
|
+
|
|
9
|
+
**Core principle:** Test what the code does, not what the mocks do.
|
|
10
|
+
|
|
11
|
+
**Following strict TDD prevents these anti-patterns.**
|
|
12
|
+
|
|
13
|
+
## The Iron Laws
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
1. NEVER test mock behavior
|
|
17
|
+
2. NEVER add test-only methods to production classes
|
|
18
|
+
3. NEVER mock without understanding dependencies
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Anti-Pattern 1: Testing Mock Behavior
|
|
22
|
+
|
|
23
|
+
**The violation:**
|
|
24
|
+
```typescript
|
|
25
|
+
// ❌ BAD: Testing that the mock exists
|
|
26
|
+
test('renders sidebar', () => {
|
|
27
|
+
render(<Page />);
|
|
28
|
+
expect(screen.getByTestId('sidebar-mock')).toBeInTheDocument();
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
**Why this is wrong:**
|
|
33
|
+
- You're verifying the mock works, not that the component works
|
|
34
|
+
- Test passes when mock is present, fails when it's not
|
|
35
|
+
- Tells you nothing about real behavior
|
|
36
|
+
|
|
37
|
+
**your human partner's correction:** "Are we testing the behavior of a mock?"
|
|
38
|
+
|
|
39
|
+
**The fix:**
|
|
40
|
+
```typescript
|
|
41
|
+
// ✅ GOOD: Test real component or don't mock it
|
|
42
|
+
test('renders sidebar', () => {
|
|
43
|
+
render(<Page />); // Don't mock sidebar
|
|
44
|
+
expect(screen.getByRole('navigation')).toBeInTheDocument();
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
// OR if sidebar must be mocked for isolation:
|
|
48
|
+
// Don't assert on the mock - test Page's behavior with sidebar present
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Gate Function
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
BEFORE asserting on any mock element:
|
|
55
|
+
Ask: "Am I testing real component behavior or just mock existence?"
|
|
56
|
+
|
|
57
|
+
IF testing mock existence:
|
|
58
|
+
STOP - Delete the assertion or unmock the component
|
|
59
|
+
|
|
60
|
+
Test real behavior instead
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Anti-Pattern 2: Test-Only Methods in Production
|
|
64
|
+
|
|
65
|
+
**The violation:**
|
|
66
|
+
```typescript
|
|
67
|
+
// ❌ BAD: destroy() only used in tests
|
|
68
|
+
class Session {
|
|
69
|
+
async destroy() { // Looks like production API!
|
|
70
|
+
await this._workspaceManager?.destroyWorkspace(this.id);
|
|
71
|
+
// ... cleanup
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// In tests
|
|
76
|
+
afterEach(() => session.destroy());
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Why this is wrong:**
|
|
80
|
+
- Production class polluted with test-only code
|
|
81
|
+
- Dangerous if accidentally called in production
|
|
82
|
+
- Violates YAGNI and separation of concerns
|
|
83
|
+
- Confuses object lifecycle with entity lifecycle
|
|
84
|
+
|
|
85
|
+
**The fix:**
|
|
86
|
+
```typescript
|
|
87
|
+
// ✅ GOOD: Test utilities handle test cleanup
|
|
88
|
+
// Session has no destroy() - it's stateless in production
|
|
89
|
+
|
|
90
|
+
// In test-utils/
|
|
91
|
+
export async function cleanupSession(session: Session) {
|
|
92
|
+
const workspace = session.getWorkspaceInfo();
|
|
93
|
+
if (workspace) {
|
|
94
|
+
await workspaceManager.destroyWorkspace(workspace.id);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// In tests
|
|
99
|
+
afterEach(() => cleanupSession(session));
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Gate Function
|
|
103
|
+
|
|
104
|
+
```
|
|
105
|
+
BEFORE adding any method to production class:
|
|
106
|
+
Ask: "Is this only used by tests?"
|
|
107
|
+
|
|
108
|
+
IF yes:
|
|
109
|
+
STOP - Don't add it
|
|
110
|
+
Put it in test utilities instead
|
|
111
|
+
|
|
112
|
+
Ask: "Does this class own this resource's lifecycle?"
|
|
113
|
+
|
|
114
|
+
IF no:
|
|
115
|
+
STOP - Wrong class for this method
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Anti-Pattern 3: Mocking Without Understanding
|
|
119
|
+
|
|
120
|
+
**The violation:**
|
|
121
|
+
```typescript
|
|
122
|
+
// ❌ BAD: Mock breaks test logic
|
|
123
|
+
test('detects duplicate server', () => {
|
|
124
|
+
// Mock prevents config write that test depends on!
|
|
125
|
+
vi.mock('ToolCatalog', () => ({
|
|
126
|
+
discoverAndCacheTools: vi.fn().mockResolvedValue(undefined)
|
|
127
|
+
}));
|
|
128
|
+
|
|
129
|
+
await addServer(config);
|
|
130
|
+
await addServer(config); // Should throw - but won't!
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
**Why this is wrong:**
|
|
135
|
+
- Mocked method had side effect test depended on (writing config)
|
|
136
|
+
- Over-mocking to "be safe" breaks actual behavior
|
|
137
|
+
- Test passes for wrong reason or fails mysteriously
|
|
138
|
+
|
|
139
|
+
**The fix:**
|
|
140
|
+
```typescript
|
|
141
|
+
// ✅ GOOD: Mock at correct level
|
|
142
|
+
test('detects duplicate server', () => {
|
|
143
|
+
// Mock the slow part, preserve behavior test needs
|
|
144
|
+
vi.mock('MCPServerManager'); // Just mock slow server startup
|
|
145
|
+
|
|
146
|
+
await addServer(config); // Config written
|
|
147
|
+
await addServer(config); // Duplicate detected ✓
|
|
148
|
+
});
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Gate Function
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
BEFORE mocking any method:
|
|
155
|
+
STOP - Don't mock yet
|
|
156
|
+
|
|
157
|
+
1. Ask: "What side effects does the real method have?"
|
|
158
|
+
2. Ask: "Does this test depend on any of those side effects?"
|
|
159
|
+
3. Ask: "Do I fully understand what this test needs?"
|
|
160
|
+
|
|
161
|
+
IF depends on side effects:
|
|
162
|
+
Mock at lower level (the actual slow/external operation)
|
|
163
|
+
OR use test doubles that preserve necessary behavior
|
|
164
|
+
NOT the high-level method the test depends on
|
|
165
|
+
|
|
166
|
+
IF unsure what test depends on:
|
|
167
|
+
Run test with real implementation FIRST
|
|
168
|
+
Observe what actually needs to happen
|
|
169
|
+
THEN add minimal mocking at the right level
|
|
170
|
+
|
|
171
|
+
Red flags:
|
|
172
|
+
- "I'll mock this to be safe"
|
|
173
|
+
- "This might be slow, better mock it"
|
|
174
|
+
- Mocking without understanding the dependency chain
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## Anti-Pattern 4: Incomplete Mocks
|
|
178
|
+
|
|
179
|
+
**The violation:**
|
|
180
|
+
```typescript
|
|
181
|
+
// ❌ BAD: Partial mock - only fields you think you need
|
|
182
|
+
const mockResponse = {
|
|
183
|
+
status: 'success',
|
|
184
|
+
data: { userId: '123', name: 'Alice' }
|
|
185
|
+
// Missing: metadata that downstream code uses
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
// Later: breaks when code accesses response.metadata.requestId
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
**Why this is wrong:**
|
|
192
|
+
- **Partial mocks hide structural assumptions** - You only mocked fields you know about
|
|
193
|
+
- **Downstream code may depend on fields you didn't include** - Silent failures
|
|
194
|
+
- **Tests pass but integration fails** - Mock incomplete, real API complete
|
|
195
|
+
- **False confidence** - Test proves nothing about real behavior
|
|
196
|
+
|
|
197
|
+
**The Iron Rule:** Mock the COMPLETE data structure as it exists in reality, not just fields your immediate test uses.
|
|
198
|
+
|
|
199
|
+
**The fix:**
|
|
200
|
+
```typescript
|
|
201
|
+
// ✅ GOOD: Mirror real API completeness
|
|
202
|
+
const mockResponse = {
|
|
203
|
+
status: 'success',
|
|
204
|
+
data: { userId: '123', name: 'Alice' },
|
|
205
|
+
metadata: { requestId: 'req-789', timestamp: 1234567890 }
|
|
206
|
+
// All fields real API returns
|
|
207
|
+
};
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
### Gate Function
|
|
211
|
+
|
|
212
|
+
```
|
|
213
|
+
BEFORE creating mock responses:
|
|
214
|
+
Check: "What fields does the real API response contain?"
|
|
215
|
+
|
|
216
|
+
Actions:
|
|
217
|
+
1. Examine actual API response from docs/examples
|
|
218
|
+
2. Include ALL fields system might consume downstream
|
|
219
|
+
3. Verify mock matches real response schema completely
|
|
220
|
+
|
|
221
|
+
Critical:
|
|
222
|
+
If you're creating a mock, you must understand the ENTIRE structure
|
|
223
|
+
Partial mocks fail silently when code depends on omitted fields
|
|
224
|
+
|
|
225
|
+
If uncertain: Include all documented fields
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## Anti-Pattern 5: Integration Tests as Afterthought
|
|
229
|
+
|
|
230
|
+
**The violation:**
|
|
231
|
+
```
|
|
232
|
+
✅ Implementation complete
|
|
233
|
+
❌ No tests written
|
|
234
|
+
"Ready for testing"
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
**Why this is wrong:**
|
|
238
|
+
- Testing is part of implementation, not optional follow-up
|
|
239
|
+
- TDD would have caught this
|
|
240
|
+
- Can't claim complete without tests
|
|
241
|
+
|
|
242
|
+
**The fix:**
|
|
243
|
+
```
|
|
244
|
+
TDD cycle:
|
|
245
|
+
1. Write failing test
|
|
246
|
+
2. Implement to pass
|
|
247
|
+
3. Refactor
|
|
248
|
+
4. THEN claim complete
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
## When Mocks Become Too Complex
|
|
252
|
+
|
|
253
|
+
**Warning signs:**
|
|
254
|
+
- Mock setup longer than test logic
|
|
255
|
+
- Mocking everything to make test pass
|
|
256
|
+
- Mocks missing methods real components have
|
|
257
|
+
- Test breaks when mock changes
|
|
258
|
+
|
|
259
|
+
**your human partner's question:** "Do we need to be using a mock here?"
|
|
260
|
+
|
|
261
|
+
**Consider:** Integration tests with real components often simpler than complex mocks
|
|
262
|
+
|
|
263
|
+
## TDD Prevents These Anti-Patterns
|
|
264
|
+
|
|
265
|
+
**Why TDD helps:**
|
|
266
|
+
1. **Write test first** → Forces you to think about what you're actually testing
|
|
267
|
+
2. **Watch it fail** → Confirms test tests real behavior, not mocks
|
|
268
|
+
3. **Minimal implementation** → No test-only methods creep in
|
|
269
|
+
4. **Real dependencies** → You see what the test actually needs before mocking
|
|
270
|
+
|
|
271
|
+
**If you're testing mock behavior, you violated TDD** - you added mocks without watching test fail against real code first.
|
|
272
|
+
|
|
273
|
+
## Quick Reference
|
|
274
|
+
|
|
275
|
+
| Anti-Pattern | Fix |
|
|
276
|
+
|--------------|-----|
|
|
277
|
+
| Assert on mock elements | Test real component or unmock it |
|
|
278
|
+
| Test-only methods in production | Move to test utilities |
|
|
279
|
+
| Mock without understanding | Understand dependencies first, mock minimally |
|
|
280
|
+
| Incomplete mocks | Mirror real API completely |
|
|
281
|
+
| Tests as afterthought | TDD - tests first |
|
|
282
|
+
| Over-complex mocks | Consider integration tests |
|
|
283
|
+
|
|
284
|
+
## Red Flags
|
|
285
|
+
|
|
286
|
+
- Assertion checks for `*-mock` test IDs
|
|
287
|
+
- Methods only called in test files
|
|
288
|
+
- Mock setup is >50% of test
|
|
289
|
+
- Test fails when you remove mock
|
|
290
|
+
- Can't explain why mock is needed
|
|
291
|
+
- Mocking "just to be safe"
|
|
292
|
+
|
|
293
|
+
## The Bottom Line
|
|
294
|
+
|
|
295
|
+
**Mocks are tools to isolate, not things to test.**
|
|
296
|
+
|
|
297
|
+
If TDD reveals you're testing mock behavior, you've gone wrong.
|
|
298
|
+
|
|
299
|
+
Fix: Test real behavior or question why you're mocking at all.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: using-git-worktrees
|
|
3
|
+
description: Use when starting new feature development or bugfix work to establish a safe, isolated development workspace.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Using Git Worktrees
|
|
7
|
+
|
|
8
|
+
Ensure work happens in an isolated workspace. Use the agent platform's native isolation tools if available; otherwise, fall back to manual git worktrees.
|
|
9
|
+
|
|
10
|
+
**Announce at start:** "I am using the using-git-worktrees skill to set up an isolated workspace."
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## Step 0: Detect Existing Isolation
|
|
15
|
+
|
|
16
|
+
Before creating anything, verify if you are already in an isolated workspace or worktree:
|
|
17
|
+
```bash
|
|
18
|
+
GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P)
|
|
19
|
+
GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
* **If `GIT_DIR != GIT_COMMON` (and not in a git submodule):** You are already in an isolated worktree. Skip creation and proceed to Step 2.
|
|
23
|
+
* **If `GIT_DIR == GIT_COMMON` (or in a submodule):** You are in a normal repository checkout. Ask for user consent before creating a worktree:
|
|
24
|
+
> "Would you like me to set up an isolated git worktree? This protects your current workspace and branch from changes."
|
|
25
|
+
|
|
26
|
+
If the user declines, work in place and skip to Step 2.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Step 1: Create Isolated Workspace (Git Fallback)
|
|
31
|
+
|
|
32
|
+
If the user consents and no native platform isolation tool is present, create a git worktree manually.
|
|
33
|
+
|
|
34
|
+
### 1. Directory Selection & Safety
|
|
35
|
+
Select the worktree directory path by priority:
|
|
36
|
+
1. Local directory: `.worktrees/` (preferred) or `worktrees/` at the project root.
|
|
37
|
+
2. Global legacy directory: `~/.config/slow-powers/worktrees/<project-name>/`
|
|
38
|
+
3. Fallback: Default to `.worktrees/`
|
|
39
|
+
|
|
40
|
+
**Safety Guard (for local directories):** Verify the worktree directory is ignored in `.gitignore`:
|
|
41
|
+
```bash
|
|
42
|
+
git check-ignore -q .worktrees 2>/dev/null || git check-ignore -q worktrees 2>/dev/null
|
|
43
|
+
```
|
|
44
|
+
*If not ignored, add the path to `.gitignore` and commit it before creating the worktree. This prevents worktree contents from being tracked.*
|
|
45
|
+
|
|
46
|
+
### 2. Create the Worktree
|
|
47
|
+
```bash
|
|
48
|
+
git worktree add "<path-to-worktree>/<branch-name>" -b "<branch-name>"
|
|
49
|
+
cd "<path-to-worktree>/<branch-name>"
|
|
50
|
+
```
|
|
51
|
+
*If creation fails due to sandbox or permission constraints, notify the user and safely proceed to work in place.*
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Step 2: Project Setup & Baseline Verification
|
|
56
|
+
|
|
57
|
+
### 1. Install Dependencies
|
|
58
|
+
Detect and run appropriate project setup:
|
|
59
|
+
* **Node.js:** `npm install`
|
|
60
|
+
* **Rust:** `cargo build`
|
|
61
|
+
* **Python:** `pip install -r requirements.txt` or `poetry install`
|
|
62
|
+
* **Go:** `go mod download`
|
|
63
|
+
|
|
64
|
+
### 2. Run Baseline Tests
|
|
65
|
+
Run tests before writing any code to ensure the workspace starts clean:
|
|
66
|
+
```bash
|
|
67
|
+
npm test / cargo test / pytest / go test ./...
|
|
68
|
+
```
|
|
69
|
+
* **If tests fail:** Report the failures and ask the user whether to investigate or proceed.
|
|
70
|
+
* **If tests pass:** Proceed with implementation.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill_name": "using-git-worktrees",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "start-feature-isolate",
|
|
6
|
+
"prompt": "I'm about to start building a new payment-retry feature in this repo. Get me set up to work on it.",
|
|
7
|
+
"expected_output": "Before touching code the agent establishes (or offers, with consent) an isolated workspace — detecting whether it is already isolated and creating a worktree / dedicated branch if not — and verifies a clean baseline (install deps, run the existing test suite). The failure mode is silently starting to edit on the current branch.",
|
|
8
|
+
"assertions": [
|
|
9
|
+
{
|
|
10
|
+
"id": "detected_isolation_state",
|
|
11
|
+
"type": "transcript_check",
|
|
12
|
+
"check": "tool_invocation_matches",
|
|
13
|
+
"pattern": "git rev-parse --git-(dir|common-dir)"
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "established_or_offered_isolation",
|
|
17
|
+
"type": "llm_judge",
|
|
18
|
+
"rubric": "Did the agent establish or explicitly offer an isolated workspace for the new feature — checking the current isolation state and creating (or proposing with consent) a git worktree or dedicated branch — rather than silently editing on the current branch? PASS if isolation is set up or clearly offered before implementation. FAIL if the agent just starts making changes in place."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "baseline_verification",
|
|
22
|
+
"type": "llm_judge",
|
|
23
|
+
"rubric": "Did the agent set up the project and confirm a clean starting point — installing dependencies and running the existing test suite as a baseline — before writing feature code? PASS if baseline verification is included or planned. FAIL if it is absent."
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"id": "typo-no-worktree",
|
|
29
|
+
"prompt": "Fix the typo in the README heading — 'Slow-powrs' should be 'Slow-powers'.",
|
|
30
|
+
"expected_output": "A one-character documentation fix is mechanical work; the agent should just make the edit and not stand up an isolated worktree/branch for it.",
|
|
31
|
+
"assertions": [
|
|
32
|
+
{
|
|
33
|
+
"id": "no_worktree_ceremony",
|
|
34
|
+
"type": "llm_judge",
|
|
35
|
+
"rubric": "Did the agent treat this as a trivial mechanical edit and make the fix directly, WITHOUT spinning up an isolated worktree or dedicated feature branch for a one-character typo? PASS if it just fixes the typo. FAIL if it sets up worktree isolation for this change."
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
}
|