@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,63 +1,63 @@
1
- ---
2
- name: statistical-inference
3
- description: Reasoning from data to a claim without overstating it — effect sizes, uncertainty, multiplicity, and what a p-value does not say
4
- additional_tools: [probability_stats, symbolic_math]
5
- ---
6
-
7
- ## Statistical inference methodology
8
-
9
- ### Report the effect, then the uncertainty, then the test
10
-
11
- The size of the difference is the finding. The interval around it is how
12
- much to trust the finding. A test statistic is the least informative of
13
- the three and is routinely reported alone. Lead with "3.2 points better,
14
- 95% CI [0.4, 6.0]", not with "significant".
15
-
16
- ### Say what the p-value is about
17
-
18
- It is the probability of data at least this extreme under the null
19
- hypothesis. It is not the probability that the null is true, not the
20
- probability the result will replicate, and not a measure of effect size. A
21
- tiny p-value from a huge sample can accompany an effect too small to
22
- matter, and that combination is one of the most common ways a true
23
- statement misleads.
24
-
25
- ### Count every comparison you made
26
-
27
- Testing twenty things at the 0.05 level and reporting the one that came
28
- out is not a finding, whether the other nineteen were preregistered,
29
- exploratory, or merely looked at. Say how many comparisons were made and
30
- correct for them, or label the result exploratory and say it needs
31
- confirmation on new data.
32
-
33
- Subgroups, alternative metrics, and "we also tried" all count.
34
-
35
- ### Check the assumptions the method actually makes
36
-
37
- Independence is the one that fails most often and is questioned least:
38
- repeated measures on the same subject, runs on the same machine, samples
39
- drawn from one crawl. Normality usually matters less than people fear;
40
- independence usually matters more.
41
-
42
- Say which assumptions you checked and which you assumed.
43
-
44
- ### Absence of significance is not evidence of absence
45
-
46
- A non-significant result with a wide interval is compatible with a large
47
- effect. Report the interval and say what the study could have detected. If
48
- you mean "we can rule out an effect larger than X", say that — it is a
49
- much stronger and much more useful claim, and it requires the interval to
50
- support it.
51
-
52
- ### Keep the estimate separate from the decision
53
-
54
- "How large is the effect" and "what should we do" are different questions
55
- with different inputs. A threshold that decides an action belongs to the
56
- decision, with its costs stated, not smuggled into the analysis as though
57
- it were a property of the data.
58
-
59
- ### Distinguish measured, modelled and assumed
60
-
61
- At each number, say which it is. A modelled quantity presented in the same
62
- voice as a measured one is the most reliable way to make an analysis look
63
- sturdier than it is.
1
+ ---
2
+ name: statistical-inference
3
+ description: Reasoning from data to a claim without overstating it — effect sizes, uncertainty, multiplicity, and what a p-value does not say
4
+ additional_tools: [probability_stats, symbolic_math]
5
+ ---
6
+
7
+ ## Statistical inference methodology
8
+
9
+ ### Report the effect, then the uncertainty, then the test
10
+
11
+ The size of the difference is the finding. The interval around it is how
12
+ much to trust the finding. A test statistic is the least informative of
13
+ the three and is routinely reported alone. Lead with "3.2 points better,
14
+ 95% CI [0.4, 6.0]", not with "significant".
15
+
16
+ ### Say what the p-value is about
17
+
18
+ It is the probability of data at least this extreme under the null
19
+ hypothesis. It is not the probability that the null is true, not the
20
+ probability the result will replicate, and not a measure of effect size. A
21
+ tiny p-value from a huge sample can accompany an effect too small to
22
+ matter, and that combination is one of the most common ways a true
23
+ statement misleads.
24
+
25
+ ### Count every comparison you made
26
+
27
+ Testing twenty things at the 0.05 level and reporting the one that came
28
+ out is not a finding, whether the other nineteen were preregistered,
29
+ exploratory, or merely looked at. Say how many comparisons were made and
30
+ correct for them, or label the result exploratory and say it needs
31
+ confirmation on new data.
32
+
33
+ Subgroups, alternative metrics, and "we also tried" all count.
34
+
35
+ ### Check the assumptions the method actually makes
36
+
37
+ Independence is the one that fails most often and is questioned least:
38
+ repeated measures on the same subject, runs on the same machine, samples
39
+ drawn from one crawl. Normality usually matters less than people fear;
40
+ independence usually matters more.
41
+
42
+ Say which assumptions you checked and which you assumed.
43
+
44
+ ### Absence of significance is not evidence of absence
45
+
46
+ A non-significant result with a wide interval is compatible with a large
47
+ effect. Report the interval and say what the study could have detected. If
48
+ you mean "we can rule out an effect larger than X", say that — it is a
49
+ much stronger and much more useful claim, and it requires the interval to
50
+ support it.
51
+
52
+ ### Keep the estimate separate from the decision
53
+
54
+ "How large is the effect" and "what should we do" are different questions
55
+ with different inputs. A threshold that decides an action belongs to the
56
+ decision, with its costs stated, not smuggled into the analysis as though
57
+ it were a property of the data.
58
+
59
+ ### Distinguish measured, modelled and assumed
60
+
61
+ At each number, say which it is. A modelled quantity presented in the same
62
+ voice as a measured one is the most reliable way to make an analysis look
63
+ sturdier than it is.
@@ -1,59 +1,59 @@
1
- ---
2
- name: technical-writing
3
- description: Writing up a result so a reader can act on it — the claim first, the evidence attached, and the limitation stated rather than implied
4
- additional_tools: [read, web_search]
5
- ---
6
-
7
- ## Technical writing methodology
8
-
9
- ### Lead with the finding
10
-
11
- The first sentence should be the thing you learned, not the thing you did.
12
- "Compaction folded a 1M-window thread at 4% of its window" is a finding;
13
- "we investigated compaction behaviour" is a table of contents. A reader
14
- who stops after one paragraph should still have the result.
15
-
16
- ### Attach the evidence to the claim
17
-
18
- Every non-obvious assertion carries, in the same place, what supports it:
19
- the measurement, the file and line, the citation, the command. Evidence
20
- gathered into a section at the end forces the reader to reconstruct which
21
- support belongs to which claim, and most will not.
22
-
23
- ### State limitations where they bite
24
-
25
- A limitation section at the end is read as a formality. The same sentence
26
- placed beside the claim it qualifies changes how the claim is read, which
27
- is the point of writing it at all. "On this dataset only" belongs in the
28
- sentence, not on page nine.
29
-
30
- ### Distinguish measured, inferred and assumed
31
-
32
- Use different words for them and use them consistently. A modelled number
33
- in the same voice as a measured one is the most reliable way to make work
34
- look sturdier than it is, and it is almost always accidental.
35
-
36
- ### Write the negative results
37
-
38
- What did not work, what was tried and abandoned, and why. It is the part
39
- most often cut and the part most useful to the next person, who will
40
- otherwise spend their time rediscovering it. A short paragraph is enough.
41
-
42
- ### One idea per paragraph, and put it first
43
-
44
- The reader is skimming. If the first sentence of each paragraph carries
45
- its point, skimming works and the document survives being read properly
46
- later. If the point arrives at the end of the paragraph, the skim returns
47
- nothing.
48
-
49
- ### Define a term once, then be boring
50
-
51
- Pick one name for each thing and reuse it. Elegant variation — calling the
52
- same object three different names across four paragraphs — reads well in
53
- prose and is actively harmful in technical writing, because the reader
54
- cannot tell whether you meant something different.
55
-
56
- ### Say what you would need to be more confident
57
-
58
- Ending with the specific thing that would settle the remaining doubt is
59
- more useful than a summary that restates what the reader just read.
1
+ ---
2
+ name: technical-writing
3
+ description: Writing up a result so a reader can act on it — the claim first, the evidence attached, and the limitation stated rather than implied
4
+ additional_tools: [read, web_search]
5
+ ---
6
+
7
+ ## Technical writing methodology
8
+
9
+ ### Lead with the finding
10
+
11
+ The first sentence should be the thing you learned, not the thing you did.
12
+ "Compaction folded a 1M-window thread at 4% of its window" is a finding;
13
+ "we investigated compaction behaviour" is a table of contents. A reader
14
+ who stops after one paragraph should still have the result.
15
+
16
+ ### Attach the evidence to the claim
17
+
18
+ Every non-obvious assertion carries, in the same place, what supports it:
19
+ the measurement, the file and line, the citation, the command. Evidence
20
+ gathered into a section at the end forces the reader to reconstruct which
21
+ support belongs to which claim, and most will not.
22
+
23
+ ### State limitations where they bite
24
+
25
+ A limitation section at the end is read as a formality. The same sentence
26
+ placed beside the claim it qualifies changes how the claim is read, which
27
+ is the point of writing it at all. "On this dataset only" belongs in the
28
+ sentence, not on page nine.
29
+
30
+ ### Distinguish measured, inferred and assumed
31
+
32
+ Use different words for them and use them consistently. A modelled number
33
+ in the same voice as a measured one is the most reliable way to make work
34
+ look sturdier than it is, and it is almost always accidental.
35
+
36
+ ### Write the negative results
37
+
38
+ What did not work, what was tried and abandoned, and why. It is the part
39
+ most often cut and the part most useful to the next person, who will
40
+ otherwise spend their time rediscovering it. A short paragraph is enough.
41
+
42
+ ### One idea per paragraph, and put it first
43
+
44
+ The reader is skimming. If the first sentence of each paragraph carries
45
+ its point, skimming works and the document survives being read properly
46
+ later. If the point arrives at the end of the paragraph, the skim returns
47
+ nothing.
48
+
49
+ ### Define a term once, then be boring
50
+
51
+ Pick one name for each thing and reuse it. Elegant variation — calling the
52
+ same object three different names across four paragraphs — reads well in
53
+ prose and is actively harmful in technical writing, because the reader
54
+ cannot tell whether you meant something different.
55
+
56
+ ### Say what you would need to be more confident
57
+
58
+ Ending with the specific thing that would settle the remaining doubt is
59
+ more useful than a summary that restates what the reader just read.
@@ -1,61 +1,61 @@
1
- ---
2
- name: code-review
3
- description: Reading a change for defects rather than for style, and reporting each one with the failure it produces
4
- additional_tools: [read, shell, read_project_doc]
5
- ---
6
-
7
- ## Code review methodology
8
-
9
- ### Establish what the change claims before reading it
10
-
11
- A diff on its own can only be read as itself. Read the commit message, the
12
- issue or the docstring first, then read the change as an answer to that
13
- claim. The most common real defect is a disagreement between the two — the
14
- message says one thing was fixed, the diff does that and one other thing
15
- nobody mentioned — and it is invisible to a reader who starts at line one
16
- of the diff.
17
-
18
- ### Read the call graph, not the changed lines
19
-
20
- A diff shows what moved. It does not show the caller three files away that
21
- passes the argument the new code no longer tolerates, the subclass that
22
- overrides the method whose contract just changed, or the second call site
23
- of the helper that was fixed in one place. Find the callers before you
24
- approve the change.
25
-
26
- ### A finding needs a failure, not a preference
27
-
28
- For each item, name the input, state or sequence that produces the wrong
29
- result. If you cannot construct one, say so and label the item a concern
30
- rather than a defect. Both are worth reporting; reporting them in the same
31
- voice makes the serious ones unfindable.
32
-
33
- Rank by consequence: wrong answers first, then silently wrong answers
34
- (worse than a crash, because nothing reports them), then security and data
35
- loss, then anything that makes the next change harder. Style last, if at
36
- all.
37
-
38
- ### Look hardest at what is absent
39
-
40
- Most surviving defects are omissions rather than errors:
41
-
42
- - The error path nobody wrote, or one that swallows the cause.
43
- - The caller that was not updated.
44
- - The invariant that used to hold and now holds only by accident.
45
- - The case the new tests cannot distinguish from the old behaviour.
46
-
47
- Ask what would have to be added for the change to be complete, not only
48
- whether what is present is correct.
49
-
50
- ### Judge the tests by whether they could fail
51
-
52
- A test that passes against the unfixed code is not testing the fix. When a
53
- change ships with tests, check at least one against the behaviour it
54
- claims to pin: would it go red if the fix were reverted? If the answer is
55
- no, that is a finding about the test, and it is often the most valuable
56
- thing in the review.
57
-
58
- ### Say when it is fine
59
-
60
- If the change is sound, say so and stop. Manufacturing findings to justify
61
- the time spent teaches the author to skim your reviews.
1
+ ---
2
+ name: code-review
3
+ description: Reading a change for defects rather than for style, and reporting each one with the failure it produces
4
+ additional_tools: [read, shell, read_project_doc]
5
+ ---
6
+
7
+ ## Code review methodology
8
+
9
+ ### Establish what the change claims before reading it
10
+
11
+ A diff on its own can only be read as itself. Read the commit message, the
12
+ issue or the docstring first, then read the change as an answer to that
13
+ claim. The most common real defect is a disagreement between the two — the
14
+ message says one thing was fixed, the diff does that and one other thing
15
+ nobody mentioned — and it is invisible to a reader who starts at line one
16
+ of the diff.
17
+
18
+ ### Read the call graph, not the changed lines
19
+
20
+ A diff shows what moved. It does not show the caller three files away that
21
+ passes the argument the new code no longer tolerates, the subclass that
22
+ overrides the method whose contract just changed, or the second call site
23
+ of the helper that was fixed in one place. Find the callers before you
24
+ approve the change.
25
+
26
+ ### A finding needs a failure, not a preference
27
+
28
+ For each item, name the input, state or sequence that produces the wrong
29
+ result. If you cannot construct one, say so and label the item a concern
30
+ rather than a defect. Both are worth reporting; reporting them in the same
31
+ voice makes the serious ones unfindable.
32
+
33
+ Rank by consequence: wrong answers first, then silently wrong answers
34
+ (worse than a crash, because nothing reports them), then security and data
35
+ loss, then anything that makes the next change harder. Style last, if at
36
+ all.
37
+
38
+ ### Look hardest at what is absent
39
+
40
+ Most surviving defects are omissions rather than errors:
41
+
42
+ - The error path nobody wrote, or one that swallows the cause.
43
+ - The caller that was not updated.
44
+ - The invariant that used to hold and now holds only by accident.
45
+ - The case the new tests cannot distinguish from the old behaviour.
46
+
47
+ Ask what would have to be added for the change to be complete, not only
48
+ whether what is present is correct.
49
+
50
+ ### Judge the tests by whether they could fail
51
+
52
+ A test that passes against the unfixed code is not testing the fix. When a
53
+ change ships with tests, check at least one against the behaviour it
54
+ claims to pin: would it go red if the fix were reverted? If the answer is
55
+ no, that is a finding about the test, and it is often the most valuable
56
+ thing in the review.
57
+
58
+ ### Say when it is fine
59
+
60
+ If the change is sound, say so and stop. Manufacturing findings to justify
61
+ the time spent teaches the author to skim your reviews.
@@ -1,61 +1,61 @@
1
- ---
2
- name: debugging
3
- description: Locating a defect by narrowing rather than guessing, and fixing it where the wrong value is produced
4
- additional_tools: [read, shell]
5
- ---
6
-
7
- ## Debugging methodology
8
-
9
- ### Reproduce it before theorising about it
10
-
11
- A defect you cannot reproduce is one you cannot confirm you fixed. Get to
12
- a command, an input or a sequence that fails reliably, and write it down.
13
- If it fails intermittently, say how often — one in two and one in a
14
- thousand are different problems with different causes.
15
-
16
- Where reproduction is genuinely impossible, say so explicitly and treat
17
- everything downstream as a hypothesis, the fix included.
18
-
19
- ### Shrink the case until nothing can be removed
20
-
21
- Cut the input, the configuration and the code path until removing anything
22
- else makes the failure disappear. What remains is usually close to the
23
- cause, and it is what makes a regression test that discriminates. Most of
24
- the value of debugging comes from this step, and most of the time is spent
25
- avoiding it.
26
-
27
- ### Read the whole error
28
-
29
- The first line of a traceback names the symptom; the cause is usually
30
- further down, or in the frame above. An exception arriving three layers
31
- from where it was created has already told you something about the
32
- boundary it crossed. Do not start editing until you can say which line
33
- produced the wrong value, as opposed to which line noticed it.
34
-
35
- ### Bisect in whichever dimension is cheapest
36
-
37
- Time (which commit), space (which module), input (which field), or
38
- configuration (which setting). Each halving is worth more than an hour of
39
- staring, and `git bisect` is only the most familiar instance of the move.
40
-
41
- ### Confirm the mechanism; correlation is not a diagnosis
42
-
43
- "It works when I change this" does not identify a cause. Before fixing, be
44
- able to say why the broken version produced the wrong result, in a
45
- sentence with a subject and a verb. Without one you cannot tell a fix from
46
- a coincidence that reorders something, and the defect returns later under
47
- a different symptom.
48
-
49
- ### Fix at the producer
50
-
51
- Correct the place that creates the bad value, not each place that copes
52
- with it. A guard at the consumer leaves the wrong value in the system,
53
- moves the failure somewhere less obvious, and has to be repeated at every
54
- future consumer. When you deliberately fix at the consumer instead —
55
- because the producer is out of reach — say that you did, and why.
56
-
57
- ### Leave the test behind, and check that it discriminates
58
-
59
- The regression test should fail against the code as it was. Verify that
60
- rather than assuming it: a test written after the fix, from the fixed
61
- code, frequently passes both ways and pins nothing.
1
+ ---
2
+ name: debugging
3
+ description: Locating a defect by narrowing rather than guessing, and fixing it where the wrong value is produced
4
+ additional_tools: [read, shell]
5
+ ---
6
+
7
+ ## Debugging methodology
8
+
9
+ ### Reproduce it before theorising about it
10
+
11
+ A defect you cannot reproduce is one you cannot confirm you fixed. Get to
12
+ a command, an input or a sequence that fails reliably, and write it down.
13
+ If it fails intermittently, say how often — one in two and one in a
14
+ thousand are different problems with different causes.
15
+
16
+ Where reproduction is genuinely impossible, say so explicitly and treat
17
+ everything downstream as a hypothesis, the fix included.
18
+
19
+ ### Shrink the case until nothing can be removed
20
+
21
+ Cut the input, the configuration and the code path until removing anything
22
+ else makes the failure disappear. What remains is usually close to the
23
+ cause, and it is what makes a regression test that discriminates. Most of
24
+ the value of debugging comes from this step, and most of the time is spent
25
+ avoiding it.
26
+
27
+ ### Read the whole error
28
+
29
+ The first line of a traceback names the symptom; the cause is usually
30
+ further down, or in the frame above. An exception arriving three layers
31
+ from where it was created has already told you something about the
32
+ boundary it crossed. Do not start editing until you can say which line
33
+ produced the wrong value, as opposed to which line noticed it.
34
+
35
+ ### Bisect in whichever dimension is cheapest
36
+
37
+ Time (which commit), space (which module), input (which field), or
38
+ configuration (which setting). Each halving is worth more than an hour of
39
+ staring, and `git bisect` is only the most familiar instance of the move.
40
+
41
+ ### Confirm the mechanism; correlation is not a diagnosis
42
+
43
+ "It works when I change this" does not identify a cause. Before fixing, be
44
+ able to say why the broken version produced the wrong result, in a
45
+ sentence with a subject and a verb. Without one you cannot tell a fix from
46
+ a coincidence that reorders something, and the defect returns later under
47
+ a different symptom.
48
+
49
+ ### Fix at the producer
50
+
51
+ Correct the place that creates the bad value, not each place that copes
52
+ with it. A guard at the consumer leaves the wrong value in the system,
53
+ moves the failure somewhere less obvious, and has to be repeated at every
54
+ future consumer. When you deliberately fix at the consumer instead —
55
+ because the producer is out of reach — say that you did, and why.
56
+
57
+ ### Leave the test behind, and check that it discriminates
58
+
59
+ The regression test should fail against the code as it was. Verify that
60
+ rather than assuming it: a test written after the fix, from the fixed
61
+ code, frequently passes both ways and pins nothing.
@@ -1,67 +1,67 @@
1
- ---
2
- name: test-design
3
- description: Writing tests that can actually fail — choosing what to pin, and checking that the assertion discriminates
4
- additional_tools: [read, shell]
5
- ---
6
-
7
- ## Test design methodology
8
-
9
- ### A test's value is entirely in what makes it fail
10
-
11
- Before writing an assertion, name the change to the production code that
12
- this test would catch. If you cannot name one, the test is decoration: it
13
- adds runtime, it adds the appearance of coverage, and it will later be
14
- cited as evidence that the behaviour is protected.
15
-
16
- The cheapest check is to break the code on purpose — revert the fix,
17
- invert the condition, delete the line — and confirm the test goes red. Do
18
- this once for every non-obvious test. It finds more than rereading does.
19
-
20
- ### Watch for the assertion that cannot discriminate
21
-
22
- An assertion is vacuous when the environment guarantees it regardless of
23
- the code. Sorting is the classic case: if the input already arrives in
24
- sorted order, a test of the sort cannot tell it from a no-op. Construct
25
- inputs that would come back wrong if the behaviour were absent —
26
- reversed, out of range, colliding, duplicated.
27
-
28
- Watch equally for a check whose passing condition is "found nothing".
29
- Such a test cannot detect that it looked in fewer places, so pin the
30
- domain in both directions: something that must be seen, and something
31
- that must not.
32
-
33
- ### Test the boundary, not the middle
34
-
35
- Off by one, empty, exactly one, exactly the limit, one over the limit, the
36
- duplicate, the null. The interior of a range is where the code is already
37
- correct.
38
-
39
- ### One reason to fail per test
40
-
41
- When a test asserts five things, its failure names one and you learn
42
- nothing about the other four. Split by the reason it would go red, not by
43
- the function it calls.
44
-
45
- ### Name the test after the behaviour
46
-
47
- `test_a_rejected_payment_leaves_the_cart_untouched` survives a rename and
48
- tells a reader what broke. `test_process_payment_2` does neither. The name
49
- is the first thing anyone reads when the suite goes red at an
50
- inconvenient moment.
51
-
52
- ### Assert on the observable, not the implementation
53
-
54
- Prefer the value a caller sees over the internal call that produced it: a
55
- test asserting that a helper was called still passes when the helper is
56
- called and its result thrown away. There is a real exception — sometimes
57
- the arguments a decision was made with are exactly what needs pinning,
58
- because an outcome check would pass for an implementation that read the
59
- wrong field and happened to agree — but take it deliberately and say so in
60
- the docstring.
61
-
62
- ### Hold the environment still
63
-
64
- A test whose result depends on filesystem ordering, wall-clock time,
65
- locale, network reachability, or which test ran before it is a source of
66
- noise that eventually gets labelled flaky and ignored. Inject the clock,
67
- pin the ordering, isolate the state.
1
+ ---
2
+ name: test-design
3
+ description: Writing tests that can actually fail — choosing what to pin, and checking that the assertion discriminates
4
+ additional_tools: [read, shell]
5
+ ---
6
+
7
+ ## Test design methodology
8
+
9
+ ### A test's value is entirely in what makes it fail
10
+
11
+ Before writing an assertion, name the change to the production code that
12
+ this test would catch. If you cannot name one, the test is decoration: it
13
+ adds runtime, it adds the appearance of coverage, and it will later be
14
+ cited as evidence that the behaviour is protected.
15
+
16
+ The cheapest check is to break the code on purpose — revert the fix,
17
+ invert the condition, delete the line — and confirm the test goes red. Do
18
+ this once for every non-obvious test. It finds more than rereading does.
19
+
20
+ ### Watch for the assertion that cannot discriminate
21
+
22
+ An assertion is vacuous when the environment guarantees it regardless of
23
+ the code. Sorting is the classic case: if the input already arrives in
24
+ sorted order, a test of the sort cannot tell it from a no-op. Construct
25
+ inputs that would come back wrong if the behaviour were absent —
26
+ reversed, out of range, colliding, duplicated.
27
+
28
+ Watch equally for a check whose passing condition is "found nothing".
29
+ Such a test cannot detect that it looked in fewer places, so pin the
30
+ domain in both directions: something that must be seen, and something
31
+ that must not.
32
+
33
+ ### Test the boundary, not the middle
34
+
35
+ Off by one, empty, exactly one, exactly the limit, one over the limit, the
36
+ duplicate, the null. The interior of a range is where the code is already
37
+ correct.
38
+
39
+ ### One reason to fail per test
40
+
41
+ When a test asserts five things, its failure names one and you learn
42
+ nothing about the other four. Split by the reason it would go red, not by
43
+ the function it calls.
44
+
45
+ ### Name the test after the behaviour
46
+
47
+ `test_a_rejected_payment_leaves_the_cart_untouched` survives a rename and
48
+ tells a reader what broke. `test_process_payment_2` does neither. The name
49
+ is the first thing anyone reads when the suite goes red at an
50
+ inconvenient moment.
51
+
52
+ ### Assert on the observable, not the implementation
53
+
54
+ Prefer the value a caller sees over the internal call that produced it: a
55
+ test asserting that a helper was called still passes when the helper is
56
+ called and its result thrown away. There is a real exception — sometimes
57
+ the arguments a decision was made with are exactly what needs pinning,
58
+ because an outcome check would pass for an implementation that read the
59
+ wrong field and happened to agree — but take it deliberately and say so in
60
+ the docstring.
61
+
62
+ ### Hold the environment still
63
+
64
+ A test whose result depends on filesystem ordering, wall-clock time,
65
+ locale, network reachability, or which test ran before it is a source of
66
+ noise that eventually gets labelled flaky and ignored. Inject the clock,
67
+ pin the ordering, isolate the state.