@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,63 +1,63 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: statistical-inference
|
|
3
|
-
description: Reasoning from data to a claim without overstating it — effect sizes, uncertainty, multiplicity, and what a p-value does not say
|
|
4
|
-
additional_tools: [probability_stats, symbolic_math]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Statistical inference methodology
|
|
8
|
-
|
|
9
|
-
### Report the effect, then the uncertainty, then the test
|
|
10
|
-
|
|
11
|
-
The size of the difference is the finding. The interval around it is how
|
|
12
|
-
much to trust the finding. A test statistic is the least informative of
|
|
13
|
-
the three and is routinely reported alone. Lead with "3.2 points better,
|
|
14
|
-
95% CI [0.4, 6.0]", not with "significant".
|
|
15
|
-
|
|
16
|
-
### Say what the p-value is about
|
|
17
|
-
|
|
18
|
-
It is the probability of data at least this extreme under the null
|
|
19
|
-
hypothesis. It is not the probability that the null is true, not the
|
|
20
|
-
probability the result will replicate, and not a measure of effect size. A
|
|
21
|
-
tiny p-value from a huge sample can accompany an effect too small to
|
|
22
|
-
matter, and that combination is one of the most common ways a true
|
|
23
|
-
statement misleads.
|
|
24
|
-
|
|
25
|
-
### Count every comparison you made
|
|
26
|
-
|
|
27
|
-
Testing twenty things at the 0.05 level and reporting the one that came
|
|
28
|
-
out is not a finding, whether the other nineteen were preregistered,
|
|
29
|
-
exploratory, or merely looked at. Say how many comparisons were made and
|
|
30
|
-
correct for them, or label the result exploratory and say it needs
|
|
31
|
-
confirmation on new data.
|
|
32
|
-
|
|
33
|
-
Subgroups, alternative metrics, and "we also tried" all count.
|
|
34
|
-
|
|
35
|
-
### Check the assumptions the method actually makes
|
|
36
|
-
|
|
37
|
-
Independence is the one that fails most often and is questioned least:
|
|
38
|
-
repeated measures on the same subject, runs on the same machine, samples
|
|
39
|
-
drawn from one crawl. Normality usually matters less than people fear;
|
|
40
|
-
independence usually matters more.
|
|
41
|
-
|
|
42
|
-
Say which assumptions you checked and which you assumed.
|
|
43
|
-
|
|
44
|
-
### Absence of significance is not evidence of absence
|
|
45
|
-
|
|
46
|
-
A non-significant result with a wide interval is compatible with a large
|
|
47
|
-
effect. Report the interval and say what the study could have detected. If
|
|
48
|
-
you mean "we can rule out an effect larger than X", say that — it is a
|
|
49
|
-
much stronger and much more useful claim, and it requires the interval to
|
|
50
|
-
support it.
|
|
51
|
-
|
|
52
|
-
### Keep the estimate separate from the decision
|
|
53
|
-
|
|
54
|
-
"How large is the effect" and "what should we do" are different questions
|
|
55
|
-
with different inputs. A threshold that decides an action belongs to the
|
|
56
|
-
decision, with its costs stated, not smuggled into the analysis as though
|
|
57
|
-
it were a property of the data.
|
|
58
|
-
|
|
59
|
-
### Distinguish measured, modelled and assumed
|
|
60
|
-
|
|
61
|
-
At each number, say which it is. A modelled quantity presented in the same
|
|
62
|
-
voice as a measured one is the most reliable way to make an analysis look
|
|
63
|
-
sturdier than it is.
|
|
1
|
+
---
|
|
2
|
+
name: statistical-inference
|
|
3
|
+
description: Reasoning from data to a claim without overstating it — effect sizes, uncertainty, multiplicity, and what a p-value does not say
|
|
4
|
+
additional_tools: [probability_stats, symbolic_math]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Statistical inference methodology
|
|
8
|
+
|
|
9
|
+
### Report the effect, then the uncertainty, then the test
|
|
10
|
+
|
|
11
|
+
The size of the difference is the finding. The interval around it is how
|
|
12
|
+
much to trust the finding. A test statistic is the least informative of
|
|
13
|
+
the three and is routinely reported alone. Lead with "3.2 points better,
|
|
14
|
+
95% CI [0.4, 6.0]", not with "significant".
|
|
15
|
+
|
|
16
|
+
### Say what the p-value is about
|
|
17
|
+
|
|
18
|
+
It is the probability of data at least this extreme under the null
|
|
19
|
+
hypothesis. It is not the probability that the null is true, not the
|
|
20
|
+
probability the result will replicate, and not a measure of effect size. A
|
|
21
|
+
tiny p-value from a huge sample can accompany an effect too small to
|
|
22
|
+
matter, and that combination is one of the most common ways a true
|
|
23
|
+
statement misleads.
|
|
24
|
+
|
|
25
|
+
### Count every comparison you made
|
|
26
|
+
|
|
27
|
+
Testing twenty things at the 0.05 level and reporting the one that came
|
|
28
|
+
out is not a finding, whether the other nineteen were preregistered,
|
|
29
|
+
exploratory, or merely looked at. Say how many comparisons were made and
|
|
30
|
+
correct for them, or label the result exploratory and say it needs
|
|
31
|
+
confirmation on new data.
|
|
32
|
+
|
|
33
|
+
Subgroups, alternative metrics, and "we also tried" all count.
|
|
34
|
+
|
|
35
|
+
### Check the assumptions the method actually makes
|
|
36
|
+
|
|
37
|
+
Independence is the one that fails most often and is questioned least:
|
|
38
|
+
repeated measures on the same subject, runs on the same machine, samples
|
|
39
|
+
drawn from one crawl. Normality usually matters less than people fear;
|
|
40
|
+
independence usually matters more.
|
|
41
|
+
|
|
42
|
+
Say which assumptions you checked and which you assumed.
|
|
43
|
+
|
|
44
|
+
### Absence of significance is not evidence of absence
|
|
45
|
+
|
|
46
|
+
A non-significant result with a wide interval is compatible with a large
|
|
47
|
+
effect. Report the interval and say what the study could have detected. If
|
|
48
|
+
you mean "we can rule out an effect larger than X", say that — it is a
|
|
49
|
+
much stronger and much more useful claim, and it requires the interval to
|
|
50
|
+
support it.
|
|
51
|
+
|
|
52
|
+
### Keep the estimate separate from the decision
|
|
53
|
+
|
|
54
|
+
"How large is the effect" and "what should we do" are different questions
|
|
55
|
+
with different inputs. A threshold that decides an action belongs to the
|
|
56
|
+
decision, with its costs stated, not smuggled into the analysis as though
|
|
57
|
+
it were a property of the data.
|
|
58
|
+
|
|
59
|
+
### Distinguish measured, modelled and assumed
|
|
60
|
+
|
|
61
|
+
At each number, say which it is. A modelled quantity presented in the same
|
|
62
|
+
voice as a measured one is the most reliable way to make an analysis look
|
|
63
|
+
sturdier than it is.
|
|
@@ -1,59 +1,59 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: technical-writing
|
|
3
|
-
description: Writing up a result so a reader can act on it — the claim first, the evidence attached, and the limitation stated rather than implied
|
|
4
|
-
additional_tools: [read, web_search]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Technical writing methodology
|
|
8
|
-
|
|
9
|
-
### Lead with the finding
|
|
10
|
-
|
|
11
|
-
The first sentence should be the thing you learned, not the thing you did.
|
|
12
|
-
"Compaction folded a 1M-window thread at 4% of its window" is a finding;
|
|
13
|
-
"we investigated compaction behaviour" is a table of contents. A reader
|
|
14
|
-
who stops after one paragraph should still have the result.
|
|
15
|
-
|
|
16
|
-
### Attach the evidence to the claim
|
|
17
|
-
|
|
18
|
-
Every non-obvious assertion carries, in the same place, what supports it:
|
|
19
|
-
the measurement, the file and line, the citation, the command. Evidence
|
|
20
|
-
gathered into a section at the end forces the reader to reconstruct which
|
|
21
|
-
support belongs to which claim, and most will not.
|
|
22
|
-
|
|
23
|
-
### State limitations where they bite
|
|
24
|
-
|
|
25
|
-
A limitation section at the end is read as a formality. The same sentence
|
|
26
|
-
placed beside the claim it qualifies changes how the claim is read, which
|
|
27
|
-
is the point of writing it at all. "On this dataset only" belongs in the
|
|
28
|
-
sentence, not on page nine.
|
|
29
|
-
|
|
30
|
-
### Distinguish measured, inferred and assumed
|
|
31
|
-
|
|
32
|
-
Use different words for them and use them consistently. A modelled number
|
|
33
|
-
in the same voice as a measured one is the most reliable way to make work
|
|
34
|
-
look sturdier than it is, and it is almost always accidental.
|
|
35
|
-
|
|
36
|
-
### Write the negative results
|
|
37
|
-
|
|
38
|
-
What did not work, what was tried and abandoned, and why. It is the part
|
|
39
|
-
most often cut and the part most useful to the next person, who will
|
|
40
|
-
otherwise spend their time rediscovering it. A short paragraph is enough.
|
|
41
|
-
|
|
42
|
-
### One idea per paragraph, and put it first
|
|
43
|
-
|
|
44
|
-
The reader is skimming. If the first sentence of each paragraph carries
|
|
45
|
-
its point, skimming works and the document survives being read properly
|
|
46
|
-
later. If the point arrives at the end of the paragraph, the skim returns
|
|
47
|
-
nothing.
|
|
48
|
-
|
|
49
|
-
### Define a term once, then be boring
|
|
50
|
-
|
|
51
|
-
Pick one name for each thing and reuse it. Elegant variation — calling the
|
|
52
|
-
same object three different names across four paragraphs — reads well in
|
|
53
|
-
prose and is actively harmful in technical writing, because the reader
|
|
54
|
-
cannot tell whether you meant something different.
|
|
55
|
-
|
|
56
|
-
### Say what you would need to be more confident
|
|
57
|
-
|
|
58
|
-
Ending with the specific thing that would settle the remaining doubt is
|
|
59
|
-
more useful than a summary that restates what the reader just read.
|
|
1
|
+
---
|
|
2
|
+
name: technical-writing
|
|
3
|
+
description: Writing up a result so a reader can act on it — the claim first, the evidence attached, and the limitation stated rather than implied
|
|
4
|
+
additional_tools: [read, web_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Technical writing methodology
|
|
8
|
+
|
|
9
|
+
### Lead with the finding
|
|
10
|
+
|
|
11
|
+
The first sentence should be the thing you learned, not the thing you did.
|
|
12
|
+
"Compaction folded a 1M-window thread at 4% of its window" is a finding;
|
|
13
|
+
"we investigated compaction behaviour" is a table of contents. A reader
|
|
14
|
+
who stops after one paragraph should still have the result.
|
|
15
|
+
|
|
16
|
+
### Attach the evidence to the claim
|
|
17
|
+
|
|
18
|
+
Every non-obvious assertion carries, in the same place, what supports it:
|
|
19
|
+
the measurement, the file and line, the citation, the command. Evidence
|
|
20
|
+
gathered into a section at the end forces the reader to reconstruct which
|
|
21
|
+
support belongs to which claim, and most will not.
|
|
22
|
+
|
|
23
|
+
### State limitations where they bite
|
|
24
|
+
|
|
25
|
+
A limitation section at the end is read as a formality. The same sentence
|
|
26
|
+
placed beside the claim it qualifies changes how the claim is read, which
|
|
27
|
+
is the point of writing it at all. "On this dataset only" belongs in the
|
|
28
|
+
sentence, not on page nine.
|
|
29
|
+
|
|
30
|
+
### Distinguish measured, inferred and assumed
|
|
31
|
+
|
|
32
|
+
Use different words for them and use them consistently. A modelled number
|
|
33
|
+
in the same voice as a measured one is the most reliable way to make work
|
|
34
|
+
look sturdier than it is, and it is almost always accidental.
|
|
35
|
+
|
|
36
|
+
### Write the negative results
|
|
37
|
+
|
|
38
|
+
What did not work, what was tried and abandoned, and why. It is the part
|
|
39
|
+
most often cut and the part most useful to the next person, who will
|
|
40
|
+
otherwise spend their time rediscovering it. A short paragraph is enough.
|
|
41
|
+
|
|
42
|
+
### One idea per paragraph, and put it first
|
|
43
|
+
|
|
44
|
+
The reader is skimming. If the first sentence of each paragraph carries
|
|
45
|
+
its point, skimming works and the document survives being read properly
|
|
46
|
+
later. If the point arrives at the end of the paragraph, the skim returns
|
|
47
|
+
nothing.
|
|
48
|
+
|
|
49
|
+
### Define a term once, then be boring
|
|
50
|
+
|
|
51
|
+
Pick one name for each thing and reuse it. Elegant variation — calling the
|
|
52
|
+
same object three different names across four paragraphs — reads well in
|
|
53
|
+
prose and is actively harmful in technical writing, because the reader
|
|
54
|
+
cannot tell whether you meant something different.
|
|
55
|
+
|
|
56
|
+
### Say what you would need to be more confident
|
|
57
|
+
|
|
58
|
+
Ending with the specific thing that would settle the remaining doubt is
|
|
59
|
+
more useful than a summary that restates what the reader just read.
|
|
@@ -1,61 +1,61 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: code-review
|
|
3
|
-
description: Reading a change for defects rather than for style, and reporting each one with the failure it produces
|
|
4
|
-
additional_tools: [read, shell, read_project_doc]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Code review methodology
|
|
8
|
-
|
|
9
|
-
### Establish what the change claims before reading it
|
|
10
|
-
|
|
11
|
-
A diff on its own can only be read as itself. Read the commit message, the
|
|
12
|
-
issue or the docstring first, then read the change as an answer to that
|
|
13
|
-
claim. The most common real defect is a disagreement between the two — the
|
|
14
|
-
message says one thing was fixed, the diff does that and one other thing
|
|
15
|
-
nobody mentioned — and it is invisible to a reader who starts at line one
|
|
16
|
-
of the diff.
|
|
17
|
-
|
|
18
|
-
### Read the call graph, not the changed lines
|
|
19
|
-
|
|
20
|
-
A diff shows what moved. It does not show the caller three files away that
|
|
21
|
-
passes the argument the new code no longer tolerates, the subclass that
|
|
22
|
-
overrides the method whose contract just changed, or the second call site
|
|
23
|
-
of the helper that was fixed in one place. Find the callers before you
|
|
24
|
-
approve the change.
|
|
25
|
-
|
|
26
|
-
### A finding needs a failure, not a preference
|
|
27
|
-
|
|
28
|
-
For each item, name the input, state or sequence that produces the wrong
|
|
29
|
-
result. If you cannot construct one, say so and label the item a concern
|
|
30
|
-
rather than a defect. Both are worth reporting; reporting them in the same
|
|
31
|
-
voice makes the serious ones unfindable.
|
|
32
|
-
|
|
33
|
-
Rank by consequence: wrong answers first, then silently wrong answers
|
|
34
|
-
(worse than a crash, because nothing reports them), then security and data
|
|
35
|
-
loss, then anything that makes the next change harder. Style last, if at
|
|
36
|
-
all.
|
|
37
|
-
|
|
38
|
-
### Look hardest at what is absent
|
|
39
|
-
|
|
40
|
-
Most surviving defects are omissions rather than errors:
|
|
41
|
-
|
|
42
|
-
- The error path nobody wrote, or one that swallows the cause.
|
|
43
|
-
- The caller that was not updated.
|
|
44
|
-
- The invariant that used to hold and now holds only by accident.
|
|
45
|
-
- The case the new tests cannot distinguish from the old behaviour.
|
|
46
|
-
|
|
47
|
-
Ask what would have to be added for the change to be complete, not only
|
|
48
|
-
whether what is present is correct.
|
|
49
|
-
|
|
50
|
-
### Judge the tests by whether they could fail
|
|
51
|
-
|
|
52
|
-
A test that passes against the unfixed code is not testing the fix. When a
|
|
53
|
-
change ships with tests, check at least one against the behaviour it
|
|
54
|
-
claims to pin: would it go red if the fix were reverted? If the answer is
|
|
55
|
-
no, that is a finding about the test, and it is often the most valuable
|
|
56
|
-
thing in the review.
|
|
57
|
-
|
|
58
|
-
### Say when it is fine
|
|
59
|
-
|
|
60
|
-
If the change is sound, say so and stop. Manufacturing findings to justify
|
|
61
|
-
the time spent teaches the author to skim your reviews.
|
|
1
|
+
---
|
|
2
|
+
name: code-review
|
|
3
|
+
description: Reading a change for defects rather than for style, and reporting each one with the failure it produces
|
|
4
|
+
additional_tools: [read, shell, read_project_doc]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Code review methodology
|
|
8
|
+
|
|
9
|
+
### Establish what the change claims before reading it
|
|
10
|
+
|
|
11
|
+
A diff on its own can only be read as itself. Read the commit message, the
|
|
12
|
+
issue or the docstring first, then read the change as an answer to that
|
|
13
|
+
claim. The most common real defect is a disagreement between the two — the
|
|
14
|
+
message says one thing was fixed, the diff does that and one other thing
|
|
15
|
+
nobody mentioned — and it is invisible to a reader who starts at line one
|
|
16
|
+
of the diff.
|
|
17
|
+
|
|
18
|
+
### Read the call graph, not the changed lines
|
|
19
|
+
|
|
20
|
+
A diff shows what moved. It does not show the caller three files away that
|
|
21
|
+
passes the argument the new code no longer tolerates, the subclass that
|
|
22
|
+
overrides the method whose contract just changed, or the second call site
|
|
23
|
+
of the helper that was fixed in one place. Find the callers before you
|
|
24
|
+
approve the change.
|
|
25
|
+
|
|
26
|
+
### A finding needs a failure, not a preference
|
|
27
|
+
|
|
28
|
+
For each item, name the input, state or sequence that produces the wrong
|
|
29
|
+
result. If you cannot construct one, say so and label the item a concern
|
|
30
|
+
rather than a defect. Both are worth reporting; reporting them in the same
|
|
31
|
+
voice makes the serious ones unfindable.
|
|
32
|
+
|
|
33
|
+
Rank by consequence: wrong answers first, then silently wrong answers
|
|
34
|
+
(worse than a crash, because nothing reports them), then security and data
|
|
35
|
+
loss, then anything that makes the next change harder. Style last, if at
|
|
36
|
+
all.
|
|
37
|
+
|
|
38
|
+
### Look hardest at what is absent
|
|
39
|
+
|
|
40
|
+
Most surviving defects are omissions rather than errors:
|
|
41
|
+
|
|
42
|
+
- The error path nobody wrote, or one that swallows the cause.
|
|
43
|
+
- The caller that was not updated.
|
|
44
|
+
- The invariant that used to hold and now holds only by accident.
|
|
45
|
+
- The case the new tests cannot distinguish from the old behaviour.
|
|
46
|
+
|
|
47
|
+
Ask what would have to be added for the change to be complete, not only
|
|
48
|
+
whether what is present is correct.
|
|
49
|
+
|
|
50
|
+
### Judge the tests by whether they could fail
|
|
51
|
+
|
|
52
|
+
A test that passes against the unfixed code is not testing the fix. When a
|
|
53
|
+
change ships with tests, check at least one against the behaviour it
|
|
54
|
+
claims to pin: would it go red if the fix were reverted? If the answer is
|
|
55
|
+
no, that is a finding about the test, and it is often the most valuable
|
|
56
|
+
thing in the review.
|
|
57
|
+
|
|
58
|
+
### Say when it is fine
|
|
59
|
+
|
|
60
|
+
If the change is sound, say so and stop. Manufacturing findings to justify
|
|
61
|
+
the time spent teaches the author to skim your reviews.
|
|
@@ -1,61 +1,61 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: debugging
|
|
3
|
-
description: Locating a defect by narrowing rather than guessing, and fixing it where the wrong value is produced
|
|
4
|
-
additional_tools: [read, shell]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Debugging methodology
|
|
8
|
-
|
|
9
|
-
### Reproduce it before theorising about it
|
|
10
|
-
|
|
11
|
-
A defect you cannot reproduce is one you cannot confirm you fixed. Get to
|
|
12
|
-
a command, an input or a sequence that fails reliably, and write it down.
|
|
13
|
-
If it fails intermittently, say how often — one in two and one in a
|
|
14
|
-
thousand are different problems with different causes.
|
|
15
|
-
|
|
16
|
-
Where reproduction is genuinely impossible, say so explicitly and treat
|
|
17
|
-
everything downstream as a hypothesis, the fix included.
|
|
18
|
-
|
|
19
|
-
### Shrink the case until nothing can be removed
|
|
20
|
-
|
|
21
|
-
Cut the input, the configuration and the code path until removing anything
|
|
22
|
-
else makes the failure disappear. What remains is usually close to the
|
|
23
|
-
cause, and it is what makes a regression test that discriminates. Most of
|
|
24
|
-
the value of debugging comes from this step, and most of the time is spent
|
|
25
|
-
avoiding it.
|
|
26
|
-
|
|
27
|
-
### Read the whole error
|
|
28
|
-
|
|
29
|
-
The first line of a traceback names the symptom; the cause is usually
|
|
30
|
-
further down, or in the frame above. An exception arriving three layers
|
|
31
|
-
from where it was created has already told you something about the
|
|
32
|
-
boundary it crossed. Do not start editing until you can say which line
|
|
33
|
-
produced the wrong value, as opposed to which line noticed it.
|
|
34
|
-
|
|
35
|
-
### Bisect in whichever dimension is cheapest
|
|
36
|
-
|
|
37
|
-
Time (which commit), space (which module), input (which field), or
|
|
38
|
-
configuration (which setting). Each halving is worth more than an hour of
|
|
39
|
-
staring, and `git bisect` is only the most familiar instance of the move.
|
|
40
|
-
|
|
41
|
-
### Confirm the mechanism; correlation is not a diagnosis
|
|
42
|
-
|
|
43
|
-
"It works when I change this" does not identify a cause. Before fixing, be
|
|
44
|
-
able to say why the broken version produced the wrong result, in a
|
|
45
|
-
sentence with a subject and a verb. Without one you cannot tell a fix from
|
|
46
|
-
a coincidence that reorders something, and the defect returns later under
|
|
47
|
-
a different symptom.
|
|
48
|
-
|
|
49
|
-
### Fix at the producer
|
|
50
|
-
|
|
51
|
-
Correct the place that creates the bad value, not each place that copes
|
|
52
|
-
with it. A guard at the consumer leaves the wrong value in the system,
|
|
53
|
-
moves the failure somewhere less obvious, and has to be repeated at every
|
|
54
|
-
future consumer. When you deliberately fix at the consumer instead —
|
|
55
|
-
because the producer is out of reach — say that you did, and why.
|
|
56
|
-
|
|
57
|
-
### Leave the test behind, and check that it discriminates
|
|
58
|
-
|
|
59
|
-
The regression test should fail against the code as it was. Verify that
|
|
60
|
-
rather than assuming it: a test written after the fix, from the fixed
|
|
61
|
-
code, frequently passes both ways and pins nothing.
|
|
1
|
+
---
|
|
2
|
+
name: debugging
|
|
3
|
+
description: Locating a defect by narrowing rather than guessing, and fixing it where the wrong value is produced
|
|
4
|
+
additional_tools: [read, shell]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Debugging methodology
|
|
8
|
+
|
|
9
|
+
### Reproduce it before theorising about it
|
|
10
|
+
|
|
11
|
+
A defect you cannot reproduce is one you cannot confirm you fixed. Get to
|
|
12
|
+
a command, an input or a sequence that fails reliably, and write it down.
|
|
13
|
+
If it fails intermittently, say how often — one in two and one in a
|
|
14
|
+
thousand are different problems with different causes.
|
|
15
|
+
|
|
16
|
+
Where reproduction is genuinely impossible, say so explicitly and treat
|
|
17
|
+
everything downstream as a hypothesis, the fix included.
|
|
18
|
+
|
|
19
|
+
### Shrink the case until nothing can be removed
|
|
20
|
+
|
|
21
|
+
Cut the input, the configuration and the code path until removing anything
|
|
22
|
+
else makes the failure disappear. What remains is usually close to the
|
|
23
|
+
cause, and it is what makes a regression test that discriminates. Most of
|
|
24
|
+
the value of debugging comes from this step, and most of the time is spent
|
|
25
|
+
avoiding it.
|
|
26
|
+
|
|
27
|
+
### Read the whole error
|
|
28
|
+
|
|
29
|
+
The first line of a traceback names the symptom; the cause is usually
|
|
30
|
+
further down, or in the frame above. An exception arriving three layers
|
|
31
|
+
from where it was created has already told you something about the
|
|
32
|
+
boundary it crossed. Do not start editing until you can say which line
|
|
33
|
+
produced the wrong value, as opposed to which line noticed it.
|
|
34
|
+
|
|
35
|
+
### Bisect in whichever dimension is cheapest
|
|
36
|
+
|
|
37
|
+
Time (which commit), space (which module), input (which field), or
|
|
38
|
+
configuration (which setting). Each halving is worth more than an hour of
|
|
39
|
+
staring, and `git bisect` is only the most familiar instance of the move.
|
|
40
|
+
|
|
41
|
+
### Confirm the mechanism; correlation is not a diagnosis
|
|
42
|
+
|
|
43
|
+
"It works when I change this" does not identify a cause. Before fixing, be
|
|
44
|
+
able to say why the broken version produced the wrong result, in a
|
|
45
|
+
sentence with a subject and a verb. Without one you cannot tell a fix from
|
|
46
|
+
a coincidence that reorders something, and the defect returns later under
|
|
47
|
+
a different symptom.
|
|
48
|
+
|
|
49
|
+
### Fix at the producer
|
|
50
|
+
|
|
51
|
+
Correct the place that creates the bad value, not each place that copes
|
|
52
|
+
with it. A guard at the consumer leaves the wrong value in the system,
|
|
53
|
+
moves the failure somewhere less obvious, and has to be repeated at every
|
|
54
|
+
future consumer. When you deliberately fix at the consumer instead —
|
|
55
|
+
because the producer is out of reach — say that you did, and why.
|
|
56
|
+
|
|
57
|
+
### Leave the test behind, and check that it discriminates
|
|
58
|
+
|
|
59
|
+
The regression test should fail against the code as it was. Verify that
|
|
60
|
+
rather than assuming it: a test written after the fix, from the fixed
|
|
61
|
+
code, frequently passes both ways and pins nothing.
|
|
@@ -1,67 +1,67 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: test-design
|
|
3
|
-
description: Writing tests that can actually fail — choosing what to pin, and checking that the assertion discriminates
|
|
4
|
-
additional_tools: [read, shell]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Test design methodology
|
|
8
|
-
|
|
9
|
-
### A test's value is entirely in what makes it fail
|
|
10
|
-
|
|
11
|
-
Before writing an assertion, name the change to the production code that
|
|
12
|
-
this test would catch. If you cannot name one, the test is decoration: it
|
|
13
|
-
adds runtime, it adds the appearance of coverage, and it will later be
|
|
14
|
-
cited as evidence that the behaviour is protected.
|
|
15
|
-
|
|
16
|
-
The cheapest check is to break the code on purpose — revert the fix,
|
|
17
|
-
invert the condition, delete the line — and confirm the test goes red. Do
|
|
18
|
-
this once for every non-obvious test. It finds more than rereading does.
|
|
19
|
-
|
|
20
|
-
### Watch for the assertion that cannot discriminate
|
|
21
|
-
|
|
22
|
-
An assertion is vacuous when the environment guarantees it regardless of
|
|
23
|
-
the code. Sorting is the classic case: if the input already arrives in
|
|
24
|
-
sorted order, a test of the sort cannot tell it from a no-op. Construct
|
|
25
|
-
inputs that would come back wrong if the behaviour were absent —
|
|
26
|
-
reversed, out of range, colliding, duplicated.
|
|
27
|
-
|
|
28
|
-
Watch equally for a check whose passing condition is "found nothing".
|
|
29
|
-
Such a test cannot detect that it looked in fewer places, so pin the
|
|
30
|
-
domain in both directions: something that must be seen, and something
|
|
31
|
-
that must not.
|
|
32
|
-
|
|
33
|
-
### Test the boundary, not the middle
|
|
34
|
-
|
|
35
|
-
Off by one, empty, exactly one, exactly the limit, one over the limit, the
|
|
36
|
-
duplicate, the null. The interior of a range is where the code is already
|
|
37
|
-
correct.
|
|
38
|
-
|
|
39
|
-
### One reason to fail per test
|
|
40
|
-
|
|
41
|
-
When a test asserts five things, its failure names one and you learn
|
|
42
|
-
nothing about the other four. Split by the reason it would go red, not by
|
|
43
|
-
the function it calls.
|
|
44
|
-
|
|
45
|
-
### Name the test after the behaviour
|
|
46
|
-
|
|
47
|
-
`test_a_rejected_payment_leaves_the_cart_untouched` survives a rename and
|
|
48
|
-
tells a reader what broke. `test_process_payment_2` does neither. The name
|
|
49
|
-
is the first thing anyone reads when the suite goes red at an
|
|
50
|
-
inconvenient moment.
|
|
51
|
-
|
|
52
|
-
### Assert on the observable, not the implementation
|
|
53
|
-
|
|
54
|
-
Prefer the value a caller sees over the internal call that produced it: a
|
|
55
|
-
test asserting that a helper was called still passes when the helper is
|
|
56
|
-
called and its result thrown away. There is a real exception — sometimes
|
|
57
|
-
the arguments a decision was made with are exactly what needs pinning,
|
|
58
|
-
because an outcome check would pass for an implementation that read the
|
|
59
|
-
wrong field and happened to agree — but take it deliberately and say so in
|
|
60
|
-
the docstring.
|
|
61
|
-
|
|
62
|
-
### Hold the environment still
|
|
63
|
-
|
|
64
|
-
A test whose result depends on filesystem ordering, wall-clock time,
|
|
65
|
-
locale, network reachability, or which test ran before it is a source of
|
|
66
|
-
noise that eventually gets labelled flaky and ignored. Inject the clock,
|
|
67
|
-
pin the ordering, isolate the state.
|
|
1
|
+
---
|
|
2
|
+
name: test-design
|
|
3
|
+
description: Writing tests that can actually fail — choosing what to pin, and checking that the assertion discriminates
|
|
4
|
+
additional_tools: [read, shell]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Test design methodology
|
|
8
|
+
|
|
9
|
+
### A test's value is entirely in what makes it fail
|
|
10
|
+
|
|
11
|
+
Before writing an assertion, name the change to the production code that
|
|
12
|
+
this test would catch. If you cannot name one, the test is decoration: it
|
|
13
|
+
adds runtime, it adds the appearance of coverage, and it will later be
|
|
14
|
+
cited as evidence that the behaviour is protected.
|
|
15
|
+
|
|
16
|
+
The cheapest check is to break the code on purpose — revert the fix,
|
|
17
|
+
invert the condition, delete the line — and confirm the test goes red. Do
|
|
18
|
+
this once for every non-obvious test. It finds more than rereading does.
|
|
19
|
+
|
|
20
|
+
### Watch for the assertion that cannot discriminate
|
|
21
|
+
|
|
22
|
+
An assertion is vacuous when the environment guarantees it regardless of
|
|
23
|
+
the code. Sorting is the classic case: if the input already arrives in
|
|
24
|
+
sorted order, a test of the sort cannot tell it from a no-op. Construct
|
|
25
|
+
inputs that would come back wrong if the behaviour were absent —
|
|
26
|
+
reversed, out of range, colliding, duplicated.
|
|
27
|
+
|
|
28
|
+
Watch equally for a check whose passing condition is "found nothing".
|
|
29
|
+
Such a test cannot detect that it looked in fewer places, so pin the
|
|
30
|
+
domain in both directions: something that must be seen, and something
|
|
31
|
+
that must not.
|
|
32
|
+
|
|
33
|
+
### Test the boundary, not the middle
|
|
34
|
+
|
|
35
|
+
Off by one, empty, exactly one, exactly the limit, one over the limit, the
|
|
36
|
+
duplicate, the null. The interior of a range is where the code is already
|
|
37
|
+
correct.
|
|
38
|
+
|
|
39
|
+
### One reason to fail per test
|
|
40
|
+
|
|
41
|
+
When a test asserts five things, its failure names one and you learn
|
|
42
|
+
nothing about the other four. Split by the reason it would go red, not by
|
|
43
|
+
the function it calls.
|
|
44
|
+
|
|
45
|
+
### Name the test after the behaviour
|
|
46
|
+
|
|
47
|
+
`test_a_rejected_payment_leaves_the_cart_untouched` survives a rename and
|
|
48
|
+
tells a reader what broke. `test_process_payment_2` does neither. The name
|
|
49
|
+
is the first thing anyone reads when the suite goes red at an
|
|
50
|
+
inconvenient moment.
|
|
51
|
+
|
|
52
|
+
### Assert on the observable, not the implementation
|
|
53
|
+
|
|
54
|
+
Prefer the value a caller sees over the internal call that produced it: a
|
|
55
|
+
test asserting that a helper was called still passes when the helper is
|
|
56
|
+
called and its result thrown away. There is a real exception — sometimes
|
|
57
|
+
the arguments a decision was made with are exactly what needs pinning,
|
|
58
|
+
because an outcome check would pass for an implementation that read the
|
|
59
|
+
wrong field and happened to agree — but take it deliberately and say so in
|
|
60
|
+
the docstring.
|
|
61
|
+
|
|
62
|
+
### Hold the environment still
|
|
63
|
+
|
|
64
|
+
A test whose result depends on filesystem ordering, wall-clock time,
|
|
65
|
+
locale, network reachability, or which test ran before it is a source of
|
|
66
|
+
noise that eventually gets labelled flaky and ignored. Inject the clock,
|
|
67
|
+
pin the ordering, isolate the state.
|