@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,67 +1,67 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: formal-specification
|
|
3
|
-
description: Saying precisely what a system must do before building it — preconditions, invariants, and the states the specification forbids
|
|
4
|
-
additional_tools: [logic, discrete_math, symbolic_math]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Formal specification methodology
|
|
8
|
-
|
|
9
|
-
### Specify the what, never the how
|
|
10
|
-
|
|
11
|
-
A specification that describes an algorithm has ruled out every other
|
|
12
|
-
correct implementation and cannot be used to judge the one that was
|
|
13
|
-
written. Say what must be true of the result. If the method genuinely
|
|
14
|
-
matters — because a caller depends on its complexity or its ordering —
|
|
15
|
-
that is itself a property to state, not a reason to describe the code.
|
|
16
|
-
|
|
17
|
-
### Write the contract as three parts
|
|
18
|
-
|
|
19
|
-
- **Precondition** — what must hold when this is called. Whose job is it
|
|
20
|
-
to establish it, the caller's or the callee's? Say which; unowned
|
|
21
|
-
preconditions are checked twice or not at all.
|
|
22
|
-
- **Postcondition** — what is true when it returns, related to the inputs.
|
|
23
|
-
- **Invariant** — what is true before and after, which the operation may
|
|
24
|
-
break in the middle and must restore.
|
|
25
|
-
|
|
26
|
-
The frame matters as much as the three: say what the operation does *not*
|
|
27
|
-
change. Most specifications are silent about it and most defects live
|
|
28
|
-
there.
|
|
29
|
-
|
|
30
|
-
### Quantifiers, explicitly, with their types
|
|
31
|
-
|
|
32
|
-
"All items are valid" is not a specification until you say all items in
|
|
33
|
-
what, valid by which predicate, and what happens for the empty collection.
|
|
34
|
-
A "for all" quietly becoming a "there exists" is the single most common
|
|
35
|
-
way a specification and its implementation come to disagree while both
|
|
36
|
-
look right.
|
|
37
|
-
|
|
38
|
-
The empty case, the singleton case and the duplicate case should each be
|
|
39
|
-
answered by the text.
|
|
40
|
-
|
|
41
|
-
### Say what is forbidden, not only what is required
|
|
42
|
-
|
|
43
|
-
The reachable-state question is usually the interesting one: which
|
|
44
|
-
combinations of values must never occur together? Those are the assertions
|
|
45
|
-
worth checking at runtime, and they are what a reviewer needs in order to
|
|
46
|
-
find the path that produces one.
|
|
47
|
-
|
|
48
|
-
### Make the failure behaviour part of the specification
|
|
49
|
-
|
|
50
|
-
Under what conditions does this refuse, and what does it leave behind when
|
|
51
|
-
it does? "Undefined behaviour on bad input" is a decision with
|
|
52
|
-
consequences, so make it explicitly — and if partial effects are possible
|
|
53
|
-
on failure, say exactly which, because a caller cannot write recovery
|
|
54
|
-
against silence.
|
|
55
|
-
|
|
56
|
-
### Prefer a testable statement to an elegant one
|
|
57
|
-
|
|
58
|
-
If a property cannot be checked — by a test, an assertion, a type, or a
|
|
59
|
-
proof — it will drift from the implementation without anyone noticing.
|
|
60
|
-
When a property genuinely resists checking, mark it as an assumption the
|
|
61
|
-
design rests on rather than as something the system guarantees.
|
|
62
|
-
|
|
63
|
-
### Version the specification with the interface
|
|
64
|
-
|
|
65
|
-
A specification that no longer matches the code is worse than none,
|
|
66
|
-
because it is believed. When behaviour changes, the specification changes
|
|
67
|
-
in the same commit, and if it does not, the change is incomplete.
|
|
1
|
+
---
|
|
2
|
+
name: formal-specification
|
|
3
|
+
description: Saying precisely what a system must do before building it — preconditions, invariants, and the states the specification forbids
|
|
4
|
+
additional_tools: [logic, discrete_math, symbolic_math]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Formal specification methodology
|
|
8
|
+
|
|
9
|
+
### Specify the what, never the how
|
|
10
|
+
|
|
11
|
+
A specification that describes an algorithm has ruled out every other
|
|
12
|
+
correct implementation and cannot be used to judge the one that was
|
|
13
|
+
written. Say what must be true of the result. If the method genuinely
|
|
14
|
+
matters — because a caller depends on its complexity or its ordering —
|
|
15
|
+
that is itself a property to state, not a reason to describe the code.
|
|
16
|
+
|
|
17
|
+
### Write the contract as three parts
|
|
18
|
+
|
|
19
|
+
- **Precondition** — what must hold when this is called. Whose job is it
|
|
20
|
+
to establish it, the caller's or the callee's? Say which; unowned
|
|
21
|
+
preconditions are checked twice or not at all.
|
|
22
|
+
- **Postcondition** — what is true when it returns, related to the inputs.
|
|
23
|
+
- **Invariant** — what is true before and after, which the operation may
|
|
24
|
+
break in the middle and must restore.
|
|
25
|
+
|
|
26
|
+
The frame matters as much as the three: say what the operation does *not*
|
|
27
|
+
change. Most specifications are silent about it and most defects live
|
|
28
|
+
there.
|
|
29
|
+
|
|
30
|
+
### Quantifiers, explicitly, with their types
|
|
31
|
+
|
|
32
|
+
"All items are valid" is not a specification until you say all items in
|
|
33
|
+
what, valid by which predicate, and what happens for the empty collection.
|
|
34
|
+
A "for all" quietly becoming a "there exists" is the single most common
|
|
35
|
+
way a specification and its implementation come to disagree while both
|
|
36
|
+
look right.
|
|
37
|
+
|
|
38
|
+
The empty case, the singleton case and the duplicate case should each be
|
|
39
|
+
answered by the text.
|
|
40
|
+
|
|
41
|
+
### Say what is forbidden, not only what is required
|
|
42
|
+
|
|
43
|
+
The reachable-state question is usually the interesting one: which
|
|
44
|
+
combinations of values must never occur together? Those are the assertions
|
|
45
|
+
worth checking at runtime, and they are what a reviewer needs in order to
|
|
46
|
+
find the path that produces one.
|
|
47
|
+
|
|
48
|
+
### Make the failure behaviour part of the specification
|
|
49
|
+
|
|
50
|
+
Under what conditions does this refuse, and what does it leave behind when
|
|
51
|
+
it does? "Undefined behaviour on bad input" is a decision with
|
|
52
|
+
consequences, so make it explicitly — and if partial effects are possible
|
|
53
|
+
on failure, say exactly which, because a caller cannot write recovery
|
|
54
|
+
against silence.
|
|
55
|
+
|
|
56
|
+
### Prefer a testable statement to an elegant one
|
|
57
|
+
|
|
58
|
+
If a property cannot be checked — by a test, an assertion, a type, or a
|
|
59
|
+
proof — it will drift from the implementation without anyone noticing.
|
|
60
|
+
When a property genuinely resists checking, mark it as an assumption the
|
|
61
|
+
design rests on rather than as something the system guarantees.
|
|
62
|
+
|
|
63
|
+
### Version the specification with the interface
|
|
64
|
+
|
|
65
|
+
A specification that no longer matches the code is worse than none,
|
|
66
|
+
because it is believed. When behaviour changes, the specification changes
|
|
67
|
+
in the same commit, and if it does not, the change is incomplete.
|
|
@@ -1,66 +1,66 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: numerical-methods
|
|
3
|
-
description: Knowing when a floating-point answer still means something — conditioning, stability, error growth, and the tests that catch a silently wrong result
|
|
4
|
-
additional_tools: [symbolic_math, linear_algebra, probability_stats]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Numerical methods methodology
|
|
8
|
-
|
|
9
|
-
### Separate the problem's conditioning from the algorithm's stability
|
|
10
|
-
|
|
11
|
-
They are different failures with different remedies. **Conditioning** is a
|
|
12
|
-
property of the problem: how much the answer moves when the input moves.
|
|
13
|
-
**Stability** is a property of the method: how much error the computation
|
|
14
|
-
adds on its own. A stable algorithm on an ill-conditioned problem still
|
|
15
|
-
returns a meaningless answer, and no amount of better code fixes it —
|
|
16
|
-
reformulating the problem does.
|
|
17
|
-
|
|
18
|
-
Say which one you are looking at before proposing a fix.
|
|
19
|
-
|
|
20
|
-
### Catastrophic cancellation is the common cause
|
|
21
|
-
|
|
22
|
-
Subtracting two nearly equal numbers destroys the leading digits and
|
|
23
|
-
leaves whatever noise they carried. It is why the textbook quadratic
|
|
24
|
-
formula fails, why computing variance as `E[x²] − E[x]²` fails, and why a
|
|
25
|
-
difference of large accumulated sums fails. Where you see a subtraction of
|
|
26
|
-
similar quantities, look for the algebraically equivalent form that does
|
|
27
|
-
not perform it.
|
|
28
|
-
|
|
29
|
-
### Never test floating-point values for equality
|
|
30
|
-
|
|
31
|
-
Compare against a tolerance, and choose it deliberately: relative
|
|
32
|
-
tolerance for values that vary in magnitude, absolute tolerance near zero
|
|
33
|
-
where relative tolerance is meaningless, both when the range spans zero.
|
|
34
|
-
A tolerance picked to make a test pass is a tolerance that will not catch
|
|
35
|
-
the regression it was written for.
|
|
36
|
-
|
|
37
|
-
### Track how error accumulates
|
|
38
|
-
|
|
39
|
-
An error bound per operation says little about a loop that runs a million
|
|
40
|
-
times. Ask whether error grows linearly, or is amplified each step — and
|
|
41
|
-
in an iterative method, whether the iteration is contracting. Report the
|
|
42
|
-
growth, not the per-step bound.
|
|
43
|
-
|
|
44
|
-
### Check the units and the scale before the algorithm
|
|
45
|
-
|
|
46
|
-
A large fraction of "numerical instability" is a variable in the wrong
|
|
47
|
-
units and a matrix whose columns differ by ten orders of magnitude.
|
|
48
|
-
Non-dimensionalise, or scale the rows and columns, before reaching for a
|
|
49
|
-
more sophisticated solver.
|
|
50
|
-
|
|
51
|
-
### Verify against something independent
|
|
52
|
-
|
|
53
|
-
Three cheap checks, roughly in order of value:
|
|
54
|
-
|
|
55
|
-
- A case with a known closed-form answer.
|
|
56
|
-
- A conservation law or invariant the result must satisfy.
|
|
57
|
-
- The same computation at higher precision, or via an independent method.
|
|
58
|
-
|
|
59
|
-
Agreement between two implementations of the *same* method proves only
|
|
60
|
-
that you transcribed it twice.
|
|
61
|
-
|
|
62
|
-
### Say what the answer's precision actually is
|
|
63
|
-
|
|
64
|
-
Reporting ten digits from a computation good to four is a claim about
|
|
65
|
-
accuracy, and it is false. Round to what the analysis supports, and say
|
|
66
|
-
what that is.
|
|
1
|
+
---
|
|
2
|
+
name: numerical-methods
|
|
3
|
+
description: Knowing when a floating-point answer still means something — conditioning, stability, error growth, and the tests that catch a silently wrong result
|
|
4
|
+
additional_tools: [symbolic_math, linear_algebra, probability_stats]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Numerical methods methodology
|
|
8
|
+
|
|
9
|
+
### Separate the problem's conditioning from the algorithm's stability
|
|
10
|
+
|
|
11
|
+
They are different failures with different remedies. **Conditioning** is a
|
|
12
|
+
property of the problem: how much the answer moves when the input moves.
|
|
13
|
+
**Stability** is a property of the method: how much error the computation
|
|
14
|
+
adds on its own. A stable algorithm on an ill-conditioned problem still
|
|
15
|
+
returns a meaningless answer, and no amount of better code fixes it —
|
|
16
|
+
reformulating the problem does.
|
|
17
|
+
|
|
18
|
+
Say which one you are looking at before proposing a fix.
|
|
19
|
+
|
|
20
|
+
### Catastrophic cancellation is the common cause
|
|
21
|
+
|
|
22
|
+
Subtracting two nearly equal numbers destroys the leading digits and
|
|
23
|
+
leaves whatever noise they carried. It is why the textbook quadratic
|
|
24
|
+
formula fails, why computing variance as `E[x²] − E[x]²` fails, and why a
|
|
25
|
+
difference of large accumulated sums fails. Where you see a subtraction of
|
|
26
|
+
similar quantities, look for the algebraically equivalent form that does
|
|
27
|
+
not perform it.
|
|
28
|
+
|
|
29
|
+
### Never test floating-point values for equality
|
|
30
|
+
|
|
31
|
+
Compare against a tolerance, and choose it deliberately: relative
|
|
32
|
+
tolerance for values that vary in magnitude, absolute tolerance near zero
|
|
33
|
+
where relative tolerance is meaningless, both when the range spans zero.
|
|
34
|
+
A tolerance picked to make a test pass is a tolerance that will not catch
|
|
35
|
+
the regression it was written for.
|
|
36
|
+
|
|
37
|
+
### Track how error accumulates
|
|
38
|
+
|
|
39
|
+
An error bound per operation says little about a loop that runs a million
|
|
40
|
+
times. Ask whether error grows linearly, or is amplified each step — and
|
|
41
|
+
in an iterative method, whether the iteration is contracting. Report the
|
|
42
|
+
growth, not the per-step bound.
|
|
43
|
+
|
|
44
|
+
### Check the units and the scale before the algorithm
|
|
45
|
+
|
|
46
|
+
A large fraction of "numerical instability" is a variable in the wrong
|
|
47
|
+
units and a matrix whose columns differ by ten orders of magnitude.
|
|
48
|
+
Non-dimensionalise, or scale the rows and columns, before reaching for a
|
|
49
|
+
more sophisticated solver.
|
|
50
|
+
|
|
51
|
+
### Verify against something independent
|
|
52
|
+
|
|
53
|
+
Three cheap checks, roughly in order of value:
|
|
54
|
+
|
|
55
|
+
- A case with a known closed-form answer.
|
|
56
|
+
- A conservation law or invariant the result must satisfy.
|
|
57
|
+
- The same computation at higher precision, or via an independent method.
|
|
58
|
+
|
|
59
|
+
Agreement between two implementations of the *same* method proves only
|
|
60
|
+
that you transcribed it twice.
|
|
61
|
+
|
|
62
|
+
### Say what the answer's precision actually is
|
|
63
|
+
|
|
64
|
+
Reporting ten digits from a computation good to four is a claim about
|
|
65
|
+
accuracy, and it is false. Round to what the analysis supports, and say
|
|
66
|
+
what that is.
|
|
@@ -1,62 +1,62 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: experiment-design
|
|
3
|
-
description: Designing a comparison that can come out either way — controls, confounds, and stating in advance what would falsify the hypothesis
|
|
4
|
-
additional_tools: [probability_stats, arxiv_search]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Experiment design methodology
|
|
8
|
-
|
|
9
|
-
### Write the hypothesis and its negation
|
|
10
|
-
|
|
11
|
-
State what you expect, and state what the world looks like if you are
|
|
12
|
-
wrong. A hypothesis with no articulable negation is not being tested, it
|
|
13
|
-
is being illustrated. Write both before collecting anything, because after
|
|
14
|
-
the data arrives every result looks like it was predicted.
|
|
15
|
-
|
|
16
|
-
### Name the comparison
|
|
17
|
-
|
|
18
|
-
An experiment measures a difference, so say against what. "The new method
|
|
19
|
-
performs well" is not a result; "the new method beats the current one on
|
|
20
|
-
this metric, on this data, by this much" is. If there is no control
|
|
21
|
-
condition, there is no experiment — there is a demonstration, which is a
|
|
22
|
-
legitimate thing to run and a different thing to report.
|
|
23
|
-
|
|
24
|
-
### Enumerate the confounds before you run it
|
|
25
|
-
|
|
26
|
-
List everything that differs between conditions besides the variable you
|
|
27
|
-
care about: hardware, ordering, time of day, who ran it, which data
|
|
28
|
-
arrived first, warm caches. Then say, for each, whether you controlled it,
|
|
29
|
-
randomised it, or accepted it. The ones you accepted belong in the write
|
|
30
|
-
up, not in a footnote.
|
|
31
|
-
|
|
32
|
-
Ordering and warm state are the two that catch experienced people
|
|
33
|
-
repeatedly.
|
|
34
|
-
|
|
35
|
-
### Fix the analysis before the data exists
|
|
36
|
-
|
|
37
|
-
Decide the metric, the exclusion rule and the stopping rule in advance.
|
|
38
|
-
Every one of those chosen after seeing results is a degree of freedom that
|
|
39
|
-
inflates the apparent effect, and none of them feels like cheating at the
|
|
40
|
-
time — dropping "the obviously broken run" is exactly how it happens.
|
|
41
|
-
|
|
42
|
-
If you change the plan after looking, say that you did and report both
|
|
43
|
-
analyses.
|
|
44
|
-
|
|
45
|
-
### Estimate what the experiment can detect
|
|
46
|
-
|
|
47
|
-
Before running, work out roughly how large an effect this sample could
|
|
48
|
-
distinguish from noise. An underpowered experiment that returns "no
|
|
49
|
-
difference" has told you nothing, and it is routinely reported as though
|
|
50
|
-
it had told you something.
|
|
51
|
-
|
|
52
|
-
### Decide what a null result means
|
|
53
|
-
|
|
54
|
-
Say in advance what you will conclude, and do, if the effect is absent. An
|
|
55
|
-
experiment whose only useful outcome is confirmation will keep being rerun
|
|
56
|
-
with small variations until it confirms.
|
|
57
|
-
|
|
58
|
-
### Record the conditions, not just the numbers
|
|
59
|
-
|
|
60
|
-
Versions, seeds, hardware, data snapshot, command line. A result whose
|
|
61
|
-
conditions were not recorded cannot be replicated by you either, three
|
|
62
|
-
weeks later.
|
|
1
|
+
---
|
|
2
|
+
name: experiment-design
|
|
3
|
+
description: Designing a comparison that can come out either way — controls, confounds, and stating in advance what would falsify the hypothesis
|
|
4
|
+
additional_tools: [probability_stats, arxiv_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Experiment design methodology
|
|
8
|
+
|
|
9
|
+
### Write the hypothesis and its negation
|
|
10
|
+
|
|
11
|
+
State what you expect, and state what the world looks like if you are
|
|
12
|
+
wrong. A hypothesis with no articulable negation is not being tested, it
|
|
13
|
+
is being illustrated. Write both before collecting anything, because after
|
|
14
|
+
the data arrives every result looks like it was predicted.
|
|
15
|
+
|
|
16
|
+
### Name the comparison
|
|
17
|
+
|
|
18
|
+
An experiment measures a difference, so say against what. "The new method
|
|
19
|
+
performs well" is not a result; "the new method beats the current one on
|
|
20
|
+
this metric, on this data, by this much" is. If there is no control
|
|
21
|
+
condition, there is no experiment — there is a demonstration, which is a
|
|
22
|
+
legitimate thing to run and a different thing to report.
|
|
23
|
+
|
|
24
|
+
### Enumerate the confounds before you run it
|
|
25
|
+
|
|
26
|
+
List everything that differs between conditions besides the variable you
|
|
27
|
+
care about: hardware, ordering, time of day, who ran it, which data
|
|
28
|
+
arrived first, warm caches. Then say, for each, whether you controlled it,
|
|
29
|
+
randomised it, or accepted it. The ones you accepted belong in the write
|
|
30
|
+
up, not in a footnote.
|
|
31
|
+
|
|
32
|
+
Ordering and warm state are the two that catch experienced people
|
|
33
|
+
repeatedly.
|
|
34
|
+
|
|
35
|
+
### Fix the analysis before the data exists
|
|
36
|
+
|
|
37
|
+
Decide the metric, the exclusion rule and the stopping rule in advance.
|
|
38
|
+
Every one of those chosen after seeing results is a degree of freedom that
|
|
39
|
+
inflates the apparent effect, and none of them feels like cheating at the
|
|
40
|
+
time — dropping "the obviously broken run" is exactly how it happens.
|
|
41
|
+
|
|
42
|
+
If you change the plan after looking, say that you did and report both
|
|
43
|
+
analyses.
|
|
44
|
+
|
|
45
|
+
### Estimate what the experiment can detect
|
|
46
|
+
|
|
47
|
+
Before running, work out roughly how large an effect this sample could
|
|
48
|
+
distinguish from noise. An underpowered experiment that returns "no
|
|
49
|
+
difference" has told you nothing, and it is routinely reported as though
|
|
50
|
+
it had told you something.
|
|
51
|
+
|
|
52
|
+
### Decide what a null result means
|
|
53
|
+
|
|
54
|
+
Say in advance what you will conclude, and do, if the effect is absent. An
|
|
55
|
+
experiment whose only useful outcome is confirmation will keep being rerun
|
|
56
|
+
with small variations until it confirms.
|
|
57
|
+
|
|
58
|
+
### Record the conditions, not just the numbers
|
|
59
|
+
|
|
60
|
+
Versions, seeds, hardware, data snapshot, command line. A result whose
|
|
61
|
+
conditions were not recorded cannot be replicated by you either, three
|
|
62
|
+
weeks later.
|
|
@@ -1,55 +1,55 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: reproducibility
|
|
3
|
-
description: Making a result something another person can obtain again — pinned environments, recorded seeds, and provenance for every input
|
|
4
|
-
additional_tools: [shell, read]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Reproducibility methodology
|
|
8
|
-
|
|
9
|
-
### The bar is another person, on another machine, later
|
|
10
|
-
|
|
11
|
-
Not "it works here". Everything that would differ between your machine and
|
|
12
|
-
theirs is either pinned or is a reason the result will not reproduce:
|
|
13
|
-
package versions, interpreter version, operating system, hardware, locale,
|
|
14
|
-
environment variables, the data.
|
|
15
|
-
|
|
16
|
-
### Pin the environment by exact version
|
|
17
|
-
|
|
18
|
-
A range resolves differently next month, so a lockfile or an exact pin is
|
|
19
|
-
what makes the record real. Record the resolved versions that actually
|
|
20
|
-
ran, not the constraints you asked for — those are different documents,
|
|
21
|
-
and only the first one reproduces anything.
|
|
22
|
-
|
|
23
|
-
### Record the seed, and check that it is the only source of randomness
|
|
24
|
-
|
|
25
|
-
Set it and log it. Then confirm the run is actually deterministic by
|
|
26
|
-
running it twice: seeds are routinely set for one library while a second
|
|
27
|
-
source of randomness — hash ordering, thread scheduling, a GPU kernel, a
|
|
28
|
-
shuffled data loader — goes unseeded. A recorded seed that does not
|
|
29
|
-
determine the output is worse than none, because it looks like control.
|
|
30
|
-
|
|
31
|
-
### Give every input a provenance
|
|
32
|
-
|
|
33
|
-
For each dataset: where it came from, when it was retrieved, which version
|
|
34
|
-
or snapshot, and what was done to it before it reached the analysis.
|
|
35
|
-
"Downloaded from the project website" is not provenance if the file at
|
|
36
|
-
that URL changes. Hash it.
|
|
37
|
-
|
|
38
|
-
### Separate the raw data from every derived form
|
|
39
|
-
|
|
40
|
-
Never modify the original in place. Derived files are outputs, regenerable
|
|
41
|
-
from the raw input by a recorded command; if a derived file cannot be
|
|
42
|
-
regenerated, it has become raw data with no provenance.
|
|
43
|
-
|
|
44
|
-
### Make the whole path executable
|
|
45
|
-
|
|
46
|
-
One command that goes from inputs to results. A sequence of manual steps
|
|
47
|
-
in a README drifts from what was actually run, silently, and the drift is
|
|
48
|
-
discovered only when someone tries to reproduce the result and fails —
|
|
49
|
-
usually the author, months later.
|
|
50
|
-
|
|
51
|
-
### Report what did not reproduce
|
|
52
|
-
|
|
53
|
-
When a rerun gives a different number, that is a finding and belongs in
|
|
54
|
-
the write up with its magnitude. Quietly rerunning until it matches is the
|
|
55
|
-
failure mode this whole methodology exists to prevent.
|
|
1
|
+
---
|
|
2
|
+
name: reproducibility
|
|
3
|
+
description: Making a result something another person can obtain again — pinned environments, recorded seeds, and provenance for every input
|
|
4
|
+
additional_tools: [shell, read]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Reproducibility methodology
|
|
8
|
+
|
|
9
|
+
### The bar is another person, on another machine, later
|
|
10
|
+
|
|
11
|
+
Not "it works here". Everything that would differ between your machine and
|
|
12
|
+
theirs is either pinned or is a reason the result will not reproduce:
|
|
13
|
+
package versions, interpreter version, operating system, hardware, locale,
|
|
14
|
+
environment variables, the data.
|
|
15
|
+
|
|
16
|
+
### Pin the environment by exact version
|
|
17
|
+
|
|
18
|
+
A range resolves differently next month, so a lockfile or an exact pin is
|
|
19
|
+
what makes the record real. Record the resolved versions that actually
|
|
20
|
+
ran, not the constraints you asked for — those are different documents,
|
|
21
|
+
and only the first one reproduces anything.
|
|
22
|
+
|
|
23
|
+
### Record the seed, and check that it is the only source of randomness
|
|
24
|
+
|
|
25
|
+
Set it and log it. Then confirm the run is actually deterministic by
|
|
26
|
+
running it twice: seeds are routinely set for one library while a second
|
|
27
|
+
source of randomness — hash ordering, thread scheduling, a GPU kernel, a
|
|
28
|
+
shuffled data loader — goes unseeded. A recorded seed that does not
|
|
29
|
+
determine the output is worse than none, because it looks like control.
|
|
30
|
+
|
|
31
|
+
### Give every input a provenance
|
|
32
|
+
|
|
33
|
+
For each dataset: where it came from, when it was retrieved, which version
|
|
34
|
+
or snapshot, and what was done to it before it reached the analysis.
|
|
35
|
+
"Downloaded from the project website" is not provenance if the file at
|
|
36
|
+
that URL changes. Hash it.
|
|
37
|
+
|
|
38
|
+
### Separate the raw data from every derived form
|
|
39
|
+
|
|
40
|
+
Never modify the original in place. Derived files are outputs, regenerable
|
|
41
|
+
from the raw input by a recorded command; if a derived file cannot be
|
|
42
|
+
regenerated, it has become raw data with no provenance.
|
|
43
|
+
|
|
44
|
+
### Make the whole path executable
|
|
45
|
+
|
|
46
|
+
One command that goes from inputs to results. A sequence of manual steps
|
|
47
|
+
in a README drifts from what was actually run, silently, and the drift is
|
|
48
|
+
discovered only when someone tries to reproduce the result and fails —
|
|
49
|
+
usually the author, months later.
|
|
50
|
+
|
|
51
|
+
### Report what did not reproduce
|
|
52
|
+
|
|
53
|
+
When a rerun gives a different number, that is a finding and belongs in
|
|
54
|
+
the write up with its magnitude. Quietly rerunning until it matches is the
|
|
55
|
+
failure mode this whole methodology exists to prevent.
|
|
@@ -1,70 +1,70 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: source-evaluation
|
|
3
|
-
description: Judging what a source is worth and saying so at the claim — who is entitled to assert it, how close it is to the point, and what it actually shows
|
|
4
|
-
additional_tools: [fetch_url, web_search, arxiv_search]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## Source evaluation methodology
|
|
8
|
-
|
|
9
|
-
### Two axes, and both are needed
|
|
10
|
-
|
|
11
|
-
A source's worth to a claim is **authority** (is this publisher entitled
|
|
12
|
-
to assert this kind of thing) multiplied by **relevance** (does this
|
|
13
|
-
passage address the claim at hand). Neither alone is a judgement: a
|
|
14
|
-
national statistics office is authoritative and useless if the page you
|
|
15
|
-
cited is about something else, and a perfectly on-point paragraph is worth
|
|
16
|
-
little if it comes from a content farm.
|
|
17
|
-
|
|
18
|
-
This is the same shape the harness's own scorer uses, deliberately, so the
|
|
19
|
-
prose and the number do not teach different things.
|
|
20
|
-
|
|
21
|
-
### Rank the publisher by what constrains it
|
|
22
|
-
|
|
23
|
-
Roughly, in descending order: restricted registries where identity is
|
|
24
|
-
verified before publication (`.gov`, `.mil`, `.edu`, `ac.uk`),
|
|
25
|
-
intergovernmental bodies, standards bodies, peer-reviewed publishers,
|
|
26
|
-
established news, preprint servers, tertiary references such as
|
|
27
|
-
encyclopaedias, general commercial sites, blog platforms, question-and-
|
|
28
|
-
answer forums, social media, content farms.
|
|
29
|
-
|
|
30
|
-
Two rules keep that from being applied naively. A preprint is not peer
|
|
31
|
-
reviewed, whatever the venue's prestige, and saying "published in" of one
|
|
32
|
-
is a factual error. A tertiary reference is a **pointer to a source**, not
|
|
33
|
-
a source: follow it and cite what it cites, or say explicitly that you are
|
|
34
|
-
relying on the summary.
|
|
35
|
-
|
|
36
|
-
### The domain is not the argument
|
|
37
|
-
|
|
38
|
-
`evilgov.com` is not a government and `example.ac` is not a university.
|
|
39
|
-
Check what the suffix actually is, not what it resembles. A hosted
|
|
40
|
-
subdomain inherits the platform's standing, not its host's.
|
|
41
|
-
|
|
42
|
-
### Verify the quote against the page
|
|
43
|
-
|
|
44
|
-
If you attribute a sentence to a source, that sentence should be findable
|
|
45
|
-
on the page you cite. Paraphrase drifts across a chain of citations
|
|
46
|
-
faster than anyone expects, and a quotation that cannot be located is a
|
|
47
|
-
finding about the claim, not a formatting problem.
|
|
48
|
-
|
|
49
|
-
### Separate what was measured from what was concluded
|
|
50
|
-
|
|
51
|
-
Papers frequently demonstrate something narrow and conclude something
|
|
52
|
-
broad, and citations then propagate the conclusion. When you find that
|
|
53
|
-
gap, report it — it is often the most valuable single observation
|
|
54
|
-
available about a body of literature.
|
|
55
|
-
|
|
56
|
-
### Adjust for a stated reason, and only a little
|
|
57
|
-
|
|
58
|
-
You may move a source up or down from its class — a retraction, a
|
|
59
|
-
correction, a known conflict of interest, an author who is the primary
|
|
60
|
-
authority on exactly this — but the adjustment must be small and it must
|
|
61
|
-
come with the reason written down beside it. An unexplained adjustment is
|
|
62
|
-
indistinguishable from a preference.
|
|
63
|
-
|
|
64
|
-
A retraction is not an adjustment. It floors the source.
|
|
65
|
-
|
|
66
|
-
### Say the strength at the claim, not in a bibliography
|
|
67
|
-
|
|
68
|
-
"Reported by a single trade publication, not independently confirmed"
|
|
69
|
-
belongs in the sentence making the claim. A reader who has to reconstruct
|
|
70
|
-
the evidential weight from a reference list will not do it.
|
|
1
|
+
---
|
|
2
|
+
name: source-evaluation
|
|
3
|
+
description: Judging what a source is worth and saying so at the claim — who is entitled to assert it, how close it is to the point, and what it actually shows
|
|
4
|
+
additional_tools: [fetch_url, web_search, arxiv_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Source evaluation methodology
|
|
8
|
+
|
|
9
|
+
### Two axes, and both are needed
|
|
10
|
+
|
|
11
|
+
A source's worth to a claim is **authority** (is this publisher entitled
|
|
12
|
+
to assert this kind of thing) multiplied by **relevance** (does this
|
|
13
|
+
passage address the claim at hand). Neither alone is a judgement: a
|
|
14
|
+
national statistics office is authoritative and useless if the page you
|
|
15
|
+
cited is about something else, and a perfectly on-point paragraph is worth
|
|
16
|
+
little if it comes from a content farm.
|
|
17
|
+
|
|
18
|
+
This is the same shape the harness's own scorer uses, deliberately, so the
|
|
19
|
+
prose and the number do not teach different things.
|
|
20
|
+
|
|
21
|
+
### Rank the publisher by what constrains it
|
|
22
|
+
|
|
23
|
+
Roughly, in descending order: restricted registries where identity is
|
|
24
|
+
verified before publication (`.gov`, `.mil`, `.edu`, `ac.uk`),
|
|
25
|
+
intergovernmental bodies, standards bodies, peer-reviewed publishers,
|
|
26
|
+
established news, preprint servers, tertiary references such as
|
|
27
|
+
encyclopaedias, general commercial sites, blog platforms, question-and-
|
|
28
|
+
answer forums, social media, content farms.
|
|
29
|
+
|
|
30
|
+
Two rules keep that from being applied naively. A preprint is not peer
|
|
31
|
+
reviewed, whatever the venue's prestige, and saying "published in" of one
|
|
32
|
+
is a factual error. A tertiary reference is a **pointer to a source**, not
|
|
33
|
+
a source: follow it and cite what it cites, or say explicitly that you are
|
|
34
|
+
relying on the summary.
|
|
35
|
+
|
|
36
|
+
### The domain is not the argument
|
|
37
|
+
|
|
38
|
+
`evilgov.com` is not a government and `example.ac` is not a university.
|
|
39
|
+
Check what the suffix actually is, not what it resembles. A hosted
|
|
40
|
+
subdomain inherits the platform's standing, not its host's.
|
|
41
|
+
|
|
42
|
+
### Verify the quote against the page
|
|
43
|
+
|
|
44
|
+
If you attribute a sentence to a source, that sentence should be findable
|
|
45
|
+
on the page you cite. Paraphrase drifts across a chain of citations
|
|
46
|
+
faster than anyone expects, and a quotation that cannot be located is a
|
|
47
|
+
finding about the claim, not a formatting problem.
|
|
48
|
+
|
|
49
|
+
### Separate what was measured from what was concluded
|
|
50
|
+
|
|
51
|
+
Papers frequently demonstrate something narrow and conclude something
|
|
52
|
+
broad, and citations then propagate the conclusion. When you find that
|
|
53
|
+
gap, report it — it is often the most valuable single observation
|
|
54
|
+
available about a body of literature.
|
|
55
|
+
|
|
56
|
+
### Adjust for a stated reason, and only a little
|
|
57
|
+
|
|
58
|
+
You may move a source up or down from its class — a retraction, a
|
|
59
|
+
correction, a known conflict of interest, an author who is the primary
|
|
60
|
+
authority on exactly this — but the adjustment must be small and it must
|
|
61
|
+
come with the reason written down beside it. An unexplained adjustment is
|
|
62
|
+
indistinguishable from a preference.
|
|
63
|
+
|
|
64
|
+
A retraction is not an adjustment. It floors the source.
|
|
65
|
+
|
|
66
|
+
### Say the strength at the claim, not in a bibliography
|
|
67
|
+
|
|
68
|
+
"Reported by a single trade publication, not independently confirmed"
|
|
69
|
+
belongs in the sentence making the claim. A reader who has to reconstruct
|
|
70
|
+
the evidential weight from a reference list will not do it.
|