@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,67 +1,67 @@
1
- ---
2
- name: formal-specification
3
- description: Saying precisely what a system must do before building it — preconditions, invariants, and the states the specification forbids
4
- additional_tools: [logic, discrete_math, symbolic_math]
5
- ---
6
-
7
- ## Formal specification methodology
8
-
9
- ### Specify the what, never the how
10
-
11
- A specification that describes an algorithm has ruled out every other
12
- correct implementation and cannot be used to judge the one that was
13
- written. Say what must be true of the result. If the method genuinely
14
- matters — because a caller depends on its complexity or its ordering —
15
- that is itself a property to state, not a reason to describe the code.
16
-
17
- ### Write the contract as three parts
18
-
19
- - **Precondition** — what must hold when this is called. Whose job is it
20
- to establish it, the caller's or the callee's? Say which; unowned
21
- preconditions are checked twice or not at all.
22
- - **Postcondition** — what is true when it returns, related to the inputs.
23
- - **Invariant** — what is true before and after, which the operation may
24
- break in the middle and must restore.
25
-
26
- The frame matters as much as the three: say what the operation does *not*
27
- change. Most specifications are silent about it and most defects live
28
- there.
29
-
30
- ### Quantifiers, explicitly, with their types
31
-
32
- "All items are valid" is not a specification until you say all items in
33
- what, valid by which predicate, and what happens for the empty collection.
34
- A "for all" quietly becoming a "there exists" is the single most common
35
- way a specification and its implementation come to disagree while both
36
- look right.
37
-
38
- The empty case, the singleton case and the duplicate case should each be
39
- answered by the text.
40
-
41
- ### Say what is forbidden, not only what is required
42
-
43
- The reachable-state question is usually the interesting one: which
44
- combinations of values must never occur together? Those are the assertions
45
- worth checking at runtime, and they are what a reviewer needs in order to
46
- find the path that produces one.
47
-
48
- ### Make the failure behaviour part of the specification
49
-
50
- Under what conditions does this refuse, and what does it leave behind when
51
- it does? "Undefined behaviour on bad input" is a decision with
52
- consequences, so make it explicitly — and if partial effects are possible
53
- on failure, say exactly which, because a caller cannot write recovery
54
- against silence.
55
-
56
- ### Prefer a testable statement to an elegant one
57
-
58
- If a property cannot be checked — by a test, an assertion, a type, or a
59
- proof — it will drift from the implementation without anyone noticing.
60
- When a property genuinely resists checking, mark it as an assumption the
61
- design rests on rather than as something the system guarantees.
62
-
63
- ### Version the specification with the interface
64
-
65
- A specification that no longer matches the code is worse than none,
66
- because it is believed. When behaviour changes, the specification changes
67
- in the same commit, and if it does not, the change is incomplete.
1
+ ---
2
+ name: formal-specification
3
+ description: Saying precisely what a system must do before building it — preconditions, invariants, and the states the specification forbids
4
+ additional_tools: [logic, discrete_math, symbolic_math]
5
+ ---
6
+
7
+ ## Formal specification methodology
8
+
9
+ ### Specify the what, never the how
10
+
11
+ A specification that describes an algorithm has ruled out every other
12
+ correct implementation and cannot be used to judge the one that was
13
+ written. Say what must be true of the result. If the method genuinely
14
+ matters — because a caller depends on its complexity or its ordering —
15
+ that is itself a property to state, not a reason to describe the code.
16
+
17
+ ### Write the contract as three parts
18
+
19
+ - **Precondition** — what must hold when this is called. Whose job is it
20
+ to establish it, the caller's or the callee's? Say which; unowned
21
+ preconditions are checked twice or not at all.
22
+ - **Postcondition** — what is true when it returns, related to the inputs.
23
+ - **Invariant** — what is true before and after, which the operation may
24
+ break in the middle and must restore.
25
+
26
+ The frame matters as much as the three: say what the operation does *not*
27
+ change. Most specifications are silent about it and most defects live
28
+ there.
29
+
30
+ ### Quantifiers, explicitly, with their types
31
+
32
+ "All items are valid" is not a specification until you say all items in
33
+ what, valid by which predicate, and what happens for the empty collection.
34
+ A "for all" quietly becoming a "there exists" is the single most common
35
+ way a specification and its implementation come to disagree while both
36
+ look right.
37
+
38
+ The empty case, the singleton case and the duplicate case should each be
39
+ answered by the text.
40
+
41
+ ### Say what is forbidden, not only what is required
42
+
43
+ The reachable-state question is usually the interesting one: which
44
+ combinations of values must never occur together? Those are the assertions
45
+ worth checking at runtime, and they are what a reviewer needs in order to
46
+ find the path that produces one.
47
+
48
+ ### Make the failure behaviour part of the specification
49
+
50
+ Under what conditions does this refuse, and what does it leave behind when
51
+ it does? "Undefined behaviour on bad input" is a decision with
52
+ consequences, so make it explicitly — and if partial effects are possible
53
+ on failure, say exactly which, because a caller cannot write recovery
54
+ against silence.
55
+
56
+ ### Prefer a testable statement to an elegant one
57
+
58
+ If a property cannot be checked — by a test, an assertion, a type, or a
59
+ proof — it will drift from the implementation without anyone noticing.
60
+ When a property genuinely resists checking, mark it as an assumption the
61
+ design rests on rather than as something the system guarantees.
62
+
63
+ ### Version the specification with the interface
64
+
65
+ A specification that no longer matches the code is worse than none,
66
+ because it is believed. When behaviour changes, the specification changes
67
+ in the same commit, and if it does not, the change is incomplete.
@@ -1,66 +1,66 @@
1
- ---
2
- name: numerical-methods
3
- description: Knowing when a floating-point answer still means something — conditioning, stability, error growth, and the tests that catch a silently wrong result
4
- additional_tools: [symbolic_math, linear_algebra, probability_stats]
5
- ---
6
-
7
- ## Numerical methods methodology
8
-
9
- ### Separate the problem's conditioning from the algorithm's stability
10
-
11
- They are different failures with different remedies. **Conditioning** is a
12
- property of the problem: how much the answer moves when the input moves.
13
- **Stability** is a property of the method: how much error the computation
14
- adds on its own. A stable algorithm on an ill-conditioned problem still
15
- returns a meaningless answer, and no amount of better code fixes it —
16
- reformulating the problem does.
17
-
18
- Say which one you are looking at before proposing a fix.
19
-
20
- ### Catastrophic cancellation is the common cause
21
-
22
- Subtracting two nearly equal numbers destroys the leading digits and
23
- leaves whatever noise they carried. It is why the textbook quadratic
24
- formula fails, why computing variance as `E[x²] − E[x]²` fails, and why a
25
- difference of large accumulated sums fails. Where you see a subtraction of
26
- similar quantities, look for the algebraically equivalent form that does
27
- not perform it.
28
-
29
- ### Never test floating-point values for equality
30
-
31
- Compare against a tolerance, and choose it deliberately: relative
32
- tolerance for values that vary in magnitude, absolute tolerance near zero
33
- where relative tolerance is meaningless, both when the range spans zero.
34
- A tolerance picked to make a test pass is a tolerance that will not catch
35
- the regression it was written for.
36
-
37
- ### Track how error accumulates
38
-
39
- An error bound per operation says little about a loop that runs a million
40
- times. Ask whether error grows linearly, or is amplified each step — and
41
- in an iterative method, whether the iteration is contracting. Report the
42
- growth, not the per-step bound.
43
-
44
- ### Check the units and the scale before the algorithm
45
-
46
- A large fraction of "numerical instability" is a variable in the wrong
47
- units and a matrix whose columns differ by ten orders of magnitude.
48
- Non-dimensionalise, or scale the rows and columns, before reaching for a
49
- more sophisticated solver.
50
-
51
- ### Verify against something independent
52
-
53
- Three cheap checks, roughly in order of value:
54
-
55
- - A case with a known closed-form answer.
56
- - A conservation law or invariant the result must satisfy.
57
- - The same computation at higher precision, or via an independent method.
58
-
59
- Agreement between two implementations of the *same* method proves only
60
- that you transcribed it twice.
61
-
62
- ### Say what the answer's precision actually is
63
-
64
- Reporting ten digits from a computation good to four is a claim about
65
- accuracy, and it is false. Round to what the analysis supports, and say
66
- what that is.
1
+ ---
2
+ name: numerical-methods
3
+ description: Knowing when a floating-point answer still means something — conditioning, stability, error growth, and the tests that catch a silently wrong result
4
+ additional_tools: [symbolic_math, linear_algebra, probability_stats]
5
+ ---
6
+
7
+ ## Numerical methods methodology
8
+
9
+ ### Separate the problem's conditioning from the algorithm's stability
10
+
11
+ They are different failures with different remedies. **Conditioning** is a
12
+ property of the problem: how much the answer moves when the input moves.
13
+ **Stability** is a property of the method: how much error the computation
14
+ adds on its own. A stable algorithm on an ill-conditioned problem still
15
+ returns a meaningless answer, and no amount of better code fixes it —
16
+ reformulating the problem does.
17
+
18
+ Say which one you are looking at before proposing a fix.
19
+
20
+ ### Catastrophic cancellation is the common cause
21
+
22
+ Subtracting two nearly equal numbers destroys the leading digits and
23
+ leaves whatever noise they carried. It is why the textbook quadratic
24
+ formula fails, why computing variance as `E[x²] − E[x]²` fails, and why a
25
+ difference of large accumulated sums fails. Where you see a subtraction of
26
+ similar quantities, look for the algebraically equivalent form that does
27
+ not perform it.
28
+
29
+ ### Never test floating-point values for equality
30
+
31
+ Compare against a tolerance, and choose it deliberately: relative
32
+ tolerance for values that vary in magnitude, absolute tolerance near zero
33
+ where relative tolerance is meaningless, both when the range spans zero.
34
+ A tolerance picked to make a test pass is a tolerance that will not catch
35
+ the regression it was written for.
36
+
37
+ ### Track how error accumulates
38
+
39
+ An error bound per operation says little about a loop that runs a million
40
+ times. Ask whether error grows linearly, or is amplified each step — and
41
+ in an iterative method, whether the iteration is contracting. Report the
42
+ growth, not the per-step bound.
43
+
44
+ ### Check the units and the scale before the algorithm
45
+
46
+ A large fraction of "numerical instability" is a variable in the wrong
47
+ units and a matrix whose columns differ by ten orders of magnitude.
48
+ Non-dimensionalise, or scale the rows and columns, before reaching for a
49
+ more sophisticated solver.
50
+
51
+ ### Verify against something independent
52
+
53
+ Three cheap checks, roughly in order of value:
54
+
55
+ - A case with a known closed-form answer.
56
+ - A conservation law or invariant the result must satisfy.
57
+ - The same computation at higher precision, or via an independent method.
58
+
59
+ Agreement between two implementations of the *same* method proves only
60
+ that you transcribed it twice.
61
+
62
+ ### Say what the answer's precision actually is
63
+
64
+ Reporting ten digits from a computation good to four is a claim about
65
+ accuracy, and it is false. Round to what the analysis supports, and say
66
+ what that is.
@@ -1,62 +1,62 @@
1
- ---
2
- name: experiment-design
3
- description: Designing a comparison that can come out either way — controls, confounds, and stating in advance what would falsify the hypothesis
4
- additional_tools: [probability_stats, arxiv_search]
5
- ---
6
-
7
- ## Experiment design methodology
8
-
9
- ### Write the hypothesis and its negation
10
-
11
- State what you expect, and state what the world looks like if you are
12
- wrong. A hypothesis with no articulable negation is not being tested, it
13
- is being illustrated. Write both before collecting anything, because after
14
- the data arrives every result looks like it was predicted.
15
-
16
- ### Name the comparison
17
-
18
- An experiment measures a difference, so say against what. "The new method
19
- performs well" is not a result; "the new method beats the current one on
20
- this metric, on this data, by this much" is. If there is no control
21
- condition, there is no experiment — there is a demonstration, which is a
22
- legitimate thing to run and a different thing to report.
23
-
24
- ### Enumerate the confounds before you run it
25
-
26
- List everything that differs between conditions besides the variable you
27
- care about: hardware, ordering, time of day, who ran it, which data
28
- arrived first, warm caches. Then say, for each, whether you controlled it,
29
- randomised it, or accepted it. The ones you accepted belong in the write
30
- up, not in a footnote.
31
-
32
- Ordering and warm state are the two that catch experienced people
33
- repeatedly.
34
-
35
- ### Fix the analysis before the data exists
36
-
37
- Decide the metric, the exclusion rule and the stopping rule in advance.
38
- Every one of those chosen after seeing results is a degree of freedom that
39
- inflates the apparent effect, and none of them feels like cheating at the
40
- time — dropping "the obviously broken run" is exactly how it happens.
41
-
42
- If you change the plan after looking, say that you did and report both
43
- analyses.
44
-
45
- ### Estimate what the experiment can detect
46
-
47
- Before running, work out roughly how large an effect this sample could
48
- distinguish from noise. An underpowered experiment that returns "no
49
- difference" has told you nothing, and it is routinely reported as though
50
- it had told you something.
51
-
52
- ### Decide what a null result means
53
-
54
- Say in advance what you will conclude, and do, if the effect is absent. An
55
- experiment whose only useful outcome is confirmation will keep being rerun
56
- with small variations until it confirms.
57
-
58
- ### Record the conditions, not just the numbers
59
-
60
- Versions, seeds, hardware, data snapshot, command line. A result whose
61
- conditions were not recorded cannot be replicated by you either, three
62
- weeks later.
1
+ ---
2
+ name: experiment-design
3
+ description: Designing a comparison that can come out either way — controls, confounds, and stating in advance what would falsify the hypothesis
4
+ additional_tools: [probability_stats, arxiv_search]
5
+ ---
6
+
7
+ ## Experiment design methodology
8
+
9
+ ### Write the hypothesis and its negation
10
+
11
+ State what you expect, and state what the world looks like if you are
12
+ wrong. A hypothesis with no articulable negation is not being tested, it
13
+ is being illustrated. Write both before collecting anything, because after
14
+ the data arrives every result looks like it was predicted.
15
+
16
+ ### Name the comparison
17
+
18
+ An experiment measures a difference, so say against what. "The new method
19
+ performs well" is not a result; "the new method beats the current one on
20
+ this metric, on this data, by this much" is. If there is no control
21
+ condition, there is no experiment — there is a demonstration, which is a
22
+ legitimate thing to run and a different thing to report.
23
+
24
+ ### Enumerate the confounds before you run it
25
+
26
+ List everything that differs between conditions besides the variable you
27
+ care about: hardware, ordering, time of day, who ran it, which data
28
+ arrived first, warm caches. Then say, for each, whether you controlled it,
29
+ randomised it, or accepted it. The ones you accepted belong in the write
30
+ up, not in a footnote.
31
+
32
+ Ordering and warm state are the two that catch experienced people
33
+ repeatedly.
34
+
35
+ ### Fix the analysis before the data exists
36
+
37
+ Decide the metric, the exclusion rule and the stopping rule in advance.
38
+ Every one of those chosen after seeing results is a degree of freedom that
39
+ inflates the apparent effect, and none of them feels like cheating at the
40
+ time — dropping "the obviously broken run" is exactly how it happens.
41
+
42
+ If you change the plan after looking, say that you did and report both
43
+ analyses.
44
+
45
+ ### Estimate what the experiment can detect
46
+
47
+ Before running, work out roughly how large an effect this sample could
48
+ distinguish from noise. An underpowered experiment that returns "no
49
+ difference" has told you nothing, and it is routinely reported as though
50
+ it had told you something.
51
+
52
+ ### Decide what a null result means
53
+
54
+ Say in advance what you will conclude, and do, if the effect is absent. An
55
+ experiment whose only useful outcome is confirmation will keep being rerun
56
+ with small variations until it confirms.
57
+
58
+ ### Record the conditions, not just the numbers
59
+
60
+ Versions, seeds, hardware, data snapshot, command line. A result whose
61
+ conditions were not recorded cannot be replicated by you either, three
62
+ weeks later.
@@ -1,55 +1,55 @@
1
- ---
2
- name: reproducibility
3
- description: Making a result something another person can obtain again — pinned environments, recorded seeds, and provenance for every input
4
- additional_tools: [shell, read]
5
- ---
6
-
7
- ## Reproducibility methodology
8
-
9
- ### The bar is another person, on another machine, later
10
-
11
- Not "it works here". Everything that would differ between your machine and
12
- theirs is either pinned or is a reason the result will not reproduce:
13
- package versions, interpreter version, operating system, hardware, locale,
14
- environment variables, the data.
15
-
16
- ### Pin the environment by exact version
17
-
18
- A range resolves differently next month, so a lockfile or an exact pin is
19
- what makes the record real. Record the resolved versions that actually
20
- ran, not the constraints you asked for — those are different documents,
21
- and only the first one reproduces anything.
22
-
23
- ### Record the seed, and check that it is the only source of randomness
24
-
25
- Set it and log it. Then confirm the run is actually deterministic by
26
- running it twice: seeds are routinely set for one library while a second
27
- source of randomness — hash ordering, thread scheduling, a GPU kernel, a
28
- shuffled data loader — goes unseeded. A recorded seed that does not
29
- determine the output is worse than none, because it looks like control.
30
-
31
- ### Give every input a provenance
32
-
33
- For each dataset: where it came from, when it was retrieved, which version
34
- or snapshot, and what was done to it before it reached the analysis.
35
- "Downloaded from the project website" is not provenance if the file at
36
- that URL changes. Hash it.
37
-
38
- ### Separate the raw data from every derived form
39
-
40
- Never modify the original in place. Derived files are outputs, regenerable
41
- from the raw input by a recorded command; if a derived file cannot be
42
- regenerated, it has become raw data with no provenance.
43
-
44
- ### Make the whole path executable
45
-
46
- One command that goes from inputs to results. A sequence of manual steps
47
- in a README drifts from what was actually run, silently, and the drift is
48
- discovered only when someone tries to reproduce the result and fails —
49
- usually the author, months later.
50
-
51
- ### Report what did not reproduce
52
-
53
- When a rerun gives a different number, that is a finding and belongs in
54
- the write up with its magnitude. Quietly rerunning until it matches is the
55
- failure mode this whole methodology exists to prevent.
1
+ ---
2
+ name: reproducibility
3
+ description: Making a result something another person can obtain again — pinned environments, recorded seeds, and provenance for every input
4
+ additional_tools: [shell, read]
5
+ ---
6
+
7
+ ## Reproducibility methodology
8
+
9
+ ### The bar is another person, on another machine, later
10
+
11
+ Not "it works here". Everything that would differ between your machine and
12
+ theirs is either pinned or is a reason the result will not reproduce:
13
+ package versions, interpreter version, operating system, hardware, locale,
14
+ environment variables, the data.
15
+
16
+ ### Pin the environment by exact version
17
+
18
+ A range resolves differently next month, so a lockfile or an exact pin is
19
+ what makes the record real. Record the resolved versions that actually
20
+ ran, not the constraints you asked for — those are different documents,
21
+ and only the first one reproduces anything.
22
+
23
+ ### Record the seed, and check that it is the only source of randomness
24
+
25
+ Set it and log it. Then confirm the run is actually deterministic by
26
+ running it twice: seeds are routinely set for one library while a second
27
+ source of randomness — hash ordering, thread scheduling, a GPU kernel, a
28
+ shuffled data loader — goes unseeded. A recorded seed that does not
29
+ determine the output is worse than none, because it looks like control.
30
+
31
+ ### Give every input a provenance
32
+
33
+ For each dataset: where it came from, when it was retrieved, which version
34
+ or snapshot, and what was done to it before it reached the analysis.
35
+ "Downloaded from the project website" is not provenance if the file at
36
+ that URL changes. Hash it.
37
+
38
+ ### Separate the raw data from every derived form
39
+
40
+ Never modify the original in place. Derived files are outputs, regenerable
41
+ from the raw input by a recorded command; if a derived file cannot be
42
+ regenerated, it has become raw data with no provenance.
43
+
44
+ ### Make the whole path executable
45
+
46
+ One command that goes from inputs to results. A sequence of manual steps
47
+ in a README drifts from what was actually run, silently, and the drift is
48
+ discovered only when someone tries to reproduce the result and fails —
49
+ usually the author, months later.
50
+
51
+ ### Report what did not reproduce
52
+
53
+ When a rerun gives a different number, that is a finding and belongs in
54
+ the write up with its magnitude. Quietly rerunning until it matches is the
55
+ failure mode this whole methodology exists to prevent.
@@ -1,70 +1,70 @@
1
- ---
2
- name: source-evaluation
3
- description: Judging what a source is worth and saying so at the claim — who is entitled to assert it, how close it is to the point, and what it actually shows
4
- additional_tools: [fetch_url, web_search, arxiv_search]
5
- ---
6
-
7
- ## Source evaluation methodology
8
-
9
- ### Two axes, and both are needed
10
-
11
- A source's worth to a claim is **authority** (is this publisher entitled
12
- to assert this kind of thing) multiplied by **relevance** (does this
13
- passage address the claim at hand). Neither alone is a judgement: a
14
- national statistics office is authoritative and useless if the page you
15
- cited is about something else, and a perfectly on-point paragraph is worth
16
- little if it comes from a content farm.
17
-
18
- This is the same shape the harness's own scorer uses, deliberately, so the
19
- prose and the number do not teach different things.
20
-
21
- ### Rank the publisher by what constrains it
22
-
23
- Roughly, in descending order: restricted registries where identity is
24
- verified before publication (`.gov`, `.mil`, `.edu`, `ac.uk`),
25
- intergovernmental bodies, standards bodies, peer-reviewed publishers,
26
- established news, preprint servers, tertiary references such as
27
- encyclopaedias, general commercial sites, blog platforms, question-and-
28
- answer forums, social media, content farms.
29
-
30
- Two rules keep that from being applied naively. A preprint is not peer
31
- reviewed, whatever the venue's prestige, and saying "published in" of one
32
- is a factual error. A tertiary reference is a **pointer to a source**, not
33
- a source: follow it and cite what it cites, or say explicitly that you are
34
- relying on the summary.
35
-
36
- ### The domain is not the argument
37
-
38
- `evilgov.com` is not a government and `example.ac` is not a university.
39
- Check what the suffix actually is, not what it resembles. A hosted
40
- subdomain inherits the platform's standing, not its host's.
41
-
42
- ### Verify the quote against the page
43
-
44
- If you attribute a sentence to a source, that sentence should be findable
45
- on the page you cite. Paraphrase drifts across a chain of citations
46
- faster than anyone expects, and a quotation that cannot be located is a
47
- finding about the claim, not a formatting problem.
48
-
49
- ### Separate what was measured from what was concluded
50
-
51
- Papers frequently demonstrate something narrow and conclude something
52
- broad, and citations then propagate the conclusion. When you find that
53
- gap, report it — it is often the most valuable single observation
54
- available about a body of literature.
55
-
56
- ### Adjust for a stated reason, and only a little
57
-
58
- You may move a source up or down from its class — a retraction, a
59
- correction, a known conflict of interest, an author who is the primary
60
- authority on exactly this — but the adjustment must be small and it must
61
- come with the reason written down beside it. An unexplained adjustment is
62
- indistinguishable from a preference.
63
-
64
- A retraction is not an adjustment. It floors the source.
65
-
66
- ### Say the strength at the claim, not in a bibliography
67
-
68
- "Reported by a single trade publication, not independently confirmed"
69
- belongs in the sentence making the claim. A reader who has to reconstruct
70
- the evidential weight from a reference list will not do it.
1
+ ---
2
+ name: source-evaluation
3
+ description: Judging what a source is worth and saying so at the claim — who is entitled to assert it, how close it is to the point, and what it actually shows
4
+ additional_tools: [fetch_url, web_search, arxiv_search]
5
+ ---
6
+
7
+ ## Source evaluation methodology
8
+
9
+ ### Two axes, and both are needed
10
+
11
+ A source's worth to a claim is **authority** (is this publisher entitled
12
+ to assert this kind of thing) multiplied by **relevance** (does this
13
+ passage address the claim at hand). Neither alone is a judgement: a
14
+ national statistics office is authoritative and useless if the page you
15
+ cited is about something else, and a perfectly on-point paragraph is worth
16
+ little if it comes from a content farm.
17
+
18
+ This is the same shape the harness's own scorer uses, deliberately, so the
19
+ prose and the number do not teach different things.
20
+
21
+ ### Rank the publisher by what constrains it
22
+
23
+ Roughly, in descending order: restricted registries where identity is
24
+ verified before publication (`.gov`, `.mil`, `.edu`, `ac.uk`),
25
+ intergovernmental bodies, standards bodies, peer-reviewed publishers,
26
+ established news, preprint servers, tertiary references such as
27
+ encyclopaedias, general commercial sites, blog platforms, question-and-
28
+ answer forums, social media, content farms.
29
+
30
+ Two rules keep that from being applied naively. A preprint is not peer
31
+ reviewed, whatever the venue's prestige, and saying "published in" of one
32
+ is a factual error. A tertiary reference is a **pointer to a source**, not
33
+ a source: follow it and cite what it cites, or say explicitly that you are
34
+ relying on the summary.
35
+
36
+ ### The domain is not the argument
37
+
38
+ `evilgov.com` is not a government and `example.ac` is not a university.
39
+ Check what the suffix actually is, not what it resembles. A hosted
40
+ subdomain inherits the platform's standing, not its host's.
41
+
42
+ ### Verify the quote against the page
43
+
44
+ If you attribute a sentence to a source, that sentence should be findable
45
+ on the page you cite. Paraphrase drifts across a chain of citations
46
+ faster than anyone expects, and a quotation that cannot be located is a
47
+ finding about the claim, not a formatting problem.
48
+
49
+ ### Separate what was measured from what was concluded
50
+
51
+ Papers frequently demonstrate something narrow and conclude something
52
+ broad, and citations then propagate the conclusion. When you find that
53
+ gap, report it — it is often the most valuable single observation
54
+ available about a body of literature.
55
+
56
+ ### Adjust for a stated reason, and only a little
57
+
58
+ You may move a source up or down from its class — a retraction, a
59
+ correction, a known conflict of interest, an author who is the primary
60
+ authority on exactly this — but the adjustment must be small and it must
61
+ come with the reason written down beside it. An unexplained adjustment is
62
+ indistinguishable from a preference.
63
+
64
+ A retraction is not an adjustment. It floors the source.
65
+
66
+ ### Say the strength at the claim, not in a bibliography
67
+
68
+ "Reported by a single trade publication, not independently confirmed"
69
+ belongs in the sentence making the claim. A reader who has to reconstruct
70
+ the evidential weight from a reference list will not do it.