@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: cryptography-verification
|
|
3
|
+
description: Checking cryptographic constructions, protocols and claimed security properties
|
|
4
|
+
additional_tools: [symbolic_math, probability_stats, arxiv_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Cryptography verification methodology
|
|
8
|
+
|
|
9
|
+
### State the model before the claim
|
|
10
|
+
|
|
11
|
+
A security claim is meaningless without its setting. Before evaluating
|
|
12
|
+
anything, pin down:
|
|
13
|
+
|
|
14
|
+
- **What the adversary can do** — passive, chosen-plaintext, chosen-
|
|
15
|
+
ciphertext, adaptive; online or offline; with or without side channels.
|
|
16
|
+
- **What "broken" would mean here** — distinguishing from random, key
|
|
17
|
+
recovery, forgery, malleability, or a distinguisher with impractical
|
|
18
|
+
data complexity.
|
|
19
|
+
- **What is assumed hard** — factoring, discrete log, LWE, a hash being a
|
|
20
|
+
random oracle, a cipher being a PRP.
|
|
21
|
+
|
|
22
|
+
A construction is not "secure"; it is secure *against this adversary*
|
|
23
|
+
*under this assumption*. Most bad cryptography analysis is a true
|
|
24
|
+
statement in the wrong model.
|
|
25
|
+
|
|
26
|
+
### Attack the joints, not the primitives
|
|
27
|
+
|
|
28
|
+
AES and SHA-256 are almost never the problem. Look at:
|
|
29
|
+
|
|
30
|
+
- **Nonce and IV handling** — reuse under CTR/GCM is catastrophic and
|
|
31
|
+
common. Ask where the nonce comes from and what happens after a restart.
|
|
32
|
+
- **Key derivation and separation** — one secret used for two purposes.
|
|
33
|
+
- **Padding and error behaviour** — anything that lets an attacker
|
|
34
|
+
distinguish failure modes.
|
|
35
|
+
- **Ordering** — encrypt-then-MAC vs MAC-then-encrypt, and whether the MAC
|
|
36
|
+
covers the associated data, the length, and the nonce.
|
|
37
|
+
- **Comparison** — non-constant-time equality on a tag or an HMAC.
|
|
38
|
+
- **Randomness source** — and whether it is seeded before first use.
|
|
39
|
+
|
|
40
|
+
### Do the arithmetic, do not assert it
|
|
41
|
+
|
|
42
|
+
When a claim rests on a number — a birthday bound, a collision
|
|
43
|
+
probability, a work factor, a data-complexity threshold — compute it.
|
|
44
|
+
Use `symbolic_math` or `probability_stats` rather than reciting a
|
|
45
|
+
remembered figure, and show the expression. Remembered constants are
|
|
46
|
+
where confidently wrong answers come from in this domain.
|
|
47
|
+
|
|
48
|
+
State work factors as base-2 logarithms and say what a bound assumes: a
|
|
49
|
+
2^128 figure that presumes single-target security is a different claim
|
|
50
|
+
from one that survives multi-target attacks.
|
|
51
|
+
|
|
52
|
+
### Never approve a hand-rolled construction
|
|
53
|
+
|
|
54
|
+
If the analysis is of a bespoke scheme, the finding is "use the standard
|
|
55
|
+
construction" plus the specific reason this one differs. Say what the
|
|
56
|
+
standard alternative is (AEAD instead of separate encrypt and MAC, a
|
|
57
|
+
password hash instead of a fast hash with a salt, an existing PAKE instead
|
|
58
|
+
of a custom handshake). Identifying that a bespoke design happens to
|
|
59
|
+
resist the one attack you thought of is not a security result.
|
|
60
|
+
|
|
61
|
+
### What you cannot conclude
|
|
62
|
+
|
|
63
|
+
You cannot conclude a scheme is secure. You can conclude that a specific
|
|
64
|
+
attack does not apply, that a claimed proof has a gap, or that a
|
|
65
|
+
construction deviates from a standard in a way with known consequences.
|
|
66
|
+
Say which of those you did.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: formal-specification
|
|
3
|
+
description: Saying precisely what a system must do before building it — preconditions, invariants, and the states the specification forbids
|
|
4
|
+
additional_tools: [logic, discrete_math, symbolic_math]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Formal specification methodology
|
|
8
|
+
|
|
9
|
+
### Specify the what, never the how
|
|
10
|
+
|
|
11
|
+
A specification that describes an algorithm has ruled out every other
|
|
12
|
+
correct implementation and cannot be used to judge the one that was
|
|
13
|
+
written. Say what must be true of the result. If the method genuinely
|
|
14
|
+
matters — because a caller depends on its complexity or its ordering —
|
|
15
|
+
that is itself a property to state, not a reason to describe the code.
|
|
16
|
+
|
|
17
|
+
### Write the contract as three parts
|
|
18
|
+
|
|
19
|
+
- **Precondition** — what must hold when this is called. Whose job is it
|
|
20
|
+
to establish it, the caller's or the callee's? Say which; unowned
|
|
21
|
+
preconditions are checked twice or not at all.
|
|
22
|
+
- **Postcondition** — what is true when it returns, related to the inputs.
|
|
23
|
+
- **Invariant** — what is true before and after, which the operation may
|
|
24
|
+
break in the middle and must restore.
|
|
25
|
+
|
|
26
|
+
The frame matters as much as the three: say what the operation does *not*
|
|
27
|
+
change. Most specifications are silent about it and most defects live
|
|
28
|
+
there.
|
|
29
|
+
|
|
30
|
+
### Quantifiers, explicitly, with their types
|
|
31
|
+
|
|
32
|
+
"All items are valid" is not a specification until you say all items in
|
|
33
|
+
what, valid by which predicate, and what happens for the empty collection.
|
|
34
|
+
A "for all" quietly becoming a "there exists" is the single most common
|
|
35
|
+
way a specification and its implementation come to disagree while both
|
|
36
|
+
look right.
|
|
37
|
+
|
|
38
|
+
The empty case, the singleton case and the duplicate case should each be
|
|
39
|
+
answered by the text.
|
|
40
|
+
|
|
41
|
+
### Say what is forbidden, not only what is required
|
|
42
|
+
|
|
43
|
+
The reachable-state question is usually the interesting one: which
|
|
44
|
+
combinations of values must never occur together? Those are the assertions
|
|
45
|
+
worth checking at runtime, and they are what a reviewer needs in order to
|
|
46
|
+
find the path that produces one.
|
|
47
|
+
|
|
48
|
+
### Make the failure behaviour part of the specification
|
|
49
|
+
|
|
50
|
+
Under what conditions does this refuse, and what does it leave behind when
|
|
51
|
+
it does? "Undefined behaviour on bad input" is a decision with
|
|
52
|
+
consequences, so make it explicitly — and if partial effects are possible
|
|
53
|
+
on failure, say exactly which, because a caller cannot write recovery
|
|
54
|
+
against silence.
|
|
55
|
+
|
|
56
|
+
### Prefer a testable statement to an elegant one
|
|
57
|
+
|
|
58
|
+
If a property cannot be checked — by a test, an assertion, a type, or a
|
|
59
|
+
proof — it will drift from the implementation without anyone noticing.
|
|
60
|
+
When a property genuinely resists checking, mark it as an assumption the
|
|
61
|
+
design rests on rather than as something the system guarantees.
|
|
62
|
+
|
|
63
|
+
### Version the specification with the interface
|
|
64
|
+
|
|
65
|
+
A specification that no longer matches the code is worse than none,
|
|
66
|
+
because it is believed. When behaviour changes, the specification changes
|
|
67
|
+
in the same commit, and if it does not, the change is incomplete.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: numerical-methods
|
|
3
|
+
description: Knowing when a floating-point answer still means something — conditioning, stability, error growth, and the tests that catch a silently wrong result
|
|
4
|
+
additional_tools: [symbolic_math, linear_algebra, probability_stats]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Numerical methods methodology
|
|
8
|
+
|
|
9
|
+
### Separate the problem's conditioning from the algorithm's stability
|
|
10
|
+
|
|
11
|
+
They are different failures with different remedies. **Conditioning** is a
|
|
12
|
+
property of the problem: how much the answer moves when the input moves.
|
|
13
|
+
**Stability** is a property of the method: how much error the computation
|
|
14
|
+
adds on its own. A stable algorithm on an ill-conditioned problem still
|
|
15
|
+
returns a meaningless answer, and no amount of better code fixes it —
|
|
16
|
+
reformulating the problem does.
|
|
17
|
+
|
|
18
|
+
Say which one you are looking at before proposing a fix.
|
|
19
|
+
|
|
20
|
+
### Catastrophic cancellation is the common cause
|
|
21
|
+
|
|
22
|
+
Subtracting two nearly equal numbers destroys the leading digits and
|
|
23
|
+
leaves whatever noise they carried. It is why the textbook quadratic
|
|
24
|
+
formula fails, why computing variance as `E[x²] − E[x]²` fails, and why a
|
|
25
|
+
difference of large accumulated sums fails. Where you see a subtraction of
|
|
26
|
+
similar quantities, look for the algebraically equivalent form that does
|
|
27
|
+
not perform it.
|
|
28
|
+
|
|
29
|
+
### Never test floating-point values for equality
|
|
30
|
+
|
|
31
|
+
Compare against a tolerance, and choose it deliberately: relative
|
|
32
|
+
tolerance for values that vary in magnitude, absolute tolerance near zero
|
|
33
|
+
where relative tolerance is meaningless, both when the range spans zero.
|
|
34
|
+
A tolerance picked to make a test pass is a tolerance that will not catch
|
|
35
|
+
the regression it was written for.
|
|
36
|
+
|
|
37
|
+
### Track how error accumulates
|
|
38
|
+
|
|
39
|
+
An error bound per operation says little about a loop that runs a million
|
|
40
|
+
times. Ask whether error grows linearly, or is amplified each step — and
|
|
41
|
+
in an iterative method, whether the iteration is contracting. Report the
|
|
42
|
+
growth, not the per-step bound.
|
|
43
|
+
|
|
44
|
+
### Check the units and the scale before the algorithm
|
|
45
|
+
|
|
46
|
+
A large fraction of "numerical instability" is a variable in the wrong
|
|
47
|
+
units and a matrix whose columns differ by ten orders of magnitude.
|
|
48
|
+
Non-dimensionalise, or scale the rows and columns, before reaching for a
|
|
49
|
+
more sophisticated solver.
|
|
50
|
+
|
|
51
|
+
### Verify against something independent
|
|
52
|
+
|
|
53
|
+
Three cheap checks, roughly in order of value:
|
|
54
|
+
|
|
55
|
+
- A case with a known closed-form answer.
|
|
56
|
+
- A conservation law or invariant the result must satisfy.
|
|
57
|
+
- The same computation at higher precision, or via an independent method.
|
|
58
|
+
|
|
59
|
+
Agreement between two implementations of the *same* method proves only
|
|
60
|
+
that you transcribed it twice.
|
|
61
|
+
|
|
62
|
+
### Say what the answer's precision actually is
|
|
63
|
+
|
|
64
|
+
Reporting ten digits from a computation good to four is a claim about
|
|
65
|
+
accuracy, and it is false. Round to what the analysis supports, and say
|
|
66
|
+
what that is.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: proof-writing
|
|
3
|
+
description: Structuring and checking a formal mathematical argument
|
|
4
|
+
additional_tools: [symbolic_math, linear_algebra, discrete_math, logic]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Proof writing methodology
|
|
8
|
+
|
|
9
|
+
### Fix the statement before proving anything
|
|
10
|
+
|
|
11
|
+
Write the claim as a single sentence with every quantifier explicit and
|
|
12
|
+
every symbol typed. Most flawed proofs are proofs of a subtly different
|
|
13
|
+
statement — one where a "for all" quietly became a "there exists", or a
|
|
14
|
+
hypothesis that was used but never stated.
|
|
15
|
+
|
|
16
|
+
List the hypotheses separately from the conclusion. If a hypothesis is
|
|
17
|
+
never used in the proof, either the proof is wrong or the statement is
|
|
18
|
+
weaker than claimed; say which.
|
|
19
|
+
|
|
20
|
+
### Choose the shape deliberately
|
|
21
|
+
|
|
22
|
+
Name the proof technique before writing: direct, contrapositive,
|
|
23
|
+
contradiction, induction, construction, exhaustion of cases. If it is
|
|
24
|
+
induction, state the induction variable, the base case, and the induction
|
|
25
|
+
hypothesis explicitly — a proof that says "by induction" and then argues
|
|
26
|
+
informally is where off-by-one errors live. If it is case analysis, show
|
|
27
|
+
that the cases are exhaustive.
|
|
28
|
+
|
|
29
|
+
### One step, one justification
|
|
30
|
+
|
|
31
|
+
Every non-trivial step cites the definition, lemma or theorem it uses. A
|
|
32
|
+
step that cannot be justified is the interesting part of the problem, not
|
|
33
|
+
a detail to move past — flag it rather than smoothing over it with
|
|
34
|
+
"clearly" or "it follows that".
|
|
35
|
+
|
|
36
|
+
Reserve "without loss of generality" for cases where you can say what the
|
|
37
|
+
symmetry is. It is frequently used to skip a case that is not actually
|
|
38
|
+
symmetric.
|
|
39
|
+
|
|
40
|
+
### Check the algebra with the tools, not from memory
|
|
41
|
+
|
|
42
|
+
Use `symbolic_math` for identities and manipulations, `linear_algebra` for
|
|
43
|
+
matrix claims, `discrete_math` for counting and number-theoretic steps,
|
|
44
|
+
and `logic` for checking that an implication chain is valid. An
|
|
45
|
+
algebraically wrong line invalidates everything after it, and it is the
|
|
46
|
+
cheapest class of error to eliminate.
|
|
47
|
+
|
|
48
|
+
### Attack your own argument before presenting it
|
|
49
|
+
|
|
50
|
+
Try the boundary cases: empty set, zero, one, equality in a strict
|
|
51
|
+
inequality, the degenerate configuration. Try to construct a
|
|
52
|
+
counterexample for ten seconds before concluding. If the proof would still
|
|
53
|
+
"work" with a hypothesis removed, the proof is probably wrong — that is
|
|
54
|
+
the sharpest self-check available.
|
|
55
|
+
|
|
56
|
+
### Report honestly
|
|
57
|
+
|
|
58
|
+
If a step is unproven, say which step, and state what would close it. A
|
|
59
|
+
proof with one clearly marked gap is a useful contribution; a proof with
|
|
60
|
+
one unmarked gap is worse than no proof, because it spends the reader's
|
|
61
|
+
trust.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: experiment-design
|
|
3
|
+
description: Designing a comparison that can come out either way — controls, confounds, and stating in advance what would falsify the hypothesis
|
|
4
|
+
additional_tools: [probability_stats, arxiv_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Experiment design methodology
|
|
8
|
+
|
|
9
|
+
### Write the hypothesis and its negation
|
|
10
|
+
|
|
11
|
+
State what you expect, and state what the world looks like if you are
|
|
12
|
+
wrong. A hypothesis with no articulable negation is not being tested, it
|
|
13
|
+
is being illustrated. Write both before collecting anything, because after
|
|
14
|
+
the data arrives every result looks like it was predicted.
|
|
15
|
+
|
|
16
|
+
### Name the comparison
|
|
17
|
+
|
|
18
|
+
An experiment measures a difference, so say against what. "The new method
|
|
19
|
+
performs well" is not a result; "the new method beats the current one on
|
|
20
|
+
this metric, on this data, by this much" is. If there is no control
|
|
21
|
+
condition, there is no experiment — there is a demonstration, which is a
|
|
22
|
+
legitimate thing to run and a different thing to report.
|
|
23
|
+
|
|
24
|
+
### Enumerate the confounds before you run it
|
|
25
|
+
|
|
26
|
+
List everything that differs between conditions besides the variable you
|
|
27
|
+
care about: hardware, ordering, time of day, who ran it, which data
|
|
28
|
+
arrived first, warm caches. Then say, for each, whether you controlled it,
|
|
29
|
+
randomised it, or accepted it. The ones you accepted belong in the write
|
|
30
|
+
up, not in a footnote.
|
|
31
|
+
|
|
32
|
+
Ordering and warm state are the two that catch experienced people
|
|
33
|
+
repeatedly.
|
|
34
|
+
|
|
35
|
+
### Fix the analysis before the data exists
|
|
36
|
+
|
|
37
|
+
Decide the metric, the exclusion rule and the stopping rule in advance.
|
|
38
|
+
Every one of those chosen after seeing results is a degree of freedom that
|
|
39
|
+
inflates the apparent effect, and none of them feels like cheating at the
|
|
40
|
+
time — dropping "the obviously broken run" is exactly how it happens.
|
|
41
|
+
|
|
42
|
+
If you change the plan after looking, say that you did and report both
|
|
43
|
+
analyses.
|
|
44
|
+
|
|
45
|
+
### Estimate what the experiment can detect
|
|
46
|
+
|
|
47
|
+
Before running, work out roughly how large an effect this sample could
|
|
48
|
+
distinguish from noise. An underpowered experiment that returns "no
|
|
49
|
+
difference" has told you nothing, and it is routinely reported as though
|
|
50
|
+
it had told you something.
|
|
51
|
+
|
|
52
|
+
### Decide what a null result means
|
|
53
|
+
|
|
54
|
+
Say in advance what you will conclude, and do, if the effect is absent. An
|
|
55
|
+
experiment whose only useful outcome is confirmation will keep being rerun
|
|
56
|
+
with small variations until it confirms.
|
|
57
|
+
|
|
58
|
+
### Record the conditions, not just the numbers
|
|
59
|
+
|
|
60
|
+
Versions, seeds, hardware, data snapshot, command line. A result whose
|
|
61
|
+
conditions were not recorded cannot be replicated by you either, three
|
|
62
|
+
weeks later.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: literature-review
|
|
3
|
+
description: Surveying a body of published work without overstating what it shows
|
|
4
|
+
additional_tools: [arxiv_search, web_search, fetch_url]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Literature review methodology
|
|
8
|
+
|
|
9
|
+
### Scope the question first
|
|
10
|
+
|
|
11
|
+
Write the question the review answers, and the boundary conditions:
|
|
12
|
+
timeframe, subfield, what counts as in scope, and what you are
|
|
13
|
+
deliberately excluding. A review without a stated boundary reads as
|
|
14
|
+
comprehensive whether or not it is, which is the failure mode that matters
|
|
15
|
+
most here.
|
|
16
|
+
|
|
17
|
+
### Search more than one way
|
|
18
|
+
|
|
19
|
+
The same body of work is indexed under different vocabulary by different
|
|
20
|
+
communities. Search by concept, by the standard term, by the older term it
|
|
21
|
+
replaced, by a landmark paper's citations forward, and by its references
|
|
22
|
+
backward. Note which searches you actually ran — a gap in coverage that is
|
|
23
|
+
recorded is a limitation, and one that is not is a false claim of
|
|
24
|
+
completeness.
|
|
25
|
+
|
|
26
|
+
### Read at the right depth, and say which
|
|
27
|
+
|
|
28
|
+
Be explicit about how deeply you engaged with each source:
|
|
29
|
+
|
|
30
|
+
- **Read** — you have the full text and the argument.
|
|
31
|
+
- **Skimmed** — abstract, figures and conclusions.
|
|
32
|
+
- **Cited only** — you know it exists and what others say it shows.
|
|
33
|
+
|
|
34
|
+
Never describe a result in the voice of having read the paper when you
|
|
35
|
+
have read only its abstract. If a claim rests on a source you could not
|
|
36
|
+
retrieve, say so at the claim, not in a footnote.
|
|
37
|
+
|
|
38
|
+
### Distinguish the finding from the framing
|
|
39
|
+
|
|
40
|
+
Separate what a paper measured from what it concluded and from how it has
|
|
41
|
+
since been cited. Drift between those three is extremely common: a result
|
|
42
|
+
demonstrated under narrow conditions becomes a general claim within two
|
|
43
|
+
citation hops. When you find that drift, it is one of the more valuable
|
|
44
|
+
things a review can report.
|
|
45
|
+
|
|
46
|
+
### Represent disagreement as disagreement
|
|
47
|
+
|
|
48
|
+
When sources conflict, report the conflict and what would settle it —
|
|
49
|
+
different populations, different measures, different assumptions,
|
|
50
|
+
different eras of the field. Averaging into a bland consensus sentence
|
|
51
|
+
destroys the most informative part of the picture. Note when apparent
|
|
52
|
+
agreement comes from a shared upstream source rather than independent
|
|
53
|
+
replication.
|
|
54
|
+
|
|
55
|
+
### Weight by evidence, not by prominence
|
|
56
|
+
|
|
57
|
+
Citation count measures attention, not correctness. Prefer replication,
|
|
58
|
+
pre-registration, adversarial collaboration and released data over venue
|
|
59
|
+
prestige. A well-cited result that has never been replicated should be
|
|
60
|
+
described that way.
|
|
61
|
+
|
|
62
|
+
### Report the shape of the field
|
|
63
|
+
|
|
64
|
+
A good review says where the evidence is thick, where it is thin, and
|
|
65
|
+
where nobody has looked. Naming the gap is often the deliverable.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: reproducibility
|
|
3
|
+
description: Making a result something another person can obtain again — pinned environments, recorded seeds, and provenance for every input
|
|
4
|
+
additional_tools: [shell, read]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Reproducibility methodology
|
|
8
|
+
|
|
9
|
+
### The bar is another person, on another machine, later
|
|
10
|
+
|
|
11
|
+
Not "it works here". Everything that would differ between your machine and
|
|
12
|
+
theirs is either pinned or is a reason the result will not reproduce:
|
|
13
|
+
package versions, interpreter version, operating system, hardware, locale,
|
|
14
|
+
environment variables, the data.
|
|
15
|
+
|
|
16
|
+
### Pin the environment by exact version
|
|
17
|
+
|
|
18
|
+
A range resolves differently next month, so a lockfile or an exact pin is
|
|
19
|
+
what makes the record real. Record the resolved versions that actually
|
|
20
|
+
ran, not the constraints you asked for — those are different documents,
|
|
21
|
+
and only the first one reproduces anything.
|
|
22
|
+
|
|
23
|
+
### Record the seed, and check that it is the only source of randomness
|
|
24
|
+
|
|
25
|
+
Set it and log it. Then confirm the run is actually deterministic by
|
|
26
|
+
running it twice: seeds are routinely set for one library while a second
|
|
27
|
+
source of randomness — hash ordering, thread scheduling, a GPU kernel, a
|
|
28
|
+
shuffled data loader — goes unseeded. A recorded seed that does not
|
|
29
|
+
determine the output is worse than none, because it looks like control.
|
|
30
|
+
|
|
31
|
+
### Give every input a provenance
|
|
32
|
+
|
|
33
|
+
For each dataset: where it came from, when it was retrieved, which version
|
|
34
|
+
or snapshot, and what was done to it before it reached the analysis.
|
|
35
|
+
"Downloaded from the project website" is not provenance if the file at
|
|
36
|
+
that URL changes. Hash it.
|
|
37
|
+
|
|
38
|
+
### Separate the raw data from every derived form
|
|
39
|
+
|
|
40
|
+
Never modify the original in place. Derived files are outputs, regenerable
|
|
41
|
+
from the raw input by a recorded command; if a derived file cannot be
|
|
42
|
+
regenerated, it has become raw data with no provenance.
|
|
43
|
+
|
|
44
|
+
### Make the whole path executable
|
|
45
|
+
|
|
46
|
+
One command that goes from inputs to results. A sequence of manual steps
|
|
47
|
+
in a README drifts from what was actually run, silently, and the drift is
|
|
48
|
+
discovered only when someone tries to reproduce the result and fails —
|
|
49
|
+
usually the author, months later.
|
|
50
|
+
|
|
51
|
+
### Report what did not reproduce
|
|
52
|
+
|
|
53
|
+
When a rerun gives a different number, that is a finding and belongs in
|
|
54
|
+
the write up with its magnitude. Quietly rerunning until it matches is the
|
|
55
|
+
failure mode this whole methodology exists to prevent.
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: source-evaluation
|
|
3
|
+
description: Judging what a source is worth and saying so at the claim — who is entitled to assert it, how close it is to the point, and what it actually shows
|
|
4
|
+
additional_tools: [fetch_url, web_search, arxiv_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Source evaluation methodology
|
|
8
|
+
|
|
9
|
+
### Two axes, and both are needed
|
|
10
|
+
|
|
11
|
+
A source's worth to a claim is **authority** (is this publisher entitled
|
|
12
|
+
to assert this kind of thing) multiplied by **relevance** (does this
|
|
13
|
+
passage address the claim at hand). Neither alone is a judgement: a
|
|
14
|
+
national statistics office is authoritative and useless if the page you
|
|
15
|
+
cited is about something else, and a perfectly on-point paragraph is worth
|
|
16
|
+
little if it comes from a content farm.
|
|
17
|
+
|
|
18
|
+
This is the same shape the harness's own scorer uses, deliberately, so the
|
|
19
|
+
prose and the number do not teach different things.
|
|
20
|
+
|
|
21
|
+
### Rank the publisher by what constrains it
|
|
22
|
+
|
|
23
|
+
Roughly, in descending order: restricted registries where identity is
|
|
24
|
+
verified before publication (`.gov`, `.mil`, `.edu`, `ac.uk`),
|
|
25
|
+
intergovernmental bodies, standards bodies, peer-reviewed publishers,
|
|
26
|
+
established news, preprint servers, tertiary references such as
|
|
27
|
+
encyclopaedias, general commercial sites, blog platforms, question-and-
|
|
28
|
+
answer forums, social media, content farms.
|
|
29
|
+
|
|
30
|
+
Two rules keep that from being applied naively. A preprint is not peer
|
|
31
|
+
reviewed, whatever the venue's prestige, and saying "published in" of one
|
|
32
|
+
is a factual error. A tertiary reference is a **pointer to a source**, not
|
|
33
|
+
a source: follow it and cite what it cites, or say explicitly that you are
|
|
34
|
+
relying on the summary.
|
|
35
|
+
|
|
36
|
+
### The domain is not the argument
|
|
37
|
+
|
|
38
|
+
`evilgov.com` is not a government and `example.ac` is not a university.
|
|
39
|
+
Check what the suffix actually is, not what it resembles. A hosted
|
|
40
|
+
subdomain inherits the platform's standing, not its host's.
|
|
41
|
+
|
|
42
|
+
### Verify the quote against the page
|
|
43
|
+
|
|
44
|
+
If you attribute a sentence to a source, that sentence should be findable
|
|
45
|
+
on the page you cite. Paraphrase drifts across a chain of citations
|
|
46
|
+
faster than anyone expects, and a quotation that cannot be located is a
|
|
47
|
+
finding about the claim, not a formatting problem.
|
|
48
|
+
|
|
49
|
+
### Separate what was measured from what was concluded
|
|
50
|
+
|
|
51
|
+
Papers frequently demonstrate something narrow and conclude something
|
|
52
|
+
broad, and citations then propagate the conclusion. When you find that
|
|
53
|
+
gap, report it — it is often the most valuable single observation
|
|
54
|
+
available about a body of literature.
|
|
55
|
+
|
|
56
|
+
### Adjust for a stated reason, and only a little
|
|
57
|
+
|
|
58
|
+
You may move a source up or down from its class — a retraction, a
|
|
59
|
+
correction, a known conflict of interest, an author who is the primary
|
|
60
|
+
authority on exactly this — but the adjustment must be small and it must
|
|
61
|
+
come with the reason written down beside it. An unexplained adjustment is
|
|
62
|
+
indistinguishable from a preference.
|
|
63
|
+
|
|
64
|
+
A retraction is not an adjustment. It floors the source.
|
|
65
|
+
|
|
66
|
+
### Say the strength at the claim, not in a bibliography
|
|
67
|
+
|
|
68
|
+
"Reported by a single trade publication, not independently confirmed"
|
|
69
|
+
belongs in the sentence making the claim. A reader who has to reconstruct
|
|
70
|
+
the evidential weight from a reference list will not do it.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: statistical-inference
|
|
3
|
+
description: Reasoning from data to a claim without overstating it — effect sizes, uncertainty, multiplicity, and what a p-value does not say
|
|
4
|
+
additional_tools: [probability_stats, symbolic_math]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Statistical inference methodology
|
|
8
|
+
|
|
9
|
+
### Report the effect, then the uncertainty, then the test
|
|
10
|
+
|
|
11
|
+
The size of the difference is the finding. The interval around it is how
|
|
12
|
+
much to trust the finding. A test statistic is the least informative of
|
|
13
|
+
the three and is routinely reported alone. Lead with "3.2 points better,
|
|
14
|
+
95% CI [0.4, 6.0]", not with "significant".
|
|
15
|
+
|
|
16
|
+
### Say what the p-value is about
|
|
17
|
+
|
|
18
|
+
It is the probability of data at least this extreme under the null
|
|
19
|
+
hypothesis. It is not the probability that the null is true, not the
|
|
20
|
+
probability the result will replicate, and not a measure of effect size. A
|
|
21
|
+
tiny p-value from a huge sample can accompany an effect too small to
|
|
22
|
+
matter, and that combination is one of the most common ways a true
|
|
23
|
+
statement misleads.
|
|
24
|
+
|
|
25
|
+
### Count every comparison you made
|
|
26
|
+
|
|
27
|
+
Testing twenty things at the 0.05 level and reporting the one that came
|
|
28
|
+
out is not a finding, whether the other nineteen were preregistered,
|
|
29
|
+
exploratory, or merely looked at. Say how many comparisons were made and
|
|
30
|
+
correct for them, or label the result exploratory and say it needs
|
|
31
|
+
confirmation on new data.
|
|
32
|
+
|
|
33
|
+
Subgroups, alternative metrics, and "we also tried" all count.
|
|
34
|
+
|
|
35
|
+
### Check the assumptions the method actually makes
|
|
36
|
+
|
|
37
|
+
Independence is the one that fails most often and is questioned least:
|
|
38
|
+
repeated measures on the same subject, runs on the same machine, samples
|
|
39
|
+
drawn from one crawl. Normality usually matters less than people fear;
|
|
40
|
+
independence usually matters more.
|
|
41
|
+
|
|
42
|
+
Say which assumptions you checked and which you assumed.
|
|
43
|
+
|
|
44
|
+
### Absence of significance is not evidence of absence
|
|
45
|
+
|
|
46
|
+
A non-significant result with a wide interval is compatible with a large
|
|
47
|
+
effect. Report the interval and say what the study could have detected. If
|
|
48
|
+
you mean "we can rule out an effect larger than X", say that — it is a
|
|
49
|
+
much stronger and much more useful claim, and it requires the interval to
|
|
50
|
+
support it.
|
|
51
|
+
|
|
52
|
+
### Keep the estimate separate from the decision
|
|
53
|
+
|
|
54
|
+
"How large is the effect" and "what should we do" are different questions
|
|
55
|
+
with different inputs. A threshold that decides an action belongs to the
|
|
56
|
+
decision, with its costs stated, not smuggled into the analysis as though
|
|
57
|
+
it were a property of the data.
|
|
58
|
+
|
|
59
|
+
### Distinguish measured, modelled and assumed
|
|
60
|
+
|
|
61
|
+
At each number, say which it is. A modelled quantity presented in the same
|
|
62
|
+
voice as a measured one is the most reliable way to make an analysis look
|
|
63
|
+
sturdier than it is.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: technical-writing
|
|
3
|
+
description: Writing up a result so a reader can act on it — the claim first, the evidence attached, and the limitation stated rather than implied
|
|
4
|
+
additional_tools: [read, web_search]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Technical writing methodology
|
|
8
|
+
|
|
9
|
+
### Lead with the finding
|
|
10
|
+
|
|
11
|
+
The first sentence should be the thing you learned, not the thing you did.
|
|
12
|
+
"Compaction folded a 1M-window thread at 4% of its window" is a finding;
|
|
13
|
+
"we investigated compaction behaviour" is a table of contents. A reader
|
|
14
|
+
who stops after one paragraph should still have the result.
|
|
15
|
+
|
|
16
|
+
### Attach the evidence to the claim
|
|
17
|
+
|
|
18
|
+
Every non-obvious assertion carries, in the same place, what supports it:
|
|
19
|
+
the measurement, the file and line, the citation, the command. Evidence
|
|
20
|
+
gathered into a section at the end forces the reader to reconstruct which
|
|
21
|
+
support belongs to which claim, and most will not.
|
|
22
|
+
|
|
23
|
+
### State limitations where they bite
|
|
24
|
+
|
|
25
|
+
A limitation section at the end is read as a formality. The same sentence
|
|
26
|
+
placed beside the claim it qualifies changes how the claim is read, which
|
|
27
|
+
is the point of writing it at all. "On this dataset only" belongs in the
|
|
28
|
+
sentence, not on page nine.
|
|
29
|
+
|
|
30
|
+
### Distinguish measured, inferred and assumed
|
|
31
|
+
|
|
32
|
+
Use different words for them and use them consistently. A modelled number
|
|
33
|
+
in the same voice as a measured one is the most reliable way to make work
|
|
34
|
+
look sturdier than it is, and it is almost always accidental.
|
|
35
|
+
|
|
36
|
+
### Write the negative results
|
|
37
|
+
|
|
38
|
+
What did not work, what was tried and abandoned, and why. It is the part
|
|
39
|
+
most often cut and the part most useful to the next person, who will
|
|
40
|
+
otherwise spend their time rediscovering it. A short paragraph is enough.
|
|
41
|
+
|
|
42
|
+
### One idea per paragraph, and put it first
|
|
43
|
+
|
|
44
|
+
The reader is skimming. If the first sentence of each paragraph carries
|
|
45
|
+
its point, skimming works and the document survives being read properly
|
|
46
|
+
later. If the point arrives at the end of the paragraph, the skim returns
|
|
47
|
+
nothing.
|
|
48
|
+
|
|
49
|
+
### Define a term once, then be boring
|
|
50
|
+
|
|
51
|
+
Pick one name for each thing and reuse it. Elegant variation — calling the
|
|
52
|
+
same object three different names across four paragraphs — reads well in
|
|
53
|
+
prose and is actively harmful in technical writing, because the reader
|
|
54
|
+
cannot tell whether you meant something different.
|
|
55
|
+
|
|
56
|
+
### Say what you would need to be more confident
|
|
57
|
+
|
|
58
|
+
Ending with the specific thing that would settle the remaining doubt is
|
|
59
|
+
more useful than a summary that restates what the reader just read.
|