@giamme/forge 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +181 -0
- package/LICENSE +21 -0
- package/README.md +1854 -2
- package/SKILL.md +122 -0
- package/VERSION +1 -0
- package/forge +74 -0
- package/package.json +26 -4
- package/references/dataset.md +329 -0
- package/references/decompose.md +667 -0
- package/references/fractal-dashboard.md +21 -0
- package/references/fractal.md +307 -0
- package/references/harnesses.md +353 -0
- package/references/jev.md +865 -0
- package/references/memory.md +228 -0
- package/references/planning.md +42 -0
- package/references/ripwire.md +47 -0
- package/references/runtime.md +64 -0
- package/references/solo.md +113 -0
- package/references/ui.md +112 -0
- package/registry.tsv +81 -0
- package/scripts/forge-artifact.sh +78 -0
- package/scripts/forge-contract.py +430 -0
- package/scripts/forge-dataset.py +6 -0
- package/scripts/forge-dispatch.sh +665 -0
- package/scripts/forge-fail-first.py +195 -0
- package/scripts/forge-fractal-dispatch.sh +4 -0
- package/scripts/forge-fractal-options.sh +54 -0
- package/scripts/forge-fractal.py +6 -0
- package/scripts/forge-guard.py +266 -0
- package/scripts/forge-install-fractal.sh +4 -0
- package/scripts/forge-install-ripwire.sh +52 -0
- package/scripts/forge-install.sh +135 -0
- package/scripts/forge-jev-options.sh +259 -0
- package/scripts/forge-jev.py +6 -0
- package/scripts/forge-lib-plans.sh +785 -0
- package/scripts/forge-lib-review.sh +494 -0
- package/scripts/forge-lib-task.sh +302 -0
- package/scripts/forge-lib-verify.sh +30 -0
- package/scripts/forge-memory.sh +575 -0
- package/scripts/forge-metrics.sh +39 -0
- package/scripts/forge-parallel-options.sh +76 -0
- package/scripts/forge-parallel.sh +1717 -0
- package/scripts/forge-plan-check.py +118 -0
- package/scripts/forge-prompt.py +45 -0
- package/scripts/forge-ripwire.py +136 -0
- package/scripts/forge-route.py +344 -0
- package/scripts/forge-runtime.py +533 -0
- package/scripts/forge-schedule.py +312 -0
- package/scripts/forge-shadow.py +305 -0
- package/scripts/forge-solo.sh +347 -0
- package/scripts/forge-ui-options.sh +42 -0
- package/scripts/forge-ui.py +435 -0
- package/scripts/forge-update.sh +71 -0
- package/scripts/forge-verify-coverage.py +956 -0
- package/scripts/forge-version.py +198 -0
- package/scripts/forge_dataset/__init__.py +439 -0
- package/scripts/forge_dataset/capture.py +330 -0
- package/scripts/forge_dataset/checks.py +109 -0
- package/scripts/forge_dataset/cli.py +233 -0
- package/scripts/forge_dataset/export.py +1009 -0
- package/scripts/forge_dataset/harvest.py +716 -0
- package/scripts/forge_dataset/importer.py +412 -0
- package/scripts/forge_dataset/outcomes.py +275 -0
- package/scripts/forge_dataset/routing.py +278 -0
- package/scripts/forge_fractal/__init__.py +83 -0
- package/scripts/forge_fractal/bridge.py +46 -0
- package/scripts/forge_fractal/cli.py +202 -0
- package/scripts/forge_fractal/dashboard.css +138 -0
- package/scripts/forge_fractal/dashboard.html +41 -0
- package/scripts/forge_fractal/dashboard.js +687 -0
- package/scripts/forge_fractal/decomposition.py +262 -0
- package/scripts/forge_fractal/execution.py +659 -0
- package/scripts/forge_fractal/inspection.py +261 -0
- package/scripts/forge_fractal/install.py +158 -0
- package/scripts/forge_fractal/invocation.py +94 -0
- package/scripts/forge_fractal/pipeline.py +124 -0
- package/scripts/forge_fractal/selection.py +179 -0
- package/scripts/forge_jev/__init__.py +376 -0
- package/scripts/forge_jev/backtest.py +322 -0
- package/scripts/forge_jev/cli.py +980 -0
- package/scripts/forge_jev/client.py +418 -0
- package/scripts/forge_jev/corpus.py +156 -0
- package/scripts/forge_jev/coupling.py +244 -0
- package/scripts/forge_jev/features.py +24 -0
- package/scripts/forge_jev/gates.py +100 -0
- package/scripts/forge_jev/memory.py +163 -0
- package/scripts/forge_jev/outcome.py +49 -0
- package/scripts/forge_jev/predict.py +204 -0
- package/scripts/forge_jev/questions.py +661 -0
- package/scripts/forge_jev/review.py +73 -0
- package/scripts/forge_jev/routing.py +271 -0
- package/scripts/forge_jev/subset.py +123 -0
- package/scripts/forge_jev/verify.py +453 -0
- package/scripts/forge_registry.py +140 -0
- package/ui/README.md +287 -0
- package/ui/THIRD_PARTY_NOTICES.md +69 -0
- package/ui/docs/api.md +888 -0
- package/ui/forge_ui/__init__.py +20 -0
- package/ui/forge_ui/__main__.py +135 -0
- package/ui/forge_ui/diffs.py +454 -0
- package/ui/forge_ui/executions.py +436 -0
- package/ui/forge_ui/export.py +204 -0
- package/ui/forge_ui/findings.py +132 -0
- package/ui/forge_ui/fractal.py +583 -0
- package/ui/forge_ui/models.py +216 -0
- package/ui/forge_ui/reader.py +1336 -0
- package/ui/forge_ui/server.py +303 -0
- package/ui/forge_ui/static/app.js +924 -0
- package/ui/forge_ui/static/cloud3d.js +646 -0
- package/ui/forge_ui/static/cloudmodel.js +375 -0
- package/ui/forge_ui/static/cloudui.js +394 -0
- package/ui/forge_ui/static/diffview.js +216 -0
- package/ui/forge_ui/static/favicon.svg +1 -0
- package/ui/forge_ui/static/files.js +776 -0
- package/ui/forge_ui/static/gantt.js +97 -0
- package/ui/forge_ui/static/graph.js +266 -0
- package/ui/forge_ui/static/index.html +55 -0
- package/ui/forge_ui/static/iso.js +765 -0
- package/ui/forge_ui/static/leaderboard.js +315 -0
- package/ui/forge_ui/static/map3d.js +45 -0
- package/ui/forge_ui/static/modelstats.js +249 -0
- package/ui/forge_ui/static/notify.js +179 -0
- package/ui/forge_ui/static/panel.js +526 -0
- package/ui/forge_ui/static/runmodel.js +651 -0
- package/ui/forge_ui/static/scene3d.js +996 -0
- package/ui/forge_ui/static/search.js +119 -0
- package/ui/forge_ui/static/style.css +618 -0
package/registry.tsv
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# forge model registry — single source of truth for alias resolution.
|
|
2
|
+
#
|
|
3
|
+
# Columns (tab-separated):
|
|
4
|
+
# alias short name you type in --dwarf / --qa
|
|
5
|
+
# harness which CLI runs it: codex | claude | openclaude | opencode | antigravity
|
|
6
|
+
# model the exact model id that harness expects
|
|
7
|
+
# effort which reasoning efforts this pairing accepts, in one of four forms:
|
|
8
|
+
# <=X any effort on the harness ladder up to and including X
|
|
9
|
+
# a,b,c exactly these efforts and no others (gaps are real: some
|
|
10
|
+
# models offer low and high but not medium)
|
|
11
|
+
# none this model takes no effort flag at all; passing one errors
|
|
12
|
+
# - harness has no validated ladder; forwarded verbatim
|
|
13
|
+
# timeout OPTIONAL fifth column: the watchdog limit in seconds for this pairing.
|
|
14
|
+
# Leave it off (existing rows do) or write `-` for "no opinion", which falls
|
|
15
|
+
# through to the built-in 2700. `0` disables the watchdog for this pairing.
|
|
16
|
+
# Precedence, highest first: --timeout, the FORGE_TIMEOUT environment
|
|
17
|
+
# variable (both used as given, and `0` there still disables), then this
|
|
18
|
+
# column, then 2700. Only this column and the built-in value are scaled by the
|
|
19
|
+
# effort that finally runs (after clamping): up to high x1, xhigh x4/3,
|
|
20
|
+
# max/ultra/ultracode x2. The result and its source are recorded as
|
|
21
|
+
# `timeout=` and `timeout_source=` in <role>.resolved.
|
|
22
|
+
# Commented examples (not enabled; write the real number for your model):
|
|
23
|
+
# # sol codex gpt-6.1-sol <=ultra 1800
|
|
24
|
+
# # opus claude opus <=max -
|
|
25
|
+
#
|
|
26
|
+
# FORGE_REGISTRY=<file> points dispatch at a different registry (useful for trying a
|
|
27
|
+
# row without editing this one); the capability cache keys on the file actually used.
|
|
28
|
+
#
|
|
29
|
+
# An effort outside the accepted set is clamped to the nearest lower one that is
|
|
30
|
+
# accepted (and reported), rather than failing a dispatch the user already paid
|
|
31
|
+
# to start. A word that is not a recognized effort at all is still an error.
|
|
32
|
+
#
|
|
33
|
+
# Row order is meaningful: the FIRST row for an alias is that alias's default
|
|
34
|
+
# harness. `--dwarf sol` means sol on codex; `--dwarf sol:high:openclaude` moves
|
|
35
|
+
# the same model to a different harness.
|
|
36
|
+
#
|
|
37
|
+
# An alias with no row here is not an error — it is passed through as a literal
|
|
38
|
+
# model id, but only when you name the harness explicitly (forge cannot guess a
|
|
39
|
+
# default harness for a model it has never heard of).
|
|
40
|
+
#
|
|
41
|
+
# alias harness model effort [timeout]
|
|
42
|
+
sol codex gpt-6.1-sol <=ultra
|
|
43
|
+
sol openclaude gpt-6.1-sol <=max
|
|
44
|
+
sol opencode github-copilot/gpt-6.1-sol -
|
|
45
|
+
astra codex gpt-6-astra <=ultra
|
|
46
|
+
astra openclaude gpt-6-astra <=max
|
|
47
|
+
astra opencode github-copilot/gpt-6-astra -
|
|
48
|
+
terra codex gpt-5.6-terra <=ultra
|
|
49
|
+
terra openclaude gpt-5.6-terra <=max
|
|
50
|
+
terra opencode github-copilot/gpt-5.6-terra -
|
|
51
|
+
luna codex gpt-6-luna <=max
|
|
52
|
+
luna openclaude gpt-6-luna <=max
|
|
53
|
+
luna opencode opencode-go/gpt-6-luna -
|
|
54
|
+
opus claude opus <=max
|
|
55
|
+
opus openclaude opus <=ultracode
|
|
56
|
+
opus antigravity claude-opus-4-6-thinking none
|
|
57
|
+
opus opencode github-copilot/claude-opus-5.5 -
|
|
58
|
+
sonnet claude sonnet <=max
|
|
59
|
+
sonnet openclaude sonnet <=ultracode
|
|
60
|
+
sonnet antigravity claude-sonnet-4-6 none
|
|
61
|
+
sonnet opencode github-copilot/claude-sonnet-5.5 -
|
|
62
|
+
fable claude fable <=max
|
|
63
|
+
fable openclaude fable <=ultracode
|
|
64
|
+
fable opencode github-copilot/claude-fable-5.1 -
|
|
65
|
+
haiku claude haiku <=max
|
|
66
|
+
haiku openclaude haiku <=ultracode
|
|
67
|
+
haiku opencode github-copilot/claude-haiku-5.5 -
|
|
68
|
+
# Antigravity (agy). Gemini models take effort as a flag on the UNSUFFIXED id;
|
|
69
|
+
# the suffixed ids agy lists (gemini-3.7-flash-low) conflict with --effort.
|
|
70
|
+
gemini antigravity gemini-3.7-flash low,medium,high
|
|
71
|
+
gemini opencode github-copilot/gemini-3.1-pro-preview -
|
|
72
|
+
gemini-flash antigravity gemini-3.7-flash low,medium,high
|
|
73
|
+
gemini-pro antigravity gemini-3.1-pro low,high
|
|
74
|
+
gemini-3.6 antigravity gemini-3.6-flash low,medium,high
|
|
75
|
+
gemini-3.5 antigravity gemini-3.5-flash low,medium,high
|
|
76
|
+
gpt-oss antigravity gpt-oss-120b medium
|
|
77
|
+
grok grok grok-4.7 <=xhigh
|
|
78
|
+
grok opencode github-copilot/grok-4.6 -
|
|
79
|
+
kimi opencode github-copilot/kimi-k3 -
|
|
80
|
+
glm opencode opencode-go/glm-5.2 -
|
|
81
|
+
qwen opencode opencode-go/qwen3.8-max -
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Shared artifact operations. A private index never stages or unstages user work.
|
|
3
|
+
forge_tree() (
|
|
4
|
+
cd "$1" || exit 1
|
|
5
|
+
local idx tree actual_index f
|
|
6
|
+
idx="$(mktemp "${TMPDIR:-/tmp}/forge-index.XXXXXX")" || exit 1
|
|
7
|
+
rm -f "$idx"
|
|
8
|
+
trap 'rm -f "$idx"' EXIT
|
|
9
|
+
actual_index="$(git rev-parse --git-path index)" || exit 1
|
|
10
|
+
export GIT_INDEX_FILE="$idx"
|
|
11
|
+
git read-tree HEAD && git add -A -- . ':(exclude).forge' || exit 1
|
|
12
|
+
# Force-added ignored files are tracked in the real index, but not in HEAD yet.
|
|
13
|
+
GIT_INDEX_FILE="$actual_index" git diff --cached --name-only --diff-filter=A -z HEAD |
|
|
14
|
+
while IFS= read -r -d '' f; do
|
|
15
|
+
case "$f" in .forge|.forge/*) continue ;; esac
|
|
16
|
+
if [ -e "$f" ] || [ -L "$f" ]; then git add -f -- "$f" || exit 1; fi
|
|
17
|
+
done || exit 1
|
|
18
|
+
tree="$(git write-tree)" || exit 1
|
|
19
|
+
printf '%s\n' "$tree"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
forge_verdict() {
|
|
23
|
+
# Only a standalone final line is authoritative; prose/quoted markers are not.
|
|
24
|
+
# The QA prompt shows each verdict followed by a dash and its meaning, and some
|
|
25
|
+
# reviewers copy that whole line; accept exactly that dash-separated gloss.
|
|
26
|
+
# CRLF replies (some harnesses on some platforms) are the same line with a stray CR.
|
|
27
|
+
awk 'NF {last=$0} END {sub(/\r$/,"",last); if(last ~ /^FORGE_VERDICT: (PASS|FAIL)([ \t]+(—|–|-)( .*)?)?[ \t]*$/) {sub(/^FORGE_VERDICT: /,"",last); sub(/[ \t].*$/,"",last); print last} else print "UNKNOWN"}' "$1"
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
forge_export_tree() ( # source repository, tree, destination; no archive attributes
|
|
31
|
+
local idx
|
|
32
|
+
idx="$(mktemp "${TMPDIR:-/tmp}/forge-export.XXXXXX")" || exit 1
|
|
33
|
+
rm -f "$idx"; trap 'rm -f "$idx"' EXIT
|
|
34
|
+
export GIT_INDEX_FILE="$idx"
|
|
35
|
+
git -C "$1" read-tree "$2" && git -C "$1" checkout-index --all --force --prefix="$3/"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
forge_review_snapshot() ( # source, baseline tree, final tree, destination
|
|
39
|
+
local src="$1" base="$2" tree="$3" dest="$4"
|
|
40
|
+
[ ! -e "$dest" ] || exit 1
|
|
41
|
+
mkdir -p "$dest" || exit 1
|
|
42
|
+
git -C "$dest" init -q || exit 1
|
|
43
|
+
base="$(git -C "$src" rev-parse "$base^{tree}")" || exit 1
|
|
44
|
+
tree="$(git -C "$src" rev-parse "$tree^{tree}")" || exit 1
|
|
45
|
+
printf '%s\n%s\n' "$base" "$tree" |
|
|
46
|
+
git -C "$src" pack-objects --revs --stdout |
|
|
47
|
+
git -C "$dest" index-pack --stdin >/dev/null || exit 1
|
|
48
|
+
local baseline final
|
|
49
|
+
baseline="$(echo baseline | git -C "$dest" -c user.name=forge -c user.email=forge@local commit-tree "$base")" || exit 1
|
|
50
|
+
final="$(echo artifact | git -C "$dest" -c user.name=forge -c user.email=forge@local commit-tree "$tree" -p "$baseline")" || exit 1
|
|
51
|
+
[ "$(git -C "$dest" rev-parse "$baseline^{tree}")" = "$base" ] || exit 1
|
|
52
|
+
echo "$baseline" > "$dest/../review.base"
|
|
53
|
+
git -C "$dest" reset --hard "$final" >/dev/null || exit 1
|
|
54
|
+
[ "$(git -C "$dest" rev-parse HEAD^{tree})" = "$tree" ]
|
|
55
|
+
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
forge_fingerprint() {
|
|
59
|
+
local tree head index
|
|
60
|
+
tree="$(forge_tree "$1")" || return 1
|
|
61
|
+
head="$(git -C "$1" rev-parse HEAD)" || return 1
|
|
62
|
+
index="$(git -C "$1" diff --cached --binary HEAD | git -C "$1" hash-object --stdin)" || return 1
|
|
63
|
+
printf '%s %s %s\n' "$head" "$tree" "$index"
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
forge_fail_first() { # review snapshot, run or task dir, repo: advisory fail-first evidence for QA
|
|
67
|
+
# Writes <dir>/failfirst.json and the reviewer's section to <dir>/failfirst.txt; never fails
|
|
68
|
+
# the run and never touches the snapshot (forge-fail-first.py exports through its own index).
|
|
69
|
+
local cmd="" reviewed
|
|
70
|
+
rm -f "$2/failfirst.json" "$2/failfirst.txt"
|
|
71
|
+
[ -s "$3/.forge/verify-subset" ] && cmd="$(cat "$3/.forge/verify-subset")"
|
|
72
|
+
reviewed="$(git -C "$1" rev-parse HEAD)" || return 0
|
|
73
|
+
python3 "$SKILL_DIR/scripts/forge-fail-first.py" run --repo "$1" --base "$(cat "$2/review.base")" \
|
|
74
|
+
--reviewed "$reviewed" --command "$cmd" --out "$2/failfirst.json" --deps-from "$3" \
|
|
75
|
+
> "$2/failfirst.txt" 2>/dev/null || : > "$2/failfirst.txt"
|
|
76
|
+
forge_dataset_event fail_first --dir "$2" --file failfirst="$2/failfirst.json"
|
|
77
|
+
return 0
|
|
78
|
+
}
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""The QA contract: one shared review prompt, a severity vocabulary and a deterministic gate.
|
|
3
|
+
|
|
4
|
+
Three subcommands, stdlib only, no network:
|
|
5
|
+
|
|
6
|
+
qa --part head|tail [--kind task|final|solo] [--threshold P0|P1|P2|P3]
|
|
7
|
+
Prints one piece of the QA prompt. The caller prints everything between and
|
|
8
|
+
after the pieces itself (scope notes, the diff fence, the memory note and the
|
|
9
|
+
"Place any learning notes before the final FORGE_VERDICT ..." sentence).
|
|
10
|
+
dwarf
|
|
11
|
+
Prints the implementer rules block that callers append to the dwarf prompt.
|
|
12
|
+
gate --verdict PASS|FAIL|UNKNOWN --qa-last FILE [--threshold Pn] [--known-issues OUT]
|
|
13
|
+
Prints ONE tab-separated line
|
|
14
|
+
decision <TAB> worst_confirmed <TAB> n_confirmed <TAB> n_unlabelled <TAB> n_below
|
|
15
|
+
and, when findings were tolerated, writes them to OUT.
|
|
16
|
+
|
|
17
|
+
Severity. Reviewers label every finding P0 (data loss, security hole, crash or wrong
|
|
18
|
+
result on the main path), P1 (wrong behaviour on realistic input, breaks an existing
|
|
19
|
+
contract), P2 (edge-case bug, unusual-timing race, missing hardening) or P3 (minor).
|
|
20
|
+
Severity is impact; confidence stays CONFIRMED vs PLAUSIBLE.
|
|
21
|
+
|
|
22
|
+
Threshold. Without one nothing changes: any CONFIRMED finding means FAIL and the gate
|
|
23
|
+
never overrides a verdict. The threshold is the LEAST severe level that still blocks:
|
|
24
|
+
`P2` means P0, P1 and P2 block and only P3 is tolerated; `P0` means only P0 blocks;
|
|
25
|
+
`P3` means everything blocks (nothing is ever tolerated). A threshold of "" is "unset".
|
|
26
|
+
|
|
27
|
+
The gate. decision is `accept` only when ALL of these hold, otherwise `keep`:
|
|
28
|
+
- a threshold is set and the verdict is FAIL (PASS and UNKNOWN are always `keep`);
|
|
29
|
+
- at least one CONFIRMED finding exists (a FAIL with none stays a FAIL);
|
|
30
|
+
- n_unlabelled == 0, and every CONFIRMED finding carries a P0-P3 label strictly less
|
|
31
|
+
severe than the threshold.
|
|
32
|
+
`accept` is a statement about the findings only; wiring it into a verdict is the
|
|
33
|
+
caller's job. Fail-safe means blocking: anything the parser cannot read confidently
|
|
34
|
+
pushes toward `keep`, never toward `accept`.
|
|
35
|
+
|
|
36
|
+
Fields: worst_confirmed is the most severe label among CONFIRMED labelled findings
|
|
37
|
+
(`-` if none); n_confirmed counts CONFIRMED findings; n_unlabelled counts CONFIRMED
|
|
38
|
+
findings with no recognisable severity, plus located findings carrying a severity but
|
|
39
|
+
no CONFIRMED/PLAUSIBLE tag at all (neither can ever be tolerated); n_below counts
|
|
40
|
+
CONFIRMED labelled findings below the threshold (0 with no threshold).
|
|
41
|
+
|
|
42
|
+
Parsing. The reply is cut into finding blocks. A block starts at a bullet (`-`, `*`,
|
|
43
|
+
`+`, `1.`, `1)`), a markdown heading, a table row, or a flush line that leads with a
|
|
44
|
+
severity/status tag and a file:line; it runs until the next such line at the same or a
|
|
45
|
+
lower indentation. Nested bullets and indented or lazy continuation lines belong to the
|
|
46
|
+
block; a flush-left paragraph after a blank line does not. A block is CONFIRMED when it
|
|
47
|
+
contains the uppercase word CONFIRMED (or a bracketed/bold `[confirmed]` in its first
|
|
48
|
+
line); CONFIRMED beats PLAUSIBLE; PLAUSIBLE-only blocks are ignored; "NOT CONFIRMED" and
|
|
49
|
+
lowercase prose do not count. The severity is the first P0-P3 token of the block's first
|
|
50
|
+
line (`[P2]`, `(P2)`, `**P2**`, `P2:`, `P2 -` or a standalone `P2`; uppercase only).
|
|
51
|
+
|
|
52
|
+
Phantom findings. A reviewer may echo the contract it was given. Three layers keep that
|
|
53
|
+
from creating findings or blocking: (1) FORGE_VERDICT and FORGE_LEARNING text is removed
|
|
54
|
+
before parsing (the verdict line itself says "CONFIRMED"); (2) any line that is a
|
|
55
|
+
contiguous fragment of this module's own contract text (all parts, kinds and thresholds,
|
|
56
|
+
whitespace-collapsed, at least 25 characters, list marker ignored) is removed, which
|
|
57
|
+
covers the example bullet, the severity definitions and the instructions even when
|
|
58
|
+
re-wrapped; (3) a block whose first line holds the literal placeholder
|
|
59
|
+
`path/file.py:123` is ignored. A leftover false match can only add a finding or a
|
|
60
|
+
blocker, and an added CONFIRMED finding without a label blocks.
|
|
61
|
+
|
|
62
|
+
Exit codes: 0 ok (even for `keep`), 2 usage, 3 unreadable QA file or unwritable OUT
|
|
63
|
+
file. On any nonzero exit callers must treat the decision as `keep`.
|
|
64
|
+
"""
|
|
65
|
+
from __future__ import annotations
|
|
66
|
+
|
|
67
|
+
import argparse
|
|
68
|
+
import collections
|
|
69
|
+
import functools
|
|
70
|
+
import os
|
|
71
|
+
import re
|
|
72
|
+
import sys
|
|
73
|
+
|
|
74
|
+
SEVERITIES = ('P0', 'P1', 'P2', 'P3')
|
|
75
|
+
PLACEHOLDER = 'path/file.py:123'
|
|
76
|
+
KINDS = ('task', 'final', 'solo')
|
|
77
|
+
|
|
78
|
+
Finding = collections.namedtuple('Finding', 'status severity located line text')
|
|
79
|
+
Finding.__doc__ = (
|
|
80
|
+
'One finding block. status: CONFIRMED, PLAUSIBLE or UNSTATED (a located, severity-'
|
|
81
|
+
'tagged block with neither status word). severity: 0-3 or None. line: 1-based '
|
|
82
|
+
'line number of the block start in the original text. text: the block, verbatim.')
|
|
83
|
+
GateResult = collections.namedtuple(
|
|
84
|
+
'GateResult', 'decision worst n_confirmed n_unlabelled n_below tolerated')
|
|
85
|
+
GateResult.__doc__ = (
|
|
86
|
+
'decision accept|keep, worst most severe confirmed label or "-", counters, and the '
|
|
87
|
+
'tuple of tolerated Findings (CONFIRMED, labelled, below the threshold).')
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# --- prompt text -----------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
def _rank(threshold) -> 'int | None':
|
|
93
|
+
if threshold is None or str(threshold).strip() == '':
|
|
94
|
+
return None
|
|
95
|
+
value = str(threshold).strip().upper()
|
|
96
|
+
if value not in SEVERITIES:
|
|
97
|
+
raise ValueError('threshold must be one of ' + ', '.join(SEVERITIES))
|
|
98
|
+
return int(value[1])
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
_NO_TRUST = [
|
|
102
|
+
"Do not trust the implementer's own reports of which checks ran or passed (the task",
|
|
103
|
+
'text may quote them); judge the diff itself.',
|
|
104
|
+
]
|
|
105
|
+
|
|
106
|
+
_HEAD = {
|
|
107
|
+
'task': [
|
|
108
|
+
'The implementer was asked to do the task above. Review the diff below for correctness',
|
|
109
|
+
'bugs: logic errors, broken edge cases, behaviour that does not match what was asked.',
|
|
110
|
+
"Also say if it solved a different problem, or touched files outside the task's scope.",
|
|
111
|
+
] + _NO_TRUST,
|
|
112
|
+
'solo': [
|
|
113
|
+
'Review the diff below for correctness bugs: logic errors, broken edge cases, wrong',
|
|
114
|
+
'behaviour versus what was asked. Also say if it solved a different problem than the',
|
|
115
|
+
"one stated, or changed files outside the goal's scope.",
|
|
116
|
+
] + _NO_TRUST,
|
|
117
|
+
'final': [
|
|
118
|
+
'This is the final, whole-run review. The diff below is the COMBINED result of several',
|
|
119
|
+
'tasks that were implemented independently and each reviewed on its own slice only.',
|
|
120
|
+
'Review it for correctness bugs, and look especially for what per-task review could not',
|
|
121
|
+
'see: interactions between tasks, contract or interface mismatches between them,',
|
|
122
|
+
'async or ordering races across modules, regressions of existing contracts, and',
|
|
123
|
+
'anything that only breaks when the pieces run together. Also say if the combined',
|
|
124
|
+
"change solved a different problem than the goal and tasks above, or touched files",
|
|
125
|
+
'outside their scope.',
|
|
126
|
+
"Do not trust the implementers' own reports of which checks ran or passed; judge the",
|
|
127
|
+
'diff itself.',
|
|
128
|
+
],
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
_FINDINGS = [
|
|
132
|
+
'Report findings only — do not edit any file. For each finding give the file and line,',
|
|
133
|
+
'what breaks, and a concrete input that triggers it. Mark each CONFIRMED if you traced',
|
|
134
|
+
'it in the code, or PLAUSIBLE if you could not fully verify it.',
|
|
135
|
+
]
|
|
136
|
+
_SOLO_EXTRA = ['If the diff is correct, say so plainly rather than inventing something to report.']
|
|
137
|
+
|
|
138
|
+
_SEVERITY = [
|
|
139
|
+
'Label every finding with its severity at the start of its own bullet, in exactly this form:',
|
|
140
|
+
' - [P1][CONFIRMED] path/file.py:123 — what breaks (trigger input)',
|
|
141
|
+
'One finding per bullet. Severity is about impact, not confidence; confidence is the',
|
|
142
|
+
'CONFIRMED or PLAUSIBLE tag.',
|
|
143
|
+
' P0 data loss, security hole, crash or wrong result on the main path',
|
|
144
|
+
' P1 wrong behaviour on realistic input, or breaks an existing contract',
|
|
145
|
+
' P2 edge-case bug, race that needs unusual timing, missing hardening',
|
|
146
|
+
' P3 minor: cosmetic, naming, nit-level correctness',
|
|
147
|
+
]
|
|
148
|
+
|
|
149
|
+
_NO_THRESHOLD = ['The severity tag does not change the verdict rule below.']
|
|
150
|
+
|
|
151
|
+
_PASS = ' FORGE_VERDICT: PASS — no confirmed correctness bug (style nits are not failures)'
|
|
152
|
+
_FAIL = ' FORGE_VERDICT: FAIL — at least one CONFIRMED correctness bug'
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _threshold_clause(rank: int) -> list:
|
|
156
|
+
name = SEVERITIES[rank]
|
|
157
|
+
blocking = ', '.join(SEVERITIES[:rank + 1])
|
|
158
|
+
if rank == len(SEVERITIES) - 1:
|
|
159
|
+
lines = ['Blocking threshold: %s. Every severity (%s) is BLOCKING, so any CONFIRMED' % (name, blocking),
|
|
160
|
+
'finding justifies FAIL.']
|
|
161
|
+
else:
|
|
162
|
+
tolerated = ', '.join(SEVERITIES[rank + 1:])
|
|
163
|
+
lines = ['Blocking threshold: %s. Findings at %s or worse (%s) are BLOCKING. A' % (name, name, blocking),
|
|
164
|
+
'CONFIRMED finding at a lower severity (%s) is recorded as a known issue and does' % tolerated,
|
|
165
|
+
'not by itself justify FAIL. Give FAIL if and only if at least one CONFIRMED',
|
|
166
|
+
'finding is blocking.']
|
|
167
|
+
return lines + ['A CONFIRMED finding without a severity tag counts as blocking.']
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def qa_text(part: str, kind: str = 'task', threshold=None) -> str:
|
|
171
|
+
"""One shared piece of the QA prompt, newline-terminated, no leading/trailing blank."""
|
|
172
|
+
if kind not in KINDS:
|
|
173
|
+
raise ValueError('kind must be one of ' + ', '.join(KINDS))
|
|
174
|
+
rank = _rank(threshold)
|
|
175
|
+
if part == 'head':
|
|
176
|
+
return '\n'.join(_HEAD[kind]) + '\n'
|
|
177
|
+
if part != 'tail':
|
|
178
|
+
raise ValueError('part must be head or tail')
|
|
179
|
+
lines = list(_FINDINGS) + (_SOLO_EXTRA if kind == 'solo' else [])
|
|
180
|
+
lines += [''] + _SEVERITY
|
|
181
|
+
lines += [''] + (_NO_THRESHOLD if rank is None else _threshold_clause(rank))
|
|
182
|
+
if rank is None:
|
|
183
|
+
verdicts = [_PASS, _FAIL]
|
|
184
|
+
else:
|
|
185
|
+
where = '%s or worse' % SEVERITIES[rank]
|
|
186
|
+
verdicts = [' FORGE_VERDICT: PASS — no CONFIRMED finding at the blocking threshold (%s)' % where,
|
|
187
|
+
' FORGE_VERDICT: FAIL — at least one CONFIRMED finding at the blocking threshold (%s)' % where]
|
|
188
|
+
lines += ['', 'End your reply with exactly one line:'] + verdicts
|
|
189
|
+
return '\n'.join(lines) + '\n'
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
_DWARF = [
|
|
193
|
+
'## Implementer rules',
|
|
194
|
+
'- Do not run git commit, checkout, switch, reset, stash or rebase. The harness captures',
|
|
195
|
+
' the working tree itself; a self-commit makes the change set ambiguous.',
|
|
196
|
+
'- Do not start background processes, and leave nothing running when you finish.',
|
|
197
|
+
'- Finish every measurement and test run BEFORE your last message and report its actual',
|
|
198
|
+
' result. If you could not run something, say so explicitly instead of claiming it passed.',
|
|
199
|
+
'- State precisely which verification commands you ran and what each one returned.',
|
|
200
|
+
]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def dwarf_text() -> str:
|
|
204
|
+
"""The implementer rules block, newline-terminated."""
|
|
205
|
+
return '\n'.join(_DWARF) + '\n'
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# --- parsing ---------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
_PROTOCOL = re.compile(r'FORGE_(?:VERDICT|LEARNING):.*$')
|
|
211
|
+
_QUOTE = re.compile(r'^\s*(?:>[ \t]?)+')
|
|
212
|
+
_MARKER = re.compile(r'^(?:[-*+•–]|\d{1,3}[.)]|#{1,6})\s+')
|
|
213
|
+
_HEADING = re.compile(r'^ {0,3}#{1,6}[ \t]+\S')
|
|
214
|
+
_BULLET = re.compile(r'^\s*(?:[-*+•–]|\d{1,3}[.)])[ \t]+\S')
|
|
215
|
+
_TABLE = re.compile(r'^\s*\|')
|
|
216
|
+
_LEAD = re.compile(r'^\s*[*_\[(]*(?:P[0-3]|CONFIRMED|PLAUSIBLE)(?![\w])')
|
|
217
|
+
_LOCATION = re.compile(r'[\w./\\-]+\.\w+:\d+|\b[Ll]ines? \d+')
|
|
218
|
+
# P0-P3 as a standalone token: not glued to a word, path or range (P1.py, src/P1/x, P1-P2).
|
|
219
|
+
_SEVERITY_TOKEN = re.compile(r'(?<![\w/.\\-])P([0-3])(?![\w/\\])(?!\.\w)(?!-\w)')
|
|
220
|
+
_CONFIRMED = re.compile(r'\bCONFIRMED\b')
|
|
221
|
+
_NOT_CONFIRMED = re.compile(r'\b(?i:not)\s+CONFIRMED\b|\bNON-CONFIRMED\b')
|
|
222
|
+
_CONFIRMED_TAG = re.compile(r'\[\s*confirmed\s*\]|\*\*\s*confirmed\s*\*\*', re.IGNORECASE)
|
|
223
|
+
_PLAUSIBLE = re.compile(r'\bPLAUSIBLE\b')
|
|
224
|
+
_MIN_ECHO = 25
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _collapse(text: str) -> str:
|
|
228
|
+
return ' '.join(text.split())
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@functools.lru_cache(maxsize=None)
|
|
232
|
+
def _contract_flat() -> str:
|
|
233
|
+
parts = [dwarf_text()]
|
|
234
|
+
for kind in KINDS:
|
|
235
|
+
for part in ('head', 'tail'):
|
|
236
|
+
for threshold in (None,) + SEVERITIES:
|
|
237
|
+
parts.append(qa_text(part, kind, threshold))
|
|
238
|
+
return _collapse(' '.join(parts))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _clean(text: str) -> list:
|
|
242
|
+
"""(line number, line) pairs with protocol lines, quotes and echoed contract removed."""
|
|
243
|
+
flat = _contract_flat()
|
|
244
|
+
out = []
|
|
245
|
+
for number, raw in enumerate(text.replace('\r\n', '\n').replace('\r', '\n').split('\n'), 1):
|
|
246
|
+
raw = raw.expandtabs(4)
|
|
247
|
+
line = _PROTOCOL.sub('', raw)
|
|
248
|
+
if line != raw and not line.strip():
|
|
249
|
+
continue
|
|
250
|
+
line = _QUOTE.sub('', line).rstrip()
|
|
251
|
+
core = _collapse(_MARKER.sub('', line.strip()))
|
|
252
|
+
if len(core) >= _MIN_ECHO and core in flat:
|
|
253
|
+
continue
|
|
254
|
+
out.append((number, line))
|
|
255
|
+
return out
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _starts_block(line: str) -> bool:
|
|
259
|
+
if _HEADING.match(line) or _BULLET.match(line) or _TABLE.match(line):
|
|
260
|
+
return True
|
|
261
|
+
return bool(_LEAD.match(line) and _LOCATION.search(line))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _blocks(lines: list) -> list:
|
|
265
|
+
blocks = [] # [start number, indent, [lines]]
|
|
266
|
+
current = None
|
|
267
|
+
blank = False
|
|
268
|
+
for number, line in lines:
|
|
269
|
+
if not line.strip():
|
|
270
|
+
blank = True
|
|
271
|
+
if current is not None:
|
|
272
|
+
current[2].append('')
|
|
273
|
+
continue
|
|
274
|
+
indent = len(line) - len(line.lstrip(' '))
|
|
275
|
+
if _starts_block(line):
|
|
276
|
+
nested = (current is not None and indent > current[1]
|
|
277
|
+
and not _HEADING.match(line))
|
|
278
|
+
if nested:
|
|
279
|
+
current[2].append(line)
|
|
280
|
+
else:
|
|
281
|
+
current = [number, indent, [line]]
|
|
282
|
+
blocks.append(current)
|
|
283
|
+
elif current is not None:
|
|
284
|
+
if blank and indent <= current[1]:
|
|
285
|
+
current = None
|
|
286
|
+
else:
|
|
287
|
+
current[2].append(line)
|
|
288
|
+
blank = False
|
|
289
|
+
return blocks
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _severity(first_line: str):
|
|
293
|
+
match = _SEVERITY_TOKEN.search(first_line)
|
|
294
|
+
return int(match.group(1)) if match else None
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _finding(number: int, lines: list):
|
|
298
|
+
while lines and not lines[-1].strip():
|
|
299
|
+
lines = lines[:-1]
|
|
300
|
+
first = lines[0]
|
|
301
|
+
if PLACEHOLDER in first:
|
|
302
|
+
return None
|
|
303
|
+
body = _NOT_CONFIRMED.sub('', '\n'.join(lines))
|
|
304
|
+
severity = _severity(first)
|
|
305
|
+
located = bool(_LOCATION.search(first))
|
|
306
|
+
if _CONFIRMED.search(body) or _CONFIRMED_TAG.search(first):
|
|
307
|
+
status = 'CONFIRMED'
|
|
308
|
+
elif _PLAUSIBLE.search(body):
|
|
309
|
+
status = 'PLAUSIBLE'
|
|
310
|
+
elif severity is not None and located:
|
|
311
|
+
status = 'UNSTATED'
|
|
312
|
+
else:
|
|
313
|
+
return None
|
|
314
|
+
return Finding(status, severity, located, number, '\n'.join(lines))
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def parse_findings(text: str) -> list:
|
|
318
|
+
"""Finding blocks of a QA reply, in order; see the module docstring for the rules."""
|
|
319
|
+
found = (_finding(start, lines) for start, _, lines in _blocks(_clean(text)))
|
|
320
|
+
return [finding for finding in found if finding is not None]
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
# --- the gate --------------------------------------------------------------------
|
|
324
|
+
|
|
325
|
+
def gate(verdict: str, text: str, threshold=None) -> GateResult:
|
|
326
|
+
"""Decide whether a QA FAIL may be accepted as PASS-with-known-issues."""
|
|
327
|
+
rank = _rank(threshold)
|
|
328
|
+
findings = parse_findings(text)
|
|
329
|
+
confirmed = [f for f in findings if f.status == 'CONFIRMED']
|
|
330
|
+
labelled = [f for f in confirmed if f.severity is not None]
|
|
331
|
+
unlabelled = (len(confirmed) - len(labelled)
|
|
332
|
+
+ sum(1 for f in findings if f.status == 'UNSTATED'))
|
|
333
|
+
worst = 'P%d' % min(f.severity for f in labelled) if labelled else '-'
|
|
334
|
+
below = tuple(f for f in labelled if rank is not None and f.severity > rank)
|
|
335
|
+
accept = (rank is not None and verdict == 'FAIL' and bool(confirmed)
|
|
336
|
+
and unlabelled == 0 and len(below) == len(confirmed))
|
|
337
|
+
return GateResult('accept' if accept else 'keep', worst, len(confirmed),
|
|
338
|
+
unlabelled, len(below), below)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def known_issues_text(threshold, tolerated) -> str:
|
|
342
|
+
name = SEVERITIES[_rank(threshold)]
|
|
343
|
+
lines = ['## Known issues (QA findings below blocking threshold %s)' % name, '',
|
|
344
|
+
'QA confirmed these findings at a severity less than %s, the blocking' % name,
|
|
345
|
+
'threshold (%s and worse block). They are recorded for follow-up and do not' % name,
|
|
346
|
+
'by themselves block the task.']
|
|
347
|
+
for finding in tolerated:
|
|
348
|
+
lines += ['', finding.text]
|
|
349
|
+
return '\n'.join(lines) + '\n'
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# --- command line ----------------------------------------------------------------
|
|
353
|
+
|
|
354
|
+
def _threshold_arg(value: str):
|
|
355
|
+
value = value.strip().upper()
|
|
356
|
+
if value == '':
|
|
357
|
+
return None
|
|
358
|
+
if value not in SEVERITIES:
|
|
359
|
+
raise argparse.ArgumentTypeError('invalid threshold %r (choose P0, P1, P2 or P3)' % value)
|
|
360
|
+
return value
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _emit(text: str) -> None:
|
|
364
|
+
sys.stdout.flush()
|
|
365
|
+
sys.stdout.buffer.write(text.encode('utf-8'))
|
|
366
|
+
sys.stdout.buffer.flush()
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _parser() -> argparse.ArgumentParser:
|
|
370
|
+
parser = argparse.ArgumentParser(
|
|
371
|
+
prog='forge-contract.py', formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
372
|
+
description='Shared QA contract, implementer rules and severity gate for forge.',
|
|
373
|
+
epilog='threshold: the LEAST severe level that still blocks. P2 = P0, P1 and P2 '
|
|
374
|
+
'block, only P3 is tolerated; P0 = only P0 blocks; P3 = everything blocks. '
|
|
375
|
+
'Unset (or empty) = no gate, today\'s semantics.')
|
|
376
|
+
sub = parser.add_subparsers(dest='command', required=True)
|
|
377
|
+
qa = sub.add_parser('qa', help='print one piece of the QA prompt')
|
|
378
|
+
qa.add_argument('--part', required=True, choices=('head', 'tail'),
|
|
379
|
+
help='head = review preamble, tail = findings format + verdict contract')
|
|
380
|
+
qa.add_argument('--kind', choices=KINDS, default='task',
|
|
381
|
+
help='task (default), final (whole-run review) or solo (solo wording)')
|
|
382
|
+
qa.add_argument('--threshold', type=_threshold_arg, default=None, metavar='P0..P3',
|
|
383
|
+
help='least severe level that still blocks (default: none)')
|
|
384
|
+
sub.add_parser('dwarf', help='print the implementer rules block')
|
|
385
|
+
gate_parser = sub.add_parser(
|
|
386
|
+
'gate', help='print: decision, worst_confirmed, n_confirmed, n_unlabelled, n_below')
|
|
387
|
+
gate_parser.add_argument('--verdict', required=True, choices=('PASS', 'FAIL', 'UNKNOWN'))
|
|
388
|
+
gate_parser.add_argument('--qa-last', required=True, metavar='FILE', help="the QA reply")
|
|
389
|
+
gate_parser.add_argument('--threshold', type=_threshold_arg, default=None, metavar='P0..P3',
|
|
390
|
+
help='least severe level that still blocks (default: none)')
|
|
391
|
+
gate_parser.add_argument('--known-issues', metavar='OUT', default=None,
|
|
392
|
+
help='write tolerated findings here when there are any')
|
|
393
|
+
return parser
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def main(argv=None) -> int:
|
|
397
|
+
args = _parser().parse_args(argv)
|
|
398
|
+
if args.command == 'qa':
|
|
399
|
+
_emit(qa_text(args.part, args.kind, args.threshold))
|
|
400
|
+
return 0
|
|
401
|
+
if args.command == 'dwarf':
|
|
402
|
+
_emit(dwarf_text())
|
|
403
|
+
return 0
|
|
404
|
+
try:
|
|
405
|
+
with open(args.qa_last, 'rb') as handle:
|
|
406
|
+
text = handle.read().decode('utf-8', 'replace')
|
|
407
|
+
except OSError as error:
|
|
408
|
+
sys.stderr.write('forge-contract: cannot read %s: %s\n' % (args.qa_last, error))
|
|
409
|
+
return 3
|
|
410
|
+
result = gate(args.verdict, text, args.threshold)
|
|
411
|
+
if args.known_issues and result.n_below >= 1:
|
|
412
|
+
temporary = args.known_issues + '.%d.tmp' % os.getpid()
|
|
413
|
+
try:
|
|
414
|
+
with open(temporary, 'w', encoding='utf-8', newline='\n') as handle:
|
|
415
|
+
handle.write(known_issues_text(args.threshold, result.tolerated))
|
|
416
|
+
os.replace(temporary, args.known_issues)
|
|
417
|
+
except OSError as error:
|
|
418
|
+
sys.stderr.write('forge-contract: cannot write %s: %s\n' % (args.known_issues, error))
|
|
419
|
+
try:
|
|
420
|
+
os.unlink(temporary)
|
|
421
|
+
except OSError:
|
|
422
|
+
pass
|
|
423
|
+
return 3
|
|
424
|
+
_emit('%s\t%s\t%d\t%d\t%d\n' % (result.decision, result.worst, result.n_confirmed,
|
|
425
|
+
result.n_unlabelled, result.n_below))
|
|
426
|
+
return 0
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
if __name__ == '__main__':
|
|
430
|
+
sys.exit(main())
|