@giamme/forge 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +181 -0
- package/LICENSE +21 -0
- package/README.md +1854 -2
- package/SKILL.md +122 -0
- package/VERSION +1 -0
- package/forge +74 -0
- package/package.json +26 -4
- package/references/dataset.md +329 -0
- package/references/decompose.md +667 -0
- package/references/fractal-dashboard.md +21 -0
- package/references/fractal.md +307 -0
- package/references/harnesses.md +353 -0
- package/references/jev.md +865 -0
- package/references/memory.md +228 -0
- package/references/planning.md +42 -0
- package/references/ripwire.md +47 -0
- package/references/runtime.md +64 -0
- package/references/solo.md +113 -0
- package/references/ui.md +112 -0
- package/registry.tsv +81 -0
- package/scripts/forge-artifact.sh +78 -0
- package/scripts/forge-contract.py +430 -0
- package/scripts/forge-dataset.py +6 -0
- package/scripts/forge-dispatch.sh +665 -0
- package/scripts/forge-fail-first.py +195 -0
- package/scripts/forge-fractal-dispatch.sh +4 -0
- package/scripts/forge-fractal-options.sh +54 -0
- package/scripts/forge-fractal.py +6 -0
- package/scripts/forge-guard.py +266 -0
- package/scripts/forge-install-fractal.sh +4 -0
- package/scripts/forge-install-ripwire.sh +52 -0
- package/scripts/forge-install.sh +135 -0
- package/scripts/forge-jev-options.sh +259 -0
- package/scripts/forge-jev.py +6 -0
- package/scripts/forge-lib-plans.sh +785 -0
- package/scripts/forge-lib-review.sh +494 -0
- package/scripts/forge-lib-task.sh +302 -0
- package/scripts/forge-lib-verify.sh +30 -0
- package/scripts/forge-memory.sh +575 -0
- package/scripts/forge-metrics.sh +39 -0
- package/scripts/forge-parallel-options.sh +76 -0
- package/scripts/forge-parallel.sh +1717 -0
- package/scripts/forge-plan-check.py +118 -0
- package/scripts/forge-prompt.py +45 -0
- package/scripts/forge-ripwire.py +136 -0
- package/scripts/forge-route.py +344 -0
- package/scripts/forge-runtime.py +533 -0
- package/scripts/forge-schedule.py +312 -0
- package/scripts/forge-shadow.py +305 -0
- package/scripts/forge-solo.sh +347 -0
- package/scripts/forge-ui-options.sh +42 -0
- package/scripts/forge-ui.py +435 -0
- package/scripts/forge-update.sh +71 -0
- package/scripts/forge-verify-coverage.py +956 -0
- package/scripts/forge-version.py +198 -0
- package/scripts/forge_dataset/__init__.py +439 -0
- package/scripts/forge_dataset/capture.py +330 -0
- package/scripts/forge_dataset/checks.py +109 -0
- package/scripts/forge_dataset/cli.py +233 -0
- package/scripts/forge_dataset/export.py +1009 -0
- package/scripts/forge_dataset/harvest.py +716 -0
- package/scripts/forge_dataset/importer.py +412 -0
- package/scripts/forge_dataset/outcomes.py +275 -0
- package/scripts/forge_dataset/routing.py +278 -0
- package/scripts/forge_fractal/__init__.py +83 -0
- package/scripts/forge_fractal/bridge.py +46 -0
- package/scripts/forge_fractal/cli.py +202 -0
- package/scripts/forge_fractal/dashboard.css +138 -0
- package/scripts/forge_fractal/dashboard.html +41 -0
- package/scripts/forge_fractal/dashboard.js +687 -0
- package/scripts/forge_fractal/decomposition.py +262 -0
- package/scripts/forge_fractal/execution.py +659 -0
- package/scripts/forge_fractal/inspection.py +261 -0
- package/scripts/forge_fractal/install.py +158 -0
- package/scripts/forge_fractal/invocation.py +94 -0
- package/scripts/forge_fractal/pipeline.py +124 -0
- package/scripts/forge_fractal/selection.py +179 -0
- package/scripts/forge_jev/__init__.py +376 -0
- package/scripts/forge_jev/backtest.py +322 -0
- package/scripts/forge_jev/cli.py +980 -0
- package/scripts/forge_jev/client.py +418 -0
- package/scripts/forge_jev/corpus.py +156 -0
- package/scripts/forge_jev/coupling.py +244 -0
- package/scripts/forge_jev/features.py +24 -0
- package/scripts/forge_jev/gates.py +100 -0
- package/scripts/forge_jev/memory.py +163 -0
- package/scripts/forge_jev/outcome.py +49 -0
- package/scripts/forge_jev/predict.py +204 -0
- package/scripts/forge_jev/questions.py +661 -0
- package/scripts/forge_jev/review.py +73 -0
- package/scripts/forge_jev/routing.py +271 -0
- package/scripts/forge_jev/subset.py +123 -0
- package/scripts/forge_jev/verify.py +453 -0
- package/scripts/forge_registry.py +140 -0
- package/ui/README.md +287 -0
- package/ui/THIRD_PARTY_NOTICES.md +69 -0
- package/ui/docs/api.md +888 -0
- package/ui/forge_ui/__init__.py +20 -0
- package/ui/forge_ui/__main__.py +135 -0
- package/ui/forge_ui/diffs.py +454 -0
- package/ui/forge_ui/executions.py +436 -0
- package/ui/forge_ui/export.py +204 -0
- package/ui/forge_ui/findings.py +132 -0
- package/ui/forge_ui/fractal.py +583 -0
- package/ui/forge_ui/models.py +216 -0
- package/ui/forge_ui/reader.py +1336 -0
- package/ui/forge_ui/server.py +303 -0
- package/ui/forge_ui/static/app.js +924 -0
- package/ui/forge_ui/static/cloud3d.js +646 -0
- package/ui/forge_ui/static/cloudmodel.js +375 -0
- package/ui/forge_ui/static/cloudui.js +394 -0
- package/ui/forge_ui/static/diffview.js +216 -0
- package/ui/forge_ui/static/favicon.svg +1 -0
- package/ui/forge_ui/static/files.js +776 -0
- package/ui/forge_ui/static/gantt.js +97 -0
- package/ui/forge_ui/static/graph.js +266 -0
- package/ui/forge_ui/static/index.html +55 -0
- package/ui/forge_ui/static/iso.js +765 -0
- package/ui/forge_ui/static/leaderboard.js +315 -0
- package/ui/forge_ui/static/map3d.js +45 -0
- package/ui/forge_ui/static/modelstats.js +249 -0
- package/ui/forge_ui/static/notify.js +179 -0
- package/ui/forge_ui/static/panel.js +526 -0
- package/ui/forge_ui/static/runmodel.js +651 -0
- package/ui/forge_ui/static/scene3d.js +996 -0
- package/ui/forge_ui/static/search.js +119 -0
- package/ui/forge_ui/static/style.css +618 -0
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
"""Verify-command discovery and failure triage: select, never generate, and judge flakes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
from . import enabled, load_config, threshold
|
|
9
|
+
from .client import ask, choice, noul, probabilities, score
|
|
10
|
+
from .questions import failure_triage, runs_tests, verify_runtime, verify_selection
|
|
11
|
+
|
|
12
|
+
CANDIDATE_CAP = 25 # keeps one discover() request small; deterministic order means the
|
|
13
|
+
# cap always drops the same (least-likely) tail, never a random one.
|
|
14
|
+
LOG_TAIL_CHARS = 4000 # the failure signature (assertion, traceback) is at the end of a
|
|
15
|
+
# log; the setup/build noise at the start is not needed to triage it.
|
|
16
|
+
|
|
17
|
+
_PKG_SCRIPT_RE = re.compile(r'^(test|check|ci|verify)(?:[:\-]|$)', re.I)
|
|
18
|
+
_MAKE_TARGETS = ('test', 'tests', 'check', 'verify', 'ci')
|
|
19
|
+
_RUNNER_DIR_RE = re.compile(r'(test|check|ci|verify)', re.I)
|
|
20
|
+
_WORKFLOW_RUN_RE = re.compile(r'^\s*(?:-\s*)?run:\s*(.+?)\s*$')
|
|
21
|
+
_WORKFLOW_TEST_HINT_RE = re.compile(
|
|
22
|
+
r'\b(pytest|npm (?:test|run\s+test\S*)|yarn (?:test|run\s+test\S*)|go test|cargo test|'
|
|
23
|
+
r'make test|tox\b|jest|vitest|mvn test|gradle test|rspec|phpunit|ctest|go\s+vet\s+&&\s+go\s+test)\b',
|
|
24
|
+
re.I)
|
|
25
|
+
_MEMORY_CMD_RES = (
|
|
26
|
+
re.compile(r'\bpytest(?:\s+-{1,2}\S+)*'),
|
|
27
|
+
re.compile(r'\bmake\s+\w[\w-]*'),
|
|
28
|
+
re.compile(r'\bnpm run\s+[\w:_-]+'),
|
|
29
|
+
re.compile(r'\bnpm test\b'),
|
|
30
|
+
re.compile(r'\byarn(?:\s+run)?\s+[\w:_-]+'),
|
|
31
|
+
re.compile(r'\bgo test(?:\s+\S+)*'),
|
|
32
|
+
re.compile(r'\bcargo test(?:\s+\S+)*'),
|
|
33
|
+
re.compile(r'\btox\b'),
|
|
34
|
+
re.compile(r'\bnox\b'),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _add(found: list[dict], seen: set[str], command: str, source: str, evidence: str) -> None:
|
|
39
|
+
command = command.strip()
|
|
40
|
+
if not command or command in seen or len(found) >= CANDIDATE_CAP:
|
|
41
|
+
return
|
|
42
|
+
seen.add(command)
|
|
43
|
+
found.append(dict(command=command, source=source, evidence=evidence))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _forge_verify(repo: Path, found, seen) -> None:
|
|
47
|
+
path = repo / '.forge' / 'verify'
|
|
48
|
+
if not path.is_file():
|
|
49
|
+
return
|
|
50
|
+
text = path.read_text(errors='replace').strip()
|
|
51
|
+
if text:
|
|
52
|
+
_add(found, seen, text, 'forge/verify', str(path))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _package_json(repo: Path, found, seen) -> None:
|
|
56
|
+
path = repo / 'package.json'
|
|
57
|
+
if not path.is_file():
|
|
58
|
+
return
|
|
59
|
+
try:
|
|
60
|
+
data = json.loads(path.read_text())
|
|
61
|
+
except (OSError, ValueError):
|
|
62
|
+
return
|
|
63
|
+
scripts = data.get('scripts') if isinstance(data, dict) else None
|
|
64
|
+
if not isinstance(scripts, dict):
|
|
65
|
+
return
|
|
66
|
+
names = [n for n in scripts if isinstance(n, str) and _PKG_SCRIPT_RE.match(n)]
|
|
67
|
+
order = {name: i for i, name in enumerate(names)}
|
|
68
|
+
# A bare 'test' is the most likely full-suite entry point; put it and its siblings
|
|
69
|
+
# ('check', 'ci', 'verify') first, then the rest in file order for determinism.
|
|
70
|
+
names = sorted(names, key=lambda n: (n not in ('test', 'check', 'ci', 'verify'), order[n]))
|
|
71
|
+
for name in names:
|
|
72
|
+
_add(found, seen, f'npm run {name}', 'package.json', str(path))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _makefile(repo: Path, found, seen) -> None:
|
|
76
|
+
path = repo / 'Makefile'
|
|
77
|
+
if not path.is_file():
|
|
78
|
+
return
|
|
79
|
+
for line in path.read_text(errors='replace').splitlines():
|
|
80
|
+
m = re.match(r'^([A-Za-z0-9_-]+)\s*:(?!=)', line)
|
|
81
|
+
if m and m.group(1).lower() in _MAKE_TARGETS:
|
|
82
|
+
_add(found, seen, f'make {m.group(1)}', 'Makefile', str(path))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _tox_and_nox(repo: Path, found, seen) -> None:
|
|
86
|
+
tox = repo / 'tox.ini'
|
|
87
|
+
if tox.is_file():
|
|
88
|
+
_add(found, seen, 'tox', 'tox.ini', str(tox))
|
|
89
|
+
nox = repo / 'noxfile.py'
|
|
90
|
+
if nox.is_file():
|
|
91
|
+
_add(found, seen, 'nox', 'noxfile.py', str(nox))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# test_foo.py / foo_test.py are individual test modules, not suite runners. Offering
|
|
95
|
+
# each one would bury the single real runner under a dozen near-duplicates and eat the
|
|
96
|
+
# candidate cap on a repo with many test files.
|
|
97
|
+
_MODULE_RE = re.compile(r'^test_.*\.py$|^.*_test\.py$')
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _runner_scripts(repo: Path, found, seen) -> None:
|
|
101
|
+
for sub in ('tests', 'scripts', 'bin'):
|
|
102
|
+
directory = repo / sub
|
|
103
|
+
if not directory.is_dir():
|
|
104
|
+
continue
|
|
105
|
+
for entry in sorted(directory.iterdir()):
|
|
106
|
+
if not entry.is_file() or not _RUNNER_DIR_RE.search(entry.name):
|
|
107
|
+
continue
|
|
108
|
+
try:
|
|
109
|
+
executable = entry.stat().st_mode & 0o111 != 0
|
|
110
|
+
except OSError:
|
|
111
|
+
continue
|
|
112
|
+
rel = entry.relative_to(repo).as_posix()
|
|
113
|
+
if executable and entry.suffix in ('', '.sh', '.py'):
|
|
114
|
+
_add(found, seen, rel, sub + '/', str(entry))
|
|
115
|
+
# A runner script that was never chmod +x is still the project's real
|
|
116
|
+
# verify command -- forge's own tests/check.sh is mode 644 and its README
|
|
117
|
+
# says to run `bash tests/check.sh`. Naming the interpreter for a file that
|
|
118
|
+
# exists is still selection, not invention, so the candidate is offered and
|
|
119
|
+
# Jev decides. Skipping it left repos like that permanently UNVERIFIED.
|
|
120
|
+
elif not executable and entry.suffix in ('.sh', '.py') and not _MODULE_RE.match(entry.name):
|
|
121
|
+
interpreter = 'bash' if entry.suffix == '.sh' else 'python3'
|
|
122
|
+
_add(found, seen, interpreter + ' ' + rel, sub + '/', str(entry))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _pytest_configured(repo: Path, found, seen) -> None:
|
|
126
|
+
candidates = (
|
|
127
|
+
(repo / 'pytest.ini', True),
|
|
128
|
+
(repo / 'tox.ini', '[pytest]'),
|
|
129
|
+
(repo / 'setup.cfg', '[tool:pytest]'),
|
|
130
|
+
(repo / 'pyproject.toml', '[tool.pytest.ini_options]'),
|
|
131
|
+
)
|
|
132
|
+
for path, marker in candidates:
|
|
133
|
+
if not path.is_file():
|
|
134
|
+
continue
|
|
135
|
+
if marker is True:
|
|
136
|
+
_add(found, seen, 'pytest -q', path.name, str(path))
|
|
137
|
+
return
|
|
138
|
+
try:
|
|
139
|
+
if marker in path.read_text(errors='replace'):
|
|
140
|
+
_add(found, seen, 'pytest -q', path.name, str(path))
|
|
141
|
+
return
|
|
142
|
+
except OSError:
|
|
143
|
+
continue
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _go_and_cargo(repo: Path, found, seen) -> None:
|
|
147
|
+
go = repo / 'go.mod'
|
|
148
|
+
if go.is_file():
|
|
149
|
+
_add(found, seen, 'go test ./...', 'go.mod', str(go))
|
|
150
|
+
cargo = repo / 'Cargo.toml'
|
|
151
|
+
if cargo.is_file():
|
|
152
|
+
_add(found, seen, 'cargo test', 'Cargo.toml', str(cargo))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _workflows(repo: Path, found, seen) -> None:
|
|
156
|
+
directory = repo / '.github' / 'workflows'
|
|
157
|
+
if not directory.is_dir():
|
|
158
|
+
return
|
|
159
|
+
for path in sorted(directory.glob('*.yml')) + sorted(directory.glob('*.yaml')):
|
|
160
|
+
try:
|
|
161
|
+
lines = path.read_text(errors='replace').splitlines()
|
|
162
|
+
except OSError:
|
|
163
|
+
continue
|
|
164
|
+
# Plain-text scan on purpose (no YAML dependency): only single-line `run: <cmd>`
|
|
165
|
+
# steps are recognised, which is the common case for a test invocation step. A
|
|
166
|
+
# malformed file just yields no matches rather than raising.
|
|
167
|
+
for line in lines:
|
|
168
|
+
m = _WORKFLOW_RUN_RE.match(line)
|
|
169
|
+
if not m:
|
|
170
|
+
continue
|
|
171
|
+
command = m.group(1).strip().strip('"\'')
|
|
172
|
+
if command and command != '|' and _WORKFLOW_TEST_HINT_RE.search(command):
|
|
173
|
+
_add(found, seen, command, 'workflow', str(path))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _memory_verify(repo: Path, found, seen) -> None:
|
|
177
|
+
path = repo / '.forge' / 'memory.md'
|
|
178
|
+
if not path.is_file():
|
|
179
|
+
return
|
|
180
|
+
for line in path.read_text(errors='replace').splitlines():
|
|
181
|
+
if 'verify |' not in line:
|
|
182
|
+
continue
|
|
183
|
+
text = line.split('verify |', 1)[1]
|
|
184
|
+
# A memory entry is prose ("pytest -q runs the suite; make test also lints and
|
|
185
|
+
# is 4x slower"), not a bare command -- pull out anything that looks like a
|
|
186
|
+
# known runner invocation rather than treating the whole sentence as a command.
|
|
187
|
+
for pattern in _MEMORY_CMD_RES:
|
|
188
|
+
for match in pattern.finditer(text):
|
|
189
|
+
_add(found, seen, match.group(0), 'memory:verify', str(path))
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def candidates(repo) -> list[dict]:
|
|
193
|
+
"""Real, existing verify commands. Each: {'command': str, 'source': str, 'evidence': str}.
|
|
194
|
+
|
|
195
|
+
Deterministic order, deduplicated, capped at CANDIDATE_CAP. Never executes anything --
|
|
196
|
+
this only enumerates what plausibly exists so Jev can select among real options.
|
|
197
|
+
"""
|
|
198
|
+
repo = Path(repo)
|
|
199
|
+
found: list[dict] = []
|
|
200
|
+
seen: set[str] = set()
|
|
201
|
+
for step in (_forge_verify, _package_json, _makefile, _tox_and_nox, _runner_scripts,
|
|
202
|
+
_pytest_configured, _go_and_cargo, _workflows, _memory_verify):
|
|
203
|
+
if len(found) >= CANDIDATE_CAP:
|
|
204
|
+
break
|
|
205
|
+
step(repo, found, seen)
|
|
206
|
+
return found[:CANDIDATE_CAP]
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _repo_files(repo: Path, limit: int = 200) -> list[str]:
|
|
210
|
+
try:
|
|
211
|
+
entries = sorted(e.name for e in repo.iterdir())
|
|
212
|
+
except OSError:
|
|
213
|
+
return []
|
|
214
|
+
return entries[:limit]
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# The headings forge-memory.sh:93 renders. Both readers below looked for the LEDGER row
|
|
218
|
+
# format instead -- `verify |` and `trap |` -- which is what ledger.tsv holds, not what
|
|
219
|
+
# `rebuild` writes into memory.md. No line in a rendered memory.md has ever contained
|
|
220
|
+
# either, so both lists were empty for every repo, always, while jev.md documented them
|
|
221
|
+
# as working. Verification discovery never saw a recorded verify command, and
|
|
222
|
+
# known_traps never corroborated anything -- which made the flake re-run unreachable,
|
|
223
|
+
# since its third condition is a trap sharing a distinctive token with the failure.
|
|
224
|
+
_MEMORY_SECTIONS = {'verify': '## Verify', 'trap': '## Known traps',
|
|
225
|
+
'finding': '## Recurring QA findings'}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _memory_facts(repo: Path, category: str) -> list[str]:
|
|
229
|
+
"""The `- ` bullets under one heading of a rendered .forge/memory.md."""
|
|
230
|
+
heading = _MEMORY_SECTIONS.get(category)
|
|
231
|
+
path = Path(repo) / '.forge' / 'memory.md'
|
|
232
|
+
if not heading or not path.is_file():
|
|
233
|
+
return []
|
|
234
|
+
try:
|
|
235
|
+
lines = path.read_text(errors='replace').splitlines()
|
|
236
|
+
except OSError:
|
|
237
|
+
return []
|
|
238
|
+
facts, inside = [], False
|
|
239
|
+
for line in lines:
|
|
240
|
+
stripped = line.strip()
|
|
241
|
+
if stripped.startswith('## '):
|
|
242
|
+
inside = stripped == heading
|
|
243
|
+
continue
|
|
244
|
+
if inside and stripped.startswith('- '):
|
|
245
|
+
fact = stripped[2:].strip()
|
|
246
|
+
# `rebuild` prefixes a recurring finding with its count ("3x: ..."), which is
|
|
247
|
+
# bookkeeping about the fact rather than part of it.
|
|
248
|
+
if fact[:1].isdigit() and 'x: ' in fact[:6]:
|
|
249
|
+
fact = fact.split('x: ', 1)[1].strip()
|
|
250
|
+
if fact:
|
|
251
|
+
facts.append(fact)
|
|
252
|
+
return facts
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _memory_verify_lines(repo: Path) -> list[str]:
|
|
256
|
+
return _memory_facts(repo, 'verify')
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def known_traps(repo) -> list[str]:
|
|
260
|
+
"""Recorded 'trap' facts from .forge/memory.md (flaky tests, generated files, sharp
|
|
261
|
+
edges), for the CLI to hand to triage() -- see references/memory.md."""
|
|
262
|
+
path = Path(repo) / '.forge' / 'memory.md'
|
|
263
|
+
if not path.is_file():
|
|
264
|
+
return []
|
|
265
|
+
try:
|
|
266
|
+
lines = path.read_text(errors='replace').splitlines()
|
|
267
|
+
except OSError:
|
|
268
|
+
return []
|
|
269
|
+
return _memory_facts(path.parent.parent, 'trap')
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
# Enough of an evidence file to tell a test command from a build one, and no more: the
|
|
273
|
+
# request already carries one Choice and two Nouls per candidate.
|
|
274
|
+
EVIDENCE_CHARS = 400
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _evidence_excerpt(candidate: dict) -> str:
|
|
278
|
+
"""What the command actually runs, read off the file that produced the candidate.
|
|
279
|
+
|
|
280
|
+
Without this the model is handed the string 'make test' plus the repo's top-level
|
|
281
|
+
file list and asked whether it runs the suite. It cannot know: the answer is inside
|
|
282
|
+
the Makefile. Measured on a repo with a textbook `test:` target, discovery picked
|
|
283
|
+
`make test` at 0.58 while leaving 0.40 on `none`, for a Choice confidence of 0.37
|
|
284
|
+
against a 0.70 gate -- so it declined, and a verifiable repo reported UNVERIFIED.
|
|
285
|
+
"""
|
|
286
|
+
path = candidate.get('evidence') or ''
|
|
287
|
+
try:
|
|
288
|
+
text = Path(path).read_text(errors='replace')
|
|
289
|
+
except OSError:
|
|
290
|
+
return ''
|
|
291
|
+
command = candidate.get('command') or ''
|
|
292
|
+
# A make target's recipe is the only part of a Makefile that answers the question,
|
|
293
|
+
# and it is rarely near the top of the file.
|
|
294
|
+
if command.startswith('make ') and Path(path).name.lower().startswith('makefile'):
|
|
295
|
+
target = command.split(None, 1)[1].strip()
|
|
296
|
+
recipe, collecting = [], False
|
|
297
|
+
for line in text.splitlines():
|
|
298
|
+
if collecting:
|
|
299
|
+
if line.startswith(('\t', ' ')) or not line.strip():
|
|
300
|
+
recipe.append(line.strip())
|
|
301
|
+
continue
|
|
302
|
+
break
|
|
303
|
+
if ':' in line and line.split(':')[0].strip() == target:
|
|
304
|
+
collecting = True
|
|
305
|
+
joined = '\n'.join(x for x in recipe if x)
|
|
306
|
+
if joined:
|
|
307
|
+
return joined[:EVIDENCE_CHARS]
|
|
308
|
+
return text[:EVIDENCE_CHARS]
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def discover(repo, *, config=None, run_dir=None) -> dict | None:
|
|
312
|
+
"""Ask Jev which candidate actually verifies this repo. None when Jev is
|
|
313
|
+
unavailable, no candidates exist, or it picks `none`.
|
|
314
|
+
-> {'command', 'confidence', 'verifiable', 'runs_tests', 'runtime_level', 'source',
|
|
315
|
+
'evidence'}
|
|
316
|
+
"""
|
|
317
|
+
config = config or load_config()
|
|
318
|
+
if not enabled('tests', config=config):
|
|
319
|
+
return None
|
|
320
|
+
repo_path = Path(repo)
|
|
321
|
+
cand = candidates(repo_path)
|
|
322
|
+
if not cand:
|
|
323
|
+
return None
|
|
324
|
+
commands = [c['command'] for c in cand]
|
|
325
|
+
|
|
326
|
+
# One request covers the Choice and a runs_tests/verify_runtime pair per candidate --
|
|
327
|
+
# they all evaluate in parallel server-side, so asking about every candidate here
|
|
328
|
+
# costs exactly one call, not one per candidate.
|
|
329
|
+
questions = {'choice': verify_selection(commands)}
|
|
330
|
+
for i in range(len(cand)):
|
|
331
|
+
questions[f'runs_{i}'] = runs_tests(commands[i])
|
|
332
|
+
questions[f'runtime_{i}'] = verify_runtime(commands[i])
|
|
333
|
+
|
|
334
|
+
# The candidates carry where they came from and what they run. Sending only the
|
|
335
|
+
# command strings threw away the single thing that answers the question being asked.
|
|
336
|
+
described = [dict(command=c['command'], source=c.get('source', ''),
|
|
337
|
+
runs=_evidence_excerpt(c)) for c in cand]
|
|
338
|
+
state = dict(repo_files=_repo_files(repo_path), candidates=described,
|
|
339
|
+
memory_verify=_memory_verify_lines(repo_path))
|
|
340
|
+
result = ask(state, questions, site='jev-verify-discover', run_dir=run_dir, config=config)
|
|
341
|
+
if result is None:
|
|
342
|
+
return None
|
|
343
|
+
|
|
344
|
+
picked, confidence = choice(result, 'choice')
|
|
345
|
+
if picked is None or picked == 'none':
|
|
346
|
+
return None
|
|
347
|
+
|
|
348
|
+
# Gate on "is this repo verifiable at all", not on "which command wins".
|
|
349
|
+
#
|
|
350
|
+
# Choice confidence answers the second question, and for a selection among several
|
|
351
|
+
# ACCEPTABLE options it is the wrong quantity: a repo offering both `bash
|
|
352
|
+
# tests/check.sh` and `python3 -m unittest discover -s tests` splits the mass
|
|
353
|
+
# 0.62/0.31 with only 0.07 on `none`, for a confidence of 0.49 against a 0.70 gate.
|
|
354
|
+
# The model is 93% sure the repo is verifiable and forge reported UNVERIFIED --
|
|
355
|
+
# because two right answers look exactly like doubt to a rubric that measures
|
|
356
|
+
# agreement on one.
|
|
357
|
+
#
|
|
358
|
+
# 1 - P(none) is the quantity `tests_act` was always meant to gate. A split between
|
|
359
|
+
# correct commands leaves it high; genuine doubt piles onto `none` and drops it.
|
|
360
|
+
# Ties are then broken by the winner, which is what Choice is for.
|
|
361
|
+
distribution = probabilities(result, 'choice')
|
|
362
|
+
act_at = threshold('tests_act', config=config)
|
|
363
|
+
if distribution:
|
|
364
|
+
if (1.0 - distribution.get('none', 0.0)) < act_at:
|
|
365
|
+
return None
|
|
366
|
+
elif confidence < act_at:
|
|
367
|
+
# No distribution in the response: fall back to the old gate rather than treat a
|
|
368
|
+
# missing field as certainty.
|
|
369
|
+
return None
|
|
370
|
+
try:
|
|
371
|
+
idx = commands.index(picked)
|
|
372
|
+
except ValueError:
|
|
373
|
+
# Choice is constrained to the options we sent, so this should not happen -- but
|
|
374
|
+
# a command we never enumerated must never be reported as "selected", so bail.
|
|
375
|
+
return None
|
|
376
|
+
|
|
377
|
+
runs_probability = noul(result, f'runs_{idx}')
|
|
378
|
+
if runs_probability is None or runs_probability < 0.5:
|
|
379
|
+
# A command that only builds or lints must not be presented as verification,
|
|
380
|
+
# even if the Choice rubric itself scored it confidently.
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
runtime_score, _runtime_conf = score(result, f'runtime_{idx}')
|
|
384
|
+
runtime_levels = questions[f'runtime_{idx}']['criteria']
|
|
385
|
+
runtime_level = None
|
|
386
|
+
if runtime_score is not None and 0 <= int(runtime_score) < len(runtime_levels):
|
|
387
|
+
runtime_level = runtime_levels[int(runtime_score)]
|
|
388
|
+
|
|
389
|
+
picked_candidate = cand[idx]
|
|
390
|
+
# `verifiable` is the quantity the gate actually used; `confidence` is kept beside it
|
|
391
|
+
# because they answer different questions and a future re-measurement needs both.
|
|
392
|
+
verifiable = (1.0 - distribution.get('none', 0.0)) if distribution else None
|
|
393
|
+
return dict(command=picked, confidence=confidence, verifiable=verifiable,
|
|
394
|
+
runs_tests=runs_probability,
|
|
395
|
+
runtime_level=runtime_level, source=picked_candidate['source'],
|
|
396
|
+
evidence=picked_candidate['evidence'])
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
# A token is distinctive enough to corroborate only if it looks like an identifier or a
|
|
400
|
+
# path -- `test_timeout`, `tests/test_api.py`, `CamelCase` -- rather than an English word
|
|
401
|
+
# that any failure log would contain.
|
|
402
|
+
_TOKEN_RE = re.compile(r'[A-Za-z_][A-Za-z0-9_./:-]{3,}')
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _distinctive(text: str) -> set[str]:
|
|
406
|
+
tokens = set()
|
|
407
|
+
for match in _TOKEN_RE.findall(text or ''):
|
|
408
|
+
# Split on path and node-id separators: a trap says `test_api.py::test_timeout`
|
|
409
|
+
# while the log says `tests/test_api.py::test_timeout`, which never compare equal
|
|
410
|
+
# as whole tokens. Atoms do.
|
|
411
|
+
for token in re.split(r'[/:]+', match):
|
|
412
|
+
token = token.strip('.-_')
|
|
413
|
+
if len(token) < 4:
|
|
414
|
+
continue
|
|
415
|
+
# A bare short word like `tests` matches every repo; require either an
|
|
416
|
+
# identifier/path shape or enough length to be specific.
|
|
417
|
+
if '_' in token or '.' in token or len(token) >= 8:
|
|
418
|
+
tokens.add(token)
|
|
419
|
+
return tokens
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def corroborating_traps(log_tail: str, traps) -> list[str]:
|
|
423
|
+
"""Traps that share a distinctive token with the failure output.
|
|
424
|
+
|
|
425
|
+
The re-run gate needs evidence that THIS failure matches a recorded trap, not merely
|
|
426
|
+
that the repo has recorded some trap at some point. Without that, one flaky test
|
|
427
|
+
noted months ago would excuse every future failure in the repo -- which is precisely
|
|
428
|
+
how a real regression gets waved through.
|
|
429
|
+
"""
|
|
430
|
+
haystack = _distinctive(log_tail)
|
|
431
|
+
return [trap for trap in (traps or []) if _distinctive(trap) & haystack]
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def triage(repo, *, command, log_tail, traps, config=None, run_dir=None) -> dict | None:
|
|
435
|
+
"""Classify a verification failure. -> {'verdict', 'confidence'} or None."""
|
|
436
|
+
config = config or load_config()
|
|
437
|
+
if not enabled('tests', config=config):
|
|
438
|
+
return None
|
|
439
|
+
tail = (log_tail or '')[-LOG_TAIL_CHARS:]
|
|
440
|
+
# The diff is never sent: recognising a flake needs the failure signature (timeout,
|
|
441
|
+
# ordering, port-in-use, a named trap), not the change itself, and the diff is a
|
|
442
|
+
# larger payload for no benefit here.
|
|
443
|
+
state = dict(command=command, log_tail=tail, known_traps=list(traps or []))
|
|
444
|
+
result = ask(state, {'triage': failure_triage(list(traps or []))}, site='jev-verify-triage',
|
|
445
|
+
run_dir=run_dir, config=config)
|
|
446
|
+
if result is None:
|
|
447
|
+
return None
|
|
448
|
+
verdict, confidence = choice(result, 'triage')
|
|
449
|
+
if verdict is None:
|
|
450
|
+
return None
|
|
451
|
+
matched = corroborating_traps(tail, traps)
|
|
452
|
+
return dict(verdict=verdict, confidence=confidence,
|
|
453
|
+
corroborated=bool(matched), corroborating_traps=matched)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Where a repo's run directories live, outside the repo.
|
|
2
|
+
|
|
3
|
+
``$XDG_CONFIG_HOME/forge/runs/<sha256(realpath(repo))[:16]>.txt`` holds one absolute
|
|
4
|
+
run-dir path per line, with a ``#`` header on a new file. ``forge jev``, ``forge
|
|
5
|
+
dataset import --registry`` and the forge-ui launcher all use it, so it does not
|
|
6
|
+
live inside the Jev package.
|
|
7
|
+
|
|
8
|
+
Run as a script, ``forge_registry.py register <run-dir>`` records a run dir from the
|
|
9
|
+
shell runners (forge-ui-options.sh ``forge_ui_register``): every plan, run and solo run
|
|
10
|
+
is registered, not only the ones opened with ``--ui``, because forge-ui's map of a
|
|
11
|
+
repo's runs reads this registry. ``FORGE_RUN_REGISTRY=off`` turns that off.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import os
|
|
17
|
+
import stat
|
|
18
|
+
import sys
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def registry_dir() -> Path:
|
|
23
|
+
"""The ``runs`` directory under forge's config home.
|
|
24
|
+
|
|
25
|
+
Resolved the same way ``forge_jev.config_path()`` resolves ``jev.json``:
|
|
26
|
+
``XDG_CONFIG_HOME`` when set and not empty, otherwise ``~/.config``, then
|
|
27
|
+
``expanduser()``. An empty value counts as unset, as the XDG spec says and as
|
|
28
|
+
forge-ui's run list reads it; taken literally it named a directory below the
|
|
29
|
+
current one, usually the user's repo. ``config_path()`` is
|
|
30
|
+
``<that>/forge/jev.json``; the registry is the sibling ``runs`` directory.
|
|
31
|
+
"""
|
|
32
|
+
config = (Path(os.environ.get('XDG_CONFIG_HOME') or str(Path.home() / '.config')).expanduser()
|
|
33
|
+
/ 'forge/jev.json')
|
|
34
|
+
return config.parent / 'runs'
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def repo_key(repo) -> str:
|
|
38
|
+
"""16 hex chars of ``sha256(realpath(repo))``. The registry file name and the dataset key."""
|
|
39
|
+
return hashlib.sha256(str(Path(repo).resolve()).encode()).hexdigest()[:16]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def runs_registry(repo) -> Path:
|
|
43
|
+
"""Where this repo's run-dir list lives -- beside the forge config, never in the repo.
|
|
44
|
+
|
|
45
|
+
The first version of this wrote <repo>/.forge/jev-runs.txt, which left the user's
|
|
46
|
+
working tree dirty and made `integrate` refuse to run: "working tree is dirty --
|
|
47
|
+
commit or stash before integrating". Forge's contract is that it does not touch the
|
|
48
|
+
user's tree, and a file recording where forge keeps its own data is forge's
|
|
49
|
+
bookkeeping, not the project's.
|
|
50
|
+
"""
|
|
51
|
+
return registry_dir() / (repo_key(repo) + '.txt')
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def register_run_dir(repo: Path, run_dir: Path) -> None:
|
|
55
|
+
"""Record an absolute run-dir path for this repo, deduplicated.
|
|
56
|
+
|
|
57
|
+
Never raises and never blocks: failing to note where a run lives must not fail the run,
|
|
58
|
+
and ``forge ui open`` calls this before any of its own timeouts.
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
# A relative XDG_CONFIG_HOME resolves below the current directory -- for a runner or
|
|
62
|
+
# `forge ui open`, usually the user's repo. Forge does not write into the user's tree.
|
|
63
|
+
if not registry_dir().is_absolute():
|
|
64
|
+
return
|
|
65
|
+
registry = runs_registry(repo)
|
|
66
|
+
registry.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
line = str(Path(run_dir).resolve())
|
|
68
|
+
existing = []
|
|
69
|
+
if registry.is_file():
|
|
70
|
+
existing = [x.strip() for x in registry.read_text(errors='replace').splitlines()]
|
|
71
|
+
if line in existing:
|
|
72
|
+
return
|
|
73
|
+
header = [] if existing else ['# forge run dirs for ' + str(Path(repo).resolve())]
|
|
74
|
+
# O_NONBLOCK: a FIFO with no reader fails at once (ENXIO) instead of blocking the
|
|
75
|
+
# open, and anything that is not a regular file is left alone.
|
|
76
|
+
fd = os.open(registry, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NONBLOCK, 0o644)
|
|
77
|
+
with os.fdopen(fd, 'a') as handle:
|
|
78
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
79
|
+
return
|
|
80
|
+
for entry in header:
|
|
81
|
+
handle.write(entry + '\n')
|
|
82
|
+
handle.write(line + '\n')
|
|
83
|
+
except (OSError, ValueError, RuntimeError):
|
|
84
|
+
# ValueError: a NUL byte in a path; RuntimeError: a symlink loop (pathlib before 3.13)
|
|
85
|
+
pass
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# A path, not a file's worth of data: the first line of <run-dir>/repo is all that is read.
|
|
89
|
+
_REPO_LINE_MAX = 64 * 1024
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _repo_of(run_dir: Path) -> Path | None:
|
|
93
|
+
"""The repo named on the first line of ``<run-dir>/repo``, or None. Never blocks.
|
|
94
|
+
|
|
95
|
+
The runners write that file themselves, but a run dir is any directory a user passes,
|
|
96
|
+
so the read is as careful as forge-ui's: O_NONBLOCK so a FIFO with no writer cannot
|
|
97
|
+
hold the open, and nothing but a regular file is read (a FIFO, a device or a directory
|
|
98
|
+
named ``repo`` is ignored).
|
|
99
|
+
"""
|
|
100
|
+
fd = os.open(run_dir / 'repo', os.O_RDONLY | os.O_NONBLOCK)
|
|
101
|
+
with os.fdopen(fd, 'rb') as handle:
|
|
102
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
103
|
+
return None
|
|
104
|
+
line = handle.readline(_REPO_LINE_MAX).decode(errors='replace').strip()
|
|
105
|
+
# Empty, or text no path can hold, or relative: a relative repo would be resolved
|
|
106
|
+
# against whatever directory the runner happened to start in, and keyed to the wrong repo.
|
|
107
|
+
if not line or '\x00' in line or not os.path.isabs(line):
|
|
108
|
+
return None
|
|
109
|
+
return Path(line)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _register(run_dir_arg: str) -> None:
|
|
113
|
+
if os.environ.get('FORGE_RUN_REGISTRY', '').strip().lower() == 'off':
|
|
114
|
+
return
|
|
115
|
+
run_dir = Path(run_dir_arg).resolve()
|
|
116
|
+
repo = _repo_of(run_dir)
|
|
117
|
+
if repo is not None:
|
|
118
|
+
register_run_dir(repo, run_dir)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def main() -> int:
|
|
122
|
+
"""``register <run-dir>``: record the run dir under the repo its ``repo`` file names.
|
|
123
|
+
|
|
124
|
+
Always exits 0 and prints nothing. The runners call it on every plan, run and solo run,
|
|
125
|
+
and noting where a run lives must never fail, block, slow or change one: a missing
|
|
126
|
+
run dir, an unreadable or hostile ``repo`` file, an unwritable registry and arguments
|
|
127
|
+
it does not understand are all silently nothing. Ctrl-C is not swallowed, so an
|
|
128
|
+
interrupted runner still stops.
|
|
129
|
+
"""
|
|
130
|
+
try:
|
|
131
|
+
args = sys.argv[1:]
|
|
132
|
+
if len(args) == 2 and args[0] == 'register':
|
|
133
|
+
_register(args[1])
|
|
134
|
+
except Exception: # noqa: BLE001 -- see the docstring: nothing here may reach the run
|
|
135
|
+
pass
|
|
136
|
+
return 0
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
if __name__ == '__main__':
|
|
140
|
+
sys.exit(main())
|