@giamme/forge 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/SKILL.md +19 -1
- package/VERSION +1 -1
- package/docs/artifacts.md +17 -2
- package/docs/commands.md +50 -0
- package/docs/dataset.md +9 -0
- package/docs/decomposed-runs.md +42 -0
- package/docs/forge-ui.md +42 -6
- package/docs/reviews.md +14 -0
- package/forge +15 -0
- package/package.json +1 -1
- package/references/dataset.md +18 -1
- package/references/decompose.md +19 -1
- package/references/ui.md +27 -5
- package/scripts/forge-config.py +347 -0
- package/scripts/forge-control.py +645 -0
- package/scripts/forge-issues.py +806 -0
- package/scripts/forge-lib-task.sh +234 -35
- package/scripts/forge-parallel-options.sh +3 -1
- package/scripts/forge-parallel.sh +3 -2
- package/scripts/forge-route.py +184 -27
- package/scripts/forge-runs-rm.py +840 -0
- package/scripts/forge-runs.py +19 -1
- package/scripts/forge-schedule.py +227 -11
- package/scripts/forge-ui.py +85 -19
- package/scripts/forge_dataset/__init__.py +77 -6
- package/scripts/forge_dataset/capture.py +5 -2
- package/scripts/forge_dataset/harvest.py +22 -0
- package/scripts/forge_dataset/importer.py +419 -17
- package/scripts/forge_registry.py +168 -16
- package/ui/README.md +37 -4
- package/ui/docs/api.md +184 -6
- package/ui/forge_ui/__main__.py +10 -2
- package/ui/forge_ui/control.py +575 -0
- package/ui/forge_ui/export.py +17 -3
- package/ui/forge_ui/reader.py +26 -1
- package/ui/forge_ui/server.py +125 -4
- package/ui/forge_ui/static/app.js +80 -8
- package/ui/forge_ui/static/controls.js +619 -0
- package/ui/forge_ui/static/findingsview.js +412 -0
- package/ui/forge_ui/static/index.html +7 -1
- package/ui/forge_ui/static/panel.js +24 -4
- package/ui/forge_ui/static/rundelete.js +388 -0
- package/ui/forge_ui/static/settings.js +399 -0
- package/ui/forge_ui/static/style.css +177 -2
- package/ui/forge_ui/static/uikit.js +161 -0
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""forge issues <run>: file the review findings a run left open as GitHub issues, with plain `gh`.
|
|
3
|
+
|
|
4
|
+
A run can end with findings nobody will act on: CONFIRMED findings `--qa-threshold` tolerated
|
|
5
|
+
(`tasks/<id>/known_issues.md`), PLAUSIBLE findings in a passed task's `qa.last` (with
|
|
6
|
+
`--include-plausible`), and whatever a whole-run review left open. This drafts one issue per
|
|
7
|
+
finding (or per task/run with `--group`) and creates them only with `--apply`.
|
|
8
|
+
|
|
9
|
+
<run> is a plan dir, a task dir, a review dir (`<plan>/final-review`), known_issues.md or qa.last.
|
|
10
|
+
Tasks that did not pass are skipped: their findings still block them.
|
|
11
|
+
|
|
12
|
+
Each issue carries a hidden `<!-- forge:finding fp=... -->` marker. The fingerprint is the one
|
|
13
|
+
gh-proman computes, and both tools skip a finding either of them already filed, in any state.
|
|
14
|
+
When the gh-proman skill is configured for the repository, prefer `gh-proman findings`: it also
|
|
15
|
+
adds the issues to the team's project board with a priority. This command is the fallback.
|
|
16
|
+
|
|
17
|
+
--json lists every finding in `findings` (fp, severity, origin, plan, task, text, `filed`, ...). --offline never
|
|
18
|
+
runs `gh` (no filed state, publication of commits judged from local refs only). --only FP[,FP] keeps those
|
|
19
|
+
fingerprints. --all-runs reads <run> as a repository (or a plan dir naming one) and collects every run in its
|
|
20
|
+
registry, the newest plan winning for a finding present in several. A caller filing an explicit selection
|
|
21
|
+
passes `--max-issues <len(selection)>`; the default is 10.
|
|
22
|
+
|
|
23
|
+
--dismiss FP[,FP] / --undismiss FP[,FP] (repeatable, with an optional --reason TEXT of at most 500 characters) mark
|
|
24
|
+
findings that need no work, or take the mark back, then exit; they need no network and accept a fingerprint that
|
|
25
|
+
matches no current finding. A dismissed finding stays in `findings` with `dismissed` = {at, reason} (null otherwise)
|
|
26
|
+
and is left out of the drafts unless --include-dismissed is given. The marks live outside the repository, in
|
|
27
|
+
`${XDG_STATE_HOME:-~/.local/state}/forge/findings/<repo_key>.json` (an empty or relative XDG_STATE_HOME counts as
|
|
28
|
+
unset); the store directory and file must not be symlinks, and the file must be a regular one. The repository they
|
|
29
|
+
belong to is the one <run> resolves to; --repo does not change it.
|
|
30
|
+
|
|
31
|
+
The repository is the run's own, whatever form <run> takes: a file or task dir resolves to its plan dir and that plan's
|
|
32
|
+
`repo` file, a solo run dir to its own `repo` file, and --all-runs to the repository it names. Listing, filing and
|
|
33
|
+
(un)dismissing all resolve it the same way, so a mark lands in the store a later listing of the run reads. Only a run
|
|
34
|
+
that records no repository falls back to the current directory, and the output says so (`warnings`).
|
|
35
|
+
|
|
36
|
+
Two processes must not file for one repository at once, or both see a fingerprint as unfiled and create it. --apply
|
|
37
|
+
holds `<repo_key>.lock` (next to the store, `flock`) from the `gh issue list` lookup to the last `gh issue create`,
|
|
38
|
+
and each --dismiss/--undismiss holds it around its read-modify-write. A busy lock is retried for 10 s. Listing
|
|
39
|
+
without --apply takes no lock.
|
|
40
|
+
|
|
41
|
+
Exit codes: 0 ok, 2 usage, `gh` or dismissal store error, 3 busy (another filing or dismissal holds the lock).
|
|
42
|
+
"""
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import argparse
|
|
46
|
+
import contextlib
|
|
47
|
+
import errno
|
|
48
|
+
import fcntl
|
|
49
|
+
import hashlib
|
|
50
|
+
import json
|
|
51
|
+
import os
|
|
52
|
+
import re
|
|
53
|
+
import stat
|
|
54
|
+
import subprocess
|
|
55
|
+
import sys
|
|
56
|
+
import tempfile
|
|
57
|
+
import time
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
|
|
60
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
61
|
+
from forge_registry import repo_key, runs_registry # noqa: E402
|
|
62
|
+
|
|
63
|
+
FINDING_RE = re.compile(r"^\s*[-*]\s*\[(P[0-3])\]\s*\[(CONFIRMED|PLAUSIBLE)\]\s*(\S.*?)\s+[—–]+\s+(.+?)\s*$", re.M)
|
|
64
|
+
# gh-proman writes `gh-proman:finding`, this command `forge:finding`; the fingerprints match.
|
|
65
|
+
FP_MARK_RE = re.compile(r"<!-- (?:gh-proman|forge):finding fp=([0-9a-f]{12})")
|
|
66
|
+
DONE = {"PASS", "MERGED"}
|
|
67
|
+
SEVERITIES = ["P0", "P1", "P2", "P3"]
|
|
68
|
+
FP_RE = re.compile(r"[0-9a-f]{12}")
|
|
69
|
+
LOCK_WAIT = 10.0 # seconds a busy repository lock is retried before `forge issues` gives up (exit 3)
|
|
70
|
+
LOCK_POLL = 0.1
|
|
71
|
+
REASON_MAX = 500
|
|
72
|
+
STORE_MAX = 4 << 20 # a dismissal store is a few hundred bytes per finding; anything bigger is not one
|
|
73
|
+
|
|
74
|
+
TEXT = {
|
|
75
|
+
"en": {
|
|
76
|
+
"plausible": "To verify: ",
|
|
77
|
+
"tolerated": "Forge review finding left open: it was below the blocking threshold {threshold}, so the "
|
|
78
|
+
"work was merged without fixing it.",
|
|
79
|
+
"review": "Forge whole-run review finding left open: check whether it was already fixed before working on it.",
|
|
80
|
+
"plausible_ctx": "PLAUSIBLE forge review finding (low confidence): verify it before fixing. If it turns out "
|
|
81
|
+
"to be wrong, close the issue saying why.",
|
|
82
|
+
"group": "Forge review findings left open in run `{plan}` ({n}).",
|
|
83
|
+
"group_title": "Non-blocking forge findings: {scope}",
|
|
84
|
+
"origin": "Origin: forge `{plan}`, task `{task}`, severity {severity}, reviewed by `{reviewer}`{commit}.",
|
|
85
|
+
"headings": ["Context", "Finding", "Tasks", "Closing criteria", "References"],
|
|
86
|
+
"tasks": ["Reproduce the described case", "Fix it", "Add a regression test that fails before the fix"],
|
|
87
|
+
"tasks_plausible": ["Check whether the problem is real (reproduce or refute it)",
|
|
88
|
+
"If real: fix it and add a regression test"],
|
|
89
|
+
"closing": ["The described case no longer reproduces"],
|
|
90
|
+
"closing_plausible": ["The problem is fixed, or refuted with the reasoning in a comment"],
|
|
91
|
+
"code": "Code",
|
|
92
|
+
"unpublished": "(commit not pushed to GitHub yet)",
|
|
93
|
+
},
|
|
94
|
+
"it": {
|
|
95
|
+
"plausible": "Da verificare: ",
|
|
96
|
+
"tolerated": "Segnalazione della review di forge rimasta aperta: era sotto la soglia bloccante {threshold}, "
|
|
97
|
+
"quindi il lavoro è stato integrato senza correggerla.",
|
|
98
|
+
"review": "Segnalazione della review complessiva di forge rimasta aperta: verificare se è già stata "
|
|
99
|
+
"corretta prima di lavorarci.",
|
|
100
|
+
"plausible_ctx": "Segnalazione PLAUSIBILE della review di forge (confidenza bassa): va verificata prima di "
|
|
101
|
+
"correggere. Se risulta infondata, chiudi l'issue spiegando perché.",
|
|
102
|
+
"group": "Segnalazioni della review di forge rimaste aperte nel run `{plan}` ({n}).",
|
|
103
|
+
"group_title": "Segnalazioni forge non bloccanti: {scope}",
|
|
104
|
+
"origin": "Origine: forge `{plan}`, task `{task}`, gravità {severity}, review di `{reviewer}`{commit}.",
|
|
105
|
+
"headings": ["Contesto", "Segnalazione", "Attività", "Criteri di chiusura", "Riferimenti"],
|
|
106
|
+
"tasks": ["Riprodurre il caso descritto", "Correggere",
|
|
107
|
+
"Aggiungere un test di regressione che fallisce prima della correzione"],
|
|
108
|
+
"tasks_plausible": ["Verificare se il problema è reale (riprodurlo o smentirlo)",
|
|
109
|
+
"Se reale: correggere e aggiungere un test di regressione"],
|
|
110
|
+
"closing": ["Il caso descritto non si riproduce più"],
|
|
111
|
+
"closing_plausible": ["Il problema è corretto, oppure smentito con motivazione in un commento"],
|
|
112
|
+
"code": "Codice",
|
|
113
|
+
"unpublished": "(commit non ancora pubblicato su GitHub)",
|
|
114
|
+
},
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class Fail(Exception):
|
|
119
|
+
pass
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Busy(Fail):
|
|
123
|
+
"""The repository lock is held by another filing or dismissal (exit 3, not 2)."""
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def read(path: Path) -> str:
|
|
127
|
+
try:
|
|
128
|
+
return path.read_text(encoding="utf-8").strip()
|
|
129
|
+
except OSError:
|
|
130
|
+
return ""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run(cmd, cwd=None, stdin=None, check=True):
|
|
134
|
+
env = dict(os.environ, LC_ALL="C", GH_PROMPT_DISABLED="1", GH_NO_UPDATE_NOTIFIER="1")
|
|
135
|
+
try:
|
|
136
|
+
res = subprocess.run(cmd, cwd=cwd, input=stdin, capture_output=True, text=True, env=env, timeout=120)
|
|
137
|
+
except FileNotFoundError:
|
|
138
|
+
raise Fail(f"{cmd[0]} is not installed")
|
|
139
|
+
if check and res.returncode != 0:
|
|
140
|
+
raise Fail(f"{' '.join(cmd[:3])} failed: {(res.stderr or res.stdout).strip()[:400]}")
|
|
141
|
+
return res
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------------------------------------- collect
|
|
145
|
+
|
|
146
|
+
def clean_markdown(text):
|
|
147
|
+
text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
|
|
148
|
+
return re.sub(r"(\*\*|__|`)", "", text).strip()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def parse_findings(text):
|
|
152
|
+
out = []
|
|
153
|
+
for m in FINDING_RE.finditer(text or ""):
|
|
154
|
+
loc = m.group(3).strip()
|
|
155
|
+
link = re.fullmatch(r"\[([^\]]+)\]\([^)]*\)", loc)
|
|
156
|
+
loc = (link.group(1) if link else loc).strip("` ")
|
|
157
|
+
lm = re.match(r"(.+?):(\d+)(?:[-:]\d+)?$", loc)
|
|
158
|
+
out.append({"severity": m.group(1), "confidence": m.group(2), "file": lm.group(1) if lm else loc,
|
|
159
|
+
"line": int(lm.group(2)) if lm else None, "text": clean_markdown(m.group(4)),
|
|
160
|
+
"raw": f"[{m.group(1)}][{m.group(2)}] {loc} — {m.group(4).strip()}"})
|
|
161
|
+
return out
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def reviewer_spec(task_dir: Path):
|
|
165
|
+
kv = dict(line.split("=", 1) for line in read(task_dir / "qa.resolved").splitlines() if "=" in line)
|
|
166
|
+
return ":".join(x for x in (kv.get("alias"), kv.get("effort"), kv.get("harness")) if x) or "?"
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def task_findings(task_dir: Path, plan_dir, include_plausible, reviewers, review=None):
|
|
170
|
+
"""`review` True: task_dir is a whole-run review dir (it may hold a known_issues.md when a threshold applied)."""
|
|
171
|
+
status = (read(task_dir / "status").split() or [""])[0]
|
|
172
|
+
results = {}
|
|
173
|
+
if plan_dir and (plan_dir / "results.tsv").is_file():
|
|
174
|
+
for line in read(plan_dir / "results.tsv").splitlines():
|
|
175
|
+
cols = line.split("\t")
|
|
176
|
+
if len(cols) >= 2:
|
|
177
|
+
results[cols[0]] = cols[1]
|
|
178
|
+
task = task_dir.name
|
|
179
|
+
state = results.get(task, status)
|
|
180
|
+
meta = {"task": task, "commit": read(task_dir / "merged") or read(task_dir / "reviewed.commit"),
|
|
181
|
+
"reviewer": reviewers.get(task) or reviewer_spec(task_dir)}
|
|
182
|
+
whole_run = review is True
|
|
183
|
+
if review is None:
|
|
184
|
+
review = not status and task not in results and not (task_dir / "known_issues.md").exists()
|
|
185
|
+
if review:
|
|
186
|
+
# A review with no task behind it (whole-run review): everything it found is still open.
|
|
187
|
+
found = [{**f, **meta, "task": None if whole_run else task, "origin": "review" if f["confidence"] == "CONFIRMED" else "plausible", "threshold": "-"}
|
|
188
|
+
for f in parse_findings(read(task_dir / "qa.last"))
|
|
189
|
+
if f["confidence"] == "CONFIRMED" or include_plausible]
|
|
190
|
+
return found, None
|
|
191
|
+
if state not in DONE and status not in DONE:
|
|
192
|
+
return [], f"{task}: status {state or 'unknown'}, its findings still block it"
|
|
193
|
+
threshold = read(task_dir / "qa_threshold") or (read(plan_dir / "qa_threshold") if plan_dir else "") or "?"
|
|
194
|
+
meta["threshold"] = threshold
|
|
195
|
+
found = [{**f, **meta, "origin": "tolerated"} for f in parse_findings(read(task_dir / "known_issues.md"))
|
|
196
|
+
if f["confidence"] == "CONFIRMED"]
|
|
197
|
+
if include_plausible:
|
|
198
|
+
found += [{**f, **meta, "origin": "plausible"} for f in parse_findings(read(task_dir / "qa.last"))
|
|
199
|
+
if f["confidence"] == "PLAUSIBLE"]
|
|
200
|
+
return found, None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def mtime(path: Path):
|
|
204
|
+
try:
|
|
205
|
+
return path.stat().st_mtime
|
|
206
|
+
except OSError:
|
|
207
|
+
return 0.0
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def whole_run_review(plan_dir: Path):
|
|
211
|
+
"""The plan's canonical latest whole-run review dir, or None.
|
|
212
|
+
|
|
213
|
+
`review` publishes `<plan>/final-review` (a link to the newest `final-review-<epoch>-<pid>/`), and
|
|
214
|
+
`integrate --final-review` writes `integrate-*/final-review/` without touching it unless it publishes.
|
|
215
|
+
Any other final-review-* dir is superseded.
|
|
216
|
+
"""
|
|
217
|
+
def usable(d):
|
|
218
|
+
return d if (d / "qa.last").is_file() else None
|
|
219
|
+
link = (plan_dir / "final-review").resolve() if (plan_dir / "final-review").is_dir() else None
|
|
220
|
+
published = usable(link) if link else None
|
|
221
|
+
# A published review without qa.last (infra stop, dispatch start failure) is still the latest attempt:
|
|
222
|
+
# an integrate review must beat its own mtime, and nothing older may stand in for it.
|
|
223
|
+
stamp = (mtime(published / "qa.last") if published else
|
|
224
|
+
max(mtime(link / "qa.input"), mtime(link)) if link else None)
|
|
225
|
+
newest = None
|
|
226
|
+
for d in plan_dir.glob("integrate-*/final-review"):
|
|
227
|
+
d = d.resolve()
|
|
228
|
+
if usable(d) and (newest is None or mtime(d / "qa.last") > mtime(newest / "qa.last")):
|
|
229
|
+
newest = d
|
|
230
|
+
if newest and (stamp is None or (newest != link and mtime(newest / "qa.last") > stamp)):
|
|
231
|
+
return newest
|
|
232
|
+
return published
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def collect_plan(plan_dir: Path, include_plausible):
|
|
236
|
+
tasks = sorted(d for d in (plan_dir / "tasks").iterdir() if d.is_dir() and not d.name.startswith("_"))
|
|
237
|
+
reviewers = {}
|
|
238
|
+
if (plan_dir / "tasks.tsv").is_file():
|
|
239
|
+
for line in read(plan_dir / "tasks.tsv").splitlines():
|
|
240
|
+
cols = line.split("\t")
|
|
241
|
+
if len(cols) >= 6 and not line.startswith("#"):
|
|
242
|
+
reviewers[cols[0]] = cols[5]
|
|
243
|
+
found, notes = [], []
|
|
244
|
+
for task_dir in tasks:
|
|
245
|
+
items, note = task_findings(task_dir, plan_dir, include_plausible, reviewers)
|
|
246
|
+
found += items
|
|
247
|
+
if note:
|
|
248
|
+
notes.append(note)
|
|
249
|
+
review = whole_run_review(plan_dir)
|
|
250
|
+
if review:
|
|
251
|
+
items, _ = task_findings(review, plan_dir, include_plausible, {}, review=True)
|
|
252
|
+
found += [{**f, "task": None} for f in items]
|
|
253
|
+
for f in found:
|
|
254
|
+
f["plan"], f["plan_dir"] = plan_dir.name, str(plan_dir)
|
|
255
|
+
return found, notes
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def collect_all(path: Path, include_plausible):
|
|
259
|
+
plan_file = path / "repo"
|
|
260
|
+
repo = Path(read(plan_file)) if plan_file.is_file() else path
|
|
261
|
+
registry = runs_registry(repo)
|
|
262
|
+
lines = read(registry).splitlines() if registry.is_file() else []
|
|
263
|
+
plans, notes = [], []
|
|
264
|
+
for line in lines:
|
|
265
|
+
line = line.strip()
|
|
266
|
+
if not line or line.startswith("#"):
|
|
267
|
+
continue
|
|
268
|
+
d = Path(line)
|
|
269
|
+
if not d.is_dir():
|
|
270
|
+
notes.append(f"{line}: run directory is missing, skipped")
|
|
271
|
+
elif not (d / "tasks").is_dir():
|
|
272
|
+
notes.append(f"{line}: no tasks/ directory, skipped")
|
|
273
|
+
elif d not in plans:
|
|
274
|
+
plans.append(d)
|
|
275
|
+
plans.sort(key=lambda d: mtime(d), reverse=True) # newest first: the first occurrence of a fingerprint wins
|
|
276
|
+
found = []
|
|
277
|
+
for d in plans:
|
|
278
|
+
items, plan_notes = collect_plan(d, include_plausible)
|
|
279
|
+
found += items
|
|
280
|
+
notes += [f"{d.name}: {n}" for n in plan_notes]
|
|
281
|
+
return {"plan": f"{repo.name} (all runs)", "repo_path": str(repo)}, found, notes
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def recorded_repo(run_dir: Path):
|
|
285
|
+
"""The repository a plan dir or solo run dir records in its `repo` file; None when it records none."""
|
|
286
|
+
return read(run_dir / "repo") or None
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def collect(path: Path, include_plausible, all_runs=False):
|
|
290
|
+
"""(meta, findings, notes); meta["repo_path"] is the run's recorded repository, None when it records none."""
|
|
291
|
+
path = path.resolve()
|
|
292
|
+
if all_runs:
|
|
293
|
+
if not path.is_dir():
|
|
294
|
+
raise Fail(f"{path}: --all-runs needs a repository or plan directory")
|
|
295
|
+
return collect_all(path, include_plausible)
|
|
296
|
+
if path.is_file():
|
|
297
|
+
if path.name not in ("known_issues.md", "qa.last"):
|
|
298
|
+
raise Fail(f"{path}: expected a forge plan or task directory, known_issues.md or qa.last")
|
|
299
|
+
parent = path.parent
|
|
300
|
+
is_task = parent.parent.name == "tasks"
|
|
301
|
+
if is_task:
|
|
302
|
+
plan_root = parent.parent.parent
|
|
303
|
+
elif (parent / "tasks").is_dir():
|
|
304
|
+
plan_root = parent
|
|
305
|
+
elif parent.parent.name.startswith("integrate-") and (parent.parent.parent / "tasks").is_dir():
|
|
306
|
+
plan_root = parent.parent.parent
|
|
307
|
+
elif (parent.parent / "tasks").is_dir():
|
|
308
|
+
plan_root = parent.parent
|
|
309
|
+
else:
|
|
310
|
+
plan_root = parent
|
|
311
|
+
is_review = not is_task and not read(parent / "status")
|
|
312
|
+
meta = {"task": None if is_review else parent.name, "threshold": "-" if is_review else read(parent / "qa_threshold") or "?",
|
|
313
|
+
"commit": read(parent / "merged") or read(parent / "reviewed.commit"),
|
|
314
|
+
"reviewer": reviewer_spec(parent)}
|
|
315
|
+
meta["plan"], meta["plan_dir"] = (plan_root.name, str(plan_root))
|
|
316
|
+
if is_review:
|
|
317
|
+
found = [{**f, **meta, "origin": "review" if f["confidence"] == "CONFIRMED" else "plausible"}
|
|
318
|
+
for f in parse_findings(read(path)) if f["confidence"] == "CONFIRMED" or include_plausible]
|
|
319
|
+
else:
|
|
320
|
+
found = [{**f, **meta, "origin": "tolerated" if f["confidence"] == "CONFIRMED" else "plausible"}
|
|
321
|
+
for f in parse_findings(read(path)) if f["confidence"] == "CONFIRMED" or include_plausible]
|
|
322
|
+
# plan_root is the plan dir, or for a file outside any plan (a solo run's) the dir holding it
|
|
323
|
+
return {"plan": plan_root.name, "repo_path": recorded_repo(plan_root)}, found, []
|
|
324
|
+
if (path / "tasks").is_dir():
|
|
325
|
+
found, notes = collect_plan(path, include_plausible)
|
|
326
|
+
return {"plan": path.name, "repo_path": recorded_repo(path)}, found, notes
|
|
327
|
+
if not ((path / "qa.last").is_file() or (path / "known_issues.md").is_file()):
|
|
328
|
+
raise Fail(f"{path} is not a forge run: no tasks/, qa.last or known_issues.md")
|
|
329
|
+
if path.parent.name == "tasks":
|
|
330
|
+
plan_dir = path.parent.parent
|
|
331
|
+
elif (path.parent / "tasks").is_dir():
|
|
332
|
+
plan_dir = path.parent
|
|
333
|
+
elif path.parent.name.startswith("integrate-") and (path.parent.parent / "tasks").is_dir():
|
|
334
|
+
plan_dir = path.parent.parent
|
|
335
|
+
else:
|
|
336
|
+
plan_dir = None
|
|
337
|
+
reviewers = {}
|
|
338
|
+
if plan_dir and (plan_dir / "tasks.tsv").is_file():
|
|
339
|
+
for line in read(plan_dir / "tasks.tsv").splitlines():
|
|
340
|
+
cols = line.split("\t")
|
|
341
|
+
if len(cols) >= 6 and not line.startswith("#"):
|
|
342
|
+
reviewers[cols[0]] = cols[5]
|
|
343
|
+
found, note = task_findings(path, plan_dir, include_plausible, reviewers,
|
|
344
|
+
review=True if path.parent.name != "tasks" and not read(path / "status") else None)
|
|
345
|
+
for f in found:
|
|
346
|
+
f["plan"], f["plan_dir"] = (plan_dir or path).name, str(plan_dir or path)
|
|
347
|
+
# a task or review dir in a plan: the plan's repository; a solo run dir: its own
|
|
348
|
+
return {"plan": (plan_dir or path).name, "repo_path": recorded_repo(plan_dir or path)}, \
|
|
349
|
+
found, [note] if note else []
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def run_repository(meta):
|
|
353
|
+
"""(repo_path, warnings): the run's recorded repository, else the current directory with a warning saying so.
|
|
354
|
+
|
|
355
|
+
Listing, filing and (un)dismissing all go through here, so they agree on the dismissal store.
|
|
356
|
+
"""
|
|
357
|
+
if meta["repo_path"]:
|
|
358
|
+
return meta["repo_path"], []
|
|
359
|
+
cwd = os.getcwd()
|
|
360
|
+
return cwd, [f"run {meta['plan']} records no repository; using the current directory {cwd}"]
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
# ---------------------------------------------------------------------------------------------- draft
|
|
364
|
+
|
|
365
|
+
def fingerprint(slug, f):
|
|
366
|
+
"""Identical to gh-proman's finding_fp: keep the two in step."""
|
|
367
|
+
norm = re.sub(r"\s+", " ", f["text"].lower())[:160]
|
|
368
|
+
return hashlib.sha1(f"{slug.lower()}|{f['file']}|{norm}".encode("utf-8")).hexdigest()[:12]
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def github_slug(repo_path):
|
|
372
|
+
res = run(["git", "-C", repo_path, "remote", "get-url", "origin"], check=False)
|
|
373
|
+
m = re.search(r"github\.com[:/]([^/\s]+)/([^/\s]+?)(?:\.git)?/?$", res.stdout.strip())
|
|
374
|
+
return f"{m.group(1)}/{m.group(2)}" if m else None
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def title_for(f, lang):
|
|
378
|
+
first = re.split(r"(?<=[.!?])\s", f["text"], maxsplit=1)[0].rstrip(". ")
|
|
379
|
+
if len(first) > 90:
|
|
380
|
+
first = first[:87].rsplit(" ", 1)[0] + "…"
|
|
381
|
+
prefix = TEXT[lang]["plausible"] if f["origin"] == "plausible" else ""
|
|
382
|
+
return f"[{f['severity']}] {prefix}{os.path.basename(f['file'])}: {first}"
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def escape(text):
|
|
386
|
+
return text.replace("<!--", "<!--").replace("-->", "-->")
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def published(slug, repo_path, commit, offline=False):
|
|
390
|
+
"""True when the commit is on GitHub; with `offline`, only remote-tracking refs can prove it."""
|
|
391
|
+
if repo_path and run(["git", "-C", repo_path, "branch", "-r", "--contains", commit], check=False).stdout.strip():
|
|
392
|
+
return True
|
|
393
|
+
if offline:
|
|
394
|
+
return False
|
|
395
|
+
return run(["gh", "api", f"repos/{slug}/commits/{commit}", "--jq", ".sha"], check=False).returncode == 0
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def body_for(slug, repo_path, items, lang, offline=False):
|
|
399
|
+
t = TEXT[lang]
|
|
400
|
+
h = t["headings"]
|
|
401
|
+
first = items[0]
|
|
402
|
+
plan = first["plan"]
|
|
403
|
+
plausible = all(f["origin"] == "plausible" for f in items)
|
|
404
|
+
if len(items) > 1:
|
|
405
|
+
context = t["group"].format(plan=plan, n=len(items))
|
|
406
|
+
elif plausible:
|
|
407
|
+
context = t["plausible_ctx"]
|
|
408
|
+
elif first["origin"] == "review":
|
|
409
|
+
context = t["review"]
|
|
410
|
+
else:
|
|
411
|
+
context = t["tolerated"].format(threshold=first["threshold"])
|
|
412
|
+
commit = f", commit `{first['commit'][:12]}`" if first.get("commit") else ""
|
|
413
|
+
lines = [f"## {h[0]}", context, "",
|
|
414
|
+
t["origin"].format(plan=plan, task=first["task"] or "final-review", severity=first["severity"],
|
|
415
|
+
reviewer=first["reviewer"], commit=commit), "", f"## {h[1]}"]
|
|
416
|
+
for f in items:
|
|
417
|
+
lines += [f"> {escape(f['raw'])}", ""]
|
|
418
|
+
lines.append(f"## {h[2]}")
|
|
419
|
+
if len(items) == 1:
|
|
420
|
+
lines += [f"- [ ] {x}" for x in (t["tasks_plausible"] if plausible else t["tasks"])]
|
|
421
|
+
else:
|
|
422
|
+
lines += [f"- [ ] [{f['severity']}] `{f['file']}{':' + str(f['line']) if f['line'] else ''}`: "
|
|
423
|
+
f"{escape(f['text'])[:160]}" for f in items]
|
|
424
|
+
lines += ["", f"## {h[3]}"] + [f"- [ ] {x}" for x in (t["closing_plausible"] if plausible else t["closing"])]
|
|
425
|
+
lines += ["", f"## {h[4]}"]
|
|
426
|
+
for f in items:
|
|
427
|
+
loc = f"{f['file']}{':' + str(f['line']) if f['line'] else ''}"
|
|
428
|
+
if f.get("commit") and published(slug, repo_path, f["commit"], offline):
|
|
429
|
+
anchor = f"#L{f['line']}" if f["line"] else ""
|
|
430
|
+
lines.append(f"- {t['code']}: https://github.com/{slug}/blob/{f['commit']}/{f['file']}{anchor}")
|
|
431
|
+
else:
|
|
432
|
+
lines.append(f"- {t['code']}: `{loc}` {t['unpublished'] if f.get('commit') else ''}".rstrip())
|
|
433
|
+
lines.append("")
|
|
434
|
+
lines += [f"<!-- forge:finding fp={f['fp']} source=forge:{f['plan']}/{f['task'] or 'final-review'} -->" for f in items]
|
|
435
|
+
return "\n".join(lines)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def existing_issues(slug):
|
|
439
|
+
res = run(["gh", "issue", "list", "--repo", slug, "--state", "all", "--limit", "1000",
|
|
440
|
+
"--json", "number,url,state,title,body"])
|
|
441
|
+
return json.loads(res.stdout or "[]")
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def norm_title(title):
|
|
445
|
+
return re.sub(r"\W+", " ", re.sub(r"^\[P[0-3]\]\s*", "", title).lower()).strip()
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# ---------------------------------------------------------------------------------------------- state
|
|
449
|
+
|
|
450
|
+
def findings_dir() -> Path:
|
|
451
|
+
"""`${XDG_STATE_HOME:-~/.local/state}/forge/findings`; an empty or relative XDG_STATE_HOME counts as unset.
|
|
452
|
+
|
|
453
|
+
Forge keeps its own data out of the user's tree, and a relative state home would name a directory below
|
|
454
|
+
the current one, usually the repository.
|
|
455
|
+
"""
|
|
456
|
+
for candidate in (os.environ.get("XDG_STATE_HOME") or "", "~/.local/state"):
|
|
457
|
+
try:
|
|
458
|
+
base = Path(candidate).expanduser() if candidate else None
|
|
459
|
+
except RuntimeError: # no home directory to expand "~" against
|
|
460
|
+
base = None
|
|
461
|
+
if base is not None and base.is_absolute():
|
|
462
|
+
return base / "forge" / "findings"
|
|
463
|
+
raise Fail("cannot place the dismissal store: no home directory and no absolute XDG_STATE_HOME")
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def store_dir(create):
|
|
467
|
+
"""The findings directory after an lstat check that it is a real directory, not a symlink; None when absent."""
|
|
468
|
+
path = findings_dir()
|
|
469
|
+
if create:
|
|
470
|
+
try:
|
|
471
|
+
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
472
|
+
except OSError as exc:
|
|
473
|
+
raise Fail(f"{path}: cannot create the dismissal store directory: {exc.strerror or exc}")
|
|
474
|
+
try:
|
|
475
|
+
mode = os.lstat(path).st_mode
|
|
476
|
+
except FileNotFoundError:
|
|
477
|
+
if create:
|
|
478
|
+
raise Fail(f"{path}: the dismissal store directory vanished")
|
|
479
|
+
return None
|
|
480
|
+
except OSError as exc:
|
|
481
|
+
raise Fail(f"{path}: cannot use the dismissal store directory: {exc.strerror}")
|
|
482
|
+
if not stat.S_ISDIR(mode):
|
|
483
|
+
raise Fail(f"{path}: the dismissal store directory must be a real directory, not a symlink")
|
|
484
|
+
return path
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def load_dismissed(key):
|
|
488
|
+
"""The repository's dismissals, `{fp: {"at": epoch, "reason": str}}`; empty when it has none yet."""
|
|
489
|
+
base = store_dir(create=False)
|
|
490
|
+
if base is None:
|
|
491
|
+
return {}
|
|
492
|
+
path = base / f"{key}.json"
|
|
493
|
+
not_regular = f"{path}: the dismissal store must be a regular file, not a symlink or special file"
|
|
494
|
+
try:
|
|
495
|
+
if not stat.S_ISREG(os.lstat(path).st_mode):
|
|
496
|
+
raise Fail(not_regular)
|
|
497
|
+
# NOFOLLOW again for a swap after the lstat; NONBLOCK so a FIFO cannot hold the open
|
|
498
|
+
with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as fh:
|
|
499
|
+
if not stat.S_ISREG(os.fstat(fh.fileno()).st_mode):
|
|
500
|
+
raise Fail(not_regular)
|
|
501
|
+
raw = fh.read(STORE_MAX + 1)
|
|
502
|
+
except FileNotFoundError:
|
|
503
|
+
return {}
|
|
504
|
+
except OSError as exc:
|
|
505
|
+
raise Fail(f"{path}: cannot read the dismissal store: {exc.strerror}")
|
|
506
|
+
try:
|
|
507
|
+
data = json.loads(raw.decode("utf-8")) if len(raw) <= STORE_MAX else None
|
|
508
|
+
except ValueError:
|
|
509
|
+
data = None
|
|
510
|
+
if not (isinstance(data, dict) and all(
|
|
511
|
+
FP_RE.fullmatch(fp) and isinstance(v, dict) and isinstance(v.get("reason"), str)
|
|
512
|
+
and isinstance(v.get("at"), (int, float)) and not isinstance(v["at"], bool) for fp, v in data.items())):
|
|
513
|
+
raise Fail(f"{path}: not a dismissal store ({{fingerprint: {{at, reason}}}}); fix or remove it")
|
|
514
|
+
return {fp: {"at": v["at"], "reason": v["reason"]} for fp, v in data.items()}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def save_dismissed(key, data):
|
|
518
|
+
"""Replace the store atomically: a temp file in the same directory, fsynced, then renamed over it."""
|
|
519
|
+
base = store_dir(create=True)
|
|
520
|
+
path = base / f"{key}.json"
|
|
521
|
+
try:
|
|
522
|
+
fd, tmp = tempfile.mkstemp(prefix=f".{key}.", suffix=".tmp", dir=base)
|
|
523
|
+
try:
|
|
524
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
525
|
+
json.dump(data, fh, indent=2, sort_keys=True, ensure_ascii=False)
|
|
526
|
+
fh.write("\n")
|
|
527
|
+
fh.flush()
|
|
528
|
+
os.fsync(fh.fileno())
|
|
529
|
+
os.replace(tmp, path)
|
|
530
|
+
except BaseException:
|
|
531
|
+
with contextlib.suppress(OSError):
|
|
532
|
+
os.unlink(tmp)
|
|
533
|
+
raise
|
|
534
|
+
except OSError as exc:
|
|
535
|
+
raise Fail(f"{path}: cannot write the dismissal store: {exc.strerror}")
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
@contextlib.contextmanager
|
|
539
|
+
def repo_lock(key, label):
|
|
540
|
+
"""Hold the repository's `<key>.lock` (flock, next to the store); busy for LOCK_WAIT seconds raises Busy.
|
|
541
|
+
|
|
542
|
+
The lock is its own file, opened in place and never replaced, because the store is: a lock on the store
|
|
543
|
+
file would stay with the old inode after an atomic save.
|
|
544
|
+
"""
|
|
545
|
+
path = store_dir(create=True) / f"{key}.lock"
|
|
546
|
+
try:
|
|
547
|
+
fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600)
|
|
548
|
+
except OSError as exc:
|
|
549
|
+
raise Fail(f"{path}: cannot open the repository lock ({exc.strerror}); it must not be a symlink")
|
|
550
|
+
try:
|
|
551
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
552
|
+
raise Fail(f"{path}: the repository lock must be a regular file")
|
|
553
|
+
deadline = time.monotonic() + LOCK_WAIT
|
|
554
|
+
while True:
|
|
555
|
+
try:
|
|
556
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
557
|
+
break
|
|
558
|
+
except OSError as exc:
|
|
559
|
+
if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
|
|
560
|
+
raise Fail(f"{path}: cannot lock the repository: {exc.strerror}")
|
|
561
|
+
if time.monotonic() >= deadline:
|
|
562
|
+
raise Busy(f"busy: another filing or dismissal for {label} is in progress")
|
|
563
|
+
time.sleep(LOCK_POLL)
|
|
564
|
+
yield
|
|
565
|
+
finally:
|
|
566
|
+
os.close(fd) # closing the descriptor releases the flock
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def dismiss(args):
|
|
570
|
+
"""--dismiss / --undismiss: update the repository's store under its lock. No network: <run> is only read to
|
|
571
|
+
find the repository it resolves to."""
|
|
572
|
+
meta, _found, _notes = collect(Path(args.run), False, args.all_runs)
|
|
573
|
+
repo_path, warnings = run_repository(meta)
|
|
574
|
+
key = repo_key(repo_path)
|
|
575
|
+
with repo_lock(key, args.repo or github_slug(repo_path) or repo_path):
|
|
576
|
+
data = load_dismissed(key)
|
|
577
|
+
before = json.dumps(data, sort_keys=True)
|
|
578
|
+
now = int(time.time())
|
|
579
|
+
for fp in args.dismiss:
|
|
580
|
+
# dismissing again without a --reason keeps the earlier mark, with its reason and date
|
|
581
|
+
if fp not in data or args.reason is not None:
|
|
582
|
+
data[fp] = {"at": now, "reason": args.reason or ""}
|
|
583
|
+
for fp in args.undismiss:
|
|
584
|
+
data.pop(fp, None)
|
|
585
|
+
if json.dumps(data, sort_keys=True) != before:
|
|
586
|
+
save_dismissed(key, data)
|
|
587
|
+
return {"ok": True, "dismissed": args.dismiss, "undismissed": args.undismiss, "warnings": warnings}
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
# ---------------------------------------------------------------------------------------------- main
|
|
591
|
+
|
|
592
|
+
def split_fps(values):
|
|
593
|
+
"""Repeatable, comma-separated fingerprints, deduplicated in order; None when one is not 12 hex characters."""
|
|
594
|
+
fps = [x.strip() for v in values for x in v.split(",")]
|
|
595
|
+
return None if any(not FP_RE.fullmatch(x) for x in fps) else list(dict.fromkeys(fps))
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def main(argv=None):
|
|
599
|
+
ap = argparse.ArgumentParser(prog="forge issues", description=__doc__.split("\n\n")[0],
|
|
600
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
601
|
+
epilog="\n\n".join(__doc__.split("\n\n")[1:]))
|
|
602
|
+
ap.add_argument("run", help="plan dir, task dir, review dir, known_issues.md or qa.last "
|
|
603
|
+
"(with --all-runs: a repository, or a plan dir naming one)")
|
|
604
|
+
ap.add_argument("--include-plausible", action="store_true", help="also file PLAUSIBLE findings, as 'To verify'")
|
|
605
|
+
ap.add_argument("--min-severity", choices=SEVERITIES, help="file only findings at least this severe")
|
|
606
|
+
ap.add_argument("--group", choices=["finding", "task", "run"], default="finding",
|
|
607
|
+
help="one issue per finding (default), per task, or one checklist issue per run")
|
|
608
|
+
ap.add_argument("--repo", help="owner/repo (default: the run's repository, from its origin remote)")
|
|
609
|
+
ap.add_argument("--lang", choices=sorted(TEXT), default="en", help="language of the issue text (default en)")
|
|
610
|
+
ap.add_argument("--label", action="append", default=[], help="label to add if the repo has it; repeatable")
|
|
611
|
+
ap.add_argument("--apply", action="store_true", help="create the issues (default: show drafts only)")
|
|
612
|
+
ap.add_argument("--confirm-new", action="store_true", help="create even when an issue with the same title exists")
|
|
613
|
+
ap.add_argument("--max-issues", type=int, default=10,
|
|
614
|
+
help="create at most this many per call (default 10); filing a --only selection: pass its size")
|
|
615
|
+
ap.add_argument("--offline", action="store_true", help="never run `gh`: no filed state, local refs only")
|
|
616
|
+
ap.add_argument("--only", action="append", default=[], metavar="FP",
|
|
617
|
+
help="keep only this 12-hex fingerprint; repeatable or comma-separated")
|
|
618
|
+
ap.add_argument("--all-runs", action="store_true", help="collect every run in the repository's run registry")
|
|
619
|
+
ap.add_argument("--dismiss", action="append", default=[], metavar="FP",
|
|
620
|
+
help="mark this 12-hex fingerprint as needing no work, then exit; repeatable or comma-separated")
|
|
621
|
+
ap.add_argument("--undismiss", action="append", default=[], metavar="FP",
|
|
622
|
+
help="take the mark back, then exit; repeatable or comma-separated")
|
|
623
|
+
ap.add_argument("--reason", help=f"why --dismiss marks them (at most {REASON_MAX} characters)")
|
|
624
|
+
ap.add_argument("--include-dismissed", action="store_true", help="draft dismissed findings too")
|
|
625
|
+
ap.add_argument("--json", action="store_true", help="machine-readable output")
|
|
626
|
+
args = ap.parse_args(argv)
|
|
627
|
+
if args.offline and args.apply:
|
|
628
|
+
ap.error("--offline cannot be combined with --apply")
|
|
629
|
+
only = split_fps(args.only)
|
|
630
|
+
if only is None:
|
|
631
|
+
ap.error("--only takes 12-hex-character fingerprints, repeated or comma-separated")
|
|
632
|
+
args.only = only
|
|
633
|
+
args.dismiss, args.undismiss = split_fps(args.dismiss), split_fps(args.undismiss)
|
|
634
|
+
if args.dismiss is None or args.undismiss is None:
|
|
635
|
+
ap.error("--dismiss and --undismiss take 12-hex-character fingerprints, repeated or comma-separated")
|
|
636
|
+
dismissing = bool(args.dismiss or args.undismiss)
|
|
637
|
+
if dismissing and args.apply:
|
|
638
|
+
ap.error("--dismiss and --undismiss cannot be combined with --apply")
|
|
639
|
+
if set(args.dismiss) & set(args.undismiss):
|
|
640
|
+
ap.error("a fingerprint cannot be both --dismiss and --undismiss")
|
|
641
|
+
if args.reason is not None:
|
|
642
|
+
if not args.dismiss:
|
|
643
|
+
ap.error("--reason goes with --dismiss")
|
|
644
|
+
try:
|
|
645
|
+
args.reason.encode("utf-8")
|
|
646
|
+
except UnicodeEncodeError:
|
|
647
|
+
ap.error("--reason must be valid UTF-8")
|
|
648
|
+
if len(args.reason) > REASON_MAX:
|
|
649
|
+
ap.error(f"--reason is at most {REASON_MAX} characters")
|
|
650
|
+
try:
|
|
651
|
+
out = dismiss(args) if dismissing else file_findings(args)
|
|
652
|
+
except Fail as exc:
|
|
653
|
+
print(f"forge issues: {exc}", file=sys.stderr)
|
|
654
|
+
return 3 if isinstance(exc, Busy) else 2
|
|
655
|
+
if args.json:
|
|
656
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
657
|
+
elif dismissing:
|
|
658
|
+
for warning in out["warnings"]:
|
|
659
|
+
print(f"warning: {warning}")
|
|
660
|
+
for verb in ("dismissed", "undismissed"):
|
|
661
|
+
if out[verb]:
|
|
662
|
+
print(f"{verb} {len(out[verb])} finding(s): {', '.join(out[verb])}")
|
|
663
|
+
else:
|
|
664
|
+
print_human(out, args.apply)
|
|
665
|
+
return 0
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def finding_entry(f, filed=None, dismissed=None):
|
|
669
|
+
return {"fp": f["fp"], "severity": f["severity"], "confidence": f["confidence"], "origin": f["origin"],
|
|
670
|
+
"plan": f["plan"], "plan_dir": f["plan_dir"], "task": f["task"], "file": f["file"],
|
|
671
|
+
"summary": (f["text"].splitlines() or [""])[0][:160], "text": f["text"],
|
|
672
|
+
"reviewer": f["reviewer"], "commit": f.get("commit") or "",
|
|
673
|
+
"filed": {"url": filed["url"], "state": filed["state"]} if filed else None,
|
|
674
|
+
"dismissed": dismissed}
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def file_findings(args):
|
|
678
|
+
meta, found, notes = collect(Path(args.run), args.include_plausible, args.all_runs)
|
|
679
|
+
repo_path, warnings = run_repository(meta)
|
|
680
|
+
slug = args.repo or github_slug(repo_path)
|
|
681
|
+
if not slug:
|
|
682
|
+
raise Fail("; ".join(warnings + [f"{repo_path} has no GitHub origin remote; pass --repo owner/repo"]))
|
|
683
|
+
if args.min_severity:
|
|
684
|
+
cut = SEVERITIES.index(args.min_severity)
|
|
685
|
+
found = [f for f in found if SEVERITIES.index(f["severity"]) <= cut]
|
|
686
|
+
# Filing is list-then-create, so no other filing or dismissal for this repository may run in between: the
|
|
687
|
+
# lock covers the lookup, the drafts (which ask `gh` whether commits are published) and every create.
|
|
688
|
+
with repo_lock(repo_key(repo_path), slug) if args.apply else contextlib.nullcontext():
|
|
689
|
+
return list_and_file(args, meta, found, notes, repo_path, slug, warnings)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def list_and_file(args, meta, found, notes, repo_path, slug, warnings):
|
|
693
|
+
offline = args.offline
|
|
694
|
+
dismissed = load_dismissed(repo_key(repo_path))
|
|
695
|
+
issues = [] if offline else existing_issues(slug)
|
|
696
|
+
filed = {fp: issue for issue in issues for fp in FP_MARK_RE.findall(issue.get("body") or "")}
|
|
697
|
+
only = set(args.only)
|
|
698
|
+
seen, todo, already, entries = set(), [], [], []
|
|
699
|
+
known = set()
|
|
700
|
+
for f in found:
|
|
701
|
+
f["fp"] = fingerprint(slug, f)
|
|
702
|
+
known.add(f["fp"])
|
|
703
|
+
if f["fp"] in seen or (only and f["fp"] not in only):
|
|
704
|
+
continue
|
|
705
|
+
seen.add(f["fp"])
|
|
706
|
+
issue = filed.get(f["fp"])
|
|
707
|
+
mark = dismissed.get(f["fp"])
|
|
708
|
+
entries.append(finding_entry(f, issue, mark))
|
|
709
|
+
if issue:
|
|
710
|
+
already.append({"finding": f["raw"], "issue": issue["url"], "state": issue["state"]})
|
|
711
|
+
elif not mark or args.include_dismissed:
|
|
712
|
+
todo.append(f)
|
|
713
|
+
notes = notes + [f"--only {fp}: no such open finding" for fp in args.only if fp not in known]
|
|
714
|
+
entries.sort(key=lambda e: SEVERITIES.index(e["severity"]))
|
|
715
|
+
keyed = {}
|
|
716
|
+
for f in todo:
|
|
717
|
+
key = id(f) if args.group == "finding" else (f["plan"], f["origin"], f["task"] if args.group == "task" else "")
|
|
718
|
+
keyed.setdefault(key, []).append(f)
|
|
719
|
+
open_titles = {norm_title(i["title"]): i["url"] for i in issues if i.get("state") == "OPEN"}
|
|
720
|
+
t = TEXT[args.lang]
|
|
721
|
+
drafts = []
|
|
722
|
+
for items in keyed.values():
|
|
723
|
+
items.sort(key=lambda f: SEVERITIES.index(f["severity"]))
|
|
724
|
+
if len(items) == 1:
|
|
725
|
+
title = title_for(items[0], args.lang)
|
|
726
|
+
else:
|
|
727
|
+
scope = (items[0]["task"] or items[0]["plan"]) if args.group == "task" else items[0]["plan"]
|
|
728
|
+
prefix = t["plausible"] if items[0]["origin"] == "plausible" else ""
|
|
729
|
+
title = f"[{items[0]['severity']}] {prefix}{t['group_title'].format(scope=scope)}"
|
|
730
|
+
drafts.append({"title": title, "severity": items[0]["severity"], "origin": items[0]["origin"],
|
|
731
|
+
"findings": [f["raw"] for f in items], "fingerprints": [f["fp"] for f in items],
|
|
732
|
+
"same_title": None if offline else open_titles.get(norm_title(title)),
|
|
733
|
+
"body": body_for(slug, repo_path if meta["repo_path"] else None, items, args.lang, offline)})
|
|
734
|
+
drafts.sort(key=lambda d: SEVERITIES.index(d["severity"]))
|
|
735
|
+
out = {"repo": slug, "plan": meta["plan"], "notes": notes, "warnings": warnings,
|
|
736
|
+
"summary": {"findings": len(seen), "already_filed": len(already),
|
|
737
|
+
"dismissed": sum(1 for e in entries if e["dismissed"]), "drafts": len(drafts),
|
|
738
|
+
**{o: sum(1 for f in todo if f["origin"] == o) for o in ("tolerated", "review", "plausible")}},
|
|
739
|
+
"findings": entries, "already_filed": already, "drafts": drafts}
|
|
740
|
+
if args.apply:
|
|
741
|
+
out["created"] = create(slug, drafts, args)
|
|
742
|
+
for d in drafts:
|
|
743
|
+
d.pop("body")
|
|
744
|
+
return out
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def create(slug, drafts, args):
|
|
748
|
+
labels = []
|
|
749
|
+
if args.label:
|
|
750
|
+
res = run(["gh", "label", "list", "--repo", slug, "--limit", "500", "--json", "name"], check=False)
|
|
751
|
+
have = {x["name"] for x in json.loads(res.stdout or "[]")} if res.returncode == 0 else set()
|
|
752
|
+
labels = [x for x in args.label if x in have]
|
|
753
|
+
created = []
|
|
754
|
+
for i, d in enumerate(drafts):
|
|
755
|
+
if i >= args.max_issues:
|
|
756
|
+
created.append({"title": d["title"], "skipped": f"over --max-issues {args.max_issues}; run again"})
|
|
757
|
+
continue
|
|
758
|
+
if d["same_title"] and not args.confirm_new:
|
|
759
|
+
created.append({"title": d["title"], "skipped": "an open issue has the same title", "issue": d["same_title"]})
|
|
760
|
+
continue
|
|
761
|
+
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") as fh:
|
|
762
|
+
fh.write(d["body"])
|
|
763
|
+
try:
|
|
764
|
+
cmd = ["gh", "issue", "create", "--repo", slug, "--title", d["title"], "--body-file", fh.name]
|
|
765
|
+
for label in labels:
|
|
766
|
+
cmd += ["--label", label]
|
|
767
|
+
res = run(cmd)
|
|
768
|
+
finally:
|
|
769
|
+
os.unlink(fh.name)
|
|
770
|
+
url = next((w for w in res.stdout.split() if w.startswith("https://")), res.stdout.strip())
|
|
771
|
+
created.append({"title": d["title"], "url": url})
|
|
772
|
+
missing = sorted(set(args.label) - set(labels))
|
|
773
|
+
if missing:
|
|
774
|
+
created.append({"note": f"labels not in {slug}, left off: {', '.join(missing)}"})
|
|
775
|
+
return created
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def print_human(out, applied):
|
|
779
|
+
s = out["summary"]
|
|
780
|
+
dismissed = f", {s['dismissed']} dismissed" if s["dismissed"] else ""
|
|
781
|
+
print(f"{out['repo']} · run {out['plan']}: {s['findings']} finding(s), {s['already_filed']} already filed{dismissed}, "
|
|
782
|
+
f"{s['drafts']} draft(s) ({s['tolerated']} tolerated, {s['review']} review, {s['plausible']} plausible)")
|
|
783
|
+
for warning in out["warnings"]:
|
|
784
|
+
print(f" warning {warning}")
|
|
785
|
+
for note in out["notes"]:
|
|
786
|
+
print(f" skipped {note}")
|
|
787
|
+
for a in out["already_filed"]:
|
|
788
|
+
print(f" filed {a['issue']} ({a['state'].lower()}): {a['finding'][:100]}")
|
|
789
|
+
if applied:
|
|
790
|
+
for c in out["created"]:
|
|
791
|
+
if "note" in c:
|
|
792
|
+
print(f" note {c['note']}")
|
|
793
|
+
elif "url" in c:
|
|
794
|
+
print(f" created {c['url']} {c['title']}")
|
|
795
|
+
else:
|
|
796
|
+
print(f" skipped {c['title']}: {c['skipped']}{' ' + c['issue'] if c.get('issue') else ''}")
|
|
797
|
+
return
|
|
798
|
+
for d in out["drafts"]:
|
|
799
|
+
flag = f" (open issue with the same title: {d['same_title']})" if d["same_title"] else ""
|
|
800
|
+
print(f" draft {d['title']}{flag}")
|
|
801
|
+
if out["drafts"]:
|
|
802
|
+
print("Nothing created: add --apply to file these.")
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
if __name__ == "__main__":
|
|
806
|
+
sys.exit(main())
|