@jagreehal/shepherd 0.13.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +22 -5
- package/README.md +2 -0
- package/package.json +1 -1
- package/skills/ci-repair/SKILL.md +59 -13
- package/skills/garden/SKILL.md +22 -3
- package/skills/git-archaeology/SKILL.md +188 -0
- package/skills/incident/SKILL.md +10 -0
- package/skills/review-maintainability/SKILL.md +1 -1
- package/skills/shepherd/SKILL.md +42 -8
- package/skills/shepherd/references/dispatch.md +14 -6
- package/skills/swarm/SKILL.md +125 -13
- package/skills/swarm/references/formats.md +8 -4
- package/skills/triage/SKILL.md +72 -15
- package/src/cilog.ts +55 -0
- package/src/cli.ts +81 -7
- package/src/diff.ts +70 -0
- package/src/eval.ts +57 -1
- package/src/garden.ts +39 -4
- package/src/leads.ts +234 -0
- package/src/local.ts +76 -12
- package/src/stack.ts +134 -0
- package/src/summary.ts +188 -10
- package/templates/shepherd-incident.yml +2 -0
- package/templates/shepherd.yml +11 -2
package/skills/swarm/SKILL.md
CHANGED
|
@@ -62,8 +62,10 @@ router runs (`references/custom-lenses.md`).
|
|
|
62
62
|
- **Reviewers read; they never act on the world.** No agent makes a network
|
|
63
63
|
request beyond `gh` and `git` for this repository, writes to any outside
|
|
64
64
|
service, or runs PR code outside Step 2's checks. A claim about an outside
|
|
65
|
-
API's behaviour is stated as an open question, never tested live.
|
|
66
|
-
|
|
65
|
+
API's behaviour is stated as an open question, never tested live. The test
|
|
66
|
+
for any call, a tool added after this was written included: would it still
|
|
67
|
+
be a no-op if reverted? If not, a reviewer does not make it. Put this rule in
|
|
68
|
+
every agent brief.
|
|
67
69
|
|
|
68
70
|
## Preview mode
|
|
69
71
|
|
|
@@ -160,9 +162,21 @@ When `shepherd` invokes you it passes `diff_path` (already the stacked delta whe
|
|
|
160
162
|
`skip_checks` (true when it already ran Step 2 this round), and `since_sha` (the
|
|
161
163
|
head swarm last reviewed, on round 2 onwards). Use them and do not re-derive.
|
|
162
164
|
|
|
163
|
-
**Later rounds review the change, not the PR again.** With a `since_sha`,
|
|
164
|
-
`
|
|
165
|
-
|
|
165
|
+
**Later rounds review the change, not the PR again.** With a `since_sha`, run
|
|
166
|
+
`shepherd interdiff <since_sha> <base_ref> --head <head_sha>` (for a stacked
|
|
167
|
+
PR, the parent's head in place of `<base_ref>`). Add `--since-base <sha>` from
|
|
168
|
+
`shepherd reviewed summary.md` (the bot's own summary comment, the one you pass
|
|
169
|
+
as `--previous`) when its `head_sha` matches `since_sha`: the base tip that
|
|
170
|
+
round reviewed against, so a stacked parent force-pushed since keeps its old
|
|
171
|
+
change out of the focus. Its `focus` is the round's
|
|
172
|
+
focus: how the PR's own diff moved since the reviewed head, `+` and `-` marking
|
|
173
|
+
what changed between the two versions, without the base's commits a merge or
|
|
174
|
+
rebase brought in, which `git diff <since_sha>..<head_sha>` would carry (`text`
|
|
175
|
+
is the same delta as raw `git range-diff`). With `unchanged` true, the PR's change is the one already
|
|
176
|
+
reviewed: report the round dry and post nothing. When the command exits
|
|
177
|
+
non-zero, fall back to `git diff <since_sha>..<head_sha>` and say so in the
|
|
178
|
+
summary. The router reviews the focus in full (its `files` first) and the
|
|
179
|
+
rest of the PR only for HIGH or CRITICAL issues. A
|
|
166
180
|
fresh read of unchanged code finds a new edge case every round, and the loop
|
|
167
181
|
never converges.
|
|
168
182
|
|
|
@@ -280,13 +294,47 @@ ends `could_not_run`. Without the CLI, build the same list by hand from
|
|
|
280
294
|
PR closes an issue or its body, commits or branch point at an issue or spec
|
|
281
295
|
(`../review-spec/SKILL.md` says where specs live).
|
|
282
296
|
|
|
297
|
+
Before dispatching, list where the change reaches beyond the diff:
|
|
298
|
+
|
|
299
|
+
```bash
|
|
300
|
+
shepherd impact "$diff_path" > impact.json
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
Its `lines` name each changed export with the files outside the PR that use
|
|
304
|
+
it, each changed source file with its tests, or "no test file matches its
|
|
305
|
+
name", and each identifier whose count the diff changed with the unchanged
|
|
306
|
+
lines still naming it, docs included. Give them to the router and the
|
|
307
|
+
correctness lens as leads to check: a caller that breaks belongs at the call
|
|
308
|
+
site's line, and a name the diff renamed or removed that a line still uses is a
|
|
309
|
+
stale reference, runtime breakage before wrong docs before stale comments.
|
|
310
|
+
Leads say where to look; they never replace reading the diff, and an empty list
|
|
311
|
+
proves nothing.
|
|
312
|
+
|
|
313
|
+
Then write the diff reviewers read, so nobody counts a line from a hunk header:
|
|
314
|
+
|
|
315
|
+
```bash
|
|
316
|
+
shepherd number "$diff_path" > "$diff_path.numbered"
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
Each hunk line reads `| OLD | NEW | TYPE | code`, under a contents list of each
|
|
320
|
+
file's lines. A file marked `generated` needs no line-by-line read when its
|
|
321
|
+
content is what the marker says; the marker reads the path alone, so read source
|
|
322
|
+
code under `dist/` or `coverage/` as source. Pass this
|
|
323
|
+
path to the router and every lens as the diff. `shepherd coverage` and
|
|
324
|
+
`shepherd inline` keep reading `$diff_path`. When the command fails, pass
|
|
325
|
+
`$diff_path`.
|
|
326
|
+
|
|
283
327
|
Dispatch the router and every lens on that list **in one message**, in the
|
|
284
328
|
foreground, so they run in parallel and all return inside this turn. Each
|
|
285
329
|
lens runs one rung above the router unless it has a `models:` pin. Each lens
|
|
286
330
|
gets its lens skill body (a custom lens gets the lens brief from
|
|
287
331
|
`references/custom-lenses.md`), the diff path, its scope, and
|
|
288
332
|
`CHECK_FINDINGS`, and is told it is the only reviewer for that scope. It never
|
|
289
|
-
learns about the router or the other lenses, so it cannot anchor on them.
|
|
333
|
+
learns about the router or the other lenses, so it cannot anchor on them. Give
|
|
334
|
+
it the diff and its scope, never a summary of the PR, a reading list or a
|
|
335
|
+
judgement of your own: each frames it to confirm instead of question. A lens
|
|
336
|
+
whose diff is empty or unreadable says so and ends `failed`, naming the path; it
|
|
337
|
+
never guesses a `git diff` of its own.
|
|
290
338
|
First-wave lenses do not count toward the cap. A personal lens's findings
|
|
291
339
|
(from `~/.config/shepherd/lenses.yml`) never post: they skip Steps 5 to 7 and
|
|
292
340
|
print under "Personal lenses" in the Step 8 report.
|
|
@@ -309,6 +357,12 @@ a rung for them. Its brief:
|
|
|
309
357
|
ordinary input and the boundary the promise names through the code before
|
|
310
358
|
anything else. A broken promise outranks naming, style and speculative
|
|
311
359
|
enhancements.
|
|
360
|
+
- Report what no line shows, as a finding with line `general`: something the
|
|
361
|
+
change should have done and did not (a caller left on the old contract, a
|
|
362
|
+
migration with no rollback), an order that must hold (deleting a column
|
|
363
|
+
before the code stops reading it), or a scope question the diff raises. A
|
|
364
|
+
concern that does anchor to a line goes on that line, however broad its
|
|
365
|
+
effect.
|
|
312
366
|
- Review the whole diff for security, simplicity, maintainability, slop, and
|
|
313
367
|
any bug you see. The correctness lens and the custom lenses named in your
|
|
314
368
|
brief review this diff beside you; do not plan them for the scopes they
|
|
@@ -322,6 +376,12 @@ a rung for them. Its brief:
|
|
|
322
376
|
what your own pass cannot cover safely. Delegate one rung above yours; name
|
|
323
377
|
a higher rung only when the reason says why the lower one cannot cover it.
|
|
324
378
|
An empty plan on a small, low-danger diff is the cheap path working.
|
|
379
|
+
- Each delegation asks one falsifiable `question` that could change your grade,
|
|
380
|
+
framed by its domain: "billing: can a retried webhook charge twice?", not
|
|
381
|
+
"check correctness" (a named domain recalls failure modes a generic pass
|
|
382
|
+
misses). "Another look" is no question. Orthogonal questions get separate
|
|
383
|
+
delegations; never fold them into one broad ask. Diff size is no proxy for
|
|
384
|
+
how many you need.
|
|
325
385
|
- **One delegation is mandatory** when you grade danger HIGH or CRITICAL, and,
|
|
326
386
|
whatever your grade, when the diff adds or changes code in auth,
|
|
327
387
|
permissions, secrets, billing, migrations, concurrency, CI/deploy workflows,
|
|
@@ -351,7 +411,9 @@ a rung for them. Its brief:
|
|
|
351
411
|
|
|
352
412
|
## Step 4: Second wave
|
|
353
413
|
|
|
354
|
-
When the router returns, run its plan as written,
|
|
414
|
+
When the router returns, run its plan as written, each lens with its
|
|
415
|
+
delegation's `question` and the evidence wanted for or against it, with one
|
|
416
|
+
addition: when the
|
|
355
417
|
plan lacks a delegation Step 3 makes mandatory (a HIGH or CRITICAL grade, a
|
|
356
418
|
risky area), add it, one rung above the router, scoped to the hunks behind
|
|
357
419
|
that grade. Never drop a planned delegation, and never re-grade a finding
|
|
@@ -402,7 +464,14 @@ evidence.
|
|
|
402
464
|
caller in the repository passes it or the promise covers it.
|
|
403
465
|
- It lists each read-only command it ran (`rg`, `sed -n`, `git show`, `git log
|
|
404
466
|
-L`) with the one line of output that settled the question. They go in the
|
|
405
|
-
finding's `checked`, so a reader can audit the claim without redoing it.
|
|
467
|
+
finding's `checked`, so a reader can audit the claim without redoing it. For
|
|
468
|
+
code that was deleted or moved, where `git blame` shows nothing, use
|
|
469
|
+
`../git-archaeology/SKILL.md`.
|
|
470
|
+
- A finding whose only fix adds ceremony is `refuted`, with the code that shows
|
|
471
|
+
it: a check for a case that cannot happen, an abstraction used once, a comment
|
|
472
|
+
restating the code, a test asserting a tautology. A fix must be sound, correct
|
|
473
|
+
and simple; one that trades simplicity for nominal correctness leaves the code
|
|
474
|
+
worse. A real bug with a heavy fix stays, and the finding names a lighter one.
|
|
406
475
|
- Judge new code by its intended use. A function the PR adds and exports exists
|
|
407
476
|
to be called; "nothing calls it yet" never lowers a finding's severity.
|
|
408
477
|
- A finding the verifier cannot settle without outside facts (an API's
|
|
@@ -432,21 +501,30 @@ other HIGH, such as "`firstName` returns the second word".
|
|
|
432
501
|
|
|
433
502
|
Check every finding's line against the diff before it posts. Cheap models
|
|
434
503
|
misnumber lines: find the code the finding quotes and use its real line on the
|
|
435
|
-
new side of the diff.
|
|
504
|
+
new side of the diff, or the old side for a `LEFT` finding.
|
|
436
505
|
|
|
437
506
|
## Step 6: Merge and grade
|
|
438
507
|
|
|
508
|
+
- **In scope.** A finding counts when its root cause sits in lines this PR
|
|
509
|
+
added or changed, or the PR introduced or amplified it, and the finding says
|
|
510
|
+
how. Drop the rest. Filter here, never in a lens brief: a lens told to skip
|
|
511
|
+
old code also misses the regression a change made worse.
|
|
439
512
|
- **Dedupe by root cause.** Two findings are one issue when one change to the
|
|
440
513
|
code fixes both, wherever they sit; report it once, at the line the fix
|
|
441
514
|
changes. Distinct bugs stay separate even on neighbouring lines. A missing
|
|
442
515
|
regression test for a bug belongs in that bug's finding (name the test to
|
|
443
516
|
add), not in a finding of its own. Two or more independent lenses agreeing
|
|
444
517
|
makes it **convergent**; note every lens that found it.
|
|
518
|
+
- **Flag the class.** When a finding above NIT is a pattern that can repeat,
|
|
519
|
+
search the lines this PR added for it. A further hit joins the finding when
|
|
520
|
+
the worked example holds there too: list its `file:line` in the body and word the fix to cover them all, so one
|
|
521
|
+
round closes the class. Places that need different fixes stay separate
|
|
522
|
+
findings.
|
|
445
523
|
- **Conflicts with open threads.** A finding whose fix would keep or bring
|
|
446
524
|
back a problem an open thread names (from any reviewer) says so in its body
|
|
447
525
|
and names that thread: the author sees both before choosing.
|
|
448
526
|
- **Undone fixes.** From round 2 on, read the Shepherd threads resolved in
|
|
449
|
-
earlier rounds and the
|
|
527
|
+
earlier rounds and the round's focus (the interdiff above). A finding whose
|
|
450
528
|
code brings back a problem one of them named sets `reintroduces` to that
|
|
451
529
|
thread's URL and keeps at least that thread's severity: a fix the author
|
|
452
530
|
already accepted came undone, often from an agent's later edit.
|
|
@@ -473,6 +551,22 @@ The verdict is advice to the author. It never becomes a GitHub approval.
|
|
|
473
551
|
|
|
474
552
|
Write the round as JSON first (`references/formats.md`): the inline review and
|
|
475
553
|
the summary are both rendered from it, so neither is ever hand-written Markdown.
|
|
554
|
+
Set `base_sha` to the full SHA of the tip this round reviewed against (the
|
|
555
|
+
parent's head for a stacked PR, else `git rev-parse origin/<base_ref>`); the
|
|
556
|
+
summary records it for the next round's `--since-base`.
|
|
557
|
+
|
|
558
|
+
Then check every changed range reached a lens that finished:
|
|
559
|
+
|
|
560
|
+
```bash
|
|
561
|
+
shepherd coverage round.json --diff "$diff_path"
|
|
562
|
+
```
|
|
563
|
+
|
|
564
|
+
When it exits non-zero, dispatch the correctness lens once over the
|
|
565
|
+
`uncovered` ranges as its scope (`file:start-end`), merge its findings through
|
|
566
|
+
Steps 5 and 6, add it to `lenses`, and run the check again. Ranges still
|
|
567
|
+
uncovered go into `lenses` as one entry, `{"lens": "coverage", "scope":
|
|
568
|
+
[<uncovered>], "status": "could_not_run"}`, so the summary marks the round
|
|
569
|
+
incomplete. A round with a range no reviewer reached is never dry.
|
|
476
570
|
With `$SHEPHERD_HANDOFF` set, write it to `$SHEPHERD_HANDOFF/round.json`,
|
|
477
571
|
then check it before you stop:
|
|
478
572
|
|
|
@@ -489,7 +583,7 @@ record (`stages`, `candidates`). `verdict` is `blocked`, `changes`, `nits` or
|
|
|
489
583
|
publish job renders and posts both. Otherwise post every inline finding as **one** review, pinned to the reviewed commit:
|
|
490
584
|
|
|
491
585
|
```bash
|
|
492
|
-
shepherd inline round.json > review.json
|
|
586
|
+
shepherd inline round.json --diff "$diff_path" > review.json
|
|
493
587
|
gh api repos/<owner>/<repo>/pulls/<number>/reviews --method POST --input review.json \
|
|
494
588
|
--jq '[.id]' > review-id.json
|
|
495
589
|
```
|
|
@@ -502,11 +596,27 @@ findings on the same lines are named under it. Run it from the reviewed
|
|
|
502
596
|
checkout, so it can place each `suggestion` on the lines it rewrites, at their
|
|
503
597
|
indentation. A finding on a line
|
|
504
598
|
outside the diff cannot anchor inline: keep its `line` and set
|
|
505
|
-
`"inline": false`.
|
|
599
|
+
`"inline": false`. A finding about code the PR deletes sets `"side": "LEFT"`
|
|
600
|
+
with the line's OLD number, so it anchors on the removal; it carries no
|
|
601
|
+
`suggestion`. With `--diff`, `shepherd inline` also leaves out any
|
|
602
|
+
finding whose lines fall outside the diff's hunks, since GitHub rejects the
|
|
603
|
+
whole review over one; the summary still carries it. When `review.json` has no `comments`, skip the post.
|
|
506
604
|
Otherwise read the new threads' URLs
|
|
507
605
|
(`gh api repos/<owner>/<repo>/pulls/<number>/reviews/<id>/comments --jq '.[] | {path, line, html_url}'`)
|
|
508
606
|
into each finding's `url`, so the summary links every finding to its thread.
|
|
509
607
|
|
|
608
|
+
When the post fails, read why before changing anything:
|
|
609
|
+
|
|
610
|
+
- A 5xx, or a 422 saying `internal error occurred`, is GitHub's fault, not the
|
|
611
|
+
review's. Send the same payload again after a few seconds, twice at most.
|
|
612
|
+
Never drop or edit a comment to get past it.
|
|
613
|
+
- A 422 naming a `pending review` means this account holds an unsubmitted draft
|
|
614
|
+
on the PR, and GitHub allows one. Never delete it: it may be a person's work.
|
|
615
|
+
Skip the inline review, say so in the summary, and post the summary, which
|
|
616
|
+
carries every finding.
|
|
617
|
+
- Any other 422 on the comments: the head moved, or a `suggestion` block is
|
|
618
|
+
malformed. Refresh the diff, or set those findings `"inline": false`.
|
|
619
|
+
|
|
510
620
|
Every run, including the first of a new session, fetches the unresolved
|
|
511
621
|
Shepherd threads first (triage's Step 3 query) and does not post a finding that
|
|
512
622
|
one of them already carries: the same issue by Step 6's root-cause rule. Name it in
|
|
@@ -571,7 +681,9 @@ overwritten.
|
|
|
571
681
|
## Step 8: Report
|
|
572
682
|
|
|
573
683
|
Standalone: print the verdict, counts by severity, convergent findings, which
|
|
574
|
-
lenses ran on which rung, and anything dropped by verification.
|
|
684
|
+
lenses ran on which rung, and anything dropped by verification. Read the PR's
|
|
685
|
+
head once more: when it moved past `head_sha`, say `new commits since
|
|
686
|
+
<short_sha>, not reviewed`, so nobody reads this round as covering them.
|
|
575
687
|
|
|
576
688
|
As a `shepherd` sub-step: end with exactly this and nothing after it:
|
|
577
689
|
|
|
@@ -35,7 +35,7 @@ DELEGATION_PLAN:
|
|
|
35
35
|
danger: <LOW|MEDIUM|HIGH|CRITICAL>
|
|
36
36
|
confidence: <HIGH|MEDIUM|LOW>
|
|
37
37
|
delegations:
|
|
38
|
-
- lens: <correctness|security|simplicity|maintainability|slop|spec|custom lens name> | rung: <ladder rung> | scope: <paths or hunks, or "full"> | reason: <one line>
|
|
38
|
+
- lens: <correctness|security|simplicity|maintainability|slop|spec|custom lens name> | rung: <ladder rung> | scope: <paths or hunks, or "full"> | question: <one falsifiable question, framed by its domain> | reason: <one line>
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
An empty `delegations:` list means the router's findings are the review.
|
|
@@ -55,10 +55,11 @@ Never write the summary's Markdown yourself. Write the round as JSON and let
|
|
|
55
55
|
```json
|
|
56
56
|
{
|
|
57
57
|
"head_sha": "<reviewed head>",
|
|
58
|
+
"base_sha": "<full SHA of the base tip reviewed against: the parent's head for a stacked PR>",
|
|
58
59
|
"verdict": "blocked|changes|nits|good",
|
|
59
60
|
"why": "<one or two sentences: is it safe to merge, and what must change first>",
|
|
60
61
|
"confidence": 4,
|
|
61
|
-
"findings": [{"file": "src/a.ts", "line": 3, "severity": "HIGH", "lens": "correctness/data", "title": "<a claim the reader can check>", "body": "<why it matters, the fix>", "evidence": {"input": "<what triggers it>", "expected": "<what should happen>", "actual": "<what happens>"}, "verified": "<what the verifier quoted>", "suggestion": "<replacement lines, only when the fix replaces exactly line 3>", "url": "<its inline thread>", "inline": true, "convergent": false, "checked": [{"command": "rg -n 'firstName' src", "result": "<the line that settled it>"}], "docs": [{"url": "<https URL you are certain of>", "title": "<page title>"}], "reintroduces": "<a resolved thread whose fix this undoes>"}],
|
|
62
|
+
"findings": [{"file": "src/a.ts", "line": 3, "side": "RIGHT", "severity": "HIGH", "lens": "correctness/data", "title": "<a claim the reader can check>", "body": "<why it matters, the fix>", "evidence": {"input": "<what triggers it>", "expected": "<what should happen>", "actual": "<what happens>"}, "verified": "<what the verifier quoted>", "suggestion": "<replacement lines, only when the fix replaces exactly line 3>", "url": "<its inline thread>", "inline": true, "convergent": false, "checked": [{"command": "rg -n 'firstName' src", "result": "<the line that settled it>"}], "docs": [{"url": "<https URL you are certain of>", "title": "<page title>"}], "reintroduces": "<a resolved thread whose fix this undoes>"}],
|
|
62
63
|
"pre_merge": [{"check": "Title|Description|Linked issue|Scope|Tests", "status": "pass|warn|fail", "detail": "<one line>"}],
|
|
63
64
|
"walkthrough": ["<what the PR does, one bullet per behaviour, for a reader who never saw it>"],
|
|
64
65
|
"changes": [{"files": ["src/a.ts", "src/a.test.ts"], "summary": "<what changed in these files>"}],
|
|
@@ -87,7 +88,9 @@ The new fields are optional: a round without them renders as before. A field
|
|
|
87
88
|
written as `null` reads as absent and takes its default. Write `evidence` as
|
|
88
89
|
the object above; text that labels its input, expected and actual result is
|
|
89
90
|
read too. A finding without a `title` takes its body's first sentence, and
|
|
90
|
-
`line` with `line_end` reads as `"line-line_end"`.
|
|
91
|
+
`line` with `line_end` reads as `"line-line_end"`. `side` is `RIGHT` unless the
|
|
92
|
+
finding is on a line the diff removes: `LEFT`, with `line` its old number. A
|
|
93
|
+
`LEFT` finding posts inline only when `shepherd inline` is given `--diff`.
|
|
91
94
|
The renderers hold the round to what they can check. A `url`, `reintroduces`
|
|
92
95
|
or still-open link that names no review comment on the PR, or is not a
|
|
93
96
|
GitHub URL, loses its link.
|
|
@@ -117,4 +120,5 @@ checks, lenses, and earlier rounds folded one line each. With `--fanout` (in CI,
|
|
|
117
120
|
`could_not_run` and the round says it is incomplete. With `--body` it keeps the
|
|
118
121
|
round number but leaves the history out: the publish job folds the comment it
|
|
119
122
|
replaces in itself, so the history would otherwise appear twice. It escapes markers, mentions
|
|
120
|
-
and images in every string you give it
|
|
123
|
+
and images in every string you give it, and cuts a comment that would pass
|
|
124
|
+
GitHub's size limit.
|
package/skills/triage/SKILL.md
CHANGED
|
@@ -116,8 +116,9 @@ gh api graphql -f query='
|
|
|
116
116
|
pageInfo { hasNextPage endCursor }
|
|
117
117
|
nodes {
|
|
118
118
|
id isResolved isOutdated
|
|
119
|
-
comments(first:
|
|
120
|
-
|
|
119
|
+
comments(first:50) {
|
|
120
|
+
totalCount
|
|
121
|
+
nodes { id databaseId author { login __typename } body path line }
|
|
121
122
|
}
|
|
122
123
|
}
|
|
123
124
|
}
|
|
@@ -131,6 +132,8 @@ gh api graphql -f query='
|
|
|
131
132
|
author: .comments.nodes[0].author.login,
|
|
132
133
|
body_head: (.comments.nodes[0].body[:1500]),
|
|
133
134
|
body_truncated: ((.comments.nodes[0].body | length) > 1500),
|
|
135
|
+
first_comment: .comments.nodes[0].id,
|
|
136
|
+
comments_truncated: (.comments.totalCount > 50),
|
|
134
137
|
participants: ([.comments.nodes[] | {login: .author.login, type: .author.__typename,
|
|
135
138
|
automated: (.body[:300] | contains("🤖 Automated comment by"))}] | unique)}]}'
|
|
136
139
|
```
|
|
@@ -139,15 +142,33 @@ Follow `endCursor` while `hasNextPage` is true. Do not re-classify ids already
|
|
|
139
142
|
in `deferred_threads`, but keep their path, line, and concern: they fence code
|
|
140
143
|
in Step 5.
|
|
141
144
|
|
|
145
|
+
A thread with `comments_truncated` true has comments this query did not read,
|
|
146
|
+
and any of them may be a person's: treat it as human.
|
|
147
|
+
|
|
148
|
+
Read 👍s separately, in one query per 100 `first_comment` ids of all-automated
|
|
149
|
+
threads. Nesting reactions under every comment above would make each page cost
|
|
150
|
+
dozens of API points:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
gh api graphql -f query='
|
|
154
|
+
query($ids:[ID!]!) { nodes(ids:$ids) { ... on PullRequestReviewComment {
|
|
155
|
+
id reactions(content: THUMBS_UP, first: 100) { nodes { user { login } } } } } }' \
|
|
156
|
+
-f 'ids[]=<first_comment>' --jq '.data.nodes[] | {id, thumbs_up: [.reactions.nodes[].user.login]}'
|
|
157
|
+
```
|
|
158
|
+
|
|
142
159
|
## Step 4: Classify each thread
|
|
143
160
|
|
|
144
161
|
**Who is in it.** A comment is automated when its author is a bot account
|
|
145
162
|
(`__typename == "Bot"`, a login ending `[bot]`, or a known review bot:
|
|
146
163
|
greptile, coderabbit, cursor, copilot, sonarcloud, codescene, sourcery,
|
|
147
164
|
ellipsis, claude, chatgpt-codex-connector, github-actions, dependabot,
|
|
148
|
-
renovate), **or** its body carries `🤖 Automated comment by
|
|
149
|
-
|
|
150
|
-
|
|
165
|
+
renovate), **or** its body carries `🤖 Automated comment by` and an account that
|
|
166
|
+
can already push wrote it: the logged-in user, or a login where `gh api
|
|
167
|
+
repos/<owner>/<repo>/collaborators/<login>/permission --jq .permission` says
|
|
168
|
+
`admin` or `write` (ask once per login). The header is the check that matters
|
|
169
|
+
for Shepherd's own comments, which post through the author's account typed
|
|
170
|
+
`User`, but anyone can paste it: from an account without write access, or one
|
|
171
|
+
whose access you could not read, the comment is human, however it is labelled.
|
|
151
172
|
|
|
152
173
|
If any comment in the thread is not automated, the whole thread is **human**:
|
|
153
174
|
defer it untouched. A person who replied to a bot has joined the conversation.
|
|
@@ -162,6 +183,14 @@ changes included, and resolve it with the commit in the report. A question
|
|
|
162
183
|
("could we...?", "should this...?") is not an instruction: defer it. Anyone
|
|
163
184
|
else in the thread makes it human again.
|
|
164
185
|
|
|
186
|
+
A 👍 is the same instruction without the typing. When the logged-in user is in
|
|
187
|
+
`thumbs_up` on an all-automated thread's first comment, they accepted that
|
|
188
|
+
finding: treat it as their instruction to apply it, in place of the pair
|
|
189
|
+
ladder. Every other rule still holds (policy, `escalate.paths`, fences, a
|
|
190
|
+
second model's `accept` for a risky change). Logins compare case-insensitively.
|
|
191
|
+
A 👍 from anyone else is not an
|
|
192
|
+
instruction and changes nothing: reactions are not participants.
|
|
193
|
+
|
|
165
194
|
**Team policy.** Load it once (`../shepherd/references/policy.md`). Its
|
|
166
195
|
`fixes.mode` caps Step 5: under `suggest` edit nothing, under `local` commit,
|
|
167
196
|
push and resolve nothing (a fix in the working tree has not landed; report
|
|
@@ -208,14 +237,18 @@ unless the PR's own change depends on the broken code.
|
|
|
208
237
|
needs proof from the code or the repository. A claim about outside behaviour
|
|
209
238
|
(an API's rules, a service's limits) that you cannot check is not wrong:
|
|
210
239
|
fix it when the fix is cheap and harmless, otherwise defer it with the claim
|
|
211
|
-
stated. Never resolve it as wrong.
|
|
240
|
+
stated. Never resolve it as wrong. A fix that would only add ceremony (a
|
|
241
|
+
check for a case the code shows cannot happen, an abstraction used once, a
|
|
242
|
+
comment restating the code, a tautological test) is wrong in this sense:
|
|
243
|
+
resolve it with the code that proves it, rather than apply it because a
|
|
244
|
+
reviewer asked.
|
|
212
245
|
- **Ambiguous**: architecture, broad scope, or a design choice.
|
|
213
246
|
|
|
214
247
|
**Repeat threads.** An all-automated thread whose concern matches one a
|
|
215
248
|
`Shepherd:` commit on this PR already claimed to fix (a resolved thread naming
|
|
216
249
|
the commit, or the commit's message) is proof the fix did not hold. Before
|
|
217
250
|
fixing again, say why it did not (it fixed a sibling path, the wrong root cause,
|
|
218
|
-
or was reverted), and never resolve it as already fixed on the same evidence.
|
|
251
|
+
or was reverted; `../git-archaeology/SKILL.md` finds what removed it), and never resolve it as already fixed on the same evidence.
|
|
219
252
|
When two `Shepherd:` commits have already tried to fix the same concern, both
|
|
220
253
|
rested on a premise that is wrong: write the premise down, test it directly
|
|
221
254
|
(one command, one read of the code it names), and fix only what that shows. If
|
|
@@ -235,7 +268,10 @@ dismiss them. Never change code only to quiet a bot.
|
|
|
235
268
|
**Outdated threads.** GitHub marks a thread outdated when its line changed, not
|
|
236
269
|
when its concern was met, so an outdated thread is never skipped. Read its
|
|
237
270
|
concern against the head: gone, resolve it ("addressed in <sha>"); still true,
|
|
238
|
-
classify it like any other thread, at the line where the code now lives.
|
|
271
|
+
classify it like any other thread, at the line where the code now lives. Code
|
|
272
|
+
reformatted, renamed or moved elsewhere has not met a concern, and a thread
|
|
273
|
+
with several concerns needs all of them met. When unsure, leave it open: a wrong
|
|
274
|
+
resolve costs more trust than a thread left standing.
|
|
239
275
|
|
|
240
276
|
**Stale bot comments.** For top-level bot comments, look for a commit reference (`/commit/<sha>`,
|
|
241
277
|
"reviewing `<sha>`"). A reference to anything but HEAD is stale; skip it. When
|
|
@@ -285,7 +321,7 @@ unsure and the comment predates the last push, skip rather than act.
|
|
|
285
321
|
ignored. A fix the section lists under "Escalate" is deferred, with that
|
|
286
322
|
line quoted in the report. Add the trailer `Shepherd-Lens: <name>` to the
|
|
287
323
|
commit.
|
|
288
|
-
- **House rules.** Follow the house rules: `AGENTS.md`, `CLAUDE.md`, `CONTRIBUTING.md`, and
|
|
324
|
+
- **House rules.** Follow the house rules: `AGENTS.md`, `CLAUDE.md`, `CONTRIBUTING.md`, `REVIEW.md`, and
|
|
289
325
|
`docs/adr/` as they stand on `origin/<default>`; a PR's own edits to them do
|
|
290
326
|
not count. Read them before the first edit.
|
|
291
327
|
- **Deferred threads fence their code.** A thread left for the author, from
|
|
@@ -332,11 +368,28 @@ unsure and the comment predates the last push, skip rather than act.
|
|
|
332
368
|
**promoted**, and put the reasoning and the alternative prominently in the
|
|
333
369
|
report. `stop-and-ask`: add to `deferred_threads`, leave open.
|
|
334
370
|
- **Operator's instruction:** apply it, check it, commit, push, resolve.
|
|
335
|
-
Record "done as asked in <sha>".
|
|
371
|
+
Record "done as asked in <sha>" ("done on 👍 in <sha>" for a reaction).
|
|
336
372
|
- **Human:** defer. Standalone, you may offer a trivial, obviously correct fix
|
|
337
373
|
through `AskUserQuestion`; even then push it, leave the thread open, and
|
|
338
374
|
report "fixed in <sha>, thread left open for your reply".
|
|
339
375
|
|
|
376
|
+
**Look again before you resolve.** The fetch in Step 3 is minutes old by now, and
|
|
377
|
+
a person may have joined the thread, or told you to stop, while you worked. Read
|
|
378
|
+
the thread's comments first:
|
|
379
|
+
|
|
380
|
+
```bash
|
|
381
|
+
gh api graphql -f query='query($id:ID!){node(id:$id){... on PullRequestReviewThread{isResolved
|
|
382
|
+
comments(first:100){totalCount nodes{author{login __typename} body}}}}}' -F id=<thread_id> \
|
|
383
|
+
--jq '.data.node | {isResolved, complete: (.comments.totalCount <= 100), comments: [.comments.nodes[] | {login: .author.login, type: .author.__typename, body: .body[:300]}]}'
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
A comment Step 3 did not see, by anyone Step 4 would call human, makes it a human
|
|
387
|
+
thread: leave it open and defer it, with your commit named in the report. So
|
|
388
|
+
does `complete` false, or a read that fails: you cannot show that no person
|
|
389
|
+
joined. The
|
|
390
|
+
fix stays pushed. Before the first push, do the same for every thread you are
|
|
391
|
+
fixing, and hold back a fix a person has since asked you to change or stop.
|
|
392
|
+
|
|
340
393
|
Resolve a thread with:
|
|
341
394
|
|
|
342
395
|
```bash
|
|
@@ -348,8 +401,10 @@ signal. A deferred line in the report is cheaper than a wrong push.
|
|
|
348
401
|
|
|
349
402
|
### stamp's verdict
|
|
350
403
|
|
|
351
|
-
`stamp` posts its verdict as a top-level review
|
|
352
|
-
|
|
404
|
+
`stamp` posts its verdict as a top-level review listing every issue. On `REFUSED`
|
|
405
|
+
and `ESCALATE` it also pins each issue whose line is in the diff as an inline
|
|
406
|
+
thread, whose comment carries `<!-- stamp-run:`. Read the latest review on the
|
|
407
|
+
current head:
|
|
353
408
|
|
|
354
409
|
```bash
|
|
355
410
|
gh pr view <number> --json latestReviews \
|
|
@@ -358,8 +413,9 @@ gh pr view <number> --json latestReviews \
|
|
|
358
413
|
|
|
359
414
|
- **REFUSED with issues:** treat each item in the **Issues** section (a
|
|
360
415
|
`<details>` block titled `Issues: <n>`, numbered or bulleted) as a bot thread and classify it the
|
|
361
|
-
same way.
|
|
362
|
-
|
|
416
|
+
same way. An inline stamp thread repeats one of these items: count the pair
|
|
417
|
+
once and resolve the thread with it. Fixes push; stamp re-reviews on the push
|
|
418
|
+
and resolves its older threads, keeping any a person replied in.
|
|
363
419
|
- **REFUSED by a gate** (the **Gates** section shows a failed gate such as
|
|
364
420
|
`deny-list`, `secrets`, or `size`): a gate decides whether stamp approves,
|
|
365
421
|
not whether the code may be fixed. Keep fixing every thread as usual; a hard
|
|
@@ -368,7 +424,8 @@ gh pr view <number> --json latestReviews \
|
|
|
368
424
|
anything: a placeholder in place of a flagged value, moved, split, or renamed
|
|
369
425
|
files, an `AGENT_APPROVALS.md`. Report the gate's message for the author; a
|
|
370
426
|
deny-listed area still needs the human review stamp asks for, fixed or not.
|
|
371
|
-
- **ESCALATE:** defer; it names the human assurance it
|
|
427
|
+
- **ESCALATE:** defer, inline threads included; it names the human assurance it
|
|
428
|
+
needs.
|
|
372
429
|
|
|
373
430
|
## Step 6: Report
|
|
374
431
|
|
package/src/cilog.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// A failed job's log, indexed.
|
|
2
|
+
|
|
3
|
+
export type LogLine = { line: number; type: "error" | "warning" | "failure" | "trace"; content: string };
|
|
4
|
+
|
|
5
|
+
export type LogAnalysis = { total_lines: number; index: LogLine[]; excerpt: { start_line: number; end_line: number; content: string } };
|
|
6
|
+
|
|
7
|
+
const PATTERNS: { type: LogLine["type"]; pattern: RegExp; skip?: RegExp }[] = [
|
|
8
|
+
{ type: "error", pattern: /##\[error\]/i },
|
|
9
|
+
{ type: "error", pattern: /\bError:/i },
|
|
10
|
+
{ type: "error", pattern: /\bERR_/i },
|
|
11
|
+
{ type: "error", pattern: /exit code [1-9]/i },
|
|
12
|
+
{ type: "warning", pattern: /##\[warning\]/i },
|
|
13
|
+
{ type: "warning", pattern: /\bWARN\b/i, skip: /apt|dpkg|Reading package/i },
|
|
14
|
+
{ type: "failure", pattern: /\d+ failed/i },
|
|
15
|
+
{ type: "failure", pattern: /FAIL\b/i },
|
|
16
|
+
{ type: "failure", pattern: /✕|✗|×/ },
|
|
17
|
+
{ type: "trace", pattern: /^\s+at\s+/i },
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
const MAX_INDEX = 200;
|
|
21
|
+
|
|
22
|
+
const MAX_LINE = 120;
|
|
23
|
+
|
|
24
|
+
const EXCERPT = 80;
|
|
25
|
+
|
|
26
|
+
const AFTER = 5;
|
|
27
|
+
|
|
28
|
+
// `gh run view --log-failed` opens each line with the job, the step and a timestamp; a raw job log with the timestamp.
|
|
29
|
+
const PREFIX = /^(?:[^\t]*\t[^\t]*\t)?\d{4}-\d{2}-\d{2}T[\d:.]+Z ?/;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A map of a failed job's log: an index of its error, warning, failure and first stack-trace lines,
|
|
33
|
+
* and a window around the last `##[error]` (the tail when there is none). `line` numbers the file given, so a reader
|
|
34
|
+
* goes straight there instead of loading the log.
|
|
35
|
+
*/
|
|
36
|
+
export function analyzeLog(log: string): LogAnalysis {
|
|
37
|
+
// eslint-disable-next-line no-control-regex -- ANSI escapes are control characters
|
|
38
|
+
const lines = log.replace(/\x1b\[[0-9;]*m/g, "").split("\n").map((l) => l.replace(PREFIX, ""));
|
|
39
|
+
const index: LogLine[] = [];
|
|
40
|
+
|
|
41
|
+
for (const [i, l] of lines.entries()) {
|
|
42
|
+
const hit = PATTERNS.find((p) => p.pattern.test(l) && !p.skip?.test(l));
|
|
43
|
+
|
|
44
|
+
if (!hit || index.length >= MAX_INDEX || (hit.type === "trace" && index.at(-1)?.type === "trace")) continue;
|
|
45
|
+
const text = l.trim();
|
|
46
|
+
|
|
47
|
+
index.push({ line: i + 1, type: hit.type, content: text.length > MAX_LINE ? `${text.slice(0, MAX_LINE - 3)}...` : text });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const last = lines.map((l) => /##\[error\]/i.test(l)).lastIndexOf(true);
|
|
51
|
+
const start = Math.max(0, last === -1 ? lines.length - EXCERPT : last - (EXCERPT - AFTER));
|
|
52
|
+
const end = last === -1 ? lines.length : Math.min(lines.length, last + AFTER);
|
|
53
|
+
|
|
54
|
+
return { total_lines: lines.length, index, excerpt: { start_line: start + 1, end_line: end, content: lines.slice(start, end).join("\n") } };
|
|
55
|
+
}
|