@sleepy_panda_srl/portulan 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/cli/README.md +145 -358
- package/cli/advisory.mjs +354 -0
- package/cli/comments.mjs +932 -0
- package/cli/compile.mjs +1719 -1699
- package/cli/context.mjs +883 -0
- package/cli/control-chars.mjs +12 -301
- package/cli/discover.mjs +17 -574
- package/cli/doctor.mjs +269 -1080
- package/cli/drills.mjs +92 -504
- package/cli/feedback.mjs +16 -237
- package/cli/finish.mjs +437 -0
- package/cli/form.mjs +644 -0
- package/cli/fuzz-shell.mjs +20 -305
- package/cli/gate.mjs +7 -199
- package/cli/goldens.mjs +26 -184
- package/cli/index.mjs +305 -655
- package/cli/init.mjs +243 -566
- package/cli/inside.mjs +3 -28
- package/cli/instructions.mjs +829 -0
- package/cli/ledger.mjs +882 -0
- package/cli/librarian.mjs +135 -599
- package/cli/manifest.mjs +2 -39
- package/cli/mutants.mjs +9 -170
- package/cli/new.mjs +12 -243
- package/cli/pack-identity.mjs +5 -74
- package/cli/pack-version.mjs +13 -255
- package/cli/plugin-lint.mjs +42 -658
- package/cli/portulan.mjs +6 -125
- package/cli/recipe-set.mjs +15 -245
- package/cli/release-eval.mjs +28 -348
- package/cli/review-meter.mjs +29 -265
- package/cli/rule-carriers.mjs +12 -175
- package/cli/sessions.mjs +82 -0
- package/cli/skill-goldens.mjs +10 -176
- package/cli/skills-set.mjs +12 -414
- package/cli/stop-gate.mjs +83 -514
- package/cli/symbols.mjs +1132 -0
- package/cli/telemetry.mjs +82 -505
- package/cli/upgrade.mjs +171 -314
- package/cli/vendor.mjs +146 -476
- package/cli/version-carriers.mjs +8 -86
- package/core/README.md +1 -1
- package/core/engine.md +3 -9
- package/core/operating/autonomy.md +1 -1
- package/core/operating/context.md +258 -0
- package/core/operating/evolution.md +3 -3
- package/core/operating/loop.md +13 -20
- package/core/operating/memory.md +12 -3
- package/core/operating/sessions.md +100 -0
- package/core/personas/README.md +33 -0
- package/core/personas/librarian.md +2 -2
- package/core/skills/clarify/SKILL.md +1 -1
- package/core/skills/consolidate/SKILL.md +5 -3
- package/core/templates/handoff.md +28 -15
- package/core/templates/memory-entry.md +1 -0
- package/evals/README.md +271 -845
- package/evals/releases/0.2.0.json +146 -0
- package/evals/releases/0.2.0.md +67 -0
- package/evals/releases/0.3.0.json +150 -0
- package/evals/releases/0.3.0.md +68 -0
- package/evals/releases/README.md +3 -3
- package/package.json +5 -2
- package/packs/.claude-plugin/plugin.json +2 -1
- package/packs/README.md +7 -4
- package/packs/rituals/checkpoints/README.md +1 -1
- package/packs/tools/README.md +8 -8
- package/packs/tools/github/verify/actions-pinned.sh +10 -55
- package/packs/tools/local-llm/README.md +136 -0
- package/packs/tools/local-llm/pack.json +32 -0
- package/packs/tools/local-llm/personas/local-worker.md +44 -0
- package/packs/tools/local-llm/runner/local-llm.mjs +1665 -0
- package/packs/tools/local-llm/skills/delegate-local/SKILL.md +64 -0
- package/packs/tools/local-llm/verify/README.md +18 -0
- package/spec/README.md +113 -6
- package/spec/migrations/0001-repository-declares-its-tree.mjs +1 -26
- package/spec/migrations/0002-bundle-fallback-path.mjs +13 -74
- package/spec/migrations/0003-handoff-index-not-kept.mjs +66 -0
- package/spec/migrations/0004-changelog-fragments.mjs +53 -0
- package/spec/migrations/0005-session-log-retired.mjs +69 -0
- package/spec/migrations/0006-boot-card.mjs +77 -0
- package/spec/migrations/0007-guidance-compiled.mjs +31 -0
- package/spec/migrations/0008-card-reading.mjs +55 -0
- package/spec/migrations/0009-instruction-sections-on-read.mjs +60 -0
- package/spec/migrations/0010-comment-rail.mjs +57 -0
- package/spec/migrations/README.md +45 -10
- package/spec/pack.schema.json +1 -1
- package/spec/slots.md +343 -27
- package/spec/workspace.schema.json +145 -4
package/cli/review-meter.mjs
CHANGED
|
@@ -1,133 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// The review-loop meter
|
|
3
|
-
// instead of counted by hand.
|
|
2
|
+
// The review-loop meter: the historical figures of this repository's Copilot review loop, derived from its reviews.
|
|
4
3
|
//
|
|
5
|
-
//
|
|
6
|
-
// pushes per round, empty-round rate.* The argument for the clause is in
|
|
7
|
-
// `../docs/milestones/m08.md`; what belongs here is what each figure IS, which of them the API can
|
|
8
|
-
// answer, and — at more length than is comfortable — which of them it cannot.
|
|
9
|
-
//
|
|
10
|
-
// ## Why this tool exists at all
|
|
11
|
-
//
|
|
12
|
-
// `../.portulan/memory/a-review-loop-needs-a-bound.md` bounds the loop on a table of figures — 110
|
|
13
|
-
// submissions over 30 pull requests, 29% of them finding nothing — every one of which was counted by
|
|
14
|
-
// hand on 2026-07-28. That record says of itself, in its own *Why it holds* section, **"Nothing
|
|
15
|
-
// checks it — discipline, not a rail"**, citing
|
|
16
|
-
// `../.portulan/memory/a-mandate-nothing-checks-is-already-broken.md`. The 2026-07-28 amendment
|
|
17
|
-
// answers it in as many words: *"the telemetry clause is where that checker's home is, and naming the
|
|
18
|
-
// home is what this amendment does rather than claiming the checker exists."* This file is that
|
|
19
|
-
// checker arriving at the home the amendment named.
|
|
20
|
-
//
|
|
21
|
-
// A hand-counted figure whose subject keeps growing is this repository's most-repeated defect — it
|
|
22
|
-
// has now been repaired in the recipe counts of `../.portulan/repos/portulan.md`, the CLI roster of
|
|
23
|
-
// `./README.md`, and the operator total in `../evals/README.md`, each time the same way: **delete the
|
|
24
|
-
// tally and name the command that derives it.** The loop's figures are the last hand-maintained set
|
|
25
|
-
// of any consequence, and they are the ones a rule leans on.
|
|
26
|
-
//
|
|
27
|
-
// ## The word "round" moved, and the criterion predates the move
|
|
28
|
-
//
|
|
29
|
-
// This is the first thing to settle, because two of the criterion's three figures are named in a unit
|
|
30
|
-
// that was redefined after the criterion was written, and reading them in today's unit yields a
|
|
31
|
-
// different tool.
|
|
32
|
-
//
|
|
33
|
-
// The criterion was written **2026-07-28**. On **2026-07-30** the maintainer defined a round as *a
|
|
34
|
-
// Copilot review the working session answers with a push*
|
|
35
|
-
// (`../.portulan/handoffs/2026-07-30-a-round-gets-its-definition.md`), and the rule's table was
|
|
36
|
-
// **re-labelled, not re-counted**: its figures had always counted **submissions** — every review
|
|
37
|
-
// Copilot submits, one per push under `review_on_push: true`, including on the branch as opened —
|
|
38
|
-
// and the table's own units note says so. The rule's `Retire when:` line settles it beyond argument,
|
|
39
|
-
// naming its threshold in *"the submission units of the table above, **not fix-rounds**"*.
|
|
40
|
-
//
|
|
41
|
-
// So the criterion's *"rounds per pull request"* is **submissions per pull request**, and that is
|
|
42
|
-
// what this tool computes under that name. It never prints the bare word "rounds" for a figure,
|
|
43
|
-
// because the word has meant two things here and a figure whose unit is ambiguous is the exact defect
|
|
44
|
-
// #119 was opened to repair.
|
|
45
|
-
//
|
|
46
|
-
// ## What the API can answer, and the one thing it cannot
|
|
47
|
-
//
|
|
48
|
-
// **Submissions: exactly.** A review on `/pulls/N/reviews` whose author is the reviewer. Two traps,
|
|
49
|
-
// both measured on this repository rather than reasoned about:
|
|
50
|
-
//
|
|
51
|
-
// * **One actor, two logins.** `copilot-pull-request-reviewer[bot]` on `/reviews`, plain `Copilot`
|
|
52
|
-
// on `/pulls/N/comments`; a filter on either returns zero from the other, which is how #105's
|
|
53
|
-
// count was first mis-measured as zero. Matching is therefore a case-insensitive prefix over both
|
|
54
|
-
// surfaces, never an equality against one spelling.
|
|
55
|
-
// * **Our own reviews are on that endpoint too.** The agent identity's replies and derived verdicts
|
|
56
|
-
// are submitted as REVIEWS, so an unfiltered count is inflated by our own traffic: **at merge,
|
|
57
|
-
// seven of #105's fifteen review objects are `portulan-agent[bot]`, and ninety of #342's hundred
|
|
58
|
-
// and two.** Both figures are stamped *at merge* on purpose — the pre-commit checkpoint measured
|
|
59
|
-
// them and found this file carrying *six of fifteen* and *seventy-four of eighty-one*, which were
|
|
60
|
-
// the counts **mid-loop** (74/81 was true at 17:58:40Z on #342, which merged at 18:49:40Z). A
|
|
61
|
-
// hand-counted figure about a subject still growing is the exact defect this whole tool exists to
|
|
62
|
-
// retire, reproduced inside the fix for it.
|
|
63
|
-
// That is also why every read here is **paginated** — page 1 of a busy pull request is measurably
|
|
64
|
-
// stale, and on #342 page 1 carries three reviewer entries while the one on head is the twelfth.
|
|
65
|
-
//
|
|
66
|
-
// **Pushes: as a floor.** Under `review_on_push: true` each push draws one submission, so the
|
|
67
|
-
// distinct heads the reviewer judged are the pushes it saw. It is a floor rather than a count,
|
|
68
|
-
// because a push that drew no review — the ruleset not requesting one, a review still in flight at
|
|
69
|
-
// merge — leaves no trace here, and a force-push destroys the commits it replaced. Printed as a floor
|
|
70
|
-
// and never as a total.
|
|
71
|
-
//
|
|
72
|
-
// **Fix-rounds: NOT AT ALL, and this is the finding rather than a gap to apologise for.** A fix-round
|
|
73
|
-
// is a push that *answers* a submission, and whether a push answers one is a fact about its contents.
|
|
74
|
-
// Two measured demonstrations, both from the pull request that produced the definition:
|
|
75
|
-
//
|
|
76
|
-
// * On #105, `08d7d10` answered review 4's inline finding and **was never a reviewed head** — it
|
|
77
|
-
// rode inside the next push. Enumerating heads cannot see it. The 2026-07-30 ruling states the
|
|
78
|
-
// method that can: *"Count pushes, then look inside each one."*
|
|
79
|
-
// * Also on #105, the push at `cff3e4e0` follows a submission whose body ran to 4,087 bytes and
|
|
80
|
-
// answers none of it — it carried records. A rule keyed on *"a finding-bearing submission
|
|
81
|
-
// preceded this push"* calls that a fix-round; the maintainer's table calls it *no*.
|
|
82
|
-
//
|
|
83
|
-
// So this tool does not compute fix-rounds, does not estimate them, and does not print a figure that
|
|
84
|
-
// could be mistaken for them. The criterion's *"pushes per round"* is reported in the one unit
|
|
85
|
-
// available without adjudication — **pushes per finding-bearing submission** — under that name, with
|
|
86
|
-
// the difference stated in the output rather than left for a reader to assume away.
|
|
87
|
-
//
|
|
88
|
-
// ## The empty-round rate is reported as a BOUND, and the reason is a layering rule
|
|
89
|
-
//
|
|
90
|
-
// The rule's table counts submissions that found **nothing at all**. Finding nothing has two halves:
|
|
91
|
-
// no inline comment thread, *and* no suppressed low-confidence note in the review body. The first
|
|
92
|
-
// half is structural and this tool computes it exactly. The second is decided by a matcher — the awk
|
|
93
|
-
// in `../.github/workflows/copilot-review.yml`, fixtured in
|
|
94
|
-
// `../.portulan/verify/workflow-filters.mjs` — that is deliberately reduced to **one** carrier, and
|
|
95
|
-
// that carrier is a workspace-layer gate while this file is engine.
|
|
96
|
-
//
|
|
97
|
-
// Re-implementing it here would put a second spelling of one rule on the other side of the
|
|
98
|
-
// engine/workspace boundary, where neither could see the other drift. That is this repository's
|
|
99
|
-
// signature defect and `../.portulan/proposals/0027` exists to refuse it. So the tool reports what it
|
|
100
|
-
// can compute exactly and names the relation:
|
|
101
|
-
//
|
|
102
|
-
// submissions that found nothing ≤ submissions with no inline comment
|
|
103
|
-
//
|
|
104
|
-
// The right-hand side is what this prints, under its own name, as an **upper bound**.
|
|
105
|
-
//
|
|
106
|
-
// **What the layering rule does NOT establish is that the exact rate is out of reach**, and saying so
|
|
107
|
-
// is the difference between a limit and an excuse. The tree already holds a lift-and-run consumer of
|
|
108
|
-
// these very programs: `../.portulan/verify/workflow-filters.mjs` extracts each single-quoted awk
|
|
109
|
-
// program out of the workflow's parsed `run:` scalars and executes it through the real `awk`. `--fetch`
|
|
110
|
-
// already spawns, so a body could be piped through the **lifted** program at capture time and stored
|
|
111
|
-
// as one integer — no second spelling, and still no bodies in the snapshot. That is the closing move,
|
|
112
|
-
// it is a workspace-side consumer rather than a copy, and it is **not built here for budget** (one
|
|
113
|
-
// clause, one session) rather than because the boundary forbids it. Tracked as
|
|
114
|
-
// https://github.com/sleepy-panda-srl/portulan/issues/355 — filed, and the number is here because
|
|
115
|
-
// "filed rather than built" naming no filing is a claim in the past tense about an issue that does
|
|
116
|
-
// not exist, which this repository has already shipped once.
|
|
117
|
-
//
|
|
118
|
-
// ## What this tool is NOT
|
|
119
|
-
//
|
|
120
|
-
// **It is a meter, not a bound.** Rule 4 of the record above stops a loop at two fix-rounds; nothing
|
|
121
|
-
// here stops anything. The record's own honest-limits section already says the judgement it depends
|
|
122
|
-
// on is the interested party's about its own work, and a tool that reports after the merge does not
|
|
123
|
-
// change that. Nor does it adjudicate rule 4's **sibling** exemption, which is a judgement about
|
|
124
|
-
// whether one finding's governing rule was already enforced at another site — so *rounds past the
|
|
125
|
-
// bound* is not computable and is not claimed.
|
|
126
|
-
//
|
|
127
|
-
// **It reports on a snapshot, never on the network.** The computation takes data and the fetch is a
|
|
128
|
-
// separate mode, for the reason `./goldens.mjs` takes fixtures: a figure a verify recipe rails must be
|
|
129
|
-
// re-derivable from the tree, on a machine with no token and no network, and must not move because
|
|
130
|
-
// GitHub was slow. `--fetch` is the only mode that talks to anything.
|
|
4
|
+
// Fix-rounds are not derivable from the API: whether a push answers a submission is a fact about its contents.
|
|
131
5
|
|
|
132
6
|
import { spawnSync } from "node:child_process";
|
|
133
7
|
import fs from "node:fs";
|
|
@@ -136,27 +10,20 @@ import { pathToFileURL } from "node:url";
|
|
|
136
10
|
|
|
137
11
|
export const SNAPSHOT_VERSION = "1";
|
|
138
12
|
|
|
139
|
-
//
|
|
140
|
-
// spellings — `copilot-pull-request-reviewer[bot]` and `Copilot` — start with it, and a third
|
|
141
|
-
// spelling of the same actor would too. The cost of the looser test is that a human login beginning
|
|
142
|
-
// "copilot" would be counted; the cost of the tighter one was a whole round read as missing (#154).
|
|
13
|
+
// A prefix, not an equality: the reviewer is `copilot-pull-request-reviewer[bot]` on reviews and `Copilot` on comments.
|
|
143
14
|
export const REVIEWER_PREFIX = "copilot";
|
|
144
15
|
|
|
145
|
-
// The threshold the
|
|
146
|
-
// 2.0 for a full milestone*. Carried here as a number so the tool can say which side of it a window
|
|
147
|
-
// falls on; the record remains the authority on what crossing it means.
|
|
16
|
+
// The retirement threshold the review-loop rule stated: submissions per pull request below this for a full milestone.
|
|
148
17
|
export const RETIRE_THRESHOLD = 2.0;
|
|
149
18
|
|
|
150
19
|
const isReviewer = (login) =>
|
|
151
20
|
typeof login === "string" && login.toLowerCase().startsWith(REVIEWER_PREFIX);
|
|
152
21
|
|
|
153
22
|
// ---------------------------------------------------------------------------------------------
|
|
154
|
-
// The computation
|
|
155
|
-
// a file, runs a process, or knows what year it is.
|
|
23
|
+
// The computation: pure, a snapshot in and numbers out
|
|
156
24
|
// ---------------------------------------------------------------------------------------------
|
|
157
25
|
|
|
158
|
-
//
|
|
159
|
-
// distinct heads those reviews judged, which is a floor for the same reason stated in the header.
|
|
26
|
+
// `pushes` is a floor: a push no review judged leaves no trace here.
|
|
160
27
|
export function meterPullRequest(pr) {
|
|
161
28
|
const submissions = pr.submissions ?? [];
|
|
162
29
|
const heads = new Set();
|
|
@@ -174,9 +41,7 @@ export function meterPullRequest(pr) {
|
|
|
174
41
|
};
|
|
175
42
|
}
|
|
176
43
|
|
|
177
|
-
//
|
|
178
|
-
// empty: a corpus with no finding-bearing submission has no pushes-per-submission, and printing 0
|
|
179
|
-
// would be a claim about a loop nobody ran. `verify-preconditions-fail-closed` in prose.
|
|
44
|
+
// A ratio over an empty denominator is null, never 0: an unmeasured loop is not a measured zero.
|
|
180
45
|
export function meter(snapshot) {
|
|
181
46
|
const perPullRequest = (snapshot.pullRequests ?? []).map(meterPullRequest);
|
|
182
47
|
const total = (key) => perPullRequest.reduce((sum, p) => sum + p[key], 0);
|
|
@@ -200,28 +65,9 @@ export function meter(snapshot) {
|
|
|
200
65
|
submissionsPerPullRequest: ratio(submissions, pullRequests),
|
|
201
66
|
noInlineRate: ratio(noInline, submissions),
|
|
202
67
|
pushesPerPullRequest: ratio(pushes, pullRequests),
|
|
203
|
-
// **The criterion's own "pushes per round" in submission units, stated rather than inferred.**
|
|
204
|
-
// Under `review_on_push: true` it is 1.00 by construction, which is why the row below it exists
|
|
205
|
-
// at all — but a reader owed the criterion's literal figure should not have to derive it from a
|
|
206
|
-
// paragraph about a coincidence. The pre-commit checkpoint asked for it in as many words.
|
|
207
68
|
pushesPerSubmission: ratio(pushes, submissions),
|
|
208
69
|
pushesPerFindingBearingSubmission: ratio(pushes, findingBearing),
|
|
209
|
-
// **Is the push figure carrying any information the submission figure does not?**
|
|
210
|
-
//
|
|
211
|
-
// Under `review_on_push: true` a push draws a submission, so the two coincide whenever every
|
|
212
|
-
// submission judged its own head — and when they do, `pushesPerFindingBearingSubmission`
|
|
213
|
-
// is not a second measurement at all: it reduces to `1 / (1 - noInlineRate)`, an identity.
|
|
214
|
-
// Measured 2026-08-26 over the thirty most recently merged pull requests: 140 submissions and
|
|
215
|
-
// 140 pushes, exactly equal, and 140/46 = 3.04 = 1/(1 - 0.671).
|
|
216
|
-
//
|
|
217
|
-
// Three figures of which two are algebraically the same figure is a table that reads as more
|
|
218
|
-
// evidence than it holds. So the coincidence is DETECTED and stated rather than left for a
|
|
219
|
-
// reader to notice, which is the same discipline `./goldens.mjs` applies when it prints the
|
|
220
|
-
// per-path census including the zeroes.
|
|
221
70
|
pushesCoincideWithSubmissions: pushes === submissions,
|
|
222
|
-
// Which side of the record's own retirement threshold this window falls on. `null` where the
|
|
223
|
-
// ratio is null — an unmeasured window is not a window measuring below the threshold, and
|
|
224
|
-
// conflating the two is how a rule gets retired on an empty corpus.
|
|
225
71
|
belowRetireThreshold:
|
|
226
72
|
ratio(submissions, pullRequests) === null
|
|
227
73
|
? null
|
|
@@ -231,8 +77,7 @@ export function meter(snapshot) {
|
|
|
231
77
|
}
|
|
232
78
|
|
|
233
79
|
// ---------------------------------------------------------------------------------------------
|
|
234
|
-
// The snapshot's
|
|
235
|
-
// "could not judge" — and never exit 1, which would report a review loop that nobody measured.
|
|
80
|
+
// The snapshot's shape, checked before metering: malformed is exit 2, never 1
|
|
236
81
|
// ---------------------------------------------------------------------------------------------
|
|
237
82
|
|
|
238
83
|
export function validateSnapshot(snapshot) {
|
|
@@ -256,19 +101,6 @@ export function validateSnapshot(snapshot) {
|
|
|
256
101
|
problems.push("pullRequests is not an array");
|
|
257
102
|
return problems;
|
|
258
103
|
}
|
|
259
|
-
// The `window` is printed in the register as a claim about what was sampled, so a window that
|
|
260
|
-
// disagrees with the corpus beneath it is a false heading over true figures. Measured at the
|
|
261
|
-
// pre-commit checkpoint: `{merged: 300}` over thirty pull requests rendered *"300 most recently
|
|
262
|
-
// merged"* above *"| Pull requests | count | 30 |"*, rail green.
|
|
263
|
-
//
|
|
264
|
-
// **Zero is a legal window and the floor here used to be one.** A repository with no merged pull
|
|
265
|
-
// requests yet has an empty review loop, which is a true measurement rather than a failed one —
|
|
266
|
-
// and `meter()` already answers it correctly, returning `null` for every ratio and `null` for the
|
|
267
|
-
// retirement verdict, because *"an unmeasured window is not a window measuring below the
|
|
268
|
-
// threshold"*. Refusing it made this tool reject a snapshot **its own `--fetch` can write**, which
|
|
269
|
-
// is a validator disagreeing with its own producer. Copilot round 2 on #357, arriving through the
|
|
270
|
-
// suppressed channel — the half no gate sees until this repository's own promotion step makes it
|
|
271
|
-
// a thread.
|
|
272
104
|
if (!Number.isInteger(snapshot.window?.merged) || snapshot.window.merged < 0) {
|
|
273
105
|
problems.push(
|
|
274
106
|
`window.merged is ${JSON.stringify(snapshot.window?.merged)}; the register prints it as the size of the sample`,
|
|
@@ -279,11 +111,6 @@ export function validateSnapshot(snapshot) {
|
|
|
279
111
|
);
|
|
280
112
|
}
|
|
281
113
|
const seen = new Set();
|
|
282
|
-
// **The window is BY MERGE DATE, and the snapshot must prove it.** `gh pr list` orders by pull
|
|
283
|
-
// request NUMBER, and the two disagree: the first capture taken here carried three merge-order
|
|
284
|
-
// inversions and sampled a corpus that was not the one its own register named — found at the
|
|
285
|
-
// pre-commit checkpoint, from evidence inside the committed snapshot. A rail that only checked
|
|
286
|
-
// the arithmetic would have re-published the same wrong window every time it was regenerated.
|
|
287
114
|
let previous = null;
|
|
288
115
|
for (const pr of snapshot.pullRequests) {
|
|
289
116
|
if (!Number.isInteger(pr?.number)) {
|
|
@@ -292,12 +119,6 @@ export function validateSnapshot(snapshot) {
|
|
|
292
119
|
}
|
|
293
120
|
if (seen.has(pr.number)) problems.push(`pull request ${pr.number} appears twice`);
|
|
294
121
|
seen.add(pr.number);
|
|
295
|
-
// **`mergedAt` is parsed, not compared as text.** It was compared lexicographically, which is
|
|
296
|
-
// right for ISO-8601 and right for nothing else: any string orders against any other string,
|
|
297
|
-
// so a snapshot carrying `"yesterday"` or a `DD/MM/YYYY` stamp would have ordered cleanly and
|
|
298
|
-
// the register would still have claimed the window was *by merge date*. A check that cannot
|
|
299
|
-
// fail on a malformed input is not checking that input. Copilot round 2 on #357, through the
|
|
300
|
-
// suppressed channel.
|
|
301
122
|
const merged = typeof pr.mergedAt === "string" ? Date.parse(pr.mergedAt) : Number.NaN;
|
|
302
123
|
if (Number.isNaN(merged)) {
|
|
303
124
|
problems.push(
|
|
@@ -319,9 +140,6 @@ export function validateSnapshot(snapshot) {
|
|
|
319
140
|
}
|
|
320
141
|
for (const s of pr.submissions) {
|
|
321
142
|
if (!isReviewer(s?.login)) {
|
|
322
|
-
// A snapshot carrying a non-reviewer review means the fetch filter drifted, and the
|
|
323
|
-
// figures would be inflated by our own traffic exactly as the header describes. That
|
|
324
|
-
// is a snapshot this tool cannot judge, not a loop with a high count.
|
|
325
143
|
problems.push(
|
|
326
144
|
`pull request ${pr.number} carries a submission by ${JSON.stringify(s?.login)}, which is not the reviewer`,
|
|
327
145
|
);
|
|
@@ -331,11 +149,6 @@ export function validateSnapshot(snapshot) {
|
|
|
331
149
|
`pull request ${pr.number} has a submission with no inline count: ${JSON.stringify(s?.inline)}`,
|
|
332
150
|
);
|
|
333
151
|
}
|
|
334
|
-
// **`head` is contracted exactly as `inline` is, and for the same sentence.** It was not,
|
|
335
|
-
// and the gap was measured at the pre-commit checkpoint: strip every `head` and the tool
|
|
336
|
-
// exits 0 printing `pushes 0` and regenerates a register carrying that zero — which is
|
|
337
|
-
// what `meter()`'s own comment forbids one field over, *"printing 0 would be a claim about
|
|
338
|
-
// a loop nobody ran"*. Two of the criterion's three metrics rest on this field.
|
|
339
152
|
if (typeof s?.head !== "string" || s.head.length === 0) {
|
|
340
153
|
problems.push(
|
|
341
154
|
`pull request ${pr.number} has a submission with no head sha: ${JSON.stringify(s?.head)} — pushes are counted from it`,
|
|
@@ -347,12 +160,7 @@ export function validateSnapshot(snapshot) {
|
|
|
347
160
|
}
|
|
348
161
|
|
|
349
162
|
// ---------------------------------------------------------------------------------------------
|
|
350
|
-
// The register: the figures as a committed document, regenerated and byte-compared
|
|
351
|
-
//
|
|
352
|
-
// The point of writing them down at all is that a snapshot is JSON nobody reads, and the figures are
|
|
353
|
-
// quoted by a rule. The point of byte-comparing them is that a published figure which can drift from
|
|
354
|
-
// its own data is the hand-maintained tally in a new costume — `./index.mjs` holds the memory index
|
|
355
|
-
// this way for the same reason.
|
|
163
|
+
// The register: the figures as a committed document, regenerated and byte-compared
|
|
356
164
|
// ---------------------------------------------------------------------------------------------
|
|
357
165
|
|
|
358
166
|
const round2 = (n) => (n === null ? "—" : (Math.round(n * 100) / 100).toFixed(2));
|
|
@@ -366,10 +174,14 @@ export function renderRegister(m) {
|
|
|
366
174
|
lines.push("> it is regenerated and byte-compared, so a hand-edit survives exactly until the next run.");
|
|
367
175
|
lines.push(">");
|
|
368
176
|
lines.push("> **Every figure here is in SUBMISSION units** — every review the reviewer submits, one");
|
|
369
|
-
lines.push("> per push, including on the branch as opened. It is not the fix-round unit that");
|
|
370
|
-
lines.push(">
|
|
371
|
-
lines.push(">
|
|
372
|
-
lines.push(">
|
|
177
|
+
lines.push("> per push, including on the branch as opened. It is not the fix-round unit that rule 4");
|
|
178
|
+
lines.push("> of the retired `a-review-loop-needs-a-bound.md` bounded, and no figure here may be read");
|
|
179
|
+
lines.push("> as one. See `../../cli/review-meter.mjs` for why fix-rounds are not derivable from the");
|
|
180
|
+
lines.push("> API at all.");
|
|
181
|
+
lines.push(">");
|
|
182
|
+
lines.push("> **This register is historical.** It measures the Copilot review loop, which left this");
|
|
183
|
+
lines.push("> repository's review path on 2026-10-30, by the maintainer's decision; the record that");
|
|
184
|
+
lines.push("> bounded the loop retired with it.");
|
|
373
185
|
lines.push("");
|
|
374
186
|
lines.push(`- **Repository:** \`${m.repository}\``);
|
|
375
187
|
lines.push(`- **Captured:** ${m.captured}`);
|
|
@@ -402,8 +214,9 @@ export function renderRegister(m) {
|
|
|
402
214
|
lines.push("## Against the record's own retirement threshold");
|
|
403
215
|
lines.push("");
|
|
404
216
|
lines.push(
|
|
405
|
-
`\`a-review-loop-needs-a-bound.md\`
|
|
406
|
-
`**${RETIRE_THRESHOLD.toFixed(1)}** for a full milestone
|
|
217
|
+
`\`a-review-loop-needs-a-bound.md\` was to retire when submissions per pull request measured below ` +
|
|
218
|
+
`**${RETIRE_THRESHOLD.toFixed(1)}** for a full milestone, and retired on 2026-10-30 on its other ` +
|
|
219
|
+
`condition, Copilot review leaving the review path. This window measures ` +
|
|
407
220
|
`**${round2(m.submissionsPerPullRequest)}**, which is ` +
|
|
408
221
|
`${m.belowRetireThreshold === null ? "unmeasured" : m.belowRetireThreshold ? "**below**" : "**at or above**"} it.`,
|
|
409
222
|
);
|
|
@@ -413,7 +226,7 @@ export function renderRegister(m) {
|
|
|
413
226
|
"field the API carries, so this tool measures a window of merged pull requests and the " +
|
|
414
227
|
"record's *for a full milestone* clause is not evaluated here. Reading this row as the " +
|
|
415
228
|
"retirement condition met would be reading a different measure than the one the record " +
|
|
416
|
-
"
|
|
229
|
+
"stated.",
|
|
417
230
|
);
|
|
418
231
|
lines.push("");
|
|
419
232
|
lines.push("## Per pull request");
|
|
@@ -438,21 +251,9 @@ const gh = (args) => {
|
|
|
438
251
|
return out.stdout;
|
|
439
252
|
};
|
|
440
253
|
|
|
441
|
-
//
|
|
442
|
-
// orders by pull request NUMBER, and number order is not merge order: the first capture taken here
|
|
443
|
-
// carried three inversions — #346 before #345, #343 before #342, #312 before #310 — so the sampled
|
|
444
|
-
// corpus contained #303 and excluded #301, while the register above it claimed *"the 30 most recently
|
|
445
|
-
// merged"*. Every published figure was therefore against a corpus its own heading did not name.
|
|
446
|
-
// Found at the pre-commit checkpoint, from evidence inside the committed snapshot: `mergedAt` was
|
|
447
|
-
// already being captured and nothing was sorting on it.
|
|
448
|
-
//
|
|
449
|
-
// So a POOL is listed and the window is taken from it by merge date. Sorted descending, ties broken by
|
|
450
|
-
// number descending so the order is total and a re-capture of the same data is byte-stable.
|
|
254
|
+
// By merge date, which `gh pr list`'s number order is not; ties by number, so a re-capture is byte-stable.
|
|
451
255
|
export function selectWindow(listed, limit) {
|
|
452
|
-
//
|
|
453
|
-
// producer and the validator must order by the same relation, or the fetch can write a window the
|
|
454
|
-
// validator then refuses. An unparsable stamp sorts last rather than throwing — the validator is
|
|
455
|
-
// where it is reported, and a fetch that crashed on one bad row would lose the other twenty-nine.
|
|
256
|
+
// An unparsable stamp sorts last rather than throwing: the validator reports it, and the other rows still land.
|
|
456
257
|
const at = (x) => {
|
|
457
258
|
const t = Date.parse(x?.mergedAt);
|
|
458
259
|
return Number.isNaN(t) ? -Infinity : t;
|
|
@@ -462,9 +263,7 @@ export function selectWindow(listed, limit) {
|
|
|
462
263
|
.slice(0, limit);
|
|
463
264
|
}
|
|
464
265
|
|
|
465
|
-
//
|
|
466
|
-
// the two surfaces together across the login split. Pure, so the grouping and the login filter are
|
|
467
|
-
// testable without a network: they are where two of the measured traps live.
|
|
266
|
+
// `pull_request_review_id` ties an inline comment to its review across the two logins.
|
|
468
267
|
export function shapeSubmissions(reviews, comments) {
|
|
469
268
|
const inlineByReview = new Map();
|
|
470
269
|
for (const c of comments) {
|
|
@@ -478,35 +277,25 @@ export function shapeSubmissions(reviews, comments) {
|
|
|
478
277
|
id: r.id,
|
|
479
278
|
login: r.user.login,
|
|
480
279
|
state: r.state,
|
|
481
|
-
//
|
|
482
|
-
// read from the REVIEW and never from a comment. That a review's own `commit_id` holds
|
|
483
|
-
// still is an assumption with an open issue on it — #253, and the falsifier is stated at
|
|
484
|
-
// length in this file's header rather than only here.
|
|
280
|
+
// From the review, never a comment: a comment's `commit_id` drifts onto later heads.
|
|
485
281
|
head: r.commit_id,
|
|
486
282
|
at: r.submitted_at,
|
|
487
283
|
inline: inlineByReview.get(r.id) ?? 0,
|
|
488
284
|
}));
|
|
489
285
|
}
|
|
490
286
|
|
|
491
|
-
// `--paginate` on both surfaces
|
|
492
|
-
// here: on #342 ninety of a hundred and two review objects at merge were our own, so an unpaginated
|
|
493
|
-
// read returns three reviewer entries and the one on head is the twelfth.
|
|
287
|
+
// `--paginate` on both surfaces: page 1 of a busy pull request misses its latest reviews.
|
|
494
288
|
export function fetchSnapshot({ repository, limit, pool, now }) {
|
|
495
289
|
const listed = JSON.parse(
|
|
496
290
|
gh(["pr", "list", "--repo", repository, "--state", "merged", "--limit", String(pool), "--json", "number,mergedAt"]),
|
|
497
291
|
);
|
|
498
|
-
// **A saturated pool cannot prove the window.** Where the listing came back short of the pool it
|
|
499
|
-
// is every merged pull request there is, and the window is provably the newest N; where it came
|
|
500
|
-
// back full, an older-numbered pull request merged recently could sit outside it. Recorded rather
|
|
501
|
-
// than assumed away, so a reader can tell which of the two they are holding.
|
|
502
292
|
const saturated = listed.length >= pool;
|
|
503
293
|
const window = selectWindow(listed, limit);
|
|
504
294
|
const pullRequests = [];
|
|
505
295
|
for (const { number, mergedAt } of window) {
|
|
506
296
|
const reviews = JSON.parse(gh(["api", "--paginate", `repos/${repository}/pulls/${number}/reviews`]));
|
|
507
297
|
const comments = JSON.parse(gh(["api", "--paginate", `repos/${repository}/pulls/${number}/comments`]));
|
|
508
|
-
// No bodies
|
|
509
|
-
// content into a committed file for no measurement's sake.
|
|
298
|
+
// No bodies: review prose has no place in a committed snapshot.
|
|
510
299
|
pullRequests.push({ number, mergedAt, submissions: shapeSubmissions(reviews, comments) });
|
|
511
300
|
}
|
|
512
301
|
return {
|
|
@@ -577,12 +366,6 @@ export function run(argv = process.argv.slice(2), io = console) {
|
|
|
577
366
|
io.error("review-meter: --check and --write ask for opposite things; pick one");
|
|
578
367
|
return 2;
|
|
579
368
|
}
|
|
580
|
-
// **`--check` and `--write` are about the REGISTER, so without one they asked for nothing and got
|
|
581
|
-
// exit 0.** A mistyped invocation — the flag typed, the path forgotten — printed the report and
|
|
582
|
-
// returned green, so a person believing they had byte-compared the register had run no comparison
|
|
583
|
-
// at all. That is a false green in the one place this tool exists to remove one, and it is the same
|
|
584
|
-
// shape as the entry-guard defect further down this file: the command runs, says nothing is wrong,
|
|
585
|
-
// and did not do the thing. Copilot round 1 on #357.
|
|
586
369
|
if ((opts.check || opts.write) && !opts.register) {
|
|
587
370
|
io.error(`review-meter: ${opts.check ? "--check" : "--write"} needs --register <file> — it is the register that is written and compared`);
|
|
588
371
|
io.error("Without it this flag would do nothing and still exit 0.");
|
|
@@ -598,12 +381,6 @@ export function run(argv = process.argv.slice(2), io = console) {
|
|
|
598
381
|
io.error(`review-meter: --limit must be a positive integer, not ${JSON.stringify(opts.limit)}`);
|
|
599
382
|
return 2;
|
|
600
383
|
}
|
|
601
|
-
// **Strictly greater, and the check used to say `<` while the prose beneath it argued `<=`.**
|
|
602
|
-
// A pool the size of the window cannot contain a pull request outside the window, so it is
|
|
603
|
-
// number order wearing the window's name — which is the defect the pool exists to remove. The
|
|
604
|
-
// guard permitted it anyway, so the suite's own case for this refusal fell straight through
|
|
605
|
-
// into the real fetch path and passed on `gh` failing instead. Copilot round 3 on #357: a test
|
|
606
|
-
// passing for the wrong reason, the same class round 1 found one site over.
|
|
607
384
|
if (!Number.isInteger(opts.pool) || opts.pool <= opts.limit) {
|
|
608
385
|
io.error(`review-meter: --pool must be an integer greater than --limit (${opts.limit}), not ${JSON.stringify(opts.pool)}`);
|
|
609
386
|
io.error("The pool is listed by pull request NUMBER and the window is taken from it by merge date,");
|
|
@@ -673,9 +450,6 @@ export function run(argv = process.argv.slice(2), io = console) {
|
|
|
673
450
|
io.log(" independent measurement — it is 1 / (1 - the no-inline rate). Two rows, one figure.");
|
|
674
451
|
}
|
|
675
452
|
|
|
676
|
-
// The limits are printed on every run rather than left in a README, for the reason
|
|
677
|
-
// `./goldens.mjs` prints its presence-versus-adequacy limit on every green: an exit code that
|
|
678
|
-
// implies more than it means is how a figure gets quoted as something it is not.
|
|
679
453
|
io.log("");
|
|
680
454
|
io.log(" Every figure above is in SUBMISSION units and none of them is a fix-round count.");
|
|
681
455
|
io.log(" Fix-rounds are not derivable here: a fix can ride inside another push, and a records");
|
|
@@ -715,15 +489,7 @@ export function run(argv = process.argv.slice(2), io = console) {
|
|
|
715
489
|
return 0;
|
|
716
490
|
}
|
|
717
491
|
|
|
718
|
-
//
|
|
719
|
-
// form, and this file shipped that spelling for exactly one measurement: `--fetch` against the live
|
|
720
|
-
// repository printed nothing, exited 0, and wrote no snapshot, because `import.meta.url`
|
|
721
|
-
// percent-encodes and this working copy lives under a path with spaces. **A green that is the tool
|
|
722
|
-
// never starting** — the fourth time this repository has met it, and the reason the form is copied
|
|
723
|
-
// rather than re-derived.
|
|
724
|
-
//
|
|
725
|
-
// The realpath fallback covers the symlink an npm `bin` produces, in a `try` because a missing path
|
|
726
|
-
// must answer no rather than throw.
|
|
492
|
+
// As file URLs, since `import.meta.url` percent-encodes a space; through realpath too, for an npm `bin` symlink.
|
|
727
493
|
function isMain() {
|
|
728
494
|
const invoked = process.argv[1];
|
|
729
495
|
if (!invoked) return false;
|
|
@@ -735,7 +501,5 @@ function isMain() {
|
|
|
735
501
|
}
|
|
736
502
|
}
|
|
737
503
|
|
|
738
|
-
// `process.exitCode` rather than `process.exit`,
|
|
739
|
-
// outright can truncate a pipe that has not drained, and a truncated line IS exit 0 with no output —
|
|
740
|
-
// the precise shape of the false green the guard above was fixed for.
|
|
504
|
+
// `process.exitCode` rather than `process.exit`, so a pipe that has not drained is not cut short.
|
|
741
505
|
if (isMain()) process.exitCode = run(process.argv.slice(2));
|