@lumoai/cli 1.47.0 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/SKILL.md +40 -0
- package/assets/skill/references/criteria.md +25 -3
- package/assets/skill/references/fidelity.md +32 -0
- package/assets/skill/references/outcome.md +65 -0
- package/assets/skill/references/verify.md +12 -11
- package/dist/cli/src/commands/criteria-audit.js +52 -0
- package/dist/cli/src/commands/fidelity.js +108 -0
- package/dist/cli/src/commands/outcome.js +225 -0
- package/dist/cli/src/commands/task-criteria-list.js +8 -1
- package/dist/cli/src/commands/task-criteria-set.js +40 -1
- package/dist/cli/src/index.js +38 -0
- package/dist/shared/src/referent-kind.js +321 -0
- package/package.json +1 -1
package/assets/skill/SKILL.md
CHANGED
|
@@ -31,6 +31,8 @@ The command catalog below is a **map**: it lists every command grouped by domain
|
|
|
31
31
|
| `task criteria set/list`, drafting the acceptance contract | [references/criteria.md](references/criteria.md) |
|
|
32
32
|
| `verify`, `task status` — machine verification loop, claim-done flow, self-check/resume | [references/verify.md](references/verify.md) |
|
|
33
33
|
| `cost` — per-operation (per-tool) token cost read-out; `task lineage` Top-5 | [references/task-context.md](references/task-context.md) |
|
|
34
|
+
| `outcome show/record/rate` — post-hoc outcome well (falsifier oracle, rejection-only) | [references/outcome.md](references/outcome.md) |
|
|
35
|
+
| `fidelity show` — per-delivery grounding + independence change-pattern read-out | [references/fidelity.md](references/fidelity.md) |
|
|
34
36
|
| `project list`, `milestone*` | [references/milestones.md](references/milestones.md) |
|
|
35
37
|
| `doc create/update/list/move/bind/share/import` (CRUD) | [references/docs.md](references/docs.md) |
|
|
36
38
|
| `doc show --raw/--section`, `doc patch/append/diff/rebuild-source` (editing live docs) | [references/doc-editing.md](references/doc-editing.md) |
|
|
@@ -92,6 +94,25 @@ The command catalog below is a **map**: it lists every command grouped by domain
|
|
|
92
94
|
|
|
93
95
|
- `lumo cost [--task <id>|--session <id>|--since <date>] [--by tool|model|member|session] [--json]` — per-operation token cost read-out, attributing each model step's token delta to the tool(s) it ran. Scope is mutually exclusive (default = workspace last-30-days). For the per-task Top-5 inline, see `lumo task lineage`.
|
|
94
96
|
|
|
97
|
+
**Criteria audit** — workspace-level referent-kind analytics
|
|
98
|
+
|
|
99
|
+
- `lumo criteria audit` — workspace-level read-out: count of acceptance criteria by effective referent kind (EXTERNAL_FACT / AGENT_CONSTRUCTED_STATE / PENDING_OUTCOME / UNVERIFIED_ASSERTION / UNCLASSIFIED) + the self-confirming-green ratio
|
|
100
|
+
|
|
101
|
+
**Outcome well (post-hoc oracle)** — see [outcome.md](references/outcome.md)
|
|
102
|
+
|
|
103
|
+
- `lumo outcome show <task>` — read the post-hoc outcome well: the **falsifier** verdict (`REJECTED` | `INCONCLUSIVE` — never a "pass") plus its backing rejection signals. `INCONCLUSIVE` = no rejection on record, **not** "satisfied".
|
|
104
|
+
- `lumo outcome record <task> --note "<what reality did>" [--kind <kind>]` — record a human-observed post-hoc **rejection** of a delivery (revert / rollback / CI regression / downstream redirect / bypass). Append-only; **there is no "mark satisfied" counterpart** — the well only asserts rejection. `--note` is required (the observed referent).
|
|
105
|
+
- `lumo outcome rate [--min N] [--json]` — workspace **trust × post-hoc-fate** join (LUM-603, LUM-606): per delivery-time **verification-friction** bucket (`no-verification` / `single-round` / `multi-round-or-sendback` — two raw counts frozen at delivery, replacing the saturating forecast-confidence axis), the post-hoc REJECTED rate from the well. Honest by construction — thin buckets read `insufficient` (no fabricated rate), the single-round-vs-multi comparison stays `inconclusive` until CIs separate, and the rate is a **lower bound** (no signal = INCONCLUSIVE, never satisfied). Bucket names are value-neutral — direction is decided by the correlation, never welded into the label. Under-sampled today by design — the skeleton produces real strata once the well fills.
|
|
106
|
+
|
|
107
|
+
**Fidelity read-out (mechanical change-pattern)** — see [fidelity.md](references/fidelity.md)
|
|
108
|
+
|
|
109
|
+
- `lumo fidelity show <task> [--json]` — per delivery snapshot (LUM-609), a **composition table, not a single score**: (1) the **grounding** of the frozen contract — grounded (`EXTERNAL_FACT`, a machine-checkable upper bound) / self-confirming (`AGENT_CONSTRUCTED_STATE` + `UNVERIFIED_ASSERTION`) / inconclusive (`PENDING_OUTCOME` + `UNCLASSIFIED`, surfaced not swallowed) + the full effective-kind tally; (2) the **independence** signal — criteria `ADDED`/`UPDATED` at round>0 (after work started) that the delivery rode on = backward-inference suspects. Purely mechanical (timing + op + grounding) — **no direction verdict**. A no-snapshot task is reported explicitly (not a pass). Zero new storage — reads the snapshot + `CRITERION_CHANGED` trail.
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
lumo fidelity show LUM-42
|
|
113
|
+
lumo fidelity show LUM-42 --json
|
|
114
|
+
```
|
|
115
|
+
|
|
95
116
|
**Artifacts & Figma** — see [artifacts-figma.md](references/artifacts-figma.md)
|
|
96
117
|
|
|
97
118
|
- `lumo task artifact add/update/list/show/rm` — record spec/plan products on a task
|
|
@@ -146,6 +167,25 @@ The command catalog below is a **map**: it lists every command grouped by domain
|
|
|
146
167
|
- `lumo worktree rm <LUM-N> --yes` — remove a worktree (keeps the branch unless `--delete-branch`)
|
|
147
168
|
- `lumo worktree list` — list `.worktrees/` worktrees (task id, branch, dirty, node_modules link)
|
|
148
169
|
|
|
170
|
+
## Criteria audit
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
lumo criteria audit
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
`lumo criteria audit` — workspace-level acceptance-criteria analytics. Calls `GET /api/criteria/audit` and prints two things:
|
|
177
|
+
|
|
178
|
+
1. **Declared-kind distribution** — count of criteria by effective referent kind:
|
|
179
|
+
- `EXTERNAL_FACT` — declared as externally observable (git SHA, CI status, DB row) and the checkpointer invokes real external tooling
|
|
180
|
+
- `AGENT_CONSTRUCTED_STATE` — passes by checking output the agent itself produced
|
|
181
|
+
- `PENDING_OUTCOME` — deferred to post-hoc falsifier
|
|
182
|
+
- `UNVERIFIED_ASSERTION` — declared `EXTERNAL_FACT` but the checkpointer does **not** actually invoke external tooling (git/CI/HTTP); auto-downgraded. As of LUM-605 this is an **enforced gate, not just a label**: `lumo task criteria set` warns on it, and `lumo verify` blocks a PASS on it with **422 (no round burned)** until it's re-grounded or honestly re-declared `AGENT_CONSTRUCTED_STATE` (see references/criteria.md "referentKind")
|
|
183
|
+
- `UNCLASSIFIED` — legacy rows predating the `referentKind` field (null in DB)
|
|
184
|
+
|
|
185
|
+
2. **Self-confirming-green headline ratio** — `(AGENT_CONSTRUCTED_STATE + UNVERIFIED_ASSERTION) / classified`. Criteria in these two buckets pass by confirming only the agent's own work, not an independent external state.
|
|
186
|
+
|
|
187
|
+
**When to suggest**: when the user wants to audit how much of the workspace's "green" criteria are self-confirming vs externally anchored; when diagnosing whether acceptance criteria have real external hooks or are circular.
|
|
188
|
+
|
|
149
189
|
## Commands & flags that do NOT exist (common mistakes)
|
|
150
190
|
|
|
151
191
|
Don't guess these:
|
|
@@ -60,6 +60,21 @@ The 3–7 range is calibrated for typical multi-file tasks. Criterion count must
|
|
|
60
60
|
|
|
61
61
|
### judgeSteps — agent-drafted judging steps for HUMAN criteria
|
|
62
62
|
|
|
63
|
+
**`referentKind` — what the criterion anchors (required, and a real gate as of LUM-605):**
|
|
64
|
+
|
|
65
|
+
Every criterion declares one `referentKind` — the only dimension that decides whether a PASS is independent or self-confirming. Required on every item (the API rejects a missing one with 400):
|
|
66
|
+
|
|
67
|
+
- `EXTERNAL_FACT` — anchored on a fact the agent cannot author: a git SHA / diff, CI status, an HTTP probe, a DB row, `prisma migrate status`. **Only honest when the checkpointer actually invokes external tooling** (`git`/`gh`/`curl`/`psql`/…) AND its output or exit status reaches the verdict (piped into an assertion, captured in `$(…)`, or the terminal command whose exit IS the verdict). A no-op or output-discarded external command (`git --version && cat frozen.txt | grep`) does **not** count.
|
|
68
|
+
- `AGENT_CONSTRUCTED_STATE` — passes by checking output the agent itself produced: **your own jest/vitest/tsc/node/tsx tests, build, type-check**. This is the honest kind for almost every code task's checks. Green here is self-confirming — that's fine, just label it truthfully.
|
|
69
|
+
- `PENDING_OUTCOME` — truth lands post-delivery (a downstream regression, a revert). Has no check today: it **cannot** be MACHINE (the API rejects PENDING_OUTCOME + MACHINE). Draft it HUMAN or leave it out of the encoded contract.
|
|
70
|
+
|
|
71
|
+
**The gate (LUM-605):** declaring `EXTERNAL_FACT` + MACHINE on a checkpointer that doesn't ground external (e.g. `npx jest …`) is the LUM-601 fakeGit shape — it would launder a self-confirming green into a "verified external fact". The system now downgrades it to `UNVERIFIED_ASSERTION` and:
|
|
72
|
+
|
|
73
|
+
- at **`criteria set`** — stores it but returns a non-rejecting ⚠ warning naming the criterion (re-ground or re-declare);
|
|
74
|
+
- at **`lumo verify`** — a PASS on it is **blocked with 422 (no round burned)**; the round does not land and the task does not move to IN_REVIEW until you either re-ground the checkpointer against a real external fact, or honestly re-declare it `AGENT_CONSTRUCTED_STATE`. Both fixes are free (no round consumed).
|
|
75
|
+
|
|
76
|
+
Don't reach for `EXTERNAL_FACT` to look stronger — a jest test is `AGENT_CONSTRUCTED_STATE`, full stop. Reserve `EXTERNAL_FACT` for checks that truly read external state, or use a HUMAN criterion for "a person ran the real system and observed X" (HUMAN is never mechanically downgraded).
|
|
77
|
+
|
|
63
78
|
A HUMAN criterion is judged by a person, not a checkpointer — so don't hand them a bare assertion ("the copy reads naturally") and make them reverse-engineer what to do. Attach **`judgeSteps`**: short, human-readable instructions the adjudication card renders verbatim (light markdown, URLs made clickable). Structured labour is yours; the human just follows the steps.
|
|
64
79
|
|
|
65
80
|
**Shape — 1–3 steps, always in this order:**
|
|
@@ -78,6 +93,7 @@ A HUMAN criterion is judged by a person, not a checkpointer — so don't hand th
|
|
|
78
93
|
{
|
|
79
94
|
"statement": "The acceptance tab reads as plain operational instructions, not agent-voice assertions",
|
|
80
95
|
"verifierType": "HUMAN",
|
|
96
|
+
"referentKind": "AGENT_CONSTRUCTED_STATE",
|
|
81
97
|
"judgeSteps": "Open the task's Acceptance tab (the IN_REVIEW card). For each HUMAN criterion, read its 'How to judge' block. Pass if each one names where to look, what to do, and what counts as pass vs send-back; send back if any reads as a bare assertion with no steps."
|
|
82
98
|
}
|
|
83
99
|
```
|
|
@@ -97,12 +113,14 @@ Two real-repo invariant examples — `prisma/migrations/` files never deleted (t
|
|
|
97
113
|
{
|
|
98
114
|
"statement": "No file under prisma/migrations/ is deleted by this change (vs origin/main)",
|
|
99
115
|
"verifierType": "MACHINE",
|
|
100
|
-
"checkpointer": "bash -c \"test -z \\\"$(git diff --diff-filter=D --name-only origin/main -- prisma/migrations/)\\\"\""
|
|
116
|
+
"checkpointer": "bash -c \"test -z \\\"$(git diff --diff-filter=D --name-only origin/main -- prisma/migrations/)\\\"\"",
|
|
117
|
+
"referentKind": "EXTERNAL_FACT"
|
|
101
118
|
},
|
|
102
119
|
{
|
|
103
120
|
"statement": "Live-doc keeps its table structure after the edit (no rows/headings dropped)",
|
|
104
121
|
"verifierType": "MACHINE",
|
|
105
|
-
"checkpointer": "npx tsx scripts/verify-live-doc.ts <docId> docs/live-docs/<file>.md"
|
|
122
|
+
"checkpointer": "npx tsx scripts/verify-live-doc.ts <docId> docs/live-docs/<file>.md",
|
|
123
|
+
"referentKind": "AGENT_CONSTRUCTED_STATE"
|
|
106
124
|
}
|
|
107
125
|
]
|
|
108
126
|
```
|
|
@@ -120,17 +138,20 @@ One invariant criterion is usually enough — it's the guardrail, not the whole
|
|
|
120
138
|
{
|
|
121
139
|
"statement": "PUT /api/tasks/[id]/criteria rejects a second AGENT_DRAFT submission with 409",
|
|
122
140
|
"verifierType": "MACHINE",
|
|
123
|
-
"checkpointer": "npx jest __tests__/task-criteria.service.test.ts -t 'agent lock'"
|
|
141
|
+
"checkpointer": "npx jest __tests__/task-criteria.service.test.ts -t 'agent lock'",
|
|
142
|
+
"referentKind": "AGENT_CONSTRUCTED_STATE"
|
|
124
143
|
},
|
|
125
144
|
{
|
|
126
145
|
"statement": "The criteria section reads naturally as part of the task statement",
|
|
127
146
|
"verifierType": "HUMAN",
|
|
147
|
+
"referentKind": "AGENT_CONSTRUCTED_STATE",
|
|
128
148
|
"judgeSteps": "Open the task's Acceptance tab. Read the contract top to bottom. Pass if it scans as one coherent definition of done; send back if a criterion reads as boilerplate or contradicts another."
|
|
129
149
|
},
|
|
130
150
|
{
|
|
131
151
|
"statement": "Session-start injection shows the contract ahead of memory",
|
|
132
152
|
"verifierType": "MACHINE",
|
|
133
153
|
"checkpointer": "npx jest __tests__/cli/hook-runner-session-start-stdout.test.ts",
|
|
154
|
+
"referentKind": "AGENT_CONSTRUCTED_STATE",
|
|
134
155
|
"evidenceRequired": true
|
|
135
156
|
}
|
|
136
157
|
]
|
|
@@ -141,6 +162,7 @@ Fields:
|
|
|
141
162
|
- `statement` — required, ≤2000 chars.
|
|
142
163
|
- `verifierType` — `"MACHINE"` | `"HUMAN"`.
|
|
143
164
|
- `checkpointer` — required for MACHINE.
|
|
165
|
+
- `referentKind` — **required**, `"EXTERNAL_FACT"` | `"AGENT_CONSTRUCTED_STATE"` | `"PENDING_OUTCOME"` (see "referentKind" above). Your own tests/build/type-check are `AGENT_CONSTRUCTED_STATE`; `EXTERNAL_FACT` is gated (warned at set, 422 at verify) unless the checkpointer truly grounds external; `PENDING_OUTCOME` can't be MACHINE.
|
|
144
166
|
- `evidenceRequired` — optional, default false.
|
|
145
167
|
- `judgeSteps` — optional, ≤2000 chars; agent-drafted human-judging steps for a HUMAN criterion (see "judgeSteps" above).
|
|
146
168
|
- `id` — only in `--human` revisions (see below).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Fidelity read-outs (mechanical change-pattern)
|
|
2
|
+
|
|
3
|
+
`lumo fidelity` is the fidelity-axis read surface — pure, read-only analytics over what the acceptance loop already records, in the same family as `lumo criteria audit` and `lumo outcome rate`. It adds **no storage and no write path**; it only reads the LUM-609 delivery snapshots and the `CRITERION_CHANGED` audit trail.
|
|
4
|
+
|
|
5
|
+
## `lumo fidelity show <task>`
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
lumo fidelity show LUM-42
|
|
9
|
+
lumo fidelity show LUM-42 --json
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
For **each delivery** of the task (one append-only `TaskCriteriaSnapshot` per fresh IN_REVIEW entry — a reopen→re-deliver appends another), it discloses two compositions and deliberately stops there. There is **no single fidelity score and no good/bad direction verdict** — the read-out is purely mechanical (timing + op + checkpointer grounding; it never reads statement or diff semantics).
|
|
13
|
+
|
|
14
|
+
### 1. Grounding composition
|
|
15
|
+
|
|
16
|
+
Runs the same `effectiveReferentKind` classifier the LUM-605 verify gate uses over the **frozen** contract (delivery-time truth, not the since-edited `latest`), rolled into three buckets plus the full per-kind tally:
|
|
17
|
+
|
|
18
|
+
- **grounded** — `EXTERNAL_FACT` (anchored on a fact the agent cannot author: a git SHA/diff, CI status, a DB row). This count is a **machine-checkable _upper bound_** — the checkpointer heuristic can over-count external grounding, never under-count it.
|
|
19
|
+
- **self-confirming** — `AGENT_CONSTRUCTED_STATE` + `UNVERIFIED_ASSERTION` (passes by checking the agent's own output, incl. a declared-EXTERNAL_FACT checkpointer that doesn't truly ground external).
|
|
20
|
+
- **inconclusive** — `PENDING_OUTCOME` + `UNCLASSIFIED` (not mechanically resolvable here). **Surfaced, never silently swallowed.**
|
|
21
|
+
|
|
22
|
+
### 2. Independence signal
|
|
23
|
+
|
|
24
|
+
A criterion is a **backward-inference suspect** when it is both (a) in this delivery's frozen contract and (b) was `ADDED`/`UPDATED` at **round>0** (after work started) at or before the freeze — i.e. the contract the delivery rode on was bent after the work began rather than fixed up front. round-0 (initial-draft) criteria are clean; `DELETED` ops and edits recorded after the freeze (a later cycle) don't count. Each suspect shows its `op`, `round`, and `causeTag`. The read-out **counts and discloses — it does not judge** whether a given edit was legitimate sharpening or tampering (that direction call is left to a human / a later axis).
|
|
25
|
+
|
|
26
|
+
### Caveat (printed on every report)
|
|
27
|
+
|
|
28
|
+
The time anchor is the **work-start boundary** (`Task.workStartedAt`), so the independence signal detects a contract edited _after work started_ — not specifically _after output was produced_. A task with **no delivery snapshot** (never entered IN_REVIEW, or predates the LUM-609 freeze) is reported explicitly — "nothing to read yet", which is **not** a pass.
|
|
29
|
+
|
|
30
|
+
`--json` emits the full report (`taskId`, `workStartedAt`, `hasSnapshots`, `deliveries[]` with `grounding`/`independence`, `caveat`) for scripting.
|
|
31
|
+
|
|
32
|
+
**When to suggest**: when the user wants to see how externally-grounded a task's delivered acceptance contract actually was, or whether criteria were added/changed after work began (independence). It's a disclosure tool, not a gate — it never blocks DONE and emits no verdict.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Outcome well (post-hoc oracle) — `lumo outcome`
|
|
2
|
+
|
|
3
|
+
The **post-hoc outcome well** (LUM-598) is the single external oracle for correctness + fidelity. It records the real-world fate a delivery met **after** it shipped — facts the agent cannot construct and that occur post-delivery (a revert, a rollback, CI turning red, a reopen, a downstream redirect of the design).
|
|
4
|
+
|
|
5
|
+
## The one invariant: falsifier, not verifier
|
|
6
|
+
|
|
7
|
+
The well **only** ever produces a rejection. Its verdict space is exactly:
|
|
8
|
+
|
|
9
|
+
- `REJECTED` — the well holds ≥1 rejection signal (reality revoked / redid / bypassed the delivery).
|
|
10
|
+
- `INCONCLUSIVE` — no rejection on record.
|
|
11
|
+
|
|
12
|
+
There is **no `SATISFIED` / `PASS`**. "N sprints and nobody touched it" is the _absence_ of a rejection — satisfied, merely-tolerated, and never-noticed are externally indistinguishable — so it reads `INCONCLUSIVE`, never a pass. Treat `INCONCLUSIVE` as "could not confirm a rejection", not as approval.
|
|
13
|
+
|
|
14
|
+
## Collection (where signals come from)
|
|
15
|
+
|
|
16
|
+
- **Automatic** — a DONE task reopened (the lineage `REWORKED` transition) is folded into the well as a `REOPENED` signal. No action needed; it happens server-side.
|
|
17
|
+
- **Manual** — a human who observed reality reject a delivery records it with `lumo outcome record`. The `--note` is mandatory: it is the observed referent, frozen as evidence.
|
|
18
|
+
|
|
19
|
+
## `lumo outcome show <task>`
|
|
20
|
+
|
|
21
|
+
Read the well for a task — the verdict plus its backing signals.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
lumo outcome show LUM-42
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
An empty well prints `INCONCLUSIVE` with the reminder that silence is not a pass. A non-empty well prints `REJECTED` and lists each signal (date, kind, source, note).
|
|
28
|
+
|
|
29
|
+
## `lumo outcome record <task> --note "<what reality did>" [--kind <kind>]`
|
|
30
|
+
|
|
31
|
+
Record one human-observed post-hoc rejection. Append-only — every call inserts a new signal; nothing is ever mutated or cleared.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
lumo outcome record LUM-42 --note "reverted in #812 after a prod incident" --kind reverted
|
|
35
|
+
lumo outcome record LUM-42 --note "downstream rewrote the API shape this task introduced" --kind downstream_redirect
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
- `--kind` (case-insensitive, default `manual`): `reverted | rolled_back | ci_regression | downstream_redirect | bypassed | manual`. `reopened` is **not** a manual kind — it is owned by the automatic lineage fold.
|
|
39
|
+
- `--occurred-at <iso>` — when the event actually happened (defaults to now).
|
|
40
|
+
|
|
41
|
+
## `lumo outcome rate [--min N] [--json]`
|
|
42
|
+
|
|
43
|
+
The workspace-level **trust × post-hoc-fate** join (LUM-603, LUM-606). For every delivery that has a frozen forecast snapshot, it buckets the delivery by its **delivery-time verification friction** — two raw counts (`verificationRounds`, `sendBackCount`) frozen at delivery from the append-only `VerificationRun` log, cut at `capturedAt`, then bucketed value-neutrally into `no-verification` / `single-round` / `multi-round-or-sendback` — and reports each bucket's post-hoc **REJECTED** rate from the well. The friction axis replaced the original sample-size forecast-confidence axis (LUM-606), which saturated: every mature-project delivery landed in `high`, so `high`-vs-`low` could never compare. The question it answers once data is sufficient: _do deliveries that took more verification friction get rejected by reality more (fragility) or less (scrutiny) often?_ The bucket names carry **no** good/bad direction — the correlation, not the label, decides.
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
lumo outcome rate
|
|
47
|
+
lumo outcome rate --min 20 --json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
This is the falsifier discipline applied at the **query layer** — it refuses to manufacture a finding off a thin sample:
|
|
51
|
+
|
|
52
|
+
- A bucket with fewer than `--min` (default 10) deliveries reads **`insufficient`** with the rate withheld — the raw `rejected/delivered` counts are still shown, but never divided into a believable proportion.
|
|
53
|
+
- The per-bucket rate is an explicit **lower bound** on true failure: the well only records rejections, so a zero-signal delivery is `INCONCLUSIVE` (counted in the denominator), never "satisfied".
|
|
54
|
+
- The **`single-round` vs `multi-round-or-sendback`** comparison asserts a direction **only** when both buckets are measured _and_ their 95% confidence intervals are disjoint; otherwise it stays **`inconclusive`**. `no-verification` is never a comparison arm (silence is not a friction level).
|
|
55
|
+
|
|
56
|
+
**Expect `inconclusive` today.** The well is freshly wired and the discriminating end of the friction axis is sparse (multi-round/send-back deliveries are rare) — that is the designed state, not a bug. The same command produces real strata once enough post-hoc signals accumulate against the populated friction buckets (the deferred outcome the skeleton exists to read).
|
|
57
|
+
|
|
58
|
+
> Scope note: friction is frozen **at delivery** and cut at `capturedAt`, so it is replayable and never reads a drifted current value. A delivery with no `VerificationRun` before capture is its own `no-verification` bucket — absence of verification is never folded into `single-round` (silence ≠ a clean pass).
|
|
59
|
+
|
|
60
|
+
## When to suggest
|
|
61
|
+
|
|
62
|
+
- A teammate says a shipped task **was reverted / rolled back / broke prod / got redone / was bypassed** — record it with `lumo outcome record` so the well carries reality's verdict.
|
|
63
|
+
- Auditing whether a "done" task actually held up — `lumo outcome show <task>` reads the well.
|
|
64
|
+
- Asking whether **verification friction predicts post-hoc fate** (do harder-to-verify deliveries fail more, or less?) — `lumo outcome rate`. Expect `inconclusive` until the well fills; it will never print a correlation the sample can't support.
|
|
65
|
+
- **Never** use it to mark something good: the well has no positive verdict by design. To assert a delivery met its contract, that is the acceptance loop (`lumo verify` / `lumo task status`), not the well.
|
|
@@ -21,10 +21,10 @@ lumo verify --note "…" --timeout 900 # per-checkpointer timeout in seconds
|
|
|
21
21
|
**`--note` is required (LUM-597)** whenever a round will actually be posted (the
|
|
22
22
|
task has MACHINE criteria). It is your one-line self-report — what you did and
|
|
23
23
|
why it is ready ("我改了 X,因为 Y 可验收"). When the round all-passes and the
|
|
24
|
-
task flips to
|
|
24
|
+
task flips to IN*REVIEW, the note is frozen as the task's **claim** (provenance
|
|
25
25
|
`AGENT` — the汇报者's own voice, not a summarizer paraphrase) and is checked
|
|
26
26
|
against the diff for faithfulness. Capturing the claim is deterministic ("不填不
|
|
27
|
-
让 post"); whether it is
|
|
27
|
+
让 post"); whether it is \_truthful* is the faithfulness audit's job — so don't
|
|
28
28
|
under- or over-state it. A missing/blank note is refused **before** the round is
|
|
29
29
|
posted, so no round is burned; just re-run with one. (An old CLI that omits it
|
|
30
30
|
still verifies — the claim degrades to the synthesized run-summary fallback.)
|
|
@@ -61,15 +61,16 @@ errors.
|
|
|
61
61
|
|
|
62
62
|
## Edge cases
|
|
63
63
|
|
|
64
|
-
| Case | Behavior
|
|
65
|
-
| ----------------------------------------------- |
|
|
66
|
-
| **No contract yet** | Error pointing at `lumo task criteria set`; draft the contract first (criteria.md golden rule).
|
|
67
|
-
| **HUMAN-only contract** (zero MACHINE criteria) | Nothing to run; CLI says so and suggests `lumo task update <id> --status in_review` for human review. No server write happens.
|
|
68
|
-
| **Partial round** | A round must cover every MACHINE criterion; the CLI always runs all of them and the server rejects partial rounds.
|
|
69
|
-
| **`REVIEW_ADDED` criteria** | Criteria added during review appear in the contract and are picked up automatically by the next round.
|
|
70
|
-
| **Session bound to a different task** | Server returns 409, surfaced as an error. No advisory printed; the verify round is rejected outright.
|
|
71
|
-
| **
|
|
72
|
-
| **
|
|
64
|
+
| Case | Behavior |
|
|
65
|
+
| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
66
|
+
| **No contract yet** | Error pointing at `lumo task criteria set`; draft the contract first (criteria.md golden rule). |
|
|
67
|
+
| **HUMAN-only contract** (zero MACHINE criteria) | Nothing to run; CLI says so and suggests `lumo task update <id> --status in_review` for human review. No server write happens. |
|
|
68
|
+
| **Partial round** | A round must cover every MACHINE criterion; the CLI always runs all of them and the server rejects partial rounds. |
|
|
69
|
+
| **`REVIEW_ADDED` criteria** | Criteria added during review appear in the contract and are picked up automatically by the next round. |
|
|
70
|
+
| **Session bound to a different task** | Server returns 409, surfaced as an error. No advisory printed; the verify round is rejected outright. |
|
|
71
|
+
| **Ungrounded EXTERNAL_FACT PASS** (LUM-605) | A reported PASS on a criterion declared `EXTERNAL_FACT` whose checkpointer doesn't invoke external tooling (e.g. `npx jest …`) is blocked with **422 — no round burned**; the task does **not** move to IN_REVIEW. Fix it for free: re-ground the checkpointer against a real external fact, or honestly re-declare `referentKind` as `AGENT_CONSTRUCTED_STATE` (`lumo task criteria set`), then re-run. See criteria.md "referentKind". |
|
|
72
|
+
| **Provably-unbound session** | Response carries `bindingAdvisory: 'unbound'`; prints `⚠ Working unbound — this verify ran from a Claude Code session not attached to the task.` Recorded as a `SESSION_BINDING_MISSING` boundary crossing (visible in `lumo task status` open crossings). Run `lumo session attach <LUM-N>` before the next verify. |
|
|
73
|
+
| **Unconfirmed session binding** | `bindingAdvisory: 'unconfirmed'` → softer advisory `⚠ Could not confirm this session is attached to the task.` Same remediation: `lumo session attach <LUM-N>`. |
|
|
73
74
|
|
|
74
75
|
## Round discipline
|
|
75
76
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.formatAuditReport = formatAuditReport;
|
|
4
|
+
exports.criteriaAudit = criteriaAudit;
|
|
5
|
+
const config_1 = require("../lib/config");
|
|
6
|
+
const api_1 = require("../lib/api");
|
|
7
|
+
const KIND_ORDER = [
|
|
8
|
+
'EXTERNAL_FACT',
|
|
9
|
+
'AGENT_CONSTRUCTED_STATE',
|
|
10
|
+
'PENDING_OUTCOME',
|
|
11
|
+
'UNVERIFIED_ASSERTION',
|
|
12
|
+
'UNCLASSIFIED',
|
|
13
|
+
];
|
|
14
|
+
function formatAuditReport(r) {
|
|
15
|
+
const lines = ['Criteria referent-kind audit (workspace)', ''];
|
|
16
|
+
for (const k of KIND_ORDER) {
|
|
17
|
+
const n = r.byEffectiveKind[k] ?? 0;
|
|
18
|
+
const pct = r.total === 0 ? 0 : Math.round((n / r.total) * 100);
|
|
19
|
+
lines.push(` ${k.padEnd(24)} ${String(n).padStart(4)} (${pct}%)`);
|
|
20
|
+
}
|
|
21
|
+
lines.push('');
|
|
22
|
+
const pct = Math.round(r.selfConfirmingGreenRatio * 100);
|
|
23
|
+
lines.push(` self-confirming green: ${pct}% of ${r.classified} classified criteria`);
|
|
24
|
+
lines.push(` (AGENT_CONSTRUCTED_STATE + UNVERIFIED_ASSERTION — green that confirms only the agent's own work)`);
|
|
25
|
+
return lines.join('\n') + '\n';
|
|
26
|
+
}
|
|
27
|
+
/** `lumo criteria audit` — workspace-level referent-kind distribution. */
|
|
28
|
+
async function criteriaAudit() {
|
|
29
|
+
const creds = (0, config_1.readCredentials)();
|
|
30
|
+
if (!creds) {
|
|
31
|
+
console.error('Error: not logged in. Run `lumo auth login` first.');
|
|
32
|
+
return 1;
|
|
33
|
+
}
|
|
34
|
+
const base = (0, api_1.trimTrailingSlash)((0, api_1.resolveAuthedApiUrl)(creds.apiUrl));
|
|
35
|
+
let res;
|
|
36
|
+
try {
|
|
37
|
+
res = await fetch(`${base}/api/criteria/audit`, {
|
|
38
|
+
headers: { Authorization: `Bearer ${creds.token}` },
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
43
|
+
console.error(`Error: could not reach Lumo API (${msg})`);
|
|
44
|
+
return 1;
|
|
45
|
+
}
|
|
46
|
+
if (!res.ok) {
|
|
47
|
+
console.error(`Error: criteria audit failed (HTTP ${res.status})`);
|
|
48
|
+
return 1;
|
|
49
|
+
}
|
|
50
|
+
const report = (await res.json());
|
|
51
|
+
process.stdout.write(formatAuditReport(report));
|
|
52
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.fidelityShow = fidelityShow;
|
|
4
|
+
const config_1 = require("../lib/config");
|
|
5
|
+
const api_1 = require("../lib/api");
|
|
6
|
+
const sanitize_1 = require("../lib/sanitize");
|
|
7
|
+
/**
|
|
8
|
+
* `lumo fidelity show <task>` — the LUM-610 mechanical change-pattern read-out
|
|
9
|
+
* (fidelity axis, second block, on the LUM-609 delivery-snapshot foundation).
|
|
10
|
+
*
|
|
11
|
+
* Per delivery snapshot it discloses two compositions and NOTHING else — no
|
|
12
|
+
* single score, no good/bad verdict (approach (a): purely mechanical):
|
|
13
|
+
* • grounding — how much of the FROZEN contract is externally grounded
|
|
14
|
+
* (EXTERNAL_FACT) vs self-confirming (agent-authored) vs inconclusive.
|
|
15
|
+
* `grounded` is a machine-checkable UPPER BOUND.
|
|
16
|
+
* • independence — criteria ADDED/UPDATED after work started (round>0) that
|
|
17
|
+
* the delivery rode on: backward-inference suspects. Direction left blank.
|
|
18
|
+
*/
|
|
19
|
+
const EFFECTIVE_KINDS = [
|
|
20
|
+
'EXTERNAL_FACT',
|
|
21
|
+
'AGENT_CONSTRUCTED_STATE',
|
|
22
|
+
'UNVERIFIED_ASSERTION',
|
|
23
|
+
'PENDING_OUTCOME',
|
|
24
|
+
'UNCLASSIFIED',
|
|
25
|
+
];
|
|
26
|
+
function authBase() {
|
|
27
|
+
const creds = (0, config_1.readCredentials)();
|
|
28
|
+
if (!creds)
|
|
29
|
+
return { error: 'not logged in. Run `lumo auth login` first.' };
|
|
30
|
+
const base = (0, api_1.trimTrailingSlash)((0, api_1.resolveAuthedApiUrl)(creds.apiUrl));
|
|
31
|
+
const headers = {
|
|
32
|
+
Authorization: `Bearer ${creds.token}`,
|
|
33
|
+
};
|
|
34
|
+
const sessionId = process.env.CLAUDE_CODE_SESSION_ID;
|
|
35
|
+
if (sessionId)
|
|
36
|
+
headers['X-Lumo-Session-Id'] = sessionId;
|
|
37
|
+
return { base, headers };
|
|
38
|
+
}
|
|
39
|
+
function formatReport(report, taskId) {
|
|
40
|
+
const lines = [];
|
|
41
|
+
lines.push(`Fidelity change-pattern — ${(0, sanitize_1.sanitizeField)(taskId)}`);
|
|
42
|
+
if (!report.hasSnapshots) {
|
|
43
|
+
lines.push(' No delivery snapshot on record (task never entered IN_REVIEW, or ' +
|
|
44
|
+
'predates the LUM-609 freeze). Nothing to read yet — not a pass.');
|
|
45
|
+
lines.push('');
|
|
46
|
+
lines.push(` ${report.caveat}`);
|
|
47
|
+
return lines.join('\n') + '\n';
|
|
48
|
+
}
|
|
49
|
+
report.deliveries.forEach((d, i) => {
|
|
50
|
+
const when = d.frozenAt.slice(0, 19).replace('T', ' ');
|
|
51
|
+
lines.push('');
|
|
52
|
+
lines.push(` Delivery ${i + 1}/${report.deliveries.length} · frozen ${when}Z · ${(0, sanitize_1.sanitizeField)(d.trigger)} · ${d.total} criteria`);
|
|
53
|
+
const g = d.grounding;
|
|
54
|
+
lines.push(` grounding: grounded ${g.grounded} · self-confirming ${g.selfConfirming} · inconclusive ${g.inconclusive} (grounded = machine-checkable upper bound)`);
|
|
55
|
+
const kinds = EFFECTIVE_KINDS.filter(k => g.byEffectiveKind[k] > 0)
|
|
56
|
+
.map(k => `${k} ${g.byEffectiveKind[k]}`)
|
|
57
|
+
.join(', ');
|
|
58
|
+
lines.push(` effective kinds: ${kinds || '(none)'}`);
|
|
59
|
+
const ind = d.independence;
|
|
60
|
+
lines.push(` independence: ${ind.postStartChanges}/${ind.total} criteria ADDED/UPDATED after work started (backward-inference suspects; direction not judged)`);
|
|
61
|
+
for (const s of ind.suspects) {
|
|
62
|
+
const cause = s.causeTag ? ` · ${(0, sanitize_1.sanitizeField)(s.causeTag)}` : '';
|
|
63
|
+
lines.push(` ↳ ${(0, sanitize_1.sanitizeField)(s.criterionId)} ${s.op}@round${s.round}${cause} (${s.at.slice(0, 10)})`);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
lines.push('');
|
|
67
|
+
lines.push(` ${report.caveat}`);
|
|
68
|
+
return lines.join('\n') + '\n';
|
|
69
|
+
}
|
|
70
|
+
async function fidelityShow(taskId, options = {}) {
|
|
71
|
+
if (!taskId || taskId.trim() === '') {
|
|
72
|
+
console.error('Error: a task is required: lumo fidelity show <task>');
|
|
73
|
+
return 1;
|
|
74
|
+
}
|
|
75
|
+
const auth = authBase();
|
|
76
|
+
if ('error' in auth) {
|
|
77
|
+
console.error(`Error: ${auth.error}`);
|
|
78
|
+
return 1;
|
|
79
|
+
}
|
|
80
|
+
let res;
|
|
81
|
+
try {
|
|
82
|
+
res = await fetch(`${auth.base}/api/tasks/${encodeURIComponent(taskId)}/fidelity`, { headers: auth.headers });
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
86
|
+
console.error(`Error: could not reach Lumo API (${msg})`);
|
|
87
|
+
return 1;
|
|
88
|
+
}
|
|
89
|
+
if (res.status === 401) {
|
|
90
|
+
console.error('Error: API key invalid or revoked. Run `lumo auth login`.');
|
|
91
|
+
return 1;
|
|
92
|
+
}
|
|
93
|
+
if (res.status === 404) {
|
|
94
|
+
console.error(`Error: task ${(0, sanitize_1.sanitizeField)(taskId)} not found.`);
|
|
95
|
+
return 1;
|
|
96
|
+
}
|
|
97
|
+
if (!res.ok) {
|
|
98
|
+
console.error(`Error: could not read the fidelity change-pattern (HTTP ${res.status}).`);
|
|
99
|
+
return 1;
|
|
100
|
+
}
|
|
101
|
+
const report = (await res.json());
|
|
102
|
+
if (options.json) {
|
|
103
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
process.stdout.write(formatReport(report, taskId));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.outcomeRecord = outcomeRecord;
|
|
4
|
+
exports.outcomeShow = outcomeShow;
|
|
5
|
+
exports.outcomeRate = outcomeRate;
|
|
6
|
+
const config_1 = require("../lib/config");
|
|
7
|
+
const api_1 = require("../lib/api");
|
|
8
|
+
const sanitize_1 = require("../lib/sanitize");
|
|
9
|
+
/**
|
|
10
|
+
* `lumo outcome` — the post-hoc outcome well (LUM-598).
|
|
11
|
+
*
|
|
12
|
+
* The well is the single external oracle for correctness + fidelity: it records
|
|
13
|
+
* the real-world fate a delivery met AFTER it shipped. It is a FALSIFIER, not a
|
|
14
|
+
* verifier — it only ever reads REJECTED (reality revoked/redid/bypassed the
|
|
15
|
+
* work) or INCONCLUSIVE (no rejection on record). There is deliberately no
|
|
16
|
+
* "mark satisfied": the absence of a rejection is not a pass.
|
|
17
|
+
*/
|
|
18
|
+
const MANUAL_KINDS = [
|
|
19
|
+
'REVERTED',
|
|
20
|
+
'ROLLED_BACK',
|
|
21
|
+
'CI_REGRESSION',
|
|
22
|
+
'DOWNSTREAM_REDIRECT',
|
|
23
|
+
'BYPASSED',
|
|
24
|
+
'MANUAL',
|
|
25
|
+
];
|
|
26
|
+
function authBase() {
|
|
27
|
+
const creds = (0, config_1.readCredentials)();
|
|
28
|
+
if (!creds)
|
|
29
|
+
return { error: 'not logged in. Run `lumo auth login` first.' };
|
|
30
|
+
const base = (0, api_1.trimTrailingSlash)((0, api_1.resolveAuthedApiUrl)(creds.apiUrl));
|
|
31
|
+
const headers = {
|
|
32
|
+
Authorization: `Bearer ${creds.token}`,
|
|
33
|
+
};
|
|
34
|
+
const sessionId = process.env.CLAUDE_CODE_SESSION_ID;
|
|
35
|
+
if (sessionId)
|
|
36
|
+
headers['X-Lumo-Session-Id'] = sessionId;
|
|
37
|
+
return { base, headers };
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* `lumo outcome record <task> --note "<what reality did>" [--kind <kind>]` —
|
|
41
|
+
* record a human-observed post-hoc REJECTION of a delivery. Append-only. The
|
|
42
|
+
* note is mandatory (the observed referent); the default kind is MANUAL.
|
|
43
|
+
*/
|
|
44
|
+
async function outcomeRecord(taskId, options = {}) {
|
|
45
|
+
if (!taskId || taskId.trim() === '') {
|
|
46
|
+
console.error('Error: a task is required: lumo outcome record <task> --note "…"');
|
|
47
|
+
return 1;
|
|
48
|
+
}
|
|
49
|
+
const note = options.note?.trim();
|
|
50
|
+
if (!note) {
|
|
51
|
+
console.error('Error: --note "<what reality did>" is required (the observed rejection, e.g. "reverted in #812 after prod incident").');
|
|
52
|
+
return 1;
|
|
53
|
+
}
|
|
54
|
+
const kind = (options.kind?.trim().toUpperCase() ||
|
|
55
|
+
'MANUAL');
|
|
56
|
+
if (!MANUAL_KINDS.includes(kind)) {
|
|
57
|
+
console.error(`Error: --kind must be one of: ${MANUAL_KINDS.map(k => k.toLowerCase()).join(', ')}`);
|
|
58
|
+
return 1;
|
|
59
|
+
}
|
|
60
|
+
const auth = authBase();
|
|
61
|
+
if ('error' in auth) {
|
|
62
|
+
console.error(`Error: ${auth.error}`);
|
|
63
|
+
return 1;
|
|
64
|
+
}
|
|
65
|
+
const payload = { kind, note };
|
|
66
|
+
if (options.occurredAt?.trim())
|
|
67
|
+
payload.occurredAt = options.occurredAt.trim();
|
|
68
|
+
let res;
|
|
69
|
+
try {
|
|
70
|
+
res = await fetch(`${auth.base}/api/tasks/${encodeURIComponent(taskId)}/outcome-signals`, {
|
|
71
|
+
method: 'POST',
|
|
72
|
+
headers: { ...auth.headers, 'Content-Type': 'application/json' },
|
|
73
|
+
body: JSON.stringify(payload),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
78
|
+
console.error(`Error: could not reach Lumo API (${msg})`);
|
|
79
|
+
return 1;
|
|
80
|
+
}
|
|
81
|
+
if (res.status === 401) {
|
|
82
|
+
console.error('Error: API key invalid or revoked. Run `lumo auth login`.');
|
|
83
|
+
return 1;
|
|
84
|
+
}
|
|
85
|
+
if (!res.ok) {
|
|
86
|
+
const errBody = (await res.json().catch(() => null));
|
|
87
|
+
const detail = errBody && typeof errBody.error === 'string'
|
|
88
|
+
? (0, sanitize_1.sanitizeField)(errBody.error)
|
|
89
|
+
: '';
|
|
90
|
+
console.error(`Error: outcome not recorded (HTTP ${res.status})${detail ? ` — ${detail}` : ''}`);
|
|
91
|
+
return 1;
|
|
92
|
+
}
|
|
93
|
+
process.stdout.write(`✓ Recorded a ${(0, sanitize_1.sanitizeField)(kind.toLowerCase())} rejection on ${(0, sanitize_1.sanitizeField)(taskId)}.\n` +
|
|
94
|
+
' The well is a falsifier — this marks the delivery REJECTED by reality; ' +
|
|
95
|
+
'it is append-only and there is no "satisfied" counterpart.\n');
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* `lumo outcome show <task>` — read the well for a task: the falsifier verdict
|
|
100
|
+
* plus its backing rejection signals.
|
|
101
|
+
*/
|
|
102
|
+
async function outcomeShow(taskId) {
|
|
103
|
+
if (!taskId || taskId.trim() === '') {
|
|
104
|
+
console.error('Error: a task is required: lumo outcome show <task>');
|
|
105
|
+
return 1;
|
|
106
|
+
}
|
|
107
|
+
const auth = authBase();
|
|
108
|
+
if ('error' in auth) {
|
|
109
|
+
console.error(`Error: ${auth.error}`);
|
|
110
|
+
return 1;
|
|
111
|
+
}
|
|
112
|
+
let res;
|
|
113
|
+
try {
|
|
114
|
+
res = await fetch(`${auth.base}/api/tasks/${encodeURIComponent(taskId)}/outcome-signals`, { headers: auth.headers });
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
118
|
+
console.error(`Error: could not reach Lumo API (${msg})`);
|
|
119
|
+
return 1;
|
|
120
|
+
}
|
|
121
|
+
if (res.status === 401) {
|
|
122
|
+
console.error('Error: API key invalid or revoked. Run `lumo auth login`.');
|
|
123
|
+
return 1;
|
|
124
|
+
}
|
|
125
|
+
if (!res.ok) {
|
|
126
|
+
console.error(`Error: could not read the well (HTTP ${res.status}).`);
|
|
127
|
+
return 1;
|
|
128
|
+
}
|
|
129
|
+
const well = (await res.json());
|
|
130
|
+
const mark = well.verdict === 'REJECTED' ? '✗' : '·';
|
|
131
|
+
process.stdout.write(`${mark} Outcome well — ${(0, sanitize_1.sanitizeField)(taskId)}: ${well.verdict}\n`);
|
|
132
|
+
if (well.verdict === 'INCONCLUSIVE') {
|
|
133
|
+
process.stdout.write(' No rejection on record. INCONCLUSIVE ≠ satisfied — the well only ' +
|
|
134
|
+
'asserts rejection; silence is not a pass.\n');
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
for (const s of well.signals) {
|
|
138
|
+
const when = (s.occurredAt ?? s.detectedAt).slice(0, 10);
|
|
139
|
+
const note = s.evidence?.note ? ` — ${(0, sanitize_1.sanitizeField)(s.evidence.note)}` : '';
|
|
140
|
+
process.stdout.write(` ${when} ${(0, sanitize_1.sanitizeField)(s.kind)} (${(0, sanitize_1.sanitizeField)(s.source)})${note}\n`);
|
|
141
|
+
}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
/** Width of the widest bucket label, for aligned columns. */
|
|
145
|
+
const BUCKET_LABEL_WIDTH = 'multi-round-or-sendback'.length;
|
|
146
|
+
/** Render the report client-side (same pattern as `lumo cost` / `outcome show`). */
|
|
147
|
+
function formatRate(report) {
|
|
148
|
+
const lines = [];
|
|
149
|
+
lines.push('Trust × post-hoc fate — delivery-time verification friction vs the outcome well');
|
|
150
|
+
lines.push(' (rejection rate is a LOWER BOUND: the well only records rejections; ' +
|
|
151
|
+
'no signal = INCONCLUSIVE, never "satisfied")');
|
|
152
|
+
lines.push(` ${report.totalDeliveries} snapshotted deliveries · min ${report.minSamples} per bracket`);
|
|
153
|
+
if (report.totalDeliveries === 0) {
|
|
154
|
+
lines.push(' (no delivery has a forecast snapshot yet — nothing to join)');
|
|
155
|
+
return lines.join('\n') + '\n';
|
|
156
|
+
}
|
|
157
|
+
for (const b of report.brackets) {
|
|
158
|
+
if (b.status === 'measured') {
|
|
159
|
+
const pct = ((b.rejectionRate ?? 0) * 100).toFixed(1);
|
|
160
|
+
const lo = ((b.interval?.lower ?? 0) * 100).toFixed(1);
|
|
161
|
+
const hi = ((b.interval?.upper ?? 0) * 100).toFixed(1);
|
|
162
|
+
lines.push(` ${b.bracket.padEnd(BUCKET_LABEL_WIDTH)} ≥${pct}% rejected (${b.rejected}/${b.delivered}, 95% CI ${lo}–${hi}%)`);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
lines.push(` ${b.bracket.padEnd(BUCKET_LABEL_WIDTH)} insufficient (${b.rejected}/${b.delivered} deliveries < ${report.minSamples} — rate withheld)`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const verdict = report.comparison.status === 'measured'
|
|
169
|
+
? `→ ${(0, sanitize_1.sanitizeField)(report.comparison.reason)}`
|
|
170
|
+
: `→ inconclusive: ${(0, sanitize_1.sanitizeField)(report.comparison.reason)}`;
|
|
171
|
+
lines.push(` ${verdict}`);
|
|
172
|
+
return lines.join('\n') + '\n';
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* `lumo outcome rate [--min N] [--json]` — read the workspace trust × post-hoc-
|
|
176
|
+
* fate join: per delivery-time verification-friction bucket (LUM-606:
|
|
177
|
+
* no-verification / single-round / multi-round-or-sendback, frozen at delivery),
|
|
178
|
+
* the post-hoc REJECTED rate from the outcome well. Honest by construction — thin
|
|
179
|
+
* buckets read `insufficient` (no fabricated rate) and the single-round
|
|
180
|
+
* vs multi-round-or-sendback comparison stays `inconclusive` until the well has
|
|
181
|
+
* enough signal to tell the buckets apart.
|
|
182
|
+
*/
|
|
183
|
+
async function outcomeRate(options = {}) {
|
|
184
|
+
const auth = authBase();
|
|
185
|
+
if ('error' in auth) {
|
|
186
|
+
console.error(`Error: ${auth.error}`);
|
|
187
|
+
return 1;
|
|
188
|
+
}
|
|
189
|
+
let min;
|
|
190
|
+
if (options.min !== undefined && options.min.trim() !== '') {
|
|
191
|
+
const n = Number(options.min);
|
|
192
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
193
|
+
console.error('Error: --min must be a positive integer');
|
|
194
|
+
return 1;
|
|
195
|
+
}
|
|
196
|
+
min = n;
|
|
197
|
+
}
|
|
198
|
+
const qs = min !== undefined ? `?min=${min}` : '';
|
|
199
|
+
let res;
|
|
200
|
+
try {
|
|
201
|
+
res = await fetch(`${auth.base}/api/outcome/rate${qs}`, {
|
|
202
|
+
headers: auth.headers,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
207
|
+
console.error(`Error: could not reach Lumo API (${msg})`);
|
|
208
|
+
return 1;
|
|
209
|
+
}
|
|
210
|
+
if (res.status === 401) {
|
|
211
|
+
console.error('Error: API key invalid or revoked. Run `lumo auth login`.');
|
|
212
|
+
return 1;
|
|
213
|
+
}
|
|
214
|
+
if (!res.ok) {
|
|
215
|
+
console.error(`Error: could not read the outcome rate (HTTP ${res.status}).`);
|
|
216
|
+
return 1;
|
|
217
|
+
}
|
|
218
|
+
const report = (await res.json());
|
|
219
|
+
if (options.json) {
|
|
220
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
process.stdout.write(formatRate(report));
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
@@ -5,6 +5,7 @@ exports.taskCriteriaList = taskCriteriaList;
|
|
|
5
5
|
const config_1 = require("../lib/config");
|
|
6
6
|
const api_1 = require("../lib/api");
|
|
7
7
|
const sanitize_1 = require("../lib/sanitize");
|
|
8
|
+
const referent_kind_1 = require("../../../shared/src/referent-kind");
|
|
8
9
|
/**
|
|
9
10
|
* Render criteria rows for stdout. One line per criterion —
|
|
10
11
|
* `<id> [TYPE] SOURCE@rN statement` — plus an indented checkpointer line
|
|
@@ -16,7 +17,13 @@ function formatCriteriaRows(criteria) {
|
|
|
16
17
|
for (const c of criteria) {
|
|
17
18
|
const provenance = `${c.source}@r${c.addedAtRound}`;
|
|
18
19
|
const evidence = c.evidenceRequired ? ' [evidence]' : '';
|
|
19
|
-
|
|
20
|
+
const eff = (0, referent_kind_1.effectiveReferentKind)({
|
|
21
|
+
declared: c.referentKind ?? null,
|
|
22
|
+
verifierType: c.verifierType,
|
|
23
|
+
checkpointer: c.checkpointer,
|
|
24
|
+
});
|
|
25
|
+
const kindTag = ` ⟨${eff}⟩`;
|
|
26
|
+
lines.push(`${c.id} [${c.verifierType}] ${provenance}${evidence}${kindTag} ${(0, sanitize_1.sanitizeField)(c.statement)}`);
|
|
20
27
|
if (c.checkpointer) {
|
|
21
28
|
lines.push(` ↳ check: ${(0, sanitize_1.sanitizeField)(c.checkpointer)}`);
|
|
22
29
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyReadback = verifyReadback;
|
|
3
4
|
exports.taskCriteriaSet = taskCriteriaSet;
|
|
4
5
|
const config_1 = require("../lib/config");
|
|
5
6
|
const api_1 = require("../lib/api");
|
|
@@ -7,6 +8,31 @@ const doc_input_1 = require("../lib/doc-input");
|
|
|
7
8
|
const path_guard_1 = require("../lib/path-guard");
|
|
8
9
|
const sanitize_1 = require("../lib/sanitize");
|
|
9
10
|
const task_criteria_list_1 = require("./task-criteria-list");
|
|
11
|
+
/**
|
|
12
|
+
* Landing-integrity read-back (LUM-602): a 200 OK proves absence of error,
|
|
13
|
+
* not presence of correct content. Compare what the server stored (echoed in
|
|
14
|
+
* the PUT response) field-by-field against what we submitted; any divergence
|
|
15
|
+
* means the value was corrupted in transport/landing.
|
|
16
|
+
*/
|
|
17
|
+
function verifyReadback(submitted, stored) {
|
|
18
|
+
const issues = [];
|
|
19
|
+
const storedByStatement = new Map(stored.map(c => [c.statement, c]));
|
|
20
|
+
for (const s of submitted) {
|
|
21
|
+
const got = storedByStatement.get(s.statement);
|
|
22
|
+
if (!got) {
|
|
23
|
+
issues.push(`criterion not found after write: "${s.statement.slice(0, 60)}"`);
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
const wantCk = s.checkpointer ?? null;
|
|
27
|
+
if ((got.checkpointer ?? null) !== wantCk) {
|
|
28
|
+
issues.push(`checkpointer corrupted in landing for "${s.statement.slice(0, 40)}": sent ${JSON.stringify(wantCk)}, stored ${JSON.stringify(got.checkpointer ?? null)}`);
|
|
29
|
+
}
|
|
30
|
+
if (s.referentKind != null && got.referentKind !== s.referentKind) {
|
|
31
|
+
issues.push(`referentKind corrupted in landing for "${s.statement.slice(0, 40)}": sent ${s.referentKind}, stored ${got.referentKind}`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return issues;
|
|
35
|
+
}
|
|
10
36
|
const CAUSE_TAGS = [
|
|
11
37
|
'NEW_INFO',
|
|
12
38
|
'SCOPE_CHANGE',
|
|
@@ -28,7 +54,7 @@ function parseCriteriaJson(raw) {
|
|
|
28
54
|
if (!Array.isArray(parsed) || parsed.length === 0) {
|
|
29
55
|
return {
|
|
30
56
|
ok: false,
|
|
31
|
-
error: 'expected a non-empty JSON array of criteria, e.g. [{"statement":"…","verifierType":"MACHINE","checkpointer":"npx jest …"}]',
|
|
57
|
+
error: 'expected a non-empty JSON array of criteria, e.g. [{"statement":"…","verifierType":"MACHINE","checkpointer":"npx jest …","referentKind":"AGENT_CONSTRUCTED_STATE"}]',
|
|
32
58
|
};
|
|
33
59
|
}
|
|
34
60
|
return { ok: true, items: parsed };
|
|
@@ -171,4 +197,17 @@ async function taskCriteriaSet(identifier, options) {
|
|
|
171
197
|
if (data.judgeStepsWarning) {
|
|
172
198
|
process.stdout.write(`⚠ ${(0, sanitize_1.sanitizeField)(data.judgeStepsWarning)}\n`);
|
|
173
199
|
}
|
|
200
|
+
// LUM-605: EXTERNAL_FACT criteria whose checkpointer doesn't ground external
|
|
201
|
+
// — stored, but the verify loop will refuse a PASS on them until re-grounded.
|
|
202
|
+
if (data.referentWarning) {
|
|
203
|
+
process.stdout.write(`⚠ ${(0, sanitize_1.sanitizeField)(data.referentWarning)}\n`);
|
|
204
|
+
}
|
|
205
|
+
const readbackIssues = verifyReadback(criteriaItems, data.criteria);
|
|
206
|
+
if (readbackIssues.length > 0) {
|
|
207
|
+
for (const issue of readbackIssues) {
|
|
208
|
+
console.error(`⚠ landing-integrity: ${(0, sanitize_1.sanitizeField)(issue)}`);
|
|
209
|
+
}
|
|
210
|
+
console.error('Error: stored contract does not match what was submitted — do not trust the success receipt. Re-run and verify.');
|
|
211
|
+
return 1;
|
|
212
|
+
}
|
|
174
213
|
}
|
package/dist/cli/src/index.js
CHANGED
|
@@ -47,9 +47,12 @@ const session_attach_1 = require("./commands/session-attach");
|
|
|
47
47
|
const session_status_1 = require("./commands/session-status");
|
|
48
48
|
const next_1 = require("./commands/next");
|
|
49
49
|
const cost_1 = require("./commands/cost");
|
|
50
|
+
const criteria_audit_1 = require("./commands/criteria-audit");
|
|
51
|
+
const fidelity_1 = require("./commands/fidelity");
|
|
50
52
|
const verify_1 = require("./commands/verify");
|
|
51
53
|
const verdict_1 = require("./commands/verdict");
|
|
52
54
|
const crossing_explain_1 = require("./commands/crossing-explain");
|
|
55
|
+
const outcome_1 = require("./commands/outcome");
|
|
53
56
|
const task_context_1 = require("./commands/task-context");
|
|
54
57
|
const task_create_1 = require("./commands/task-create");
|
|
55
58
|
const task_update_1 = require("./commands/task-update");
|
|
@@ -242,6 +245,26 @@ crossing
|
|
|
242
245
|
.description('Append an agent self-explanation ("申辩") to a boundary crossing (LUM-542). Append-only and for the human reviewer — it never clears the crossing or unblocks Done (a human dispositions that). Targets a crossing on the session-bound task.')
|
|
243
246
|
.requiredOption('--note <text>', 'The explanation to record (the rationale for the action / why it may be a false positive)')
|
|
244
247
|
.action(wrap((id, options) => (0, crossing_explain_1.crossingExplain)(id, options)));
|
|
248
|
+
const outcome = program
|
|
249
|
+
.command('outcome')
|
|
250
|
+
.description('Read / record the post-hoc outcome well (LUM-598)');
|
|
251
|
+
outcome
|
|
252
|
+
.command('show <task>')
|
|
253
|
+
.description('Read the post-hoc outcome well for a task: the falsifier verdict (REJECTED | INCONCLUSIVE — never a "pass") plus its backing rejection signals. INCONCLUSIVE means no rejection on record, NOT satisfied.')
|
|
254
|
+
.action(wrap((task) => (0, outcome_1.outcomeShow)(task)));
|
|
255
|
+
outcome
|
|
256
|
+
.command('record <task>')
|
|
257
|
+
.description('Record a human-observed post-hoc REJECTION of a delivery (revert / rollback / CI regression / downstream redirect / bypass). Append-only; there is no "mark satisfied" counterpart — the well only asserts rejection.')
|
|
258
|
+
.requiredOption('--note <text>', 'What reality did (the observed referent), e.g. "reverted in #812 after a prod incident"')
|
|
259
|
+
.option('--kind <kind>', 'reverted | rolled_back | ci_regression | downstream_redirect | bypassed | manual (case-insensitive; default: manual)')
|
|
260
|
+
.option('--occurred-at <iso>', 'When the event happened (ISO 8601; defaults to now)')
|
|
261
|
+
.action(wrap((task, options) => (0, outcome_1.outcomeRecord)(task, options)));
|
|
262
|
+
outcome
|
|
263
|
+
.command('rate')
|
|
264
|
+
.description('Trust × post-hoc fate: per delivery-time forecast-confidence bracket, the post-hoc REJECTED rate from the outcome well. Honest by construction — thin brackets read "insufficient" and the high-vs-low comparison stays "inconclusive" until the well has enough signal. The rate is a LOWER BOUND (no signal = INCONCLUSIVE, never satisfied).')
|
|
265
|
+
.option('--min <N>', 'Per-bracket sample floor below which a rate is withheld (default 10)')
|
|
266
|
+
.option('--json', 'Emit the report as JSON')
|
|
267
|
+
.action(wrap(options => (0, outcome_1.outcomeRate)(options)));
|
|
245
268
|
program
|
|
246
269
|
.command('next')
|
|
247
270
|
.description('Recommend the next task(s) to work on, ranked by priority, active sprint, and due date. Prints top N (default 3); pick one and run `session attach` + `task context`.')
|
|
@@ -256,6 +279,21 @@ program
|
|
|
256
279
|
.option('--by <dim>', 'Headline grouping: tool | model | member | session (case-insensitive; default tool)')
|
|
257
280
|
.option('--json', 'Emit the versioned payload as JSON')
|
|
258
281
|
.action(wrap(options => (0, cost_1.cost)(options)));
|
|
282
|
+
const criteria = program
|
|
283
|
+
.command('criteria')
|
|
284
|
+
.description('Workspace-level acceptance-criteria analytics');
|
|
285
|
+
criteria
|
|
286
|
+
.command('audit')
|
|
287
|
+
.description('Show the workspace referent-kind distribution + self-confirming-green ratio (GET /api/criteria/audit)')
|
|
288
|
+
.action(wrap(() => (0, criteria_audit_1.criteriaAudit)()));
|
|
289
|
+
const fidelity = program
|
|
290
|
+
.command('fidelity')
|
|
291
|
+
.description('Mechanical fidelity read-outs (LUM-610)');
|
|
292
|
+
fidelity
|
|
293
|
+
.command('show <task>')
|
|
294
|
+
.description('Per delivery snapshot (LUM-609): the grounding composition of the frozen contract (grounded / self-confirming / inconclusive) and the independence signal (criteria ADDED/UPDATED after work started = backward-inference suspects). A composition table, not a single score — direction is never judged. inconclusive is surfaced, never swallowed.')
|
|
295
|
+
.option('--json', 'Emit the report as JSON')
|
|
296
|
+
.action(wrap((task, options) => (0, fidelity_1.fidelityShow)(task, options)));
|
|
259
297
|
const session = program
|
|
260
298
|
.command('session')
|
|
261
299
|
.description('Manage per-terminal coding-session context');
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.REFERENT_KIND_DECLARABLE = void 0;
|
|
4
|
+
exports.classifyCheckpointerGrounding = classifyCheckpointerGrounding;
|
|
5
|
+
exports.effectiveReferentKind = effectiveReferentKind;
|
|
6
|
+
/**
|
|
7
|
+
* referentKind (LUM-602): what a criterion anchors — the only dimension that
|
|
8
|
+
* determines independence. Dependency-light (no @prisma/client) so both the
|
|
9
|
+
* Next.js server and the CLI can import it; the Prisma `ReferentKind` enum
|
|
10
|
+
* mirrors REFERENT_KIND_DECLARABLE exactly.
|
|
11
|
+
*/
|
|
12
|
+
exports.REFERENT_KIND_DECLARABLE = [
|
|
13
|
+
'EXTERNAL_FACT',
|
|
14
|
+
'AGENT_CONSTRUCTED_STATE',
|
|
15
|
+
'PENDING_OUTCOME',
|
|
16
|
+
];
|
|
17
|
+
/** Tools whose output the agent cannot author — external facts. */
|
|
18
|
+
const EXTERNAL_TOOLS = [
|
|
19
|
+
'git',
|
|
20
|
+
'gh',
|
|
21
|
+
'curl',
|
|
22
|
+
'wget',
|
|
23
|
+
'http',
|
|
24
|
+
'https',
|
|
25
|
+
'psql',
|
|
26
|
+
'dig',
|
|
27
|
+
'nc',
|
|
28
|
+
];
|
|
29
|
+
/** Agent's own test/type/build harness — green here is self-confirming. */
|
|
30
|
+
const HARNESS_TOOLS = [
|
|
31
|
+
'jest',
|
|
32
|
+
'vitest',
|
|
33
|
+
'mocha',
|
|
34
|
+
'pytest',
|
|
35
|
+
'tsc',
|
|
36
|
+
'node',
|
|
37
|
+
'ts-node',
|
|
38
|
+
'tsx',
|
|
39
|
+
];
|
|
40
|
+
/**
|
|
41
|
+
* `prisma migrate status` is EXTERNAL (queries DB/migration history); a bare
|
|
42
|
+
* `prisma generate` is harness. Matched as a two-word phrase before the
|
|
43
|
+
* single-word scan.
|
|
44
|
+
*/
|
|
45
|
+
const EXTERNAL_PHRASES = ['prisma migrate status'];
|
|
46
|
+
function wordMatches(word, tools) {
|
|
47
|
+
// exact tool, or a tool-prefixed filename like jest-t.ts / vitest.config
|
|
48
|
+
return tools.some(t => word === t || word.startsWith(`${t}-`) || word.startsWith(`${t}.`));
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* No-op external invocations (LUM-605): forms whose exit code and output are
|
|
52
|
+
* constant w.r.t. external state (`git --version`, `gh --help`, …). An external
|
|
53
|
+
* tool invoked this way establishes no external fact, so it must never ground
|
|
54
|
+
* EXTERNAL — this is half of the C-hole fix (the other half is the consumption
|
|
55
|
+
* requirement below). Kept deliberately small/conservative: a smaller set means
|
|
56
|
+
* fewer real external checks get mistaken for no-ops (we'd rather under-flag a
|
|
57
|
+
* decoy than false-downgrade a genuine check, which enforcement now penalizes).
|
|
58
|
+
*/
|
|
59
|
+
const NOOP_FLAG_TOKENS = new Set(['--version', '--help']);
|
|
60
|
+
/** No-op SUBCOMMAND forms — only when they are the first word after the tool
|
|
61
|
+
* (`git version`, `gh help`). Not matched as a bare flag-anywhere, so we don't
|
|
62
|
+
* trip on an unrelated positional like a psql `-h <host>` (host, not help). */
|
|
63
|
+
const NOOP_SUBCOMMANDS = new Set(['version', 'help']);
|
|
64
|
+
/**
|
|
65
|
+
* Given a raw command word (possibly `npx jest`, `npm run build`, etc.),
|
|
66
|
+
* normalize to the effective tool name. Returns null for pure wrapper tokens
|
|
67
|
+
* (npx, sudo, command) that should cause us to advance to the next word.
|
|
68
|
+
*/
|
|
69
|
+
function normalizeCommandWord(word) {
|
|
70
|
+
// strip a leading path: scripts/jest-t.ts → jest-t.ts (keep basename)
|
|
71
|
+
const slash = word.lastIndexOf('/');
|
|
72
|
+
return slash >= 0 ? word.slice(slash + 1) : word;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Extract command-position words from a shell script string.
|
|
76
|
+
*
|
|
77
|
+
* Algorithm:
|
|
78
|
+
* 1. Strip `#` comments (everything from # to end of line).
|
|
79
|
+
* 2. Extract `$(...)` and backtick substitutions — they are command
|
|
80
|
+
* lists; recurse into them and add their command-position words.
|
|
81
|
+
* 3. For `bash -c '...'` / `sh -c '...'` tokens, extract the quoted
|
|
82
|
+
* payload and recurse into it as a command list.
|
|
83
|
+
* 4. Split the remaining text on command separators (|, &&, ||, ;, \n)
|
|
84
|
+
* to get individual command segments.
|
|
85
|
+
* 5. From each segment, take the first meaningful word as the command.
|
|
86
|
+
* Skip wrapper words (npx, sudo, command) and "npm run" pairs.
|
|
87
|
+
*/
|
|
88
|
+
function collectCommandWords(script) {
|
|
89
|
+
const result = [];
|
|
90
|
+
// Step 1: strip # comments (not inside quotes — simple heuristic: strip
|
|
91
|
+
// from # that is preceded by whitespace or start-of-line to end of line)
|
|
92
|
+
const noComments = script.replace(/(^|\s)#[^\n]*/gm, ' ');
|
|
93
|
+
// Step 2: recurse into $(...) substitutions
|
|
94
|
+
// We use a simple balanced-paren extractor.
|
|
95
|
+
let remaining = noComments;
|
|
96
|
+
remaining = remaining.replace(/\$\(([^)]*)\)/g, (_match, inner) => {
|
|
97
|
+
result.push(...collectCommandWords(inner));
|
|
98
|
+
return ' ';
|
|
99
|
+
});
|
|
100
|
+
// Recurse into backtick substitutions
|
|
101
|
+
remaining = remaining.replace(/`([^`]*)`/g, (_match, inner) => {
|
|
102
|
+
result.push(...collectCommandWords(inner));
|
|
103
|
+
return ' ';
|
|
104
|
+
});
|
|
105
|
+
// Step 3: handle bash -c / sh -c with a single-quoted payload.
|
|
106
|
+
// Pattern: (bash|sh) ... -c '...' or (bash|sh) ... -c "..."
|
|
107
|
+
// We do this BEFORE splitting on separators so the payload content
|
|
108
|
+
// doesn't pollute the segment split.
|
|
109
|
+
remaining = remaining.replace(/\b(?:bash|sh)\b[^'"]*-c\s+'([^']*)'/g, (_match, payload) => {
|
|
110
|
+
result.push(...collectCommandWords(payload));
|
|
111
|
+
return ' ';
|
|
112
|
+
});
|
|
113
|
+
remaining = remaining.replace(/\b(?:bash|sh)\b[^'"]*-c\s+"([^"]*)"/g, (_match, payload) => {
|
|
114
|
+
result.push(...collectCommandWords(payload));
|
|
115
|
+
return ' ';
|
|
116
|
+
});
|
|
117
|
+
// Step 4: split on command separators to get segments.
|
|
118
|
+
// Split on |, &&, ||, ;, newlines. Note: || must be checked before |.
|
|
119
|
+
const segments = remaining.split(/\|\||&&|[|;\n]/);
|
|
120
|
+
// Step 5: from each segment extract the command word.
|
|
121
|
+
for (const seg of segments) {
|
|
122
|
+
// Strip leading/trailing whitespace and any residual shell chars
|
|
123
|
+
// (quotes, $, brackets not already consumed).
|
|
124
|
+
const cleaned = seg
|
|
125
|
+
.replace(/['"]/g, ' ') // remove remaining quotes (string args)
|
|
126
|
+
.replace(/\[|\]/g, ' ') // remove [ ] test brackets
|
|
127
|
+
.replace(/\$\{?[^}]*}?/g, ' ') // remove remaining $VAR references
|
|
128
|
+
.trim();
|
|
129
|
+
if (!cleaned)
|
|
130
|
+
continue;
|
|
131
|
+
const tokens = cleaned.split(/\s+/).filter(Boolean);
|
|
132
|
+
// Walk tokens to find the effective command, skipping wrappers.
|
|
133
|
+
let i = 0;
|
|
134
|
+
while (i < tokens.length) {
|
|
135
|
+
const tok = tokens[i];
|
|
136
|
+
const lower = tok.toLowerCase();
|
|
137
|
+
if (lower === 'npx' || lower === 'sudo' || lower === 'command') {
|
|
138
|
+
i++;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (lower === 'npm' && tokens[i + 1]?.toLowerCase() === 'run') {
|
|
142
|
+
// "npm run <script>" — script name is the command
|
|
143
|
+
i += 2;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
// This is the effective command
|
|
147
|
+
const word = normalizeCommandWord(lower);
|
|
148
|
+
if (word)
|
|
149
|
+
result.push(word);
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return result;
|
|
154
|
+
}
|
|
155
|
+
function isExternalTool(word) {
|
|
156
|
+
return wordMatches(word, EXTERNAL_TOOLS);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Parse one shell command segment (already split off sequencing/pipe operators)
|
|
160
|
+
* into its effective tool + whether it is a no-op invocation. Mirrors the
|
|
161
|
+
* wrapper-skipping of {@link collectCommandWords} (npx / sudo / command / npm
|
|
162
|
+
* run), then reads the post-tool tokens to flag no-op forms (LUM-605).
|
|
163
|
+
*/
|
|
164
|
+
function parseInvocation(segment) {
|
|
165
|
+
const cleaned = segment
|
|
166
|
+
.replace(/['"]/g, ' ')
|
|
167
|
+
.replace(/\[|\]/g, ' ')
|
|
168
|
+
.replace(/\$\{?[^}]*}?/g, ' ')
|
|
169
|
+
.trim();
|
|
170
|
+
if (!cleaned)
|
|
171
|
+
return null;
|
|
172
|
+
const tokens = cleaned.split(/\s+/).filter(Boolean);
|
|
173
|
+
let i = 0;
|
|
174
|
+
while (i < tokens.length) {
|
|
175
|
+
const lower = tokens[i].toLowerCase();
|
|
176
|
+
if (lower === 'npx' || lower === 'sudo' || lower === 'command') {
|
|
177
|
+
i++;
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (lower === 'npm' && tokens[i + 1]?.toLowerCase() === 'run') {
|
|
181
|
+
i += 2;
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
if (i >= tokens.length)
|
|
187
|
+
return null;
|
|
188
|
+
const tool = normalizeCommandWord(tokens[i].toLowerCase());
|
|
189
|
+
const rest = tokens.slice(i + 1).map(t => t.toLowerCase());
|
|
190
|
+
const isNoop = rest.some(t => NOOP_FLAG_TOKENS.has(t)) ||
|
|
191
|
+
(rest.length > 0 && NOOP_SUBCOMMANDS.has(rest[0]));
|
|
192
|
+
return { tool, isNoop };
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Does this script contain a CONSUMED, non-no-op external command (LUM-605)?
|
|
196
|
+
* "Consumed" — the discriminator that closes the C hole — means the external
|
|
197
|
+
* command's output or exit actually reaches the check's verdict:
|
|
198
|
+
* - captured in a `$(...)` / backtick substitution (output → data → assertion);
|
|
199
|
+
* - piped into a downstream stage (`ext | …`);
|
|
200
|
+
* - the terminal command of the whole script (its exit status IS the verdict).
|
|
201
|
+
* A non-terminal external word whose output is discarded (the decoy
|
|
202
|
+
* `git --version && cat frozen.txt | grep`), or any no-op invocation, does NOT
|
|
203
|
+
* ground EXTERNAL.
|
|
204
|
+
*/
|
|
205
|
+
function groundsExternal(script) {
|
|
206
|
+
const noComments = script.replace(/(^|\s)#[^\n]*/gm, ' ');
|
|
207
|
+
return scanScript(noComments);
|
|
208
|
+
}
|
|
209
|
+
/** Pull out substitutions / `bash -c` payloads (recursing into them) then walk
|
|
210
|
+
* statements → `&&`/`||` segments → pipeline stages to test each consumed slot. */
|
|
211
|
+
function scanScript(text) {
|
|
212
|
+
let remaining = text;
|
|
213
|
+
let foundInNested = false;
|
|
214
|
+
remaining = remaining.replace(/\$\(([^)]*)\)/g, (_m, inner) => {
|
|
215
|
+
if (scanConsumed(inner))
|
|
216
|
+
foundInNested = true;
|
|
217
|
+
return ' ';
|
|
218
|
+
});
|
|
219
|
+
remaining = remaining.replace(/`([^`]*)`/g, (_m, inner) => {
|
|
220
|
+
if (scanConsumed(inner))
|
|
221
|
+
foundInNested = true;
|
|
222
|
+
return ' ';
|
|
223
|
+
});
|
|
224
|
+
remaining = remaining.replace(/\b(?:bash|sh)\b[^'"]*-c\s+'([^']*)'/g, (_m, payload) => {
|
|
225
|
+
if (scanScript(payload))
|
|
226
|
+
foundInNested = true;
|
|
227
|
+
return ' ';
|
|
228
|
+
});
|
|
229
|
+
remaining = remaining.replace(/\b(?:bash|sh)\b[^'"]*-c\s+"([^"]*)"/g, (_m, payload) => {
|
|
230
|
+
if (scanScript(payload))
|
|
231
|
+
foundInNested = true;
|
|
232
|
+
return ' ';
|
|
233
|
+
});
|
|
234
|
+
if (foundInNested)
|
|
235
|
+
return true;
|
|
236
|
+
const statements = remaining
|
|
237
|
+
.split(/;|\n/)
|
|
238
|
+
.map(s => s.trim())
|
|
239
|
+
.filter(Boolean);
|
|
240
|
+
for (let si = 0; si < statements.length; si++) {
|
|
241
|
+
const terminalStatement = si === statements.length - 1;
|
|
242
|
+
const segments = statements[si].split(/&&|\|\|/)
|
|
243
|
+
.map(s => s.trim())
|
|
244
|
+
.filter(Boolean);
|
|
245
|
+
for (let gi = 0; gi < segments.length; gi++) {
|
|
246
|
+
const terminalSegment = gi === segments.length - 1;
|
|
247
|
+
const stages = segments[gi].split(/\|/)
|
|
248
|
+
.map(s => s.trim())
|
|
249
|
+
.filter(Boolean);
|
|
250
|
+
for (let pi = 0; pi < stages.length; pi++) {
|
|
251
|
+
const finalStage = pi === stages.length - 1;
|
|
252
|
+
const inv = parseInvocation(stages[pi]);
|
|
253
|
+
if (!inv || !isExternalTool(inv.tool) || inv.isNoop)
|
|
254
|
+
continue;
|
|
255
|
+
const pipedIntoConsumer = !finalStage;
|
|
256
|
+
const exitIsVerdict = finalStage && terminalSegment && terminalStatement;
|
|
257
|
+
if (pipedIntoConsumer || exitIsVerdict)
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
/** Substitution context: the captured output is data fed to the surrounding
|
|
265
|
+
* assertion, so a non-no-op external command word in ANY position is consumed. */
|
|
266
|
+
function scanConsumed(inner) {
|
|
267
|
+
let remaining = inner;
|
|
268
|
+
let found = false;
|
|
269
|
+
remaining = remaining.replace(/\$\(([^)]*)\)/g, (_m, x) => {
|
|
270
|
+
if (scanConsumed(x))
|
|
271
|
+
found = true;
|
|
272
|
+
return ' ';
|
|
273
|
+
});
|
|
274
|
+
remaining = remaining.replace(/`([^`]*)`/g, (_m, x) => {
|
|
275
|
+
if (scanConsumed(x))
|
|
276
|
+
found = true;
|
|
277
|
+
return ' ';
|
|
278
|
+
});
|
|
279
|
+
if (found)
|
|
280
|
+
return true;
|
|
281
|
+
const segments = remaining
|
|
282
|
+
.split(/&&|\|\||;|\||\n/)
|
|
283
|
+
.map(s => s.trim())
|
|
284
|
+
.filter(Boolean);
|
|
285
|
+
for (const seg of segments) {
|
|
286
|
+
const inv = parseInvocation(seg);
|
|
287
|
+
if (inv && isExternalTool(inv.tool) && !inv.isNoop)
|
|
288
|
+
return true;
|
|
289
|
+
}
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
function classifyCheckpointerGrounding(checkpointer) {
|
|
293
|
+
if (!checkpointer || checkpointer.trim() === '')
|
|
294
|
+
return 'NONE';
|
|
295
|
+
const lower = checkpointer.toLowerCase();
|
|
296
|
+
// Fast path for known two-word external phrases (e.g. prisma migrate status)
|
|
297
|
+
if (EXTERNAL_PHRASES.some(p => lower.includes(p)))
|
|
298
|
+
return 'EXTERNAL';
|
|
299
|
+
const words = collectCommandWords(lower);
|
|
300
|
+
if (words.length === 0)
|
|
301
|
+
return 'NONE';
|
|
302
|
+
// Strongest signal wins: EXTERNAL > AGENT_HARNESS > ASSERTION_ONLY. EXTERNAL
|
|
303
|
+
// is gated on a CONSUMED, non-no-op external command (LUM-605) — a no-op or
|
|
304
|
+
// output-discarded external word falls through to harness/assertion.
|
|
305
|
+
if (groundsExternal(lower))
|
|
306
|
+
return 'EXTERNAL';
|
|
307
|
+
if (words.some(w => wordMatches(w, HARNESS_TOOLS)))
|
|
308
|
+
return 'AGENT_HARNESS';
|
|
309
|
+
return 'ASSERTION_ONLY';
|
|
310
|
+
}
|
|
311
|
+
function effectiveReferentKind(args) {
|
|
312
|
+
const { declared, verifierType, checkpointer } = args;
|
|
313
|
+
if (declared === null)
|
|
314
|
+
return 'UNCLASSIFIED';
|
|
315
|
+
if (declared === 'EXTERNAL_FACT' && verifierType === 'MACHINE') {
|
|
316
|
+
if (classifyCheckpointerGrounding(checkpointer) !== 'EXTERNAL') {
|
|
317
|
+
return 'UNVERIFIED_ASSERTION';
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
return declared;
|
|
321
|
+
}
|