@windyroad/risk-scorer 0.17.4 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +13 -2
- package/bin/install.mjs +42 -8
- package/hooks/codex-agent-completion.mjs +82 -0
- package/hooks/hooks.json +5 -2
- package/hooks/risk-scorer-dispatch.sh +3 -0
- package/hooks/risk-scorer-scaffold-nudge.sh +15 -4
- package/package.json +5 -2
- package/scripts/codex-agents.mjs +208 -0
- package/scripts/sync-codex-skills.mjs +8 -3
- package/skills/assess-external-comms/SKILL.md +5 -4
- package/skills/assess-inbound-report/SKILL.md +5 -4
- package/skills/assess-release/SKILL.md +5 -4
- package/skills/assess-wip/SKILL.md +5 -4
- package/skills/bootstrap-catalog/SKILL.md +5 -4
- package/skills/create-risk/SKILL.md +5 -4
- package/skills/external-comms/SKILL.md +5 -4
- package/skills/pipeline/SKILL.md +5 -4
- package/skills/update-policy/SKILL.md +6 -5
- package/skills/wip/SKILL.md +5 -4
- package/agents/test/inbound-report-contract.bats +0 -225
- package/agents/test/risk-scorer-above-appetite-stop.bats +0 -80
- package/agents/test/risk-scorer-catalog-consumption.bats +0 -138
- package/agents/test/risk-scorer-commit-verdict.bats +0 -66
- package/agents/test/risk-scorer-monitoring-not-a-control.bats +0 -76
- package/agents/test/risk-scorer-reducing-bypass-criteria.bats +0 -62
- package/agents/test/risk-scorer-register-hint.bats +0 -111
- package/agents/test/risk-scorer-structured-remediations.bats +0 -123
- package/agents/test/risk-scorer-user-stated-preconditions.bats +0 -89
- package/hooks/test/ci-status-gate.bats +0 -234
- package/hooks/test/external-comms-gate.bats +0 -528
- package/hooks/test/external-comms-key-substance.bats +0 -74
- package/hooks/test/git-push-gate.bats +0 -82
- package/hooks/test/pipeline-state-changeset-partition.bats +0 -181
- package/hooks/test/pipeline-state-hash.bats +0 -195
- package/hooks/test/plan-risk-guidance-once-per-session.bats +0 -95
- package/hooks/test/reducing-marker-persistence.bats +0 -236
- package/hooks/test/risk-gate.bats +0 -330
- package/hooks/test/risk-policy-project-root.bats +0 -20
- package/hooks/test/risk-score-commit-gate-adr-031-bypass.bats +0 -98
- package/hooks/test/risk-score-commit-gate-cadence-staleness.bats +0 -105
- package/hooks/test/risk-score-mark-external-comms-prompt-parse.bats +0 -94
- package/hooks/test/risk-score-mark-register-queue.bats +0 -253
- package/hooks/test/risk-score-mark.bats +0 -112
- package/hooks/test/risk-scorer-hook-dispatch.bats +0 -77
- package/hooks/test/risk-scorer-no-stop-hook.bats +0 -19
- package/hooks/test/risk-scorer-scaffold-nudge.bats +0 -132
- package/hooks/test/slide-marker-on-subprocess-return.bats +0 -118
- package/hooks/test/wip-risk-mark.bats +0 -64
- package/scripts/test/drain-register-queue.bats +0 -359
- package/scripts/test/evaluate-graduation.bats +0 -533
- package/scripts/test/extract-risks-from-reports.bats +0 -366
- package/scripts/test/restage-commit.bats +0 -137
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:assess-wip
|
|
7
3
|
description: On-demand WIP risk nudge. Scores the current uncommitted diff for pipeline risk. Use during development to catch high-risk changes before committing.
|
|
8
4
|
allowed-tools: Read, Glob, Grep, Bash, request_user_input, Skill
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# WIP Risk Assessment Skill
|
|
12
13
|
|
|
13
14
|
Run a WIP (work-in-progress) risk assessment on demand. Scores the current uncommitted diff — staged and unstaged — for pipeline risk. Use during development to get early feedback before committing.
|
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:bootstrap-catalog
|
|
7
3
|
description: Bootstrap docs/risks/ standing-risk catalog from existing .risk-reports/ corpus. Walks reports, dedupes by ADR-056 slug, emits one R<NNN>-<slug>.active.md per unique slug with ## Source Evidence block citing originating reports. Idempotent — re-runs are no-ops by file-existence per slug. One-shot per project lifetime; install-updates Step 6.5 auto-triggers when catalog is empty AND .risk-reports/ is non-empty AND RISK-POLICY.md is present.
|
|
@@ -9,6 +5,11 @@ allowed-tools: Read, Write, Edit, Bash, Glob, Grep
|
|
|
9
5
|
maturity: proposed
|
|
10
6
|
---
|
|
11
7
|
|
|
8
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
9
|
+
|
|
10
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
11
|
+
|
|
12
|
+
|
|
12
13
|
# Risk Catalog Bootstrap
|
|
13
14
|
|
|
14
15
|
Walk the project's `.risk-reports/*.md` corpus, dedupe by ADR-056 risk-slug, and emit one `R<NNN>-<slug>.active.md` register entry per unique slug with a `## Source Evidence` block citing originating reports. This is the **one-shot historical backfill** that Phase 3 of ADR-047's roadmap deferred — now landed via ADR-059.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:create-risk
|
|
7
3
|
description: Create a new standing-risk entry in docs/risks/. Examines existing risks, gathers impact/likelihood/controls from the user, writes a file using the entry shape inlined in this skill (no TEMPLATE.md dependency — the entry shape is owned by this skill per user direction 2026-05-04), and updates the register index.
|
|
8
4
|
allowed-tools: Read, Write, Edit, Bash, Glob, Grep, request_user_input
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# Risk Register Entry Generator
|
|
12
13
|
|
|
13
14
|
Create a new standing-risk file in `docs/risks/` using the entry shape inlined in this skill (Step 5 below). The register captures persistent risks (distinct from the ephemeral per-change reports in `.risk-reports/`), and its criteria come from `RISK-POLICY.md`. **No `docs/risks/TEMPLATE.md` exists** — per user direction 2026-05-04 the entry shape lives in this skill (and the bootstrap-catalog extractor that reuses the same shape) so the directory is purely the inventory, not its own scaffolding.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:external-comms
|
|
7
3
|
description: Invokable SKILL wrapper around the wr-risk-scorer:external-comms leak-review agent. Delegates to the agent via the Agent tool and returns the agent's structured EXTERNAL_COMMS_RISK_VERDICT. Internal-use plumbing used by `/wr-risk-scorer:assess-external-comms` per ADR-015's Confirmation literal phrasing. End users should invoke `/wr-risk-scorer:assess-external-comms` instead.
|
|
8
4
|
allowed-tools: Read, Glob, Grep, Bash, Agent
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# External-Comms Leak Review Skill (Wrapper)
|
|
12
13
|
|
|
13
14
|
This SKILL is an **invokable wrapper** around the `wr-risk-scorer:external-comms` agent. It exists so consumer SKILLs can invoke the leak reviewer via the **Skill tool** with `skill: wr-risk-scorer:external-comms` — matching ADR-015's Confirmation literal phrasing.
|
package/skills/pipeline/SKILL.md
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:pipeline
|
|
7
3
|
description: Invokable SKILL wrapper around the wr-risk-scorer:pipeline scoring agent. Delegates to the agent via the Agent tool and returns the agent's structured RISK_SCORES output. Internal-use plumbing used by `/wr-risk-scorer:assess-release` and any other consumer SKILL that needs Skill-tool-shaped invocation of the pipeline scorer per ADR-015's Confirmation literal phrasing. End users should invoke `/wr-risk-scorer:assess-release` instead.
|
|
8
4
|
allowed-tools: Read, Glob, Bash, Agent
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# Pipeline Scoring Skill (Wrapper)
|
|
12
13
|
|
|
13
14
|
This SKILL is an **invokable wrapper** around the `wr-risk-scorer:pipeline` agent. It exists so consumer SKILLs can invoke the pipeline scorer via the **Skill tool** with `skill: wr-risk-scorer:pipeline` — matching ADR-015's Confirmation literal phrasing.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:update-policy
|
|
7
3
|
description: Create or update the project's RISK-POLICY.md per ISO 31000 and the risk-scorer agent. Examines the project to derive business-specific impact levels.
|
|
8
4
|
allowed-tools: Read, Write, Edit, Bash, Glob, Grep, request_user_input, Agent
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# Risk Policy Generator
|
|
12
13
|
|
|
13
14
|
Create or update `RISK-POLICY.md` per ISO 31000, tailored to this project's business context. The risk-scorer agent reads this file to score pipeline actions (commit, push, release).
|
|
@@ -168,7 +169,7 @@ Run the Codex policy reviewer with this prompt:
|
|
|
168
169
|
>
|
|
169
170
|
> [paste the full draft policy content here]
|
|
170
171
|
|
|
171
|
-
Use a native Codex subagent workflow
|
|
172
|
+
Use a native Codex subagent workflow with the installed custom agent named `wr-risk-scorer:policy`. If it is not visible in the current session, restart Codex; do not substitute a different agent identity because the policy marker hook consumes it.
|
|
172
173
|
|
|
173
174
|
The risk-scorer will check:
|
|
174
175
|
- Impact levels describe business consequences (not file categories)
|
package/skills/wip/SKILL.md
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
2
|
-
|
|
3
|
-
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow: spawn the matching installed Codex custom agent when available, otherwise spawn the built-in `default` subagent and instruct it to read the plugin's sibling `agents/*.md` instructions in full before returning the same structured verdict block.
|
|
4
|
-
|
|
5
1
|
---
|
|
6
2
|
name: wr-risk-scorer:wip
|
|
7
3
|
description: Invokable SKILL wrapper around the wr-risk-scorer:wip nudge agent. Delegates to the agent via the Agent tool and returns the agent's structured WIP risk verdict. Internal-use plumbing used by `/wr-risk-scorer:assess-wip` per ADR-015's Confirmation literal phrasing. End users should invoke `/wr-risk-scorer:assess-wip` instead.
|
|
8
4
|
allowed-tools: Read, Glob, Bash, Agent
|
|
9
5
|
---
|
|
10
6
|
|
|
7
|
+
<!-- Generated from packages/risk-scorer/skills/*/SKILL.md by packages/risk-scorer/scripts/sync-codex-skills.mjs during npm pack. Do not edit packaged output directly. -->
|
|
8
|
+
|
|
9
|
+
> Codex runtime note: use `request_user_input` only in Plan Mode where this skill needs structured user input. Outside Plan Mode, ask one concise direct question only when no safe assumption exists. If a step refers to Claude-style agent dispatch or `subagent_type`, use a native Codex subagent workflow and spawn the matching installed `wr-risk-scorer:<mode>` custom agent. Wait for it, then close that same agent so the completion hook receives its final response. Do not substitute the built-in `default` agent when a hook consumes the subagent identity; restart Codex if the installed agent is not yet visible.
|
|
10
|
+
|
|
11
|
+
|
|
11
12
|
# WIP Scoring Skill (Wrapper)
|
|
12
13
|
|
|
13
14
|
This SKILL is an **invokable wrapper** around the `wr-risk-scorer:wip` agent. It exists so consumer SKILLs can invoke the WIP nudge scorer via the **Skill tool** with `skill: wr-risk-scorer:wip` — matching ADR-015's Confirmation literal phrasing.
|
|
@@ -1,225 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
# Contract assertions for the wr-risk-scorer:inbound-report subagent
|
|
3
|
-
# (RFC-004 Slice B). Sibling of wr-risk-scorer:external-comms — NOT
|
|
4
|
-
# extension. Reviews INBOUND third-party prose on two axes (Request-risk +
|
|
5
|
-
# Fix-risk) per RISK-POLICY.md § Inbound Report Risk Classes.
|
|
6
|
-
#
|
|
7
|
-
# Structural assertions — Permitted Exception to the source-grep ban
|
|
8
|
-
# per ADR-005 / P011 / ADR-037 / ADR-052 § Surface 2. Subagent prompt
|
|
9
|
-
# prose governs LLM-driven verdict behaviour; behavioural-replay
|
|
10
|
-
# testing requires a synthetic agent harness (P012 / P176). Until that
|
|
11
|
-
# harness lands, contract bats assert the load-bearing rubric + structured
|
|
12
|
-
# verdict format are present so future edits don't silently strip them.
|
|
13
|
-
#
|
|
14
|
-
# @problem P079
|
|
15
|
-
# @rfc RFC-004
|
|
16
|
-
# @adr ADR-062 (inbound discovery + assessment pipeline — § Sibling subagent)
|
|
17
|
-
# @adr ADR-015 (on-demand assessment skills — § Scope table)
|
|
18
|
-
# @adr ADR-026 (grounding discipline — every FAIL verdict cites policy class)
|
|
19
|
-
# @adr ADR-029 (diagnose before implement — hypothesis / evidence / structured verdict)
|
|
20
|
-
# @adr ADR-052 (behavioural-tests default + Permitted Exception)
|
|
21
|
-
# @jtbd JTBD-301 (acknowledgement contract grounded in policy classes)
|
|
22
|
-
# @jtbd JTBD-001 (mechanical-stage carve-out via structured verdict)
|
|
23
|
-
|
|
24
|
-
setup() {
|
|
25
|
-
AGENTS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
26
|
-
AGENT_FILE="${AGENTS_DIR}/inbound-report.md"
|
|
27
|
-
POLICY_FILE="$(cd "${AGENTS_DIR}/../../.." && pwd)/RISK-POLICY.md"
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
31
|
-
# Frontmatter + tool surface
|
|
32
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
33
|
-
|
|
34
|
-
@test "inbound-report.md exists and has frontmatter (RFC-004 Slice B)" {
|
|
35
|
-
[ -f "$AGENT_FILE" ]
|
|
36
|
-
run head -1 "$AGENT_FILE"
|
|
37
|
-
[ "$status" -eq 0 ]
|
|
38
|
-
[ "$output" = "---" ]
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
@test "frontmatter name is 'inbound-report' (sibling of external-comms)" {
|
|
42
|
-
run grep -nE '^name: inbound-report$' "$AGENT_FILE"
|
|
43
|
-
[ "$status" -eq 0 ]
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
@test "frontmatter tools are read-only (Read, Glob, Grep)" {
|
|
47
|
-
# Per ADR-062 § Sibling subagent: read-only contract; subagent emits
|
|
48
|
-
# verdict, PostToolUse hook owns marker writes.
|
|
49
|
-
run grep -nE ' - Read' "$AGENT_FILE"
|
|
50
|
-
[ "$status" -eq 0 ]
|
|
51
|
-
run grep -nE ' - Glob' "$AGENT_FILE"
|
|
52
|
-
[ "$status" -eq 0 ]
|
|
53
|
-
run grep -nE ' - Grep' "$AGENT_FILE"
|
|
54
|
-
[ "$status" -eq 0 ]
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
@test "frontmatter tools do NOT include Write / Edit / Bash (read-only invariant)" {
|
|
58
|
-
run grep -nE '^ - (Write|Edit|Bash)$' "$AGENT_FILE"
|
|
59
|
-
[ "$status" -ne 0 ]
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
@test "frontmatter model is inherit" {
|
|
63
|
-
run grep -nE '^model: inherit$' "$AGENT_FILE"
|
|
64
|
-
[ "$status" -eq 0 ]
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
68
|
-
# Sibling-not-extension positioning (ADR-062 § Sibling subagent)
|
|
69
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
70
|
-
|
|
71
|
-
@test "agent prose names sibling-not-extension positioning vs external-comms" {
|
|
72
|
-
# ADR-062 explicitly carves the inbound-report subagent as a sibling
|
|
73
|
-
# (NOT extension) of external-comms. Protects JTBD-101 plugin-developer
|
|
74
|
-
# constraint "must not break existing plugins" by preserving
|
|
75
|
-
# external-comms scope-purity.
|
|
76
|
-
run grep -inE 'sibling.*external-comms|external-comms.*sibling' "$AGENT_FILE"
|
|
77
|
-
[ "$status" -eq 0 ]
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
@test "agent prose names the inbound-direction framing (third-party prose flowing INWARD)" {
|
|
81
|
-
run grep -inE 'INWARD|inbound prose|third-party prose' "$AGENT_FILE"
|
|
82
|
-
[ "$status" -eq 0 ]
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
86
|
-
# Two-axis review structure (Request-risk + Fix-risk)
|
|
87
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
88
|
-
|
|
89
|
-
@test "Axis 1 Request-risk documented (attack-vector axis)" {
|
|
90
|
-
run grep -nE 'Axis 1.*Request-risk' "$AGENT_FILE"
|
|
91
|
-
[ "$status" -eq 0 ]
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
@test "Axis 1 enumerates info-extraction / backdoor request / malicious-code injection classes" {
|
|
95
|
-
run grep -inE 'Info-extraction' "$AGENT_FILE"
|
|
96
|
-
[ "$status" -eq 0 ]
|
|
97
|
-
run grep -inE 'Backdoor request' "$AGENT_FILE"
|
|
98
|
-
[ "$status" -eq 0 ]
|
|
99
|
-
run grep -inE 'Malicious-code injection' "$AGENT_FILE"
|
|
100
|
-
[ "$status" -eq 0 ]
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
@test "Axis 2 Fix-risk documented (work-to-be-weighed axis)" {
|
|
104
|
-
run grep -nE 'Axis 2.*Fix-risk' "$AGENT_FILE"
|
|
105
|
-
[ "$status" -eq 0 ]
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
@test "Axis 2 enumerates privilege escalation / removal-of-safety-check / adopter-attack-surface-expansion classes" {
|
|
109
|
-
run grep -inE 'Privilege escalation' "$AGENT_FILE"
|
|
110
|
-
[ "$status" -eq 0 ]
|
|
111
|
-
run grep -inE 'Removal of load-bearing safety check' "$AGENT_FILE"
|
|
112
|
-
[ "$status" -eq 0 ]
|
|
113
|
-
run grep -inE 'Adopter-attack-surface expansion' "$AGENT_FILE"
|
|
114
|
-
[ "$status" -eq 0 ]
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
118
|
-
# Structured verdict block (consumed by assessment-pipeline branch routing)
|
|
119
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
120
|
-
|
|
121
|
-
@test "verdict block defines INBOUND_REPORT_VERDICT" {
|
|
122
|
-
run grep -nE 'INBOUND_REPORT_VERDICT' "$AGENT_FILE"
|
|
123
|
-
[ "$status" -eq 0 ]
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
@test "verdict block defines INBOUND_REPORT_KEY (sha256 hex for marker matching)" {
|
|
127
|
-
run grep -nE 'INBOUND_REPORT_KEY' "$AGENT_FILE"
|
|
128
|
-
[ "$status" -eq 0 ]
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
@test "verdict block defines INBOUND_REPORT_CLASS (one of four classifications)" {
|
|
132
|
-
run grep -nE 'INBOUND_REPORT_CLASS' "$AGENT_FILE"
|
|
133
|
-
[ "$status" -eq 0 ]
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
@test "verdict block defines INBOUND_REPORT_REASON for FAIL path" {
|
|
137
|
-
run grep -nE 'INBOUND_REPORT_REASON' "$AGENT_FILE"
|
|
138
|
-
[ "$status" -eq 0 ]
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
142
|
-
# Four classifications enumerated (branch-routing vocabulary)
|
|
143
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
144
|
-
|
|
145
|
-
@test "classification safe-low-fix-risk enumerated" {
|
|
146
|
-
run grep -nE 'safe-low-fix-risk' "$AGENT_FILE"
|
|
147
|
-
[ "$status" -eq 0 ]
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
@test "classification safe-high-fix-risk enumerated" {
|
|
151
|
-
run grep -nE 'safe-high-fix-risk' "$AGENT_FILE"
|
|
152
|
-
[ "$status" -eq 0 ]
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
@test "classification above-threshold-risk enumerated" {
|
|
156
|
-
run grep -nE 'above-threshold-risk' "$AGENT_FILE"
|
|
157
|
-
[ "$status" -eq 0 ]
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
@test "classification clear-malicious-request enumerated" {
|
|
161
|
-
run grep -nE 'clear-malicious-request' "$AGENT_FILE"
|
|
162
|
-
[ "$status" -eq 0 ]
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
166
|
-
# Grounding discipline (ADR-026)
|
|
167
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
168
|
-
|
|
169
|
-
@test "FAIL verdict requires citing the specific RISK-POLICY.md class" {
|
|
170
|
-
run grep -inE 'cite|class violated' "$AGENT_FILE"
|
|
171
|
-
[ "$status" -eq 0 ]
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
@test "agent prose cites ADR-026 grounding discipline" {
|
|
175
|
-
run grep -nE 'ADR-026' "$AGENT_FILE"
|
|
176
|
-
[ "$status" -eq 0 ]
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
180
|
-
# Read-only constraints + marker boundary
|
|
181
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
182
|
-
|
|
183
|
-
@test "agent declares read-only (no file writes / commits / draft modifications)" {
|
|
184
|
-
run grep -inE 'read-only' "$AGENT_FILE"
|
|
185
|
-
[ "$status" -eq 0 ]
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
@test "agent forbids self-writing to /tmp/ or marker locations" {
|
|
189
|
-
# PostToolUse hook owns marker writes per ADR-009; the subagent
|
|
190
|
-
# emits the verdict and the hook computes the marker key.
|
|
191
|
-
run grep -inE 'NOT write to /tmp|PostToolUse hook owns' "$AGENT_FILE"
|
|
192
|
-
[ "$status" -eq 0 ]
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
196
|
-
# Mechanical-stage carve-out integration (P132 — pipeline branch routing)
|
|
197
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
198
|
-
|
|
199
|
-
@test "agent prose names the mechanical-stage carve-out integration (P132)" {
|
|
200
|
-
run grep -nE 'P132|mechanical' "$AGENT_FILE"
|
|
201
|
-
[ "$status" -eq 0 ]
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
@test "agent does NOT make a block-list decision (P123 scope carve-out)" {
|
|
205
|
-
# Block-list enforcement is a separate ticket's concern; this subagent's
|
|
206
|
-
# verdict feeds the audit-log via the assessment-pipeline's clear-malicious
|
|
207
|
-
# branch and stops there.
|
|
208
|
-
run grep -inE 'NOT make a block-list|P123' "$AGENT_FILE"
|
|
209
|
-
[ "$status" -eq 0 ]
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
213
|
-
# RISK-POLICY.md integration (the policy classes the agent grounds verdicts against)
|
|
214
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
215
|
-
|
|
216
|
-
@test "RISK-POLICY.md has the '## Inbound Report Risk Classes' section the agent reads" {
|
|
217
|
-
[ -f "$POLICY_FILE" ]
|
|
218
|
-
run grep -nE '^## Inbound Report Risk Classes$' "$POLICY_FILE"
|
|
219
|
-
[ "$status" -eq 0 ]
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
@test "agent prose references RISK-POLICY.md § Inbound Report Risk Classes" {
|
|
223
|
-
run grep -nE 'Inbound Report Risk Classes' "$AGENT_FILE"
|
|
224
|
-
[ "$status" -eq 0 ]
|
|
225
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
# Doc-lint guard: risk-scorer agent prompts must contain an explicit
|
|
3
|
-
# STOP / do-not-proceed directive in their Above-Appetite sections.
|
|
4
|
-
#
|
|
5
|
-
# Structural assertions — Permitted Exception to the source-grep ban (ADR-005 / P011).
|
|
6
|
-
# These tests assert that the pipeline, wip, and plan scorer prompts forbid
|
|
7
|
-
# "Proceed", "Continue", or "You may ship" nudges when cumulative risk
|
|
8
|
-
# exceeds appetite.
|
|
9
|
-
#
|
|
10
|
-
# Background: P037 identified that scorer reports could include "Proceed
|
|
11
|
-
# with release" or similar nudge language even when residual risk exceeded
|
|
12
|
-
# appetite. The hook gate then correctly blocked the action, but only after
|
|
13
|
-
# the agent wasted tool calls and tokens acting on the nudge. The scorer
|
|
14
|
-
# is not the primary decision-maker, but its verbal verdict must match the
|
|
15
|
-
# structured score — ambiguous "proceed" language undermines this.
|
|
16
|
-
#
|
|
17
|
-
# The Below-Appetite Output Rule (ADR-013 Rule 5) already requires silent
|
|
18
|
-
# policy-authorised release when all scores are within appetite. This guard
|
|
19
|
-
# enforces the inverse: an explicit STOP directive above appetite.
|
|
20
|
-
#
|
|
21
|
-
# Cross-reference:
|
|
22
|
-
# P037: docs/problems/037-risk-scorer-proceeds-above-appetite.open.md
|
|
23
|
-
# ADR-013: docs/decisions/013-structured-user-interaction-for-governance-decisions.proposed.md
|
|
24
|
-
# @jtbd JTBD-001 (enforce governance without slowing down)
|
|
25
|
-
# @jtbd JTBD-002 (ship with confidence — verbal verdict must match structured score)
|
|
26
|
-
# @jtbd JTBD-202 (pre-flight governance — structured output is the only sanctioned channel)
|
|
27
|
-
|
|
28
|
-
setup() {
|
|
29
|
-
AGENTS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
30
|
-
PIPELINE="${AGENTS_DIR}/pipeline.md"
|
|
31
|
-
WIP="${AGENTS_DIR}/wip.md"
|
|
32
|
-
PLAN="${AGENTS_DIR}/plan.md"
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
36
|
-
# pipeline.md: Above-Appetite STOP directive
|
|
37
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
38
|
-
|
|
39
|
-
@test "pipeline.md Above-Appetite section contains explicit STOP directive" {
|
|
40
|
-
# Must contain the word STOP (or BLOCKED) as the verdict above appetite.
|
|
41
|
-
run grep -qE "STOP|BLOCKED" "$PIPELINE"
|
|
42
|
-
[ "$status" -eq 0 ]
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
@test "pipeline.md Above-Appetite section forbids Proceed nudges" {
|
|
46
|
-
# Must explicitly forbid emitting "Proceed" / "Continue" nudges
|
|
47
|
-
# when risk exceeds appetite.
|
|
48
|
-
run grep -qE "[Dd]o NOT emit.*Proceed|forbid.*Proceed|not emit.*Continue|must not.*proceed" "$PIPELINE"
|
|
49
|
-
[ "$status" -eq 0 ]
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
53
|
-
# wip.md: Above-Appetite STOP directive
|
|
54
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
55
|
-
|
|
56
|
-
@test "wip.md Above-Appetite section contains explicit STOP directive" {
|
|
57
|
-
# PAUSE is the wip-mode verdict equivalent of STOP.
|
|
58
|
-
run grep -qE "STOP|BLOCKED|PAUSE" "$WIP"
|
|
59
|
-
[ "$status" -eq 0 ]
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
@test "wip.md Above-Appetite section forbids Proceed nudges" {
|
|
63
|
-
run grep -qE "[Dd]o NOT emit.*Proceed|forbid.*Proceed|not emit.*Continue|must not.*proceed" "$WIP"
|
|
64
|
-
[ "$status" -eq 0 ]
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
68
|
-
# plan.md: FAIL directive (plan-mode equivalent of STOP)
|
|
69
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
70
|
-
|
|
71
|
-
@test "plan.md FAIL section contains explicit STOP directive" {
|
|
72
|
-
# FAIL is the plan-mode verdict; reinforces STOP language.
|
|
73
|
-
run grep -qE "STOP|BLOCKED|FAIL" "$PLAN"
|
|
74
|
-
[ "$status" -eq 0 ]
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
@test "plan.md FAIL section forbids Proceed nudges" {
|
|
78
|
-
run grep -qE "[Dd]o NOT emit.*Proceed|forbid.*Proceed|not emit.*Continue|must not.*proceed" "$PLAN"
|
|
79
|
-
[ "$status" -eq 0 ]
|
|
80
|
-
}
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bats
|
|
2
|
-
# Doc-lint guard: pipeline scorer MUST define the catalog consumption protocol
|
|
3
|
-
# per ADR-059 — read docs/risks/ first, hybrid filter (slug-token primary,
|
|
4
|
-
# judgement fallback), residual reconciliation (per-action residual in
|
|
5
|
-
# RISK_SCORES, catalog lifetime baseline in risk-item block), per-run
|
|
6
|
-
# CATALOG_HIT_RATE observability line.
|
|
7
|
-
#
|
|
8
|
-
# Structural assertions — Permitted Exception to the source-grep ban (ADR-005 / P011).
|
|
9
|
-
# Agent prompts are specification documents; behavioural verification of an LLM's
|
|
10
|
-
# output is out of scope for bats — the contract document is what consuming
|
|
11
|
-
# orchestrators and reviewers rely on. This pattern matches existing tests in
|
|
12
|
-
# this directory (see risk-scorer-register-hint.bats).
|
|
13
|
-
#
|
|
14
|
-
# Cross-reference:
|
|
15
|
-
# ADR-059: docs/decisions/059-pipeline-consume-catalog-and-bootstrap-from-reports.proposed.md
|
|
16
|
-
# ADR-056: docs/decisions/056-risk-register-back-channel-write-contract.proposed.md (slug primitive consumed)
|
|
17
|
-
# ADR-015: docs/decisions/015-on-demand-assessment-skills.proposed.md (pure-scorer contract preserved)
|
|
18
|
-
# ADR-026: docs/decisions/026-agent-output-grounding.proposed.md
|
|
19
|
-
# P168: docs/problems/168-risk-scorer-doesnt-consume-catalog-or-bootstrap.known-error.md
|
|
20
|
-
# P167: docs/problems/167-risk-register-aggregate-reads-as-dont-ship.known-error.md
|
|
21
|
-
# @jtbd JTBD-001 (enforce governance without slowing down — closes missed-risk-class hazard)
|
|
22
|
-
# @jtbd JTBD-202 (pre-flight governance — catalog as ISO 31000/27001 audit-trail artefact)
|
|
23
|
-
|
|
24
|
-
setup() {
|
|
25
|
-
AGENTS_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
|
26
|
-
PIPELINE="${AGENTS_DIR}/pipeline.md"
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
30
|
-
# Contract surface: Catalog Consumption Protocol section exists
|
|
31
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
32
|
-
|
|
33
|
-
@test "pipeline.md defines Catalog Consumption Protocol section" {
|
|
34
|
-
run grep -q "## Catalog Consumption Protocol" "$PIPELINE"
|
|
35
|
-
[ "$status" -eq 0 ]
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
@test "pipeline.md cites ADR-059 in the catalog protocol section" {
|
|
39
|
-
run grep -q "ADR-059" "$PIPELINE"
|
|
40
|
-
[ "$status" -eq 0 ]
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
@test "pipeline.md names docs/risks/ as the catalog read source" {
|
|
44
|
-
run grep -qE "READ.*docs/risks/|read.*standing-risk catalog at .docs/risks/" "$PIPELINE"
|
|
45
|
-
[ "$status" -eq 0 ]
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
49
|
-
# Hybrid filter: slug-token-match primary, judgement fallback
|
|
50
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
51
|
-
|
|
52
|
-
@test "pipeline.md describes slug-token-match as primary filter path" {
|
|
53
|
-
run grep -qE "[Ss]lug-token-match.*primary|[Ss]lug-token-match \(primary" "$PIPELINE"
|
|
54
|
-
[ "$status" -eq 0 ]
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
@test "pipeline.md describes judgement as fallback filter path" {
|
|
58
|
-
run grep -qE "[Jj]udgement.*fallback|[Ff]ree-form judgement.*fallback" "$PIPELINE"
|
|
59
|
-
[ "$status" -eq 0 ]
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
63
|
-
# Risk Item Format: Catalog match + Catalog baseline lines
|
|
64
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
65
|
-
|
|
66
|
-
@test "pipeline.md Risk Item Format includes Catalog match line" {
|
|
67
|
-
run grep -q "Catalog match:" "$PIPELINE"
|
|
68
|
-
[ "$status" -eq 0 ]
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
@test "pipeline.md Risk Item Format includes Catalog baseline line" {
|
|
72
|
-
run grep -q "Catalog baseline:" "$PIPELINE"
|
|
73
|
-
[ "$status" -eq 0 ]
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
@test "pipeline.md names the three Catalog match values" {
|
|
77
|
-
# slug-token | judgement | none — matches the ADR-059 verdict E3 contract.
|
|
78
|
-
run grep -qE "slug-token.*judgement.*none|slug-token \| judgement \| none" "$PIPELINE"
|
|
79
|
-
[ "$status" -eq 0 ]
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
83
|
-
# Residual reconciliation: per-action residual in RISK_SCORES, baseline contextual
|
|
84
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
85
|
-
|
|
86
|
-
@test "pipeline.md names per-action residual as RISK_SCORES output" {
|
|
87
|
-
run grep -qE "RISK_SCORES.*per-action residual|per-action residual.*RISK_SCORES" "$PIPELINE"
|
|
88
|
-
[ "$status" -eq 0 ]
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
@test "pipeline.md describes catalog lifetime baseline as context not RISK_SCORES" {
|
|
92
|
-
run grep -qE "lifetime baseline|Catalog baseline:" "$PIPELINE"
|
|
93
|
-
[ "$status" -eq 0 ]
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
97
|
-
# Hit-rate observability: CATALOG_HIT_RATE line emitted per run
|
|
98
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
99
|
-
|
|
100
|
-
@test "pipeline.md defines CATALOG_HIT_RATE observability line" {
|
|
101
|
-
run grep -q "CATALOG_HIT_RATE:" "$PIPELINE"
|
|
102
|
-
[ "$status" -eq 0 ]
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
@test "pipeline.md names the CATALOG_HIT_RATE matched + missed columns" {
|
|
106
|
-
run grep -qE "matched=N missed=M|CATALOG_HIT_RATE: matched" "$PIPELINE"
|
|
107
|
-
[ "$status" -eq 0 ]
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
111
|
-
# Empty catalog handling: nudge but do NOT halt or inflate residual
|
|
112
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
113
|
-
|
|
114
|
-
@test "pipeline.md handles empty catalog with nudge not halt" {
|
|
115
|
-
run grep -qE "[Ee]mpty catalog|catalog is empty.*nudge|do NOT halt" "$PIPELINE"
|
|
116
|
-
[ "$status" -eq 0 ]
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
@test "pipeline.md cites bootstrap-catalog skill in empty catalog nudge" {
|
|
120
|
-
run grep -qE "bootstrap-catalog|/install-updates.*bootstrap" "$PIPELINE"
|
|
121
|
-
[ "$status" -eq 0 ]
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
125
|
-
# Pure-scorer contract preserved: no Write tool grant added
|
|
126
|
-
# ──────────────────────────────────────────────────────────────────────────────
|
|
127
|
-
|
|
128
|
-
@test "pipeline.md preserves pure-scorer contract (Read + Glob only)" {
|
|
129
|
-
# The agent's tool grant must remain Read + Glob per ADR-015.
|
|
130
|
-
# Adding Write would break the architectural boundary ADR-059 verdict F2 preserves.
|
|
131
|
-
run grep -qE "^ - Read$" "$PIPELINE"
|
|
132
|
-
[ "$status" -eq 0 ]
|
|
133
|
-
run grep -qE "^ - Glob$" "$PIPELINE"
|
|
134
|
-
[ "$status" -eq 0 ]
|
|
135
|
-
# Negative: Write tool MUST NOT appear in tool grant
|
|
136
|
-
run grep -qE "^ - Write$" "$PIPELINE"
|
|
137
|
-
[ "$status" -ne 0 ]
|
|
138
|
-
}
|