@chrono-meta/fh-gate 2.15.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/regression/probes_live.yaml +137 -0
- package/.claude/rules/.residency-patterns.defaults +7 -0
- package/.claude/rules/fh_4axis_gate.md +88 -1
- package/.claude-plugin/marketplace.json +8 -2
- package/AGENTS.md +27 -0
- package/CATALOG.md +17 -0
- package/CLAUDE.md +13 -3
- package/README.ja.md +51 -7
- package/README.ko.md +48 -7
- package/README.md +43 -6
- package/README.zh.md +45 -8
- package/docs/STANDARDS_ALIGNMENT.md +61 -0
- package/docs/USER_GUIDE.md +3 -0
- package/docs/USE_CASES.md +50 -0
- package/docs/model_tier_expectations.md +60 -0
- package/knowledge/shared/harness-core/fh_three_layer_canon.md +77 -10
- package/knowledge/shared/harness-core/field_verdict_crossfamily_gate.md +18 -0
- package/knowledge/shared/harness-core/harness_incubator_doctrine.md +9 -0
- package/knowledge/shared/harness-core/iso_ai_standards_crosswalk.md +139 -0
- package/knowledge/shared/harness-core/measurement-integrity-checklist.md +35 -1
- package/knowledge/shared/harness-core/ship_readiness_gate.md +19 -2
- package/knowledge/shared/learnings/subagent_invocations_log.yaml +489 -3
- package/package.json +48 -2
- package/plugins/fh-commons/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-commons/skills/preprep/README.md +4 -1
- package/plugins/fh-commons/skills/preprep/SKILL.md +99 -3
- package/plugins/fh-commons/skills/preprep/diagram_from_json.py +154 -0
- package/plugins/fh-commons/skills/preprep/fixtures/fixture_R3_negative.pptx +0 -0
- package/plugins/fh-commons/skills/preprep/fixtures/fixture_R3_positive.pptx +0 -0
- package/plugins/fh-commons/skills/preprep/fixtures/mk_slide_fixtures.py +179 -0
- package/plugins/fh-commons/skills/preprep/interslide_deps.py +98 -9
- package/plugins/fh-commons/skills/preprep/lane_adjacent_dup.py +4 -1
- package/plugins/fh-commons/skills/preprep/lane_diagram.py +95 -0
- package/plugins/fh-commons/skills/preprep/lane_geometry.py +181 -0
- package/plugins/fh-commons/skills/preprep/lane_promise.py +4 -1
- package/plugins/fh-commons/skills/preprep/lane_slide_refs.py +134 -0
- package/plugins/fh-commons/skills/preprep/lane_slide_relations.py +300 -0
- package/plugins/fh-commons/skills/preprep/preprep.py +129 -8
- package/plugins/fh-commons/skills/preprep/preprep_wire.py +210 -0
- package/plugins/fh-commons/skills/preprep/presentation_checklist.md +4 -0
- package/plugins/fh-commons/skills/preprep/surfaces.example.yaml +18 -0
- package/plugins/fh-commons/skills/preprep/test_preprep_lanes_rp.py +184 -0
- package/plugins/fh-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-meta/CHANGELOG.md +67 -0
- package/plugins/fh-meta/skills/auto-decorrelation/SKILL.md +87 -2
- package/plugins/fh-meta/skills/frontier-digest/SKILL.md +2 -2
- package/plugins/fh-meta/skills/frontier-digest/SKILL_detail.md +25 -5
- package/plugins/fh-meta/skills/hub-cc-pr-reviewer/SKILL.md +11 -0
- package/plugins/fh-meta/skills/hub-cc-pr-reviewer/SKILL_detail.md +3 -0
- package/plugins/fh-qp/.claude-plugin/plugin.json +22 -0
- package/plugins/fh-qp/README.md +71 -0
- package/plugins/fh-qp/fixtures/evidence_known_clean.txt +3 -0
- package/plugins/fh-qp/fixtures/evidence_known_dirty.txt +6 -0
- package/plugins/fh-qp/fixtures/reach_known_partial.tsv +4 -0
- package/plugins/fh-qp/fixtures/reach_known_reached.tsv +4 -0
- package/plugins/fh-qp/fixtures/reach_known_wall.tsv +3 -0
- package/plugins/fh-qp/fixtures/verdicts_known_bad_branch.tsv +2 -0
- package/plugins/fh-qp/fixtures/verdicts_known_bad_vacuous_machine.tsv +2 -0
- package/plugins/fh-qp/fixtures/verdicts_known_good.tsv +4 -0
- package/plugins/fh-qp/fixtures/verdicts_verify_only.tsv +3 -0
- package/plugins/fh-qp/qp_profile.example.yaml +29 -0
- package/plugins/fh-qp/scripts/qp_tools.sh +216 -0
- package/plugins/fh-qp/skills/qp/SKILL.md +65 -0
- package/plugins/fh-qp/skills/qp-plan/SKILL.md +42 -0
- package/plugins/fh-qp/skills/qp-regress/SKILL.md +45 -0
- package/plugins/fh-qp/skills/qp-run/SKILL.md +49 -0
- package/scripts/backtick_guard.sh +194 -0
- package/scripts/chamber_run.sh +14 -5
- package/scripts/com.forge-harness.live-eval.plist +84 -0
- package/scripts/compaction_probe.sh +9 -35
- package/scripts/context_continuity_score.sh +49 -7
- package/scripts/directional_diff_gate.sh +14 -2
- package/scripts/fh-gate.sh +3 -3
- package/scripts/files_manifest_shipping_check.sh +19 -0
- package/scripts/frontier_digest_autopilot.sh +4 -1
- package/scripts/gate_pathspec_check.sh +1 -1
- package/scripts/map_postprocess.py +90 -0
- package/scripts/outbound_query_guard.sh +131 -0
- package/scripts/outbound_query_hook.sh +373 -0
- package/scripts/package_coverage_check.sh +79 -18
- package/scripts/pipe_verdict_guard.sh +41 -1
- package/scripts/probe_live_eval.sh +240 -0
- package/scripts/probe_live_eval_lib.py +579 -0
- package/scripts/proposal_hook.sh +192 -0
- package/scripts/public_surface_scan_files.sh +11 -2
- package/scripts/push_zone_check.sh +78 -0
- package/scripts/residency_closure_scan.py +252 -0
- package/scripts/revert_probe.sh +250 -0
- package/scripts/selfcheck.sh +106 -3
- package/scripts/session_close_check.sh +100 -0
- package/scripts/sim_isolated_run.sh +195 -9
- package/scripts/test_action_yml_lanes.sh +97 -0
- package/scripts/test_backtick_guard_lanes.sh +115 -0
- package/scripts/test_degrade_scan_shell_probes.sh +7 -7
- package/scripts/test_fh_qp_lanes.sh +105 -0
- package/scripts/test_files_manifest_shipping_lanes.sh +5 -5
- package/scripts/test_gate_two_verdicts_lanes.sh +139 -0
- package/scripts/test_heavy_classifier_lanes.sh +1 -1
- package/scripts/test_lane_runner_lanes.sh +59 -33
- package/scripts/test_map_postprocess_lanes.sh +143 -0
- package/scripts/test_mapped_tracks_lanes.sh +1 -1
- package/scripts/test_marker_affected_lanes.sh +93 -0
- package/scripts/test_marker_crossfamily_lanes.sh +90 -6
- package/scripts/test_marker_oracle_lanes.sh +136 -0
- package/scripts/test_marker_soul_check_lanes.sh +24 -0
- package/scripts/test_node_check_lanes.sh +34 -34
- package/scripts/test_outbound_query_hook_lanes.sh +433 -0
- package/scripts/test_outbound_query_lanes.sh +87 -0
- package/scripts/test_package_coverage_lanes.sh +53 -27
- package/scripts/test_pipe_verdict_guard_lanes.sh +31 -5
- package/scripts/test_precommit_pointer_index_lanes.sh +33 -0
- package/scripts/test_preprep_diagram_lanes.sh +87 -0
- package/scripts/test_preprep_drift_anchor.sh +16 -7
- package/scripts/test_preprep_drift_anchor_lanes.sh +23 -0
- package/scripts/test_preprep_slide_refs_lanes.sh +169 -0
- package/scripts/test_probe_live_eval_lanes.sh +437 -0
- package/scripts/test_proposal_hook_lanes.sh +57 -0
- package/scripts/test_push_zone_lanes.sh +304 -0
- package/scripts/test_residency_closure_lanes.sh +70 -0
- package/scripts/test_revert_probe_lanes.sh +146 -0
- package/scripts/test_session_close_lanes.sh +3 -5
- package/scripts/test_sim_isolated_run_lanes.sh +136 -0
- package/scripts/test_utterance_intake_lanes.sh +414 -0
- package/scripts/test_worktree_reclaim_lanes.sh +70 -0
- package/scripts/transcript_utterances.py +222 -0
- package/scripts/utterance_intake.sh +424 -0
- package/scripts/utterance_landing_check.sh +2 -2
- package/scripts/validate_yaml.sh +27 -0
- package/scripts/worktree_reclaim.sh +95 -0
- package/templates/.git-hooks/pre-commit +380 -5
- package/templates/.git-hooks/pre-push +91 -0
- package/templates/RED_TEAM_REPORT.md +49 -0
- package/templates/settings.PreToolUse.snippet.json +101 -1
- package/plugins/fh-commons/README.md +0 -38
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: qp
|
|
3
|
+
description: QP (Quality Platform) router — takes a web URL or a desktop app name and runs the plan → run → regress loop (qp-plan, qp-run, qp-regress) end to end, or routes to one stage. Classifies the target (public vs profile-required), probes which adapter the session has (Playwright MCP for web, computer-use MCP for desktop), and writes every artifact with a typed engine/adapter marker. Triggers on "test this website for me", "run a QA pass on this app", "check the app end to end", "이 사이트 QA 돌려줘", "이 앱 테스트해줘", "회귀 돌려줘".
|
|
4
|
+
user-invocable: true
|
|
5
|
+
allowed-tools: ["Read", "Bash", "Write", "Glob", "Grep"]
|
|
6
|
+
model: sonnet
|
|
7
|
+
origin: chamber run fh-qa-generic (2026-09-05) — generic edition of a field QA harness's P·A·R loop; zero domain constants
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# qp — Quality Platform router
|
|
11
|
+
|
|
12
|
+
One entry point for people who want to **dynamically test a web or desktop app with a QA lens from inside Claude Code**.
|
|
13
|
+
Three stages, each its own skill, each leaving files:
|
|
14
|
+
|
|
15
|
+
| stage | skill | you get |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| **P**lan | `qp-plan` | `qp/plan/inventory.md` (surfaces the app exposes) + `qp/plan/tcs.tsv` (test cases) |
|
|
18
|
+
| **A**utomation (run) | `qp-run` | `qp/run/<ts>/verdicts.tsv` + masked evidence (DOM snapshots or screenshots) |
|
|
19
|
+
| **R**egression | `qp-regress` | `qp/regress/<ts>/surface_reach.txt` + per-TC delta vs the previous run |
|
|
20
|
+
|
|
21
|
+
Glossary (one line each, because a first-time reader asked): **TC** = test case · **inventory** = the list of screens/routes/menus the app exposes · **MTM** = each finding is labeled *as planned* / *differs from the plan document* / *code differs* · **precondition** = what must already be true before a step runs (logged in, page loaded), checked not assumed — a failure before it is met is `BLOCKED`, not `FAIL` · **surface_reach** = how many TCs actually left the entry screen · **engine** = which machinery drove the run (see §Engine).
|
|
22
|
+
|
|
23
|
+
## Step 0 — classify the target (mechanical, never by eye)
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
bash plugins/fh-qp/scripts/qp_tools.sh target-class <https://… | app:NAME> [--profile <file>]
|
|
27
|
+
```
|
|
28
|
+
| result | rc | what you do |
|
|
29
|
+
|---|---|---|
|
|
30
|
+
| `PUBLIC` | 0 | proceed |
|
|
31
|
+
| `PROFILE_OK` | 0 | proceed with the profile |
|
|
32
|
+
| `PROFILE_REQUIRED` | 4 | **stop.** Say: *"this target is not public; QP needs a profile file — copy `plugins/fh-qp/qp_profile.example.yaml` to a gitignored path, fill the host/app, and re-run with `--profile`."* Do not improvise a run against a private host. |
|
|
33
|
+
| `UNKNOWN` | 10 | stop, report the target form QP accepts |
|
|
34
|
+
|
|
35
|
+
## Step 1 — probe the adapter (mechanical; unknown ≠ present)
|
|
36
|
+
|
|
37
|
+
List the MCP tool names this session actually has (from your tool list — do not guess), then:
|
|
38
|
+
```
|
|
39
|
+
bash plugins/fh-qp/scripts/qp_tools.sh adapter-probe --need web|desktop --tools "<comma-separated tool names>"
|
|
40
|
+
```
|
|
41
|
+
`ADAPTER=… evidence=dom|pixel` → proceed and **write that line into every artifact header**.
|
|
42
|
+
`HARNESS_ERROR` (rc 10) → **stop.** Write `qp/HARNESS_ERROR.txt` with the reason and tell the user which MCP to connect (Playwright MCP for web · computer-use MCP for desktop). This is not a pass and not a skip.
|
|
43
|
+
|
|
44
|
+
## Engine — typed capability vs MCP fallback (form C)
|
|
45
|
+
|
|
46
|
+
- If a QA harness has **registered** a typed capability whose id ends in `:par` (check `.claude/capabilities/**/*.cap` and `bash scripts/cluster_capability_scan.sh` when in an FH hub), call its `entry` and merge constraints **strictest-wins** (`capability_composition_contract.md`). Engine marker: `engine=capability:<id>`.
|
|
47
|
+
- Otherwise drive the target yourself through the adapter from Step 1. Engine marker: `engine=mcp-fallback adapter=<name> evidence=dom|pixel`.
|
|
48
|
+
- **As of 2026-09-05 no such capability is registered anywhere** — the fallback is the only path that exists. If the user explicitly asks for the capability engine (`--engine capability`) and none is registered → `HARNESS_ERROR reason=capability-not-registered`, never a silent fallback.
|
|
49
|
+
- Desktop (`evidence=pixel`) is the **weaker edition**: no DOM, so `verify` closes only by screenshot text you can read, and surface_reach hashes screenshots, not DOM. Say so in the report header.
|
|
50
|
+
|
|
51
|
+
## Step 2 — run the stages
|
|
52
|
+
|
|
53
|
+
`qp <target>` = plan → run → regress in order, stopping at the first `HARNESS_ERROR`. `qp plan|run|regress <target>` = one stage. Each stage skill carries its own Done When.
|
|
54
|
+
|
|
55
|
+
## Done When
|
|
56
|
+
1. Target classified with a `target-class` line recorded in `qp/plan/inventory.md` header · **[mandatory-pass]** — rc is the check.
|
|
57
|
+
2. Adapter probed and `ADAPTER=` or `HARNESS_ERROR` written to a file · **[mandatory-pass]** — file exists and matches the probe output.
|
|
58
|
+
3. Every artifact header carries `engine=` and `evidence=` · **[measured]** — `grep -L 'engine=' qp/**/*.md qp/**/*.tsv` is empty.
|
|
59
|
+
4. Stages chained in P→A→R order with no stage skipped silently · **[judged — pairing: the lane `scripts/test_fh_qp_lanes.sh` and the per-stage Done When files]**.
|
|
60
|
+
|
|
61
|
+
## Independently executable
|
|
62
|
+
Needs only `bash` + the session's MCP adapter. No other FH skill is required. Inside an FH hub the capability scan is optional.
|
|
63
|
+
|
|
64
|
+
## Not this skill
|
|
65
|
+
Mobile (deferred). Verifying FH itself (use FH's own gates). Fixing the app.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: qp-plan
|
|
3
|
+
description: Prepare stage of QP — builds a surface inventory of a web page or desktop app (routes, menus, forms, dialogs), checks it is MECE (no overlap, nothing missing that is visible), and designs test cases as a TSV with preconditions and expected values whose SOURCE is named. Works without a spec document (then expected values come from the inventory, not a plan). Triggers on "what should I test on this page", "make a test plan for this app", "list the screens of this app", "테스트 케이스 뽑아줘", "이 화면 인벤토리 만들어줘".
|
|
4
|
+
user-invocable: true
|
|
5
|
+
allowed-tools: ["Read", "Bash", "Write", "Glob", "Grep"]
|
|
6
|
+
model: sonnet
|
|
7
|
+
origin: chamber run fh-qa-generic (2026-09-05)
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# qp-plan — Prepare (inventory → MECE → TCs)
|
|
11
|
+
|
|
12
|
+
Input: a target that passed `qp_tools.sh target-class` (PUBLIC or PROFILE_OK) and an adapter line from `adapter-probe`. Optional: a plan/spec document path (`--plan <file>`). Optional profile.
|
|
13
|
+
|
|
14
|
+
## 1. Surface inventory — observe, do not guess
|
|
15
|
+
Open the entry route with the adapter (web: navigate + accessibility snapshot; desktop: screenshot). Record every **navigable surface** you can see: links/routes, menu items, buttons that open a dialog, forms, tabs. One row each:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
| id | kind (route·menu·dialog·form·tab) | label as shown | how reached (from which surface, which action) | observed? (yes/no) |
|
|
19
|
+
```
|
|
20
|
+
Rules: ⓐ **do not invent routes** — a surface is listed only if you saw its trigger; ⓑ parameterised routes (`/item/:id`) are listed as `ROUTE_PARAMETERIZED`, not filled with a made-up id; ⓒ if the entry route did not load → stop with `HARNESS_ERROR reason=entry-unreachable`. Write `qp/plan/inventory.md` with the header lines `target-class: …` (the tool's literal output), `ADAPTER=… evidence=dom|pixel` (the probe's literal output), `engine=mcp-fallback` (or `engine=capability:<id>` only if the `qp` router found a registered capability — never a browser name; a floor-tier sim wrote `engine=chromium` when this line did not say so, 2026-09-05), `plan_doc: <path|NONE>`.
|
|
21
|
+
|
|
22
|
+
## 2. MECE check
|
|
23
|
+
Walk the inventory once more against the snapshot: any visible navigable element not in the list → add it (**missing**); any two rows reaching the same surface → merge (**overlap**). Write the count of each fix under `## MECE` in inventory.md. Zero fixes is a legitimate value; an unstated count is not.
|
|
24
|
+
|
|
25
|
+
## 3. Test-case design — `qp/plan/tcs.tsv`
|
|
26
|
+
Tab-separated, header row:
|
|
27
|
+
```
|
|
28
|
+
tc_id surface_id precondition step_verb step_target expected expected_source priority
|
|
29
|
+
```
|
|
30
|
+
- `step_verb` ∈ `navigate click input verify` (one verb per row; a multi-step TC is several rows sharing `tc_id`).
|
|
31
|
+
- `expected_source` ∈ `PLAN_DOC INVENTORY CODE HUMAN` — **where the expected value came from**. With no plan document every row is `INVENTORY` (or `HUMAN` if the user told you). This field is what lets `qp-run` label a finding *differs from plan* vs *code differs*; a TC without it cannot be run.
|
|
32
|
+
- `precondition` is what must be true before step 1 (e.g. `at:/`, `logged-in`). Write `none` explicitly when there is none.
|
|
33
|
+
- **At least one TC must carry a state-changing verb (`click` or `input`)** — a verify-only plan never exercises the runner (`qp_tools.sh run-verbs` enforces this on the run output).
|
|
34
|
+
- Minimum: inventory ≥ 5 surfaces, TCs ≥ 3 for a first pass; say when the target is too small to reach that (a one-page site can be).
|
|
35
|
+
|
|
36
|
+
## Done When
|
|
37
|
+
1. `qp/plan/inventory.md` exists with the four header lines and ≥1 observed surface · **[mandatory-pass]**.
|
|
38
|
+
2. `qp/plan/tcs.tsv` parses: every row has 8 fields, `step_verb` and `expected_source` in their enums, ≥1 click/input row · **[measured]** — `awk -F'\t' 'NF!=8' tcs.tsv` prints nothing; `run-verbs` on the later verdicts is OK.
|
|
39
|
+
3. No surface in the inventory was invented (each has `observed? yes` or a `ROUTE_PARAMETERIZED` mark) · **[judged — pairing: a second session re-opens the entry route and diffs the inventory; the lane's residency/zero-constant grep on the plan files]**.
|
|
40
|
+
|
|
41
|
+
## Independently executable
|
|
42
|
+
Yes — bash + adapter. Does not need `qp-run`.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: qp-regress
|
|
3
|
+
description: Regression stage of QP — re-executes a previous run's test cases unchanged, computes surface_reach (how many TCs left the entry screen, as a ratio over ALL TCs), and reports the per-TC delta against the earlier run so a "no change" claim is per case, not an aggregate. Triggers on "did anything regress since last run", "re-run last week's QA pass", "compare this run with the previous one", "회귀 확인해줘", "지난 런이랑 비교해줘".
|
|
4
|
+
user-invocable: true
|
|
5
|
+
allowed-tools: ["Read", "Bash", "Write", "Glob", "Grep"]
|
|
6
|
+
model: sonnet
|
|
7
|
+
origin: chamber run fh-qa-generic (2026-09-05)
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# qp-regress — Regression (re-run unchanged → surface_reach → delta)
|
|
11
|
+
|
|
12
|
+
Input: a baseline `qp/run/<ts>/` and the same `qp/plan/tcs.tsv`. Output: `qp/regress/<ts>/`.
|
|
13
|
+
|
|
14
|
+
## 1. Re-run unchanged
|
|
15
|
+
Execute the baseline's TCs with `qp-run` **without editing them**. Editing a TC to make it pass is a new run, not a regression — say so if the user asks for it.
|
|
16
|
+
|
|
17
|
+
## 2. surface_reach — did the batch leave the entry screen?
|
|
18
|
+
```
|
|
19
|
+
bash plugins/fh-qp/scripts/qp_tools.sh surface-reach qp/run/<new-ts>/evidence.tsv
|
|
20
|
+
```
|
|
21
|
+
| result | meaning |
|
|
22
|
+
|---|---|
|
|
23
|
+
| `REACHED n/n` | every TC observed a screen other than the modal (entry) one |
|
|
24
|
+
| `PARTIAL k/n` | some did — report **which** did not, by tc_id |
|
|
25
|
+
| `NOT_REACHED 0/n` | the whole batch sat on one screen (a login wall, a dead env). Every FAIL in that run is **unmeasured**, not a detection — write that sentence in the report |
|
|
26
|
+
| `UNMEASURED` (rc 10) | no evidence file / empty — not a zero |
|
|
27
|
+
Denominator is **all TCs**, including ones that never executed; write `surface_reach.txt` with the raw line.
|
|
28
|
+
|
|
29
|
+
## 3. Per-TC delta — `delta.tsv`
|
|
30
|
+
```
|
|
31
|
+
tc_id baseline_status new_status changed note
|
|
32
|
+
```
|
|
33
|
+
One row per TC. `changed=yes` on any status change **or** on a same-status verdict whose `closure` moved from MACHINE to JUDGMENT (the check got weaker). A run where every row is `changed=no` may say "no regression"; a run reported only as "12 PASS both times" may not — the aggregate hides a swap.
|
|
34
|
+
|
|
35
|
+
## 4. Report — `report.md`
|
|
36
|
+
First 8 lines: target · engine/adapter/evidence · `#mtm:` state · surface_reach line · counts PASS/FAIL/BLOCKED/AMBIGUOUS new vs baseline · number of `changed=yes` · the sentence for NOT_REACHED if it applies · what was **not** measured (unexecuted TCs, dropped screenshots).
|
|
37
|
+
|
|
38
|
+
## Done When
|
|
39
|
+
1. `surface_reach.txt` holds a `REACHED|PARTIAL|NOT_REACHED|UNMEASURED` line produced by the tool · **[mandatory-pass]**.
|
|
40
|
+
2. `delta.tsv` has one row per TC in the baseline · **[measured]** — row count equals distinct tc_ids.
|
|
41
|
+
3. A NOT_REACHED run's report contains the "FAILs are unmeasured" sentence · **[measured]** — grep.
|
|
42
|
+
4. The delta's `changed` column reflects real behavior change, not selector drift · **[judged — pairing: for each `changed=yes`, the before/after DOM snapshots are attached and a second session confirms the element moved, not the locator]**.
|
|
43
|
+
|
|
44
|
+
## Independently executable
|
|
45
|
+
Yes — needs a baseline run dir in the documented shape and bash.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: qp-run
|
|
3
|
+
description: Automation stage of QP — executes the test cases from qp-plan against the live app through the session's adapter (Playwright MCP for web, computer-use MCP for desktop), records one verdict per step with a closed status, an MTM branch, whether the step was closed by machine evidence or left to judgment, and masks emails/tokens/passwords out of the evidence before saving. Triggers on "run these test cases", "execute the QA plan against the site", "drive the app through the TCs", "TC 실행해줘", "플랜대로 돌려줘".
|
|
4
|
+
user-invocable: true
|
|
5
|
+
allowed-tools: ["Read", "Bash", "Write", "Glob", "Grep"]
|
|
6
|
+
model: sonnet
|
|
7
|
+
origin: chamber run fh-qa-generic (2026-09-05)
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# qp-run — Automation (execute → verdict → mask)
|
|
11
|
+
|
|
12
|
+
Input: `qp/plan/tcs.tsv` + the adapter/engine lines from `qp/plan/inventory.md`. Output dir: `qp/run/<YYYYMMDD-HHMM>/`.
|
|
13
|
+
|
|
14
|
+
## 0. MTM state — decide once, write it first
|
|
15
|
+
`#mtm:` is the first line of `verdicts.tsv`:
|
|
16
|
+
| value | when |
|
|
17
|
+
|---|---|
|
|
18
|
+
| `ACTIVE` | a plan document was given **and** you opened it (path recorded) |
|
|
19
|
+
| `UNAVAILABLE` | no plan document — expected values come from the inventory; the branch *differs from plan* is **unreachable** and the vocabulary collapses to AS_PLANNED / CODE_DIFFERS |
|
|
20
|
+
| `FAILED` | a plan document was given but could not be read — say so; do not fall back silently |
|
|
21
|
+
| `DISABLED` | the user turned it off (reason recorded) |
|
|
22
|
+
`UNAVAILABLE` and `FAILED` are different events; never merge them.
|
|
23
|
+
|
|
24
|
+
## 1. Precondition first — attribution, not blocking
|
|
25
|
+
Before step 1 of each TC, establish its `precondition`. Navigation you can do (`at:/about`) you do; state you cannot create (`logged-in`, `has-an-order`) you mark `UNMET_UNEXECUTABLE`. A step-1 failure under an unmet precondition is **`BLOCKED`**, not `FAIL` — you never reached the screen, so it is not a claim about the app. Only step 1 is re-attributed; later steps failed after arrival.
|
|
26
|
+
|
|
27
|
+
## 2. Execute each step, observe before and after
|
|
28
|
+
web: accessibility snapshot before → action → snapshot after. desktop: screenshot before/after. Save each observation as `<tc_id>_step<N>_{before,after}.{md,png}` and append its stable id from `bash plugins/fh-qp/scripts/qp_tools.sh screen-id <file>` (a content hash — Playwright MCP ref tokens change every navigation, so a raw file hash is NOT stable; measured 2026-09-05) to `evidence.tsv` (`tc_id<TAB>step<TAB>screen_hash`) — this file is what `qp-regress` reads.
|
|
29
|
+
|
|
30
|
+
## 3. Verdict row per step — `verdicts.tsv` (7 tab-separated fields)
|
|
31
|
+
```
|
|
32
|
+
tc_id status branch closure verb assertion expected_source
|
|
33
|
+
```
|
|
34
|
+
- `status` ∈ `PASS FAIL BLOCKED AMBIGUOUS` (AMBIGUOUS = could not map the step to one element: 0 candidates or a tie).
|
|
35
|
+
- `branch` ∈ `AS_PLANNED DIFFERS_FROM_PLAN CODE_DIFFERS NONE`. `DIFFERS_FROM_PLAN` only under `#mtm: ACTIVE` and only with `expected_source=PLAN_DOC`. `CODE_DIFFERS` needs `expected_source` `CODE` or `PLAN_DOC`. `NONE` only for BLOCKED/AMBIGUOUS.
|
|
36
|
+
- `closure` = `MACHINE` **only if** `assertion` names the check that closed it (`text-visible:About`, `url==…`, `element-count==3`). An empty assertion with `MACHINE` is invalid — that is the muscle-not-skeleton hole this field exists to close. Everything you decided by reading a screenshot is `JUDGMENT`.
|
|
37
|
+
- Validate before you finish: `bash plugins/fh-qp/scripts/qp_tools.sh mtm-check verdicts.tsv` must print `OK`; `run-verbs verdicts.tsv` must print `OK` (≥1 click/input actually executed).
|
|
38
|
+
|
|
39
|
+
## 4. Mask, then save — original 0
|
|
40
|
+
Every text artifact goes through `qp_tools.sh mask <in> <out>` and only `<out>` is kept; delete `<in>`. `RESIDUE` (rc 5) → do not keep the file; report which artifact. Typed secret values (from the profile's `*_env`) are read from the environment at type-time and never written. Screenshots that show a filled password field are kept only if the field is masked in the image (crop or overlay); otherwise drop the screenshot and say so.
|
|
41
|
+
|
|
42
|
+
## Done When
|
|
43
|
+
1. `verdicts.tsv` passes `mtm-check` and `run-verbs` · **[mandatory-pass]** — rc 0 both.
|
|
44
|
+
2. `evidence.tsv` has ≥1 row per executed TC · **[measured]**.
|
|
45
|
+
3. Masking: `grep -cE '@|Bearer|eyJ' <saved artifacts>` is 0 outside `__REDACTED_*__` markers · **[measured]** — the lane reproduces it on a known-dirty fixture.
|
|
46
|
+
4. Every BLOCKED verdict names its unmet precondition; every JUDGMENT closure says what was read · **[judged — pairing: `qp-regress` re-runs the same TCs and flags verdicts that flip without a DOM change; a second session reviews BLOCKED rows against the before-snapshots]**.
|
|
47
|
+
|
|
48
|
+
## Independently executable
|
|
49
|
+
Yes, given a `tcs.tsv` in the documented shape (hand-written is fine) and an adapter.
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# backtick_guard.sh — PreToolUse(Bash) advisory: a backtick inside a shell double-quoting context.
|
|
3
|
+
#
|
|
4
|
+
# THE DEFECT
|
|
5
|
+
# In an unquoted heredoc body (`<<EOF`) or a "double-quoted string", a backtick is COMMAND
|
|
6
|
+
# SUBSTITUTION: the text between the backticks is REPLACED by that command's output. Written as
|
|
7
|
+
# markup (`--flag`, `file.sh`), it names no command, so the output is empty and the text is
|
|
8
|
+
# DELETED — the sentence stays grammatical, only its subject is gone. The one signal is a
|
|
9
|
+
# `command not found` line at the TOP of the output, where it reads as unrelated noise. Every
|
|
10
|
+
# record hook (marker · manifest · completed-log) checks a field's PRESENCE, not its completeness,
|
|
11
|
+
# so the hole commits. Measured 7×: 2026-08-10 (lane stub, stderr noise) · 2026-09-01 ×3 (marker,
|
|
12
|
+
# failure-message string, seal) · 2026-09-02 ×4 (marker, RESULT doc, fh_completed echo ×2).
|
|
13
|
+
#
|
|
14
|
+
# WHY A HOOK AND NOT A MEMORY RULE
|
|
15
|
+
# The memory rule existed since 08-10 and was re-read the day of each recurrence. It failed every
|
|
16
|
+
# time for the same reason: recall is grep, and the actor's task carried a different NAME (writing
|
|
17
|
+
# a marker · a failure message · a seal) than the rule's title (heredoc). N=7 ≥ 3 → mechanize
|
|
18
|
+
# (weekly_audit_2026-09-02 HIGH #1). The surface is the Bash tool call, where every recurrence
|
|
19
|
+
# lived (see KNOWN RESIDUALS for the two that may not have been).
|
|
20
|
+
#
|
|
21
|
+
# TWO RULES
|
|
22
|
+
# BT1 — unquoted heredoc body: `<<TAG` / `<<-TAG` whose tag is NOT quoted (`'TAG'` `"TAG"` `\TAG`).
|
|
23
|
+
# A `\`` inside is literal and not flagged. Body ends at a line equal to TAG (`<<-` strips
|
|
24
|
+
# leading tabs). Several heredocs on one line are queued in order (shell semantics).
|
|
25
|
+
# BT2 — double-quoted string containing an unescaped backtick. Single-quoted text is literal.
|
|
26
|
+
# `$( … )` inside double quotes re-enters normal parsing, so a single-quoted backtick
|
|
27
|
+
# there is literal and not flagged.
|
|
28
|
+
# Both rules come from ONE quote-aware state machine (N · single · double · $'…' · heredoc
|
|
29
|
+
# body), not a regex — the defect IS a quoting context, so a quote-blind matcher would flag the
|
|
30
|
+
# exact prescription (`printf '%s' '…`…`…'`). A heredoc operator counts only in normal context,
|
|
31
|
+
# comments (`#` at a word start) are skipped, and a quoted heredoc body is skipped whole.
|
|
32
|
+
# KNOWN RESIDUALS (named, not hidden — several found by the Axis-2 pass of 2026-09-03):
|
|
33
|
+
# · a bare backtick outside any quote (V=`date`), or inside `$( )` re-entered from double quotes,
|
|
34
|
+
# is live substitution written on purpose — NOT flagged (an earlier header said BT2; the code
|
|
35
|
+
# never did, and the code is the intent).
|
|
36
|
+
# · `# noqa: backtick` exempts the WHOLE payload, including when the phrase appears inside a
|
|
37
|
+
# record being written (quoting this header's own prescription into a marker exempts that
|
|
38
|
+
# marker's payload). Same accepted residual as destructive_pre_gate's noqa.
|
|
39
|
+
# · python3 broken/absent → CMD="" → silent exit 0 even under FH_BACKTICK_BLOCK=1: block mode
|
|
40
|
+
# fails OPEN on a dead interpreter, the same accepted trade as pipe_verdict_guard (the
|
|
41
|
+
# alternative blocks every Bash call on such a machine).
|
|
42
|
+
# · surface = the Bash tool call. Of the 7 measured recurrences, at least 5 were composed Bash
|
|
43
|
+
# commands; the 2026-08-10 one lived in a shipped lane file (`test_sidecar_calibrate_lanes.sh`,
|
|
44
|
+
# git log -S confirms) and the 09-01 failure-message one in a script — if those were authored
|
|
45
|
+
# through Write/Edit, this hook is not on that path. Coverage claim is therefore «the composed
|
|
46
|
+
# command surface», not 7/7; a file-side scanner is a separate, unbuilt instrument.
|
|
47
|
+
# · three JSON-emitting PreToolUse(Bash) hooks now fire on every call (pipe_verdict ·
|
|
48
|
+
# destructive_pre_gate · this one); concurrent emission is unverified at runtime (LOW).
|
|
49
|
+
#
|
|
50
|
+
# DEGRADE DIRECTION: advisory. Warns and exits 0 — a mangled write is re-runnable, and a false block
|
|
51
|
+
# on the developer's shell trains --no-verify on the hooks that guard irreversible surfaces.
|
|
52
|
+
# FH_BACKTICK_BLOCK=1 escalates to exit 2. Unparseable payload → silent (not a finding).
|
|
53
|
+
# DELIVERY: JSON on stdout — additionalContext (model) + systemMessage (user), no
|
|
54
|
+
# permissionDecision (same contract as pipe_verdict_guard; see its header for why).
|
|
55
|
+
#
|
|
56
|
+
# PRESCRIPTION (memory feedback_unquoted_heredoc_backtick_executes, 4th revision):
|
|
57
|
+
# ① heredoc → `<<'EOF'`; a value that must expand (hash, time) is computed FIRST into a variable
|
|
58
|
+
# and substituted after, or printed on its own line — never opened unquoted for one value.
|
|
59
|
+
# ② one-line append → `printf '%s\n' '…'` (single quotes), not `echo "…"`.
|
|
60
|
+
#
|
|
61
|
+
# Usage:
|
|
62
|
+
# hook: PreToolUse matcher "Bash" → bash scripts/backtick_guard.sh
|
|
63
|
+
# test: printf '%s' "<command>" | bash scripts/backtick_guard.sh --stdin-raw
|
|
64
|
+
# Opt out on a single call with a trailing `# noqa: backtick` (exempts the whole payload).
|
|
65
|
+
|
|
66
|
+
set -u
|
|
67
|
+
|
|
68
|
+
CMD=""
|
|
69
|
+
if [ "${1:-}" = "--stdin-raw" ]; then
|
|
70
|
+
CMD=$(cat)
|
|
71
|
+
else
|
|
72
|
+
RAW=$(cat)
|
|
73
|
+
CMD=$(printf '%s' "$RAW" | python3 -c '
|
|
74
|
+
import json,sys
|
|
75
|
+
try: d = json.load(sys.stdin)
|
|
76
|
+
except Exception: sys.exit(0)
|
|
77
|
+
if d.get("tool_name") != "Bash": sys.exit(0)
|
|
78
|
+
sys.stdout.buffer.write((d.get("tool_input", {}).get("command", "") or "").encode("utf-8"))
|
|
79
|
+
' 2>/dev/null) || CMD=""
|
|
80
|
+
fi
|
|
81
|
+
[ -n "$CMD" ] || exit 0
|
|
82
|
+
printf '%s' "$CMD" | grep -qE '#[[:space:]]*noqa:?[[:space:]]*backtick' && exit 0
|
|
83
|
+
|
|
84
|
+
# The scanner. Emits one line per finding: "<rule>\t<line>\t<snippet>". Empty output = clean.
|
|
85
|
+
hits=$(printf '%s' "$CMD" | PYTHONIOENCODING=utf-8 python3 -c '
|
|
86
|
+
import re, sys
|
|
87
|
+
text = sys.stdin.read()
|
|
88
|
+
L = len(text)
|
|
89
|
+
findings = []
|
|
90
|
+
# ONE quote-aware pass. Contexts: N normal · S single-quoted · D double-quoted · A $\x27…\x27 ANSI-C.
|
|
91
|
+
# A heredoc operator is recognised ONLY in N (so `"<<EOF"` in a commit message opens nothing), and
|
|
92
|
+
# its body is consumed line-by-line when the operator line ends — quoted bodies are skipped whole,
|
|
93
|
+
# unquoted bodies are scanned for a live backtick (`\\` escapes the next char, so `\\\\`+backtick is live).
|
|
94
|
+
HD = re.compile(r"<<(-?)[ \t]*(?:\x27([^\x27\n]*)\x27|\"([^\"\n]*)\"|\\\\([A-Za-z_][A-Za-z0-9_]*)|([A-Za-z_][A-Za-z0-9_]*))")
|
|
95
|
+
st = ["N"]; depth = [] # depth: paren depth per $( ) nesting opened from D
|
|
96
|
+
pending = [] # (tag, quoted, strip_tabs) heredocs opened on the current line, in order
|
|
97
|
+
line = 1
|
|
98
|
+
k = 0
|
|
99
|
+
def snippet(i):
|
|
100
|
+
return text[max(0, i-30):i+31].replace("\n", " ").strip()[:90]
|
|
101
|
+
while k < L:
|
|
102
|
+
c = text[k]
|
|
103
|
+
top = st[-1]
|
|
104
|
+
if c == "\n":
|
|
105
|
+
line += 1; k += 1
|
|
106
|
+
if pending and top == "N":
|
|
107
|
+
for tag, quoted, strip_tabs in pending:
|
|
108
|
+
while k < L:
|
|
109
|
+
e = text.find("\n", k)
|
|
110
|
+
if e < 0: e = L
|
|
111
|
+
ln = text[k:e]
|
|
112
|
+
cmp_ = ln.lstrip("\t") if strip_tabs else ln
|
|
113
|
+
if cmp_ == tag:
|
|
114
|
+
k = e + 1; line += 1; break
|
|
115
|
+
if not quoted:
|
|
116
|
+
j = 0
|
|
117
|
+
while j < len(ln):
|
|
118
|
+
if ln[j] == "\\": j += 2; continue
|
|
119
|
+
if ln[j] == "`":
|
|
120
|
+
findings.append(("BT1", line, ln.strip()[:90])); break
|
|
121
|
+
j += 1
|
|
122
|
+
k = e + 1; line += 1
|
|
123
|
+
pending = []
|
|
124
|
+
continue
|
|
125
|
+
if top == "S":
|
|
126
|
+
if c == "\x27": st.pop()
|
|
127
|
+
k += 1; continue
|
|
128
|
+
if top == "A":
|
|
129
|
+
if c == "\\": k += 2; continue
|
|
130
|
+
if c == "\x27": st.pop()
|
|
131
|
+
k += 1; continue
|
|
132
|
+
if c == "\\":
|
|
133
|
+
k += 2; continue
|
|
134
|
+
if top == "D":
|
|
135
|
+
if c == "\"": st.pop(); k += 1; continue
|
|
136
|
+
if text.startswith("$(", k): st.append("N"); depth.append(1); k += 2; continue
|
|
137
|
+
if c == "`": findings.append(("BT2", line, snippet(k))); k += 1; continue
|
|
138
|
+
k += 1; continue
|
|
139
|
+
# top == N
|
|
140
|
+
if c == "#" and (k == 0 or text[k-1] in " \t\n;&|(" ) and not depth:
|
|
141
|
+
e = text.find("\n", k); k = L if e < 0 else e; continue
|
|
142
|
+
if text.startswith("$\x27", k): st.append("A"); k += 2; continue
|
|
143
|
+
if c == "\x27": st.append("S"); k += 1; continue
|
|
144
|
+
if c == "\"": st.append("D"); k += 1; continue
|
|
145
|
+
if text.startswith("$(", k):
|
|
146
|
+
if depth: depth[-1] += 1
|
|
147
|
+
k += 2; continue
|
|
148
|
+
if c == "(" and depth: depth[-1] += 1; k += 1; continue
|
|
149
|
+
if c == ")" and depth:
|
|
150
|
+
depth[-1] -= 1
|
|
151
|
+
if depth[-1] == 0: depth.pop(); st.pop()
|
|
152
|
+
k += 1; continue
|
|
153
|
+
if c == "<" and text.startswith("<<", k) and not text.startswith("<<<", k) and (k == 0 or text[k-1] != "<"):
|
|
154
|
+
m = HD.match(text, k)
|
|
155
|
+
if m:
|
|
156
|
+
dash, q1, q2, esc, bare = m.groups()
|
|
157
|
+
tag = q1 if q1 is not None else (q2 if q2 is not None else (esc if esc is not None else bare))
|
|
158
|
+
pending.append((tag, (q1 is not None) or (q2 is not None) or (esc is not None), dash == "-"))
|
|
159
|
+
k = m.end(); continue
|
|
160
|
+
k += 1
|
|
161
|
+
seen = set()
|
|
162
|
+
for r, l, s in findings:
|
|
163
|
+
if (r, l) in seen: continue
|
|
164
|
+
seen.add((r, l)); print("%s\t%d\t%s" % (r, l, s))
|
|
165
|
+
' 2>/dev/null) || hits=""
|
|
166
|
+
[ -n "$hits" ] || exit 0
|
|
167
|
+
|
|
168
|
+
msg=" ⚠️ BACKTICK — 셸 이중인용 문맥 안의 백틱은 «명령 치환»이다: 그 자리 텍스트가 명령 출력으로 바뀐다(명령 없으면 삭제·있으면 오삽입). 실측 7회, 마커·기록에 구멍이 뚫린 채 커밋됐다.
|
|
169
|
+
"
|
|
170
|
+
while IFS=$'\t' read -r rule ln snip; do
|
|
171
|
+
[ -n "$rule" ] || continue
|
|
172
|
+
case "$rule" in
|
|
173
|
+
BT1) what="비인용 heredoc 본문";;
|
|
174
|
+
BT2) what="큰따옴표 문자열";;
|
|
175
|
+
*) what="$rule";;
|
|
176
|
+
esac
|
|
177
|
+
msg="${msg} ${rule} L${ln} (${what}): ${snip}
|
|
178
|
+
"
|
|
179
|
+
done <<< "$hits"
|
|
180
|
+
msg="${msg} 처방: heredoc 은 <<'EOF' 로 열고 확장할 값(해시·시각)은 «먼저 변수로 계산해» 뒤에 치환 · 한 줄 append 는 printf '%s\\n' '…'(작은따옴표). 의도된 치환이면 # noqa: backtick
|
|
181
|
+
"
|
|
182
|
+
|
|
183
|
+
if [ "${FH_BACKTICK_BLOCK:-0}" = "1" ]; then
|
|
184
|
+
printf '%s' "$msg" >&2
|
|
185
|
+
exit 2
|
|
186
|
+
fi
|
|
187
|
+
json_out=$(printf '%s' "$msg" | PYTHONIOENCODING=utf-8 python3 -c '
|
|
188
|
+
import json, sys
|
|
189
|
+
h = sys.stdin.read()
|
|
190
|
+
print(json.dumps({"systemMessage": h, "hookSpecificOutput": {"hookEventName": "PreToolUse", "additionalContext": h}}))
|
|
191
|
+
' 2>/dev/null)
|
|
192
|
+
if [ -n "$json_out" ]; then printf '%s\n' "$json_out"; exit 0; fi
|
|
193
|
+
printf '%s' "$msg" >&2
|
|
194
|
+
exit 0
|
package/scripts/chamber_run.sh
CHANGED
|
@@ -168,12 +168,17 @@ fi
|
|
|
168
168
|
_stamp "step-4-done"; echo " ✓ step 4: $NPERS blind persona sections present (isolation-gate satisfied)"
|
|
169
169
|
_witness_record "$WS/SIM_NOTES.md"
|
|
170
170
|
|
|
171
|
-
# STEP 5 — Emission Gate. Require a VERDICT: EMIT | PARTIAL-EMIT | KILL.
|
|
171
|
+
# STEP 5 — Emission Gate. Require a VERDICT: EMIT | PARTIAL-EMIT | KILL | CURATED | NOT-APPLICABLE.
|
|
172
|
+
# 🟥 2026-09-04: the doctrine re-routed two outcomes on 2026-08-17 (§3-SCREEN-2026-08-17 — net-new
|
|
173
|
+
# shortfall → CURATED, judgment-shaped candidate → NOT-APPLICABLE) and this regex kept parsing the
|
|
174
|
+
# OLD vocabulary for 18 days; run #16 had to file «KILL + CURATED 재료 동봉» to get past it, and the
|
|
175
|
+
# pre-registered taxonomy known-pair (5 old runs → CURATE) could not be replayed at all. A runner that
|
|
176
|
+
# rejects the doctrine's own words is [[feedback_rule_misdescribes_its_own_machine]] in the code half.
|
|
172
177
|
if [ ! -f "$WS/EMISSION_VERDICT.md" ]; then
|
|
173
178
|
cat > "$WS/EMISSION_VERDICT.md" <<EOF
|
|
174
179
|
# Emission Gate Verdict — $SLUG (chamber run)
|
|
175
180
|
|
|
176
|
-
VERDICT: <EMIT | PARTIAL-EMIT | KILL>
|
|
181
|
+
VERDICT: <EMIT | PARTIAL-EMIT | KILL | CURATED | NOT-APPLICABLE>
|
|
177
182
|
|
|
178
183
|
## Judged: does the simulation hold? (+ mechanical anchor: overlap grep / gate verdicts / reproduced flows)
|
|
179
184
|
|
|
@@ -183,11 +188,11 @@ EOF
|
|
|
183
188
|
echo " ⛔ step 5 BLOCKED: decide WITH the operator (HITL), record VERDICT in $WS/EMISSION_VERDICT.md, re-run."; exit 1
|
|
184
189
|
fi
|
|
185
190
|
# PARTIAL-EMIT listed FIRST in every alternation so it is never mis-extracted as its EMIT substring.
|
|
186
|
-
VERDICT=$(grep -ioE '^VERDICT:[[:space:]]*(PARTIAL-EMIT|EMIT|KILL)' "$WS/EMISSION_VERDICT.md" 2>/dev/null | head -1 | grep -ioE 'PARTIAL-EMIT|EMIT|KILL' | head -1 | tr 'a-z' 'A-Z')
|
|
191
|
+
VERDICT=$(grep -ioE '^VERDICT:[[:space:]]*(PARTIAL-EMIT|EMIT|KILL|CURATED|NOT-APPLICABLE)' "$WS/EMISSION_VERDICT.md" 2>/dev/null | head -1 | grep -ioE 'PARTIAL-EMIT|EMIT|KILL|CURATED|NOT-APPLICABLE' | head -1 | tr 'a-z' 'A-Z')
|
|
187
192
|
# a bare "## Verdict:" prose line (run #3 style) also counts if it names KILL/EMIT
|
|
188
|
-
[ -z "$VERDICT" ] && VERDICT=$(grep -ioE 'VERDICT[: *]+\**(PARTIAL-EMIT|EMIT|KILL)' "$WS/EMISSION_VERDICT.md" 2>/dev/null | grep -ioE 'PARTIAL-EMIT|EMIT|KILL' | head -1 | tr 'a-z' 'A-Z')
|
|
193
|
+
[ -z "$VERDICT" ] && VERDICT=$(grep -ioE 'VERDICT[: *]+\**(PARTIAL-EMIT|EMIT|KILL|CURATED|NOT-APPLICABLE)' "$WS/EMISSION_VERDICT.md" 2>/dev/null | grep -ioE 'PARTIAL-EMIT|EMIT|KILL|CURATED|NOT-APPLICABLE' | head -1 | tr 'a-z' 'A-Z')
|
|
189
194
|
if [ -z "$VERDICT" ]; then
|
|
190
|
-
echo " ⛔ step 5 BLOCKED: no VERDICT (EMIT|PARTIAL-EMIT|KILL) found in $WS/EMISSION_VERDICT.md, re-run."; exit 1
|
|
195
|
+
echo " ⛔ step 5 BLOCKED: no VERDICT (EMIT|PARTIAL-EMIT|KILL|CURATED|NOT-APPLICABLE) found in $WS/EMISSION_VERDICT.md, re-run."; exit 1
|
|
191
196
|
fi
|
|
192
197
|
_stamp "step-5-done"; echo " ✓ step 5: Emission Gate verdict = $VERDICT"
|
|
193
198
|
_witness_record "$WS/EMISSION_VERDICT.md"
|
|
@@ -269,6 +274,10 @@ case "$VERDICT" in
|
|
|
269
274
|
echo " existing asset / the skeleton (no new asset). Workspace stays as evidence." ;;
|
|
270
275
|
KILL) echo "TERMINUS (KILL): first-class success — a cheap run prevented a speculative/reinvention build."
|
|
271
276
|
echo " No emit. Workspace stays as the evidence record; seen-filter will skip re-listing it." ;;
|
|
277
|
+
CURATED) echo "TERMINUS (CURATED): net-new shortfall is not a kill (§3-SCREEN-2026-08-17) — hand the maker the"
|
|
278
|
+
echo " prior-art list, the closest existing asset, and the delta it does not cover." ;;
|
|
279
|
+
NOT-APPLICABLE) echo "TERMINUS (NOT-APPLICABLE): judgment-shaped candidate — not this incubator's output form."
|
|
280
|
+
echo " Route to doctrine (a rule/lens), not to a build. Workspace stays as evidence." ;;
|
|
272
281
|
esac
|
|
273
282
|
echo "chamber run '$SLUG' COMPLETE (STATUS: step-7-done, verdict $VERDICT)."
|
|
274
283
|
# EMIT/PARTIAL-EMIT 인데 순서 증인이 없으면 **비영 종료**한다. KILL 은 영향 없다 —
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
3
|
+
<plist version="1.0">
|
|
4
|
+
<dict>
|
|
5
|
+
<!-- Replace every /path/to/... below with your own absolute paths before installing.
|
|
6
|
+
Follows the same template shape as scripts/com.forge-harness.daily-report.plist —
|
|
7
|
+
see that file's own comments for the general launchd-on-macOS caveats (sleep/wake
|
|
8
|
+
does not guarantee the exact minute, StandardOutPath dirs must pre-exist). -->
|
|
9
|
+
<key>Label</key>
|
|
10
|
+
<string>com.forge-harness.live-eval</string>
|
|
11
|
+
|
|
12
|
+
<key>ProgramArguments</key>
|
|
13
|
+
<array>
|
|
14
|
+
<string>/bin/bash</string>
|
|
15
|
+
<string>/path/to/forge-harness/scripts/probe_live_eval.sh</string>
|
|
16
|
+
<!-- 🟥 COST + API-KEY DECISION IS THE OPERATOR'S — this template ships with NO default
|
|
17
|
+
--subset/--ids limiter, i.e. installing it as-is runs the FULL selected set (12
|
|
18
|
+
probes x 2 calls = 24 live `claude -p` invocations) every night. Read
|
|
19
|
+
scripts/probe_live_eval.sh's own header (§COST) before installing, and add a
|
|
20
|
+
--subset N or --ids P1,P2 argument here if a nightly full-set run is not what you
|
|
21
|
+
want. This plist is a template, not a recommendation of scope. -->
|
|
22
|
+
<!-- 🟥 reps=3 (2026-09-06). 이 인자가 없으면 reps=1 이고, 그때의 pass_rate 로는 문턱을
|
|
23
|
+
정할 수 없다 — 실측: 유효 런 3 개 재채점에서 12 프로브 중 5 개가 flaky, 같은 코퍼스
|
|
24
|
+
15 분 간격 두 런에서 4 개가 뒤집혔고 관측 pass_rate 는 0.50/0.67/0.67 이었다(단일 rep
|
|
25
|
+
노이즈 폭 > 문턱까지의 거리). 판정은 과반, 분산은 리포트 `reps(pass/ran)` 칸에 남는다.
|
|
26
|
+
⚠️ 비용이 3 배다: 24 → **72** 회 `claude -p`. 줄이려면 --subset/--ids 를 같이 줘라. -->
|
|
27
|
+
<string>--reps</string>
|
|
28
|
+
<string>3</string>
|
|
29
|
+
</array>
|
|
30
|
+
|
|
31
|
+
<!-- 매일 02:30 (야간, 다른 cron 과 안 겹치는 시간대 — frontier-digest=09:00, daily-report=07:30
|
|
32
|
+
과 분리). 잠자는 중이면 launchd 가 깨어난 뒤 한 번 돌린다(정각 보장 아님). -->
|
|
33
|
+
<key>StartCalendarInterval</key>
|
|
34
|
+
<dict>
|
|
35
|
+
<key>Hour</key><integer>2</integer>
|
|
36
|
+
<key>Minute</key><integer>30</integer>
|
|
37
|
+
</dict>
|
|
38
|
+
|
|
39
|
+
<key>RunAtLoad</key>
|
|
40
|
+
<false/>
|
|
41
|
+
|
|
42
|
+
<key>StandardOutPath</key>
|
|
43
|
+
<string>/path/to/forge-harness/tracks/_meta/logs/live_eval_out.log</string>
|
|
44
|
+
<key>StandardErrorPath</key>
|
|
45
|
+
<string>/path/to/forge-harness/tracks/_meta/logs/live_eval_err.log</string>
|
|
46
|
+
|
|
47
|
+
<key>EnvironmentVariables</key>
|
|
48
|
+
<dict>
|
|
49
|
+
<key>PATH</key>
|
|
50
|
+
<!-- must include wherever `claude` and `python3` resolve for this user.
|
|
51
|
+
/opt/homebrew/bin included so Homebrew's `gtimeout` (Apple Silicon prefix; Intel's
|
|
52
|
+
/usr/local/bin is already below) resolves — stock macOS ships no `timeout(1)` at
|
|
53
|
+
all, and scripts/probe_live_eval.sh's runner (sim_isolated_run.sh) falls back to a
|
|
54
|
+
bash-native watchdog if NEITHER resolves, so this is an optimization, not a hard
|
|
55
|
+
requirement: the run still enforces its timeout without this entry, just via the
|
|
56
|
+
slower fallback path. See sim_isolated_run.sh's own §timeout(1) RESOLUTION header
|
|
57
|
+
(measured 2026-09-05: the first unpatched run here died 12/12 FAILED-TO-RUN because
|
|
58
|
+
this PATH had neither `timeout` nor `gtimeout` AND the fallback did not exist yet). -->
|
|
59
|
+
<string>/path/to/home/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string>
|
|
60
|
+
<key>HOME</key>
|
|
61
|
+
<string>/path/to/home</string>
|
|
62
|
+
</dict>
|
|
63
|
+
</dict>
|
|
64
|
+
</plist>
|
|
65
|
+
|
|
66
|
+
<!--
|
|
67
|
+
INSTALL (manual — no launchd_wiring_check.sh render- helper exists for this plist yet; unlike
|
|
68
|
+
com.forge-harness.daily-report.plist, this one is not wired into that renderer, so every
|
|
69
|
+
/path/to/... above must be hand-edited before `launchctl load`):
|
|
70
|
+
|
|
71
|
+
1. cp scripts/com.forge-harness.live-eval.plist ~/Library/LaunchAgents/com.forge-harness.live-eval.plist
|
|
72
|
+
2. Edit every /path/to/... above to this checkout's absolute path and your $HOME.
|
|
73
|
+
3. mkdir -p tracks/_meta/logs (StandardOutPath/StandardErrorPath dirs must pre-exist)
|
|
74
|
+
4. launchctl load ~/Library/LaunchAgents/com.forge-harness.live-eval.plist
|
|
75
|
+
5. Verify: launchctl list | grep com.forge-harness.live-eval
|
|
76
|
+
|
|
77
|
+
UNINSTALL:
|
|
78
|
+
launchctl unload ~/Library/LaunchAgents/com.forge-harness.live-eval.plist
|
|
79
|
+
rm ~/Library/LaunchAgents/com.forge-harness.live-eval.plist
|
|
80
|
+
|
|
81
|
+
The nightly run appends its record to tracks/_meta/live_eval_<date>.md (created fresh each run —
|
|
82
|
+
not append-only; a stale prior day's file is left alone, per date, exactly like
|
|
83
|
+
tracks/_meta/daily_report_<date>.md and tracks/_meta/frontier_digest_<date>.md).
|
|
84
|
+
-->
|
|
@@ -70,41 +70,15 @@ do_seal() {
|
|
|
70
70
|
|
|
71
71
|
echo "## 운영자 발화 (이 세션)"
|
|
72
72
|
if [ -f "$transcript" ]; then
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
c=m.get('content')
|
|
83
|
-
# ⚠️ 초판은 "실발화=str · 툴결과=list" 로 갈랐다. **틀렸다** — 이미지/파일을 첨부한 실발화는
|
|
84
|
-
# list 다(high 리뷰 실측: 전사본 25개에서 그런 발화 18건이 구조적으로 안 보였다). 더 나쁜 건
|
|
85
|
-
# self-test 픽스처가 같은 가정을 인코딩해서 **초록이 그 결함을 보증**했다는 것이다.
|
|
86
|
-
# 이제 list 는 text 블록을 꺼내 쓰고, tool_result 만 제외한다.
|
|
87
|
-
if isinstance(c,str):
|
|
88
|
-
t=c
|
|
89
|
-
elif isinstance(c,list):
|
|
90
|
-
if any(isinstance(b,dict) and b.get('type')=='tool_result' for b in c):
|
|
91
|
-
skipped_tool+=1; continue
|
|
92
|
-
parts=[b.get('text','') for b in c if isinstance(b,dict) and b.get('type')=='text']
|
|
93
|
-
if not parts:
|
|
94
|
-
skipped_other+=1; continue # 이미지-only 등 — 셈에서 지우지 않고 센다
|
|
95
|
-
t=' '.join(parts)
|
|
96
|
-
else:
|
|
97
|
-
skipped_other+=1; continue
|
|
98
|
-
t=' '.join(t.split())
|
|
99
|
-
if not t: skipped_other+=1; continue
|
|
100
|
-
if t.startswith('<') or t.startswith('/'): # 슬래시 커맨드·메타 봉투 제외
|
|
101
|
-
skipped_meta+=1; continue
|
|
102
|
-
n+=1
|
|
103
|
-
print(f"{n}. {t[:200]}")
|
|
104
|
-
# **제외분을 반드시 인쇄한다.** 합계만 찍으면 그 원장이 완전한 것처럼 읽힌다 — `not found ≠ 0`.
|
|
105
|
-
print(f"\n합계: {n}건" if n else "- (발화 0건)")
|
|
106
|
-
print(f"제외: tool_result {skipped_tool} · 메타/커맨드 {skipped_meta} · 텍스트없음 {skipped_other}")
|
|
107
|
-
PY
|
|
73
|
+
# 🟥 추출 로직은 `scripts/transcript_utterances.py` 가 **단일 소스**다 (2026-09-05).
|
|
74
|
+
# 인라인 heredoc 이었을 때 두 번째 소비처(`utterance_intake.sh` — 발화 착지 검사)를
|
|
75
|
+
# 붙이려면 사본이 생겼고, 전처리 두 벌은 «한쪽만 통과하는 입력이 다른 쪽에서 무음
|
|
76
|
+
# 드롭» 이다(`[[feedback_divergent_leniency_duplicate_normalizers]]`).
|
|
77
|
+
# 출력은 **바이트 동일**해야 한다 — `scripts/test_utterance_intake_lanes.sh` L10 이
|
|
78
|
+
# 골든으로 고정하고, L12/L12b 가 되돌림으로 «정말 이 파일을 통해 도는가» 를 잰다.
|
|
79
|
+
# 추출기가 없거나 죽으면 비영 종료 → 아래 폴백 문구가 그대로 뜬다(무음 아님).
|
|
80
|
+
python3 "$REPO_ROOT/scripts/transcript_utterances.py" "$transcript" --format seal 2>/dev/null \
|
|
81
|
+
|| echo "- (전사본 파싱 실패 — 원본: $transcript)"
|
|
108
82
|
else
|
|
109
83
|
echo "- 🟥 전사본 경로 없음: $transcript"
|
|
110
84
|
fi
|