hookgate 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +10 -0
- package/CHANGELOG.md +99 -0
- package/README.md +148 -23
- package/bin/hookgate.mjs +89 -43
- package/bin/lib/config.mjs +174 -0
- package/bin/lib/doctor.mjs +58 -0
- package/bin/lib/gates.mjs +129 -0
- package/bin/lib/handlers.mjs +150 -0
- package/bin/lib/harness.mjs +64 -0
- package/bin/lib/jev.mjs +50 -0
- package/bin/lib/redact.mjs +34 -0
- package/bin/lib/report.mjs +70 -0
- package/bin/lib/store.mjs +149 -0
- package/codex/hooks.json +43 -0
- package/hooks/hooks.json +10 -14
- package/package.json +6 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hookgate",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.3",
|
|
4
4
|
"description": "Calibrated, sub-second decisions in Claude Code's hooks: a PreToolUse gate on shell commands and a Stop gate on unverified completion, answered by TypeSafe's Jev with a confidence score, escalating to the human when unsure.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Allan Nava",
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "hookgate",
|
|
3
|
+
"version": "0.0.3",
|
|
4
|
+
"description": "Calibrated, sub-second decisions in Codex CLI's hooks: a PreToolUse gate on shell commands and a Stop gate on unverified completion, answered by TypeSafe's Jev with a confidence score, blocking with a reason when unsure.",
|
|
5
|
+
"author": { "name": "Allan Nava", "url": "https://github.com/Allan-Nava" },
|
|
6
|
+
"homepage": "https://github.com/Allan-Nava/hookgate",
|
|
7
|
+
"repository": "https://github.com/Allan-Nava/hookgate",
|
|
8
|
+
"license": "MIT",
|
|
9
|
+
"hooks": "codex/hooks.json"
|
|
10
|
+
}
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to hookgate. The format is [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
|
|
4
|
+
versions follow [SemVer](https://semver.org/). Items reference their `HG-n` backlog id.
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.0.3] — 2026-09-22
|
|
9
|
+
|
|
10
|
+
Everything since the bootstrap: both gates, Codex, the audit of 2026-09-22, the seven
|
|
11
|
+
bugs the first measurements found, and the scorecard that proves each fix. Audit mode
|
|
12
|
+
by default: the benchmark has not run, so nothing blocks until it has.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
- `mode` defaults to `audit` — judge, log, fall through — until HG-4 has measured the
|
|
16
|
+
thresholds; `"mode": "enforce"` in `~/.hookgate.json` opts in. `0.1.0` flips the
|
|
17
|
+
default back (HG-5, HG-31).
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
- Numeric and enum config values were never checked, so a string or `null` flowed into
|
|
21
|
+
`setTimeout` and `truncate`; every field now has a rule, a bad value is reported as a
|
|
22
|
+
problem (`hookgate doctor` fails on it) and the default takes over. A 200 whose body
|
|
23
|
+
is not JSON is classified `malformed` rather than `network` (HG-29).
|
|
24
|
+
- The audit log dropped the token usage Jev reported, so cost per decision could not
|
|
25
|
+
be computed. Every judged decision logs `inputTokens` (0 on a cache hit), and
|
|
26
|
+
`hookgate report` prints tokens, total cost and cost per judged decision at the
|
|
27
|
+
published input price, one dated constant shared with `evals/run.mjs` (HG-28).
|
|
28
|
+
- A repository's own `.hookgate.json` could switch the gates off, go audit, or set a
|
|
29
|
+
model or timeout that fails open. Configuration now has a trust order — defaults,
|
|
30
|
+
`~/.hookgate.json`, `HOOKGATE_CONFIG`, then the repository file, then the environment
|
|
31
|
+
— and the repository file may only tighten; what it tried and could not do is listed
|
|
32
|
+
by `hookgate doctor` (HG-27).
|
|
33
|
+
- Rule promotion counted a chained command's verdict against its first word and could
|
|
34
|
+
propose a prefix rule for an interpreter or wrapper. Only one simple command — no
|
|
35
|
+
chaining, no pipe, no substitution outside quotes — counts, and a never-promote set
|
|
36
|
+
(`python3`, `bash`, `sudo`, `xargs`, `find`…) excludes programs that run whatever
|
|
37
|
+
follows; the log carries `promotable` beside the descriptive `prefix` (HG-24, HG-25).
|
|
38
|
+
- A command longer than `maxStateChars` reached Jev with its middle elided, so anything
|
|
39
|
+
hidden there was never judged. The command gate now answers `ask` for such a command
|
|
40
|
+
without a request; audit mode logs `skipped: too-long` and falls through (HG-26).
|
|
41
|
+
- The completion prefilter read English only, so a final message in another language
|
|
42
|
+
never reached Jev: 766 of 1,229 real stops were Italian claims it skipped. The lexicon
|
|
43
|
+
is now per language (English, Italian), always on, with ambiguous words anchored to
|
|
44
|
+
their claim form, and `completion.lexicon` adds patterns from the config. The measured
|
|
45
|
+
skip rate is 29%, not the 81% first reported (HG-23).
|
|
46
|
+
|
|
47
|
+
### Added
|
|
48
|
+
- `evals/scorecard.mjs` and `evals/fixtures/`: one metric per open bug, from labelled
|
|
49
|
+
fixtures, no key; `scorecard.yml` scores base and head of every pull request and
|
|
50
|
+
posts the delta, failing on a regression (HG-30).
|
|
51
|
+
- The command gate (HG-2), the completion gate (HG-3) and the off-by-default
|
|
52
|
+
injection screen (HG-15, code only), decided by Jev with calibrated confidence.
|
|
53
|
+
- Audit mode and `hookgate report` (HG-10); per-session decision cache (HG-12); rule
|
|
54
|
+
promotion in the harness's own syntax (HG-13); `hookgate doctor` (HG-14);
|
|
55
|
+
`failClosed` opt-in (HG-9); Codex CLI adapter aligned to the documented hook contract
|
|
56
|
+
— `permissionDecision` shape, `ask` as pass-through plus `systemMessage` or
|
|
57
|
+
`codex.askAs: deny`, injection feedback via `decision: block` — with e2e tests under
|
|
58
|
+
`PLUGIN_ROOT`; harness also detected from the stdin shape and `HOOKGATE_HARNESS`;
|
|
59
|
+
`codex/hooks.json` in Codex's single-command form; config also read from
|
|
60
|
+
`.hookgate.json` and `.codex/hookgate.json`; verified live on Codex 0.155.1, which has
|
|
61
|
+
dropped plugin-bundled hooks — `hookgate print-hooks` writes the repo-level
|
|
62
|
+
`.codex/hooks.json` with absolute paths (HG-11).
|
|
63
|
+
- `evals/`: 79 hand-labelled commands, 20 tool outputs, and the benchmark runner with
|
|
64
|
+
a `type: prompt` baseline (HG-4, tooling only — the run is blocked on a key).
|
|
65
|
+
- The repository as a knowledge graph under `graphify-out/`, versioned and queryable
|
|
66
|
+
(HG-22).
|
|
67
|
+
- README section "What leaves the machine, exactly" (HG-17); `HOOKGATE_ENDPOINT` for
|
|
68
|
+
proxies and end-to-end tests (HG-19); this changelog (HG-20); CodeQL and Renovate
|
|
69
|
+
digest pinning for actions (HG-21).
|
|
70
|
+
|
|
71
|
+
### Added (measured, no key)
|
|
72
|
+
- `evals/local.mjs`: hook overhead and transcript-derived counts; `--baseline-only` on
|
|
73
|
+
the runner; the README Benchmark section carries the incumbent's ten-command numbers,
|
|
74
|
+
the plugin's own cost, and what 102 real sessions say (HG-16, half).
|
|
75
|
+
|
|
76
|
+
### Changed
|
|
77
|
+
- Command-prefix logic skips `cd … &&`, `cd …;`, newline-separated `cd` hops and
|
|
78
|
+
`VAR=value` assignments before naming a prefix; `cd` no longer dominates promotion.
|
|
79
|
+
- The completion gate asks Jev only when the final message claims completion; a local
|
|
80
|
+
prefilter skips questions and partial reports, logged as `skipped` so `report` shows
|
|
81
|
+
the share that never reached the network (brief Q7).
|
|
82
|
+
- Audit log rotates at 8 MB; session files are written atomically and pruned after
|
|
83
|
+
seven days (HG-18).
|
|
84
|
+
|
|
85
|
+
### Fixed
|
|
86
|
+
- A malformed `TYPESAFE_API_KEY` is reported as such instead of an opaque fetch error.
|
|
87
|
+
- The rule-promotion message under Codex used Claude Code's answer shape.
|
|
88
|
+
- The benchmark runner passed an undefined `CLAUDECODE` to the child `claude`.
|
|
89
|
+
- Three CodeQL findings from the first scan: an unanchored regex in `check`, a
|
|
90
|
+
one-pass HTML-comment strip in the backlog tool that could leave a `<!--`, and a
|
|
91
|
+
check-then-use on the audit log file.
|
|
92
|
+
|
|
93
|
+
## [0.0.2] — 2026-09-22
|
|
94
|
+
|
|
95
|
+
### Added
|
|
96
|
+
- The scaffold: manifests, `hooks.json` with two inert handlers, `check`, CI on Node
|
|
97
|
+
18/20/22/24, release by tag over npm trusted publishing, Pages site generated from
|
|
98
|
+
the README, `BACKLOG.md` as the single source of truth with the one-way issue sync.
|
|
99
|
+
`0.0.1` is burnt on npm by a version published and unpublished in March.
|
package/README.md
CHANGED
|
@@ -14,10 +14,13 @@ on shell commands and a `Stop` gate on unverified claims of completion, answered
|
|
|
14
14
|
confidence score instead of text — in about 100 ms. Confident: act. Unsure: ask the
|
|
15
15
|
human. Unreachable: get out of the way.
|
|
16
16
|
|
|
17
|
-
> **Status:
|
|
18
|
-
>
|
|
19
|
-
>
|
|
20
|
-
>
|
|
17
|
+
> **Status: gates implemented, benchmark pending.** The command gate, the completion
|
|
18
|
+
> gate and an off-by-default injection screen are in, with audit mode, a per-session
|
|
19
|
+
> cache, rule promotion, `doctor` and `report`, on Claude Code and Codex CLI. What is
|
|
20
|
+
> still missing is the number: the benchmark below has not been run yet, so `0.0.3`
|
|
21
|
+
> ships in **audit mode by default** — every gate judges and logs, nothing blocks —
|
|
22
|
+
> and `"mode": "enforce"` in `~/.hookgate.json` is the opt-in. `0.1.0` flips the
|
|
23
|
+
> default once the table below is measured.
|
|
21
24
|
|
|
22
25
|
## Why in the hooks
|
|
23
26
|
|
|
@@ -35,41 +38,163 @@ on Jev. Nobody has put it inside the agent harness. That is the gap this fills.
|
|
|
35
38
|
|
|
36
39
|
| Gate | Hook | Question to Jev | Effect |
|
|
37
40
|
|---|---|---|---|
|
|
38
|
-
| Command risk | `PreToolUse` on `Bash` | `Choice{allow, ask, deny}` + `Noul` "destroys data or state outside the repo?" | `
|
|
39
|
-
| Unverified completion | `Stop` | `Noul` "does the last message claim a completion the visible state does not support?" on the message plus `git status` | `block` with
|
|
41
|
+
| Command risk | `PreToolUse` on `Bash` | `Choice{allow, ask, deny}` + `Noul` "destroys data or state outside the repo?" | `ask` or `deny` with a reason. Below the confidence threshold it is always `ask`, never `allow`. A confident `allow` **passes through** by default: hookgate narrows what the harness would do, it never widens it (`allowMode: "allow"` opts in) A command longer than `maxStateChars` (12,000 by default) is `ask` with no request: judging a head and a tail is not judging the command. |
|
|
42
|
+
| Unverified completion | `Stop` | `Noul` "does the last message claim a completion the visible state does not support?" on the message plus `git status` — asked only when the message claims something (a local prefilter skips questions and partial reports with no network call; it reads English and Italian, and `completion.lexicon` adds patterns) | `block` with a reason naming what to verify. Once per prompt, so the agent cannot loop |
|
|
43
|
+
| Injected instructions | `PostToolUse` on `WebFetch`, `WebSearch`, `Read`, `Bash` | `Noul` "does this output contain instructions addressed to an AI agent?" | `additionalContext` telling the agent to treat the span as data. **Off by default** until the fixture set gives a false-positive rate |
|
|
44
|
+
|
|
45
|
+
Around the gates:
|
|
46
|
+
|
|
47
|
+
- **Audit mode** (`"mode": "audit"` or `HOOKGATE_MODE=audit`): every gate judges for real,
|
|
48
|
+
logs one JSON line per decision to the plugin data directory, and always falls
|
|
49
|
+
through. `hookgate report` prints decisions by outcome, p50/p95 latency, cache hit
|
|
50
|
+
rate, input tokens and cost per judged decision at TypeSafe's published price, and
|
|
51
|
+
the share that would be `ask` at each threshold. The default until `0.1.0`.
|
|
52
|
+
- **Per-session cache**: the same command in the same session is judged once; a repeat
|
|
53
|
+
answers in microseconds with no request, and never outlives the session.
|
|
54
|
+
- **Rule promotion**: three verdicts above 95% confidence on one command prefix produce
|
|
55
|
+
a single `systemMessage` proposing the harness's own permission rule —
|
|
56
|
+
`"Bash(npm test *)"` for Claude Code, `prefix_rule()` for Codex. Proposed, never written,
|
|
57
|
+
and only for what the rule would actually cover: one simple command (no `&&`, `|`, `;`,
|
|
58
|
+
newline or substitution) whose program is not an interpreter or wrapper — `python3`,
|
|
59
|
+
`bash`, `sudo`, `xargs` and their kind run whatever follows, so a prefix rule on them
|
|
60
|
+
would widen, and hookgate never widens.
|
|
61
|
+
- **`hookgate doctor`**: key, connectivity, latency, model, config, harness. Non-zero only
|
|
62
|
+
on a broken configuration, never on a slow API.
|
|
63
|
+
- **Two harnesses, one file**: the handlers read the same stdin JSON under Claude Code
|
|
64
|
+
and Codex CLI and answer in each one's shape, verified against Codex's hooks
|
|
65
|
+
documentation. Codex has no `ask` on `PreToolUse`, so a below-threshold answer passes
|
|
66
|
+
through with the concern surfaced as a `systemMessage` (`"codex": {"askAs": "deny"}`
|
|
67
|
+
refuses instead), and the injection screen uses Codex's `decision: "block"` feedback
|
|
68
|
+
in place of `additionalContext`. Exercised against a live Codex 0.155.1 (see Install).
|
|
69
|
+
|
|
70
|
+
Configuration is read in trust order: the defaults, then `~/.hookgate.json` (yours),
|
|
71
|
+
then `HOOKGATE_CONFIG` if set, then the repository's `.hookgate.json` (or
|
|
72
|
+
`.claude/hookgate.json`, `.codex/hookgate.json`), then `HOOKGATE_MODE`, `HOOKGATE_MODEL`
|
|
73
|
+
and `HOOKGATE_FAIL_CLOSED=1`. **The repository's file may only tighten** what the layers
|
|
74
|
+
above it say: enable a gate, raise the confidence bar, go `enforce`, turn `failClosed`
|
|
75
|
+
on, add lexicon patterns, ask Codex to deny instead of pass through. A repository you
|
|
76
|
+
just cloned cannot switch the gate off, put it in audit mode, lower a threshold or point
|
|
77
|
+
it at a model or a timeout that would fail open; `hookgate doctor` lists what it tried.
|
|
78
|
+
Every key is optional:
|
|
79
|
+
|
|
80
|
+
```json
|
|
81
|
+
{
|
|
82
|
+
"mode": "audit",
|
|
83
|
+
"model": "jev-latest",
|
|
84
|
+
"timeoutMs": 2000,
|
|
85
|
+
"failClosed": false,
|
|
86
|
+
"allowMode": "passthrough",
|
|
87
|
+
"thresholds": { "confidence": 0.7, "destructive": 0.5, "unverified": 0.7, "injection": 0.7 },
|
|
88
|
+
"gates": { "command": true, "completion": true, "injection": false },
|
|
89
|
+
"completion": { "prefilter": true, "lexicon": [] }
|
|
90
|
+
}
|
|
91
|
+
```
|
|
40
92
|
|
|
41
|
-
|
|
42
|
-
[TypeSafe's confidence guide](https://docs.typesafe.ai/confidence.md)
|
|
93
|
+
Every value is checked: a bad one is reported by `hookgate doctor` and the default takes
|
|
94
|
+
its place, so the gate never runs arithmetic on a string. Thresholds scale with risk, as [TypeSafe's confidence guide](https://docs.typesafe.ai/confidence.md)
|
|
95
|
+
recommends: the defaults are conservative and the benchmark is what moves them.
|
|
43
96
|
|
|
44
97
|
**Fail-open, always.** No `TYPESAFE_API_KEY`, no network, a timeout, a 5xx or a bug
|
|
45
|
-
in this plugin means *no decision*: exit 0, empty stdout, and
|
|
98
|
+
in this plugin means *no decision*: exit 0, empty stdout, and the harness's normal
|
|
46
99
|
permission flow applies as if hookgate were not installed. A gate that stalls the
|
|
47
|
-
agent is worse than none.
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
100
|
+
agent is worse than none. `failClosed: true` is the explicit opt-in under which an
|
|
101
|
+
unreachable API makes the command gate `ask`.
|
|
102
|
+
|
|
103
|
+
**What leaves the machine, exactly.** One HTTPS POST per decision to
|
|
104
|
+
`api.typesafe.ai`, carrying only what the question needs: for the command gate the
|
|
105
|
+
shell command, its description if the agent wrote one, the last two segments of the
|
|
106
|
+
working directory and the permission mode; for the completion gate the agent's final
|
|
107
|
+
message, the stop reason and the first sixty lines of `git status --porcelain`; for
|
|
108
|
+
the injection screen the tool's output. Never the transcript, never file contents the
|
|
109
|
+
agent did not just fetch, never the session id. The audit log on disk keeps the
|
|
110
|
+
verdicts and a command *prefix*, not the command. TypeSafe's handling of what it
|
|
111
|
+
receives is theirs to state: [typesafe.ai legal](https://docs.typesafe.ai/legal.md).
|
|
112
|
+
`HOOKGATE_ENDPOINT` points the plugin at a proxy of your own if that matters.
|
|
113
|
+
|
|
114
|
+
**State never carries secrets.** Commands and tool outputs can contain tokens; key
|
|
115
|
+
shapes, bearer headers, `KEY=value` assignments, URL passwords and private keys are
|
|
116
|
+
redacted before anything leaves the machine, and state is truncated well under Jev's
|
|
51
117
|
32k-token limit.
|
|
52
118
|
|
|
53
119
|
## Install
|
|
54
120
|
|
|
55
|
-
Not published yet. Once it is:
|
|
56
|
-
|
|
57
121
|
```
|
|
58
122
|
/plugin marketplace add Allan-Nava/hookgate
|
|
59
123
|
/plugin install hookgate@hookgate
|
|
60
124
|
```
|
|
61
125
|
|
|
62
|
-
|
|
63
|
-
|
|
126
|
+
with `TYPESAFE_API_KEY` in the environment Claude Code runs in, then `hookgate doctor`
|
|
127
|
+
from the plugin directory to see what it sees.
|
|
128
|
+
|
|
129
|
+
**Codex CLI** (0.155 and later dropped plugin-bundled hooks, so hooks are per repository
|
|
130
|
+
or per user):
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
npm install -g hookgate
|
|
134
|
+
hookgate print-hooks > .codex/hooks.json # or ~/.codex/hooks.json
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Codex asks to trust the hooks file once; `--dangerously-bypass-hook-trust` skips that
|
|
138
|
+
for automation you already vet. Verified live on 2026-09-22 with Codex 0.155.1: the
|
|
139
|
+
command gate refused `rm -rf ~/…` ("Command blocked by PreToolUse hook: hookgate:
|
|
140
|
+
refused at 97% confidence") and let a `git push --force` through with the concern as a
|
|
141
|
+
`systemMessage`; the completion gate ran on `Stop`. Zero dependencies, Node 18 or later,
|
|
142
|
+
one `fetch` to `POST https://api.typesafe.ai/v1/systemone`. The package on npm is the
|
|
143
|
+
same tree, for `npx hookgate doctor` and `npx hookgate report`.
|
|
64
144
|
|
|
65
145
|
## Benchmark
|
|
66
146
|
|
|
67
|
-
The
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
147
|
+
The Jev half is not run yet — it needs a TypeSafe API key. What has been measured,
|
|
148
|
+
one machine, one day:
|
|
149
|
+
|
|
150
|
+
**The incumbent.** A `type: prompt`-style judge on `claude-opus-5`, one `claude -p`
|
|
151
|
+
per command from an empty directory, on the first ten labelled commands (all `safe`):
|
|
152
|
+
|
|
153
|
+
| System | Agreement with labels | p50 | p95 | Cost per decision |
|
|
154
|
+
|---|---:|---:|---:|---:|
|
|
155
|
+
| `type: prompt` hook (claude-opus-5) | 100% | 3,532 ms | 4,675 ms | $0.129 |
|
|
156
|
+
|
|
157
|
+
Ten commands, 2026-09-22. The cost is the CLI's own `total_cost_usd`, which carries the
|
|
158
|
+
CLI's system prompt on every call — the floor a prompt hook pays, not a model price.
|
|
159
|
+
|
|
160
|
+
**The plugin's own cost**, thirty spawns each, `evals/local.mjs`:
|
|
161
|
+
|
|
162
|
+
| Case | p50 | p95 |
|
|
163
|
+
|---|---:|---:|
|
|
164
|
+
| no key: fall-through | 59 ms | 61 ms |
|
|
165
|
+
| cache hit, no request | 61 ms | 62 ms |
|
|
166
|
+
| full round trip to a local fake Jev | 79 ms | 81 ms |
|
|
167
|
+
|
|
168
|
+
So the fixed price of having hookgate installed is one Node start, about 60 ms per
|
|
169
|
+
`Bash` call; Jev's own latency (70–500 ms by TypeSafe's numbers) comes on top and is
|
|
170
|
+
the part the key will tell.
|
|
171
|
+
|
|
172
|
+
**What real sessions say**, from 102 local Claude Code transcripts, counts only,
|
|
173
|
+
nothing sent anywhere:
|
|
174
|
+
|
|
175
|
+
- The completion prefilter skips **29%** of stops: of 1,229 assistant turns that ended
|
|
176
|
+
with a human reply, 878 claimed completion in English or Italian. The first measurement
|
|
177
|
+
said 81%, with an English-only lexicon reading Italian transcripts: 766 of those
|
|
178
|
+
"skipped" stops were claims it could not read (HG-23). Gate 2 asks Jev on roughly two
|
|
179
|
+
stops in three; the prefilter buys less than it seemed, and now says so.
|
|
180
|
+
- The per-session cache's ceiling is **~0%**: 99 exact repeats in 27,111 shell commands.
|
|
181
|
+
Real commands vary; the cache stays because it is free, not because it pays.
|
|
182
|
+
- The redactor changes **6%** of commands — keys, tokens, URL passwords are there to
|
|
183
|
+
be caught.
|
|
184
|
+
- A promoted rule may cover **6%** of commands: 1,744 of 27,147 are one simple command
|
|
185
|
+
whose program is not an interpreter (HG-24, HG-25). `sed`, `grep`, `cat`, `head`,
|
|
186
|
+
`tail` lead. The other 94% chain, pipe or run a script — a prefix rule on them would
|
|
187
|
+
say more than the judgement did.
|
|
188
|
+
|
|
189
|
+
**Every fix carries its number.** `node evals/scorecard.mjs` scores each open bug from
|
|
190
|
+
committed fixtures — no key, no network — and CI runs it on the base and the head of
|
|
191
|
+
every pull request, posting the delta. A fix moves one line to 100%; a regression on
|
|
192
|
+
any line fails the job.
|
|
193
|
+
|
|
194
|
+
Full runner: `node evals/run.mjs commands --baseline` over all 79 labelled commands and
|
|
195
|
+
`node evals/run.mjs injection` over the 20 outputs, once a key exists; the pre-stated
|
|
196
|
+
rule stands — under about 90% agreement the command gate ships `ask`-only. Until then,
|
|
197
|
+
run in audit mode.
|
|
73
198
|
|
|
74
199
|
## Two design notes
|
|
75
200
|
|
package/bin/hookgate.mjs
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// hookgate — calibrated decision gates for Claude Code hooks.
|
|
2
|
+
// hookgate — calibrated decision gates for Claude Code (and Codex CLI) hooks.
|
|
3
3
|
//
|
|
4
|
-
// hookgate check validate the manifests, hooks
|
|
5
|
-
// hookgate pre-tool-use PreToolUse handler
|
|
6
|
-
// hookgate stop Stop handler
|
|
4
|
+
// hookgate check validate the manifests, hooks files and this package
|
|
5
|
+
// hookgate pre-tool-use PreToolUse handler: the command-risk gate (stdin JSON)
|
|
6
|
+
// hookgate stop Stop handler: the completion gate (stdin JSON)
|
|
7
|
+
// hookgate post-tool-use PostToolUse handler: the injection screen (stdin JSON, off by default)
|
|
8
|
+
// hookgate doctor key, connectivity, latency, model, config, harness
|
|
9
|
+
// hookgate report decisions by outcome, latency, ask share — from the audit log
|
|
10
|
+
// hookgate print-hooks a .codex/hooks.json for this checkout, absolute paths (Codex
|
|
11
|
+
// 0.155 dropped plugin-bundled hooks; repo or user hooks remain)
|
|
7
12
|
// hookgate help
|
|
8
13
|
//
|
|
9
|
-
// Every handler FAILS OPEN. Exit 0 with no JSON on stdout means "no decision":
|
|
10
|
-
//
|
|
14
|
+
// Every handler FAILS OPEN. Exit 0 with no JSON on stdout means "no decision": the
|
|
15
|
+
// harness's ordinary permission flow applies, exactly as if the plugin were not
|
|
11
16
|
// installed. A missing TYPESAFE_API_KEY, a timeout, a 5xx or a bug in this file must
|
|
12
|
-
// never block the user's work. Fail-closed is an explicit opt-in
|
|
17
|
+
// never block the user's work. Fail-closed is an explicit opt-in (`failClosed: true`).
|
|
13
18
|
//
|
|
14
|
-
// Zero dependencies, Node 18+: a hook starts on every tool call, so start-up cost
|
|
15
|
-
//
|
|
19
|
+
// Zero dependencies, Node 18+: a hook starts on every tool call, so start-up cost is
|
|
20
|
+
// the cost. The API is one POST with fetch, which Node 18 has.
|
|
16
21
|
|
|
17
22
|
import { existsSync, readFileSync } from 'node:fs'
|
|
18
23
|
import { dirname, join, resolve } from 'node:path'
|
|
@@ -21,15 +26,8 @@ import { fileURLToPath } from 'node:url'
|
|
|
21
26
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
22
27
|
const read = (p) => readFileSync(join(ROOT, p), 'utf8')
|
|
23
28
|
const json = (p) => JSON.parse(read(p))
|
|
24
|
-
|
|
25
29
|
const [cmd = 'help'] = process.argv.slice(2)
|
|
26
30
|
|
|
27
|
-
// --- hook handlers ----------------------------------------------------------
|
|
28
|
-
//
|
|
29
|
-
// Not implemented yet: the gates are being designed under thoughts/HG-1-jev-gates/
|
|
30
|
-
// through the QRSPI workflow. Until then both handlers fall through — the plugin is
|
|
31
|
-
// installable and inert, which is what fail-open means.
|
|
32
|
-
|
|
33
31
|
async function readStdin() {
|
|
34
32
|
let s = ''
|
|
35
33
|
for await (const chunk of process.stdin) s += chunk
|
|
@@ -42,58 +40,84 @@ async function readStdin() {
|
|
|
42
40
|
|
|
43
41
|
async function handler(name) {
|
|
44
42
|
const input = await readStdin()
|
|
45
|
-
if (!input
|
|
46
|
-
|
|
43
|
+
if (!input) process.exit(0)
|
|
44
|
+
try {
|
|
45
|
+
const { preToolUse, stop, postToolUse } = await import('./lib/handlers.mjs')
|
|
46
|
+
const fn = { 'pre-tool-use': preToolUse, stop, 'post-tool-use': postToolUse }[name]
|
|
47
|
+
const out = await fn(input)
|
|
48
|
+
if (out) process.stdout.write(`${JSON.stringify(out)}\n`)
|
|
49
|
+
} catch (e) {
|
|
50
|
+
// The last line of defence: a bug here is a debug-log line, not a blocked agent.
|
|
51
|
+
process.stderr.write(`hookgate: ${name} failed open: ${e.message}\n`)
|
|
52
|
+
}
|
|
47
53
|
process.exit(0)
|
|
48
54
|
}
|
|
49
55
|
|
|
50
|
-
// --- check ------------------------------------------------------------------
|
|
51
|
-
|
|
52
56
|
const HOOK_EVENTS = new Set(['PreToolUse', 'PostToolUse', 'Stop', 'SubagentStop', 'UserPromptSubmit', 'PermissionRequest'])
|
|
57
|
+
const HANDLERS = new Set(['pre-tool-use', 'stop', 'post-tool-use'])
|
|
58
|
+
|
|
59
|
+
function checkHooksFile(path, rootVar, fail) {
|
|
60
|
+
const hooks = json(path)
|
|
61
|
+
for (const [event, entries] of Object.entries(hooks.hooks ?? {})) {
|
|
62
|
+
if (!HOOK_EVENTS.has(event)) fail(`${path}: unknown event ${event}`)
|
|
63
|
+
for (const entry of entries) {
|
|
64
|
+
for (const h of entry.hooks ?? []) {
|
|
65
|
+
if (h.type !== 'command') fail(`${path} ${event}: only command hooks are used here (got ${h.type})`)
|
|
66
|
+
// Claude Code takes command + args; Codex documents a single command string.
|
|
67
|
+
const handler = Array.isArray(h.args) ? h.args[0] : h.command.split(/\s+/).at(-1)
|
|
68
|
+
const bin = Array.isArray(h.args) ? h.command : h.command.replace(/^node\s+"?/, '').replace(/"?\s+\S+$/, '')
|
|
69
|
+
if (bin !== `${rootVar}/bin/hookgate.mjs`) fail(`${path} ${event}: command must run ${rootVar}/bin/hookgate.mjs, got ${h.command}`)
|
|
70
|
+
if (!HANDLERS.has(handler)) fail(`${path} ${event}: the command must end in a handler this file implements, got ${handler}`)
|
|
71
|
+
if (typeof h.timeout !== 'number' || h.timeout > 10) fail(`${path} ${event}: timeout must be set and at most 10 s — a gate that stalls the agent is worse than none`)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (!hooks.hooks?.PreToolUse?.some((e) => e.matcher === 'Bash')) fail(`${path}: the PreToolUse gate must match Bash`)
|
|
76
|
+
if (!hooks.hooks?.Stop) fail(`${path}: the Stop gate is missing`)
|
|
77
|
+
return hooks
|
|
78
|
+
}
|
|
53
79
|
|
|
54
80
|
function check() {
|
|
55
81
|
const errors = []
|
|
56
82
|
const fail = (m) => errors.push(m)
|
|
57
|
-
|
|
58
83
|
const pkg = json('package.json')
|
|
59
84
|
const plugin = json('.claude-plugin/plugin.json')
|
|
60
85
|
const market = json('.claude-plugin/marketplace.json')
|
|
86
|
+
const codex = json('.codex-plugin/plugin.json')
|
|
61
87
|
|
|
62
|
-
const versions = { 'package.json': pkg.version, 'plugin.json': plugin.version, 'marketplace.json': market.metadata?.version }
|
|
88
|
+
const versions = { 'package.json': pkg.version, 'plugin.json': plugin.version, 'marketplace.json': market.metadata?.version, 'codex plugin.json': codex.version }
|
|
63
89
|
if (new Set(Object.values(versions)).size !== 1) fail(`versions differ: ${JSON.stringify(versions)}`)
|
|
64
|
-
if (pkg.name !== 'hookgate' || plugin.name !== 'hookgate') fail('package.json and plugin.json must
|
|
90
|
+
if (pkg.name !== 'hookgate' || plugin.name !== 'hookgate' || codex.name !== 'hookgate') fail('package.json, plugin.json and .codex-plugin/plugin.json must all be named hookgate')
|
|
65
91
|
if (!(market.plugins ?? []).some((p) => p.name === 'hookgate' && p.source === './')) fail('marketplace.json must list the hookgate plugin with source "./"')
|
|
66
|
-
if (
|
|
92
|
+
if (!/^(?:git\+)?https:\/\/github\.com\/Allan-Nava\/hookgate(?:\.git)?$/.test(pkg.repository?.url ?? '')) fail('package.json#repository must be the GitHub repo URL, exactly')
|
|
67
93
|
if (pkg.dependencies && Object.keys(pkg.dependencies).length) fail('no runtime dependencies — a hook runs on every tool call')
|
|
68
|
-
for (const f of ['bin', 'hooks', '.claude-plugin', 'README.md', 'LICENSE']) if (!pkg.files?.includes(f)) fail(`package.json#files is missing ${f}`)
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
94
|
+
for (const f of ['bin', 'hooks', 'codex', '.claude-plugin', '.codex-plugin', 'README.md', 'CHANGELOG.md', 'LICENSE']) if (!pkg.files?.includes(f)) fail(`package.json#files is missing ${f}`)
|
|
95
|
+
|
|
96
|
+
const claude = checkHooksFile('hooks/hooks.json', '${CLAUDE_PLUGIN_ROOT}', fail)
|
|
97
|
+
const codexHooks = checkHooksFile('codex/hooks.json', '${PLUGIN_ROOT}', fail)
|
|
98
|
+
const handlersOf = (h) => Object.values(h.hooks).flatMap((es) => es.flatMap((e) => e.hooks.map((x) => (Array.isArray(x.args) ? x.args[0] : x.command.split(/\s+/).at(-1))))).sort().join(',')
|
|
99
|
+
if (handlersOf(claude) !== handlersOf(codexHooks)) fail('hooks/hooks.json and codex/hooks.json must register the same handlers')
|
|
100
|
+
|
|
101
|
+
for (const f of ['README.md', 'CONTRIBUTING.md', 'CLAUDE.md', 'LICENSE', 'BACKLOG.md', 'ROADMAP.md', 'CHANGELOG.md']) if (!existsSync(join(ROOT, f))) fail(`${f} is missing`)
|
|
102
|
+
if (existsSync(join(ROOT, 'CHANGELOG.md'))) {
|
|
103
|
+
const log = read('CHANGELOG.md')
|
|
104
|
+
if (!/^## \[Unreleased\]/m.test(log)) fail('CHANGELOG.md needs an [Unreleased] section — the release bump renames it')
|
|
105
|
+
if (!log.includes(`## [${pkg.version}]`) && pkg.version !== '0.0.0') fail(`CHANGELOG.md has no section for ${pkg.version}`)
|
|
81
106
|
}
|
|
82
|
-
if (
|
|
83
|
-
|
|
84
|
-
for (const f of ['README.md', 'CONTRIBUTING.md', 'CLAUDE.md', 'LICENSE']) if (!existsSync(join(ROOT, f))) fail(`${f} is missing`)
|
|
107
|
+
if (!/what leaves the machine/i.test(read('README.md'))) fail('README.md must state what leaves the machine')
|
|
85
108
|
if (!/fail-open|fails open/i.test(read('README.md'))) fail('README.md must state the fail-open rule')
|
|
109
|
+
for (const m of ['bin/lib/config.mjs', 'bin/lib/gates.mjs', 'bin/lib/handlers.mjs', 'bin/lib/harness.mjs', 'bin/lib/jev.mjs', 'bin/lib/redact.mjs', 'bin/lib/store.mjs', 'bin/lib/report.mjs', 'bin/lib/doctor.mjs']) if (!existsSync(join(ROOT, m))) fail(`${m} is missing`)
|
|
86
110
|
|
|
87
111
|
if (errors.length) {
|
|
88
112
|
for (const e of errors) console.error(`✗ ${e}`)
|
|
89
113
|
process.exit(1)
|
|
90
114
|
}
|
|
91
|
-
const n = Object.values(
|
|
92
|
-
console.log(`ok — ${n} hooks, manifests in sync at ${pkg.version}`)
|
|
115
|
+
const n = Object.values(claude.hooks).reduce((a, es) => a + es.reduce((b, e) => b + e.hooks.length, 0), 0)
|
|
116
|
+
console.log(`ok — ${n} hooks, two harnesses, manifests in sync at ${pkg.version}`)
|
|
93
117
|
}
|
|
94
118
|
|
|
95
119
|
function help() {
|
|
96
|
-
console.log(read('bin/hookgate.mjs').split('\n').slice(1,
|
|
120
|
+
console.log(read('bin/hookgate.mjs').split('\n').slice(1, 10).map((l) => l.replace(/^\/\/ ?/, '')).join('\n'))
|
|
97
121
|
}
|
|
98
122
|
|
|
99
123
|
switch (cmd) {
|
|
@@ -102,8 +126,30 @@ switch (cmd) {
|
|
|
102
126
|
break
|
|
103
127
|
case 'pre-tool-use':
|
|
104
128
|
case 'stop':
|
|
129
|
+
case 'post-tool-use':
|
|
105
130
|
await handler(cmd)
|
|
106
131
|
break
|
|
132
|
+
case 'doctor': {
|
|
133
|
+
const { doctor } = await import('./lib/doctor.mjs')
|
|
134
|
+
const { lines, broken } = await doctor()
|
|
135
|
+
console.log(`hookgate doctor\n${lines.join('\n')}`)
|
|
136
|
+
process.exit(broken ? 1 : 0)
|
|
137
|
+
break
|
|
138
|
+
}
|
|
139
|
+
case 'print-hooks': {
|
|
140
|
+
const tpl = json('codex/hooks.json')
|
|
141
|
+
const bin = join(ROOT, 'bin', 'hookgate.mjs')
|
|
142
|
+
for (const entries of Object.values(tpl.hooks)) for (const e of entries) for (const h of e.hooks) h.command = h.command.replace('${PLUGIN_ROOT}/bin/hookgate.mjs', bin)
|
|
143
|
+
tpl.description = `hookgate — calibrated gates for Codex CLI, pointing at ${bin}. Save as .codex/hooks.json in the repository (or ~/.codex/hooks.json) and trust it when Codex asks.`
|
|
144
|
+
console.log(JSON.stringify(tpl, null, 2))
|
|
145
|
+
break
|
|
146
|
+
}
|
|
147
|
+
case 'report': {
|
|
148
|
+
const { report } = await import('./lib/report.mjs')
|
|
149
|
+
const { dataDir } = await import('./lib/harness.mjs')
|
|
150
|
+
console.log(report(dataDir()))
|
|
151
|
+
break
|
|
152
|
+
}
|
|
107
153
|
default:
|
|
108
154
|
help()
|
|
109
155
|
}
|