agentwrangler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +116 -0
- package/dist/apply/jobs.js +429 -0
- package/dist/apply/open-terminal-child.mjs +98 -0
- package/dist/apply/open-terminal.js +221 -0
- package/dist/apply/settings-gen.js +35 -0
- package/dist/cli/agentwrangler.js +18 -0
- package/dist/daemon/config.js +51 -0
- package/dist/daemon/http.js +258 -0
- package/dist/daemon/index.js +372 -0
- package/dist/daemon/outcomes-pass.js +82 -0
- package/dist/daemon/readiness.js +15 -0
- package/dist/daemon/router.js +756 -0
- package/dist/daemon/static.js +146 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrations/001_observe.sql +196 -0
- package/dist/db/migrations/002_indexes.sql +6 -0
- package/dist/db/migrations/003_context_inventory_history.sql +20 -0
- package/dist/db/migrations/004_apply_jobs.sql +17 -0
- package/dist/db/migrations/005_tool_event_metadata.sql +17 -0
- package/dist/db/migrations/006_d7_query_indexes.sql +9 -0
- package/dist/db/migrations/007_work_item_branch_keys.sql +11 -0
- package/dist/db/migrations/008_thinking_tokens.sql +1 -0
- package/dist/db/migrations/009_user_turn_count.sql +1 -0
- package/dist/db/migrations/010_workspace_cwd.sql +1 -0
- package/dist/db/migrations/011_reports.sql +1 -0
- package/dist/db/migrations/012_reconcile_indexes.sql +2 -0
- package/dist/db/migrations/013_friction_fields.sql +5 -0
- package/dist/db/migrations/014_session_churn.sql +11 -0
- package/dist/db/migrations/015_gap_aggregates.sql +6 -0
- package/dist/db/open.js +30 -0
- package/dist/detector/benchmark-anchors.js +36 -0
- package/dist/detector/calibration.js +302 -0
- package/dist/detector/context-history-retention.js +312 -0
- package/dist/detector/context-probe.js +574 -0
- package/dist/detector/d1-source-identity.js +25 -0
- package/dist/detector/detectors/d10_catalog_footprint.js +146 -0
- package/dist/detector/detectors/d1_ctx_always_loaded.js +203 -0
- package/dist/detector/detectors/d2_session_long_full_context.js +119 -0
- package/dist/detector/detectors/d4_model_mismatch.js +258 -0
- package/dist/detector/detectors/d5_limit_burn_forecast.js +138 -0
- package/dist/detector/detectors/d6_tool_result_bloat.js +301 -0
- package/dist/detector/detectors/d7_loop_retry_waste.js +345 -0
- package/dist/detector/detectors/d8_cache_write_churn.js +201 -0
- package/dist/detector/detectors/d9_idle_background_session.js +101 -0
- package/dist/detector/engine.js +88 -0
- package/dist/detector/index.js +17 -0
- package/dist/detector/measurement.js +426 -0
- package/dist/detector/practice-registry.js +259 -0
- package/dist/detector/registry.js +32 -0
- package/dist/detector/savings.js +249 -0
- package/dist/detector/types.js +14 -0
- package/dist/evidence/common/approved-input.js +632 -0
- package/dist/evidence/common/boundary.js +84 -0
- package/dist/evidence/common/canonical.js +55 -0
- package/dist/evidence/common/redaction.js +321 -0
- package/dist/evidence/common/sqlite.js +25 -0
- package/dist/evidence/common/state.js +29 -0
- package/dist/evidence/cond1/cli.js +289 -0
- package/dist/evidence/cond1/packet.js +407 -0
- package/dist/evidence/cond1/prepare.js +295 -0
- package/dist/evidence/cond1/score.js +349 -0
- package/dist/evidence/cond1/types.js +1 -0
- package/dist/evidence/create-approval.js +365 -0
- package/dist/evidence/create-scratch.js +542 -0
- package/dist/evidence/d7/cli.js +113 -0
- package/dist/evidence/d7/measure.js +193 -0
- package/dist/evidence/d7/types.js +1 -0
- package/dist/evidence/discover-approval.js +492 -0
- package/dist/evidence/g2/adjudicate.js +20 -0
- package/dist/evidence/g2/cli.js +207 -0
- package/dist/evidence/g2/kappa.js +39 -0
- package/dist/evidence/g2/pipeline.js +92 -0
- package/dist/evidence/g2/store.js +14 -0
- package/dist/evidence/github/client.js +1 -0
- package/dist/evidence/github/gh-cli-client.js +301 -0
- package/dist/evidence/r3/cli.js +209 -0
- package/dist/evidence/r3/evaluate.js +417 -0
- package/dist/evidence/r3/packet.js +162 -0
- package/dist/evidence/r3/prepare.js +405 -0
- package/dist/evidence/r3/score.js +341 -0
- package/dist/evidence/r3/transcript.js +155 -0
- package/dist/evidence/r3/types.js +4 -0
- package/dist/hook/context-budget-hook.mjs +138 -0
- package/dist/hook/danger-guard-denylist.json +27 -0
- package/dist/hook/danger-guard-hook.mjs +167 -0
- package/dist/hook/install.js +0 -0
- package/dist/hook/limit-burn-hook.mjs +127 -0
- package/dist/hook/loop-guard-hook.mjs +104 -0
- package/dist/hook/precompact-checkpoint-hook.mjs +123 -0
- package/dist/ingest/churn-collector.js +122 -0
- package/dist/ingest/detector-hook.js +52 -0
- package/dist/ingest/discovery.js +207 -0
- package/dist/ingest/health.js +43 -0
- package/dist/ingest/index.js +28 -0
- package/dist/ingest/ingestor.js +509 -0
- package/dist/ingest/parser.js +344 -0
- package/dist/ingest/pricing.js +153 -0
- package/dist/ingest/reconcile.js +52 -0
- package/dist/ingest/tail.js +152 -0
- package/dist/ingest/types.js +24 -0
- package/dist/ingest/workspace-mapping.js +114 -0
- package/dist/oauth/anthropic-api-key.js +88 -0
- package/dist/oauth/count-tokens.js +86 -0
- package/dist/oauth/credentials.js +171 -0
- package/dist/oauth/judge-g2-client.js +154 -0
- package/dist/oauth/usage.js +167 -0
- package/dist/outcomes/branch-key.js +49 -0
- package/dist/outcomes/conclusions.js +45 -0
- package/dist/outcomes/derive.js +94 -0
- package/dist/outcomes/finding-extractors.js +131 -0
- package/dist/outcomes/findings.js +237 -0
- package/dist/outcomes/github/client.js +367 -0
- package/dist/outcomes/github/credential.js +195 -0
- package/dist/outcomes/github/gh-cli-client.js +340 -0
- package/dist/outcomes/linker.js +486 -0
- package/dist/outcomes/pool.js +24 -0
- package/dist/outcomes/sync.js +276 -0
- package/dist/query/api/agents-liveness.js +182 -0
- package/dist/query/api/burn-status.js +50 -0
- package/dist/query/api/context-budget.js +114 -0
- package/dist/query/api/context-composition.js +67 -0
- package/dist/query/api/cost-per-success.js +104 -0
- package/dist/query/api/delivery.js +92 -0
- package/dist/query/api/effectiveness.js +254 -0
- package/dist/query/api/efficiency-headroom.js +74 -0
- package/dist/query/api/headroom-trend.js +105 -0
- package/dist/query/api/hook-config.js +75 -0
- package/dist/query/api/hook-install.js +8 -0
- package/dist/query/api/hot-sessions.js +17 -0
- package/dist/query/api/idle-sessions.js +52 -0
- package/dist/query/api/index.js +40 -0
- package/dist/query/api/loop-guard.js +90 -0
- package/dist/query/api/offload-share.js +41 -0
- package/dist/query/api/outcomes.js +218 -0
- package/dist/query/api/overview.js +535 -0
- package/dist/query/api/rec-prompt.js +138 -0
- package/dist/query/api/recommendations-ledger.js +111 -0
- package/dist/query/api/recommendations.js +514 -0
- package/dist/query/api/reports.js +78 -0
- package/dist/query/api/self-churn.js +77 -0
- package/dist/query/api/self-percentiles.js +109 -0
- package/dist/query/api/session-drivers.js +153 -0
- package/dist/query/api/settings.js +85 -0
- package/dist/query/api/spend-flavor.js +234 -0
- package/dist/query/api/trends.js +155 -0
- package/dist/query/cap-weighted.js +119 -0
- package/dist/query/db-context.js +42 -0
- package/dist/query/envelope.js +71 -0
- package/dist/query/forecast.js +191 -0
- package/dist/query/settings-store.js +441 -0
- package/dist/query/spend.js +171 -0
- package/dist/query/trends.js +194 -0
- package/dist/ui/assets/index-DnRKgc21.css +1 -0
- package/dist/ui/assets/index-h1Q1wWq5.js +168 -0
- package/dist/ui/index.html +39 -0
- package/package.json +59 -0
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { loadApprovedEvidenceInput, publishApprovedOutput, } from "../common/approved-input.js";
|
|
4
|
+
import { canonicalJson, sha256Bytes, sha256Canonical } from "../common/canonical.js";
|
|
5
|
+
const SHA256_RE = /^[0-9a-f]{64}$/u;
|
|
6
|
+
const COVERAGE_THRESHOLD = 0.8;
|
|
7
|
+
const PRECISION_THRESHOLD = 0.95;
|
|
8
|
+
const VERDICTS = new Set(["CORRECT", "INCORRECT", "UNCERTAIN"]);
|
|
9
|
+
const REASONS = new Set([
|
|
10
|
+
"TIMELINE",
|
|
11
|
+
"COMMIT_SIGNAL",
|
|
12
|
+
"KNOWN_WORK_ITEM",
|
|
13
|
+
"CONFLICT",
|
|
14
|
+
"INSUFFICIENT",
|
|
15
|
+
]);
|
|
16
|
+
function refuse(code) {
|
|
17
|
+
throw new Error(code);
|
|
18
|
+
}
|
|
19
|
+
function retainedFailure(error) {
|
|
20
|
+
const message = error instanceof Error && /^[a-z0-9_]+$/u.test(error.message)
|
|
21
|
+
? error.message
|
|
22
|
+
: "r3_score_failure";
|
|
23
|
+
return new Error(`${message}_scratch_verification_retained`);
|
|
24
|
+
}
|
|
25
|
+
function exactKeys(value, expected, code) {
|
|
26
|
+
const actual = Object.keys(value).sort();
|
|
27
|
+
const wanted = [...expected].sort();
|
|
28
|
+
if (actual.length !== wanted.length || actual.some((key, i) => key !== wanted[i]))
|
|
29
|
+
refuse(code);
|
|
30
|
+
}
|
|
31
|
+
function record(value, code) {
|
|
32
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
33
|
+
refuse(code);
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
function readJson(state, file, digest, code) {
|
|
37
|
+
if (!SHA256_RE.test(digest))
|
|
38
|
+
refuse(`${code}_digest_invalid`);
|
|
39
|
+
const stateReal = fs.realpathSync.native(state);
|
|
40
|
+
const fileReal = fs.realpathSync.native(file);
|
|
41
|
+
if (path.dirname(fileReal) !== stateReal)
|
|
42
|
+
refuse(`${code}_path_invalid`);
|
|
43
|
+
const bytes = fs.readFileSync(fileReal);
|
|
44
|
+
if (sha256Bytes(bytes) !== digest)
|
|
45
|
+
refuse(`${code}_digest_mismatch`);
|
|
46
|
+
try {
|
|
47
|
+
return JSON.parse(bytes.toString("utf8"));
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
refuse(`${code}_json_invalid`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function parseAggregate(value) {
|
|
54
|
+
const parsed = record(value, "r3_score_aggregate_invalid");
|
|
55
|
+
exactKeys(parsed, [
|
|
56
|
+
"campaign",
|
|
57
|
+
"status",
|
|
58
|
+
"identity",
|
|
59
|
+
"backfill",
|
|
60
|
+
"denominator",
|
|
61
|
+
"shadow",
|
|
62
|
+
"adjudication",
|
|
63
|
+
"projected",
|
|
64
|
+
"byRepoAlias",
|
|
65
|
+
"deterministicRerun",
|
|
66
|
+
"privacy",
|
|
67
|
+
], "r3_score_aggregate_keys_invalid");
|
|
68
|
+
if (parsed.campaign !== "R3_U4_BRANCH_LINKAGE" ||
|
|
69
|
+
(parsed.status !== "PREPARED" && parsed.status !== "DATA_INSUFFICIENT") ||
|
|
70
|
+
!Array.isArray(parsed.byRepoAlias))
|
|
71
|
+
refuse("r3_score_aggregate_value_invalid");
|
|
72
|
+
const denominator = record(parsed.denominator, "r3_score_denominator_invalid");
|
|
73
|
+
exactKeys(denominator, [
|
|
74
|
+
"reconciledSessionsN",
|
|
75
|
+
"outcomeBearingSessionsN",
|
|
76
|
+
"baselineLinkedOutcomeBearingN",
|
|
77
|
+
"excludedNotReconciledN",
|
|
78
|
+
"excludedNoBashN",
|
|
79
|
+
"excludedNoWorkItemsN",
|
|
80
|
+
], "r3_score_denominator_keys_invalid");
|
|
81
|
+
for (const count of Object.values(denominator)) {
|
|
82
|
+
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0)
|
|
83
|
+
refuse("r3_score_denominator_value_invalid");
|
|
84
|
+
}
|
|
85
|
+
const privacy = record(parsed.privacy, "r3_score_privacy_invalid");
|
|
86
|
+
exactKeys(privacy, ["linkTableUnchanged", "rawRefN", "transcriptPathN", "tokenN"], "r3_score_privacy_keys_invalid");
|
|
87
|
+
if (typeof privacy.linkTableUnchanged !== "boolean" ||
|
|
88
|
+
privacy.rawRefN !== 0 ||
|
|
89
|
+
privacy.transcriptPathN !== 0 ||
|
|
90
|
+
privacy.tokenN !== 0)
|
|
91
|
+
refuse("r3_score_privacy_value_invalid");
|
|
92
|
+
const shadow = record(parsed.shadow, "r3_score_shadow_invalid");
|
|
93
|
+
exactKeys(shadow, [
|
|
94
|
+
"candidateReasonN",
|
|
95
|
+
"dispositionReasonN",
|
|
96
|
+
"uniqueUpliftCandidatesN",
|
|
97
|
+
"higherPrecedenceCandidatesN",
|
|
98
|
+
"backtestEligibleN",
|
|
99
|
+
"backtestAgreementN",
|
|
100
|
+
"backtestDisagreementN",
|
|
101
|
+
"transcriptSucceededN",
|
|
102
|
+
"transcriptFailureReasonN",
|
|
103
|
+
], "r3_score_shadow_keys_invalid");
|
|
104
|
+
for (const key of [
|
|
105
|
+
"uniqueUpliftCandidatesN",
|
|
106
|
+
"higherPrecedenceCandidatesN",
|
|
107
|
+
"backtestEligibleN",
|
|
108
|
+
"backtestAgreementN",
|
|
109
|
+
"backtestDisagreementN",
|
|
110
|
+
"transcriptSucceededN",
|
|
111
|
+
]) {
|
|
112
|
+
if (typeof shadow[key] !== "number" || !Number.isSafeInteger(shadow[key]) || shadow[key] < 0)
|
|
113
|
+
refuse("r3_score_shadow_value_invalid");
|
|
114
|
+
}
|
|
115
|
+
const rerun = record(parsed.deterministicRerun, "r3_score_rerun_invalid");
|
|
116
|
+
exactKeys(rerun, ["firstCanonicalSha256", "secondCanonicalSha256", "equal"], "r3_score_rerun_keys_invalid");
|
|
117
|
+
if (typeof rerun.firstCanonicalSha256 !== "string" ||
|
|
118
|
+
!SHA256_RE.test(rerun.firstCanonicalSha256) ||
|
|
119
|
+
typeof rerun.secondCanonicalSha256 !== "string" ||
|
|
120
|
+
!SHA256_RE.test(rerun.secondCanonicalSha256) ||
|
|
121
|
+
typeof rerun.equal !== "boolean")
|
|
122
|
+
refuse("r3_score_rerun_value_invalid");
|
|
123
|
+
const aliases = new Set();
|
|
124
|
+
for (const raw of parsed.byRepoAlias) {
|
|
125
|
+
const cohort = record(raw, "r3_score_cohort_invalid");
|
|
126
|
+
exactKeys(cohort, [
|
|
127
|
+
"repoAlias",
|
|
128
|
+
"outcomeBearingSessionsN",
|
|
129
|
+
"baselineLinkedOutcomeBearingN",
|
|
130
|
+
"candidateReasonN",
|
|
131
|
+
"dispositionReasonN",
|
|
132
|
+
"requiredAdjudicationN",
|
|
133
|
+
"adjudicatedN",
|
|
134
|
+
"correctN",
|
|
135
|
+
"incorrectN",
|
|
136
|
+
"uncertainN",
|
|
137
|
+
], "r3_score_cohort_keys_invalid");
|
|
138
|
+
if (typeof cohort.repoAlias !== "string" ||
|
|
139
|
+
cohort.repoAlias.length === 0 ||
|
|
140
|
+
aliases.has(cohort.repoAlias))
|
|
141
|
+
refuse("r3_score_cohort_alias_invalid");
|
|
142
|
+
aliases.add(cohort.repoAlias);
|
|
143
|
+
}
|
|
144
|
+
return value;
|
|
145
|
+
}
|
|
146
|
+
function parseKey(value) {
|
|
147
|
+
const parsed = record(value, "r3_score_key_invalid");
|
|
148
|
+
exactKeys(parsed, [
|
|
149
|
+
"campaignId",
|
|
150
|
+
"packetVersion",
|
|
151
|
+
"scorerVersion",
|
|
152
|
+
"randomizationSeed",
|
|
153
|
+
"randomizationSeedCommitment",
|
|
154
|
+
"aggregateSha256",
|
|
155
|
+
"entries",
|
|
156
|
+
], "r3_score_key_keys_invalid");
|
|
157
|
+
if (typeof parsed.campaignId !== "string" ||
|
|
158
|
+
parsed.campaignId.length === 0 ||
|
|
159
|
+
parsed.packetVersion !== "r3-u4-blinded-v1" ||
|
|
160
|
+
parsed.scorerVersion !== "r3-u4-score-v1" ||
|
|
161
|
+
typeof parsed.randomizationSeed !== "string" ||
|
|
162
|
+
!/^[0-9a-f]{64,}$/u.test(parsed.randomizationSeed) ||
|
|
163
|
+
typeof parsed.randomizationSeedCommitment !== "string" ||
|
|
164
|
+
!SHA256_RE.test(parsed.randomizationSeedCommitment) ||
|
|
165
|
+
typeof parsed.aggregateSha256 !== "string" ||
|
|
166
|
+
!SHA256_RE.test(parsed.aggregateSha256) ||
|
|
167
|
+
!Array.isArray(parsed.entries))
|
|
168
|
+
refuse("r3_score_key_value_invalid");
|
|
169
|
+
const ids = new Set();
|
|
170
|
+
for (const raw of parsed.entries) {
|
|
171
|
+
const entry = record(raw, "r3_score_key_entry_invalid");
|
|
172
|
+
exactKeys(entry, [
|
|
173
|
+
"candidateId",
|
|
174
|
+
"sessionId",
|
|
175
|
+
"candidateWorkItemId",
|
|
176
|
+
"repoAlias",
|
|
177
|
+
"cohort",
|
|
178
|
+
"backtestCorrect",
|
|
179
|
+
"redactionFailed",
|
|
180
|
+
], "r3_score_key_entry_keys_invalid");
|
|
181
|
+
if (typeof entry.candidateId !== "string" ||
|
|
182
|
+
entry.candidateId.length === 0 ||
|
|
183
|
+
ids.has(entry.candidateId) ||
|
|
184
|
+
typeof entry.sessionId !== "string" ||
|
|
185
|
+
entry.sessionId.length === 0 ||
|
|
186
|
+
typeof entry.candidateWorkItemId !== "string" ||
|
|
187
|
+
entry.candidateWorkItemId.length === 0 ||
|
|
188
|
+
typeof entry.repoAlias !== "string" ||
|
|
189
|
+
entry.repoAlias.length === 0 ||
|
|
190
|
+
(entry.cohort !== "UPLIFT" && entry.cohort !== "BACKTEST") ||
|
|
191
|
+
!(typeof entry.backtestCorrect === "boolean" || entry.backtestCorrect === null) ||
|
|
192
|
+
typeof entry.redactionFailed !== "boolean")
|
|
193
|
+
refuse("r3_score_key_entry_value_invalid");
|
|
194
|
+
ids.add(entry.candidateId);
|
|
195
|
+
}
|
|
196
|
+
return value;
|
|
197
|
+
}
|
|
198
|
+
function parseVerdicts(value) {
|
|
199
|
+
if (!Array.isArray(value))
|
|
200
|
+
refuse("r3_verdicts_invalid");
|
|
201
|
+
const ids = new Set();
|
|
202
|
+
return value.map((raw) => {
|
|
203
|
+
const row = record(raw, "r3_verdict_invalid");
|
|
204
|
+
exactKeys(row, ["candidateId", "verdict", "adjudicatorAlias", "reasonCode"], "r3_verdict_keys_invalid");
|
|
205
|
+
if (typeof row.candidateId !== "string" ||
|
|
206
|
+
row.candidateId.length === 0 ||
|
|
207
|
+
ids.has(row.candidateId) ||
|
|
208
|
+
typeof row.verdict !== "string" ||
|
|
209
|
+
!VERDICTS.has(row.verdict) ||
|
|
210
|
+
typeof row.adjudicatorAlias !== "string" ||
|
|
211
|
+
row.adjudicatorAlias.length === 0 ||
|
|
212
|
+
typeof row.reasonCode !== "string" ||
|
|
213
|
+
!REASONS.has(row.reasonCode))
|
|
214
|
+
refuse("r3_verdict_value_invalid");
|
|
215
|
+
ids.add(row.candidateId);
|
|
216
|
+
return row;
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
/** Score one exact packet key and complete one-adjudicator verdict set. */
|
|
220
|
+
export async function scoreR3(request) {
|
|
221
|
+
const approved = await loadApprovedEvidenceInput(request.approvedInput);
|
|
222
|
+
try {
|
|
223
|
+
const aggregate = parseAggregate(readJson(approved.scratchStatePath, request.aggregatePath, request.aggregateSha256, "r3_aggregate"));
|
|
224
|
+
const key = parseKey(readJson(approved.scratchStatePath, request.sealedKeyPath, request.sealedKeySha256, "r3_key"));
|
|
225
|
+
const verdicts = parseVerdicts(readJson(approved.scratchStatePath, request.verdictsPath, request.verdictsSha256, "r3_verdicts"));
|
|
226
|
+
if (key.aggregateSha256 !== request.aggregateSha256)
|
|
227
|
+
refuse("r3_score_aggregate_binding_mismatch");
|
|
228
|
+
const expectedCampaignId = sha256Canonical({
|
|
229
|
+
identity: aggregate.identity,
|
|
230
|
+
aggregate: request.aggregateSha256,
|
|
231
|
+
}).slice(0, 32);
|
|
232
|
+
if (key.campaignId !== expectedCampaignId)
|
|
233
|
+
refuse("r3_score_campaign_binding_mismatch");
|
|
234
|
+
if (sha256Bytes(Buffer.from(key.randomizationSeed, "hex")) !== key.randomizationSeedCommitment)
|
|
235
|
+
refuse("r3_score_seed_commitment_mismatch");
|
|
236
|
+
const entries = new Map(key.entries.map((entry) => [entry.candidateId, entry]));
|
|
237
|
+
const upliftEntryN = key.entries.filter((entry) => entry.cohort === "UPLIFT").length;
|
|
238
|
+
const backtestEntryN = key.entries.length - upliftEntryN;
|
|
239
|
+
if (upliftEntryN !== aggregate.shadow.uniqueUpliftCandidatesN ||
|
|
240
|
+
backtestEntryN !== Math.min(aggregate.shadow.backtestEligibleN, 100))
|
|
241
|
+
refuse("r3_score_packet_population_mismatch");
|
|
242
|
+
const verdictMap = new Map();
|
|
243
|
+
const adjudicators = new Set();
|
|
244
|
+
for (const verdict of verdicts) {
|
|
245
|
+
if (!entries.has(verdict.candidateId))
|
|
246
|
+
refuse("r3_score_unknown_verdict");
|
|
247
|
+
verdictMap.set(verdict.candidateId, verdict);
|
|
248
|
+
adjudicators.add(verdict.adjudicatorAlias);
|
|
249
|
+
}
|
|
250
|
+
if (verdicts.length > 0 && adjudicators.size !== 1)
|
|
251
|
+
refuse("r3_score_adjudicator_count_invalid");
|
|
252
|
+
let correctN = 0;
|
|
253
|
+
let incorrectN = 0;
|
|
254
|
+
let uncertainN = 0;
|
|
255
|
+
let correctUpliftN = 0;
|
|
256
|
+
let redactionViolation = false;
|
|
257
|
+
const cohortByAlias = new Map(aggregate.byRepoAlias.map((cohort) => [
|
|
258
|
+
cohort.repoAlias,
|
|
259
|
+
{
|
|
260
|
+
...cohort,
|
|
261
|
+
requiredAdjudicationN: 0,
|
|
262
|
+
adjudicatedN: 0,
|
|
263
|
+
correctN: 0,
|
|
264
|
+
incorrectN: 0,
|
|
265
|
+
uncertainN: 0,
|
|
266
|
+
},
|
|
267
|
+
]));
|
|
268
|
+
for (const entry of key.entries) {
|
|
269
|
+
const verdict = verdictMap.get(entry.candidateId);
|
|
270
|
+
const effective = entry.redactionFailed ? "UNCERTAIN" : verdict?.verdict;
|
|
271
|
+
if (entry.redactionFailed && verdict !== undefined && verdict.verdict !== "UNCERTAIN")
|
|
272
|
+
redactionViolation = true;
|
|
273
|
+
if (effective === "CORRECT") {
|
|
274
|
+
correctN += 1;
|
|
275
|
+
if (entry.cohort === "UPLIFT")
|
|
276
|
+
correctUpliftN += 1;
|
|
277
|
+
}
|
|
278
|
+
else if (effective === "INCORRECT")
|
|
279
|
+
incorrectN += 1;
|
|
280
|
+
else if (effective === "UNCERTAIN")
|
|
281
|
+
uncertainN += 1;
|
|
282
|
+
const cohort = cohortByAlias.get(entry.repoAlias);
|
|
283
|
+
if (cohort === undefined)
|
|
284
|
+
refuse("r3_score_unknown_repo_alias");
|
|
285
|
+
cohort.requiredAdjudicationN += 1;
|
|
286
|
+
cohort.adjudicatedN += verdict === undefined ? 0 : 1;
|
|
287
|
+
if (effective === "CORRECT")
|
|
288
|
+
cohort.correctN += 1;
|
|
289
|
+
else if (effective === "INCORRECT")
|
|
290
|
+
cohort.incorrectN += 1;
|
|
291
|
+
else if (effective === "UNCERTAIN")
|
|
292
|
+
cohort.uncertainN += 1;
|
|
293
|
+
}
|
|
294
|
+
const complete = verdictMap.size === entries.size && adjudicators.size === (entries.size === 0 ? 0 : 1);
|
|
295
|
+
const precisionDenominator = correctN + incorrectN;
|
|
296
|
+
const precision = complete && uncertainN === 0 && precisionDenominator > 0
|
|
297
|
+
? correctN / precisionDenominator
|
|
298
|
+
: null;
|
|
299
|
+
const linkedNumerator = aggregate.denominator.baselineLinkedOutcomeBearingN + correctUpliftN;
|
|
300
|
+
const coverage = aggregate.denominator.outcomeBearingSessionsN > 0
|
|
301
|
+
? linkedNumerator / aggregate.denominator.outcomeBearingSessionsN
|
|
302
|
+
: null;
|
|
303
|
+
const insufficient = aggregate.status === "DATA_INSUFFICIENT" ||
|
|
304
|
+
!complete ||
|
|
305
|
+
uncertainN > 0 ||
|
|
306
|
+
redactionViolation ||
|
|
307
|
+
precision === null ||
|
|
308
|
+
coverage === null;
|
|
309
|
+
const status = insufficient
|
|
310
|
+
? "DATA_INSUFFICIENT"
|
|
311
|
+
: coverage >= COVERAGE_THRESHOLD && precision >= PRECISION_THRESHOLD
|
|
312
|
+
? "PASS"
|
|
313
|
+
: "FAIL";
|
|
314
|
+
const report = {
|
|
315
|
+
...aggregate,
|
|
316
|
+
status,
|
|
317
|
+
adjudication: {
|
|
318
|
+
requiredN: entries.size,
|
|
319
|
+
adjudicatedN: verdictMap.size,
|
|
320
|
+
correctN,
|
|
321
|
+
incorrectN,
|
|
322
|
+
uncertainN,
|
|
323
|
+
precisionNumerator: correctN,
|
|
324
|
+
precisionDenominator,
|
|
325
|
+
},
|
|
326
|
+
projected: {
|
|
327
|
+
correctUniqueUpliftN: correctUpliftN,
|
|
328
|
+
linkedNumerator,
|
|
329
|
+
outcomeBearingDenominator: aggregate.denominator.outcomeBearingSessionsN,
|
|
330
|
+
coverage,
|
|
331
|
+
precision,
|
|
332
|
+
},
|
|
333
|
+
byRepoAlias: [...cohortByAlias.values()].sort((a, b) => a.repoAlias < b.repoAlias ? -1 : a.repoAlias > b.repoAlias ? 1 : 0),
|
|
334
|
+
};
|
|
335
|
+
publishApprovedOutput(approved, request.out, canonicalJson(report));
|
|
336
|
+
return report;
|
|
337
|
+
}
|
|
338
|
+
catch (error) {
|
|
339
|
+
throw retainedFailure(error);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as readline from "node:readline";
|
|
4
|
+
import { Transform } from "node:stream";
|
|
5
|
+
import { fingerprintBranchRef } from "../../outcomes/branch-key.js";
|
|
6
|
+
const MAX_PR_LINKS = 64;
|
|
7
|
+
const MAX_REPOSITORY_BYTES = 255;
|
|
8
|
+
function identityOf(stat) {
|
|
9
|
+
return {
|
|
10
|
+
device: stat.dev.toString(),
|
|
11
|
+
inode: stat.ino.toString(),
|
|
12
|
+
size: stat.size.toString(),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function sameIdentity(left, right) {
|
|
16
|
+
return left.device === right.device && left.inode === right.inode && left.inode !== "0";
|
|
17
|
+
}
|
|
18
|
+
function classifyReadError(error) {
|
|
19
|
+
return error?.code === "ENOENT" ? "MISSING" : "UNREADABLE";
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Stream one descriptor-attested transcript and retain only bounded structural signals.
|
|
23
|
+
* The pathname and raw refs never cross the result boundary.
|
|
24
|
+
*/
|
|
25
|
+
export async function harvestFrozenTranscript(entry) {
|
|
26
|
+
let handle;
|
|
27
|
+
try {
|
|
28
|
+
handle = await fs.promises.open(entry.path, "r");
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
return { ok: false, reason: classifyReadError(error) };
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
let before;
|
|
35
|
+
try {
|
|
36
|
+
before = await handle.stat({ bigint: true });
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return { ok: false, reason: "UNREADABLE" };
|
|
40
|
+
}
|
|
41
|
+
if (!before.isFile()) {
|
|
42
|
+
return { ok: false, reason: "UNREADABLE" };
|
|
43
|
+
}
|
|
44
|
+
if (!sameIdentity(identityOf(before), entry.identity)) {
|
|
45
|
+
return { ok: false, reason: "REPLACED" };
|
|
46
|
+
}
|
|
47
|
+
if (before.size.toString() !== entry.identity.size) {
|
|
48
|
+
return { ok: false, reason: "CHANGED" };
|
|
49
|
+
}
|
|
50
|
+
const links = [];
|
|
51
|
+
const branchKeys = new Set();
|
|
52
|
+
let malformedLines = 0;
|
|
53
|
+
let projectionLimitExceeded = false;
|
|
54
|
+
const digest = createHash("sha256");
|
|
55
|
+
const hashingStream = new Transform({
|
|
56
|
+
transform(chunk, _encoding, callback) {
|
|
57
|
+
digest.update(chunk);
|
|
58
|
+
callback(null, chunk);
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
const input = handle.createReadStream({ autoClose: false, encoding: "utf8" });
|
|
62
|
+
input.pipe(hashingStream);
|
|
63
|
+
const lines = readline.createInterface({
|
|
64
|
+
input: hashingStream,
|
|
65
|
+
crlfDelay: Number.POSITIVE_INFINITY,
|
|
66
|
+
});
|
|
67
|
+
try {
|
|
68
|
+
for await (const line of lines) {
|
|
69
|
+
const trimmed = line.trim();
|
|
70
|
+
if (trimmed.length === 0)
|
|
71
|
+
continue;
|
|
72
|
+
let record;
|
|
73
|
+
try {
|
|
74
|
+
const parsed = JSON.parse(trimmed);
|
|
75
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
76
|
+
malformedLines += 1;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
record = parsed;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
malformedLines += 1;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
if (branchKeys.size < 2) {
|
|
86
|
+
const branchKey = fingerprintBranchRef(record.gitBranch);
|
|
87
|
+
if (branchKey !== null)
|
|
88
|
+
branchKeys.add(branchKey);
|
|
89
|
+
}
|
|
90
|
+
if (record.type === "pr-link") {
|
|
91
|
+
const repository = record.prRepository;
|
|
92
|
+
if (typeof record.prNumber === "number" &&
|
|
93
|
+
Number.isSafeInteger(record.prNumber) &&
|
|
94
|
+
record.prNumber > 0 &&
|
|
95
|
+
typeof repository === "string" &&
|
|
96
|
+
Buffer.byteLength(repository, "utf8") <= MAX_REPOSITORY_BYTES &&
|
|
97
|
+
/^[^/\s]+\/[^/\s]+$/u.test(repository) &&
|
|
98
|
+
links.length < MAX_PR_LINKS) {
|
|
99
|
+
links.push({ prNumber: record.prNumber, prRepository: repository });
|
|
100
|
+
}
|
|
101
|
+
else if (typeof record.prNumber === "number" &&
|
|
102
|
+
Number.isSafeInteger(record.prNumber) &&
|
|
103
|
+
record.prNumber > 0 &&
|
|
104
|
+
typeof repository === "string" &&
|
|
105
|
+
Buffer.byteLength(repository, "utf8") <= MAX_REPOSITORY_BYTES &&
|
|
106
|
+
/^[^/\s]+\/[^/\s]+$/u.test(repository)) {
|
|
107
|
+
projectionLimitExceeded = true;
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
malformedLines += 1;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return { ok: false, reason: "UNREADABLE" };
|
|
117
|
+
}
|
|
118
|
+
let after;
|
|
119
|
+
try {
|
|
120
|
+
after = await handle.stat({ bigint: true });
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
return { ok: false, reason: "UNREADABLE" };
|
|
124
|
+
}
|
|
125
|
+
if (!sameIdentity(identityOf(after), entry.identity)) {
|
|
126
|
+
return { ok: false, reason: "REPLACED" };
|
|
127
|
+
}
|
|
128
|
+
if (after.size !== before.size ||
|
|
129
|
+
after.mtimeNs !== before.mtimeNs ||
|
|
130
|
+
after.ctimeNs !== before.ctimeNs ||
|
|
131
|
+
digest.digest("hex") !== entry.sha256) {
|
|
132
|
+
return { ok: false, reason: "CHANGED" };
|
|
133
|
+
}
|
|
134
|
+
let pathnameAfter;
|
|
135
|
+
try {
|
|
136
|
+
pathnameAfter = await fs.promises.stat(entry.path, { bigint: true });
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
return { ok: false, reason: classifyReadError(error) };
|
|
140
|
+
}
|
|
141
|
+
if (!sameIdentity(identityOf(pathnameAfter), entry.identity)) {
|
|
142
|
+
return { ok: false, reason: "REPLACED" };
|
|
143
|
+
}
|
|
144
|
+
if (pathnameAfter.size.toString() !== entry.identity.size) {
|
|
145
|
+
return { ok: false, reason: "CHANGED" };
|
|
146
|
+
}
|
|
147
|
+
if (projectionLimitExceeded) {
|
|
148
|
+
return { ok: false, reason: "LIMIT_EXCEEDED" };
|
|
149
|
+
}
|
|
150
|
+
return { ok: true, projection: { links, branchKeys, malformedLines } };
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
await handle.close().catch(() => undefined);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fail-open Claude Code PreToolUse hook for AgentWrangler context budgets.
|
|
3
|
+
* It intentionally has no package dependencies, never exits non-zero, and never
|
|
4
|
+
* denies a tool call — both thresholds are warn-only (a deny at the hard threshold
|
|
5
|
+
* would block the very checkpoint/clear it prescribes).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import * as fs from "node:fs";
|
|
9
|
+
import * as http from "node:http";
|
|
10
|
+
import * as os from "node:os";
|
|
11
|
+
import * as path from "node:path";
|
|
12
|
+
import process from "node:process";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
|
|
15
|
+
const REQUEST_TIMEOUT_MS = 500;
|
|
16
|
+
const SOFT_DEBOUNCE_MS = 60_000;
|
|
17
|
+
|
|
18
|
+
/** Convert an endpoint stage to the only hook output it may emit. */
|
|
19
|
+
export function stageToStdout(stage) {
|
|
20
|
+
if (stage === "hard") {
|
|
21
|
+
return JSON.stringify({
|
|
22
|
+
hookSpecificOutput: {
|
|
23
|
+
hookEventName: "PreToolUse",
|
|
24
|
+
permissionDecision: "allow",
|
|
25
|
+
permissionDecisionReason: "Context budget at its urgent threshold.",
|
|
26
|
+
},
|
|
27
|
+
additionalContext:
|
|
28
|
+
"Context budget is at its urgent threshold. Checkpoint now, then run /clear.",
|
|
29
|
+
systemMessage:
|
|
30
|
+
"AgentWrangler: context budget at its urgent threshold. Checkpoint now, then run /clear.",
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
if (stage === "soft") {
|
|
34
|
+
return JSON.stringify({
|
|
35
|
+
hookSpecificOutput: {
|
|
36
|
+
hookEventName: "PreToolUse",
|
|
37
|
+
permissionDecision: "allow",
|
|
38
|
+
permissionDecisionReason: "Context budget approaching its limit.",
|
|
39
|
+
},
|
|
40
|
+
additionalContext:
|
|
41
|
+
"Context budget is approaching its limit. Checkpoint, then run /clear or /compact soon.",
|
|
42
|
+
systemMessage:
|
|
43
|
+
"AgentWrangler: context budget is approaching its limit. Checkpoint, then run /clear or /compact soon.",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
return "";
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function markerPath(sessionId) {
|
|
50
|
+
const markerRoot =
|
|
51
|
+
process.env.AW_HOOK_MARKER_DIR ?? path.join(os.homedir(), ".agentwrangler", "hook-markers");
|
|
52
|
+
return path.join(markerRoot, `context-budget-${encodeURIComponent(sessionId)}.marker`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function shouldEmitSoft(sessionId) {
|
|
56
|
+
const file = markerPath(sessionId);
|
|
57
|
+
const now = Date.now();
|
|
58
|
+
try {
|
|
59
|
+
const previous = Number(fs.readFileSync(file, "utf8"));
|
|
60
|
+
if (Number.isFinite(previous) && now - previous < SOFT_DEBOUNCE_MS) return false;
|
|
61
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
62
|
+
fs.writeFileSync(file, String(now), { encoding: "utf8", mode: 0o600 });
|
|
63
|
+
return true;
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (error && typeof error === "object" && error.code === "ENOENT") {
|
|
66
|
+
try {
|
|
67
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
68
|
+
fs.writeFileSync(file, String(now), { encoding: "utf8", mode: 0o600 });
|
|
69
|
+
return true;
|
|
70
|
+
} catch {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function readStdin() {
|
|
79
|
+
return new Promise((resolve, reject) => {
|
|
80
|
+
let raw = "";
|
|
81
|
+
process.stdin.setEncoding("utf8");
|
|
82
|
+
process.stdin.on("data", (chunk) => {
|
|
83
|
+
raw += chunk;
|
|
84
|
+
});
|
|
85
|
+
process.stdin.on("end", () => resolve(raw));
|
|
86
|
+
process.stdin.on("error", reject);
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function fetchBudget(sessionId) {
|
|
91
|
+
return new Promise((resolve, reject) => {
|
|
92
|
+
const port = process.env.AW_PORT ?? "47821";
|
|
93
|
+
const request = http.get(
|
|
94
|
+
`http://127.0.0.1:${port}/api/context-budget?session_id=${encodeURIComponent(sessionId)}`,
|
|
95
|
+
(response) => {
|
|
96
|
+
if (response.statusCode !== 200) {
|
|
97
|
+
response.resume();
|
|
98
|
+
reject(new Error("non-200 response"));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let raw = "";
|
|
102
|
+
response.setEncoding("utf8");
|
|
103
|
+
response.on("data", (chunk) => {
|
|
104
|
+
raw += chunk;
|
|
105
|
+
});
|
|
106
|
+
response.on("end", () => {
|
|
107
|
+
try {
|
|
108
|
+
const body = JSON.parse(raw);
|
|
109
|
+
resolve(body?.data);
|
|
110
|
+
} catch {
|
|
111
|
+
reject(new Error("invalid response"));
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
},
|
|
115
|
+
);
|
|
116
|
+
request.setTimeout(REQUEST_TIMEOUT_MS, () => request.destroy(new Error("timeout")));
|
|
117
|
+
request.on("error", reject);
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function main() {
|
|
122
|
+
try {
|
|
123
|
+
const input = JSON.parse(await readStdin());
|
|
124
|
+
const sessionId = input?.session_id;
|
|
125
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) return;
|
|
126
|
+
const budget = await fetchBudget(sessionId);
|
|
127
|
+
const stage = budget?.stage;
|
|
128
|
+
if (stage === "soft" && !shouldEmitSoft(sessionId)) return;
|
|
129
|
+
const output = stageToStdout(stage);
|
|
130
|
+
if (output) process.stdout.write(output);
|
|
131
|
+
} catch {
|
|
132
|
+
// Hooks must never block Claude Code if AgentWrangler is unavailable.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const invoked =
|
|
137
|
+
process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
|
138
|
+
if (invoked) void main();
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_comment": "AgentWrangler dangerous-command guard deny-list. Extend this file directly; it is data, not code. Each rule has a `pattern` (a JavaScript RegExp source, tested against the Bash command) and a static `reason` (never the command text). `deny` = hard-block via permissionDecision:deny (bypass-mode-proof); `ask` = prompt for confirmation. Deny is checked before ask. An invalid regex or bad shape makes the whole file fall back to the embedded baseline in danger-guard-hook.mjs (fail-closed on the deny tier).",
|
|
3
|
+
"deny": [
|
|
4
|
+
{
|
|
5
|
+
"pattern": "(?:^|[\\s;&|(])rm\\s+(?:-\\S*\\s+)*-\\S*r\\S*\\s+(?:[^;&|]*\\s)?(?:/|~|\\$HOME)(?:\\s|/|\\*|$)",
|
|
6
|
+
"reason": "recursive delete targeting your home directory or the filesystem root"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"pattern": "git\\s+push\\b(?=[^;&|]*(?:--force|-f)\\b)(?=[^;&|]*\\b(?:main|master)\\b)",
|
|
10
|
+
"reason": "force-push to a protected branch (main/master)"
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"pattern": "\\b(?:curl|wget)\\b[^;&|]*\\|\\s*(?:sudo\\s+)?(?:sh|bash|zsh|dash|ksh)\\b",
|
|
14
|
+
"reason": "piping a downloaded script straight into a shell"
|
|
15
|
+
}
|
|
16
|
+
],
|
|
17
|
+
"ask": [
|
|
18
|
+
{
|
|
19
|
+
"pattern": "(?:^|[\\s;&|(])rm\\s+(?:-\\S*\\s+)*-\\S*r\\S*\\b",
|
|
20
|
+
"reason": "recursive delete — confirm the target before running"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"pattern": "git\\s+push\\b(?=[^;&|]*(?:--force|-f)\\b)",
|
|
24
|
+
"reason": "force-push — confirm the branch before running"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|