agentwrangler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +116 -0
- package/dist/apply/jobs.js +429 -0
- package/dist/apply/open-terminal-child.mjs +98 -0
- package/dist/apply/open-terminal.js +221 -0
- package/dist/apply/settings-gen.js +35 -0
- package/dist/cli/agentwrangler.js +18 -0
- package/dist/daemon/config.js +51 -0
- package/dist/daemon/http.js +258 -0
- package/dist/daemon/index.js +372 -0
- package/dist/daemon/outcomes-pass.js +82 -0
- package/dist/daemon/readiness.js +15 -0
- package/dist/daemon/router.js +756 -0
- package/dist/daemon/static.js +146 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrations/001_observe.sql +196 -0
- package/dist/db/migrations/002_indexes.sql +6 -0
- package/dist/db/migrations/003_context_inventory_history.sql +20 -0
- package/dist/db/migrations/004_apply_jobs.sql +17 -0
- package/dist/db/migrations/005_tool_event_metadata.sql +17 -0
- package/dist/db/migrations/006_d7_query_indexes.sql +9 -0
- package/dist/db/migrations/007_work_item_branch_keys.sql +11 -0
- package/dist/db/migrations/008_thinking_tokens.sql +1 -0
- package/dist/db/migrations/009_user_turn_count.sql +1 -0
- package/dist/db/migrations/010_workspace_cwd.sql +1 -0
- package/dist/db/migrations/011_reports.sql +1 -0
- package/dist/db/migrations/012_reconcile_indexes.sql +2 -0
- package/dist/db/migrations/013_friction_fields.sql +5 -0
- package/dist/db/migrations/014_session_churn.sql +11 -0
- package/dist/db/migrations/015_gap_aggregates.sql +6 -0
- package/dist/db/open.js +30 -0
- package/dist/detector/benchmark-anchors.js +36 -0
- package/dist/detector/calibration.js +302 -0
- package/dist/detector/context-history-retention.js +312 -0
- package/dist/detector/context-probe.js +574 -0
- package/dist/detector/d1-source-identity.js +25 -0
- package/dist/detector/detectors/d10_catalog_footprint.js +146 -0
- package/dist/detector/detectors/d1_ctx_always_loaded.js +203 -0
- package/dist/detector/detectors/d2_session_long_full_context.js +119 -0
- package/dist/detector/detectors/d4_model_mismatch.js +258 -0
- package/dist/detector/detectors/d5_limit_burn_forecast.js +138 -0
- package/dist/detector/detectors/d6_tool_result_bloat.js +301 -0
- package/dist/detector/detectors/d7_loop_retry_waste.js +345 -0
- package/dist/detector/detectors/d8_cache_write_churn.js +201 -0
- package/dist/detector/detectors/d9_idle_background_session.js +101 -0
- package/dist/detector/engine.js +88 -0
- package/dist/detector/index.js +17 -0
- package/dist/detector/measurement.js +426 -0
- package/dist/detector/practice-registry.js +259 -0
- package/dist/detector/registry.js +32 -0
- package/dist/detector/savings.js +249 -0
- package/dist/detector/types.js +14 -0
- package/dist/evidence/common/approved-input.js +632 -0
- package/dist/evidence/common/boundary.js +84 -0
- package/dist/evidence/common/canonical.js +55 -0
- package/dist/evidence/common/redaction.js +321 -0
- package/dist/evidence/common/sqlite.js +25 -0
- package/dist/evidence/common/state.js +29 -0
- package/dist/evidence/cond1/cli.js +289 -0
- package/dist/evidence/cond1/packet.js +407 -0
- package/dist/evidence/cond1/prepare.js +295 -0
- package/dist/evidence/cond1/score.js +349 -0
- package/dist/evidence/cond1/types.js +1 -0
- package/dist/evidence/create-approval.js +365 -0
- package/dist/evidence/create-scratch.js +542 -0
- package/dist/evidence/d7/cli.js +113 -0
- package/dist/evidence/d7/measure.js +193 -0
- package/dist/evidence/d7/types.js +1 -0
- package/dist/evidence/discover-approval.js +492 -0
- package/dist/evidence/g2/adjudicate.js +20 -0
- package/dist/evidence/g2/cli.js +207 -0
- package/dist/evidence/g2/kappa.js +39 -0
- package/dist/evidence/g2/pipeline.js +92 -0
- package/dist/evidence/g2/store.js +14 -0
- package/dist/evidence/github/client.js +1 -0
- package/dist/evidence/github/gh-cli-client.js +301 -0
- package/dist/evidence/r3/cli.js +209 -0
- package/dist/evidence/r3/evaluate.js +417 -0
- package/dist/evidence/r3/packet.js +162 -0
- package/dist/evidence/r3/prepare.js +405 -0
- package/dist/evidence/r3/score.js +341 -0
- package/dist/evidence/r3/transcript.js +155 -0
- package/dist/evidence/r3/types.js +4 -0
- package/dist/hook/context-budget-hook.mjs +138 -0
- package/dist/hook/danger-guard-denylist.json +27 -0
- package/dist/hook/danger-guard-hook.mjs +167 -0
- package/dist/hook/install.js +0 -0
- package/dist/hook/limit-burn-hook.mjs +127 -0
- package/dist/hook/loop-guard-hook.mjs +104 -0
- package/dist/hook/precompact-checkpoint-hook.mjs +123 -0
- package/dist/ingest/churn-collector.js +122 -0
- package/dist/ingest/detector-hook.js +52 -0
- package/dist/ingest/discovery.js +207 -0
- package/dist/ingest/health.js +43 -0
- package/dist/ingest/index.js +28 -0
- package/dist/ingest/ingestor.js +509 -0
- package/dist/ingest/parser.js +344 -0
- package/dist/ingest/pricing.js +153 -0
- package/dist/ingest/reconcile.js +52 -0
- package/dist/ingest/tail.js +152 -0
- package/dist/ingest/types.js +24 -0
- package/dist/ingest/workspace-mapping.js +114 -0
- package/dist/oauth/anthropic-api-key.js +88 -0
- package/dist/oauth/count-tokens.js +86 -0
- package/dist/oauth/credentials.js +171 -0
- package/dist/oauth/judge-g2-client.js +154 -0
- package/dist/oauth/usage.js +167 -0
- package/dist/outcomes/branch-key.js +49 -0
- package/dist/outcomes/conclusions.js +45 -0
- package/dist/outcomes/derive.js +94 -0
- package/dist/outcomes/finding-extractors.js +131 -0
- package/dist/outcomes/findings.js +237 -0
- package/dist/outcomes/github/client.js +367 -0
- package/dist/outcomes/github/credential.js +195 -0
- package/dist/outcomes/github/gh-cli-client.js +340 -0
- package/dist/outcomes/linker.js +486 -0
- package/dist/outcomes/pool.js +24 -0
- package/dist/outcomes/sync.js +276 -0
- package/dist/query/api/agents-liveness.js +182 -0
- package/dist/query/api/burn-status.js +50 -0
- package/dist/query/api/context-budget.js +114 -0
- package/dist/query/api/context-composition.js +67 -0
- package/dist/query/api/cost-per-success.js +104 -0
- package/dist/query/api/delivery.js +92 -0
- package/dist/query/api/effectiveness.js +254 -0
- package/dist/query/api/efficiency-headroom.js +74 -0
- package/dist/query/api/headroom-trend.js +105 -0
- package/dist/query/api/hook-config.js +75 -0
- package/dist/query/api/hook-install.js +8 -0
- package/dist/query/api/hot-sessions.js +17 -0
- package/dist/query/api/idle-sessions.js +52 -0
- package/dist/query/api/index.js +40 -0
- package/dist/query/api/loop-guard.js +90 -0
- package/dist/query/api/offload-share.js +41 -0
- package/dist/query/api/outcomes.js +218 -0
- package/dist/query/api/overview.js +535 -0
- package/dist/query/api/rec-prompt.js +138 -0
- package/dist/query/api/recommendations-ledger.js +111 -0
- package/dist/query/api/recommendations.js +514 -0
- package/dist/query/api/reports.js +78 -0
- package/dist/query/api/self-churn.js +77 -0
- package/dist/query/api/self-percentiles.js +109 -0
- package/dist/query/api/session-drivers.js +153 -0
- package/dist/query/api/settings.js +85 -0
- package/dist/query/api/spend-flavor.js +234 -0
- package/dist/query/api/trends.js +155 -0
- package/dist/query/cap-weighted.js +119 -0
- package/dist/query/db-context.js +42 -0
- package/dist/query/envelope.js +71 -0
- package/dist/query/forecast.js +191 -0
- package/dist/query/settings-store.js +441 -0
- package/dist/query/spend.js +171 -0
- package/dist/query/trends.js +194 -0
- package/dist/ui/assets/index-DnRKgc21.css +1 -0
- package/dist/ui/assets/index-h1Q1wWq5.js +168 -0
- package/dist/ui/index.html +39 -0
- package/package.json +59 -0
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/outcomes/findings.ts — FindingsExtractor E1/E2/E3.
|
|
3
|
+
*
|
|
4
|
+
* All three extractors ship EXPERIMENTAL and are excluded from gated/deferral
|
|
5
|
+
* denominators (COND-1 conservative — plan §3, §6 Q6).
|
|
6
|
+
*
|
|
7
|
+
* E1 UNRESOLVED_THREAD: GraphQL isResolved; unresolved@merge → DEFERRED,
|
|
8
|
+
* resolved-pre-merge → ADDRESSED; evidence_ref = thread id.
|
|
9
|
+
* E2 DEFERRAL_SECTION: regex on in-memory PR body; each item → DEFERRED;
|
|
10
|
+
* evidence_ref = keyword-class+index (SEC-101: fixed enum + position, never
|
|
11
|
+
* the raw heading text).
|
|
12
|
+
* E3 DIFF_MARKER: TODO/FIXME on in-memory diff (added lines only), excluding
|
|
13
|
+
* vendored/lockfile globs; evidence_ref = file:line@commit.
|
|
14
|
+
*
|
|
15
|
+
* Bodies and diffs are parsed in-memory and discarded — never stored (SEC-101).
|
|
16
|
+
* Findings written: source/extractor_version/evidence_ref/raised_at/status.
|
|
17
|
+
* Clearance: cleared_at/cleared_by set when a subsequent run finds them resolved.
|
|
18
|
+
*
|
|
19
|
+
* Pass bounding (latency): bodies/diffs of TERMINAL work items are immutable,
|
|
20
|
+
* so E2/E3 fetch them only for items never yet extracted at this synced_at
|
|
21
|
+
* (per-repo user_config watermark `gh_findings_seen:<owner>/<repo>`, same
|
|
22
|
+
* pattern as the sync gh_watermark). E1 CAN change post-merge (resolution
|
|
23
|
+
* clears DEFERRED), so review threads are re-fetched for OPEN items and
|
|
24
|
+
* newly-synced items every pass, and terminal items at most once per
|
|
25
|
+
* RECHECK_TTL, deterministically staggered by hash of work_item_id (per-item
|
|
26
|
+
* last-check marker `gh_e1_check:<work_item_id>` in user_config — no schema
|
|
27
|
+
* migration). First-ever pass behaves exactly like an unbounded pass.
|
|
28
|
+
*/
|
|
29
|
+
import { EXTRACTOR_VERSIONS, extractDeferralFindings, extractDiffMarkerFindings, projectReviewThreadFindings, } from "./finding-extractors.js";
|
|
30
|
+
import { DEFAULT_CONCURRENCY, mapWithConcurrency } from "./pool.js";
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
// DB helpers
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
function insertFinding(db, findingId, workItemId, source, severity, status, evidenceRef, confidence, raisedAt, extractorVersion) {
|
|
35
|
+
db.prepare(`
|
|
36
|
+
INSERT OR IGNORE INTO review_findings
|
|
37
|
+
(finding_id, work_item_id, source, severity, status, evidence_ref,
|
|
38
|
+
confidence, raised_at, extractor_version)
|
|
39
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
40
|
+
`).run(findingId, workItemId, source, severity, status, evidenceRef, confidence, raisedAt, extractorVersion);
|
|
41
|
+
}
|
|
42
|
+
function clearFinding(db, findingId, clearedBy) {
|
|
43
|
+
db.prepare("UPDATE review_findings SET cleared_at = ?, cleared_by = ? WHERE finding_id = ? AND cleared_at IS NULL").run(new Date().toISOString(), clearedBy, findingId);
|
|
44
|
+
}
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
// Main extractor pass
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
/** Default terminal-item E1 recheck TTL: 7 days. */
|
|
49
|
+
export const DEFAULT_RECHECK_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
50
|
+
/** FNV-1a 32-bit — deterministic stagger phase per work_item_id. */
|
|
51
|
+
function hash32(s) {
|
|
52
|
+
let h = 0x811c9dc5;
|
|
53
|
+
for (let i = 0; i < s.length; i++) {
|
|
54
|
+
h ^= s.charCodeAt(i);
|
|
55
|
+
h = Math.imul(h, 0x01000193);
|
|
56
|
+
}
|
|
57
|
+
return h >>> 0;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Run E1/E2/E3 extractors for all synced work_items in the given workspace.
|
|
61
|
+
* All finding rows carry extractor_version and evidence_ref; no body/diff content stored.
|
|
62
|
+
*/
|
|
63
|
+
export async function extractFindings(db, client, repoOwner, repoName, options = {}) {
|
|
64
|
+
if (!client.enabled)
|
|
65
|
+
return;
|
|
66
|
+
const nowDate = (options.now ?? (() => new Date()))();
|
|
67
|
+
const recheckTtlMs = options.recheckTtlMs ?? DEFAULT_RECHECK_TTL_MS;
|
|
68
|
+
const nowMs = nowDate.getTime();
|
|
69
|
+
const now = nowDate.toISOString();
|
|
70
|
+
const workItems = db
|
|
71
|
+
.prepare(`SELECT wi.work_item_id, wi.number, wi.state, wi.final_commit, wi.synced_at
|
|
72
|
+
FROM work_items wi
|
|
73
|
+
JOIN workspaces w USING (workspace_id)
|
|
74
|
+
WHERE w.repo_owner = ? AND w.repo_name = ?`)
|
|
75
|
+
.all(repoOwner, repoName);
|
|
76
|
+
// Per-repo "seen" watermark: max work_items.synced_at whose terminal
|
|
77
|
+
// body/diff extraction completed successfully. Failed fetches must remain
|
|
78
|
+
// retryable, or transient GitHub errors can permanently hide findings.
|
|
79
|
+
const seenKey = `gh_findings_seen:${repoOwner}/${repoName}`;
|
|
80
|
+
const seenRow = db.prepare("SELECT value FROM user_config WHERE key = ?").get(seenKey);
|
|
81
|
+
const seenWm = seenRow?.value ?? undefined;
|
|
82
|
+
// Hoist prepared statements outside the per-work-item loop to avoid
|
|
83
|
+
// re-creating statement handles on every iteration (performance).
|
|
84
|
+
const selectE1 = db.prepare("SELECT finding_id, status FROM review_findings WHERE finding_id = ?");
|
|
85
|
+
const selectExists = db.prepare("SELECT 1 FROM review_findings WHERE finding_id = ?");
|
|
86
|
+
const selectE1Check = db.prepare("SELECT value FROM user_config WHERE key = ?");
|
|
87
|
+
const ensureConfig = db.prepare(`
|
|
88
|
+
INSERT OR IGNORE INTO user_config (key, value, updated_at) VALUES (?, ?, ?)
|
|
89
|
+
`);
|
|
90
|
+
const updateConfig = db.prepare("UPDATE user_config SET value = ?, updated_at = ? WHERE key = ?");
|
|
91
|
+
// Per-item due decisions (sync, before any network I/O).
|
|
92
|
+
const plans = workItems.map((wi) => {
|
|
93
|
+
const isNew = seenWm === undefined || wi.synced_at === null || wi.synced_at > seenWm;
|
|
94
|
+
const isOpen = wi.state === "OPEN";
|
|
95
|
+
// E1: OPEN items and newly-synced items always; terminal items at most
|
|
96
|
+
// once per recheckTtlMs, staggered deterministically by hash so each pass
|
|
97
|
+
// re-checks roughly total/TTL worth instead of everything at once.
|
|
98
|
+
let e1Due = isNew || isOpen;
|
|
99
|
+
if (!e1Due) {
|
|
100
|
+
const checkKey = `gh_e1_check:${wi.work_item_id}`;
|
|
101
|
+
const last = selectE1Check.get(checkKey);
|
|
102
|
+
if (last?.value === undefined || last.value === null) {
|
|
103
|
+
e1Due = true;
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
const lastMs = Date.parse(last.value);
|
|
107
|
+
// Interval is recheckTtlMs * (1 + phase), phase ∈ [0,1): strictly at
|
|
108
|
+
// least TTL between terminal rechecks, evenly spread across passes.
|
|
109
|
+
const phase = (hash32(wi.work_item_id) % 1000) / 1000;
|
|
110
|
+
e1Due = Number.isNaN(lastMs) || nowMs - lastMs >= recheckTtlMs * (1 + phase);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
// E2/E3: terminal bodies/diffs are immutable → fetch only when never yet
|
|
114
|
+
// extracted at this synced_at (or still OPEN and thus mutable).
|
|
115
|
+
const e23Due = isNew || isOpen;
|
|
116
|
+
return { wi, isOpen, e1Due, e23Due };
|
|
117
|
+
});
|
|
118
|
+
const extractionResults = await mapWithConcurrency(plans, DEFAULT_CONCURRENCY, async (plan) => {
|
|
119
|
+
const { wi, isOpen, e1Due, e23Due } = plan;
|
|
120
|
+
const prNumber = wi.number;
|
|
121
|
+
let e23Complete = true;
|
|
122
|
+
// E1: Unresolved review threads (GraphQL)
|
|
123
|
+
if (e1Due) {
|
|
124
|
+
const threadsResult = await client.getReviewThreads(repoOwner, repoName, prNumber);
|
|
125
|
+
if (threadsResult.ok) {
|
|
126
|
+
for (const finding of projectReviewThreadFindings(wi.work_item_id, threadsResult.data)) {
|
|
127
|
+
const findingId = finding.sourceFindingId;
|
|
128
|
+
// Merged PRs: unresolved@merge → DEFERRED; resolved → ADDRESSED
|
|
129
|
+
const status = finding.status;
|
|
130
|
+
// Check if finding already exists
|
|
131
|
+
const existing = selectE1.get(findingId);
|
|
132
|
+
if (existing === undefined) {
|
|
133
|
+
insertFinding(db, findingId, wi.work_item_id, "UNRESOLVED_THREAD", "UNKNOWN", status, finding.evidenceRef, null, now, EXTRACTOR_VERSIONS.E1);
|
|
134
|
+
}
|
|
135
|
+
else if (existing.status === "DEFERRED" && status === "ADDRESSED") {
|
|
136
|
+
// Thread got resolved → clear it
|
|
137
|
+
clearFinding(db, findingId, wi.final_commit ?? wi.work_item_id);
|
|
138
|
+
db.prepare("UPDATE review_findings SET status = ? WHERE finding_id = ?").run("ADDRESSED", findingId);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Record last E1 check for terminal items only after a successful
|
|
142
|
+
// GraphQL read. OPEN items are checked every pass, so their marker
|
|
143
|
+
// would be dead weight.
|
|
144
|
+
if (!isOpen) {
|
|
145
|
+
const checkKey = `gh_e1_check:${wi.work_item_id}`;
|
|
146
|
+
ensureConfig.run(checkKey, now, now);
|
|
147
|
+
updateConfig.run(now, now, checkKey);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (!e23Due)
|
|
152
|
+
return { syncedAt: wi.synced_at, isOpen, e23Due, e23Complete };
|
|
153
|
+
// E2: Deferral sections in body (in-memory only)
|
|
154
|
+
{
|
|
155
|
+
const bodyResult = await client.getPRBody(repoOwner, repoName, prNumber);
|
|
156
|
+
if (bodyResult.ok) {
|
|
157
|
+
if (bodyResult.data.length > 0) {
|
|
158
|
+
const items = extractDeferralFindings(bodyResult.data, wi.work_item_id);
|
|
159
|
+
for (let idx = 0; idx < items.length; idx++) {
|
|
160
|
+
const item = items[idx];
|
|
161
|
+
if (item === undefined)
|
|
162
|
+
continue;
|
|
163
|
+
const findingId = `e2:${wi.work_item_id}:${idx}`;
|
|
164
|
+
const existing = selectExists.get(findingId);
|
|
165
|
+
if (existing === undefined) {
|
|
166
|
+
insertFinding(db, findingId, wi.work_item_id, "DEFERRAL_SECTION", "UNKNOWN", item.status, item.evidenceRef, null, now, EXTRACTOR_VERSIONS.E2);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// body variable goes out of scope here — content discarded
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
e23Complete = false;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// E3: TODO/FIXME in diff (in-memory only)
|
|
177
|
+
if (wi.state === "MERGED" && wi.final_commit !== null) {
|
|
178
|
+
const diffResult = await client.getPRDiff(repoOwner, repoName, prNumber);
|
|
179
|
+
if (diffResult.ok) {
|
|
180
|
+
if (diffResult.data.length > 0) {
|
|
181
|
+
const markers = extractDiffMarkerFindings(diffResult.data, wi.final_commit, wi.work_item_id);
|
|
182
|
+
for (let idx = 0; idx < markers.length; idx++) {
|
|
183
|
+
const marker = markers[idx];
|
|
184
|
+
if (marker === undefined)
|
|
185
|
+
continue;
|
|
186
|
+
const findingId = `e3:${wi.work_item_id}:${idx}`;
|
|
187
|
+
const existing = selectExists.get(findingId);
|
|
188
|
+
if (existing === undefined) {
|
|
189
|
+
insertFinding(db, findingId, wi.work_item_id, "DIFF_MARKER", "LOW", "UNKNOWN", marker.evidenceRef, null, now, EXTRACTOR_VERSIONS.E3);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// diff variable goes out of scope here — content discarded
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
e23Complete = false;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return { syncedAt: wi.synced_at, isOpen, e23Due, e23Complete };
|
|
200
|
+
});
|
|
201
|
+
// Advance the seen watermark only through synced_at groups whose terminal
|
|
202
|
+
// body/diff extraction completed. syncWorkItems writes the same synced_at for
|
|
203
|
+
// a fetched page, so a single failed item must hold the whole group retryable.
|
|
204
|
+
let newestSeen = seenWm ?? "1970-01-01T00:00:00Z";
|
|
205
|
+
const terminalDueBySyncedAt = new Map();
|
|
206
|
+
for (const result of extractionResults) {
|
|
207
|
+
if (!result.e23Due || result.isOpen || result.syncedAt === null)
|
|
208
|
+
continue;
|
|
209
|
+
const previous = terminalDueBySyncedAt.get(result.syncedAt) ?? true;
|
|
210
|
+
terminalDueBySyncedAt.set(result.syncedAt, previous && result.e23Complete);
|
|
211
|
+
}
|
|
212
|
+
const syncedGroups = Array.from(terminalDueBySyncedAt.entries()).sort(([a], [b]) => a.localeCompare(b));
|
|
213
|
+
for (const [syncedAt, complete] of syncedGroups) {
|
|
214
|
+
if (syncedAt <= newestSeen)
|
|
215
|
+
continue;
|
|
216
|
+
if (!complete)
|
|
217
|
+
break;
|
|
218
|
+
newestSeen = syncedAt;
|
|
219
|
+
}
|
|
220
|
+
if (workItems.length > 0) {
|
|
221
|
+
ensureConfig.run(seenKey, newestSeen, now);
|
|
222
|
+
updateConfig.run(newestSeen, now, seenKey);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Run extractors for all mapped workspaces.
|
|
227
|
+
*/
|
|
228
|
+
export async function extractAllFindings(db, client) {
|
|
229
|
+
if (!client.enabled)
|
|
230
|
+
return;
|
|
231
|
+
const workspaces = db
|
|
232
|
+
.prepare("SELECT repo_owner, repo_name FROM workspaces WHERE repo_owner IS NOT NULL AND repo_name IS NOT NULL")
|
|
233
|
+
.all();
|
|
234
|
+
for (const ws of workspaces) {
|
|
235
|
+
await extractFindings(db, client, ws.repo_owner, ws.repo_name);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/outcomes/github/client.ts — GitHub sync client.
|
|
3
|
+
*
|
|
4
|
+
* Uses fetch (REST + GraphQL) — NOT Octokit (plan §6 Q3: no new dep).
|
|
5
|
+
* Injected token + injected fetch for testability.
|
|
6
|
+
*
|
|
7
|
+
* Disabled state: when token is ok:false, ALL methods return {ok:false,reason}
|
|
8
|
+
* and make ZERO network calls.
|
|
9
|
+
*
|
|
10
|
+
* Clean degradation (HARD constraint): never throws; always {ok}|{ok:false,reason}.
|
|
11
|
+
*/
|
|
12
|
+
import { fingerprintBranchRef } from "../branch-key.js";
|
|
13
|
+
import { aggregateCheckConclusion, normalizePRBody } from "../conclusions.js";
|
|
14
|
+
function isRecord(value) {
|
|
15
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16
|
+
}
|
|
17
|
+
function isNullableString(value) {
|
|
18
|
+
return typeof value === "string" || value === null;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Validate and privacy-project a REST pull-request payload.
|
|
22
|
+
* The raw head ref is fingerprinted while local to this function and is never
|
|
23
|
+
* included in either the returned value or an error reason.
|
|
24
|
+
*/
|
|
25
|
+
function projectPR(value) {
|
|
26
|
+
if (!isRecord(value) || !isRecord(value.head) || typeof value.head.ref !== "string")
|
|
27
|
+
return null;
|
|
28
|
+
if (!Number.isInteger(value.number) ||
|
|
29
|
+
(value.state !== "open" && value.state !== "closed" && value.state !== "merged") ||
|
|
30
|
+
!isNullableString(value.merged_at) ||
|
|
31
|
+
!isNullableString(value.closed_at) ||
|
|
32
|
+
typeof value.created_at !== "string" ||
|
|
33
|
+
!isNullableString(value.merge_commit_sha) ||
|
|
34
|
+
typeof value.head.sha !== "string") {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
number: value.number,
|
|
39
|
+
state: value.state,
|
|
40
|
+
merged_at: value.merged_at,
|
|
41
|
+
closed_at: value.closed_at,
|
|
42
|
+
created_at: value.created_at,
|
|
43
|
+
merge_commit_sha: value.merge_commit_sha,
|
|
44
|
+
head: { sha: value.head.sha, refKey: fingerprintBranchRef(value.head.ref) },
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/** Validate a complete page before exposing any projected PRs. */
|
|
48
|
+
export function projectPRPage(value) {
|
|
49
|
+
if (!Array.isArray(value))
|
|
50
|
+
return { ok: false, reason: "github-pr-payload-invalid" };
|
|
51
|
+
const projected = [];
|
|
52
|
+
for (const item of value) {
|
|
53
|
+
const pr = projectPR(item);
|
|
54
|
+
if (pr === null)
|
|
55
|
+
return { ok: false, reason: "github-pr-payload-invalid" };
|
|
56
|
+
projected.push(pr);
|
|
57
|
+
}
|
|
58
|
+
return { ok: true, data: projected };
|
|
59
|
+
}
|
|
60
|
+
/** Validate and project only the privacy-safe branch key from one PR payload. */
|
|
61
|
+
export function projectPRHeadKey(value) {
|
|
62
|
+
if (!isRecord(value) || !isRecord(value.head) || typeof value.head.ref !== "string") {
|
|
63
|
+
return { ok: false, reason: "github-pr-payload-invalid" };
|
|
64
|
+
}
|
|
65
|
+
return { ok: true, data: fingerprintBranchRef(value.head.ref) };
|
|
66
|
+
}
|
|
67
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
|
|
68
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
69
|
+
const DEFAULT_RETRY_BACKOFF_MS = 500;
|
|
70
|
+
// ---------------------------------------------------------------------------
|
|
71
|
+
// Client
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
const GH_REST = "https://api.github.com";
|
|
74
|
+
const GH_GRAPHQL = "https://api.github.com/graphql";
|
|
75
|
+
export class GithubSyncClient {
|
|
76
|
+
token;
|
|
77
|
+
fetchFn;
|
|
78
|
+
etags = new Map();
|
|
79
|
+
requestTimeoutMs;
|
|
80
|
+
maxAttempts;
|
|
81
|
+
retryBackoffMs;
|
|
82
|
+
constructor(tokenResult, fetchFn = globalThis.fetch, options = {}) {
|
|
83
|
+
this.token = tokenResult.ok ? tokenResult.data : null;
|
|
84
|
+
this.fetchFn = fetchFn;
|
|
85
|
+
this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
86
|
+
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
87
|
+
this.retryBackoffMs = options.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
|
|
88
|
+
}
|
|
89
|
+
get enabled() {
|
|
90
|
+
return this.token !== null;
|
|
91
|
+
}
|
|
92
|
+
disabledResult() {
|
|
93
|
+
return { ok: false, reason: "github-token-unset" };
|
|
94
|
+
}
|
|
95
|
+
headers(extra) {
|
|
96
|
+
return {
|
|
97
|
+
Authorization: `Bearer ${this.token ?? ""}`,
|
|
98
|
+
Accept: "application/vnd.github+json",
|
|
99
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
100
|
+
...extra,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* fetch with a bounded per-attempt timeout and retry-with-backoff.
|
|
105
|
+
* Each attempt arms a fresh AbortSignal.timeout so a request that was aborted
|
|
106
|
+
* because the loop was momentarily starved gets a clean retry once the loop
|
|
107
|
+
* drains. Re-throws the last error after maxAttempts so the caller's try/catch
|
|
108
|
+
* still maps it to {ok:false,reason} (clean degradation — never hangs the pass).
|
|
109
|
+
* Only thrown errors (abort/network) are retried; an HTTP error response is
|
|
110
|
+
* returned to the caller unchanged (no retry on 4xx/5xx).
|
|
111
|
+
*/
|
|
112
|
+
async fetchWithRetry(url, init) {
|
|
113
|
+
let lastErr;
|
|
114
|
+
for (let attempt = 1; attempt <= this.maxAttempts; attempt++) {
|
|
115
|
+
try {
|
|
116
|
+
return await this.fetchFn(url, {
|
|
117
|
+
...init,
|
|
118
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
catch (e) {
|
|
122
|
+
lastErr = e;
|
|
123
|
+
if (attempt < this.maxAttempts) {
|
|
124
|
+
await new Promise((r) => setTimeout(r, this.retryBackoffMs * attempt));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
throw lastErr;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* List PRs for a repo updated since the watermark ISO-8601 date.
|
|
132
|
+
* Returns up to 100 PRs per call (no pagination for now — watermark minimises volume).
|
|
133
|
+
*/
|
|
134
|
+
async listPRs(owner, repo, since) {
|
|
135
|
+
if (!this.enabled)
|
|
136
|
+
return this.disabledResult();
|
|
137
|
+
const qs = new URLSearchParams({
|
|
138
|
+
state: "all",
|
|
139
|
+
sort: "updated",
|
|
140
|
+
direction: "desc",
|
|
141
|
+
per_page: "100",
|
|
142
|
+
});
|
|
143
|
+
if (since !== undefined)
|
|
144
|
+
qs.set("since", since);
|
|
145
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/pulls?${qs}`;
|
|
146
|
+
const hdrs = this.headers();
|
|
147
|
+
const etag = this.etags.get(url);
|
|
148
|
+
if (etag !== undefined)
|
|
149
|
+
hdrs["If-None-Match"] = etag;
|
|
150
|
+
try {
|
|
151
|
+
const res = await this.fetchWithRetry(url, { headers: hdrs });
|
|
152
|
+
if (res.status === 304)
|
|
153
|
+
return { ok: true, data: [] }; // not modified
|
|
154
|
+
if (!res.ok)
|
|
155
|
+
return { ok: false, reason: `github-api-error:${res.status}` };
|
|
156
|
+
let payload;
|
|
157
|
+
try {
|
|
158
|
+
payload = await res.json();
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return { ok: false, reason: "github-pr-payload-invalid" };
|
|
162
|
+
}
|
|
163
|
+
const projected = projectPRPage(payload);
|
|
164
|
+
if (!projected.ok)
|
|
165
|
+
return projected;
|
|
166
|
+
const newEtag = res.headers.get("ETag");
|
|
167
|
+
if (newEtag !== null)
|
|
168
|
+
this.etags.set(url, newEtag);
|
|
169
|
+
return projected;
|
|
170
|
+
}
|
|
171
|
+
catch (e) {
|
|
172
|
+
return {
|
|
173
|
+
ok: false,
|
|
174
|
+
reason: `github-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async getPRHeadKey(owner, repo, prNumber) {
|
|
179
|
+
if (!this.enabled)
|
|
180
|
+
return this.disabledResult();
|
|
181
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/pulls/${prNumber}`;
|
|
182
|
+
try {
|
|
183
|
+
const res = await this.fetchWithRetry(url, { headers: this.headers() });
|
|
184
|
+
if (!res.ok)
|
|
185
|
+
return { ok: false, reason: `github-pr-error:${res.status}` };
|
|
186
|
+
try {
|
|
187
|
+
return projectPRHeadKey(await res.json());
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return { ok: false, reason: "github-pr-payload-invalid" };
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
catch (e) {
|
|
194
|
+
return {
|
|
195
|
+
ok: false,
|
|
196
|
+
reason: `github-pr-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Get check-run conclusions for the HEAD sha of a PR.
|
|
202
|
+
* Returns aggregated conclusion: SUCCESS|FAILURE|PENDING|NONE.
|
|
203
|
+
*/
|
|
204
|
+
async getCheckConclusion(owner, repo, ref) {
|
|
205
|
+
if (!this.enabled)
|
|
206
|
+
return this.disabledResult();
|
|
207
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/commits/${ref}/check-runs?per_page=100`;
|
|
208
|
+
try {
|
|
209
|
+
const res = await this.fetchWithRetry(url, { headers: this.headers() });
|
|
210
|
+
if (!res.ok)
|
|
211
|
+
return { ok: false, reason: `github-checks-error:${res.status}` };
|
|
212
|
+
const body = (await res.json());
|
|
213
|
+
return aggregateCheckConclusion(body);
|
|
214
|
+
}
|
|
215
|
+
catch (e) {
|
|
216
|
+
return {
|
|
217
|
+
ok: false,
|
|
218
|
+
reason: `github-checks-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* List commit SHAs for a PR (for SHA_OVERLAP linkage).
|
|
224
|
+
*/
|
|
225
|
+
async listPRCommits(owner, repo, prNumber) {
|
|
226
|
+
if (!this.enabled)
|
|
227
|
+
return this.disabledResult();
|
|
228
|
+
// GitHub caps commits per page at 100; PRs with >100 commits truncate the SHA set
|
|
229
|
+
// (pre-existing limitation, same on both transports).
|
|
230
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/pulls/${prNumber}/commits?per_page=100`;
|
|
231
|
+
try {
|
|
232
|
+
const res = await this.fetchWithRetry(url, { headers: this.headers() });
|
|
233
|
+
if (!res.ok)
|
|
234
|
+
return { ok: false, reason: `github-commits-error:${res.status}` };
|
|
235
|
+
const data = (await res.json());
|
|
236
|
+
return {
|
|
237
|
+
ok: true,
|
|
238
|
+
data: data.map((c) => ({ sha: c.sha, message: c.commit?.message ?? "" })),
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
catch (e) {
|
|
242
|
+
return {
|
|
243
|
+
ok: false,
|
|
244
|
+
reason: `github-commits-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
async listPullsForCommit(owner, repo, sha) {
|
|
249
|
+
if (!this.enabled)
|
|
250
|
+
return this.disabledResult();
|
|
251
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/commits/${sha}/pulls`;
|
|
252
|
+
try {
|
|
253
|
+
const res = await this.fetchWithRetry(url, {
|
|
254
|
+
headers: { ...this.headers(), Accept: "application/vnd.github+json" },
|
|
255
|
+
});
|
|
256
|
+
if (res.status === 404)
|
|
257
|
+
return { ok: true, data: [] };
|
|
258
|
+
if (!res.ok)
|
|
259
|
+
return { ok: false, reason: `github-commit-pulls-error:${res.status}` };
|
|
260
|
+
const data = (await res.json());
|
|
261
|
+
return { ok: true, data: data.map((p) => p.number) };
|
|
262
|
+
}
|
|
263
|
+
catch (e) {
|
|
264
|
+
return {
|
|
265
|
+
ok: false,
|
|
266
|
+
reason: `github-commit-pulls-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Fetch PR body for E2 extraction (in-memory only; body never stored).
|
|
272
|
+
*/
|
|
273
|
+
async getPRBody(owner, repo, prNumber) {
|
|
274
|
+
if (!this.enabled)
|
|
275
|
+
return this.disabledResult();
|
|
276
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/pulls/${prNumber}`;
|
|
277
|
+
try {
|
|
278
|
+
const res = await this.fetchWithRetry(url, { headers: this.headers() });
|
|
279
|
+
if (!res.ok)
|
|
280
|
+
return { ok: false, reason: `github-pr-error:${res.status}` };
|
|
281
|
+
const data = (await res.json());
|
|
282
|
+
return { ok: true, data: normalizePRBody(data) };
|
|
283
|
+
}
|
|
284
|
+
catch (e) {
|
|
285
|
+
return {
|
|
286
|
+
ok: false,
|
|
287
|
+
reason: `github-pr-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Fetch diff for E3 extraction (in-memory only; diff never stored).
|
|
293
|
+
*/
|
|
294
|
+
async getPRDiff(owner, repo, prNumber) {
|
|
295
|
+
if (!this.enabled)
|
|
296
|
+
return this.disabledResult();
|
|
297
|
+
const url = `${GH_REST}/repos/${owner}/${repo}/pulls/${prNumber}`;
|
|
298
|
+
const hdrs = { ...this.headers(), Accept: "application/vnd.github.diff" };
|
|
299
|
+
try {
|
|
300
|
+
const res = await this.fetchWithRetry(url, { headers: hdrs });
|
|
301
|
+
if (!res.ok)
|
|
302
|
+
return { ok: false, reason: `github-diff-error:${res.status}` };
|
|
303
|
+
const diff = await res.text();
|
|
304
|
+
return { ok: true, data: diff };
|
|
305
|
+
}
|
|
306
|
+
catch (e) {
|
|
307
|
+
return {
|
|
308
|
+
ok: false,
|
|
309
|
+
reason: `github-diff-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Fetch review thread resolution status via GraphQL (E1 extraction).
|
|
315
|
+
* Returns list of {id, isResolved} for the PR.
|
|
316
|
+
* Paginates automatically so PRs with >100 review threads are not truncated.
|
|
317
|
+
*/
|
|
318
|
+
async getReviewThreads(owner, repo, prNumber) {
|
|
319
|
+
if (!this.enabled)
|
|
320
|
+
return this.disabledResult();
|
|
321
|
+
const query = `
|
|
322
|
+
query($owner:String!, $repo:String!, $pr:Int!, $cursor:String) {
|
|
323
|
+
repository(owner:$owner, name:$repo) {
|
|
324
|
+
pullRequest(number:$pr) {
|
|
325
|
+
reviewThreads(first:100, after:$cursor) {
|
|
326
|
+
pageInfo { hasNextPage endCursor }
|
|
327
|
+
nodes { id isResolved }
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
`;
|
|
333
|
+
try {
|
|
334
|
+
const allThreads = [];
|
|
335
|
+
let cursor = null;
|
|
336
|
+
do {
|
|
337
|
+
const res = await this.fetchWithRetry(GH_GRAPHQL, {
|
|
338
|
+
method: "POST",
|
|
339
|
+
headers: { ...this.headers(), "Content-Type": "application/json" },
|
|
340
|
+
body: JSON.stringify({ query, variables: { owner, repo, pr: prNumber, cursor } }),
|
|
341
|
+
});
|
|
342
|
+
if (!res.ok)
|
|
343
|
+
return { ok: false, reason: `github-graphql-error:${res.status}` };
|
|
344
|
+
const body = (await res.json());
|
|
345
|
+
if (body.errors !== undefined && body.errors.length > 0) {
|
|
346
|
+
return {
|
|
347
|
+
ok: false,
|
|
348
|
+
reason: `github-graphql-errors: ${body.errors.map((e) => e.message).join("; ")}`,
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
const reviewThreads = body.data?.repository?.pullRequest?.reviewThreads;
|
|
352
|
+
allThreads.push(...(reviewThreads?.nodes ?? []));
|
|
353
|
+
const pageInfo = reviewThreads?.pageInfo;
|
|
354
|
+
if (pageInfo?.hasNextPage !== true)
|
|
355
|
+
break;
|
|
356
|
+
cursor = pageInfo.endCursor ?? null;
|
|
357
|
+
} while (cursor !== null);
|
|
358
|
+
return { ok: true, data: allThreads };
|
|
359
|
+
}
|
|
360
|
+
catch (e) {
|
|
361
|
+
return {
|
|
362
|
+
ok: false,
|
|
363
|
+
reason: `github-graphql-fetch-error: ${e instanceof Error ? e.message : String(e)}`,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|