@obsunified/collector 2.2.0 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.js +1 -1
- package/dist/{chunk-KMEZKGSO.js → chunk-2FALTDCE.js} +12 -4
- package/dist/chunk-2FALTDCE.js.map +1 -0
- package/dist/index.d.ts +61 -3
- package/dist/index.js +10933 -9769
- package/dist/index.js.map +1 -1
- package/package.json +2 -3
- package/src/__tests__/autosre-closed-loop.test.ts +1554 -17
- package/src/auth/dashboard-auth.test.ts +61 -0
- package/src/auth/dashboard-auth.ts +17 -2
- package/src/lib/ai-store/calls.ts +43 -23
- package/src/lib/ai-store.ts +12 -1
- package/src/lib/analyses-runner.ts +44 -7
- package/src/lib/autosre-change-correlation.ts +39 -12
- package/src/lib/autosre-findings.ts +237 -44
- package/src/lib/autosre-ontology-cache.test.ts +78 -0
- package/src/lib/autosre-ontology.ts +52 -28
- package/src/lib/autosre-remediation-gate.ts +942 -348
- package/src/lib/housekeeping.test.ts +70 -0
- package/src/lib/housekeeping.ts +91 -83
- package/src/lib/projects-store.test.ts +38 -1
- package/src/lib/projects-store.ts +23 -4
- package/src/lib/sql-db-postgres.integration.test.ts +75 -2
- package/src/lib/store/ingest.ts +8 -56
- package/src/lib/usage-store.ts +9 -3
- package/src/migrations/051_incident_active_dedup_unique.sql +31 -0
- package/src/migrations/052_remediation_callback.sql +10 -0
- package/src/migrations/053_remediation_callback_delivery.sql +7 -0
- package/src/migrations/alert-state-notified.test.ts +3 -3
- package/src/migrations-postgres/051_incident_active_dedup_unique.sql +31 -0
- package/src/migrations-postgres/052_remediation_callback.sql +10 -0
- package/src/migrations-postgres/053_remediation_callback_delivery.sql +7 -0
- package/src/plugins/ai-call-by-id.test.ts +73 -0
- package/src/plugins/ai-receiver.ts +10 -0
- package/src/plugins/alerts-evaluator.ts +15 -3
- package/src/plugins/autosre-routes.ts +374 -108
- package/src/plugins/connected-routes/deep-link-shapes.test.ts +44 -0
- package/src/plugins/connected-routes/manifest.test.ts +142 -0
- package/src/plugins/connected-routes/manifest.ts +123 -6
- package/src/plugins/connected-routes-links.test.ts +96 -0
- package/src/plugins/connected-routes.test.ts +3 -3
- package/src/plugins/connected-routes.ts +56 -83
- package/src/plugins/evidence-retrieval-routes.test.ts +32 -0
- package/src/plugins/evidence-retrieval-routes.ts +397 -25
- package/dist/chunk-KMEZKGSO.js.map +0 -1
|
@@ -4,6 +4,8 @@ import type {
|
|
|
4
4
|
ChangeEventKind,
|
|
5
5
|
CohortAttributionVerdict,
|
|
6
6
|
EvidenceEntityKind,
|
|
7
|
+
GrantCallbackDelivery,
|
|
8
|
+
GrantDeliveryResult,
|
|
7
9
|
InvestigationFindingInput,
|
|
8
10
|
JsonValue,
|
|
9
11
|
RemediationActionKind,
|
|
@@ -11,11 +13,18 @@ import type {
|
|
|
11
13
|
RemediationProposalStatus,
|
|
12
14
|
RemediationStateSpec,
|
|
13
15
|
} from "@obsunified/types";
|
|
16
|
+
import {
|
|
17
|
+
ChangeEventKindSchema,
|
|
18
|
+
EvidencePredicateSchema,
|
|
19
|
+
RemediationActionKindSchema,
|
|
20
|
+
} from "@obsunified/types";
|
|
14
21
|
import { AgentAutonomyLevel } from "@obsunified/types/constants";
|
|
15
22
|
import type { CollectorPlugin } from "../framework/collector";
|
|
23
|
+
import type { CollectorEnv } from "../framework/env";
|
|
16
24
|
import {
|
|
17
25
|
computeChangeBlastRadius,
|
|
18
26
|
getChangeEvent,
|
|
27
|
+
InvalidTimestampError,
|
|
19
28
|
listChangeEvents,
|
|
20
29
|
recordChangeEvent,
|
|
21
30
|
} from "../lib/autosre-change-correlation";
|
|
@@ -39,36 +48,27 @@ import {
|
|
|
39
48
|
evaluateRemediationWatchdogs,
|
|
40
49
|
failRemediationProposal,
|
|
41
50
|
getIncidentInvestigation,
|
|
51
|
+
getRemediationCallback,
|
|
42
52
|
getRemediationProposal,
|
|
43
53
|
listIncidentInvestigations,
|
|
44
54
|
listRemediationProposals,
|
|
45
55
|
openOrDeduplicateIncident,
|
|
46
56
|
proposeRemediationAction,
|
|
47
57
|
RemediationGateError,
|
|
58
|
+
recordCallbackDelivery,
|
|
48
59
|
rejectRemediationProposal,
|
|
49
60
|
} from "../lib/autosre-remediation-gate";
|
|
61
|
+
import { sha256Hex } from "../lib/hash";
|
|
50
62
|
import { sqlDbFor } from "../lib/sql-db";
|
|
63
|
+
import { assertSafeWebhookUrl } from "../lib/webhook-url";
|
|
51
64
|
import { getProjectId } from "./_context";
|
|
52
65
|
import { readJsonObject } from "./_json";
|
|
53
66
|
|
|
54
|
-
const CHANGE_KINDS = new Set<
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
"prompt_rollout",
|
|
60
|
-
"agent_version",
|
|
61
|
-
"schema_migration",
|
|
62
|
-
"infra_scale",
|
|
63
|
-
]);
|
|
64
|
-
|
|
65
|
-
const REMEDIATION_ACTION_KINDS = new Set<RemediationActionKind>([
|
|
66
|
-
"open_code_pr",
|
|
67
|
-
"toggle_feature_flag",
|
|
68
|
-
"rollback_deploy",
|
|
69
|
-
"apply_config",
|
|
70
|
-
"promote_agent_prompt",
|
|
71
|
-
]);
|
|
67
|
+
const CHANGE_KINDS = new Set<string>(ChangeEventKindSchema.options);
|
|
68
|
+
|
|
69
|
+
const REMEDIATION_ACTION_KINDS = new Set<string>(
|
|
70
|
+
RemediationActionKindSchema.options,
|
|
71
|
+
);
|
|
72
72
|
|
|
73
73
|
const parseLimit = (raw: string | undefined, fallback = 50): number => {
|
|
74
74
|
const parsed = Number.parseInt(raw ?? String(fallback), 10);
|
|
@@ -85,9 +85,11 @@ const isAgentAutonomyLevel = (value: unknown): value is AgentAutonomyLevel =>
|
|
|
85
85
|
* require an authenticated human dashboard session (`obs_session` cookie) and
|
|
86
86
|
* reject programmatic agent/MCP bearer tokens (`403 Forbidden`).
|
|
87
87
|
*/
|
|
88
|
-
function requireHumanDashboardSession(c: {
|
|
88
|
+
async function requireHumanDashboardSession(c: {
|
|
89
89
|
req: { header: (name: string) => string | undefined };
|
|
90
|
-
}):
|
|
90
|
+
}): Promise<
|
|
91
|
+
{ ok: true; sessionActor: string } | { ok: false; message: string }
|
|
92
|
+
> {
|
|
91
93
|
const authHeader = c.req.header("Authorization");
|
|
92
94
|
if (authHeader) {
|
|
93
95
|
return {
|
|
@@ -98,23 +100,139 @@ function requireHumanDashboardSession(c: {
|
|
|
98
100
|
}
|
|
99
101
|
const cookieHeader = c.req.header("Cookie") ?? "";
|
|
100
102
|
const match = cookieHeader.match(/(?:^|;\s*)obs_session=([^;]*)/);
|
|
101
|
-
if (!match
|
|
103
|
+
if (!match?.[1]) {
|
|
102
104
|
return {
|
|
103
105
|
ok: false,
|
|
104
106
|
message:
|
|
105
107
|
"Human approval requires an authenticated human dashboard session (obs_session cookie).",
|
|
106
108
|
};
|
|
107
109
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
// The recorded approver is derived from the session itself. Client-supplied
|
|
111
|
+
// identities (X-Obs-User-Actor, body.approvedByUserId) are never trusted:
|
|
112
|
+
// they would let any session holder attribute an approval to someone else.
|
|
113
|
+
// Only a hash prefix is stored so the cookie value never reaches the DB.
|
|
114
|
+
let rawSession = match[1];
|
|
115
|
+
try {
|
|
116
|
+
rawSession = decodeURIComponent(rawSession);
|
|
117
|
+
} catch {
|
|
118
|
+
// keep the raw value; it is only hashed
|
|
119
|
+
}
|
|
120
|
+
const sessionActor = `dashboard_session:${(await sha256Hex(rawSession)).slice(0, 16)}`;
|
|
112
121
|
return { ok: true, sessionActor };
|
|
113
122
|
}
|
|
114
123
|
|
|
115
124
|
export const autosreRoutesPlugin: CollectorPlugin = {
|
|
116
125
|
name: "autosre-routes",
|
|
117
126
|
register(app, runtime) {
|
|
127
|
+
/**
|
|
128
|
+
* Grant handoff: the plaintext grant token exists only in the response of
|
|
129
|
+
* the request that mints it (propose with an autonomous grant, or human
|
|
130
|
+
* approve). Incidents do not store a webhook, so the caller may pass
|
|
131
|
+
* `webhookUrl` (+ optional `secret`) in that same request and the token is
|
|
132
|
+
* pushed to the runner over the SSRF-guarded, HMAC-signed incident webhook.
|
|
133
|
+
* Without a `webhookUrl` the token is simply returned in the response.
|
|
134
|
+
*
|
|
135
|
+
* A `webhookUrl` (+ `secret`) given to `propose` for a proposal that is
|
|
136
|
+
* parked for human approval is persisted on the proposal and used when
|
|
137
|
+
* the approve request carries none. A callback on the approve request
|
|
138
|
+
* wins over the stored one. The stored callback is cleared on approve
|
|
139
|
+
* or reject, and only its host is ever returned by GET.
|
|
140
|
+
*/
|
|
141
|
+
const deliverGrant = async (
|
|
142
|
+
env: CollectorEnv | undefined,
|
|
143
|
+
db: ReturnType<typeof sqlDbFor>,
|
|
144
|
+
projectId: string,
|
|
145
|
+
body: Record<string, unknown>,
|
|
146
|
+
proposal: {
|
|
147
|
+
id: string;
|
|
148
|
+
incidentId: string;
|
|
149
|
+
environment: string;
|
|
150
|
+
status: string;
|
|
151
|
+
grantToken?: string;
|
|
152
|
+
},
|
|
153
|
+
/** Callback persisted at propose time; used when the request has none. */
|
|
154
|
+
stored?: { url: string; secret?: string } | null,
|
|
155
|
+
) => {
|
|
156
|
+
const bodyUrl =
|
|
157
|
+
typeof body.webhookUrl === "string" ? body.webhookUrl.trim() : "";
|
|
158
|
+
const target = bodyUrl
|
|
159
|
+
? {
|
|
160
|
+
url: bodyUrl,
|
|
161
|
+
secret: typeof body.secret === "string" ? body.secret : undefined,
|
|
162
|
+
}
|
|
163
|
+
: (stored ?? null);
|
|
164
|
+
const notAttempted: GrantDeliveryResult = {
|
|
165
|
+
attempted: false,
|
|
166
|
+
delivered: false,
|
|
167
|
+
attempts: 0,
|
|
168
|
+
host: null,
|
|
169
|
+
};
|
|
170
|
+
if (!target || !proposal.grantToken) {
|
|
171
|
+
return { webhookDelivery: undefined, grantDelivery: notAttempted };
|
|
172
|
+
}
|
|
173
|
+
if (proposal.status !== "grant_issued") {
|
|
174
|
+
return { webhookDelivery: undefined, grantDelivery: notAttempted };
|
|
175
|
+
}
|
|
176
|
+
let host: string | null = null;
|
|
177
|
+
try {
|
|
178
|
+
host = new URL(target.url).host || null;
|
|
179
|
+
} catch {
|
|
180
|
+
host = null;
|
|
181
|
+
}
|
|
182
|
+
const webhookDelivery = await dispatchIncidentWebhook(
|
|
183
|
+
db,
|
|
184
|
+
projectId,
|
|
185
|
+
proposal.incidentId,
|
|
186
|
+
{
|
|
187
|
+
url: target.url,
|
|
188
|
+
secret: target.secret,
|
|
189
|
+
environment: proposal.environment,
|
|
190
|
+
allowPrivateNetworks: env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
|
|
191
|
+
logger: runtime.logger,
|
|
192
|
+
grant: {
|
|
193
|
+
proposalId: proposal.id,
|
|
194
|
+
status: proposal.status,
|
|
195
|
+
grantToken: proposal.grantToken,
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
);
|
|
199
|
+
// Error text is surfaced to the human: strip anything URL-shaped and
|
|
200
|
+
// the secret so neither the query string nor the HMAC key can leak.
|
|
201
|
+
let error = webhookDelivery.error
|
|
202
|
+
? webhookDelivery.error.replace(
|
|
203
|
+
/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi,
|
|
204
|
+
"<url>",
|
|
205
|
+
)
|
|
206
|
+
: undefined;
|
|
207
|
+
if (error && target.secret) {
|
|
208
|
+
error = error.split(target.secret).join("<redacted>");
|
|
209
|
+
}
|
|
210
|
+
const record = await recordCallbackDelivery(
|
|
211
|
+
db,
|
|
212
|
+
projectId,
|
|
213
|
+
proposal.environment,
|
|
214
|
+
proposal.id,
|
|
215
|
+
{
|
|
216
|
+
delivered: webhookDelivery.delivered,
|
|
217
|
+
attempts: webhookDelivery.attempts,
|
|
218
|
+
error,
|
|
219
|
+
},
|
|
220
|
+
);
|
|
221
|
+
(
|
|
222
|
+
proposal as { callbackDelivery?: GrantCallbackDelivery | null }
|
|
223
|
+
).callbackDelivery = record;
|
|
224
|
+
const grantDelivery: GrantDeliveryResult = {
|
|
225
|
+
attempted: true,
|
|
226
|
+
delivered: webhookDelivery.delivered,
|
|
227
|
+
attempts: webhookDelivery.attempts,
|
|
228
|
+
host,
|
|
229
|
+
...(webhookDelivery.delivered
|
|
230
|
+
? {}
|
|
231
|
+
: { error: error ?? "unknown error" }),
|
|
232
|
+
};
|
|
233
|
+
return { webhookDelivery, grantDelivery };
|
|
234
|
+
};
|
|
235
|
+
|
|
118
236
|
// 1. Change Ingestion (Ingest Auth /v1/changes)
|
|
119
237
|
app.post("/v1/changes", async (c) => {
|
|
120
238
|
const projectId = getProjectId(c);
|
|
@@ -122,7 +240,8 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
122
240
|
if (!body) {
|
|
123
241
|
return c.json({ error: "Invalid JSON body" }, 400);
|
|
124
242
|
}
|
|
125
|
-
const rawKind =
|
|
243
|
+
const rawKind =
|
|
244
|
+
typeof body.kind === "string" ? body.kind.trim() : "deploy";
|
|
126
245
|
const normalizedKind = (
|
|
127
246
|
rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind
|
|
128
247
|
) as ChangeEventKind;
|
|
@@ -181,47 +300,58 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
181
300
|
? body.actor
|
|
182
301
|
: null;
|
|
183
302
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
typeof body.
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
303
|
+
let changeEvent: Awaited<ReturnType<typeof recordChangeEvent>>;
|
|
304
|
+
try {
|
|
305
|
+
changeEvent = await recordChangeEvent(db, projectId, {
|
|
306
|
+
id: typeof body.id === "string" ? body.id : undefined,
|
|
307
|
+
environment:
|
|
308
|
+
typeof body.environment === "string" ? body.environment : undefined,
|
|
309
|
+
kind: normalizedKind,
|
|
310
|
+
source:
|
|
311
|
+
body.source === "inferred_telemetry" ||
|
|
312
|
+
body.source === "autosre_remediation" ||
|
|
313
|
+
body.source === "ci_webhook"
|
|
314
|
+
? body.source
|
|
315
|
+
: "explicit_api",
|
|
316
|
+
targetNodeId:
|
|
317
|
+
typeof body.targetNodeId === "string"
|
|
318
|
+
? body.targetNodeId
|
|
319
|
+
: undefined,
|
|
320
|
+
serviceName,
|
|
321
|
+
namespace,
|
|
322
|
+
version: typeof body.version === "string" ? body.version : null,
|
|
323
|
+
previousVersion:
|
|
324
|
+
typeof body.previousVersion === "string"
|
|
325
|
+
? body.previousVersion
|
|
326
|
+
: null,
|
|
327
|
+
commitSha: typeof body.commitSha === "string" ? body.commitSha : null,
|
|
328
|
+
previousCommitSha:
|
|
329
|
+
typeof body.previousCommitSha === "string"
|
|
330
|
+
? body.previousCommitSha
|
|
331
|
+
: null,
|
|
332
|
+
configDigest:
|
|
333
|
+
typeof body.configDigest === "string" ? body.configDigest : null,
|
|
334
|
+
summary,
|
|
335
|
+
actorId,
|
|
336
|
+
repositoryUrl,
|
|
337
|
+
pullRequestUrl,
|
|
338
|
+
causedByActionId:
|
|
339
|
+
typeof body.causedByActionId === "string"
|
|
340
|
+
? body.causedByActionId
|
|
341
|
+
: null,
|
|
342
|
+
agentRunId:
|
|
343
|
+
typeof body.agentRunId === "string" ? body.agentRunId : null,
|
|
344
|
+
touchedPaths,
|
|
345
|
+
attributes,
|
|
346
|
+
occurredAt:
|
|
347
|
+
typeof body.occurredAt === "string" ? body.occurredAt : undefined,
|
|
348
|
+
});
|
|
349
|
+
} catch (err) {
|
|
350
|
+
if (err instanceof InvalidTimestampError) {
|
|
351
|
+
return c.json({ error: err.message }, 400);
|
|
352
|
+
}
|
|
353
|
+
throw err;
|
|
354
|
+
}
|
|
225
355
|
|
|
226
356
|
return c.json({ changeEvent, change: changeEvent }, 201);
|
|
227
357
|
});
|
|
@@ -254,23 +384,27 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
254
384
|
const rawKind = c.req.query("kind");
|
|
255
385
|
const normalizedKind =
|
|
256
386
|
rawKind === "agent_prompt_release" ? "prompt_rollout" : rawKind;
|
|
257
|
-
if (
|
|
258
|
-
normalizedKind &&
|
|
259
|
-
!CHANGE_KINDS.has(normalizedKind as ChangeEventKind)
|
|
260
|
-
) {
|
|
387
|
+
if (normalizedKind && !CHANGE_KINDS.has(normalizedKind)) {
|
|
261
388
|
return c.json({ error: `Invalid kind: ${rawKind}` }, 400);
|
|
262
389
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
390
|
+
try {
|
|
391
|
+
const changes = await listChangeEvents(db, {
|
|
392
|
+
projectId,
|
|
393
|
+
environment: c.req.query("environment") || undefined,
|
|
394
|
+
serviceName:
|
|
395
|
+
c.req.query("serviceName") || c.req.query("service") || undefined,
|
|
396
|
+
targetNodeId: c.req.query("targetNodeId") || undefined,
|
|
397
|
+
kind: normalizedKind || undefined,
|
|
398
|
+
since: c.req.query("since") || undefined,
|
|
399
|
+
limit: parseLimit(c.req.query("limit"), 50),
|
|
400
|
+
});
|
|
401
|
+
return c.json({ changes });
|
|
402
|
+
} catch (err) {
|
|
403
|
+
if (err instanceof InvalidTimestampError) {
|
|
404
|
+
return c.json({ error: err.message }, 400);
|
|
405
|
+
}
|
|
406
|
+
throw err;
|
|
407
|
+
}
|
|
274
408
|
});
|
|
275
409
|
|
|
276
410
|
app.get("/internal/changes/:id/blast-radius", async (c) => {
|
|
@@ -394,10 +528,13 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
394
528
|
typeof body.leaseDurationSeconds === "number" &&
|
|
395
529
|
Number.isFinite(body.leaseDurationSeconds)
|
|
396
530
|
? body.leaseDurationSeconds
|
|
397
|
-
: typeof body.
|
|
398
|
-
Number.isFinite(body.
|
|
399
|
-
?
|
|
400
|
-
:
|
|
531
|
+
: typeof body.ttlSeconds === "number" &&
|
|
532
|
+
Number.isFinite(body.ttlSeconds)
|
|
533
|
+
? body.ttlSeconds
|
|
534
|
+
: typeof body.leaseDurationMs === "number" &&
|
|
535
|
+
Number.isFinite(body.leaseDurationMs)
|
|
536
|
+
? Math.ceil(body.leaseDurationMs / 1000)
|
|
537
|
+
: undefined;
|
|
401
538
|
const result = await acquireIncidentLease(db, projectId, incidentId, {
|
|
402
539
|
environment:
|
|
403
540
|
typeof body.environment === "string" ? body.environment : undefined,
|
|
@@ -424,9 +561,12 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
424
561
|
const result = await dispatchIncidentWebhook(db, projectId, incidentId, {
|
|
425
562
|
url: webhookUrl,
|
|
426
563
|
secret: typeof body?.secret === "string" ? body.secret : undefined,
|
|
427
|
-
|
|
428
|
-
body?.
|
|
429
|
-
|
|
564
|
+
environment:
|
|
565
|
+
typeof body?.environment === "string" ? body.environment : undefined,
|
|
566
|
+
// Operator-level setting only: a request body must never be able to
|
|
567
|
+
// turn off the SSRF guard.
|
|
568
|
+
allowPrivateNetworks: c.env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
|
|
569
|
+
logger: runtime.logger,
|
|
430
570
|
});
|
|
431
571
|
return c.json(result, result.delivered ? 200 : 502);
|
|
432
572
|
});
|
|
@@ -497,9 +637,33 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
497
637
|
);
|
|
498
638
|
}
|
|
499
639
|
|
|
640
|
+
if (body.causalChain !== undefined && !Array.isArray(body.causalChain)) {
|
|
641
|
+
return c.json({ error: "causalChain must be an array" }, 400);
|
|
642
|
+
}
|
|
643
|
+
for (const [i, triple] of (
|
|
644
|
+
(body.causalChain as unknown[] | undefined) ?? []
|
|
645
|
+
).entries()) {
|
|
646
|
+
const predicate =
|
|
647
|
+
triple && typeof triple === "object"
|
|
648
|
+
? (triple as Record<string, unknown>).predicate
|
|
649
|
+
: undefined;
|
|
650
|
+
if (!EvidencePredicateSchema.safeParse(predicate).success) {
|
|
651
|
+
return c.json(
|
|
652
|
+
{
|
|
653
|
+
error: `causalChain[${i}].predicate is not a known evidence predicate: ${String(predicate)}`,
|
|
654
|
+
},
|
|
655
|
+
400,
|
|
656
|
+
);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
500
660
|
const db = sqlDbFor(c.env);
|
|
501
661
|
const input: InvestigationFindingInput = {
|
|
502
662
|
id: typeof body.id === "string" ? body.id : undefined,
|
|
663
|
+
idempotencyKey:
|
|
664
|
+
typeof body.idempotencyKey === "string" && body.idempotencyKey.trim()
|
|
665
|
+
? body.idempotencyKey.trim()
|
|
666
|
+
: undefined,
|
|
503
667
|
environment:
|
|
504
668
|
typeof body.environment === "string" ? body.environment : undefined,
|
|
505
669
|
incidentId:
|
|
@@ -548,11 +712,16 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
548
712
|
(id): id is string => typeof id === "string",
|
|
549
713
|
)
|
|
550
714
|
: [],
|
|
715
|
+
codeReferences: Array.isArray(body.codeReferences)
|
|
716
|
+
? (body.codeReferences as InvestigationFindingInput["codeReferences"])
|
|
717
|
+
: undefined,
|
|
551
718
|
agentRunId:
|
|
552
719
|
typeof body.agentRunId === "string" ? body.agentRunId : null,
|
|
553
720
|
};
|
|
554
721
|
|
|
555
|
-
const finding = await recordInvestigationFinding(db, projectId, input
|
|
722
|
+
const finding = await recordInvestigationFinding(db, projectId, input, {
|
|
723
|
+
logger: runtime.logger,
|
|
724
|
+
});
|
|
556
725
|
return c.json({ finding, id: finding.id }, 201);
|
|
557
726
|
});
|
|
558
727
|
|
|
@@ -615,6 +784,40 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
615
784
|
}
|
|
616
785
|
});
|
|
617
786
|
|
|
787
|
+
// Lightweight existence check so callers can validate counterfactual
|
|
788
|
+
// prerequisites before persisting a finding.
|
|
789
|
+
app.get("/internal/evals/runs/:id", async (c) => {
|
|
790
|
+
const projectId = getProjectId(c);
|
|
791
|
+
const runId = c.req.param("id");
|
|
792
|
+
const db = sqlDbFor(c.env);
|
|
793
|
+
const row = await db
|
|
794
|
+
.prepare(
|
|
795
|
+
`SELECT id, eval_case_id, service_version, commit_sha,
|
|
796
|
+
pass_count, fail_count
|
|
797
|
+
FROM eval_runs WHERE project_id = ? AND id = ? LIMIT 1`,
|
|
798
|
+
)
|
|
799
|
+
.bind(projectId, runId)
|
|
800
|
+
.first<{
|
|
801
|
+
id: string;
|
|
802
|
+
eval_case_id: string | null;
|
|
803
|
+
service_version: string | null;
|
|
804
|
+
commit_sha: string | null;
|
|
805
|
+
pass_count: number | null;
|
|
806
|
+
fail_count: number | null;
|
|
807
|
+
}>();
|
|
808
|
+
if (!row) return c.json({ error: "Eval run not found" }, 404);
|
|
809
|
+
return c.json({
|
|
810
|
+
run: {
|
|
811
|
+
id: row.id,
|
|
812
|
+
evalCaseId: row.eval_case_id,
|
|
813
|
+
serviceVersion: row.service_version,
|
|
814
|
+
commitSha: row.commit_sha,
|
|
815
|
+
passCount: Number(row.pass_count ?? 0),
|
|
816
|
+
failCount: Number(row.fail_count ?? 0),
|
|
817
|
+
},
|
|
818
|
+
});
|
|
819
|
+
});
|
|
820
|
+
|
|
618
821
|
// 6. Remediation Proposals, Human Approval, Claim Grant, Watchdog
|
|
619
822
|
app.get("/internal/remediations", async (c) => {
|
|
620
823
|
const projectId = getProjectId(c);
|
|
@@ -687,6 +890,22 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
687
890
|
: "autonomous_write";
|
|
688
891
|
|
|
689
892
|
const db = sqlDbFor(c.env);
|
|
893
|
+
const callbackUrl =
|
|
894
|
+
typeof body.webhookUrl === "string" ? body.webhookUrl.trim() : "";
|
|
895
|
+
if (callbackUrl) {
|
|
896
|
+
try {
|
|
897
|
+
assertSafeWebhookUrl(callbackUrl, {
|
|
898
|
+
allowPrivateNetworks: c.env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
|
|
899
|
+
});
|
|
900
|
+
} catch (err) {
|
|
901
|
+
return c.json(
|
|
902
|
+
{
|
|
903
|
+
error: `webhookUrl rejected: ${err instanceof Error ? err.message : String(err)}`,
|
|
904
|
+
},
|
|
905
|
+
400,
|
|
906
|
+
);
|
|
907
|
+
}
|
|
908
|
+
}
|
|
690
909
|
const input: RemediationProposalInput = {
|
|
691
910
|
environment:
|
|
692
911
|
typeof body.environment === "string" ? body.environment : undefined,
|
|
@@ -701,12 +920,32 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
701
920
|
};
|
|
702
921
|
|
|
703
922
|
try {
|
|
704
|
-
const proposal = await proposeRemediationAction(db, projectId, input
|
|
923
|
+
const proposal = await proposeRemediationAction(db, projectId, input, {
|
|
924
|
+
logger: runtime.logger,
|
|
925
|
+
callback: callbackUrl
|
|
926
|
+
? {
|
|
927
|
+
url: callbackUrl,
|
|
928
|
+
secret:
|
|
929
|
+
typeof body.secret === "string" && body.secret
|
|
930
|
+
? body.secret
|
|
931
|
+
: undefined,
|
|
932
|
+
}
|
|
933
|
+
: undefined,
|
|
934
|
+
});
|
|
935
|
+
const { webhookDelivery, grantDelivery } = await deliverGrant(
|
|
936
|
+
c.env,
|
|
937
|
+
db,
|
|
938
|
+
projectId,
|
|
939
|
+
body,
|
|
940
|
+
proposal,
|
|
941
|
+
);
|
|
705
942
|
return c.json(
|
|
706
943
|
{
|
|
707
944
|
proposal,
|
|
708
945
|
...proposal,
|
|
709
946
|
grantToken: proposal.grantToken ?? null,
|
|
947
|
+
grantDelivery,
|
|
948
|
+
...(webhookDelivery ? { webhookDelivery } : {}),
|
|
710
949
|
},
|
|
711
950
|
201,
|
|
712
951
|
);
|
|
@@ -721,7 +960,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
721
960
|
});
|
|
722
961
|
|
|
723
962
|
app.post("/internal/remediations/:id/approve", async (c) => {
|
|
724
|
-
const sessionCheck = requireHumanDashboardSession(c);
|
|
963
|
+
const sessionCheck = await requireHumanDashboardSession(c);
|
|
725
964
|
if (!sessionCheck.ok) {
|
|
726
965
|
return c.json(
|
|
727
966
|
{ error: "Forbidden", message: sessionCheck.message },
|
|
@@ -735,17 +974,17 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
735
974
|
{};
|
|
736
975
|
const environment =
|
|
737
976
|
typeof body.environment === "string" ? body.environment : undefined;
|
|
738
|
-
const approvedByUserId =
|
|
739
|
-
typeof body.approvedByUserId === "string" &&
|
|
740
|
-
body.approvedByUserId.trim()
|
|
741
|
-
? body.approvedByUserId.trim()
|
|
742
|
-
: typeof body.approvedBySession === "string" &&
|
|
743
|
-
body.approvedBySession.trim()
|
|
744
|
-
? body.approvedBySession.trim()
|
|
745
|
-
: sessionCheck.sessionActor;
|
|
977
|
+
const approvedByUserId = sessionCheck.sessionActor;
|
|
746
978
|
|
|
747
979
|
const db = sqlDbFor(c.env);
|
|
748
980
|
try {
|
|
981
|
+
// Read before approving: the approve UPDATE clears the stored callback.
|
|
982
|
+
const storedCallback = await getRemediationCallback(
|
|
983
|
+
db,
|
|
984
|
+
projectId,
|
|
985
|
+
proposalId,
|
|
986
|
+
environment,
|
|
987
|
+
);
|
|
749
988
|
const proposal = await approveRemediationProposal(
|
|
750
989
|
db,
|
|
751
990
|
projectId,
|
|
@@ -755,10 +994,20 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
755
994
|
approvedByUserId,
|
|
756
995
|
},
|
|
757
996
|
);
|
|
997
|
+
const { webhookDelivery, grantDelivery } = await deliverGrant(
|
|
998
|
+
c.env,
|
|
999
|
+
db,
|
|
1000
|
+
projectId,
|
|
1001
|
+
body,
|
|
1002
|
+
proposal,
|
|
1003
|
+
storedCallback,
|
|
1004
|
+
);
|
|
758
1005
|
return c.json({
|
|
759
1006
|
proposal,
|
|
760
1007
|
...proposal,
|
|
761
1008
|
grantToken: proposal.grantToken ?? null,
|
|
1009
|
+
grantDelivery,
|
|
1010
|
+
...(webhookDelivery ? { webhookDelivery } : {}),
|
|
762
1011
|
});
|
|
763
1012
|
} catch (err) {
|
|
764
1013
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -771,7 +1020,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
771
1020
|
});
|
|
772
1021
|
|
|
773
1022
|
app.post("/internal/remediations/:id/reject", async (c) => {
|
|
774
|
-
const sessionCheck = requireHumanDashboardSession(c);
|
|
1023
|
+
const sessionCheck = await requireHumanDashboardSession(c);
|
|
775
1024
|
if (!sessionCheck.ok) {
|
|
776
1025
|
return c.json(
|
|
777
1026
|
{ error: "Forbidden", message: sessionCheck.message },
|
|
@@ -785,8 +1034,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
785
1034
|
{};
|
|
786
1035
|
const environment =
|
|
787
1036
|
typeof body.environment === "string" ? body.environment : undefined;
|
|
788
|
-
const reason =
|
|
789
|
-
typeof body.reason === "string" ? body.reason : undefined;
|
|
1037
|
+
const reason = typeof body.reason === "string" ? body.reason : undefined;
|
|
790
1038
|
|
|
791
1039
|
const db = sqlDbFor(c.env);
|
|
792
1040
|
try {
|
|
@@ -837,6 +1085,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
837
1085
|
grantToken,
|
|
838
1086
|
runnerId:
|
|
839
1087
|
typeof body.runnerId === "string" ? body.runnerId : undefined,
|
|
1088
|
+
logger: runtime.logger,
|
|
840
1089
|
},
|
|
841
1090
|
);
|
|
842
1091
|
return c.json({ claimed: true, proposal, ...proposal });
|
|
@@ -846,7 +1095,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
846
1095
|
err instanceof RemediationGateError
|
|
847
1096
|
? (err.statusCode as 400 | 403 | 404 | 409 | 410)
|
|
848
1097
|
: 409;
|
|
849
|
-
return c.json(
|
|
1098
|
+
return c.json(
|
|
1099
|
+
{ claimed: false, error: message, reason: message },
|
|
1100
|
+
status,
|
|
1101
|
+
);
|
|
850
1102
|
}
|
|
851
1103
|
});
|
|
852
1104
|
|
|
@@ -856,6 +1108,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
856
1108
|
const body =
|
|
857
1109
|
(await readJsonObject<Record<string, unknown>>(c).catch(() => null)) ??
|
|
858
1110
|
{};
|
|
1111
|
+
const runnerId =
|
|
1112
|
+
typeof body.runnerId === "string" && body.runnerId.trim()
|
|
1113
|
+
? body.runnerId.trim()
|
|
1114
|
+
: "autosre-runner";
|
|
859
1115
|
const db = sqlDbFor(c.env);
|
|
860
1116
|
try {
|
|
861
1117
|
const proposal = await completeRemediationProposal(
|
|
@@ -867,6 +1123,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
867
1123
|
typeof body.environment === "string"
|
|
868
1124
|
? body.environment
|
|
869
1125
|
: undefined,
|
|
1126
|
+
runnerId,
|
|
870
1127
|
resultingChangeId:
|
|
871
1128
|
typeof body.resultingChangeId === "string"
|
|
872
1129
|
? body.resultingChangeId
|
|
@@ -894,6 +1151,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
894
1151
|
const body =
|
|
895
1152
|
(await readJsonObject<Record<string, unknown>>(c).catch(() => null)) ??
|
|
896
1153
|
{};
|
|
1154
|
+
const runnerId =
|
|
1155
|
+
typeof body.runnerId === "string" && body.runnerId.trim()
|
|
1156
|
+
? body.runnerId.trim()
|
|
1157
|
+
: "autosre-runner";
|
|
897
1158
|
const db = sqlDbFor(c.env);
|
|
898
1159
|
try {
|
|
899
1160
|
const proposal = await failRemediationProposal(
|
|
@@ -905,6 +1166,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
905
1166
|
typeof body.environment === "string"
|
|
906
1167
|
? body.environment
|
|
907
1168
|
: undefined,
|
|
1169
|
+
runnerId,
|
|
908
1170
|
failureReason:
|
|
909
1171
|
typeof body.failureReason === "string"
|
|
910
1172
|
? body.failureReason
|
|
@@ -916,7 +1178,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
916
1178
|
const message = err instanceof Error ? err.message : String(err);
|
|
917
1179
|
const status =
|
|
918
1180
|
err instanceof RemediationGateError
|
|
919
|
-
? (err.statusCode as 400 | 404)
|
|
1181
|
+
? (err.statusCode as 400 | 404 | 409)
|
|
920
1182
|
: 400;
|
|
921
1183
|
return c.json({ error: message }, status);
|
|
922
1184
|
}
|
|
@@ -927,12 +1189,16 @@ export const autosreRoutesPlugin: CollectorPlugin = {
|
|
|
927
1189
|
const body =
|
|
928
1190
|
(await readJsonObject<Record<string, unknown>>(c).catch(() => null)) ??
|
|
929
1191
|
{};
|
|
1192
|
+
// Scoped to the caller's project; the clock is never client-controlled
|
|
1193
|
+
// and the traffic floor can only be raised, never lowered, by a caller.
|
|
930
1194
|
const summary = await evaluateRemediationWatchdogs(db, {
|
|
931
|
-
|
|
1195
|
+
projectId: getProjectId(c),
|
|
932
1196
|
minComparableRequests:
|
|
933
|
-
typeof body.minComparableRequests === "number"
|
|
934
|
-
|
|
1197
|
+
typeof body.minComparableRequests === "number" &&
|
|
1198
|
+
Number.isFinite(body.minComparableRequests)
|
|
1199
|
+
? Math.max(50, body.minComparableRequests)
|
|
935
1200
|
: undefined,
|
|
1201
|
+
logger: runtime.logger,
|
|
936
1202
|
});
|
|
937
1203
|
return c.json({ summary, ...summary });
|
|
938
1204
|
});
|