@obsunified/collector 2.2.1 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.js +1 -1
- package/dist/{chunk-KMEZKGSO.js → chunk-2FALTDCE.js} +12 -4
- package/dist/chunk-2FALTDCE.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +497 -96
- package/dist/index.js.map +1 -1
- package/package.json +2 -3
- package/src/__tests__/autosre-closed-loop.test.ts +590 -0
- package/src/auth/dashboard-auth.test.ts +61 -0
- package/src/auth/dashboard-auth.ts +17 -2
- package/src/lib/ai-store/calls.ts +43 -23
- package/src/lib/ai-store.ts +12 -1
- package/src/lib/autosre-findings.ts +169 -14
- package/src/lib/autosre-remediation-gate.ts +156 -26
- package/src/lib/projects-store.test.ts +38 -1
- package/src/lib/projects-store.ts +23 -4
- package/src/lib/sql-db-postgres.integration.test.ts +75 -2
- package/src/lib/usage-store.ts +9 -3
- package/src/migrations/052_remediation_callback.sql +10 -0
- package/src/migrations/053_remediation_callback_delivery.sql +7 -0
- package/src/migrations-postgres/052_remediation_callback.sql +10 -0
- package/src/migrations-postgres/053_remediation_callback_delivery.sql +7 -0
- package/src/plugins/ai-call-by-id.test.ts +73 -0
- package/src/plugins/ai-receiver.ts +10 -0
- package/src/plugins/autosre-routes.ts +167 -16
- package/src/plugins/connected-routes/deep-link-shapes.test.ts +44 -0
- package/src/plugins/connected-routes/manifest.test.ts +142 -0
- package/src/plugins/connected-routes/manifest.ts +127 -7
- package/src/plugins/connected-routes-links.test.ts +96 -0
- package/src/plugins/connected-routes.test.ts +3 -3
- package/src/plugins/connected-routes.ts +11 -8
- package/src/plugins/evidence-retrieval-routes.ts +30 -0
- package/dist/chunk-KMEZKGSO.js.map +0 -1
|
@@ -293,6 +293,21 @@ export function createDashboardAuth(config?: DashboardAuthConfig): {
|
|
|
293
293
|
: new LoginRateLimiter(config?.loginRateLimit ?? {});
|
|
294
294
|
let warnedUnkeyedClient = false;
|
|
295
295
|
|
|
296
|
+
// `Secure` when the request itself is https, or when the operator
|
|
297
|
+
// declared a trusted TLS-terminating proxy (`trustProxyHeaders`, i.e.
|
|
298
|
+
// TRUST_PROXY) that reports `X-Forwarded-Proto: https`.
|
|
299
|
+
const isSecureRequest = (
|
|
300
|
+
c: Context<{ Bindings: CollectorEnv }>,
|
|
301
|
+
limitConfig: { trustProxyHeaders?: boolean } | undefined,
|
|
302
|
+
): boolean =>
|
|
303
|
+
c.req.url.startsWith("https:") ||
|
|
304
|
+
(limitConfig?.trustProxyHeaders === true &&
|
|
305
|
+
c.req
|
|
306
|
+
.header("x-forwarded-proto")
|
|
307
|
+
?.split(",")[0]
|
|
308
|
+
?.trim()
|
|
309
|
+
.toLowerCase() === "https");
|
|
310
|
+
|
|
296
311
|
app.post("/auth/login", async (c) => {
|
|
297
312
|
const configuredPassword = config?.password ?? c.env.DASHBOARD_PASSWORD;
|
|
298
313
|
if (!configuredPassword) {
|
|
@@ -342,7 +357,7 @@ export function createDashboardAuth(config?: DashboardAuthConfig): {
|
|
|
342
357
|
const sig = await sign(payload, configuredPassword);
|
|
343
358
|
const cookieValue = `${payload}.${sig}`;
|
|
344
359
|
|
|
345
|
-
const isSecure = c
|
|
360
|
+
const isSecure = isSecureRequest(c, config?.loginRateLimit || undefined);
|
|
346
361
|
c.header(
|
|
347
362
|
"Set-Cookie",
|
|
348
363
|
`${SESSION_COOKIE}=${encodeURIComponent(cookieValue)}; Path=/; HttpOnly; SameSite=Strict; Max-Age=${SESSION_MAX_AGE}${isSecure ? "; Secure" : ""}`,
|
|
@@ -405,7 +420,7 @@ export function createDashboardAuth(config?: DashboardAuthConfig): {
|
|
|
405
420
|
});
|
|
406
421
|
|
|
407
422
|
app.post("/auth/logout", (c) => {
|
|
408
|
-
const isSecure = c
|
|
423
|
+
const isSecure = isSecureRequest(c, config?.loginRateLimit || undefined);
|
|
409
424
|
c.header(
|
|
410
425
|
"Set-Cookie",
|
|
411
426
|
`${SESSION_COOKIE}=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0${isSecure ? "; Secure" : ""}`,
|
|
@@ -53,6 +53,48 @@ export async function ingestAICallBatch(
|
|
|
53
53
|
await db.batch(batch);
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
function mapAICallRow(row: AICallRow): AICallRecord {
|
|
57
|
+
return {
|
|
58
|
+
projectId: row.project_id ?? "default",
|
|
59
|
+
callId: row.call_id,
|
|
60
|
+
traceId: row.trace_id,
|
|
61
|
+
spanId: row.span_id,
|
|
62
|
+
serviceName: row.service_name,
|
|
63
|
+
modelName: row.model_name,
|
|
64
|
+
provider: row.provider,
|
|
65
|
+
callType: row.call_type,
|
|
66
|
+
requestJson: row.request_json,
|
|
67
|
+
responseJson: row.response_json,
|
|
68
|
+
promptTokens: row.prompt_tokens,
|
|
69
|
+
completionTokens: row.completion_tokens,
|
|
70
|
+
totalCostUsd: row.total_cost_usd,
|
|
71
|
+
latencyMs: row.latency_ms,
|
|
72
|
+
isError: row.is_error === 1 || row.is_error === true,
|
|
73
|
+
errorMessage: row.error_message,
|
|
74
|
+
occurredAt: row.occurred_at,
|
|
75
|
+
receivedAt: row.received_at,
|
|
76
|
+
expiresAt: row.expires_at,
|
|
77
|
+
sessionId: row.session_id ?? null,
|
|
78
|
+
interactionId: row.interaction_id ?? null,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** One call by id, scoped to the project; `null` when absent. */
|
|
83
|
+
export async function getAICallById(
|
|
84
|
+
db: SqlDb,
|
|
85
|
+
projectId: string,
|
|
86
|
+
callId: string,
|
|
87
|
+
): Promise<AICallRecord | null> {
|
|
88
|
+
if (!projectId) throw new Error("AIStore.getAICall: projectId is required");
|
|
89
|
+
const row = await db
|
|
90
|
+
.prepare(
|
|
91
|
+
`SELECT * FROM ai_calls WHERE project_id = ? AND call_id = ? LIMIT 1`,
|
|
92
|
+
)
|
|
93
|
+
.bind(projectId, callId)
|
|
94
|
+
.first<AICallRow>();
|
|
95
|
+
return row ? mapAICallRow(row) : null;
|
|
96
|
+
}
|
|
97
|
+
|
|
56
98
|
export async function getAICallsOverview(
|
|
57
99
|
db: SqlDb,
|
|
58
100
|
options: AICallsOverviewOptions,
|
|
@@ -109,29 +151,7 @@ export async function getAICallsOverview(
|
|
|
109
151
|
.bind(...summaryParams)
|
|
110
152
|
.first<AICallSummaryRow>()) || {};
|
|
111
153
|
|
|
112
|
-
const calls: AICallRecord[] = (results.results || []).map(
|
|
113
|
-
projectId: row.project_id ?? "default",
|
|
114
|
-
callId: row.call_id,
|
|
115
|
-
traceId: row.trace_id,
|
|
116
|
-
spanId: row.span_id,
|
|
117
|
-
serviceName: row.service_name,
|
|
118
|
-
modelName: row.model_name,
|
|
119
|
-
provider: row.provider,
|
|
120
|
-
callType: row.call_type,
|
|
121
|
-
requestJson: row.request_json,
|
|
122
|
-
responseJson: row.response_json,
|
|
123
|
-
promptTokens: row.prompt_tokens,
|
|
124
|
-
completionTokens: row.completion_tokens,
|
|
125
|
-
totalCostUsd: row.total_cost_usd,
|
|
126
|
-
latencyMs: row.latency_ms,
|
|
127
|
-
isError: row.is_error === 1 || row.is_error === true,
|
|
128
|
-
errorMessage: row.error_message,
|
|
129
|
-
occurredAt: row.occurred_at,
|
|
130
|
-
receivedAt: row.received_at,
|
|
131
|
-
expiresAt: row.expires_at,
|
|
132
|
-
sessionId: row.session_id ?? null,
|
|
133
|
-
interactionId: row.interaction_id ?? null,
|
|
134
|
-
}));
|
|
154
|
+
const calls: AICallRecord[] = (results.results || []).map(mapAICallRow);
|
|
135
155
|
|
|
136
156
|
return {
|
|
137
157
|
calls,
|
package/src/lib/ai-store.ts
CHANGED
|
@@ -10,7 +10,11 @@ import type {
|
|
|
10
10
|
AISpansOverviewOptions,
|
|
11
11
|
AISpansOverviewResponse,
|
|
12
12
|
} from "@obsunified/types";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
getAICallById,
|
|
15
|
+
getAICallsOverview,
|
|
16
|
+
ingestAICallBatch,
|
|
17
|
+
} from "./ai-store/calls";
|
|
14
18
|
import {
|
|
15
19
|
getAIEvaluation,
|
|
16
20
|
ingestAIEvaluations,
|
|
@@ -37,6 +41,13 @@ export class AIStore {
|
|
|
37
41
|
return getAICallsOverview(this.db, options);
|
|
38
42
|
}
|
|
39
43
|
|
|
44
|
+
async getAICall(
|
|
45
|
+
projectId: string,
|
|
46
|
+
callId: string,
|
|
47
|
+
): Promise<AICallRecord | null> {
|
|
48
|
+
return getAICallById(this.db, projectId, callId);
|
|
49
|
+
}
|
|
50
|
+
|
|
40
51
|
async purgeExpired(): Promise<number> {
|
|
41
52
|
return purgeExpiredAIData(this.db);
|
|
42
53
|
}
|
|
@@ -15,10 +15,11 @@ import type {
|
|
|
15
15
|
import {
|
|
16
16
|
buildCanonicalNodeId,
|
|
17
17
|
buildFindingFingerprint,
|
|
18
|
+
OntologyNodeKindSchema,
|
|
18
19
|
} from "@obsunified/types";
|
|
19
20
|
import { consoleLogger, type Logger } from "../framework/logger";
|
|
20
21
|
import { computeChangeBlastRadius } from "./autosre-change-correlation";
|
|
21
|
-
import { randomHex } from "./hash";
|
|
22
|
+
import { randomHex, sha256Hex } from "./hash";
|
|
22
23
|
import { parseJsonArray, parseJsonRecord } from "./json";
|
|
23
24
|
import type { SqlDb } from "./sql-db";
|
|
24
25
|
|
|
@@ -221,6 +222,97 @@ function resolveSafeCausalProofTier(
|
|
|
221
222
|
return "unverified_locus";
|
|
222
223
|
}
|
|
223
224
|
|
|
225
|
+
export interface FaultLocusSpanRow {
|
|
226
|
+
service_name: string | null;
|
|
227
|
+
span_name: string;
|
|
228
|
+
status_message: string | null;
|
|
229
|
+
error_signature?: string | null;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Looks up the span a finding's fault locus refers to (newest match wins). */
|
|
233
|
+
export async function loadFaultLocusSpan(
|
|
234
|
+
db: SqlDb,
|
|
235
|
+
projectId: string,
|
|
236
|
+
environment: string,
|
|
237
|
+
spanId: string,
|
|
238
|
+
traceId?: string,
|
|
239
|
+
): Promise<FaultLocusSpanRow | null> {
|
|
240
|
+
if (!spanId) return null;
|
|
241
|
+
return db
|
|
242
|
+
.prepare(
|
|
243
|
+
`SELECT service_name, span_name, status_message, error_signature
|
|
244
|
+
FROM telemetry_spans
|
|
245
|
+
WHERE project_id = ? AND environment = ? AND span_id = ?
|
|
246
|
+
${traceId ? "AND trace_id = ?" : ""}
|
|
247
|
+
ORDER BY start_time DESC
|
|
248
|
+
LIMIT 1`,
|
|
249
|
+
)
|
|
250
|
+
.bind(
|
|
251
|
+
...(traceId
|
|
252
|
+
? [projectId, environment, spanId, traceId]
|
|
253
|
+
: [projectId, environment, spanId]),
|
|
254
|
+
)
|
|
255
|
+
.first<FaultLocusSpanRow>();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Fingerprint inputs derivable from a span; empty fields are left undefined. */
|
|
259
|
+
export type SpanFingerprintInput = Pick<
|
|
260
|
+
FindingFingerprintInput,
|
|
261
|
+
"environment" | "originatingNodeId" | "downstreamTargetNodeId"
|
|
262
|
+
> &
|
|
263
|
+
Partial<
|
|
264
|
+
Pick<FindingFingerprintInput, "normalizedOperation" | "errorSignature">
|
|
265
|
+
>;
|
|
266
|
+
|
|
267
|
+
/** True for `kind:env:ns:name...` ids built by buildCanonicalNodeId. */
|
|
268
|
+
function looksLikeCanonicalNodeId(id: string): boolean {
|
|
269
|
+
const parts = id.split(":");
|
|
270
|
+
return (
|
|
271
|
+
parts.length >= 4 &&
|
|
272
|
+
(OntologyNodeKindSchema.options as readonly string[]).includes(parts[0])
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Splits a `<traceId>:<spanId>` span locus. Returns null for anything that
|
|
278
|
+
* is a canonical node id or does not have exactly two non-empty parts.
|
|
279
|
+
*/
|
|
280
|
+
function splitTraceSpanLocus(
|
|
281
|
+
id: string,
|
|
282
|
+
): { traceId: string; spanId: string } | null {
|
|
283
|
+
if (looksLikeCanonicalNodeId(id)) return null;
|
|
284
|
+
const parts = id.split(":");
|
|
285
|
+
if (parts.length !== 2 || !parts[0] || !parts[1]) return null;
|
|
286
|
+
return { traceId: parts[0], spanId: parts[1] };
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Stable recurrence-fingerprint inputs for a fault-locus span: the service node,
|
|
291
|
+
* the span name as the operation and the span's error signature (falling back
|
|
292
|
+
* to its status message). No instance ids participate, so two traces failing
|
|
293
|
+
* the same way on the same service/route share a fingerprint. Used by both
|
|
294
|
+
* finding creation and the evidence bundle's prior-finding lookup.
|
|
295
|
+
*/
|
|
296
|
+
export function fingerprintInputFromSpan(
|
|
297
|
+
span: FaultLocusSpanRow,
|
|
298
|
+
environment: string,
|
|
299
|
+
): SpanFingerprintInput {
|
|
300
|
+
return {
|
|
301
|
+
environment,
|
|
302
|
+
originatingNodeId: buildCanonicalNodeId({
|
|
303
|
+
kind: "service",
|
|
304
|
+
environment,
|
|
305
|
+
serviceName: span.service_name || "unknown-service",
|
|
306
|
+
}).id,
|
|
307
|
+
// Empty strings are "absent": returning undefined lets the caller's
|
|
308
|
+
// `??` chain reach its title/summary fallbacks.
|
|
309
|
+
normalizedOperation: span.span_name?.trim() || undefined,
|
|
310
|
+
errorSignature:
|
|
311
|
+
span.error_signature?.trim() || span.status_message?.trim() || undefined,
|
|
312
|
+
downstreamTargetNodeId: null,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
224
316
|
export async function recordInvestigationFinding(
|
|
225
317
|
db: SqlDb,
|
|
226
318
|
projectId: string,
|
|
@@ -237,23 +329,86 @@ export async function recordInvestigationFinding(
|
|
|
237
329
|
.trim()
|
|
238
330
|
.toLowerCase();
|
|
239
331
|
|
|
240
|
-
|
|
241
|
-
|
|
332
|
+
// Idempotent retries: one key maps to one finding id per project and
|
|
333
|
+
// environment, and a replay returns the stored finding untouched.
|
|
334
|
+
const idempotencyKey = input.idempotencyKey?.trim();
|
|
335
|
+
let idempotencyId: string | undefined;
|
|
336
|
+
if (idempotencyKey) {
|
|
337
|
+
idempotencyId = `fnd_k${(await sha256Hex(`${projectId}\u0000${environment}\u0000${idempotencyKey}`)).slice(0, 20)}`;
|
|
338
|
+
const replay = await getInvestigationFinding(
|
|
339
|
+
db,
|
|
340
|
+
projectId,
|
|
341
|
+
idempotencyId,
|
|
342
|
+
environment,
|
|
343
|
+
);
|
|
344
|
+
if (replay) return replay;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Findings pointing at a span id (no colon) must not mint a per-incident
|
|
348
|
+
// fingerprint. Resolve the span to its service/operation/error signature so
|
|
349
|
+
// every incident on the same service + route + error shares one identity.
|
|
350
|
+
// Caller-supplied fingerprint fields always win; unknown spans fall back to
|
|
351
|
+
// treating the locus id as a service name.
|
|
352
|
+
const supplied: Partial<FindingFingerprintInput> =
|
|
353
|
+
input.fingerprintInput ?? {};
|
|
354
|
+
const locusId = input.faultLocusEntityId.trim();
|
|
355
|
+
let spanDerived: Partial<FindingFingerprintInput> = {};
|
|
356
|
+
const traceSpan =
|
|
357
|
+
input.faultLocusEntityKind === "span" ? splitTraceSpanLocus(locusId) : null;
|
|
358
|
+
if (traceSpan) {
|
|
359
|
+
const spanRow = await loadFaultLocusSpan(
|
|
360
|
+
db,
|
|
361
|
+
projectId,
|
|
362
|
+
environment,
|
|
363
|
+
traceSpan.spanId,
|
|
364
|
+
traceSpan.traceId,
|
|
365
|
+
);
|
|
366
|
+
if (spanRow) spanDerived = fingerprintInputFromSpan(spanRow, environment);
|
|
367
|
+
} else if (
|
|
368
|
+
!locusId.includes(":") &&
|
|
369
|
+
input.faultLocusEntityKind !== "service"
|
|
370
|
+
) {
|
|
371
|
+
const spanRow = await loadFaultLocusSpan(
|
|
372
|
+
db,
|
|
373
|
+
projectId,
|
|
374
|
+
environment,
|
|
375
|
+
locusId,
|
|
376
|
+
);
|
|
377
|
+
if (spanRow) spanDerived = fingerprintInputFromSpan(spanRow, environment);
|
|
378
|
+
}
|
|
379
|
+
const defaultOriginNode = locusId.includes(":")
|
|
380
|
+
? traceSpan
|
|
381
|
+
? buildCanonicalNodeId({
|
|
382
|
+
kind: "service",
|
|
383
|
+
environment,
|
|
384
|
+
serviceName: locusId,
|
|
385
|
+
}).id
|
|
386
|
+
: locusId
|
|
242
387
|
: buildCanonicalNodeId({
|
|
243
388
|
kind: "service",
|
|
244
389
|
environment,
|
|
245
|
-
serviceName:
|
|
390
|
+
serviceName: locusId,
|
|
246
391
|
}).id;
|
|
247
392
|
|
|
248
|
-
const fpResult = buildFindingFingerprint(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
originatingNodeId
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
393
|
+
const fpResult = buildFindingFingerprint({
|
|
394
|
+
environment: supplied.environment ?? environment,
|
|
395
|
+
originatingNodeId:
|
|
396
|
+
supplied.originatingNodeId ??
|
|
397
|
+
spanDerived.originatingNodeId ??
|
|
398
|
+
defaultOriginNode,
|
|
399
|
+
normalizedOperation:
|
|
400
|
+
supplied.normalizedOperation ??
|
|
401
|
+
spanDerived.normalizedOperation ??
|
|
402
|
+
input.title,
|
|
403
|
+
errorSignature:
|
|
404
|
+
supplied.errorSignature ??
|
|
405
|
+
spanDerived.errorSignature ??
|
|
406
|
+
input.summary.slice(0, 120),
|
|
407
|
+
downstreamTargetNodeId:
|
|
408
|
+
supplied.downstreamTargetNodeId ??
|
|
409
|
+
spanDerived.downstreamTargetNodeId ??
|
|
410
|
+
null,
|
|
411
|
+
});
|
|
257
412
|
|
|
258
413
|
// Check for prior verified findings with the same stable fingerprint
|
|
259
414
|
const priorRow = await db
|
|
@@ -346,7 +501,7 @@ export async function recordInvestigationFinding(
|
|
|
346
501
|
finalConfidence: confidence,
|
|
347
502
|
};
|
|
348
503
|
|
|
349
|
-
const id = input.id?.trim() || `fnd_${randomHex(10)}
|
|
504
|
+
const id = idempotencyId ?? (input.id?.trim() || `fnd_${randomHex(10)}`);
|
|
350
505
|
const expiresAt = new Date(Date.now() + 90 * 24 * 3600_000).toISOString();
|
|
351
506
|
|
|
352
507
|
await db
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
CausalTriple,
|
|
3
3
|
EvidenceEntityKind,
|
|
4
|
+
GrantCallbackDelivery,
|
|
4
5
|
IncidentInvestigation,
|
|
5
6
|
IncidentStatus,
|
|
6
7
|
RemediationActionKind,
|
|
@@ -538,8 +539,15 @@ export async function dispatchIncidentWebhook(
|
|
|
538
539
|
fetchImpl?: typeof fetch;
|
|
539
540
|
logger?: Logger;
|
|
540
541
|
grant?: { proposalId: string; status: string; grantToken: string };
|
|
542
|
+
/** Test hook: replaces the real backoff sleep between grant attempts. */
|
|
543
|
+
sleep?: (ms: number) => Promise<void>;
|
|
541
544
|
},
|
|
542
|
-
): Promise<{
|
|
545
|
+
): Promise<{
|
|
546
|
+
delivered: boolean;
|
|
547
|
+
status?: number;
|
|
548
|
+
error?: string;
|
|
549
|
+
attempts: number;
|
|
550
|
+
}> {
|
|
543
551
|
const logger = webhook.logger ?? consoleLogger;
|
|
544
552
|
const incident = await getIncidentInvestigation(
|
|
545
553
|
db,
|
|
@@ -548,10 +556,14 @@ export async function dispatchIncidentWebhook(
|
|
|
548
556
|
webhook.environment,
|
|
549
557
|
);
|
|
550
558
|
if (!incident) {
|
|
551
|
-
return { delivered: false, error: "Incident not found" };
|
|
559
|
+
return { delivered: false, error: "Incident not found", attempts: 0 };
|
|
552
560
|
}
|
|
553
561
|
if (!webhook.grant && incident.dispatchAttempts >= 5) {
|
|
554
|
-
return {
|
|
562
|
+
return {
|
|
563
|
+
delivered: false,
|
|
564
|
+
error: "MAX_DISPATCH_ATTEMPTS (5) exceeded",
|
|
565
|
+
attempts: 0,
|
|
566
|
+
};
|
|
555
567
|
}
|
|
556
568
|
|
|
557
569
|
let safeUrl: URL;
|
|
@@ -581,7 +593,7 @@ export async function dispatchIncidentWebhook(
|
|
|
581
593
|
incidentId,
|
|
582
594
|
error,
|
|
583
595
|
});
|
|
584
|
-
return { delivered: false, error };
|
|
596
|
+
return { delivered: false, error, attempts: 0 };
|
|
585
597
|
}
|
|
586
598
|
|
|
587
599
|
const nowIso = new Date().toISOString();
|
|
@@ -620,40 +632,59 @@ export async function dispatchIncidentWebhook(
|
|
|
620
632
|
}
|
|
621
633
|
|
|
622
634
|
const fetchFn = webhook.fetchImpl ?? fetch;
|
|
635
|
+
const sleep =
|
|
636
|
+
webhook.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
|
|
637
|
+
// Grant handoffs retry on network error, timeout and 5xx (never 4xx or a
|
|
638
|
+
// redirect); each attempt keeps the manual-redirect and 5 s timeout rules.
|
|
639
|
+
const maxAttempts = webhook.grant ? GRANT_DELIVERY_BACKOFF_MS.length + 1 : 1;
|
|
623
640
|
let delivered = false;
|
|
624
641
|
let status: number | undefined;
|
|
625
642
|
let errorMsg: string | undefined;
|
|
643
|
+
let attempts = 0;
|
|
626
644
|
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
645
|
+
while (attempts < maxAttempts) {
|
|
646
|
+
attempts++;
|
|
647
|
+
let retryable = false;
|
|
648
|
+
status = undefined;
|
|
649
|
+
errorMsg = undefined;
|
|
650
|
+
try {
|
|
651
|
+
const res = await fetchFn(safeUrl.toString(), {
|
|
652
|
+
method: "POST",
|
|
653
|
+
headers,
|
|
654
|
+
body,
|
|
655
|
+
// Never follow redirects: a 3xx could re-aim the POST (and its
|
|
656
|
+
// signed body, which may carry a grant token) at another host.
|
|
657
|
+
redirect: "manual",
|
|
658
|
+
signal: AbortSignal.timeout(5000),
|
|
659
|
+
});
|
|
660
|
+
status = res.status;
|
|
661
|
+
if (res.status >= 300 && res.status < 400) {
|
|
662
|
+
errorMsg = `Redirect (HTTP ${res.status}) refused`;
|
|
663
|
+
} else {
|
|
664
|
+
delivered = res.ok;
|
|
665
|
+
if (!res.ok) {
|
|
666
|
+
errorMsg = `HTTP ${res.status}`;
|
|
667
|
+
retryable = res.status >= 500;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
} catch (err) {
|
|
671
|
+
errorMsg = err instanceof Error ? err.message : String(err);
|
|
672
|
+
retryable = true;
|
|
643
673
|
}
|
|
644
|
-
|
|
645
|
-
|
|
674
|
+
if (delivered || !retryable || attempts >= maxAttempts) break;
|
|
675
|
+
await sleep(GRANT_DELIVERY_BACKOFF_MS[attempts - 1]);
|
|
646
676
|
}
|
|
647
677
|
|
|
648
678
|
if (!delivered) {
|
|
649
679
|
logger.warn("[autosre] incident webhook delivery failed", {
|
|
650
680
|
incidentId,
|
|
651
681
|
error: errorMsg,
|
|
682
|
+
attempts,
|
|
652
683
|
});
|
|
653
684
|
}
|
|
654
685
|
|
|
655
686
|
if (webhook.grant) {
|
|
656
|
-
return { delivered, status, error: errorMsg };
|
|
687
|
+
return { delivered, status, error: errorMsg, attempts };
|
|
657
688
|
}
|
|
658
689
|
|
|
659
690
|
await db
|
|
@@ -675,7 +706,37 @@ export async function dispatchIncidentWebhook(
|
|
|
675
706
|
)
|
|
676
707
|
.run();
|
|
677
708
|
|
|
678
|
-
return { delivered, status, error: errorMsg };
|
|
709
|
+
return { delivered, status, error: errorMsg, attempts };
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/** Backoff before attempt 2 and 3 of a grant webhook delivery. */
|
|
713
|
+
const GRANT_DELIVERY_BACKOFF_MS = [500, 1500];
|
|
714
|
+
|
|
715
|
+
/**
|
|
716
|
+
* Persists the last grant-webhook delivery outcome on the proposal. The JSON
|
|
717
|
+
* holds only status/attempts/error/time, never the URL or secret.
|
|
718
|
+
*/
|
|
719
|
+
export async function recordCallbackDelivery(
|
|
720
|
+
db: SqlDb,
|
|
721
|
+
projectId: string,
|
|
722
|
+
environment: string,
|
|
723
|
+
proposalId: string,
|
|
724
|
+
outcome: { delivered: boolean; attempts: number; error?: string },
|
|
725
|
+
): Promise<GrantCallbackDelivery> {
|
|
726
|
+
const record: GrantCallbackDelivery = {
|
|
727
|
+
status: outcome.delivered ? "delivered" : "failed",
|
|
728
|
+
attempts: outcome.attempts,
|
|
729
|
+
lastError: outcome.delivered ? null : (outcome.error ?? null),
|
|
730
|
+
at: new Date().toISOString(),
|
|
731
|
+
};
|
|
732
|
+
await db
|
|
733
|
+
.prepare(
|
|
734
|
+
`UPDATE remediation_proposals SET callback_delivery = ?
|
|
735
|
+
WHERE project_id = ? AND environment = ? AND id = ?`,
|
|
736
|
+
)
|
|
737
|
+
.bind(JSON.stringify(record), projectId, environment, proposalId)
|
|
738
|
+
.run();
|
|
739
|
+
return record;
|
|
679
740
|
}
|
|
680
741
|
|
|
681
742
|
interface ProposalRow {
|
|
@@ -703,11 +764,42 @@ interface ProposalRow {
|
|
|
703
764
|
remediation_agent_run_id: string | null;
|
|
704
765
|
resulting_change_id: string | null;
|
|
705
766
|
failure_reason: string | null;
|
|
767
|
+
callback_url?: string | null;
|
|
768
|
+
callback_secret?: string | null;
|
|
769
|
+
callback_delivery?: string | null;
|
|
706
770
|
created_at: string;
|
|
707
771
|
updated_at: string;
|
|
708
772
|
expires_at: string | null;
|
|
709
773
|
}
|
|
710
774
|
|
|
775
|
+
/** Host (never path, query or credentials) of a stored grant callback URL. */
|
|
776
|
+
function callbackHostOf(url: string | null | undefined): string | null {
|
|
777
|
+
if (!url) return null;
|
|
778
|
+
try {
|
|
779
|
+
return new URL(url).host || null;
|
|
780
|
+
} catch {
|
|
781
|
+
return null;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
function parseCallbackDelivery(
|
|
786
|
+
raw: string | null | undefined,
|
|
787
|
+
): GrantCallbackDelivery | null {
|
|
788
|
+
if (!raw) return null;
|
|
789
|
+
try {
|
|
790
|
+
const v = JSON.parse(raw) as Partial<GrantCallbackDelivery>;
|
|
791
|
+
if (v.status !== "delivered" && v.status !== "failed") return null;
|
|
792
|
+
return {
|
|
793
|
+
status: v.status,
|
|
794
|
+
attempts: Number(v.attempts ?? 0),
|
|
795
|
+
lastError: typeof v.lastError === "string" ? v.lastError : null,
|
|
796
|
+
at: String(v.at ?? ""),
|
|
797
|
+
};
|
|
798
|
+
} catch {
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
|
|
711
803
|
export const mapProposalRow = (
|
|
712
804
|
row: ProposalRow,
|
|
713
805
|
grantToken?: string,
|
|
@@ -739,6 +831,8 @@ export const mapProposalRow = (
|
|
|
739
831
|
remediationAgentRunId: row.remediation_agent_run_id,
|
|
740
832
|
resultingChangeId: row.resulting_change_id,
|
|
741
833
|
failureReason: row.failure_reason,
|
|
834
|
+
callbackHost: callbackHostOf(row.callback_url),
|
|
835
|
+
callbackDelivery: parseCallbackDelivery(row.callback_delivery),
|
|
742
836
|
createdAt: row.created_at,
|
|
743
837
|
updatedAt: row.updated_at,
|
|
744
838
|
expiresAt: row.expires_at,
|
|
@@ -847,7 +941,11 @@ export async function proposeRemediationAction(
|
|
|
847
941
|
db: SqlDb,
|
|
848
942
|
projectId: string,
|
|
849
943
|
input: RemediationProposalInput,
|
|
850
|
-
options: {
|
|
944
|
+
options: {
|
|
945
|
+
logger?: Logger;
|
|
946
|
+
/** Persisted on a pending proposal and used when it is approved later. */
|
|
947
|
+
callback?: { url: string; secret?: string };
|
|
948
|
+
} = {},
|
|
851
949
|
): Promise<RemediationProposal> {
|
|
852
950
|
const logger = options.logger ?? consoleLogger;
|
|
853
951
|
const nowIso = new Date().toISOString();
|
|
@@ -977,6 +1075,12 @@ export async function proposeRemediationAction(
|
|
|
977
1075
|
status = "rejected";
|
|
978
1076
|
}
|
|
979
1077
|
|
|
1078
|
+
// Only a proposal that still awaits a human can use a stored callback; an
|
|
1079
|
+
// autonomous grant is delivered inline by the propose request itself.
|
|
1080
|
+
const storedCallback =
|
|
1081
|
+
status === "pending_human_approval" && options.callback?.url
|
|
1082
|
+
? options.callback
|
|
1083
|
+
: null;
|
|
980
1084
|
const id = `rem_${randomHex(10)}`;
|
|
981
1085
|
const expiresAt = new Date(Date.now() + 30 * 24 * 3600_000).toISOString();
|
|
982
1086
|
|
|
@@ -997,8 +1101,9 @@ export async function proposeRemediationAction(
|
|
|
997
1101
|
expected_current_state_json, desired_target_state_json,
|
|
998
1102
|
requested_autonomy, effective_autonomy, policy_decision_reason,
|
|
999
1103
|
status, grant_token_hash, grant_expires_at,
|
|
1104
|
+
callback_url, callback_secret,
|
|
1000
1105
|
created_at, updated_at, expires_at
|
|
1001
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1106
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1002
1107
|
ON CONFLICT DO NOTHING`,
|
|
1003
1108
|
)
|
|
1004
1109
|
.bind(
|
|
@@ -1018,6 +1123,8 @@ export async function proposeRemediationAction(
|
|
|
1018
1123
|
status,
|
|
1019
1124
|
grantTokenHash,
|
|
1020
1125
|
grantExpiresAt,
|
|
1126
|
+
storedCallback?.url ?? null,
|
|
1127
|
+
storedCallback?.secret ?? null,
|
|
1021
1128
|
nowIso,
|
|
1022
1129
|
nowIso,
|
|
1023
1130
|
expiresAt,
|
|
@@ -1123,6 +1230,25 @@ export async function listRemediationProposals(
|
|
|
1123
1230
|
return rows.results.map((r) => mapProposalRow(r));
|
|
1124
1231
|
}
|
|
1125
1232
|
|
|
1233
|
+
/**
|
|
1234
|
+
* Reads the grant callback persisted at propose time. Server-side use only:
|
|
1235
|
+
* the secret is never mapped into API responses. Call before
|
|
1236
|
+
* `approveRemediationProposal`, which clears the stored callback.
|
|
1237
|
+
*/
|
|
1238
|
+
export async function getRemediationCallback(
|
|
1239
|
+
db: SqlDb,
|
|
1240
|
+
projectId: string,
|
|
1241
|
+
proposalId: string,
|
|
1242
|
+
environment?: string,
|
|
1243
|
+
): Promise<{ url: string; secret?: string } | null> {
|
|
1244
|
+
const row = await loadProposalRow(db, projectId, proposalId, environment);
|
|
1245
|
+
if (!row?.callback_url) return null;
|
|
1246
|
+
return {
|
|
1247
|
+
url: row.callback_url,
|
|
1248
|
+
...(row.callback_secret ? { secret: row.callback_secret } : {}),
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1126
1252
|
/**
|
|
1127
1253
|
* Human dashboard session approval for `pending_human_approval` proposals
|
|
1128
1254
|
* (RFC 0013 § 6.B & P1 #4). Mints a single-use `grantToken`.
|
|
@@ -1169,6 +1295,8 @@ export async function approveRemediationProposal(
|
|
|
1169
1295
|
grant_expires_at = ?,
|
|
1170
1296
|
approved_by_user_id = ?,
|
|
1171
1297
|
approved_at = ?,
|
|
1298
|
+
callback_url = NULL,
|
|
1299
|
+
callback_secret = NULL,
|
|
1172
1300
|
updated_at = ?
|
|
1173
1301
|
WHERE project_id = ? AND environment = ? AND id = ?
|
|
1174
1302
|
AND status = 'pending_human_approval'`,
|
|
@@ -1241,6 +1369,8 @@ export async function rejectRemediationProposal(
|
|
|
1241
1369
|
`UPDATE remediation_proposals
|
|
1242
1370
|
SET status = 'rejected',
|
|
1243
1371
|
grant_token_hash = NULL,
|
|
1372
|
+
callback_url = NULL,
|
|
1373
|
+
callback_secret = NULL,
|
|
1244
1374
|
failure_reason = ?,
|
|
1245
1375
|
updated_at = ?
|
|
1246
1376
|
WHERE project_id = ? AND environment = ? AND id = ?
|