@openlup/platform 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -138
- package/README.md +80 -3
- package/dist/composition/nodeHost.d.ts +1 -1
- package/dist/composition/nodeHost.js +3 -2
- package/dist/lifecycle/appUpgrade.d.ts +52 -0
- package/dist/lifecycle/appUpgrade.js +180 -0
- package/dist/lifecycle/index.d.ts +2 -0
- package/dist/lifecycle/index.js +2 -0
- package/dist/lifecycle/postgresInstallQualification.d.ts +51 -0
- package/dist/lifecycle/postgresInstallQualification.js +74 -0
- package/dist/migrations/postgresRunner.d.ts +9 -0
- package/dist/migrations/postgresRunner.js +17 -10
- package/dist/operations/adminAlertsSchemas.d.ts +153 -0
- package/dist/operations/adminAlertsSchemas.js +69 -0
- package/dist/operations/adminAlertsView.d.ts +73 -0
- package/dist/operations/adminAlertsView.js +184 -0
- package/dist/operations/alertPagingPolicy.d.ts +15 -0
- package/dist/operations/alertPagingPolicy.js +43 -0
- package/dist/operations/alertThrottlePolicy.d.ts +3 -0
- package/dist/operations/alertThrottlePolicy.js +18 -0
- package/dist/operations/clients.d.ts +17 -0
- package/dist/operations/clients.js +32 -0
- package/dist/operations/contracts/admin.d.ts +93 -0
- package/dist/operations/contracts/admin.js +1 -0
- package/dist/operations/contracts/adminAlertsContracts.d.ts +40 -0
- package/dist/operations/contracts/adminAlertsContracts.js +1 -0
- package/dist/operations/contracts/alertHumanContext.d.ts +7 -0
- package/dist/operations/contracts/alertHumanContext.js +1 -0
- package/dist/operations/contracts/index.d.ts +6 -0
- package/dist/operations/contracts/index.js +6 -0
- package/dist/operations/contracts/observabilityContracts.d.ts +41 -0
- package/dist/operations/contracts/observabilityContracts.js +1 -0
- package/dist/operations/contracts/observabilityPorts.d.ts +18 -0
- package/dist/operations/contracts/observabilityPorts.js +1 -0
- package/dist/operations/contracts/ports.d.ts +19 -0
- package/dist/operations/contracts/ports.js +1 -0
- package/dist/operations/errors.d.ts +6 -0
- package/dist/operations/errors.js +6 -0
- package/dist/operations/index.d.ts +9 -0
- package/dist/operations/index.js +9 -0
- package/dist/operations/schemas.d.ts +138 -0
- package/dist/operations/schemas.js +104 -0
- package/dist/operations/server/adminAlertsHandlers.d.ts +16 -0
- package/dist/operations/server/adminAlertsHandlers.js +35 -0
- package/dist/operations/server/adminPipelineHandlers.d.ts +12 -0
- package/dist/operations/server/adminPipelineHandlers.js +48 -0
- package/dist/operations/server/adminSettingsHandlers.d.ts +14 -0
- package/dist/operations/server/adminSettingsHandlers.js +82 -0
- package/dist/operations/server/controlPlaneHandler.d.ts +7 -0
- package/dist/operations/server/controlPlaneHandler.js +54 -0
- package/dist/operations/server/controlPlaneStore.d.ts +12 -0
- package/dist/operations/server/controlPlaneStore.js +138 -0
- package/dist/operations/server/controlPlaneWatchdog.d.ts +7 -0
- package/dist/operations/server/controlPlaneWatchdog.js +20 -0
- package/dist/operations/server/index.d.ts +12 -0
- package/dist/operations/server/index.js +12 -0
- package/dist/operations/server/migrations.d.ts +14 -0
- package/dist/operations/server/migrations.js +7 -0
- package/dist/operations/server/platformWatchdogDecision.d.ts +34 -0
- package/dist/operations/server/platformWatchdogDecision.js +71 -0
- package/dist/operations/server/platformWatchdogDeliveryPolicy.d.ts +10 -0
- package/dist/operations/server/platformWatchdogDeliveryPolicy.js +53 -0
- package/dist/operations/server/platformWatchdogQueue.d.ts +11 -0
- package/dist/operations/server/platformWatchdogQueue.js +94 -0
- package/dist/operations/server/platformWatchdogService.d.ts +47 -0
- package/dist/operations/server/platformWatchdogService.js +105 -0
- package/dist/operations/validation.d.ts +2 -0
- package/dist/operations/validation.js +3 -0
- package/dist/portability/filesystemObjects.d.ts +12 -0
- package/dist/portability/filesystemObjects.js +191 -0
- package/dist/portability/index.d.ts +5 -0
- package/dist/portability/index.js +4 -0
- package/dist/portability/objects.d.ts +63 -0
- package/dist/portability/objects.js +138 -0
- package/dist/portability/postgresRows.d.ts +64 -0
- package/dist/portability/postgresRows.js +182 -0
- package/dist/scheduler/advisoryLock.js +20 -2
- package/dist/scheduler/nodeScheduler.d.ts +2 -2
- package/dist/scheduler/nodeScheduler.js +35 -6
- package/package.json +27 -2
- package/release-gates.json +73 -7
- package/sql/20260814010000_platform_control_plane.sql +286 -0
- package/src/composition/nodeHost.ts +4 -3
- package/src/lifecycle/appUpgrade.ts +186 -0
- package/src/lifecycle/index.ts +19 -0
- package/src/lifecycle/postgresInstallQualification.ts +125 -0
- package/src/migrations/postgresRunner.ts +24 -9
- package/src/operations/adminAlertsSchemas.ts +78 -0
- package/src/operations/adminAlertsView.ts +238 -0
- package/src/operations/alertPagingPolicy.ts +50 -0
- package/src/operations/alertThrottlePolicy.ts +22 -0
- package/src/operations/clients.ts +44 -0
- package/src/operations/contracts/admin.ts +24 -0
- package/src/operations/contracts/adminAlertsContracts.ts +13 -0
- package/src/operations/contracts/alertHumanContext.ts +17 -0
- package/src/operations/contracts/index.ts +6 -0
- package/src/operations/contracts/observabilityContracts.ts +40 -0
- package/src/operations/contracts/observabilityPorts.ts +30 -0
- package/src/operations/contracts/ports.ts +22 -0
- package/src/operations/errors.ts +6 -0
- package/src/operations/index.ts +9 -0
- package/src/operations/schemas.ts +122 -0
- package/src/operations/server/adminAlertsHandlers.ts +61 -0
- package/src/operations/server/adminPipelineHandlers.ts +73 -0
- package/src/operations/server/adminSettingsHandlers.ts +114 -0
- package/src/operations/server/controlPlaneHandler.ts +56 -0
- package/src/operations/server/controlPlaneStore.ts +150 -0
- package/src/operations/server/controlPlaneWatchdog.ts +28 -0
- package/src/operations/server/index.ts +12 -0
- package/src/operations/server/migrations.ts +7 -0
- package/src/operations/server/platformWatchdogDecision.ts +93 -0
- package/src/operations/server/platformWatchdogDeliveryPolicy.ts +92 -0
- package/src/operations/server/platformWatchdogQueue.ts +96 -0
- package/src/operations/server/platformWatchdogService.ts +209 -0
- package/src/operations/validation.ts +3 -0
- package/src/portability/filesystemObjects.ts +138 -0
- package/src/portability/index.ts +12 -0
- package/src/portability/objects.ts +160 -0
- package/src/portability/postgresRows.ts +169 -0
- package/src/scheduler/advisoryLock.ts +20 -5
- package/src/scheduler/nodeScheduler.ts +28 -8
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { PlatformControlPlaneConflictError, PlatformControlPlaneUnavailableError } from "../index.js";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
const rpc = (name, types) => `SELECT * FROM public.${name}(${types.map((type, index) => `$${index + 1}::${type}`).join(", ")})`;
|
|
4
|
+
export const platformControlPlaneSql = {
|
|
5
|
+
active: rpc("platform_control_operator_is_active", ["uuid"]),
|
|
6
|
+
setControl: rpc("platform_control_set_control", ["uuid", "text", "text", "text", "boolean"]),
|
|
7
|
+
heartbeat: rpc("platform_control_record_heartbeat", ["uuid", "text", "text", "text", "integer", "text", "boolean"]),
|
|
8
|
+
controls: rpc("platform_control_list_controls", ["uuid"]),
|
|
9
|
+
jobs: rpc("platform_control_list_jobs", ["uuid"]),
|
|
10
|
+
alerts: rpc("platform_control_list_alerts", ["uuid"]),
|
|
11
|
+
};
|
|
12
|
+
export function createPlatformControlPlanePort(client, operatorId) {
|
|
13
|
+
return {
|
|
14
|
+
async readControlPlane() {
|
|
15
|
+
try {
|
|
16
|
+
const [controls, jobs, alerts] = await Promise.all([
|
|
17
|
+
client.query(platformControlPlaneSql.controls, [operatorId]),
|
|
18
|
+
client.query(platformControlPlaneSql.jobs, [operatorId]),
|
|
19
|
+
client.query(platformControlPlaneSql.alerts, [operatorId]),
|
|
20
|
+
]);
|
|
21
|
+
return {
|
|
22
|
+
controls: controls.rows.map((row) => ({
|
|
23
|
+
controlKey: text(row.control_key), enabled: bool(row.enabled),
|
|
24
|
+
revision: positive(row.revision), updatedAt: timestamp(row.updated_at),
|
|
25
|
+
})),
|
|
26
|
+
jobs: jobs.rows.map((row) => ({
|
|
27
|
+
jobName: text(row.job_name), enabled: bool(row.enabled),
|
|
28
|
+
activeDriver: driver(row.active_driver), lastStatus: jobStatus(row.last_status),
|
|
29
|
+
lastStartedAt: nullableTimestamp(row.last_started_at),
|
|
30
|
+
lastFinishedAt: nullableTimestamp(row.last_finished_at),
|
|
31
|
+
lastSuccessAt: nullableTimestamp(row.last_success_at),
|
|
32
|
+
})),
|
|
33
|
+
alerts: alerts.rows.map((row) => ({
|
|
34
|
+
dedupeKey: text(row.dedupe_key), severity: severity(row.severity),
|
|
35
|
+
status: alertStatus(row.status), owner: text(row.owner), title: text(row.title),
|
|
36
|
+
firstSeenAt: timestamp(row.first_seen_at), lastSeenAt: timestamp(row.last_seen_at),
|
|
37
|
+
resolvedAt: nullableTimestamp(row.resolved_at),
|
|
38
|
+
})),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
throw mapError(error);
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
async mutateControlPlane(request) {
|
|
46
|
+
try {
|
|
47
|
+
const fingerprint = digest(JSON.stringify(request));
|
|
48
|
+
const result = request.action === "set-control"
|
|
49
|
+
? await client.query(platformControlPlaneSql.setControl, [
|
|
50
|
+
operatorId, request.idempotencyKey, fingerprint, request.controlKey, request.enabled,
|
|
51
|
+
])
|
|
52
|
+
: await client.query(platformControlPlaneSql.heartbeat, [
|
|
53
|
+
operatorId, request.idempotencyKey, fingerprint, request.health,
|
|
54
|
+
request.firingCount, request.maxSeverity, request.promotionReady,
|
|
55
|
+
]);
|
|
56
|
+
return mutation(first(result.rows), request.action);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
throw mapError(error);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export async function isPlatformControlOperatorActive(client, principalId) {
|
|
65
|
+
try {
|
|
66
|
+
return first((await client.query(platformControlPlaneSql.active, [principalId])).rows)
|
|
67
|
+
?.platform_control_operator_is_active === true;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
throw mapError(error);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function mutation(row, expected) {
|
|
74
|
+
if (!row || row.action !== expected)
|
|
75
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
76
|
+
return {
|
|
77
|
+
action: expected,
|
|
78
|
+
replayed: bool(row.replayed),
|
|
79
|
+
revision: row.revision == null ? null : positive(row.revision),
|
|
80
|
+
recordedAt: timestamp(row.recorded_at),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function digest(value) {
|
|
84
|
+
return createHash("sha256").update(value).digest("hex");
|
|
85
|
+
}
|
|
86
|
+
function first(rows) { return rows[0]; }
|
|
87
|
+
function text(value) {
|
|
88
|
+
if (typeof value !== "string" || value.length === 0)
|
|
89
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
function bool(value) {
|
|
93
|
+
if (typeof value !== "boolean")
|
|
94
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
95
|
+
return value;
|
|
96
|
+
}
|
|
97
|
+
function positive(value) {
|
|
98
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1)
|
|
99
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
function timestamp(value) {
|
|
103
|
+
const result = text(value);
|
|
104
|
+
if (!Number.isFinite(Date.parse(result)))
|
|
105
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
function nullableTimestamp(value) { return value == null ? null : timestamp(value); }
|
|
109
|
+
function driver(value) {
|
|
110
|
+
if (value === "worker" || value === "scheduler" || value === "operator")
|
|
111
|
+
return value;
|
|
112
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
113
|
+
}
|
|
114
|
+
function jobStatus(value) {
|
|
115
|
+
if (value == null)
|
|
116
|
+
return null;
|
|
117
|
+
if (value === "running" || value === "success" || value === "failed" || value === "skipped")
|
|
118
|
+
return value;
|
|
119
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
120
|
+
}
|
|
121
|
+
function severity(value) {
|
|
122
|
+
if (value === "p0" || value === "p1" || value === "p2" || value === "p3")
|
|
123
|
+
return value;
|
|
124
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
125
|
+
}
|
|
126
|
+
function alertStatus(value) {
|
|
127
|
+
if (value === "open" || value === "acknowledged" || value === "resolved")
|
|
128
|
+
return value;
|
|
129
|
+
throw new PlatformControlPlaneUnavailableError();
|
|
130
|
+
}
|
|
131
|
+
function mapError(error) {
|
|
132
|
+
if (error instanceof PlatformControlPlaneUnavailableError || error instanceof PlatformControlPlaneConflictError)
|
|
133
|
+
return error;
|
|
134
|
+
const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
|
|
135
|
+
if (code === "23505")
|
|
136
|
+
return new PlatformControlPlaneConflictError("platform_control_idempotency_conflict");
|
|
137
|
+
return new PlatformControlPlaneUnavailableError("platform_control_plane_unavailable");
|
|
138
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { PlatformControlPlanePort } from "../contracts/index.js";
|
|
2
|
+
export type DirectPlatformWatchdogResult = {
|
|
3
|
+
status: number;
|
|
4
|
+
body: Record<string, unknown>;
|
|
5
|
+
};
|
|
6
|
+
/** Existing public-bundle evaluation of persisted neutral job facts. No provider calls. */
|
|
7
|
+
export declare function runControlPlaneWatchdogTick(port: PlatformControlPlanePort, now: Date, checkOnly: boolean): Promise<DirectPlatformWatchdogResult>;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Existing public-bundle evaluation of persisted neutral job facts. No provider calls. */
|
|
2
|
+
export async function runControlPlaneWatchdogTick(port, now, checkOnly) {
|
|
3
|
+
const snapshot = await port.readControlPlane({});
|
|
4
|
+
const failedJobs = snapshot.jobs.filter((job) => job.enabled && job.lastStatus === "failed");
|
|
5
|
+
const health = failedJobs.length > 0 ? "degraded" : "healthy";
|
|
6
|
+
const recorded = checkOnly ? null : await port.mutateControlPlane({
|
|
7
|
+
action: "record-heartbeat",
|
|
8
|
+
idempotencyKey: `platform-watchdog:${now.toISOString()}`,
|
|
9
|
+
health,
|
|
10
|
+
firingCount: failedJobs.length,
|
|
11
|
+
maxSeverity: failedJobs.length > 0 ? "p2" : null,
|
|
12
|
+
promotionReady: failedJobs.length === 0,
|
|
13
|
+
});
|
|
14
|
+
return { status: 200, body: {
|
|
15
|
+
ok: true, health, firingCount: failedJobs.length,
|
|
16
|
+
failedJobs: failedJobs.map((job) => job.jobName),
|
|
17
|
+
checkedAt: now.toISOString(), heartbeatRecorded: recorded != null,
|
|
18
|
+
replayed: recorded?.replayed ?? false,
|
|
19
|
+
} };
|
|
20
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Server mechanisms; authorization, persistence and provider transports are injected. */
|
|
2
|
+
export * from "./adminSettingsHandlers.js";
|
|
3
|
+
export * from "./adminPipelineHandlers.js";
|
|
4
|
+
export * from "./adminAlertsHandlers.js";
|
|
5
|
+
export * from "./controlPlaneHandler.js";
|
|
6
|
+
export * from "./controlPlaneStore.js";
|
|
7
|
+
export * from "./platformWatchdogService.js";
|
|
8
|
+
export * from "./platformWatchdogDecision.js";
|
|
9
|
+
export * from "./platformWatchdogDeliveryPolicy.js";
|
|
10
|
+
export * from "./platformWatchdogQueue.js";
|
|
11
|
+
export * from "./controlPlaneWatchdog.js";
|
|
12
|
+
export * from "./migrations.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Server mechanisms; authorization, persistence and provider transports are injected. */
|
|
2
|
+
export * from "./adminSettingsHandlers.js";
|
|
3
|
+
export * from "./adminPipelineHandlers.js";
|
|
4
|
+
export * from "./adminAlertsHandlers.js";
|
|
5
|
+
export * from "./controlPlaneHandler.js";
|
|
6
|
+
export * from "./controlPlaneStore.js";
|
|
7
|
+
export * from "./platformWatchdogService.js";
|
|
8
|
+
export * from "./platformWatchdogDecision.js";
|
|
9
|
+
export * from "./platformWatchdogDeliveryPolicy.js";
|
|
10
|
+
export * from "./platformWatchdogQueue.js";
|
|
11
|
+
export * from "./controlPlaneWatchdog.js";
|
|
12
|
+
export * from "./migrations.js";
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** Sparse projection only: install the complete canonical native prefix, never this unit alone. */
|
|
2
|
+
export declare const OPERATIONS_MIGRATIONS: {
|
|
3
|
+
readonly native: {
|
|
4
|
+
readonly units: readonly [{
|
|
5
|
+
readonly position: 28;
|
|
6
|
+
readonly catalogPath: "db/platform/migrations/20260814010000_platform_control_plane.sql";
|
|
7
|
+
readonly sha256: "b0c8ef6424ebc6f1a84cb132c37908f4831fa40484a729c3713cfb35936559c9";
|
|
8
|
+
readonly url: URL;
|
|
9
|
+
}];
|
|
10
|
+
};
|
|
11
|
+
readonly managed: {
|
|
12
|
+
readonly verified: false;
|
|
13
|
+
};
|
|
14
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Sparse projection only: install the complete canonical native prefix, never this unit alone. */
|
|
2
|
+
export const OPERATIONS_MIGRATIONS = {
|
|
3
|
+
native: { units: [{ position: 28, catalogPath: "db/platform/migrations/20260814010000_platform_control_plane.sql",
|
|
4
|
+
sha256: "b0c8ef6424ebc6f1a84cb132c37908f4831fa40484a729c3713cfb35936559c9",
|
|
5
|
+
url: new URL("../../../sql/20260814010000_platform_control_plane.sql", import.meta.url) }] },
|
|
6
|
+
managed: { verified: false },
|
|
7
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { AlertDecision, AlertSeverity } from "../contracts/index.js";
|
|
2
|
+
import type { AlertLedgerPort, AlertSinkPort } from "../contracts/index.js";
|
|
3
|
+
export declare function emptyWatchdogDecisionCounts(): {
|
|
4
|
+
notified: number;
|
|
5
|
+
notificationFailures: number;
|
|
6
|
+
deliveryBackoffCount: number;
|
|
7
|
+
throttledNotifications: number;
|
|
8
|
+
skippedNotifications: number;
|
|
9
|
+
belowThreshold: number;
|
|
10
|
+
suppressed: number;
|
|
11
|
+
actionableCriticalCount: number;
|
|
12
|
+
suppressedCriticalCount: number;
|
|
13
|
+
actionablePageableCount: number;
|
|
14
|
+
};
|
|
15
|
+
/** One complete ledger transition; callers must serialize the same dedupe key. */
|
|
16
|
+
export declare function processWatchdogDecision<Channel extends string>(decision: AlertDecision<Channel>, { ledgerPort, sinkPort, now, pagingMinSeverity, notificationThrottleSeconds, }: {
|
|
17
|
+
ledgerPort: AlertLedgerPort<Channel>;
|
|
18
|
+
sinkPort: AlertSinkPort<Channel>;
|
|
19
|
+
now: Date;
|
|
20
|
+
pagingMinSeverity: AlertSeverity;
|
|
21
|
+
notificationThrottleSeconds?: number;
|
|
22
|
+
}): Promise<{
|
|
23
|
+
notified: number;
|
|
24
|
+
notificationFailures: number;
|
|
25
|
+
deliveryBackoffCount: number;
|
|
26
|
+
throttledNotifications: number;
|
|
27
|
+
skippedNotifications: number;
|
|
28
|
+
belowThreshold: number;
|
|
29
|
+
suppressed: number;
|
|
30
|
+
actionableCriticalCount: number;
|
|
31
|
+
suppressedCriticalCount: number;
|
|
32
|
+
actionablePageableCount: number;
|
|
33
|
+
}>;
|
|
34
|
+
export declare function isCriticalDecision(decision: AlertDecision): boolean;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { shouldPage } from "../index.js";
|
|
2
|
+
import { throttleSecondsForSeverity } from "../index.js";
|
|
3
|
+
import { isActionabilitySuppressed, isDeliveryBackoff, isPagingSuppressed, notificationSchedule, shouldAttemptNotification, } from "./platformWatchdogDeliveryPolicy.js";
|
|
4
|
+
const BELOW_PAGING_SEVERITY_OUTCOME = {
|
|
5
|
+
channel: "webhook", status: "skipped", provider: "webhook", error: "below_paging_severity",
|
|
6
|
+
};
|
|
7
|
+
const NON_PAGEABLE_DIAGNOSTIC_OUTCOME = {
|
|
8
|
+
channel: "webhook", status: "skipped", provider: "webhook", error: "non_pageable_diagnostic",
|
|
9
|
+
};
|
|
10
|
+
export function emptyWatchdogDecisionCounts() {
|
|
11
|
+
return {
|
|
12
|
+
notified: 0,
|
|
13
|
+
notificationFailures: 0,
|
|
14
|
+
deliveryBackoffCount: 0,
|
|
15
|
+
throttledNotifications: 0,
|
|
16
|
+
skippedNotifications: 0,
|
|
17
|
+
belowThreshold: 0,
|
|
18
|
+
suppressed: 0,
|
|
19
|
+
actionableCriticalCount: 0,
|
|
20
|
+
suppressedCriticalCount: 0,
|
|
21
|
+
actionablePageableCount: 0,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/** One complete ledger transition; callers must serialize the same dedupe key. */
|
|
25
|
+
export async function processWatchdogDecision(decision, { ledgerPort, sinkPort, now, pagingMinSeverity, notificationThrottleSeconds, }) {
|
|
26
|
+
const counts = emptyWatchdogDecisionCounts();
|
|
27
|
+
const alert = await ledgerPort.upsertOpenAlert(decision, now);
|
|
28
|
+
const critical = isCriticalDecision(decision);
|
|
29
|
+
const actionableSuppressed = isActionabilitySuppressed(alert, now);
|
|
30
|
+
const pagingSuppressed = isPagingSuppressed(alert, now);
|
|
31
|
+
const pageable = decision.paging !== "never" && shouldPage(decision.severity, pagingMinSeverity);
|
|
32
|
+
// Acknowledgement silences pages, but only a snooze suppresses actionability.
|
|
33
|
+
if (critical) {
|
|
34
|
+
if (actionableSuppressed)
|
|
35
|
+
counts.suppressedCriticalCount += 1;
|
|
36
|
+
else
|
|
37
|
+
counts.actionableCriticalCount += 1;
|
|
38
|
+
}
|
|
39
|
+
if (pageable && !pagingSuppressed)
|
|
40
|
+
counts.actionablePageableCount += 1;
|
|
41
|
+
if (pageable && !pagingSuppressed && isDeliveryBackoff(alert, now))
|
|
42
|
+
counts.deliveryBackoffCount += 1;
|
|
43
|
+
if (pagingSuppressed) {
|
|
44
|
+
counts.suppressed += 1;
|
|
45
|
+
return counts;
|
|
46
|
+
}
|
|
47
|
+
const throttleSeconds = notificationThrottleSeconds ?? throttleSecondsForSeverity(decision.severity);
|
|
48
|
+
if (!shouldAttemptNotification(alert, now, throttleSeconds)) {
|
|
49
|
+
counts.throttledNotifications += 1;
|
|
50
|
+
return counts;
|
|
51
|
+
}
|
|
52
|
+
// Non-pageable and below-threshold alerts retain their skipped-attempt audit.
|
|
53
|
+
const nonPageable = decision.paging === "never";
|
|
54
|
+
const paging = !nonPageable && shouldPage(decision.severity, pagingMinSeverity);
|
|
55
|
+
const outcome = paging
|
|
56
|
+
? await sinkPort.send(decision, alert)
|
|
57
|
+
: nonPageable ? NON_PAGEABLE_DIAGNOSTIC_OUTCOME : BELOW_PAGING_SEVERITY_OUTCOME;
|
|
58
|
+
if (outcome.status === "sent")
|
|
59
|
+
counts.notified += 1;
|
|
60
|
+
else if (outcome.status === "failed")
|
|
61
|
+
counts.notificationFailures += 1;
|
|
62
|
+
else
|
|
63
|
+
counts.skippedNotifications += 1;
|
|
64
|
+
if (!paging)
|
|
65
|
+
counts.belowThreshold += 1;
|
|
66
|
+
await ledgerPort.recordNotification(alert.id, outcome, now, notificationSchedule(alert, outcome, now, throttleSeconds));
|
|
67
|
+
return counts;
|
|
68
|
+
}
|
|
69
|
+
export function isCriticalDecision(decision) {
|
|
70
|
+
return decision.paging !== "never" && (decision.severity === "p0" || decision.severity === "p1");
|
|
71
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AlertDecision, AlertNotificationOutcome, OpenAlert } from "../contracts/index.js";
|
|
2
|
+
export declare function shouldAttemptNotification(alert: Pick<OpenAlert, "lastNotifiedAt" | "lastNotificationStatus" | "nextNotificationAttemptAt" | "snoozedUntil">, now: Date, throttleSeconds: number): boolean;
|
|
3
|
+
export declare function notificationSchedule(alert: Pick<OpenAlert, "notificationFailureCount">, outcome: AlertNotificationOutcome, now: Date, throttleSeconds: number): {
|
|
4
|
+
nextAttemptAt: Date;
|
|
5
|
+
failureCount: number;
|
|
6
|
+
};
|
|
7
|
+
export declare function isActionabilitySuppressed(alert: Pick<OpenAlert, "snoozedUntil">, now: Date): boolean;
|
|
8
|
+
export declare function isPagingSuppressed(alert: Pick<OpenAlert, "status" | "snoozedUntil">, now: Date): boolean;
|
|
9
|
+
export declare function isDeliveryBackoff(alert: Pick<OpenAlert, "lastNotificationStatus" | "nextNotificationAttemptAt">, now: Date): boolean;
|
|
10
|
+
export declare function isSeverityEscalation(decision: Pick<AlertDecision, "severity">, alert: Pick<OpenAlert, "severity">): boolean;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
const FAILURE_RETRY_INITIAL_SECONDS = 10 * 60;
|
|
2
|
+
const FAILURE_RETRY_MAX_SECONDS = 60 * 60;
|
|
3
|
+
export function shouldAttemptNotification(alert, now, throttleSeconds) {
|
|
4
|
+
const snoozeExpiry = alert.snoozedUntil ? Date.parse(alert.snoozedUntil) : Number.NaN;
|
|
5
|
+
const lastSentAt = alert.lastNotifiedAt ? Date.parse(alert.lastNotifiedAt) : Number.NaN;
|
|
6
|
+
if (alert.lastNotificationStatus !== "failed"
|
|
7
|
+
&& Number.isFinite(snoozeExpiry)
|
|
8
|
+
&& snoozeExpiry <= now.getTime()
|
|
9
|
+
&& (!Number.isFinite(lastSentAt) || snoozeExpiry > lastSentAt)) {
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
const nextAttemptAt = alert.nextNotificationAttemptAt
|
|
13
|
+
? Date.parse(alert.nextNotificationAttemptAt)
|
|
14
|
+
: Number.NaN;
|
|
15
|
+
if (Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime())
|
|
16
|
+
return false;
|
|
17
|
+
if (!alert.lastNotifiedAt)
|
|
18
|
+
return true;
|
|
19
|
+
return now.getTime() - new Date(alert.lastNotifiedAt).getTime() >= throttleSeconds * 1000;
|
|
20
|
+
}
|
|
21
|
+
export function notificationSchedule(alert, outcome, now, throttleSeconds) {
|
|
22
|
+
if (outcome.status === "failed") {
|
|
23
|
+
const failureCount = (alert.notificationFailureCount ?? 0) + 1;
|
|
24
|
+
const exponent = Math.max(0, Math.min(failureCount - 1, 16));
|
|
25
|
+
const retrySeconds = Math.min(FAILURE_RETRY_MAX_SECONDS, FAILURE_RETRY_INITIAL_SECONDS * 2 ** exponent);
|
|
26
|
+
return {
|
|
27
|
+
nextAttemptAt: new Date(now.getTime() + retrySeconds * 1000),
|
|
28
|
+
failureCount,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
// Sent uses the severity cadence. Skipped uses the same due field only to
|
|
33
|
+
// avoid duplicate audit rows; last_notified_at still advances on sent only.
|
|
34
|
+
nextAttemptAt: new Date(now.getTime() + throttleSeconds * 1000),
|
|
35
|
+
failureCount: 0,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
export function isActionabilitySuppressed(alert, now) {
|
|
39
|
+
return alert.snoozedUntil != null && new Date(alert.snoozedUntil).getTime() > now.getTime();
|
|
40
|
+
}
|
|
41
|
+
export function isPagingSuppressed(alert, now) {
|
|
42
|
+
return alert.status === "acknowledged" || isActionabilitySuppressed(alert, now);
|
|
43
|
+
}
|
|
44
|
+
export function isDeliveryBackoff(alert, now) {
|
|
45
|
+
if (alert.lastNotificationStatus !== "failed" || !alert.nextNotificationAttemptAt)
|
|
46
|
+
return false;
|
|
47
|
+
const nextAttemptAt = Date.parse(alert.nextNotificationAttemptAt);
|
|
48
|
+
return Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime();
|
|
49
|
+
}
|
|
50
|
+
export function isSeverityEscalation(decision, alert) {
|
|
51
|
+
const rank = { p0: 0, p1: 1, p2: 2, p3: 3 };
|
|
52
|
+
return rank[decision.severity] < rank[alert.severity];
|
|
53
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { AlertDecision } from "../contracts/index.js";
|
|
2
|
+
import type { AlertSinkPort } from "../contracts/index.js";
|
|
3
|
+
type WatchdogQueueItem = Pick<AlertDecision, "dedupeKey" | "severity" | "paging">;
|
|
4
|
+
/**
|
|
5
|
+
* Only each key's first unfinished item is eligible. A head heap prioritizes
|
|
6
|
+
* normalized severity without reordering the caller's decisions or duplicates.
|
|
7
|
+
*/
|
|
8
|
+
export declare function runWatchdogQueue(items: readonly WatchdogQueueItem[], process: (index: number) => Promise<void>): Promise<void>;
|
|
9
|
+
/** Per-tick transport limit; admitted waiters still run if an earlier send throws. */
|
|
10
|
+
export declare function createSerialWatchdogSink<Channel extends string>(sinkPort: AlertSinkPort<Channel>): AlertSinkPort<Channel>;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
const WATCHDOG_CONCURRENCY = 4;
|
|
2
|
+
const SEVERITY_RANK = { p0: 0, p1: 1, p2: 2, p3: 3 };
|
|
3
|
+
/**
|
|
4
|
+
* Only each key's first unfinished item is eligible. A head heap prioritizes
|
|
5
|
+
* normalized severity without reordering the caller's decisions or duplicates.
|
|
6
|
+
*/
|
|
7
|
+
export async function runWatchdogQueue(items, process) {
|
|
8
|
+
const next = new Array(items.length).fill(-1);
|
|
9
|
+
const lastByKey = new Map();
|
|
10
|
+
const ready = [];
|
|
11
|
+
for (let index = 0; index < items.length; index += 1) {
|
|
12
|
+
const previous = lastByKey.get(items[index].dedupeKey);
|
|
13
|
+
if (previous === undefined)
|
|
14
|
+
push(index);
|
|
15
|
+
else
|
|
16
|
+
next[previous] = index;
|
|
17
|
+
lastByKey.set(items[index].dedupeKey, index);
|
|
18
|
+
}
|
|
19
|
+
let failed = false;
|
|
20
|
+
let firstError;
|
|
21
|
+
async function worker() {
|
|
22
|
+
while (!failed) {
|
|
23
|
+
const index = pop();
|
|
24
|
+
if (index === undefined)
|
|
25
|
+
return;
|
|
26
|
+
try {
|
|
27
|
+
await process(index);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (!failed)
|
|
31
|
+
firstError = error;
|
|
32
|
+
failed = true;
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
if (!failed && next[index] !== -1)
|
|
36
|
+
push(next[index]);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Workers catch their own errors so every admitted chain settles before reject.
|
|
40
|
+
await Promise.all(Array.from({ length: WATCHDOG_CONCURRENCY }, () => worker()));
|
|
41
|
+
if (failed)
|
|
42
|
+
throw firstError;
|
|
43
|
+
function compare(left, right) {
|
|
44
|
+
const a = items[left];
|
|
45
|
+
const b = items[right];
|
|
46
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity]
|
|
47
|
+
|| Number(a.paging === "never") - Number(b.paging === "never")
|
|
48
|
+
|| a.dedupeKey.localeCompare(b.dedupeKey)
|
|
49
|
+
|| left - right;
|
|
50
|
+
}
|
|
51
|
+
function push(index) {
|
|
52
|
+
let position = ready.length;
|
|
53
|
+
ready.push(index);
|
|
54
|
+
while (position > 0) {
|
|
55
|
+
const parent = Math.floor((position - 1) / 2);
|
|
56
|
+
if (compare(ready[parent], index) <= 0)
|
|
57
|
+
break;
|
|
58
|
+
ready[position] = ready[parent];
|
|
59
|
+
position = parent;
|
|
60
|
+
}
|
|
61
|
+
ready[position] = index;
|
|
62
|
+
}
|
|
63
|
+
function pop() {
|
|
64
|
+
if (ready.length === 0)
|
|
65
|
+
return undefined;
|
|
66
|
+
const first = ready[0];
|
|
67
|
+
const last = ready.pop();
|
|
68
|
+
if (ready.length === 0)
|
|
69
|
+
return first;
|
|
70
|
+
let position = 0;
|
|
71
|
+
while (position * 2 + 1 < ready.length) {
|
|
72
|
+
let child = position * 2 + 1;
|
|
73
|
+
if (child + 1 < ready.length && compare(ready[child + 1], ready[child]) < 0)
|
|
74
|
+
child += 1;
|
|
75
|
+
if (compare(last, ready[child]) <= 0)
|
|
76
|
+
break;
|
|
77
|
+
ready[position] = ready[child];
|
|
78
|
+
position = child;
|
|
79
|
+
}
|
|
80
|
+
ready[position] = last;
|
|
81
|
+
return first;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Per-tick transport limit; admitted waiters still run if an earlier send throws. */
|
|
85
|
+
export function createSerialWatchdogSink(sinkPort) {
|
|
86
|
+
let tail = Promise.resolve();
|
|
87
|
+
return {
|
|
88
|
+
send(decision, alert) {
|
|
89
|
+
const attempt = tail.then(() => sinkPort.send(decision, alert));
|
|
90
|
+
tail = attempt.then(() => undefined, () => undefined);
|
|
91
|
+
return attempt;
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { AlertDecision, AlertSeverity, WatchdogSnapshot } from "../contracts/index.js";
|
|
2
|
+
import type { AlertLedgerPort, AlertSinkPort, ObservabilityEvidencePort } from "../contracts/index.js";
|
|
3
|
+
export type PlatformWatchdogResult<Channel extends string = string> = {
|
|
4
|
+
ok: boolean;
|
|
5
|
+
health: "healthy" | "firing";
|
|
6
|
+
checkOnly: boolean;
|
|
7
|
+
checkedAt: string;
|
|
8
|
+
alertCount: number;
|
|
9
|
+
firingCount: number;
|
|
10
|
+
/** P0/P1 incidents not covered by an active snooze and still requiring ownership. */
|
|
11
|
+
actionableCriticalCount: number;
|
|
12
|
+
/** P0/P1 incidents covered by an active snooze; acknowledged alerts stay actionable. */
|
|
13
|
+
suppressedCriticalCount: number;
|
|
14
|
+
/** Alerts that meet the current paging threshold and are not currently page-suppressed. */
|
|
15
|
+
actionablePageableCount: number;
|
|
16
|
+
maxSeverity: string | null;
|
|
17
|
+
decisions: AlertDecision<Channel>[];
|
|
18
|
+
notified: number;
|
|
19
|
+
notificationFailures: number;
|
|
20
|
+
/** Failed deliveries held until their persisted bounded retry time. */
|
|
21
|
+
deliveryBackoffCount: number;
|
|
22
|
+
throttledNotifications: number;
|
|
23
|
+
skippedNotifications: number;
|
|
24
|
+
belowThreshold: number;
|
|
25
|
+
suppressed: number;
|
|
26
|
+
muted: number;
|
|
27
|
+
resolved: number;
|
|
28
|
+
};
|
|
29
|
+
export declare function runPlatformWatchdog<Snapshot extends WatchdogSnapshot, CatalogEntry, Channel extends string = string>({ evidencePort, evaluate, ledgerPort, sinkPort, catalog, now, checkOnly, notificationThrottleSeconds, pagingMinSeverity, mutedDedupePrefixes, normalizeDecision, }: {
|
|
30
|
+
evidencePort: ObservabilityEvidencePort<Snapshot>;
|
|
31
|
+
evaluate: (snapshot: Snapshot, catalog: readonly CatalogEntry[], now: Date) => AlertDecision<Channel>[];
|
|
32
|
+
ledgerPort: AlertLedgerPort<Channel>;
|
|
33
|
+
sinkPort: AlertSinkPort<Channel>;
|
|
34
|
+
catalog: readonly CatalogEntry[];
|
|
35
|
+
now?: Date;
|
|
36
|
+
checkOnly: boolean;
|
|
37
|
+
/** Flat override; when omitted, the throttle is scaled per alert severity. */
|
|
38
|
+
notificationThrottleSeconds?: number;
|
|
39
|
+
pagingMinSeverity?: AlertSeverity;
|
|
40
|
+
/**
|
|
41
|
+
* Dedupe-key prefixes fully muted for this environment (e.g. `job_missed` on a
|
|
42
|
+
* staging host with no real crons). Muted signals are not recorded, not paged,
|
|
43
|
+
* and existing open rows for them auto-resolve — keeping the ledger clean.
|
|
44
|
+
*/
|
|
45
|
+
mutedDedupePrefixes?: readonly string[];
|
|
46
|
+
normalizeDecision?: (decision: AlertDecision<Channel>) => AlertDecision<Channel>;
|
|
47
|
+
}): Promise<PlatformWatchdogResult<Channel>>;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { DEFAULT_PAGING_MIN_SEVERITY, shouldPage, } from "../index.js";
|
|
2
|
+
import { isActionabilitySuppressed, isDeliveryBackoff, isPagingSuppressed, isSeverityEscalation, } from "./platformWatchdogDeliveryPolicy.js";
|
|
3
|
+
import { emptyWatchdogDecisionCounts, isCriticalDecision, processWatchdogDecision, } from "./platformWatchdogDecision.js";
|
|
4
|
+
import { createSerialWatchdogSink, runWatchdogQueue } from "./platformWatchdogQueue.js";
|
|
5
|
+
export async function runPlatformWatchdog({ evidencePort, evaluate, ledgerPort, sinkPort, catalog, now = new Date(), checkOnly, notificationThrottleSeconds, pagingMinSeverity = DEFAULT_PAGING_MIN_SEVERITY, mutedDedupePrefixes = [], normalizeDecision, }) {
|
|
6
|
+
const snapshot = await evidencePort.collectSnapshot(now);
|
|
7
|
+
const allDecisions = evaluate(snapshot, catalog, now);
|
|
8
|
+
const decisions = allDecisions
|
|
9
|
+
.filter((decision) => !isMuted(decision.dedupeKey, mutedDedupePrefixes))
|
|
10
|
+
.map((decision) => normalizeDecision ? normalizeDecision(decision) : decision);
|
|
11
|
+
const muted = allDecisions.length - decisions.length;
|
|
12
|
+
const firingDecisions = decisions.filter((decision) => decision.paging !== "never");
|
|
13
|
+
const firingCount = firingDecisions.length;
|
|
14
|
+
const maxSeverity = maxSeverityOf(firingDecisions);
|
|
15
|
+
const openAlerts = await ledgerPort.listOpenAlerts();
|
|
16
|
+
if (checkOnly) {
|
|
17
|
+
const alertsByDedupeKey = new Map(openAlerts.map((alert) => [alert.dedupeKey, alert]));
|
|
18
|
+
let actionableCriticalCount = 0;
|
|
19
|
+
let suppressedCriticalCount = 0;
|
|
20
|
+
let actionablePageableCount = 0;
|
|
21
|
+
let deliveryBackoffCount = 0;
|
|
22
|
+
let suppressed = 0;
|
|
23
|
+
for (const decision of decisions) {
|
|
24
|
+
const alert = alertsByDedupeKey.get(decision.dedupeKey);
|
|
25
|
+
// The mutating path clears acknowledgement, snooze and delivery cadence on
|
|
26
|
+
// escalation. Mirror that transition here so a read-only probe cannot
|
|
27
|
+
// report a stale suppression that the next real tick will remove.
|
|
28
|
+
const suppressionSurvives = alert ? !isSeverityEscalation(decision, alert) : false;
|
|
29
|
+
const actionableSuppressed = alert && suppressionSurvives
|
|
30
|
+
? isActionabilitySuppressed(alert, now)
|
|
31
|
+
: false;
|
|
32
|
+
const pagingSuppressed = alert && suppressionSurvives
|
|
33
|
+
? isPagingSuppressed(alert, now)
|
|
34
|
+
: false;
|
|
35
|
+
const critical = isCriticalDecision(decision);
|
|
36
|
+
const pageable = decision.paging !== "never" && shouldPage(decision.severity, pagingMinSeverity);
|
|
37
|
+
if (critical) {
|
|
38
|
+
if (actionableSuppressed)
|
|
39
|
+
suppressedCriticalCount += 1;
|
|
40
|
+
else
|
|
41
|
+
actionableCriticalCount += 1;
|
|
42
|
+
}
|
|
43
|
+
if (pageable && !pagingSuppressed)
|
|
44
|
+
actionablePageableCount += 1;
|
|
45
|
+
if (alert && pageable && !pagingSuppressed && isDeliveryBackoff(alert, now)) {
|
|
46
|
+
deliveryBackoffCount += 1;
|
|
47
|
+
}
|
|
48
|
+
if (pagingSuppressed)
|
|
49
|
+
suppressed += 1;
|
|
50
|
+
}
|
|
51
|
+
return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, actionableCriticalCount, suppressedCriticalCount, actionablePageableCount, notified: 0, notificationFailures: 0, deliveryBackoffCount, throttledNotifications: 0, skippedNotifications: 0, belowThreshold: 0, suppressed, muted, resolved: 0, maxSeverity });
|
|
52
|
+
}
|
|
53
|
+
const firingKeys = new Set(decisions.map((decision) => decision.dedupeKey));
|
|
54
|
+
const counts = emptyWatchdogDecisionCounts();
|
|
55
|
+
const serialSink = createSerialWatchdogSink(sinkPort);
|
|
56
|
+
await runWatchdogQueue(decisions, async (index) => {
|
|
57
|
+
const contribution = await processWatchdogDecision(decisions[index], {
|
|
58
|
+
ledgerPort, sinkPort: serialSink, now, pagingMinSeverity, notificationThrottleSeconds,
|
|
59
|
+
});
|
|
60
|
+
for (const key of Object.keys(contribution)) {
|
|
61
|
+
counts[key] += contribution[key];
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
// A rejected decision queue drains before throwing, so it never enters resolution.
|
|
65
|
+
const staleAlerts = openAlerts.filter((alert) => !firingKeys.has(alert.dedupeKey));
|
|
66
|
+
await runWatchdogQueue(staleAlerts, async (index) => {
|
|
67
|
+
await ledgerPort.resolveAlert(staleAlerts[index].dedupeKey, now);
|
|
68
|
+
});
|
|
69
|
+
return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, ...counts, muted, resolved: staleAlerts.length, maxSeverity });
|
|
70
|
+
}
|
|
71
|
+
function isMuted(dedupeKey, mutedPrefixes) {
|
|
72
|
+
return mutedPrefixes.some((prefix) => prefix.length > 0 && dedupeKey.startsWith(prefix));
|
|
73
|
+
}
|
|
74
|
+
function severityRank(severity) {
|
|
75
|
+
return { p0: 0, p1: 1, p2: 2, p3: 3 }[severity];
|
|
76
|
+
}
|
|
77
|
+
function maxSeverityOf(decisions) {
|
|
78
|
+
return decisions.reduce((current, decision) => current == null || severityRank(decision.severity) < severityRank(current) ? decision.severity : current, null);
|
|
79
|
+
}
|
|
80
|
+
function result({ checkOnly, checkedAt, decisions, firingCount, actionableCriticalCount, suppressedCriticalCount, actionablePageableCount, notified, notificationFailures, deliveryBackoffCount, throttledNotifications, skippedNotifications, belowThreshold, suppressed, muted, resolved, maxSeverity, }) {
|
|
81
|
+
return {
|
|
82
|
+
// Evaluation success is separate from the incident state. HTTP binding may
|
|
83
|
+
// still fail the request for evidence, notification transport, or config.
|
|
84
|
+
ok: true,
|
|
85
|
+
health: firingCount === 0 ? "healthy" : "firing",
|
|
86
|
+
checkOnly,
|
|
87
|
+
checkedAt,
|
|
88
|
+
alertCount: decisions.length,
|
|
89
|
+
firingCount,
|
|
90
|
+
actionableCriticalCount,
|
|
91
|
+
suppressedCriticalCount,
|
|
92
|
+
actionablePageableCount,
|
|
93
|
+
maxSeverity,
|
|
94
|
+
decisions,
|
|
95
|
+
notified,
|
|
96
|
+
notificationFailures,
|
|
97
|
+
deliveryBackoffCount,
|
|
98
|
+
throttledNotifications,
|
|
99
|
+
skippedNotifications,
|
|
100
|
+
belowThreshold,
|
|
101
|
+
suppressed,
|
|
102
|
+
muted,
|
|
103
|
+
resolved,
|
|
104
|
+
};
|
|
105
|
+
}
|