@openlup/platform 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -138
- package/README.md +80 -3
- package/dist/composition/nodeHost.d.ts +1 -1
- package/dist/composition/nodeHost.js +3 -2
- package/dist/lifecycle/appUpgrade.d.ts +52 -0
- package/dist/lifecycle/appUpgrade.js +180 -0
- package/dist/lifecycle/index.d.ts +2 -0
- package/dist/lifecycle/index.js +2 -0
- package/dist/lifecycle/postgresInstallQualification.d.ts +51 -0
- package/dist/lifecycle/postgresInstallQualification.js +74 -0
- package/dist/migrations/postgresRunner.d.ts +9 -0
- package/dist/migrations/postgresRunner.js +17 -10
- package/dist/operations/adminAlertsSchemas.d.ts +153 -0
- package/dist/operations/adminAlertsSchemas.js +69 -0
- package/dist/operations/adminAlertsView.d.ts +73 -0
- package/dist/operations/adminAlertsView.js +184 -0
- package/dist/operations/alertPagingPolicy.d.ts +15 -0
- package/dist/operations/alertPagingPolicy.js +43 -0
- package/dist/operations/alertThrottlePolicy.d.ts +3 -0
- package/dist/operations/alertThrottlePolicy.js +18 -0
- package/dist/operations/clients.d.ts +17 -0
- package/dist/operations/clients.js +32 -0
- package/dist/operations/contracts/admin.d.ts +93 -0
- package/dist/operations/contracts/admin.js +1 -0
- package/dist/operations/contracts/adminAlertsContracts.d.ts +40 -0
- package/dist/operations/contracts/adminAlertsContracts.js +1 -0
- package/dist/operations/contracts/alertHumanContext.d.ts +7 -0
- package/dist/operations/contracts/alertHumanContext.js +1 -0
- package/dist/operations/contracts/index.d.ts +6 -0
- package/dist/operations/contracts/index.js +6 -0
- package/dist/operations/contracts/observabilityContracts.d.ts +41 -0
- package/dist/operations/contracts/observabilityContracts.js +1 -0
- package/dist/operations/contracts/observabilityPorts.d.ts +18 -0
- package/dist/operations/contracts/observabilityPorts.js +1 -0
- package/dist/operations/contracts/ports.d.ts +19 -0
- package/dist/operations/contracts/ports.js +1 -0
- package/dist/operations/errors.d.ts +6 -0
- package/dist/operations/errors.js +6 -0
- package/dist/operations/index.d.ts +9 -0
- package/dist/operations/index.js +9 -0
- package/dist/operations/schemas.d.ts +138 -0
- package/dist/operations/schemas.js +104 -0
- package/dist/operations/server/adminAlertsHandlers.d.ts +16 -0
- package/dist/operations/server/adminAlertsHandlers.js +35 -0
- package/dist/operations/server/adminPipelineHandlers.d.ts +12 -0
- package/dist/operations/server/adminPipelineHandlers.js +48 -0
- package/dist/operations/server/adminSettingsHandlers.d.ts +14 -0
- package/dist/operations/server/adminSettingsHandlers.js +82 -0
- package/dist/operations/server/controlPlaneHandler.d.ts +7 -0
- package/dist/operations/server/controlPlaneHandler.js +54 -0
- package/dist/operations/server/controlPlaneStore.d.ts +12 -0
- package/dist/operations/server/controlPlaneStore.js +138 -0
- package/dist/operations/server/controlPlaneWatchdog.d.ts +7 -0
- package/dist/operations/server/controlPlaneWatchdog.js +20 -0
- package/dist/operations/server/index.d.ts +12 -0
- package/dist/operations/server/index.js +12 -0
- package/dist/operations/server/migrations.d.ts +14 -0
- package/dist/operations/server/migrations.js +7 -0
- package/dist/operations/server/platformWatchdogDecision.d.ts +34 -0
- package/dist/operations/server/platformWatchdogDecision.js +71 -0
- package/dist/operations/server/platformWatchdogDeliveryPolicy.d.ts +10 -0
- package/dist/operations/server/platformWatchdogDeliveryPolicy.js +53 -0
- package/dist/operations/server/platformWatchdogQueue.d.ts +11 -0
- package/dist/operations/server/platformWatchdogQueue.js +94 -0
- package/dist/operations/server/platformWatchdogService.d.ts +47 -0
- package/dist/operations/server/platformWatchdogService.js +105 -0
- package/dist/operations/validation.d.ts +2 -0
- package/dist/operations/validation.js +3 -0
- package/dist/portability/filesystemObjects.d.ts +12 -0
- package/dist/portability/filesystemObjects.js +191 -0
- package/dist/portability/index.d.ts +5 -0
- package/dist/portability/index.js +4 -0
- package/dist/portability/objects.d.ts +63 -0
- package/dist/portability/objects.js +138 -0
- package/dist/portability/postgresRows.d.ts +64 -0
- package/dist/portability/postgresRows.js +182 -0
- package/dist/scheduler/advisoryLock.js +20 -2
- package/dist/scheduler/nodeScheduler.d.ts +2 -2
- package/dist/scheduler/nodeScheduler.js +35 -6
- package/package.json +27 -2
- package/release-gates.json +73 -7
- package/sql/20260814010000_platform_control_plane.sql +286 -0
- package/src/composition/nodeHost.ts +4 -3
- package/src/lifecycle/appUpgrade.ts +186 -0
- package/src/lifecycle/index.ts +19 -0
- package/src/lifecycle/postgresInstallQualification.ts +125 -0
- package/src/migrations/postgresRunner.ts +24 -9
- package/src/operations/adminAlertsSchemas.ts +78 -0
- package/src/operations/adminAlertsView.ts +238 -0
- package/src/operations/alertPagingPolicy.ts +50 -0
- package/src/operations/alertThrottlePolicy.ts +22 -0
- package/src/operations/clients.ts +44 -0
- package/src/operations/contracts/admin.ts +24 -0
- package/src/operations/contracts/adminAlertsContracts.ts +13 -0
- package/src/operations/contracts/alertHumanContext.ts +17 -0
- package/src/operations/contracts/index.ts +6 -0
- package/src/operations/contracts/observabilityContracts.ts +40 -0
- package/src/operations/contracts/observabilityPorts.ts +30 -0
- package/src/operations/contracts/ports.ts +22 -0
- package/src/operations/errors.ts +6 -0
- package/src/operations/index.ts +9 -0
- package/src/operations/schemas.ts +122 -0
- package/src/operations/server/adminAlertsHandlers.ts +61 -0
- package/src/operations/server/adminPipelineHandlers.ts +73 -0
- package/src/operations/server/adminSettingsHandlers.ts +114 -0
- package/src/operations/server/controlPlaneHandler.ts +56 -0
- package/src/operations/server/controlPlaneStore.ts +150 -0
- package/src/operations/server/controlPlaneWatchdog.ts +28 -0
- package/src/operations/server/index.ts +12 -0
- package/src/operations/server/migrations.ts +7 -0
- package/src/operations/server/platformWatchdogDecision.ts +93 -0
- package/src/operations/server/platformWatchdogDeliveryPolicy.ts +92 -0
- package/src/operations/server/platformWatchdogQueue.ts +96 -0
- package/src/operations/server/platformWatchdogService.ts +209 -0
- package/src/operations/validation.ts +3 -0
- package/src/portability/filesystemObjects.ts +138 -0
- package/src/portability/index.ts +12 -0
- package/src/portability/objects.ts +160 -0
- package/src/portability/postgresRows.ts +169 -0
- package/src/scheduler/advisoryLock.ts +20 -5
- package/src/scheduler/nodeScheduler.ts +28 -8
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AlertDecision,
|
|
3
|
+
AlertNotificationOutcome,
|
|
4
|
+
OpenAlert,
|
|
5
|
+
} from "../contracts/index.js";
|
|
6
|
+
|
|
7
|
+
const FAILURE_RETRY_INITIAL_SECONDS = 10 * 60;
|
|
8
|
+
const FAILURE_RETRY_MAX_SECONDS = 60 * 60;
|
|
9
|
+
|
|
10
|
+
export function shouldAttemptNotification(
|
|
11
|
+
alert: Pick<
|
|
12
|
+
OpenAlert,
|
|
13
|
+
"lastNotifiedAt" | "lastNotificationStatus" | "nextNotificationAttemptAt" | "snoozedUntil"
|
|
14
|
+
>,
|
|
15
|
+
now: Date,
|
|
16
|
+
throttleSeconds: number,
|
|
17
|
+
): boolean {
|
|
18
|
+
const snoozeExpiry = alert.snoozedUntil ? Date.parse(alert.snoozedUntil) : Number.NaN;
|
|
19
|
+
const lastSentAt = alert.lastNotifiedAt ? Date.parse(alert.lastNotifiedAt) : Number.NaN;
|
|
20
|
+
if (
|
|
21
|
+
alert.lastNotificationStatus !== "failed"
|
|
22
|
+
&& Number.isFinite(snoozeExpiry)
|
|
23
|
+
&& snoozeExpiry <= now.getTime()
|
|
24
|
+
&& (!Number.isFinite(lastSentAt) || snoozeExpiry > lastSentAt)
|
|
25
|
+
) {
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
28
|
+
const nextAttemptAt = alert.nextNotificationAttemptAt
|
|
29
|
+
? Date.parse(alert.nextNotificationAttemptAt)
|
|
30
|
+
: Number.NaN;
|
|
31
|
+
if (Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime()) return false;
|
|
32
|
+
if (!alert.lastNotifiedAt) return true;
|
|
33
|
+
return now.getTime() - new Date(alert.lastNotifiedAt).getTime() >= throttleSeconds * 1000;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function notificationSchedule(
|
|
37
|
+
alert: Pick<OpenAlert, "notificationFailureCount">,
|
|
38
|
+
outcome: AlertNotificationOutcome,
|
|
39
|
+
now: Date,
|
|
40
|
+
throttleSeconds: number,
|
|
41
|
+
): { nextAttemptAt: Date; failureCount: number } {
|
|
42
|
+
if (outcome.status === "failed") {
|
|
43
|
+
const failureCount = (alert.notificationFailureCount ?? 0) + 1;
|
|
44
|
+
const exponent = Math.max(0, Math.min(failureCount - 1, 16));
|
|
45
|
+
const retrySeconds = Math.min(
|
|
46
|
+
FAILURE_RETRY_MAX_SECONDS,
|
|
47
|
+
FAILURE_RETRY_INITIAL_SECONDS * 2 ** exponent,
|
|
48
|
+
);
|
|
49
|
+
return {
|
|
50
|
+
nextAttemptAt: new Date(now.getTime() + retrySeconds * 1000),
|
|
51
|
+
failureCount,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return {
|
|
56
|
+
// Sent uses the severity cadence. Skipped uses the same due field only to
|
|
57
|
+
// avoid duplicate audit rows; last_notified_at still advances on sent only.
|
|
58
|
+
nextAttemptAt: new Date(now.getTime() + throttleSeconds * 1000),
|
|
59
|
+
failureCount: 0,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function isActionabilitySuppressed(
|
|
64
|
+
alert: Pick<OpenAlert, "snoozedUntil">,
|
|
65
|
+
now: Date,
|
|
66
|
+
): boolean {
|
|
67
|
+
return alert.snoozedUntil != null && new Date(alert.snoozedUntil).getTime() > now.getTime();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function isPagingSuppressed(
|
|
71
|
+
alert: Pick<OpenAlert, "status" | "snoozedUntil">,
|
|
72
|
+
now: Date,
|
|
73
|
+
): boolean {
|
|
74
|
+
return alert.status === "acknowledged" || isActionabilitySuppressed(alert, now);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function isDeliveryBackoff(
|
|
78
|
+
alert: Pick<OpenAlert, "lastNotificationStatus" | "nextNotificationAttemptAt">,
|
|
79
|
+
now: Date,
|
|
80
|
+
): boolean {
|
|
81
|
+
if (alert.lastNotificationStatus !== "failed" || !alert.nextNotificationAttemptAt) return false;
|
|
82
|
+
const nextAttemptAt = Date.parse(alert.nextNotificationAttemptAt);
|
|
83
|
+
return Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime();
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function isSeverityEscalation(
|
|
87
|
+
decision: Pick<AlertDecision, "severity">,
|
|
88
|
+
alert: Pick<OpenAlert, "severity">,
|
|
89
|
+
): boolean {
|
|
90
|
+
const rank = { p0: 0, p1: 1, p2: 2, p3: 3 };
|
|
91
|
+
return rank[decision.severity] < rank[alert.severity];
|
|
92
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { AlertDecision } from "../contracts/index.js";
|
|
2
|
+
import type { AlertSinkPort } from "../contracts/index.js";
|
|
3
|
+
|
|
4
|
+
type WatchdogQueueItem = Pick<AlertDecision, "dedupeKey" | "severity" | "paging">;
|
|
5
|
+
const WATCHDOG_CONCURRENCY = 4;
|
|
6
|
+
const SEVERITY_RANK = { p0: 0, p1: 1, p2: 2, p3: 3 };
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Only each key's first unfinished item is eligible. A head heap prioritizes
|
|
10
|
+
* normalized severity without reordering the caller's decisions or duplicates.
|
|
11
|
+
*/
|
|
12
|
+
export async function runWatchdogQueue(
|
|
13
|
+
items: readonly WatchdogQueueItem[],
|
|
14
|
+
process: (index: number) => Promise<void>,
|
|
15
|
+
): Promise<void> {
|
|
16
|
+
const next = new Array<number>(items.length).fill(-1);
|
|
17
|
+
const lastByKey = new Map<string, number>();
|
|
18
|
+
const ready: number[] = [];
|
|
19
|
+
for (let index = 0; index < items.length; index += 1) {
|
|
20
|
+
const previous = lastByKey.get(items[index].dedupeKey);
|
|
21
|
+
if (previous === undefined) push(index);
|
|
22
|
+
else next[previous] = index;
|
|
23
|
+
lastByKey.set(items[index].dedupeKey, index);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let failed = false;
|
|
27
|
+
let firstError: unknown;
|
|
28
|
+
async function worker() {
|
|
29
|
+
while (!failed) {
|
|
30
|
+
const index = pop();
|
|
31
|
+
if (index === undefined) return;
|
|
32
|
+
try {
|
|
33
|
+
await process(index);
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (!failed) firstError = error;
|
|
36
|
+
failed = true;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
if (!failed && next[index] !== -1) push(next[index]);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Workers catch their own errors so every admitted chain settles before reject.
|
|
44
|
+
await Promise.all(Array.from({ length: WATCHDOG_CONCURRENCY }, () => worker()));
|
|
45
|
+
if (failed) throw firstError;
|
|
46
|
+
|
|
47
|
+
function compare(left: number, right: number): number {
|
|
48
|
+
const a = items[left];
|
|
49
|
+
const b = items[right];
|
|
50
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity]
|
|
51
|
+
|| Number(a.paging === "never") - Number(b.paging === "never")
|
|
52
|
+
|| a.dedupeKey.localeCompare(b.dedupeKey)
|
|
53
|
+
|| left - right;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function push(index: number) {
|
|
57
|
+
let position = ready.length;
|
|
58
|
+
ready.push(index);
|
|
59
|
+
while (position > 0) {
|
|
60
|
+
const parent = Math.floor((position - 1) / 2);
|
|
61
|
+
if (compare(ready[parent], index) <= 0) break;
|
|
62
|
+
ready[position] = ready[parent];
|
|
63
|
+
position = parent;
|
|
64
|
+
}
|
|
65
|
+
ready[position] = index;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function pop(): number | undefined {
|
|
69
|
+
if (ready.length === 0) return undefined;
|
|
70
|
+
const first = ready[0];
|
|
71
|
+
const last = ready.pop()!;
|
|
72
|
+
if (ready.length === 0) return first;
|
|
73
|
+
let position = 0;
|
|
74
|
+
while (position * 2 + 1 < ready.length) {
|
|
75
|
+
let child = position * 2 + 1;
|
|
76
|
+
if (child + 1 < ready.length && compare(ready[child + 1], ready[child]) < 0) child += 1;
|
|
77
|
+
if (compare(last, ready[child]) <= 0) break;
|
|
78
|
+
ready[position] = ready[child];
|
|
79
|
+
position = child;
|
|
80
|
+
}
|
|
81
|
+
ready[position] = last;
|
|
82
|
+
return first;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Per-tick transport limit; admitted waiters still run if an earlier send throws. */
|
|
87
|
+
export function createSerialWatchdogSink<Channel extends string>(sinkPort: AlertSinkPort<Channel>): AlertSinkPort<Channel> {
|
|
88
|
+
let tail = Promise.resolve();
|
|
89
|
+
return {
|
|
90
|
+
send(decision, alert) {
|
|
91
|
+
const attempt = tail.then(() => sinkPort.send(decision, alert));
|
|
92
|
+
tail = attempt.then(() => undefined, () => undefined);
|
|
93
|
+
return attempt;
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AlertDecision,
|
|
3
|
+
AlertSeverity,
|
|
4
|
+
WatchdogSnapshot,
|
|
5
|
+
} from "../contracts/index.js";
|
|
6
|
+
import {
|
|
7
|
+
DEFAULT_PAGING_MIN_SEVERITY,
|
|
8
|
+
shouldPage,
|
|
9
|
+
} from "../index.js";
|
|
10
|
+
import type {
|
|
11
|
+
AlertLedgerPort,
|
|
12
|
+
AlertSinkPort,
|
|
13
|
+
ObservabilityEvidencePort,
|
|
14
|
+
} from "../contracts/index.js";
|
|
15
|
+
import {
|
|
16
|
+
isActionabilitySuppressed,
|
|
17
|
+
isDeliveryBackoff,
|
|
18
|
+
isPagingSuppressed,
|
|
19
|
+
isSeverityEscalation,
|
|
20
|
+
} from "./platformWatchdogDeliveryPolicy.js";
|
|
21
|
+
|
|
22
|
+
import {
|
|
23
|
+
emptyWatchdogDecisionCounts,
|
|
24
|
+
isCriticalDecision,
|
|
25
|
+
processWatchdogDecision,
|
|
26
|
+
} from "./platformWatchdogDecision.js";
|
|
27
|
+
import { createSerialWatchdogSink, runWatchdogQueue } from "./platformWatchdogQueue.js";
|
|
28
|
+
|
|
29
|
+
export type PlatformWatchdogResult<Channel extends string = string> = {
|
|
30
|
+
ok: boolean;
|
|
31
|
+
health: "healthy" | "firing";
|
|
32
|
+
checkOnly: boolean;
|
|
33
|
+
checkedAt: string;
|
|
34
|
+
alertCount: number;
|
|
35
|
+
firingCount: number;
|
|
36
|
+
/** P0/P1 incidents not covered by an active snooze and still requiring ownership. */
|
|
37
|
+
actionableCriticalCount: number;
|
|
38
|
+
/** P0/P1 incidents covered by an active snooze; acknowledged alerts stay actionable. */
|
|
39
|
+
suppressedCriticalCount: number;
|
|
40
|
+
/** Alerts that meet the current paging threshold and are not currently page-suppressed. */
|
|
41
|
+
actionablePageableCount: number;
|
|
42
|
+
maxSeverity: string | null;
|
|
43
|
+
decisions: AlertDecision<Channel>[];
|
|
44
|
+
notified: number;
|
|
45
|
+
notificationFailures: number;
|
|
46
|
+
/** Failed deliveries held until their persisted bounded retry time. */
|
|
47
|
+
deliveryBackoffCount: number;
|
|
48
|
+
throttledNotifications: number;
|
|
49
|
+
skippedNotifications: number;
|
|
50
|
+
belowThreshold: number;
|
|
51
|
+
suppressed: number;
|
|
52
|
+
muted: number;
|
|
53
|
+
resolved: number;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
export async function runPlatformWatchdog<Snapshot extends WatchdogSnapshot, CatalogEntry, Channel extends string = string>({
|
|
57
|
+
evidencePort, evaluate,
|
|
58
|
+
ledgerPort,
|
|
59
|
+
sinkPort,
|
|
60
|
+
catalog,
|
|
61
|
+
now = new Date(),
|
|
62
|
+
checkOnly,
|
|
63
|
+
notificationThrottleSeconds,
|
|
64
|
+
pagingMinSeverity = DEFAULT_PAGING_MIN_SEVERITY,
|
|
65
|
+
mutedDedupePrefixes = [],
|
|
66
|
+
normalizeDecision,
|
|
67
|
+
}: {
|
|
68
|
+
evidencePort: ObservabilityEvidencePort<Snapshot>;
|
|
69
|
+
evaluate: (snapshot: Snapshot, catalog: readonly CatalogEntry[], now: Date) => AlertDecision<Channel>[];
|
|
70
|
+
ledgerPort: AlertLedgerPort<Channel>;
|
|
71
|
+
sinkPort: AlertSinkPort<Channel>;
|
|
72
|
+
catalog: readonly CatalogEntry[];
|
|
73
|
+
now?: Date;
|
|
74
|
+
checkOnly: boolean;
|
|
75
|
+
/** Flat override; when omitted, the throttle is scaled per alert severity. */
|
|
76
|
+
notificationThrottleSeconds?: number;
|
|
77
|
+
pagingMinSeverity?: AlertSeverity;
|
|
78
|
+
/**
|
|
79
|
+
* Dedupe-key prefixes fully muted for this environment (e.g. `job_missed` on a
|
|
80
|
+
* staging host with no real crons). Muted signals are not recorded, not paged,
|
|
81
|
+
* and existing open rows for them auto-resolve — keeping the ledger clean.
|
|
82
|
+
*/
|
|
83
|
+
mutedDedupePrefixes?: readonly string[];
|
|
84
|
+
normalizeDecision?: (decision: AlertDecision<Channel>) => AlertDecision<Channel>;
|
|
85
|
+
}): Promise<PlatformWatchdogResult<Channel>> {
|
|
86
|
+
const snapshot = await evidencePort.collectSnapshot(now);
|
|
87
|
+
const allDecisions = evaluate(snapshot, catalog, now);
|
|
88
|
+
const decisions = allDecisions
|
|
89
|
+
.filter((decision) => !isMuted(decision.dedupeKey, mutedDedupePrefixes))
|
|
90
|
+
.map((decision) => normalizeDecision ? normalizeDecision(decision) : decision);
|
|
91
|
+
const muted = allDecisions.length - decisions.length;
|
|
92
|
+
const firingDecisions = decisions.filter((decision) => decision.paging !== "never");
|
|
93
|
+
const firingCount = firingDecisions.length;
|
|
94
|
+
const maxSeverity = maxSeverityOf(firingDecisions);
|
|
95
|
+
const openAlerts = await ledgerPort.listOpenAlerts();
|
|
96
|
+
|
|
97
|
+
if (checkOnly) {
|
|
98
|
+
const alertsByDedupeKey = new Map(openAlerts.map((alert) => [alert.dedupeKey, alert]));
|
|
99
|
+
let actionableCriticalCount = 0;
|
|
100
|
+
let suppressedCriticalCount = 0;
|
|
101
|
+
let actionablePageableCount = 0;
|
|
102
|
+
let deliveryBackoffCount = 0;
|
|
103
|
+
let suppressed = 0;
|
|
104
|
+
for (const decision of decisions) {
|
|
105
|
+
const alert = alertsByDedupeKey.get(decision.dedupeKey);
|
|
106
|
+
// The mutating path clears acknowledgement, snooze and delivery cadence on
|
|
107
|
+
// escalation. Mirror that transition here so a read-only probe cannot
|
|
108
|
+
// report a stale suppression that the next real tick will remove.
|
|
109
|
+
const suppressionSurvives = alert ? !isSeverityEscalation(decision, alert) : false;
|
|
110
|
+
const actionableSuppressed = alert && suppressionSurvives
|
|
111
|
+
? isActionabilitySuppressed(alert, now)
|
|
112
|
+
: false;
|
|
113
|
+
const pagingSuppressed = alert && suppressionSurvives
|
|
114
|
+
? isPagingSuppressed(alert, now)
|
|
115
|
+
: false;
|
|
116
|
+
const critical = isCriticalDecision(decision);
|
|
117
|
+
const pageable = decision.paging !== "never" && shouldPage(decision.severity, pagingMinSeverity);
|
|
118
|
+
if (critical) {
|
|
119
|
+
if (actionableSuppressed) suppressedCriticalCount += 1;
|
|
120
|
+
else actionableCriticalCount += 1;
|
|
121
|
+
}
|
|
122
|
+
if (pageable && !pagingSuppressed) actionablePageableCount += 1;
|
|
123
|
+
if (alert && pageable && !pagingSuppressed && isDeliveryBackoff(alert, now)) {
|
|
124
|
+
deliveryBackoffCount += 1;
|
|
125
|
+
}
|
|
126
|
+
if (pagingSuppressed) suppressed += 1;
|
|
127
|
+
}
|
|
128
|
+
return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, actionableCriticalCount, suppressedCriticalCount, actionablePageableCount, notified: 0, notificationFailures: 0, deliveryBackoffCount, throttledNotifications: 0, skippedNotifications: 0, belowThreshold: 0, suppressed, muted, resolved: 0, maxSeverity });
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const firingKeys = new Set(decisions.map((decision) => decision.dedupeKey));
|
|
132
|
+
const counts = emptyWatchdogDecisionCounts();
|
|
133
|
+
const serialSink = createSerialWatchdogSink(sinkPort);
|
|
134
|
+
await runWatchdogQueue(decisions, async (index) => {
|
|
135
|
+
const contribution = await processWatchdogDecision(decisions[index], {
|
|
136
|
+
ledgerPort, sinkPort: serialSink, now, pagingMinSeverity, notificationThrottleSeconds,
|
|
137
|
+
});
|
|
138
|
+
for (const key of Object.keys(contribution) as (keyof typeof counts)[]) {
|
|
139
|
+
counts[key] += contribution[key];
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// A rejected decision queue drains before throwing, so it never enters resolution.
|
|
144
|
+
const staleAlerts = openAlerts.filter((alert) => !firingKeys.has(alert.dedupeKey));
|
|
145
|
+
await runWatchdogQueue(staleAlerts, async (index) => {
|
|
146
|
+
await ledgerPort.resolveAlert(staleAlerts[index].dedupeKey, now);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, ...counts, muted, resolved: staleAlerts.length, maxSeverity });
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function isMuted(dedupeKey: string, mutedPrefixes: readonly string[]): boolean {
|
|
153
|
+
return mutedPrefixes.some((prefix) => prefix.length > 0 && dedupeKey.startsWith(prefix));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function severityRank(severity: AlertSeverity): number {
|
|
157
|
+
return { p0: 0, p1: 1, p2: 2, p3: 3 }[severity];
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function maxSeverityOf(decisions: readonly AlertDecision[]): AlertSeverity | null {
|
|
161
|
+
return decisions.reduce<AlertSeverity | null>((current, decision) =>
|
|
162
|
+
current == null || severityRank(decision.severity) < severityRank(current) ? decision.severity : current
|
|
163
|
+
, null);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function result<Channel extends string>({
|
|
167
|
+
checkOnly,
|
|
168
|
+
checkedAt,
|
|
169
|
+
decisions,
|
|
170
|
+
firingCount,
|
|
171
|
+
actionableCriticalCount,
|
|
172
|
+
suppressedCriticalCount,
|
|
173
|
+
actionablePageableCount,
|
|
174
|
+
notified,
|
|
175
|
+
notificationFailures,
|
|
176
|
+
deliveryBackoffCount,
|
|
177
|
+
throttledNotifications,
|
|
178
|
+
skippedNotifications,
|
|
179
|
+
belowThreshold,
|
|
180
|
+
suppressed,
|
|
181
|
+
muted,
|
|
182
|
+
resolved,
|
|
183
|
+
maxSeverity,
|
|
184
|
+
}: Omit<PlatformWatchdogResult<Channel>, "ok" | "health" | "alertCount">): PlatformWatchdogResult<Channel> {
|
|
185
|
+
return {
|
|
186
|
+
// Evaluation success is separate from the incident state. HTTP binding may
|
|
187
|
+
// still fail the request for evidence, notification transport, or config.
|
|
188
|
+
ok: true,
|
|
189
|
+
health: firingCount === 0 ? "healthy" : "firing",
|
|
190
|
+
checkOnly,
|
|
191
|
+
checkedAt,
|
|
192
|
+
alertCount: decisions.length,
|
|
193
|
+
firingCount,
|
|
194
|
+
actionableCriticalCount,
|
|
195
|
+
suppressedCriticalCount,
|
|
196
|
+
actionablePageableCount,
|
|
197
|
+
maxSeverity,
|
|
198
|
+
decisions,
|
|
199
|
+
notified,
|
|
200
|
+
notificationFailures,
|
|
201
|
+
deliveryBackoffCount,
|
|
202
|
+
throttledNotifications,
|
|
203
|
+
skippedNotifications,
|
|
204
|
+
belowThreshold,
|
|
205
|
+
suppressed,
|
|
206
|
+
muted,
|
|
207
|
+
resolved,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { constants } from "node:fs";
|
|
3
|
+
import { lstat, mkdir, open, realpath, rename, rm, type FileHandle } from "node:fs/promises";
|
|
4
|
+
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
encodeTransferObject, normalizeObjectTransferError, ObjectTransferError, validateTransferObject,
|
|
7
|
+
type TransferObject, type TransferObjectRead, type TransferObjectStore,
|
|
8
|
+
} from "./objects.js";
|
|
9
|
+
|
|
10
|
+
// Covers bounded metadata JSON in UTF-8 plus JSON-escaped bounded identifiers.
|
|
11
|
+
const maximumDescriptorBytes = 262144;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Private local transfer store, exclusively controlled by its owner process/account.
|
|
15
|
+
* Metadata and bytes publish together through one directory rename. Concurrent use through
|
|
16
|
+
* this adapter is supported; hostile same-UID filesystem mutation is outside that guarantee.
|
|
17
|
+
*/
|
|
18
|
+
export async function createFilesystemTransferObjects(options: {
|
|
19
|
+
root: string;
|
|
20
|
+
maximumObjectBytes: number;
|
|
21
|
+
/** Host label for this filesystem owner's private-read policy; other ACLs refuse. */
|
|
22
|
+
access: string;
|
|
23
|
+
}): Promise<TransferObjectStore> {
|
|
24
|
+
if (!isAbsolute(options.root) || !Number.isSafeInteger(options.maximumObjectBytes) || options.maximumObjectBytes < 0
|
|
25
|
+
|| typeof options.access !== "string" || !options.access.trim()) {
|
|
26
|
+
throw new ObjectTransferError("invalid_input");
|
|
27
|
+
}
|
|
28
|
+
const root = resolve(options.root), maximum = options.maximumObjectBytes, access = options.access;
|
|
29
|
+
try {
|
|
30
|
+
if (await realpath(dirname(root)) !== dirname(root)) throw new ObjectTransferError("denied");
|
|
31
|
+
await mkdir(root, { mode: 0o700 }).catch(error => { if (errno(error) !== "EEXIST") throw error; });
|
|
32
|
+
await privateDirectory(root, true);
|
|
33
|
+
} catch (error) { throw filesystemFailure(error); }
|
|
34
|
+
const location = (key: string) => join(root, createHash("sha256").update(JSON.stringify(key)).digest("hex"));
|
|
35
|
+
return {
|
|
36
|
+
async read(key, expectedVersion): Promise<TransferObjectRead> {
|
|
37
|
+
let body: FileHandle | undefined;
|
|
38
|
+
let published = false;
|
|
39
|
+
try {
|
|
40
|
+
await privateDirectory(root, true);
|
|
41
|
+
const directory = location(key);
|
|
42
|
+
await privateDirectory(directory);
|
|
43
|
+
published = true;
|
|
44
|
+
const metadata = await openPrivateFile(join(directory, "metadata.json"));
|
|
45
|
+
let object: TransferObject;
|
|
46
|
+
try {
|
|
47
|
+
if ((await metadata.stat()).size > maximumDescriptorBytes) throw new ObjectTransferError("integrity_conflict");
|
|
48
|
+
try { object = JSON.parse(await metadata.readFile("utf8")) as TransferObject; }
|
|
49
|
+
catch { throw new ObjectTransferError("integrity_conflict"); }
|
|
50
|
+
} finally { await metadata.close(); }
|
|
51
|
+
validateTransferObject(object);
|
|
52
|
+
if (object.access !== access) throw new ObjectTransferError("denied");
|
|
53
|
+
if (object.key !== key || (expectedVersion !== undefined && object.version !== expectedVersion)) throw new ObjectTransferError("version_conflict");
|
|
54
|
+
if (object.bytes > maximum) throw new ObjectTransferError("invalid_input");
|
|
55
|
+
body = await openPrivateFile(join(directory, "bytes"));
|
|
56
|
+
if ((await body.stat()).size !== object.bytes) throw new ObjectTransferError("integrity_conflict");
|
|
57
|
+
await body.close();
|
|
58
|
+
body = undefined;
|
|
59
|
+
// Open lazily: descriptor refusal must not leave an unconsumed stream holding a file.
|
|
60
|
+
return { object, body: (async function* () {
|
|
61
|
+
let opened: FileHandle | undefined;
|
|
62
|
+
try {
|
|
63
|
+
await privateDirectory(root, true);
|
|
64
|
+
await privateDirectory(directory);
|
|
65
|
+
opened = await openPrivateFile(join(directory, "bytes"));
|
|
66
|
+
if ((await opened.stat()).size !== object.bytes) throw new ObjectTransferError("integrity_conflict");
|
|
67
|
+
for await (const chunk of opened.createReadStream({ autoClose: false })) yield Buffer.from(chunk);
|
|
68
|
+
}
|
|
69
|
+
catch (error) { throw errno(error) === "ENOENT" ? new ObjectTransferError("integrity_conflict") : filesystemFailure(error); }
|
|
70
|
+
finally { await opened?.close(); }
|
|
71
|
+
})() };
|
|
72
|
+
} catch (error) {
|
|
73
|
+
await body?.close();
|
|
74
|
+
if (published && errno(error) === "ENOENT") throw new ObjectTransferError("integrity_conflict");
|
|
75
|
+
throw filesystemFailure(error);
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
async createIfAbsent(object, bytes) {
|
|
79
|
+
let temporary: string | undefined;
|
|
80
|
+
try {
|
|
81
|
+
const encoded = Buffer.from(encodeTransferObject(object));
|
|
82
|
+
if (encoded.byteLength > maximumDescriptorBytes) throw new ObjectTransferError("invalid_input");
|
|
83
|
+
if (object.access !== access) throw new ObjectTransferError("denied");
|
|
84
|
+
const key = object.key;
|
|
85
|
+
if (object.bytes > maximum || bytes.byteLength !== object.bytes
|
|
86
|
+
|| createHash("sha256").update(bytes).digest("hex") !== object.sha256) throw new ObjectTransferError("integrity_conflict");
|
|
87
|
+
await privateDirectory(root, true);
|
|
88
|
+
temporary = join(root, `.partial-${randomUUID()}`);
|
|
89
|
+
await mkdir(temporary, { mode: 0o700 });
|
|
90
|
+
await writePrivateFile(join(temporary, "metadata.json"), encoded);
|
|
91
|
+
await writePrivateFile(join(temporary, "bytes"), bytes);
|
|
92
|
+
const directoryHandle = await open(temporary, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
93
|
+
try { await directoryHandle.sync(); } finally { await directoryHandle.close(); }
|
|
94
|
+
try {
|
|
95
|
+
// Existing published objects are nonempty directories, so rename cannot replace them.
|
|
96
|
+
// Check also refuses an empty directory or symlink occupying a target key.
|
|
97
|
+
await lstat(location(key));
|
|
98
|
+
throw new ObjectTransferError("version_conflict");
|
|
99
|
+
} catch (error) { if (errno(error) !== "ENOENT") throw error; }
|
|
100
|
+
await rename(temporary, location(key));
|
|
101
|
+
temporary = undefined;
|
|
102
|
+
const rootHandle = await open(root, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
103
|
+
try { await rootHandle.sync(); } finally { await rootHandle.close(); }
|
|
104
|
+
} catch (error) { throw filesystemFailure(error); }
|
|
105
|
+
finally { if (temporary) await rm(temporary, { recursive: true, force: true }); }
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function privateDirectory(path: string, storeRoot = false): Promise<void> {
|
|
111
|
+
const stat = await lstat(path).catch(error => {
|
|
112
|
+
if (storeRoot && errno(error) === "ENOENT") throw new ObjectTransferError("unavailable");
|
|
113
|
+
throw error;
|
|
114
|
+
});
|
|
115
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o777) !== 0o700 || stat.uid !== process.getuid?.()) {
|
|
116
|
+
throw new ObjectTransferError("denied");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
async function openPrivateFile(path: string): Promise<FileHandle> {
|
|
120
|
+
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
121
|
+
try {
|
|
122
|
+
const stat = await handle.stat();
|
|
123
|
+
if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o777) !== 0o600 || stat.uid !== process.getuid?.()) throw new ObjectTransferError("denied");
|
|
124
|
+
return handle;
|
|
125
|
+
} catch (error) { await handle.close(); throw error; }
|
|
126
|
+
}
|
|
127
|
+
async function writePrivateFile(path: string, bytes: Uint8Array): Promise<void> {
|
|
128
|
+
const handle = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
129
|
+
try { await handle.writeFile(bytes); await handle.sync(); } finally { await handle.close(); }
|
|
130
|
+
}
|
|
131
|
+
function errno(error: unknown): string | undefined { return (error as NodeJS.ErrnoException | undefined)?.code; }
|
|
132
|
+
function filesystemFailure(error: unknown): ObjectTransferError {
|
|
133
|
+
if (error instanceof ObjectTransferError) return error;
|
|
134
|
+
if (errno(error) === "ENOENT") return new ObjectTransferError("absent");
|
|
135
|
+
if (["EACCES", "EPERM", "ELOOP"].includes(errno(error) ?? "")) return new ObjectTransferError("denied");
|
|
136
|
+
if (["EEXIST", "ENOTEMPTY"].includes(errno(error) ?? "")) return new ObjectTransferError("version_conflict");
|
|
137
|
+
return normalizeObjectTransferError(error);
|
|
138
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// Server-only, opt-in data movement. No executor, route or handover is activated.
|
|
2
|
+
export { copyTransferObjects, ObjectTransferError } from "./objects.js";
|
|
3
|
+
export type {
|
|
4
|
+
ObjectTransferFailure, TransferObject, TransferObjectRead, TransferObjectReader,
|
|
5
|
+
TransferObjectStore, TransferObjectInventory,
|
|
6
|
+
} from "./objects.js";
|
|
7
|
+
export { createFilesystemTransferObjects } from "./filesystemObjects.js";
|
|
8
|
+
export { captureCompatiblePostgresRows, restoreCompatiblePostgresRows, PostgresTransferError } from "./postgresRows.js";
|
|
9
|
+
export type {
|
|
10
|
+
CompatiblePostgresTable, TransferPostgresQuery, CompatiblePostgresSnapshot,
|
|
11
|
+
CompatiblePostgresTarget, CompatiblePostgresPacket, PostgresTransferFailure,
|
|
12
|
+
} from "./postgresRows.js";
|