@openlup/platform 0.28.0 → 0.30.0

package/dist/operations/server/platformWatchdogQueue.js

@@ -0,0 +1,94 @@
+const WATCHDOG_CONCURRENCY = 4;
+const SEVERITY_RANK = { p0: 0, p1: 1, p2: 2, p3: 3 };
+/**
+ * Only each key's first unfinished item is eligible. A head heap prioritizes
+ * normalized severity without reordering the caller's decisions or duplicates.
+ */
+export async function runWatchdogQueue(items, process) {
+    const next = new Array(items.length).fill(-1);
+    const lastByKey = new Map();
+    const ready = [];
+    for (let index = 0; index < items.length; index += 1) {
+        const previous = lastByKey.get(items[index].dedupeKey);
+        if (previous === undefined)
+            push(index);
+        else
+            next[previous] = index;
+        lastByKey.set(items[index].dedupeKey, index);
+    }
+    let failed = false;
+    let firstError;
+    async function worker() {
+        while (!failed) {
+            const index = pop();
+            if (index === undefined)
+                return;
+            try {
+                await process(index);
+            }
+            catch (error) {
+                if (!failed)
+                    firstError = error;
+                failed = true;
+                return;
+            }
+            if (!failed && next[index] !== -1)
+                push(next[index]);
+        }
+    }
+    // Workers catch their own errors so every admitted chain settles before reject.
+    await Promise.all(Array.from({ length: WATCHDOG_CONCURRENCY }, () => worker()));
+    if (failed)
+        throw firstError;
+    function compare(left, right) {
+        const a = items[left];
+        const b = items[right];
+        return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity]
+            || Number(a.paging === "never") - Number(b.paging === "never")
+            || a.dedupeKey.localeCompare(b.dedupeKey)
+            || left - right;
+    }
+    function push(index) {
+        let position = ready.length;
+        ready.push(index);
+        while (position > 0) {
+            const parent = Math.floor((position - 1) / 2);
+            if (compare(ready[parent], index) <= 0)
+                break;
+            ready[position] = ready[parent];
+            position = parent;
+        }
+        ready[position] = index;
+    }
+    function pop() {
+        if (ready.length === 0)
+            return undefined;
+        const first = ready[0];
+        const last = ready.pop();
+        if (ready.length === 0)
+            return first;
+        let position = 0;
+        while (position * 2 + 1 < ready.length) {
+            let child = position * 2 + 1;
+            if (child + 1 < ready.length && compare(ready[child + 1], ready[child]) < 0)
+                child += 1;
+            if (compare(last, ready[child]) <= 0)
+                break;
+            ready[position] = ready[child];
+            position = child;
+        }
+        ready[position] = last;
+        return first;
+    }
+}
+/** Per-tick transport limit; admitted waiters still run if an earlier send throws. */
+export function createSerialWatchdogSink(sinkPort) {
+    let tail = Promise.resolve();
+    return {
+        send(decision, alert) {
+            const attempt = tail.then(() => sinkPort.send(decision, alert));
+            tail = attempt.then(() => undefined, () => undefined);
+            return attempt;
+        },
+    };
+}

package/dist/operations/server/platformWatchdogService.d.ts

@@ -0,0 +1,47 @@
+import type { AlertDecision, AlertSeverity, WatchdogSnapshot } from "../contracts/index.js";
+import type { AlertLedgerPort, AlertSinkPort, ObservabilityEvidencePort } from "../contracts/index.js";
+export type PlatformWatchdogResult<Channel extends string = string> = {
+    ok: boolean;
+    health: "healthy" | "firing";
+    checkOnly: boolean;
+    checkedAt: string;
+    alertCount: number;
+    firingCount: number;
+    /** P0/P1 incidents not covered by an active snooze and still requiring ownership. */
+    actionableCriticalCount: number;
+    /** P0/P1 incidents covered by an active snooze; acknowledged alerts stay actionable. */
+    suppressedCriticalCount: number;
+    /** Alerts that meet the current paging threshold and are not currently page-suppressed. */
+    actionablePageableCount: number;
+    maxSeverity: string | null;
+    decisions: AlertDecision<Channel>[];
+    notified: number;
+    notificationFailures: number;
+    /** Failed deliveries held until their persisted bounded retry time. */
+    deliveryBackoffCount: number;
+    throttledNotifications: number;
+    skippedNotifications: number;
+    belowThreshold: number;
+    suppressed: number;
+    muted: number;
+    resolved: number;
+};
+export declare function runPlatformWatchdog<Snapshot extends WatchdogSnapshot, CatalogEntry, Channel extends string = string>({ evidencePort, evaluate, ledgerPort, sinkPort, catalog, now, checkOnly, notificationThrottleSeconds, pagingMinSeverity, mutedDedupePrefixes, normalizeDecision, }: {
+    evidencePort: ObservabilityEvidencePort<Snapshot>;
+    evaluate: (snapshot: Snapshot, catalog: readonly CatalogEntry[], now: Date) => AlertDecision<Channel>[];
+    ledgerPort: AlertLedgerPort<Channel>;
+    sinkPort: AlertSinkPort<Channel>;
+    catalog: readonly CatalogEntry[];
+    now?: Date;
+    checkOnly: boolean;
+    /** Flat override; when omitted, the throttle is scaled per alert severity. */
+    notificationThrottleSeconds?: number;
+    pagingMinSeverity?: AlertSeverity;
+    /**
+     * Dedupe-key prefixes fully muted for this environment (e.g. `job_missed` on a
+     * staging host with no real crons). Muted signals are not recorded, not paged,
+     * and existing open rows for them auto-resolve — keeping the ledger clean.
+     */
+    mutedDedupePrefixes?: readonly string[];
+    normalizeDecision?: (decision: AlertDecision<Channel>) => AlertDecision<Channel>;
+}): Promise<PlatformWatchdogResult<Channel>>;

package/dist/operations/server/platformWatchdogService.js

@@ -0,0 +1,105 @@
+import { DEFAULT_PAGING_MIN_SEVERITY, shouldPage, } from "../index.js";
+import { isActionabilitySuppressed, isDeliveryBackoff, isPagingSuppressed, isSeverityEscalation, } from "./platformWatchdogDeliveryPolicy.js";
+import { emptyWatchdogDecisionCounts, isCriticalDecision, processWatchdogDecision, } from "./platformWatchdogDecision.js";
+import { createSerialWatchdogSink, runWatchdogQueue } from "./platformWatchdogQueue.js";
+export async function runPlatformWatchdog({ evidencePort, evaluate, ledgerPort, sinkPort, catalog, now = new Date(), checkOnly, notificationThrottleSeconds, pagingMinSeverity = DEFAULT_PAGING_MIN_SEVERITY, mutedDedupePrefixes = [], normalizeDecision, }) {
+    const snapshot = await evidencePort.collectSnapshot(now);
+    const allDecisions = evaluate(snapshot, catalog, now);
+    const decisions = allDecisions
+        .filter((decision) => !isMuted(decision.dedupeKey, mutedDedupePrefixes))
+        .map((decision) => normalizeDecision ? normalizeDecision(decision) : decision);
+    const muted = allDecisions.length - decisions.length;
+    const firingDecisions = decisions.filter((decision) => decision.paging !== "never");
+    const firingCount = firingDecisions.length;
+    const maxSeverity = maxSeverityOf(firingDecisions);
+    const openAlerts = await ledgerPort.listOpenAlerts();
+    if (checkOnly) {
+        const alertsByDedupeKey = new Map(openAlerts.map((alert) => [alert.dedupeKey, alert]));
+        let actionableCriticalCount = 0;
+        let suppressedCriticalCount = 0;
+        let actionablePageableCount = 0;
+        let deliveryBackoffCount = 0;
+        let suppressed = 0;
+        for (const decision of decisions) {
+            const alert = alertsByDedupeKey.get(decision.dedupeKey);
+            // The mutating path clears acknowledgement, snooze and delivery cadence on
+            // escalation. Mirror that transition here so a read-only probe cannot
+            // report a stale suppression that the next real tick will remove.
+            const suppressionSurvives = alert ? !isSeverityEscalation(decision, alert) : false;
+            const actionableSuppressed = alert && suppressionSurvives
+                ? isActionabilitySuppressed(alert, now)
+                : false;
+            const pagingSuppressed = alert && suppressionSurvives
+                ? isPagingSuppressed(alert, now)
+                : false;
+            const critical = isCriticalDecision(decision);
+            const pageable = decision.paging !== "never" && shouldPage(decision.severity, pagingMinSeverity);
+            if (critical) {
+                if (actionableSuppressed)
+                    suppressedCriticalCount += 1;
+                else
+                    actionableCriticalCount += 1;
+            }
+            if (pageable && !pagingSuppressed)
+                actionablePageableCount += 1;
+            if (alert && pageable && !pagingSuppressed && isDeliveryBackoff(alert, now)) {
+                deliveryBackoffCount += 1;
+            }
+            if (pagingSuppressed)
+                suppressed += 1;
+        }
+        return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, actionableCriticalCount, suppressedCriticalCount, actionablePageableCount, notified: 0, notificationFailures: 0, deliveryBackoffCount, throttledNotifications: 0, skippedNotifications: 0, belowThreshold: 0, suppressed, muted, resolved: 0, maxSeverity });
+    }
+    const firingKeys = new Set(decisions.map((decision) => decision.dedupeKey));
+    const counts = emptyWatchdogDecisionCounts();
+    const serialSink = createSerialWatchdogSink(sinkPort);
+    await runWatchdogQueue(decisions, async (index) => {
+        const contribution = await processWatchdogDecision(decisions[index], {
+            ledgerPort, sinkPort: serialSink, now, pagingMinSeverity, notificationThrottleSeconds,
+        });
+        for (const key of Object.keys(contribution)) {
+            counts[key] += contribution[key];
+        }
+    });
+    // A rejected decision queue drains before throwing, so it never enters resolution.
+    const staleAlerts = openAlerts.filter((alert) => !firingKeys.has(alert.dedupeKey));
+    await runWatchdogQueue(staleAlerts, async (index) => {
+        await ledgerPort.resolveAlert(staleAlerts[index].dedupeKey, now);
+    });
+    return result({ checkOnly, checkedAt: snapshot.checkedAt, decisions, firingCount, ...counts, muted, resolved: staleAlerts.length, maxSeverity });
+}
+function isMuted(dedupeKey, mutedPrefixes) {
+    return mutedPrefixes.some((prefix) => prefix.length > 0 && dedupeKey.startsWith(prefix));
+}
+function severityRank(severity) {
+    return { p0: 0, p1: 1, p2: 2, p3: 3 }[severity];
+}
+function maxSeverityOf(decisions) {
+    return decisions.reduce((current, decision) => current == null || severityRank(decision.severity) < severityRank(current) ? decision.severity : current, null);
+}
+function result({ checkOnly, checkedAt, decisions, firingCount, actionableCriticalCount, suppressedCriticalCount, actionablePageableCount, notified, notificationFailures, deliveryBackoffCount, throttledNotifications, skippedNotifications, belowThreshold, suppressed, muted, resolved, maxSeverity, }) {
+    return {
+        // Evaluation success is separate from the incident state. HTTP binding may
+        // still fail the request for evidence, notification transport, or config.
+        ok: true,
+        health: firingCount === 0 ? "healthy" : "firing",
+        checkOnly,
+        checkedAt,
+        alertCount: decisions.length,
+        firingCount,
+        actionableCriticalCount,
+        suppressedCriticalCount,
+        actionablePageableCount,
+        maxSeverity,
+        decisions,
+        notified,
+        notificationFailures,
+        deliveryBackoffCount,
+        throttledNotifications,
+        skippedNotifications,
+        belowThreshold,
+        suppressed,
+        muted,
+        resolved,
+    };
+}

package/dist/operations/validation.d.ts

@@ -0,0 +1,2 @@
+import { z } from "zod";
+export { z };

package/dist/operations/validation.js

@@ -0,0 +1,3 @@
+import { z } from "zod";
+z.config({ jitless: true });
+export { z };

package/dist/portability/filesystemObjects.d.ts

@@ -0,0 +1,12 @@
+import { type TransferObjectStore } from "./objects.js";
+/**
+ * Private local transfer store, exclusively controlled by its owner process/account.
+ * Metadata and bytes publish together through one directory rename. Concurrent use through
+ * this adapter is supported; hostile same-UID filesystem mutation is outside that guarantee.
+ */
+export declare function createFilesystemTransferObjects(options: {
+    root: string;
+    maximumObjectBytes: number;
+    /** Host label for this filesystem owner's private-read policy; other ACLs refuse. */
+    access: string;
+}): Promise<TransferObjectStore>;

package/dist/portability/filesystemObjects.js

@@ -0,0 +1,191 @@
+import { createHash, randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, realpath, rename, rm } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { encodeTransferObject, normalizeObjectTransferError, ObjectTransferError, validateTransferObject, } from "./objects.js";
+// Covers bounded metadata JSON in UTF-8 plus JSON-escaped bounded identifiers.
+const maximumDescriptorBytes = 262144;
+/**
+ * Private local transfer store, exclusively controlled by its owner process/account.
+ * Metadata and bytes publish together through one directory rename. Concurrent use through
+ * this adapter is supported; hostile same-UID filesystem mutation is outside that guarantee.
+ */
+export async function createFilesystemTransferObjects(options) {
+    if (!isAbsolute(options.root) || !Number.isSafeInteger(options.maximumObjectBytes) || options.maximumObjectBytes < 0
+        || typeof options.access !== "string" || !options.access.trim()) {
+        throw new ObjectTransferError("invalid_input");
+    }
+    const root = resolve(options.root), maximum = options.maximumObjectBytes, access = options.access;
+    try {
+        if (await realpath(dirname(root)) !== dirname(root))
+            throw new ObjectTransferError("denied");
+        await mkdir(root, { mode: 0o700 }).catch(error => { if (errno(error) !== "EEXIST")
+            throw error; });
+        await privateDirectory(root, true);
+    }
+    catch (error) {
+        throw filesystemFailure(error);
+    }
+    const location = (key) => join(root, createHash("sha256").update(JSON.stringify(key)).digest("hex"));
+    return {
+        async read(key, expectedVersion) {
+            let body;
+            let published = false;
+            try {
+                await privateDirectory(root, true);
+                const directory = location(key);
+                await privateDirectory(directory);
+                published = true;
+                const metadata = await openPrivateFile(join(directory, "metadata.json"));
+                let object;
+                try {
+                    if ((await metadata.stat()).size > maximumDescriptorBytes)
+                        throw new ObjectTransferError("integrity_conflict");
+                    try {
+                        object = JSON.parse(await metadata.readFile("utf8"));
+                    }
+                    catch {
+                        throw new ObjectTransferError("integrity_conflict");
+                    }
+                }
+                finally {
+                    await metadata.close();
+                }
+                validateTransferObject(object);
+                if (object.access !== access)
+                    throw new ObjectTransferError("denied");
+                if (object.key !== key || (expectedVersion !== undefined && object.version !== expectedVersion))
+                    throw new ObjectTransferError("version_conflict");
+                if (object.bytes > maximum)
+                    throw new ObjectTransferError("invalid_input");
+                body = await openPrivateFile(join(directory, "bytes"));
+                if ((await body.stat()).size !== object.bytes)
+                    throw new ObjectTransferError("integrity_conflict");
+                await body.close();
+                body = undefined;
+                // Open lazily: descriptor refusal must not leave an unconsumed stream holding a file.
+                return { object, body: (async function* () {
+                        let opened;
+                        try {
+                            await privateDirectory(root, true);
+                            await privateDirectory(directory);
+                            opened = await openPrivateFile(join(directory, "bytes"));
+                            if ((await opened.stat()).size !== object.bytes)
+                                throw new ObjectTransferError("integrity_conflict");
+                            for await (const chunk of opened.createReadStream({ autoClose: false }))
+                                yield Buffer.from(chunk);
+                        }
+                        catch (error) {
+                            throw errno(error) === "ENOENT" ? new ObjectTransferError("integrity_conflict") : filesystemFailure(error);
+                        }
+                        finally {
+                            await opened?.close();
+                        }
+                    })() };
+            }
+            catch (error) {
+                await body?.close();
+                if (published && errno(error) === "ENOENT")
+                    throw new ObjectTransferError("integrity_conflict");
+                throw filesystemFailure(error);
+            }
+        },
+        async createIfAbsent(object, bytes) {
+            let temporary;
+            try {
+                const encoded = Buffer.from(encodeTransferObject(object));
+                if (encoded.byteLength > maximumDescriptorBytes)
+                    throw new ObjectTransferError("invalid_input");
+                if (object.access !== access)
+                    throw new ObjectTransferError("denied");
+                const key = object.key;
+                if (object.bytes > maximum || bytes.byteLength !== object.bytes
+                    || createHash("sha256").update(bytes).digest("hex") !== object.sha256)
+                    throw new ObjectTransferError("integrity_conflict");
+                await privateDirectory(root, true);
+                temporary = join(root, `.partial-${randomUUID()}`);
+                await mkdir(temporary, { mode: 0o700 });
+                await writePrivateFile(join(temporary, "metadata.json"), encoded);
+                await writePrivateFile(join(temporary, "bytes"), bytes);
+                const directoryHandle = await open(temporary, constants.O_RDONLY | constants.O_NOFOLLOW);
+                try {
+                    await directoryHandle.sync();
+                }
+                finally {
+                    await directoryHandle.close();
+                }
+                try {
+                    // Existing published objects are nonempty directories, so rename cannot replace them.
+                    // Check also refuses an empty directory or symlink occupying a target key.
+                    await lstat(location(key));
+                    throw new ObjectTransferError("version_conflict");
+                }
+                catch (error) {
+                    if (errno(error) !== "ENOENT")
+                        throw error;
+                }
+                await rename(temporary, location(key));
+                temporary = undefined;
+                const rootHandle = await open(root, constants.O_RDONLY | constants.O_NOFOLLOW);
+                try {
+                    await rootHandle.sync();
+                }
+                finally {
+                    await rootHandle.close();
+                }
+            }
+            catch (error) {
+                throw filesystemFailure(error);
+            }
+            finally {
+                if (temporary)
+                    await rm(temporary, { recursive: true, force: true });
+            }
+        },
+    };
+}
+async function privateDirectory(path, storeRoot = false) {
+    const stat = await lstat(path).catch(error => {
+        if (storeRoot && errno(error) === "ENOENT")
+            throw new ObjectTransferError("unavailable");
+        throw error;
+    });
+    if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o777) !== 0o700 || stat.uid !== process.getuid?.()) {
+        throw new ObjectTransferError("denied");
+    }
+}
+async function openPrivateFile(path) {
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o777) !== 0o600 || stat.uid !== process.getuid?.())
+            throw new ObjectTransferError("denied");
+        return handle;
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
+async function writePrivateFile(path, bytes) {
+    const handle = await open(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+    try {
+        await handle.writeFile(bytes);
+        await handle.sync();
+    }
+    finally {
+        await handle.close();
+    }
+}
+function errno(error) { return error?.code; }
+function filesystemFailure(error) {
+    if (error instanceof ObjectTransferError)
+        return error;
+    if (errno(error) === "ENOENT")
+        return new ObjectTransferError("absent");
+    if (["EACCES", "EPERM", "ELOOP"].includes(errno(error) ?? ""))
+        return new ObjectTransferError("denied");
+    if (["EEXIST", "ENOTEMPTY"].includes(errno(error) ?? ""))
+        return new ObjectTransferError("version_conflict");
+    return normalizeObjectTransferError(error);
+}

package/dist/portability/index.d.ts

@@ -0,0 +1,5 @@
+export { copyTransferObjects, ObjectTransferError } from "./objects.js";
+export type { ObjectTransferFailure, TransferObject, TransferObjectRead, TransferObjectReader, TransferObjectStore, TransferObjectInventory, } from "./objects.js";
+export { createFilesystemTransferObjects } from "./filesystemObjects.js";
+export { captureCompatiblePostgresRows, restoreCompatiblePostgresRows, PostgresTransferError } from "./postgresRows.js";
+export type { CompatiblePostgresTable, TransferPostgresQuery, CompatiblePostgresSnapshot, CompatiblePostgresTarget, CompatiblePostgresPacket, PostgresTransferFailure, } from "./postgresRows.js";

package/dist/portability/index.js

@@ -0,0 +1,4 @@
+// Server-only, opt-in data movement. No executor, route or handover is activated.
+export { copyTransferObjects, ObjectTransferError } from "./objects.js";
+export { createFilesystemTransferObjects } from "./filesystemObjects.js";
+export { captureCompatiblePostgresRows, restoreCompatiblePostgresRows, PostgresTransferError } from "./postgresRows.js";

package/dist/portability/objects.d.ts

@@ -0,0 +1,63 @@
+export type ObjectTransferFailure = "absent" | "denied" | "unavailable" | "version_conflict" | "integrity_conflict" | "invalid_input";
+/** Adapters distinguish missing data from permission and availability failures. */
+export declare class ObjectTransferError extends Error {
+    readonly code: ObjectTransferFailure;
+    constructor(code: ObjectTransferFailure);
+}
+export interface TransferObject {
+    key: string;
+    /** Logical immutable source version; never a claim about a target provider's native version ID. */
+    version: string;
+    bytes: number;
+    sha256: string;
+    metadata: Readonly<Record<string, string>>;
+    /** An explicit access classification, never inferred from a public URL. */
+    access: string;
+    lineage?: {
+        sourceId: string;
+        snapshotId: string;
+        sourceVersion: string;
+    };
+}
+export interface TransferObjectRead {
+    object: TransferObject;
+    body: AsyncIterable<Uint8Array>;
+}
+export interface TransferObjectReader {
+    /** Must read the requested version or refuse, never silently read a newer one. */
+    read(key: string, expectedVersion?: string): Promise<TransferObjectRead>;
+}
+export interface TransferObjectStore extends TransferObjectReader {
+    /** Atomic publication; an existing key must refuse with version_conflict. */
+    createIfAbsent(object: TransferObject, bytes: Uint8Array): Promise<void>;
+}
+export interface TransferObjectInventory {
+    sourceId: string;
+    /** Host prerequisite: complete inventory and stable versioned reads, or a held write fence. */
+    snapshot: {
+        kind: "immutable" | "quiesced";
+        id: string;
+    };
+    objects: readonly TransferObject[];
+}
+/** Canonical descriptor encoding also preserves metadata independent of property order. */
+export declare function encodeTransferObject(object: TransferObject): string;
+export declare function validateTransferObject(object: TransferObject): void;
+/** Bounded in-memory object copy; ceilings are explicit and checked before any destination mutation. */
+export declare function copyTransferObjects(input: {
+    inventory: TransferObjectInventory;
+    source: TransferObjectReader;
+    destination: TransferObjectStore;
+    accessMapping: Readonly<Record<string, string>>;
+    maximumObjectBytes: number;
+    maximumTotalBytes: number;
+    /** Called only after destination descriptor and bytes have been read back successfully. */
+    checkpoint?: (object: TransferObject) => Promise<void>;
+}): Promise<{
+    copied: number;
+    replayed: number;
+    verifiedKeys: string[];
+}>;
+/** Read exactly the declared bytes and hash; never trust a successful HEAD alone. */
+export declare function verifyRead(read: TransferObjectRead, expected: TransferObject): Promise<Uint8Array>;
+export declare function normalizeObjectTransferError(error: unknown): ObjectTransferError;