@smthrs/control 0.0.0-stage → 1.0.0-rc.4

package/dist/esm/Monitor.js

@@ -0,0 +1,415 @@
+/**
+ * Run health over the control plane: what a run's state means, and what to do
+ * about it.
+ *
+ * A control plane answers "what is this run doing". A monitor answers the
+ * question after it — "is that all right, and if not, what now" — and the two
+ * are deliberately separate modules. {@link classify} is a pure function of
+ * one observation, so the vocabulary an operator reads on a dashboard is the
+ * same one a heal loop branches on and the same one a test can enumerate.
+ * {@link run} is the loop that beats over `Control` and journals what it saw.
+ *
+ * The classification is decided from durable evidence only: the run summary
+ * the control plane projects, and the journal entries `watch` replays. Nothing
+ * here reads an in-process fiber, which is what makes a monitor able to watch
+ * a run in another process at all.
+ *
+ * @since 0.1.0
+ */
+import { Journal, JournalEvent } from "@smthrs/journal";
+import { Duration, Effect, Stream } from "effect";
+import { Control } from "./Control.js";
+import { PersistenceError } from "./ControlError.js";
+import * as SubjectHealth from "./Health.js";
+/**
+ * The journal event type the engine records when an action attempt starts.
+ *
+ * Named as a string rather than imported, exactly as in `Lineage`: a monitor
+ * reads journals, not engines.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const attemptStartedEventType = "flows.engine.attempt-started";
+/**
+ * The journal event type the engine records when an action attempt settles.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const attemptFinishedEventType = "flows.engine.attempt-finished";
+/**
+ * The journal event type one monitor beat is recorded under.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const beatEventType = "control.monitor.beat";
+/**
+ * The journal event type one applied remedy is recorded under.
+ *
+ * A beat and a heal are two records because they are two facts, and only one
+ * of them is true before the remedy runs. A single record carrying `healed`
+ * has to be written either before the remedy — leaving durable evidence of a
+ * heal that a crash one instruction later never performed — or after it, which
+ * loses the evidence of what a monitor decided when the remedy is what killed
+ * it. Splitting them keeps both.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const healedEventType = "control.monitor.healed";
+/**
+ * What a run looks like to a monitor.
+ *
+ * `healthy` covers both "moving" and "finished": a completed run needs nothing
+ * done to it, and neither does one that is making progress. The other six name
+ * a specific thing that is wrong, because a heal that cannot tell them apart
+ * would resume a run that is waiting for a human.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const Health = SubjectHealth.HealthState;
+/** How many rounds a trampoline may take before a monitor calls it runaway. */
+const defaultRoundBound = 32;
+const foldAttempt = (state, event) => {
+    if (event.kind === attemptStartedEventType)
+        state.open += 1;
+    if (event.kind === attemptFinishedEventType) {
+        state.open -= 1;
+        const payload = event.payload;
+        state.failed = typeof payload === "object" && payload !== null && !Array.isArray(payload) &&
+            payload["state"] === "failed";
+    }
+};
+/**
+ * Decides what a run's state means.
+ *
+ * The order is the order an operator would read it in, and each rule earns its
+ * place by naming a different response:
+ *
+ * | Condition | Health | Because |
+ * | --- | --- | --- |
+ * | No summary | `unknown` | Nothing to say, and nothing to do. |
+ * | `failed` | `failing` | The run itself reported the failure. |
+ * | `completed`, `cancelled` | `healthy` | A finished run needs nothing. |
+ * | `waiting-approval`, or parked on `approval` | `awaiting-human` | A human owes it an answer; no machine can supply one. |
+ * | Parked with no waiting reason | `awaiting-human` | Only an operator's own park writes no waiting reason, so a person stopped it and a person restarts it. |
+ * | `roundOrdinal` at or past the bound | `runaway-loop` | The lineage is looping without converging. |
+ * | The last settled attempt failed | `failing` | The run is still alive but its work is not landing. |
+ * | No progress for `stallBeats`, an attempt open | `wedged-node` | One attempt started and never settled: the work is stuck, not the run. |
+ * | No progress for `stallBeats` | `stalled` | Nothing is happening and nothing is in flight. |
+ * | Anything else | `healthy` | Entries are still arriving. |
+ *
+ * `awaiting-human` deliberately outranks `failing`: a run parked for approval
+ * after a failed attempt is waiting for a person, and resuming or cancelling
+ * it would take the decision away from them.
+ *
+ * A park with no reason is the same case. The engine names every park it makes
+ * — `event`, `approval`, `timer`, `quota`, `released` — so a parked run whose
+ * `waitingReason` is absent was parked by an operator through
+ * `ControlRuntime.writeStatus`. Calling that a stall and resuming it would undo
+ * a deliberate act, which is the worst thing an unattended heal loop can do.
+ *
+ * @param observation what the beat saw
+ * @category projections
+ * @since 0.1.0
+ */
+export const classify = (observation) => {
+    const attempts = { open: 0, failed: false };
+    for (const event of observation.events)
+        foldAttempt(attempts, event);
+    return classifyState(observation, attempts);
+};
+const classifyState = (observation, attempts) => {
+    const summary = observation.summary;
+    if (summary === undefined)
+        return "unknown";
+    if (summary.status === "failed")
+        return "failing";
+    if (summary.status === "completed" || summary.status === "cancelled")
+        return "healthy";
+    if (summary.status === "waiting-approval")
+        return "awaiting-human";
+    if (summary.status === "parked" && (summary.waitingReason === "approval" || summary.waitingReason === undefined)) {
+        return "awaiting-human";
+    }
+    if (SubjectHealth.waitReason(summary.status, summary.waitingReason) !== undefined)
+        return "healthy";
+    const bound = observation.roundBound ?? defaultRoundBound;
+    if (summary.roundOrdinal !== undefined && summary.roundOrdinal >= bound)
+        return "runaway-loop";
+    if (attempts.failed)
+        return "failing";
+    if (!observation.semanticProgress && observation.beatsWithoutProgress >= observation.stallBeats) {
+        return attempts.open > 0 ? "wedged-node" : "stalled";
+    }
+    return "healthy";
+};
+/**
+ * The remedy a health warrants, before `autoHeal` decides whether to apply it.
+ *
+ * A stalled or wedged run is one nobody is driving, and a resume claims it. A
+ * failing or runaway run is one that will not get better by being driven
+ * harder. Everything else is left alone.
+ *
+ * @param health what the beat classified
+ * @category projections
+ * @since 0.1.0
+ */
+export const remedyFor = (health) => {
+    switch (health) {
+        case "stalled":
+        case "wedged-node":
+            return "resume";
+        case "failing":
+        case "runaway-loop":
+            return "cancel";
+        default:
+            return "none";
+    }
+};
+const summaryOf = (runId) => Effect.flatMap(Control, (control) => control.list({ _tag: "runs", filters: { runId } }).pipe(Effect.map((listed) => listed._tag === "runs" ? listed.items[0] : undefined)));
+const terminal = (summary) => summary !== undefined &&
+    (summary.status === "completed" || summary.status === "failed" || summary.status === "cancelled");
+/**
+ * Watches one run and, when told to, heals it.
+ *
+ * The loop stops early on a terminal run, because a finished run has nothing
+ * left to observe and beating at it would fill its journal with heartbeats.
+ * Otherwise it takes `maxChecks` beats and reports what it saw.
+ *
+ * Every beat is journaled as `control.monitor.beat` before any remedy is
+ * applied, carrying the `remedy` it is about to attempt, so a monitor that
+ * crashes mid-heal leaves the evidence of what it decided. The remedy itself
+ * is journaled separately as `control.monitor.healed`, and only once the heal
+ * returned a receipt: a heal that failed, or a process that died running one,
+ * must not leave a durable record saying the run was healed.
+ *
+ * `autoHeal` is empty by default. A monitor that healed by default would be a
+ * monitor that cancels a run the first time it looks at one, which is the
+ * wrong default for a thing an operator points at production.
+ *
+ * Nothing here leases the run. Two monitors on one run both beat and both
+ * remedy, so a remedy has to be idempotent on the control plane — which the
+ * two defaults are, through `Control.resume` and `Control.cancel`
+ * idempotency keys that name the monitor, the run, and the beat. A custom
+ * `heal` owes the same property.
+ *
+ * @param options how to beat and what to heal
+ * @category constructors
+ * @since 0.1.0
+ */
+export const run = (options) => Effect.gen(function* () {
+    const control = yield* Control;
+    const journal = yield* Journal.Journal;
+    const intervalMs = options.intervalMs ?? options.healthCheck?.policy.intervalMs ?? 1_000;
+    let consecutiveProbeFailures = 0;
+    let lastPublished;
+    const maxChecks = options.maxChecks ?? 10;
+    const stallBeats = options.stallBeats ?? 3;
+    const autoHeal = options.autoHeal ?? [];
+    const monitorId = options.monitorId ?? "default";
+    const heal = options.heal ?? ((input) => input.remedy === "resume"
+        ? control.resume({
+            runId: input.runId,
+            idempotencyKey: `monitor:${monitorId}:resume:${input.runId}:${input.beat}`
+        })
+        : control.cancel({
+            runId: input.runId,
+            reason: `monitor:${input.health}`,
+            idempotencyKey: `monitor:${monitorId}:cancel:${input.runId}:${input.beat}`
+        }));
+    const emit = (eventType, payload, operation) => {
+        const unrecorded = (cause) => new PersistenceError({
+            operation: eventType,
+            message: `Failed to record ${operation} for ${options.runId}`,
+            cause
+        });
+        // The record is BUILT inside the failure channel, not before it.
+        // `JournalEvent.RunId`/`SourceId` refuse an identifier the store cannot
+        // tell apart from another one — a lone UTF-16 surrogate, an embedded NUL
+        // — by THROWING, and `monitorId` reaches here from an operator's flag. A
+        // thrown constructor is a defect, which the RPC boundary reports as an
+        // opaque `TransportError` and a caller cannot handle; a beat whose record
+        // cannot be built is the same event as a beat whose record cannot be
+        // appended, so both fail as `PersistenceError` naming the record.
+        return Effect.try({
+            try: () => new JournalEvent.Input({
+                runId: JournalEvent.RunId.make(options.runId),
+                sourceId: JournalEvent.SourceId.make(`/control/monitor/${monitorId}`),
+                eventType,
+                payload: { runId: options.runId, monitorId, ...payload }
+            }),
+            catch: unrecorded
+        }).pipe(Effect.flatMap((input) => Effect.mapError(journal.emitDurableUnfenced(input), unrecorded)), Effect.asVoid);
+    };
+    /** What the beat saw, and what it is about to do about it. */
+    const record = (beat, remedy) => emit(beatEventType, {
+        beat: beat.beat,
+        health: beat.health,
+        sequence: beat.sequence,
+        ...(remedy === "none" ? {} : { remedy })
+    }, `monitor beat ${beat.beat}`);
+    /** What the remedy did, once it returned. */
+    const recordHealed = (beat, remedy, receipt) => emit(healedEventType, { beat: beat.beat, health: beat.health, healed: remedy, receipt: receipt._tag }, `monitor heal ${beat.beat}`);
+    const beats = [];
+    let beatsWithoutProgress = 0;
+    let lastSequence = -1;
+    let checkpoint = {};
+    const attempts = { open: 0, failed: false };
+    let sequence = -1;
+    for (let beat = 0; beat < maxChecks; beat += 1) {
+        if (beat > 0 && intervalMs > 0) {
+            yield* Effect.sleep(Duration.millis(options.healthCheck === undefined
+                ? intervalMs
+                : SubjectHealth.nextDelay(options.healthCheck.policy, consecutiveProbeFailures)));
+        }
+        const summary = yield* summaryOf(options.runId);
+        const newEvents = [];
+        const sinceCursor = Math.max(0, sequence);
+        yield* control.watch({ runId: options.runId, follow: false, ...checkpoint }).pipe(Stream.runForEach((event) => Effect.sync(() => {
+            // Advance over bookkeeping too, but never count it as run progress.
+            checkpoint = event.cursor === undefined
+                ? { afterSequence: event.sequence }
+                : { afterCursor: event.cursor };
+            if (isBookkeepingEvent(event.kind))
+                return;
+            newEvents.push(event);
+            if (newEvents.length > 256)
+                newEvents.shift();
+            sequence = event.sequence;
+            foldAttempt(attempts, event);
+        })));
+        beatsWithoutProgress = sequence === lastSequence ? beatsWithoutProgress + 1 : 0;
+        lastSequence = sequence;
+        const health = classifyState({
+            ...(summary === undefined ? {} : { summary }),
+            beatsWithoutProgress,
+            stallBeats,
+            ...(options.roundBound === undefined ? {} : { roundBound: options.roundBound })
+        }, attempts);
+        if (options.healthCheck !== undefined && summary !== undefined) {
+            const check = options.healthCheck;
+            const subjectId = `run:${options.runId}`;
+            const incarnation = SubjectHealth.runIncarnation(summary);
+            const probe = SubjectHealth.evaluate(check, {
+                subjectId,
+                state: summary.status,
+                summary,
+                events: newEvents,
+                sinceCursor
+            }, { monitorId, incarnation, evidenceSeq: Math.max(0, sequence) });
+            let observation = yield* (options.withProbePermit?.(probe) ?? probe);
+            const current = yield* summaryOf(options.runId);
+            if (current === undefined || SubjectHealth.runIncarnation(current) !== incarnation) {
+                observation = { ...observation, outcome: "discarded", report: undefined, reason: "owner-changed" };
+            }
+            consecutiveProbeFailures = observation.outcome === "ok" ? 0 : consecutiveProbeFailures + 1;
+            const semanticProgress = observation.outcome === "ok" && observation.report?.activity === "working";
+            const baseHealth = classifyState({
+                summary,
+                beatsWithoutProgress,
+                stallBeats,
+                roundBound: options.roundBound,
+                semanticProgress
+            }, attempts);
+            observation = { ...observation, baseHealth };
+            const status = SubjectHealth.rollup({
+                subjectId,
+                state: summary.status,
+                incarnation,
+                waitingReason: summary.waitingReason,
+                baseHealth,
+                latest: { observation, sequence: 0 },
+                now: observation.observedAt,
+                updatedAt: summary.updatedAt
+            });
+            const changed = lastPublished === undefined || lastPublished.incarnation !== observation.incarnation ||
+                lastPublished.outcome !== observation.outcome || lastPublished.baseHealth !== observation.baseHealth ||
+                lastPublished.evidenceSeq !== observation.evidenceSeq ||
+                JSON.stringify(lastPublished.report) !== JSON.stringify(observation.report) ||
+                lastPublished.reason !== observation.reason;
+            // Renew before expiry, but do not fill the journal with identical per-probe records.
+            if (changed || observation.observedAt >= lastPublished.observedAt + check.policy.ttlMs / 2) {
+                yield* emit(SubjectHealth.statusObservedEventType, {
+                    ...observation,
+                    status: summary.status,
+                    health: status.health,
+                    activity: status.activity,
+                    attention: status.attention,
+                    freshness: status.freshness,
+                    waitingReason: summary.waitingReason ?? ""
+                }, "health observation");
+                lastPublished = observation;
+            }
+        }
+        // Checker output is observational and never participates in remedy authorization.
+        const remedy = autoHeal.includes(health) ? remedyFor(health) : "none";
+        const observed = { beat, health, sequence };
+        if (options.recordBeats !== false)
+            yield* record(observed, remedy);
+        if (remedy === "none") {
+            beats.push(observed);
+        }
+        else {
+            const receipt = yield* heal({ runId: options.runId, health, remedy, beat }).pipe(
+            // A remedy that fails must not abort the run of beats: the monitor
+            // is an unattended loop, and one failing heal leaves every other
+            // beat — and every other run it would have observed — unwatched.
+            // The failure is logged with the beat it failed on, and the beat is
+            // recorded without `healed` or `receipt`, exactly as a remedy the
+            // monitor never got to apply reports.
+            Effect.catch((failure) => Effect.annotateLogs(Effect.logWarning("A monitor remedy failed and was skipped"), { runId: options.runId, beat, health, remedy, cause: String(failure) }).pipe(Effect.as(undefined))));
+            if (receipt === undefined) {
+                // The stall evidence stands: nothing provably moved the run, so the
+                // next beat compares against the same progress mark and the remedy
+                // is retried with the beat's own idempotency key.
+                beats.push(observed);
+            }
+            else {
+                // A remedy that returned is not a remedy that was applied. `Terminal`
+                // says the run had already settled, so nothing was healed; `Conflict`
+                // says the key belonged to another mutation, so this monitor's remedy
+                // never ran. Recording either as `healed` claimed something that did
+                // not happen, and resetting the stall count on either erased the
+                // evidence the next beat needs to notice the run is still stuck.
+                const applied = receipt._tag === "Accepted" || receipt._tag === "AlreadyApplied";
+                if (applied) {
+                    // Journaled here and not a line earlier: `healed` is a claim about
+                    // something that happened, and until the receipt came back it had not.
+                    yield* recordHealed(observed, remedy, receipt);
+                    // The heal moved the run, so the next beat compares against a run
+                    // that has changed. Counting the beats before it as stall evidence
+                    // again would heal a second time for the same stall.
+                    beatsWithoutProgress = 0;
+                }
+                beats.push(applied ? { ...observed, healed: remedy, receipt } : { ...observed, receipt });
+                // A remedy that answered `Terminal` observed the run settle. The loop
+                // ends on the same evidence a terminal summary ends it on, rather than
+                // beating against a run nothing can move.
+                if (receipt._tag === "Terminal")
+                    break;
+            }
+        }
+        if (options.retainBeats !== undefined && beats.length > options.retainBeats) {
+            beats.splice(0, beats.length - options.retainBeats);
+        }
+        if (terminal(summary)) {
+            break;
+        }
+    }
+    return {
+        runId: options.runId,
+        beats,
+        health: beats[beats.length - 1]?.health ?? "unknown"
+    };
+});
+/** Observer and notification bookkeeping is never execution progress.
+ * @category predicates
+ * @since 1.0.0
+ */
+export const isBookkeepingEvent = (kind) => kind.startsWith("control.monitor.") || kind.startsWith("control.status.") ||
+    kind.startsWith("flows.alerts.") || kind.startsWith("flows.notifications.") || kind.startsWith("flows.notification.");
+//# sourceMappingURL=Monitor.js.map

package/dist/esm/Monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Monitor.js","sourceRoot":"","sources":["../../src/Monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEpD,OAAO,KAAK,aAAa,MAAM,aAAa,CAAA;AAE5C;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,8BAA8B,CAAA;AAErE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,+BAA+B,CAAA;AAEvE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,sBAAsB,CAAA;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,wBAAwB,CAAA;AAEvD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,CAAA;AAqC/C,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAO5B,MAAM,WAAW,GAAG,CAAC,KAAmB,EAAE,KAAmB,EAAQ,EAAE;IACrE,IAAI,KAAK,CAAC,IAAI,KAAK,uBAAuB;QAAE,KAAK,CAAC,IAAI,IAAI,CAAC,CAAA;IAC3D,IAAI,KAAK,CAAC,IAAI,KAAK,wBAAwB,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,IAAI,CAAC,CAAA;QACf,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAA;QAC7B,KAAK,CAAC,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YACtF,OAAwC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAA;IACnE,CAAC;AACH,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,WAAwB,EAAU,EAAE;IAC3D,MAAM,QAAQ,GAAiB,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACzD,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM;QAAE,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACpE,OAAO,aAAa,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;AAC7C,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,CAAC,WAAwC,EAAE,QAAsB,EAAU,EAAE;IACjG,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAA;IACnC,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW;QAAE,OAAO,SAAS,CAAA;IACtF,IAAI,OAAO,CAAC,MAAM,KAAK,kBAAkB;QAAE,OAAO,gBAAgB,CAAA;IAClE,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,aAAa,KAAK,UAAU,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,CAAC,EAAE,CAAC;QACjH,OAAO,gBAAgB,CAAA;IACzB,CAAC;IACD,IAAI,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACnG,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,IAAI,iBAAiB,CAAA;IACzD,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,IAAI,OAAO,CAAC,YAAY,IAAI,KAAK;QAAE,OAAO,cAAc,CAAA;IAC9F,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,SAAS,CAAA;IACrC,IAAI,CAAC,WAAW,CAAC,gBAAgB,IAAI,WAAW,CAAC,oBAAoB,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QAChG,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACtD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC,CAAA;AAUD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,MAAc,EAAU,EAAE;IAClD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,SAAS,CAAC;QACf,KAAK,aAAa;YAChB,OAAO,QAAQ,CAAA;QACjB,KAAK,SAAS,CAAC;QACf,KAAK,cAAc;YACjB,OAAO,QAAQ,CAAA;QACjB;YACE,OAAO,MAAM,CAAA;IACjB,CAAC;AACH,CAAC,CAAA;AAyFD,MAAM,SAAS,GAAG,CAAC,KAAY,EAAgE,EAAE,CAC/F,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAClC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CACrD,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAC7E,CAAC,CAAA;AAEN,MAAM,QAAQ,GAAG,CAAC,OAA+B,EAAW,EAAE,CAC5D,OAAO,KAAK,SAAS;IACrB,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAA;AAEnG;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,CACjB,OAAgB,EACgD,EAAE,CAClE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAA;IAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IACtC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK,CAAA;IACxF,IAAI,wBAAwB,GAAG,CAAC,CAAA;IAChC,IAAI,aAA0D,CAAA;IAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAA;IACzC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,SAAS,CAAA;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,MAAM,KAAK,QAAQ;QACvB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACf,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,cAAc,EAAE,WAAW,SAAS,WAAW,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE;SAC3E,CAAC;QACF,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YACf,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,WAAW,KAAK,CAAC,MAAM,EAAE;YACjC,cAAc,EAAE,WAAW,SAAS,WAAW,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE;SAC3E,CAAC,CAAC,CAAA;IAEP,MAAM,IAAI,GAAG,CACX,SAAiB,EACjB,OAAgC,EAChC,SAAiB,EACkB,EAAE;QACrC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAE,EAAE,CACpC,IAAI,gBAAgB,CAAC;YACnB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,oBAAoB,SAAS,QAAQ,OAAO,CAAC,KAAK,EAAE;YAC7D,KAAK;SACN,CAAC,CAAA;QACJ,iEAAiE;QACjE,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,uEAAuE;QACvE,0EAA0E;QAC1E,qEAAqE;QACrE,kEAAkE;QAClE,OAAO,MAAM,CAAC,GAAG,CAAC;YAChB,GAAG,EAAE,GAAG,EAAE,CACR,IAAI,YAAY,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC7C,QAAQ,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,oBAAoB,SAAS,EAAE,CAAC;gBACrE,SAAS;gBACT,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE;aACzD,CAAC;YACJ,KAAK,EAAE,UAAU;SAClB,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,CAAC,EAC1F,MAAM,CAAC,MAAM,CACd,CAAA;IACH,CAAC,CAAA;IAED,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,IAAU,EAAE,MAAc,EAAqC,EAAE,CAC/E,IAAI,CACF,aAAa,EACb;QACE,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;KACzC,EACD,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAC5B,CAAA;IAEH,6CAA6C;IAC7C,MAAM,YAAY,GAAG,CACnB,IAAU,EACV,MAAc,EACd,OAAgB,EACmB,EAAE,CACrC,IAAI,CACF,eAAe,EACf,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAC/E,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAC5B,CAAA;IAEH,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,IAAI,oBAAoB,GAAG,CAAC,CAAA;IAC5B,IAAI,YAAY,GAAG,CAAC,CAAC,CAAA;IACrB,IAAI,UAAU,GAAuD,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAiB,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACzD,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAA;IACjB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAC/C,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YAC/B,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CACjC,OAAO,CAAC,WAAW,KAAK,SAAS;gBAC/B,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAClF,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,SAAS,GAAwB,EAAE,CAAA;QACzC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;QACzC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,IAAI,CAC/E,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,oEAAoE;YACpE,UAAU,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,QAAQ,EAAE;gBACnC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;YACjC,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,OAAM;YAC1C,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACrB,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG;gBAAE,SAAS,CAAC,KAAK,EAAE,CAAA;YAC7C,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAA;YACzB,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QAC9B,CAAC,CAAC,CACH,CACF,CAAA;QACD,oBAAoB,GAAG,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC/E,YAAY,GAAG,QAAQ,CAAA;QACvB,MAAM,MAAM,GAAG,aAAa,CAAC;YAC3B,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;YAC7C,oBAAoB;YACpB,UAAU;YACV,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;SAChF,EAAE,QAAQ,CAAC,CAAA;QACZ,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAA;YACjC,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,KAAK,EAAE,CAAA;YACxC,MAAM,WAAW,GAAG,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,CAAA;YACzD,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC1C,SAAS;gBACT,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,OAAO;gBACP,MAAM,EAAE,SAAS;gBACjB,WAAW;aACZ,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;YAClE,IAAI,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAA;YACpE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAC/C,IAAI,OAAO,KAAK,SAAS,IAAI,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,WAAW,EAAE,CAAC;gBACnF,WAAW,GAAG,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,CAAA;YACpG,CAAC;YACD,wBAAwB,GAAG,WAAW,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB,GAAG,CAAC,CAAA;YAC1F,MAAM,gBAAgB,GAAG,WAAW,CAAC,OAAO,KAAK,IAAI,IAAI,WAAW,CAAC,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAA;YACnG,MAAM,UAAU,GAAG,aAAa,CAAC;gBAC/B,OAAO;gBACP,oBAAoB;gBACpB,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,gBAAgB;aACjB,EAAE,QAAQ,CAAC,CAAA;YACZ,WAAW,GAAG,EAAE,GAAG,WAAW,EAAE,UAAU,EAAE,CAAA;YAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,SAAS;gBACT,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,WAAW;gBACX,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,UAAU;gBACV,MAAM,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE;gBACpC,GAAG,EAAE,WAAW,CAAC,UAAU;gBAC3B,SAAS,EAAE,OAAO,CAAC,SAAS;aAC7B,CAAC,CAAA;YACF,MAAM,OAAO,GAAG,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,WAAW,KAAK,WAAW,CAAC,WAAW;gBAClG,aAAa,CAAC,OAAO,KAAK,WAAW,CAAC,OAAO,IAAI,aAAa,CAAC,UAAU,KAAK,WAAW,CAAC,UAAU;gBACpG,aAAa,CAAC,WAAW,KAAK,WAAW,CAAC,WAAW;gBACrD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC;gBAC3E,aAAa,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,CAAA;YAC7C,qFAAqF;YACrF,IAAI,OAAO,IAAI,WAAW,CAAC,UAAU,IAAI,aAAc,CAAC,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC5F,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,uBAAuB,EAAE;oBACjD,GAAG,WAAW;oBACd,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;iBAC3C,EAAE,oBAAoB,CAAC,CAAA;gBACxB,aAAa,GAAG,WAAW,CAAA;YAC7B,CAAC;QACH,CAAC;QACD,kFAAkF;QAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACrE,MAAM,QAAQ,GAAS,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QACjD,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK;YAAE,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAClE,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI;YAC9E,mEAAmE;YACnE,iEAAiE;YACjE,iEAAiE;YACjE,oEAAoE;YACpE,kEAAkE;YAClE,sCAAsC;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CACvB,MAAM,CAAC,YAAY,CACjB,MAAM,CAAC,UAAU,CAAC,yCAAyC,CAAC,EAC5D,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CACvE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAC7B,CACF,CAAA;YACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,oEAAoE;gBACpE,mEAAmE;gBACnE,kDAAkD;gBAClD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,sEAAsE;gBACtE,sEAAsE;gBACtE,qEAAqE;gBACrE,iEAAiE;gBACjE,iEAAiE;gBACjE,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,IAAI,KAAK,gBAAgB,CAAA;gBAChF,IAAI,OAAO,EAAE,CAAC;oBACZ,mEAAmE;oBACnE,uEAAuE;oBACvE,KAAK,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;oBAC9C,kEAAkE;oBAClE,mEAAmE;oBACnE,qDAAqD;oBACrD,oBAAoB,GAAG,CAAC,CAAA;gBAC1B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;gBACzF,sEAAsE;gBACtE,uEAAuE;gBACvE,0CAA0C;gBAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU;oBAAE,MAAK;YACxC,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAC5E,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;QACrD,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtB,MAAK;QACP,CAAC;IACH,CAAC;IACD,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,SAAS;KACrD,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAW,EAAE,CAC1D,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC;IACzE,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,sBAAsB,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAA","sourcesContent":["/**\n * Run health over the control plane: what a run's state means, and what to do\n * about it.\n *\n * A control plane answers \"what is this run doing\". A monitor answers the\n * question after it — \"is that all right, and if not, what now\" — and the two\n * are deliberately separate modules. {@link classify} is a pure function of\n * one observation, so the vocabulary an operator reads on a dashboard is the\n * same one a heal loop branches on and the same one a test can enumerate.\n * {@link run} is the loop that beats over `Control` and journals what it saw.\n *\n * The classification is decided from durable evidence only: the run summary\n * the control plane projects, and the journal entries `watch` replays. Nothing\n * here reads an in-process fiber, which is what makes a monitor able to watch\n * a run in another process at all.\n *\n * @since 0.1.0\n */\n\nimport { Journal, JournalEvent } from \"@smthrs/journal\"\nimport { Duration, Effect, Stream } from \"effect\"\nimport { Control } from \"./Control.ts\"\nimport type { ControlError } from \"./ControlError.ts\"\nimport { PersistenceError } from \"./ControlError.ts\"\nimport type { ControlEvent, Receipt, RunId, RunSummary, WatchFilter } from \"./ControlSchema.ts\"\nimport * as SubjectHealth from \"./Health.ts\"\n\n/**\n * The journal event type the engine records when an action attempt starts.\n *\n * Named as a string rather than imported, exactly as in `Lineage`: a monitor\n * reads journals, not engines.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const attemptStartedEventType = \"flows.engine.attempt-started\"\n\n/**\n * The journal event type the engine records when an action attempt settles.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const attemptFinishedEventType = \"flows.engine.attempt-finished\"\n\n/**\n * The journal event type one monitor beat is recorded under.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const beatEventType = \"control.monitor.beat\"\n\n/**\n * The journal event type one applied remedy is recorded under.\n *\n * A beat and a heal are two records because they are two facts, and only one\n * of them is true before the remedy runs. A single record carrying `healed`\n * has to be written either before the remedy — leaving durable evidence of a\n * heal that a crash one instruction later never performed — or after it, which\n * loses the evidence of what a monitor decided when the remedy is what killed\n * it. Splitting them keeps both.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const healedEventType = \"control.monitor.healed\"\n\n/**\n * What a run looks like to a monitor.\n *\n * `healthy` covers both \"moving\" and \"finished\": a completed run needs nothing\n * done to it, and neither does one that is making progress. The other six name\n * a specific thing that is wrong, because a heal that cannot tell them apart\n * would resume a run that is waiting for a human.\n *\n * @category models\n * @since 0.1.0\n */\nexport const Health = SubjectHealth.HealthState\n\n/**\n * What a run looks like to a monitor.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Health = typeof Health.Type\n\n/**\n * Everything one classification is decided from.\n *\n * `events` is the run's journal as `watch` projects it, oldest first.\n * `beatsWithoutProgress` counts consecutive beats that added no entry, and\n * `stallBeats` is how many of those make a stall. Splitting the count from the\n * threshold is what lets the same pure function serve a monitor that beats\n * every second and one that beats every hour.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Observation {\n  /** The run's projection, or nothing when the control plane has no such run. */\n  readonly summary?: RunSummary | undefined\n  readonly events: ReadonlyArray<ControlEvent>\n  readonly beatsWithoutProgress: number\n  readonly stallBeats: number\n  /**\n   * The trampoline round at which a lineage stops being a loop and starts\n   * being a runaway. A run past it is looping without converging.\n   */\n  readonly roundBound?: number | undefined\n  /** Fresh, explicitly reported semantic work; never inferred from bytes or monitor events. */\n  readonly semanticProgress?: boolean | undefined\n}\n\n/** How many rounds a trampoline may take before a monitor calls it runaway. */\nconst defaultRoundBound = 32\n\ninterface AttemptState {\n  open: number\n  failed: boolean\n}\n\nconst foldAttempt = (state: AttemptState, event: ControlEvent): void => {\n  if (event.kind === attemptStartedEventType) state.open += 1\n  if (event.kind === attemptFinishedEventType) {\n    state.open -= 1\n    const payload = event.payload\n    state.failed = typeof payload === \"object\" && payload !== null && !Array.isArray(payload) &&\n      (payload as { readonly state?: unknown })[\"state\"] === \"failed\"\n  }\n}\n\n/**\n * Decides what a run's state means.\n *\n * The order is the order an operator would read it in, and each rule earns its\n * place by naming a different response:\n *\n * | Condition | Health | Because |\n * | --- | --- | --- |\n * | No summary | `unknown` | Nothing to say, and nothing to do. |\n * | `failed` | `failing` | The run itself reported the failure. |\n * | `completed`, `cancelled` | `healthy` | A finished run needs nothing. |\n * | `waiting-approval`, or parked on `approval` | `awaiting-human` | A human owes it an answer; no machine can supply one. |\n * | Parked with no waiting reason | `awaiting-human` | Only an operator's own park writes no waiting reason, so a person stopped it and a person restarts it. |\n * | `roundOrdinal` at or past the bound | `runaway-loop` | The lineage is looping without converging. |\n * | The last settled attempt failed | `failing` | The run is still alive but its work is not landing. |\n * | No progress for `stallBeats`, an attempt open | `wedged-node` | One attempt started and never settled: the work is stuck, not the run. |\n * | No progress for `stallBeats` | `stalled` | Nothing is happening and nothing is in flight. |\n * | Anything else | `healthy` | Entries are still arriving. |\n *\n * `awaiting-human` deliberately outranks `failing`: a run parked for approval\n * after a failed attempt is waiting for a person, and resuming or cancelling\n * it would take the decision away from them.\n *\n * A park with no reason is the same case. The engine names every park it makes\n * — `event`, `approval`, `timer`, `quota`, `released` — so a parked run whose\n * `waitingReason` is absent was parked by an operator through\n * `ControlRuntime.writeStatus`. Calling that a stall and resuming it would undo\n * a deliberate act, which is the worst thing an unattended heal loop can do.\n *\n * @param observation what the beat saw\n * @category projections\n * @since 0.1.0\n */\nexport const classify = (observation: Observation): Health => {\n  const attempts: AttemptState = { open: 0, failed: false }\n  for (const event of observation.events) foldAttempt(attempts, event)\n  return classifyState(observation, attempts)\n}\n\nconst classifyState = (observation: Omit<Observation, \"events\">, attempts: AttemptState): Health => {\n  const summary = observation.summary\n  if (summary === undefined) return \"unknown\"\n  if (summary.status === \"failed\") return \"failing\"\n  if (summary.status === \"completed\" || summary.status === \"cancelled\") return \"healthy\"\n  if (summary.status === \"waiting-approval\") return \"awaiting-human\"\n  if (summary.status === \"parked\" && (summary.waitingReason === \"approval\" || summary.waitingReason === undefined)) {\n    return \"awaiting-human\"\n  }\n  if (SubjectHealth.waitReason(summary.status, summary.waitingReason) !== undefined) return \"healthy\"\n  const bound = observation.roundBound ?? defaultRoundBound\n  if (summary.roundOrdinal !== undefined && summary.roundOrdinal >= bound) return \"runaway-loop\"\n  if (attempts.failed) return \"failing\"\n  if (!observation.semanticProgress && observation.beatsWithoutProgress >= observation.stallBeats) {\n    return attempts.open > 0 ? \"wedged-node\" : \"stalled\"\n  }\n  return \"healthy\"\n}\n\n/**\n * What a monitor does about one unhealthy run.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Remedy = \"resume\" | \"cancel\" | \"none\"\n\n/**\n * The remedy a health warrants, before `autoHeal` decides whether to apply it.\n *\n * A stalled or wedged run is one nobody is driving, and a resume claims it. A\n * failing or runaway run is one that will not get better by being driven\n * harder. Everything else is left alone.\n *\n * @param health what the beat classified\n * @category projections\n * @since 0.1.0\n */\nexport const remedyFor = (health: Health): Remedy => {\n  switch (health) {\n    case \"stalled\":\n    case \"wedged-node\":\n      return \"resume\"\n    case \"failing\":\n    case \"runaway-loop\":\n      return \"cancel\"\n    default:\n      return \"none\"\n  }\n}\n\n/**\n * One beat of a monitor.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Beat {\n  /** Which beat this was, counting from zero. */\n  readonly beat: number\n  readonly health: Health\n  /** The newest journal sequence the run had, ignoring the monitor's own records. */\n  readonly sequence: number\n  /**\n   * The remedy that ran and returned, absent when none did.\n   *\n   * Present only after the heal succeeded, which is also when\n   * `control.monitor.healed` is journaled. A beat that decided on a remedy the\n   * monitor never got to apply reports nothing here.\n   */\n  readonly healed?: Remedy | undefined\n  /** The receipt the remedy returned, absent when none was applied. */\n  readonly receipt?: Receipt | undefined\n}\n\n/**\n * What a monitor found.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly runId: RunId\n  readonly beats: ReadonlyArray<Beat>\n  /** The last beat's health, or `unknown` when no beat ran. */\n  readonly health: Health\n}\n\n/**\n * How a monitor beats.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Options {\n  readonly runId: RunId\n  /** Optional configured observational checker; checker results never authorize a remedy. */\n  readonly healthCheck?: SubjectHealth.ResolvedCheck | undefined\n  /** Shared host admission limit around probes. */\n  readonly withProbePermit?: (<A>(effect: Effect.Effect<A>) => Effect.Effect<A>) | undefined\n  /** Bounded retained beat history for long-lived host monitoring. */\n  readonly retainBeats?: number | undefined\n  /** Hosts with structured status observations omit redundant legacy heartbeat records. */\n  readonly recordBeats?: boolean | undefined\n  /**\n   * Who is watching. Defaults to `default`.\n   *\n   * It reaches the journal twice: as the source of every record this monitor\n   * writes, and as `monitorId` in each payload. Two monitors pointed at one\n   * run both beat and both remedy — nothing on the control plane leases a run\n   * to one watcher — so the identity is what makes their evidence tellable\n   * apart, and what keeps their remedies on separate idempotency keys.\n   */\n  readonly monitorId?: string | undefined\n  /** How long to wait between beats. Defaults to one second. */\n  readonly intervalMs?: number | undefined\n  /** How many beats to take before reporting. Defaults to ten. */\n  readonly maxChecks?: number | undefined\n  /** How many beats without progress make a stall. Defaults to three. */\n  readonly stallBeats?: number | undefined\n  /** The round at which a trampoline is a runaway. Defaults to 32. */\n  readonly roundBound?: number | undefined\n  /** Which healths the monitor is allowed to act on. Defaults to none. */\n  readonly autoHeal?: ReadonlyArray<Health> | undefined\n  /**\n   * What to do about an unhealthy run. Defaults to `Control.resume` and\n   * `Control.cancel` per {@link remedyFor}.\n   *\n   * A heal that fails does not abort the monitor: the failure is logged, the\n   * beat is recorded without `healed` or `receipt`, and the loop moves to the\n   * next beat, where the stall evidence still stands and the remedy is\n   * retried under that beat's own idempotency key.\n   */\n  readonly heal?: (\n    input: { readonly runId: RunId; readonly health: Health; readonly remedy: Remedy; readonly beat: number }\n  ) => Effect.Effect<Receipt, ControlError>\n}\n\nconst summaryOf = (runId: RunId): Effect.Effect<RunSummary | undefined, ControlError, Control> =>\n  Effect.flatMap(Control, (control) =>\n    control.list({ _tag: \"runs\", filters: { runId } }).pipe(\n      Effect.map((listed) => listed._tag === \"runs\" ? listed.items[0] : undefined)\n    ))\n\nconst terminal = (summary: RunSummary | undefined): boolean =>\n  summary !== undefined &&\n  (summary.status === \"completed\" || summary.status === \"failed\" || summary.status === \"cancelled\")\n\n/**\n * Watches one run and, when told to, heals it.\n *\n * The loop stops early on a terminal run, because a finished run has nothing\n * left to observe and beating at it would fill its journal with heartbeats.\n * Otherwise it takes `maxChecks` beats and reports what it saw.\n *\n * Every beat is journaled as `control.monitor.beat` before any remedy is\n * applied, carrying the `remedy` it is about to attempt, so a monitor that\n * crashes mid-heal leaves the evidence of what it decided. The remedy itself\n * is journaled separately as `control.monitor.healed`, and only once the heal\n * returned a receipt: a heal that failed, or a process that died running one,\n * must not leave a durable record saying the run was healed.\n *\n * `autoHeal` is empty by default. A monitor that healed by default would be a\n * monitor that cancels a run the first time it looks at one, which is the\n * wrong default for a thing an operator points at production.\n *\n * Nothing here leases the run. Two monitors on one run both beat and both\n * remedy, so a remedy has to be idempotent on the control plane — which the\n * two defaults are, through `Control.resume` and `Control.cancel`\n * idempotency keys that name the monitor, the run, and the beat. A custom\n * `heal` owes the same property.\n *\n * @param options how to beat and what to heal\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n  options: Options\n): Effect.Effect<Report, ControlError, Control | Journal.Journal> =>\n  Effect.gen(function*() {\n    const control = yield* Control\n    const journal = yield* Journal.Journal\n    const intervalMs = options.intervalMs ?? options.healthCheck?.policy.intervalMs ?? 1_000\n    let consecutiveProbeFailures = 0\n    let lastPublished: SubjectHealth.HealthObservation | undefined\n    const maxChecks = options.maxChecks ?? 10\n    const stallBeats = options.stallBeats ?? 3\n    const autoHeal = options.autoHeal ?? []\n    const monitorId = options.monitorId ?? \"default\"\n    const heal = options.heal ?? ((input) =>\n      input.remedy === \"resume\"\n        ? control.resume({\n          runId: input.runId,\n          idempotencyKey: `monitor:${monitorId}:resume:${input.runId}:${input.beat}`\n        })\n        : control.cancel({\n          runId: input.runId,\n          reason: `monitor:${input.health}`,\n          idempotencyKey: `monitor:${monitorId}:cancel:${input.runId}:${input.beat}`\n        }))\n\n    const emit = (\n      eventType: string,\n      payload: Record<string, unknown>,\n      operation: string\n    ): Effect.Effect<void, ControlError> => {\n      const unrecorded = (cause: unknown) =>\n        new PersistenceError({\n          operation: eventType,\n          message: `Failed to record ${operation} for ${options.runId}`,\n          cause\n        })\n      // The record is BUILT inside the failure channel, not before it.\n      // `JournalEvent.RunId`/`SourceId` refuse an identifier the store cannot\n      // tell apart from another one — a lone UTF-16 surrogate, an embedded NUL\n      // — by THROWING, and `monitorId` reaches here from an operator's flag. A\n      // thrown constructor is a defect, which the RPC boundary reports as an\n      // opaque `TransportError` and a caller cannot handle; a beat whose record\n      // cannot be built is the same event as a beat whose record cannot be\n      // appended, so both fail as `PersistenceError` naming the record.\n      return Effect.try({\n        try: () =>\n          new JournalEvent.Input({\n            runId: JournalEvent.RunId.make(options.runId),\n            sourceId: JournalEvent.SourceId.make(`/control/monitor/${monitorId}`),\n            eventType,\n            payload: { runId: options.runId, monitorId, ...payload }\n          }),\n        catch: unrecorded\n      }).pipe(\n        Effect.flatMap((input) => Effect.mapError(journal.emitDurableUnfenced(input), unrecorded)),\n        Effect.asVoid\n      )\n    }\n\n    /** What the beat saw, and what it is about to do about it. */\n    const record = (beat: Beat, remedy: Remedy): Effect.Effect<void, ControlError> =>\n      emit(\n        beatEventType,\n        {\n          beat: beat.beat,\n          health: beat.health,\n          sequence: beat.sequence,\n          ...(remedy === \"none\" ? {} : { remedy })\n        },\n        `monitor beat ${beat.beat}`\n      )\n\n    /** What the remedy did, once it returned. */\n    const recordHealed = (\n      beat: Beat,\n      remedy: Remedy,\n      receipt: Receipt\n    ): Effect.Effect<void, ControlError> =>\n      emit(\n        healedEventType,\n        { beat: beat.beat, health: beat.health, healed: remedy, receipt: receipt._tag },\n        `monitor heal ${beat.beat}`\n      )\n\n    const beats: Array<Beat> = []\n    let beatsWithoutProgress = 0\n    let lastSequence = -1\n    let checkpoint: Pick<WatchFilter, \"afterCursor\" | \"afterSequence\"> = {}\n    const attempts: AttemptState = { open: 0, failed: false }\n    let sequence = -1\n    for (let beat = 0; beat < maxChecks; beat += 1) {\n      if (beat > 0 && intervalMs > 0) {\n        yield* Effect.sleep(Duration.millis(\n          options.healthCheck === undefined\n            ? intervalMs\n            : SubjectHealth.nextDelay(options.healthCheck.policy, consecutiveProbeFailures)\n        ))\n      }\n      const summary = yield* summaryOf(options.runId)\n      const newEvents: Array<ControlEvent> = []\n      const sinceCursor = Math.max(0, sequence)\n      yield* control.watch({ runId: options.runId, follow: false, ...checkpoint }).pipe(\n        Stream.runForEach((event) =>\n          Effect.sync(() => {\n            // Advance over bookkeeping too, but never count it as run progress.\n            checkpoint = event.cursor === undefined\n              ? { afterSequence: event.sequence }\n              : { afterCursor: event.cursor }\n            if (isBookkeepingEvent(event.kind)) return\n            newEvents.push(event)\n            if (newEvents.length > 256) newEvents.shift()\n            sequence = event.sequence\n            foldAttempt(attempts, event)\n          })\n        )\n      )\n      beatsWithoutProgress = sequence === lastSequence ? beatsWithoutProgress + 1 : 0\n      lastSequence = sequence\n      const health = classifyState({\n        ...(summary === undefined ? {} : { summary }),\n        beatsWithoutProgress,\n        stallBeats,\n        ...(options.roundBound === undefined ? {} : { roundBound: options.roundBound })\n      }, attempts)\n      if (options.healthCheck !== undefined && summary !== undefined) {\n        const check = options.healthCheck\n        const subjectId = `run:${options.runId}`\n        const incarnation = SubjectHealth.runIncarnation(summary)\n        const probe = SubjectHealth.evaluate(check, {\n          subjectId,\n          state: summary.status,\n          summary,\n          events: newEvents,\n          sinceCursor\n        }, { monitorId, incarnation, evidenceSeq: Math.max(0, sequence) })\n        let observation = yield* (options.withProbePermit?.(probe) ?? probe)\n        const current = yield* summaryOf(options.runId)\n        if (current === undefined || SubjectHealth.runIncarnation(current) !== incarnation) {\n          observation = { ...observation, outcome: \"discarded\", report: undefined, reason: \"owner-changed\" }\n        }\n        consecutiveProbeFailures = observation.outcome === \"ok\" ? 0 : consecutiveProbeFailures + 1\n        const semanticProgress = observation.outcome === \"ok\" && observation.report?.activity === \"working\"\n        const baseHealth = classifyState({\n          summary,\n          beatsWithoutProgress,\n          stallBeats,\n          roundBound: options.roundBound,\n          semanticProgress\n        }, attempts)\n        observation = { ...observation, baseHealth }\n        const status = SubjectHealth.rollup({\n          subjectId,\n          state: summary.status,\n          incarnation,\n          waitingReason: summary.waitingReason,\n          baseHealth,\n          latest: { observation, sequence: 0 },\n          now: observation.observedAt,\n          updatedAt: summary.updatedAt\n        })\n        const changed = lastPublished === undefined || lastPublished.incarnation !== observation.incarnation ||\n          lastPublished.outcome !== observation.outcome || lastPublished.baseHealth !== observation.baseHealth ||\n          lastPublished.evidenceSeq !== observation.evidenceSeq ||\n          JSON.stringify(lastPublished.report) !== JSON.stringify(observation.report) ||\n          lastPublished.reason !== observation.reason\n        // Renew before expiry, but do not fill the journal with identical per-probe records.\n        if (changed || observation.observedAt >= lastPublished!.observedAt + check.policy.ttlMs / 2) {\n          yield* emit(SubjectHealth.statusObservedEventType, {\n            ...observation,\n            status: summary.status,\n            health: status.health,\n            activity: status.activity,\n            attention: status.attention,\n            freshness: status.freshness,\n            waitingReason: summary.waitingReason ?? \"\"\n          }, \"health observation\")\n          lastPublished = observation\n        }\n      }\n      // Checker output is observational and never participates in remedy authorization.\n      const remedy = autoHeal.includes(health) ? remedyFor(health) : \"none\"\n      const observed: Beat = { beat, health, sequence }\n      if (options.recordBeats !== false) yield* record(observed, remedy)\n      if (remedy === \"none\") {\n        beats.push(observed)\n      } else {\n        const receipt = yield* heal({ runId: options.runId, health, remedy, beat }).pipe(\n          // A remedy that fails must not abort the run of beats: the monitor\n          // is an unattended loop, and one failing heal leaves every other\n          // beat — and every other run it would have observed — unwatched.\n          // The failure is logged with the beat it failed on, and the beat is\n          // recorded without `healed` or `receipt`, exactly as a remedy the\n          // monitor never got to apply reports.\n          Effect.catch((failure) =>\n            Effect.annotateLogs(\n              Effect.logWarning(\"A monitor remedy failed and was skipped\"),\n              { runId: options.runId, beat, health, remedy, cause: String(failure) }\n            ).pipe(Effect.as(undefined))\n          )\n        )\n        if (receipt === undefined) {\n          // The stall evidence stands: nothing provably moved the run, so the\n          // next beat compares against the same progress mark and the remedy\n          // is retried with the beat's own idempotency key.\n          beats.push(observed)\n        } else {\n          // A remedy that returned is not a remedy that was applied. `Terminal`\n          // says the run had already settled, so nothing was healed; `Conflict`\n          // says the key belonged to another mutation, so this monitor's remedy\n          // never ran. Recording either as `healed` claimed something that did\n          // not happen, and resetting the stall count on either erased the\n          // evidence the next beat needs to notice the run is still stuck.\n          const applied = receipt._tag === \"Accepted\" || receipt._tag === \"AlreadyApplied\"\n          if (applied) {\n            // Journaled here and not a line earlier: `healed` is a claim about\n            // something that happened, and until the receipt came back it had not.\n            yield* recordHealed(observed, remedy, receipt)\n            // The heal moved the run, so the next beat compares against a run\n            // that has changed. Counting the beats before it as stall evidence\n            // again would heal a second time for the same stall.\n            beatsWithoutProgress = 0\n          }\n          beats.push(applied ? { ...observed, healed: remedy, receipt } : { ...observed, receipt })\n          // A remedy that answered `Terminal` observed the run settle. The loop\n          // ends on the same evidence a terminal summary ends it on, rather than\n          // beating against a run nothing can move.\n          if (receipt._tag === \"Terminal\") break\n        }\n      }\n      if (options.retainBeats !== undefined && beats.length > options.retainBeats) {\n        beats.splice(0, beats.length - options.retainBeats)\n      }\n      if (terminal(summary)) {\n        break\n      }\n    }\n    return {\n      runId: options.runId,\n      beats,\n      health: beats[beats.length - 1]?.health ?? \"unknown\"\n    }\n  })\n\n/** Observer and notification bookkeeping is never execution progress.\n * @category predicates\n * @since 1.0.0\n */\nexport const isBookkeepingEvent = (kind: string): boolean =>\n  kind.startsWith(\"control.monitor.\") || kind.startsWith(\"control.status.\") ||\n  kind.startsWith(\"flows.alerts.\") || kind.startsWith(\"flows.notifications.\") || kind.startsWith(\"flows.notification.\")\n"]}