@mark1russell7/service 0.5.13 → 0.5.14

package/README.md

@@ -10,6 +10,7 @@ This release has the declaration, its export, the law, the health runtime, the l
 - **No defaults.** The runtime has no defaults of its own. Thus `describe()` refuses a service that does not declare its readiness, its liveness, its startup budget, its shutdown phases or its cooperative window. It also refuses a service that does not declare what a repeat signal does. The error tells which. `start()` also refuses a check without its function.
 - **One timing shape.** Each policy has a `Timing`: a debounce of polls, or a window of time. Each converts to the other (`asWindow`, `asDebounce`). `failSpan` and `recoverSpan` give the earliest and the latest time to detect a change. At 3 polls of 5 s ± 20 %, a latch fails in 8–18 s on an error, and in 12–22 s on a 4 s timeout.
 - **Checks.** A check names the step of the connection that it examines (`dial`, `round-trip`, `checkout` and others). It also tells if it uses the connection of the application, and which policy it controls. The policy has no default.
+- **The catalog of a client.** The checks of each client name the steps that they gate: `steps: { "round-trip": "readiness" }`. With `catalog: true`, the client also gives a check for each other step that it has. These checks gate nothing and poll each 30 s. A nominated step that the client does not have then fails the attach. The name of each check is the pair of the component and the step (`db.round-trip`).
 - **Idempotency for each operation.** One dependency has the two kinds of operation. A Redis GET is safe to send again. An INCR that goes again after a timeout counts two times. Thus each operation of a dependency, and each route of a controller, declares if it can go again: `true`, `false`, or keyed.
 - **Keyed operations.** A keyed operation has the window that the server keeps its key for. `mayReissue` answers from the declaration of an operation. An operation without a declaration does not go again.
 - **The law.** `evaluate` and `verdictOf` decide one rule of the timer-ordering law on one binding. A rule is data: linear inequalities over `symbol@hop`, with a margin. A verdict is `holds`, `violated`, `unbounded` (it holds only because a timer is absent) or `depends` (nothing is bound).
@@ -29,6 +30,7 @@ The format of a description is `mark1russell7.service/1`.
 
 - **Checks on their own loops.** Each check polls on a fixed delay from the start of each poll, with its jitter. Each poll has a timeout, and the runtime aborts the signal of the check at that timeout. An answer is `"ok"`, `"error"` or `"indeterminate"`. A throw is an error with a flag. An indeterminate poll measured nothing, thus it moves no streak.
 - **A pending use is not started again.** While a use of a check is pending, each later poll fails at once with the same cause. Thus a check that hangs fails a debounce of two polls or more as fast as an error does (`failSpan(timing, poll, "hang")`).
+- **The metrics.** `health.metrics()` gives the counts of the runtime, and starts no check. For each check, it counts each outcome, each throw and each forced failure. Its histogram counts the durations of the polls. The bounds are fractions of the timeout of the poll, and a poll that timed out goes into `timedOut`. Its last 8 episodes of failure each go from the first error to the next pass. The cause of an episode has no user, password or token (`redact`). It also counts each transition of the machines of the runtime.
 - **A latch for each check that gates a policy.** A debounce counts polls in sequence. A window measures time on the monotonic clock from the first failure of a sequence. A readiness latch starts tripped, thus readiness is earned. A liveness latch starts released.
 - **The start.** `start()` makes one warm-up sweep of each check, bounded by each timeout, and then starts the loops. The startup probe fails until the sweep ends. `svc.health` gives the runtime from the start of `start()`, thus the probes can answer during the sweep. The verdict says when the sweep outlasts the startup budget.
 - **The verdicts.** `health.verdict()` gives started, draining, ready, alive, each controller, each dependency and each check. The service is ready when it started, it does not drain, and each readiness latch is released. A controller is ready when its own dependencies are ready. Thus a failed dependency fails only the controllers that need it.

package/build/{builder-DawJhq4p.d.ts → builder-CsSO4LzB.d.ts}

@@ -1,6 +1,6 @@
 import { n as Clock } from "./clock-mVqhXD0b.js";
 import { c as LimiterSpec } from "./policy-G4QJ1ZZq.js";
-import { B as Gate, D as Outcome, E as CheckFn, F as CooperativeFloor, G as ServiceDescription, H as Phase, I as Coupling, L as Dependency, N as Check, R as ExitCodes, T as StartOptions, V as MessageTrigger, W as RepeatSignal, Y as Poll, Z as Timing, d as TransportImpl, ht as ProcessTrigger, k as Trace, lt as ExitTrigger, n as HealthServer, o as PhaseWork, r as Lifecycle, rt as OperationOf, s as RunOptions, t as DependencyHandle, wt as Operation, x as Health } from "./lifecycle-fgYfxrHK.js";
+import { B as Check, E as CheckFn, G as ExitCodes, H as CooperativeFloor, I as Trace, J as MessageTrigger, P as Outcome, Q as ServiceDescription, St as ProcessTrigger, T as StartOptions, U as Coupling, W as Dependency, Y as Phase, Z as RepeatSignal, d as TransportImpl, gt as ExitTrigger, it as Timing, jt as Operation, n as HealthServer, nt as Poll, o as PhaseWork, q as Gate, r as Lifecycle, s as RunOptions, t as DependencyHandle, ut as OperationOf, x as Health } from "./lifecycle-CMedxaQd.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 //#region src/profiles.d.ts
 /**
@@ -208,4 +208,4 @@ interface ServiceBuilder {
 declare const service: (name: string) => ServiceBuilder;
 //#endregion
 export { WatchdogSpec as a, Profile as c, kubernetes as d, localDev as f, Watchdog as i, daemon as l, ServiceBuilder as n, watchdog as o, service as r, PodTiming as s, ControllerBuilder as t, ephemeral as u };
-//# sourceMappingURL=builder-DawJhq4p.d.ts.map
+//# sourceMappingURL=builder-CsSO4LzB.d.ts.map

package/build/{core-BPLRDcUJ.d.ts → core-D8iXeoE4.d.ts}

@@ -1,7 +1,7 @@
 import { t as Backoff } from "./backoff-B-jGWVDm.js";
 import { n as Clock } from "./clock-mVqhXD0b.js";
-import { B as Gate, E as CheckFn, G as ServiceDescription, K as Step, L as Dependency, M as Recorder, N as Check, Y as Poll, t as DependencyHandle, wt as Operation } from "./lifecycle-fgYfxrHK.js";
-import { n as ServiceBuilder } from "./builder-DawJhq4p.js";
+import { $ as Step, B as Check, E as CheckFn, Q as ServiceDescription, W as Dependency, jt as Operation, nt as Poll, q as Gate, t as DependencyHandle, z as Recorder } from "./lifecycle-CMedxaQd.js";
+import { n as ServiceBuilder } from "./builder-CsSO4LzB.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 //#region src/client/meets.d.ts
 /** A situation is what the connection did when the failure started: a command in flight, or idle. */
@@ -89,6 +89,13 @@ interface CheckOptions<S extends Step = ClientStep> {
   readonly poll: Poll;
   /** For each step, the policy that its check controls, or null for a check that only reports. A step that is absent has no check. */
   readonly steps: Partial<Record<S, Gate | null>>;
+  /**
+   * With a catalog, the client gives a check for each step that it has (B19). The steps of `steps` are
+   * the nominated checks. Each other step reports only, and polls at the catalog poll. `true` is a poll
+   * of 30 s with the jitter and the timeout of `poll`. A nominated step that the client has no check
+   * for then fails the attach. Without a catalog, the client gives only the steps of `steps`.
+   */
+  readonly catalog?: true | Poll | undefined;
 }
 /**
  * These are the knobs that each client has, by the names of the ordering law. Each driver adds its
@@ -223,4 +230,4 @@ declare const retryProblems: (name: string, knobs: RetryKnobs) => string[];
 declare const STARTING_POLL: Poll;
 //#endregion
 export { MANIFESTATIONS as C, firstDetector as D, Situation as E, unwatched as O, Detector as S, Meeting as T, Sent as _, ClientKnobs as a, retryProblems as b, ClientSpec as c, Factory as d, ReadyWait as f, SendOptions as g, STARTING_POLL as h, ClientJoin as i, ClientStep as l, RetrySchedule as m, CheckOptions as n, ClientOptions as o, RetryKnobs as p, Client as r, ClientSocket as s, Attempt as t, DialFn as u, readyWait as v, Manifestation as w, retrySchedule as x, retryBackoff as y };
-//# sourceMappingURL=core-BPLRDcUJ.d.ts.map
+//# sourceMappingURL=core-D8iXeoE4.d.ts.map

package/build/{dial-DA8vJfh2.d.ts → dial-DJolDmOK.d.ts}

@@ -1,4 +1,4 @@
-import "./core-BPLRDcUJ.js";
+import "./core-D8iXeoE4.js";
 //#region src/client/dial.d.ts
 /**
  * A judge reads the bytes that the server sent so far. It gives null for a good answer, an error for
@@ -13,4 +13,4 @@ type Judge = (received: Buffer) => Error | null | "more";
 declare const statusLineJudge: (accept: (status: number, line: string) => Error | null) => Judge;
 //#endregion
 export { statusLineJudge as n, Judge as t };
-//# sourceMappingURL=dial-DA8vJfh2.d.ts.map
+//# sourceMappingURL=dial-DJolDmOK.d.ts.map

package/build/grpc.d.ts

@@ -1,6 +1,6 @@
-import { Y as Poll, wt as Operation } from "./lifecycle-fgYfxrHK.js";
-import { C as MANIFESTATIONS, D as firstDetector, E as Situation, O as unwatched, S as Detector, T as Meeting, _ as Sent, a as ClientKnobs, b as retryProblems, c as ClientSpec, d as Factory, f as ReadyWait, g as SendOptions, h as STARTING_POLL, i as ClientJoin, l as ClientStep, n as CheckOptions, o as ClientOptions, p as RetryKnobs, r as Client, s as ClientSocket, t as Attempt, u as DialFn, v as readyWait, w as Manifestation } from "./core-BPLRDcUJ.js";
-import { t as Judge } from "./dial-DA8vJfh2.js";
+import { jt as Operation, nt as Poll } from "./lifecycle-CMedxaQd.js";
+import { C as MANIFESTATIONS, D as firstDetector, E as Situation, O as unwatched, S as Detector, T as Meeting, _ as Sent, a as ClientKnobs, b as retryProblems, c as ClientSpec, d as Factory, f as ReadyWait, g as SendOptions, h as STARTING_POLL, i as ClientJoin, l as ClientStep, n as CheckOptions, o as ClientOptions, p as RetryKnobs, r as Client, s as ClientSocket, t as Attempt, u as DialFn, v as readyWait, w as Manifestation } from "./core-D8iXeoE4.js";
+import { t as Judge } from "./dial-DJolDmOK.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 import { Machine } from "@mark1russell7/algebra";
 //#region src/grpc/connection.d.ts

package/build/grpc.js

@@ -1,6 +1,6 @@
 import { realClock } from "./clock.js";
 import { i as heartbeatDetection } from "./timing-DGRi3XCI.js";
-import { B as zeroProblems, C as notSent, D as readyWait, F as timeProblems, G as immediate, H as NEVER, L as wholeProblems, M as sendWithReissue, N as socketEnds, P as stepChecks, R as withinBound, S as ms, U as WHAT, V as MANIFESTATIONS, W as firstDetector, X as unwatched, Y as kernelGiveUp, _ as inFlight, a as helloDial, b as message, c as ageProblems, d as clientProblems, f as clientRecorder, g as idleCloseProblem, h as finiteMs, i as authority, j as roundTripCheck, k as retryProblems, l as attachClient, m as dialCheck, n as closeTimer, r as transition, s as STARTING_POLL, t as clientKnob, u as clientBounds, w as operationTable, y as localStateCheck, z as wrapperTimerProblems } from "./machine-BhTRu3u9.js";
+import { B as zeroProblems, C as notSent, D as readyWait, F as timeProblems, G as immediate, H as NEVER, L as wholeProblems, M as sendWithReissue, N as socketEnds, P as stepChecks, R as withinBound, S as ms, U as WHAT, V as MANIFESTATIONS, W as firstDetector, X as unwatched, Y as kernelGiveUp, _ as inFlight, a as helloDial, b as message, c as ageProblems, d as clientProblems, f as clientRecorder, g as idleCloseProblem, h as finiteMs, i as authority, j as roundTripCheck, k as retryProblems, l as attachClient, m as dialCheck, n as closeTimer, r as transition, s as STARTING_POLL, t as clientKnob, u as clientBounds, w as operationTable, y as localStateCheck, z as wrapperTimerProblems } from "./machine-DZMb_QEU.js";
 //#region src/grpc/connection.ts
 /** These states are in the sequence of the enum of grpc-js: the index of a state is its value. */
 const GRPC_STATES = [

package/build/http-client.d.ts

@@ -1,6 +1,6 @@
-import { Y as Poll, wt as Operation } from "./lifecycle-fgYfxrHK.js";
-import { C as MANIFESTATIONS, D as firstDetector, E as Situation, O as unwatched, S as Detector, T as Meeting, _ as Sent, a as ClientKnobs, c as ClientSpec, d as Factory, g as SendOptions, h as STARTING_POLL, i as ClientJoin, l as ClientStep, n as CheckOptions, o as ClientOptions, r as Client, s as ClientSocket, t as Attempt, u as DialFn, w as Manifestation } from "./core-BPLRDcUJ.js";
-import { t as Judge } from "./dial-DA8vJfh2.js";
+import { jt as Operation, nt as Poll } from "./lifecycle-CMedxaQd.js";
+import { C as MANIFESTATIONS, D as firstDetector, E as Situation, O as unwatched, S as Detector, T as Meeting, _ as Sent, a as ClientKnobs, c as ClientSpec, d as Factory, g as SendOptions, h as STARTING_POLL, i as ClientJoin, l as ClientStep, n as CheckOptions, o as ClientOptions, r as Client, s as ClientSocket, t as Attempt, u as DialFn, w as Manifestation } from "./core-D8iXeoE4.js";
+import { t as Judge } from "./dial-DJolDmOK.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 import { Machine } from "@mark1russell7/algebra";
 //#region src/http-client/connection.d.ts

package/build/http-client.js

@@ -1,5 +1,5 @@
 import { realClock } from "./clock.js";
-import { B as zeroProblems, C as notSent, E as poolProblems, F as timeProblems, G as immediate, I as trackWithTimer, J as keepaliveSilent, K as keepaliveAnswered, L as wholeProblems, M as sendWithReissue, N as socketEnds, P as stepChecks, R as withinBound, S as ms, U as WHAT, V as MANIFESTATIONS, W as firstDetector, X as unwatched, Y as kernelGiveUp, _ as inFlight, a as helloDial, b as message, c as ageProblems, d as clientProblems, f as clientRecorder, g as idleCloseProblem, i as authority, j as roundTripCheck, l as attachClient, m as dialCheck, n as closeTimer, o as statusLineJudge, q as keepaliveReset, r as transition, s as STARTING_POLL, t as clientKnob, u as clientBounds, v as keepaliveOnProblems, w as operationTable, y as localStateCheck, z as wrapperTimerProblems } from "./machine-BhTRu3u9.js";
+import { B as zeroProblems, C as notSent, E as poolProblems, F as timeProblems, G as immediate, I as trackWithTimer, J as keepaliveSilent, K as keepaliveAnswered, L as wholeProblems, M as sendWithReissue, N as socketEnds, P as stepChecks, R as withinBound, S as ms, U as WHAT, V as MANIFESTATIONS, W as firstDetector, X as unwatched, Y as kernelGiveUp, _ as inFlight, a as helloDial, b as message, c as ageProblems, d as clientProblems, f as clientRecorder, g as idleCloseProblem, i as authority, j as roundTripCheck, l as attachClient, m as dialCheck, n as closeTimer, o as statusLineJudge, q as keepaliveReset, r as transition, s as STARTING_POLL, t as clientKnob, u as clientBounds, v as keepaliveOnProblems, w as operationTable, y as localStateCheck, z as wrapperTimerProblems } from "./machine-DZMb_QEU.js";
 //#region src/http-client/connection.ts
 /**
 * This factory makes an `http.Agent` of Node. It loads node:http only when a client connects.

package/build/http.d.ts

@@ -1,6 +1,6 @@
 import { n as Clock } from "./clock-mVqhXD0b.js";
 import { d as Refused } from "./policy-G4QJ1ZZq.js";
-import { d as TransportImpl, m as RequestEnd, n as HealthServer, r as Lifecycle, v as RequestStream, w as ProbeKind, x as Health, y as RequestWork } from "./lifecycle-fgYfxrHK.js";
+import { d as TransportImpl, m as RequestEnd, n as HealthServer, r as Lifecycle, v as RequestStream, w as ProbeKind, x as Health, y as RequestWork } from "./lifecycle-CMedxaQd.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 import { ListenOptions } from "node:net";
 import { IncomingMessage, Server, ServerEventMap, ServerResponse } from "node:http";

package/build/index.d.ts

@@ -1,12 +1,12 @@
 import { a as backoffDelay, i as backoffBand, n as DelayBand, o as backoffProblems, r as Jitter, t as Backoff } from "./backoff-B-jGWVDm.js";
 import { a as realClock, i as platformClock, n as Clock, r as PlatformClockOptions, t as Cancel } from "./clock-mVqhXD0b.js";
 import { S as retryAfterSeconds, _ as freeAt, a as LimitRule, b as limiterProblems, c as LimiterSpec, d as Refused, f as SlidingWindow, g as decide, h as TokenBucket, i as LimitPolicy, l as LimiterStoreSpec, m as TakeStep, n as Decision, o as Limiter, p as StoreFailure, r as KeyState, s as LimiterEviction, t as Allowed, u as RefusalCause, v as idleSpan, x as longestRetryAfter, y as limitRule } from "./policy-G4QJ1ZZq.js";
-import { $ as asWindow, A as LifecycleEvent, B as Gate, C as ProbeAnswer, Ct as Idempotency, D as Outcome, E as CheckFn, F as CooperativeFloor, G as ServiceDescription, H as Phase, I as Coupling, J as FailsBy, K as Step, L as Dependency, M as Recorder, N as Check, O as Result, P as Controller, Q as asDebounce, R as ExitCodes, S as HealthVerdict, St as processPortOf, T as StartOptions, Tt as mayReissue, U as ProgressMarks, V as MessageTrigger, W as RepeatSignal, X as Span, Y as Poll, Z as Timing, _ as RequestOptions, _t as isExitCode, a as PhaseContext, at as RouteRequest, b as CheckVerdict, bt as matchesMessage, c as STAGES, ct as ExitSignal, d as TransportImpl, dt as MessageShape, et as failSpan, f as cooperativeAbortAt, ft as PROCESS_TRIGGERS, g as RequestFn, gt as embeddedProcess, h as RequestEvent, ht as ProcessTrigger, i as LifecycleState, it as RoutePattern, j as MachineName, l as ShutdownOptions, lt as ExitTrigger, m as RequestEnd, mt as ProcessPort, n as HealthServer, nt as recoverSpan, o as PhaseWork, ot as EXIT_SIGNALS, p as stageRank, pt as ProcessHost, q as Transport, r as Lifecycle, rt as OperationOf, s as RunOptions, st as EmbeddedProcess, t as DependencyHandle, tt as heartbeatDetection, u as Stage, ut as HostStream, v as RequestStream, vt as isExitSignal, w as ProbeKind, wt as Operation, x as Health, xt as nodeProcess, y as RequestWork, yt as isMessageShape, z as FORMAT } from "./lifecycle-fgYfxrHK.js";
-import { a as WatchdogSpec, c as Profile, d as kubernetes, f as localDev, i as Watchdog, l as daemon, n as ServiceBuilder, o as watchdog, r as service, s as PodTiming, t as ControllerBuilder, u as ephemeral } from "./builder-DawJhq4p.js";
+import { $ as Step, A as EPISODES_KEPT, At as Idempotency, B as Check, C as ProbeAnswer, Ct as embeddedProcess, D as CheckMetrics, Dt as matchesMessage, E as CheckFn, Et as isMessageShape, F as Result, G as ExitCodes, H as CooperativeFloor, J as MessageTrigger, K as FORMAT, L as LifecycleEvent, M as HealthMetrics, Mt as mayReissue, N as redact, O as DURATION_FRACTIONS, Ot as nodeProcess, P as Outcome, Q as ServiceDescription, R as MachineName, S as HealthVerdict, St as ProcessTrigger, T as StartOptions, Tt as isExitSignal, U as Coupling, V as Controller, W as Dependency, X as ProgressMarks, Y as Phase, Z as RepeatSignal, _ as RequestOptions, _t as HostStream, a as PhaseContext, at as asDebounce, b as CheckVerdict, bt as ProcessHost, c as STAGES, ct as heartbeatDetection, d as TransportImpl, dt as RoutePattern, et as Transport, f as cooperativeAbortAt, ft as RouteRequest, g as RequestFn, gt as ExitTrigger, h as RequestEvent, ht as ExitSignal, i as LifecycleState, it as Timing, j as Episode, jt as Operation, k as DurationHistogram, kt as processPortOf, l as ShutdownOptions, lt as recoverSpan, m as RequestEnd, mt as EmbeddedProcess, n as HealthServer, nt as Poll, o as PhaseWork, ot as asWindow, p as stageRank, pt as EXIT_SIGNALS, q as Gate, r as Lifecycle, rt as Span, s as RunOptions, st as failSpan, t as DependencyHandle, tt as FailsBy, u as Stage, ut as OperationOf, v as RequestStream, vt as MessageShape, w as ProbeKind, wt as isExitCode, x as Health, xt as ProcessPort, y as RequestWork, yt as PROCESS_TRIGGERS, z as Recorder } from "./lifecycle-CMedxaQd.js";
+import { a as WatchdogSpec, c as Profile, d as kubernetes, f as localDev, i as Watchdog, l as daemon, n as ServiceBuilder, o as watchdog, r as service, s as PodTiming, t as ControllerBuilder, u as ephemeral } from "./builder-CsSO4LzB.js";
 import { Heartbeat, HeartbeatOptions, detectionTime, heartbeat } from "./heartbeat.js";
 import { a as keyedLimiter, i as TakeRequest, n as LimiterStore, o as memoryStore, r as MemoryStore, t as KeyedLimiter } from "./store-D5HLgBnW.js";
 import { a as limiterStateOf, i as limiterMachine, n as SLIDING_WINDOW_MACHINE, r as TOKEN_BUCKET_MACHINE, t as LimiterState } from "./limiter-B0849Vhp.js";
-import { a as JobSource, c as QueueKnobs, d as queueDrain, f as redeliverOnce, i as JobSettle, l as RedeliveryPolicy, n as JobContext, o as QueueDrain, p as UnitRequest, r as JobFailure, s as QueueDrainOptions, t as JobAction, u as RequeueCause } from "./queue-snRHK3Jm.js";
+import { a as JobSource, c as QueueKnobs, d as queueDrain, f as redeliverOnce, i as JobSettle, l as RedeliveryPolicy, n as JobContext, o as QueueDrain, p as UnitRequest, r as JobFailure, s as QueueDrainOptions, t as JobAction, u as RequeueCause } from "./queue-CndiCVaY.js";
 import { Extent, Quantity } from "@mark1russell7/quantity";
 import { replay } from "@mark1russell7/algebra/replay";
 import { Machine } from "@mark1russell7/algebra";
@@ -292,5 +292,5 @@ export declare const drainMachine: (description: ServiceDescription) => Machine;
 /** These are the static machines of the runtime, by the name that their events carry. */
 export declare const MACHINES: Readonly<Record<Exclude<MachineName, "drain" | "client">, Machine>>;
 //#endregion
-export { type Allowed, type Backoff, type Binding, type Bound, type Cancel, type Check, type CheckFn, type CheckVerdict, type Clock, type Controller, type ControllerBuilder, type CooperativeFloor, type Coupling, type Decision, type DelayBand, type Dependency, type DependencyHandle, EXIT_SIGNALS, type EmbeddedProcess, type Evaluated, type ExitCodes, type ExitSignal, type ExitTrigger, FORMAT, type FailsBy, type Finding, type Gate, type Health, type HealthServer, type HealthVerdict, type Heartbeat, type HeartbeatOptions, type HostStream, type Idempotency, type Inequality, type IterationContext, type Jitter, type JobAction, type JobContext, type JobFailure, type JobSettle, type JobSource, type KeyState, type KeyedLimiter, type Lifecycle, type LifecycleEvent, type LifecycleState, type LimitPolicy, type LimitRule, type Limiter, type LimiterEviction, type LimiterSpec, type LimiterState, type LimiterStore, type LimiterStoreSpec, type MachineName, type MemoryStore, type MessageShape, type MessageTrigger, type Operation, type OperationOf, type Outcome, PROCESS_TRIGGERS, type Phase, type PhaseContext, type PhaseWork, type PlatformClockOptions, type PodTiming, type Poll, type ProbeAnswer, type ProbeKind, type ProcessHost, type ProcessPort, type ProcessTrigger, type Profile, type ProgressMarks, type QueueDrain, type QueueDrainOptions, type QueueKnobs, type Recorder, type RedeliveryPolicy, type RefusalCause, type Refused, type RepeatSignal, type RequestEnd, type RequestEvent, type RequestFn, type RequestOptions, type RequestStream, type RequestWork, type RequeueCause, type Result, type RoutePattern, type RouteRequest, type Rule, type RunOptions, SLIDING_WINDOW_MACHINE, STAGES, type Scenario, type ServiceBuilder, type ServiceDescription, type ShutdownOptions, type Side, type SlidingWindow, type Span, type Stage, type StartOptions, type Step, type StoreFailure, TOKEN_BUCKET_MACHINE, type TakeRequest, type TakeStep, type Term, type Timing, type TokenBucket, type Transport, type TransportImpl, type UnitRequest, type Verdict, type Watchdog, type WatchdogSpec, type WorkerKnobs, type WorkerLoop, type WorkerLoopOptions, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, cooperativeAbortAt, daemon, decide, detectionTime, embeddedProcess, ephemeral, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redeliverOnce, replay, retryAfterSeconds, service, stageRank, watchdog };
+export { type Allowed, type Backoff, type Binding, type Bound, type Cancel, type Check, type CheckFn, type CheckMetrics, type CheckVerdict, type Clock, type Controller, type ControllerBuilder, type CooperativeFloor, type Coupling, DURATION_FRACTIONS, type Decision, type DelayBand, type Dependency, type DependencyHandle, type DurationHistogram, EPISODES_KEPT, EXIT_SIGNALS, type EmbeddedProcess, type Episode, type Evaluated, type ExitCodes, type ExitSignal, type ExitTrigger, FORMAT, type FailsBy, type Finding, type Gate, type Health, type HealthMetrics, type HealthServer, type HealthVerdict, type Heartbeat, type HeartbeatOptions, type HostStream, type Idempotency, type Inequality, type IterationContext, type Jitter, type JobAction, type JobContext, type JobFailure, type JobSettle, type JobSource, type KeyState, type KeyedLimiter, type Lifecycle, type LifecycleEvent, type LifecycleState, type LimitPolicy, type LimitRule, type Limiter, type LimiterEviction, type LimiterSpec, type LimiterState, type LimiterStore, type LimiterStoreSpec, type MachineName, type MemoryStore, type MessageShape, type MessageTrigger, type Operation, type OperationOf, type Outcome, PROCESS_TRIGGERS, type Phase, type PhaseContext, type PhaseWork, type PlatformClockOptions, type PodTiming, type Poll, type ProbeAnswer, type ProbeKind, type ProcessHost, type ProcessPort, type ProcessTrigger, type Profile, type ProgressMarks, type QueueDrain, type QueueDrainOptions, type QueueKnobs, type Recorder, type RedeliveryPolicy, type RefusalCause, type Refused, type RepeatSignal, type RequestEnd, type RequestEvent, type RequestFn, type RequestOptions, type RequestStream, type RequestWork, type RequeueCause, type Result, type RoutePattern, type RouteRequest, type Rule, type RunOptions, SLIDING_WINDOW_MACHINE, STAGES, type Scenario, type ServiceBuilder, type ServiceDescription, type ShutdownOptions, type Side, type SlidingWindow, type Span, type Stage, type StartOptions, type Step, type StoreFailure, TOKEN_BUCKET_MACHINE, type TakeRequest, type TakeStep, type Term, type Timing, type TokenBucket, type Transport, type TransportImpl, type UnitRequest, type Verdict, type Watchdog, type WatchdogSpec, type WorkerKnobs, type WorkerLoop, type WorkerLoopOptions, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, cooperativeAbortAt, daemon, decide, detectionTime, embeddedProcess, ephemeral, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redact, redeliverOnce, replay, retryAfterSeconds, service, stageRank, watchdog };
 //# sourceMappingURL=index.d.ts.map

package/build/index.js

@@ -136,6 +136,114 @@ const latch = (timing, clock, tripped, changed, trace) => {
 	};
 };
 //#endregion
+//#region src/metrics.ts
+/** These fractions of the timeout of a poll are the bounds of its histogram. */
+const DURATION_FRACTIONS = [
+	.05,
+	.1,
+	.25,
+	.5,
+	.75,
+	1
+];
+/** The runtime keeps this many episodes of each check. */
+const EPISODES_KEPT = 8;
+/**
+* This function removes the secrets that a cause can hold. These are the user and the password of a
+* URL, and the value of a key, for example `password=`. A driver error can hold a connection string.
+*/
+const redact = (cause) => cause.replace(/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s/@]*@/gi, "$1***@").replace(/\b(password|passwd|pwd|secret|token|api[_-]?key|auth)(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s,;&]+)/gi, "$1$2***");
+/** This function makes a meter. */
+const meter = () => {
+	const checks = /* @__PURE__ */ new Map();
+	const transitions = /* @__PURE__ */ new Map();
+	const add = (check) => {
+		if (checks.has(check.name)) return;
+		const timeout = valueIn(check.poll.timeout, "ms");
+		checks.set(check.name, {
+			ok: 0,
+			error: 0,
+			indeterminate: 0,
+			thrown: 0,
+			forced: 0,
+			le: DURATION_FRACTIONS.map((fraction) => fraction * timeout),
+			counts: DURATION_FRACTIONS.map(() => 0),
+			timedOut: 0,
+			episodes: [],
+			open: null
+		});
+	};
+	const close = (counted, end) => {
+		if (counted.open === null) return null;
+		return {
+			start: counted.open.start,
+			end,
+			failures: counted.open.failures,
+			cause: counted.open.cause
+		};
+	};
+	const poll = (check, result) => {
+		add(check);
+		const counted = checks.get(check.name);
+		counted[result.outcome] += 1;
+		if (result.thrown) counted.thrown += 1;
+		if (result.forced) counted.forced += 1;
+		if (result.timedOut) counted.timedOut += 1;
+		else {
+			const at = counted.le.findIndex((bound) => result.took <= bound);
+			const bucket = at === -1 ? counted.counts.length - 1 : at;
+			counted.counts[bucket] = (counted.counts[bucket] ?? 0) + 1;
+		}
+		if (result.outcome === "error") {
+			if (counted.open === null) counted.open = {
+				start: result.at,
+				failures: 0,
+				cause: result.cause === null ? null : redact(result.cause)
+			};
+			counted.open.failures += 1;
+		} else if (result.outcome === "ok" && counted.open !== null) {
+			counted.episodes.push(close(counted, result.at));
+			counted.open = null;
+			if (counted.episodes.length > 8) counted.episodes.shift();
+		}
+	};
+	const event = (event) => {
+		const key = `${event.machine} ${event.instance} ${event.from} -> ${event.to}`;
+		transitions.set(key, (transitions.get(key) ?? 0) + 1);
+	};
+	const metrics = () => {
+		const out = {};
+		for (const [name, counted] of checks) {
+			const open = close(counted, null);
+			out[name] = {
+				outcomes: {
+					ok: counted.ok,
+					error: counted.error,
+					indeterminate: counted.indeterminate
+				},
+				thrown: counted.thrown,
+				forced: counted.forced,
+				durations: {
+					le: [...counted.le],
+					counts: [...counted.counts],
+					timedOut: counted.timedOut
+				},
+				episodes: open === null ? [...counted.episodes] : [...counted.episodes, open].slice(-8)
+			};
+		}
+		return {
+			checks: out,
+			transitions: Object.fromEntries(transitions)
+		};
+	};
+	return {
+		add,
+		poll,
+		event,
+		metrics
+	};
+};
+//#endregion
 //#region src/poller.ts
 const message$1 = (error) => error instanceof Error ? error.message : String(error);
 /**
@@ -304,7 +412,12 @@ const healthRuntime = (description, runs, options = {}) => {
 	let draining = false;
 	let stopped = false;
 	let gate = "closed";
-	const tracer = (machine, instance) => (from, to, on) => options.record?.({
+	const counts = meter();
+	const record = (event) => {
+		counts.event(event);
+		options.record?.(event);
+	};
+	const tracer = (machine, instance) => (from, to, on) => record({
 		at: clock.now(),
 		machine,
 		instance,
@@ -316,7 +429,7 @@ const healthRuntime = (description, runs, options = {}) => {
 	const moveGate = (to, on, detail) => {
 		const from = gate;
 		gate = to;
-		options.record?.({
+		record({
 			at: clock.now(),
 			machine: "gate",
 			instance: description.name,
@@ -350,6 +463,7 @@ const healthRuntime = (description, runs, options = {}) => {
 				random,
 				forced: () => forced.has(check.name) || check.dependency !== null && forced.has(check.dependency),
 				record: (result) => {
+					counts.poll(check, result);
 					made?.record(result);
 					if (result.outcome === "ok" && check.dependency !== null) {
 						if (lateDependencies.delete(check.dependency)) changed();
@@ -359,6 +473,7 @@ const healthRuntime = (description, runs, options = {}) => {
 			})
 		};
 		entries.set(check.name, entry);
+		counts.add(check);
 		if (late && !stopped) {
 			entry.poller.pollNow();
 			entry.poller.start(entry.poller.jittered());
@@ -425,6 +540,7 @@ const healthRuntime = (description, runs, options = {}) => {
 	});
 	return {
 		verdict,
+		metrics: counts.metrics,
 		probe(kind, controller) {
 			const now = verdict();
 			if (kind === "startup") return answer(kind, now.started);
@@ -3555,6 +3671,6 @@ const MACHINES = {
 	watchdog: WATCHDOG_MACHINE
 };
 //#endregion
-export { DECLARATION_RULE, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, LATCH_MACHINE, LIMITER_RULES, MACHINES, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, TOKEN_BUCKET_MACHINE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, capKnob, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redeliverOnce, replay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, verdictOf, watchdog, workerLoop };
+export { DECLARATION_RULE, DURATION_FRACTIONS, EPISODES_KEPT, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, LATCH_MACHINE, LIMITER_RULES, MACHINES, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, TOKEN_BUCKET_MACHINE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, backoffBand, backoffDelay, backoffProblems, capKnob, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, localDev, longestRetryAfter, matchesMessage, mayReissue, memoryStore, nodeProcess, platformClock, processPortOf, queueDrain, realClock, recoverSpan, redact, redeliverOnce, replay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, verdictOf, watchdog, workerLoop };
 
 //# sourceMappingURL=index.js.map