@mark1russell7/service 0.8.19 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/build/backoff.d.ts +44 -2
- package/build/{builder-zO48Vgoj.d.ts → builder.d.ts} +14 -101
- package/build/builder.js +499 -0
- package/build/builder.js.map +1 -0
- package/build/{call-context-DDhcVvn5.d.ts → call-context.d.ts} +3 -4
- package/build/channel.d.ts +1 -1
- package/build/channel.js.map +1 -1
- package/build/{core-BSzt4RI2.d.ts → client/core.d.ts} +32 -61
- package/build/{machine-BhVeHr_m.js → client/core.js} +4 -153
- package/build/client/core.js.map +1 -0
- package/build/{dial-ChwIcvd4.d.ts → client/dial.d.ts} +3 -4
- package/build/client/dial.js +52 -0
- package/build/client/dial.js.map +1 -0
- package/build/client/machine.js +29 -0
- package/build/client/machine.js.map +1 -0
- package/build/client/meets.d.ts +35 -0
- package/build/client/meets.js +78 -0
- package/build/client/meets.js.map +1 -0
- package/build/clock.d.ts +2 -2
- package/build/clock.js +1 -1
- package/build/conformance.d.ts +122 -0
- package/build/conformance.js +469 -0
- package/build/conformance.js.map +1 -0
- package/build/contract.d.ts +28 -0
- package/build/contract.js +184 -0
- package/build/contract.js.map +1 -0
- package/build/description.d.ts +161 -0
- package/build/description.js +11 -0
- package/build/description.js.map +1 -0
- package/build/drains.js +35 -0
- package/build/drains.js.map +1 -0
- package/build/events.d.ts +25 -0
- package/build/fakes.d.ts +43 -0
- package/build/fakes.js +93 -0
- package/build/fakes.js.map +1 -0
- package/build/grpc/client.d.ts +153 -0
- package/build/grpc/client.js +474 -0
- package/build/grpc/client.js.map +1 -0
- package/build/grpc/connection.d.ts +97 -0
- package/build/grpc/connection.js +59 -0
- package/build/grpc/connection.js.map +1 -0
- package/build/grpc/machine.d.ts +12 -0
- package/build/grpc/machine.js +149 -0
- package/build/grpc/machine.js.map +1 -0
- package/build/grpc/meets.d.ts +14 -0
- package/build/grpc/meets.js +105 -0
- package/build/grpc/meets.js.map +1 -0
- package/build/grpc/starting.d.ts +33 -0
- package/build/grpc/starting.js +89 -0
- package/build/grpc/starting.js.map +1 -0
- package/build/grpc.d.ts +9 -303
- package/build/grpc.js +7 -854
- package/build/health.d.ts +93 -0
- package/build/health.js +225 -0
- package/build/health.js.map +1 -0
- package/build/heartbeat.d.ts +6 -2
- package/build/heartbeat.js.map +1 -1
- package/build/host/discovery.d.ts +61 -0
- package/build/host/discovery.js +113 -0
- package/build/host/discovery.js.map +1 -0
- package/build/host/env.d.ts +14 -0
- package/build/host/env.js +57 -0
- package/build/host/env.js.map +1 -0
- package/build/host/fakes.d.ts +82 -0
- package/build/host/fakes.js +239 -0
- package/build/host/fakes.js.map +1 -0
- package/build/host/group.d.ts +142 -0
- package/build/host/group.js +524 -0
- package/build/host/group.js.map +1 -0
- package/build/host/install.d.ts +96 -0
- package/build/host/install.js +330 -0
- package/build/host/install.js.map +1 -0
- package/build/host/lock.d.ts +78 -0
- package/build/host/lock.js +190 -0
- package/build/host/lock.js.map +1 -0
- package/build/host/logs.d.ts +26 -0
- package/build/host/logs.js +48 -0
- package/build/host/logs.js.map +1 -0
- package/build/host/node.d.ts +14 -0
- package/build/host/node.js +309 -0
- package/build/host/node.js.map +1 -0
- package/build/{ports-B1ayTL-j.d.ts → host/ports.d.ts} +15 -16
- package/build/host/ports.js +17 -0
- package/build/host/ports.js.map +1 -0
- package/build/host/probes.d.ts +51 -0
- package/build/host/probes.js +139 -0
- package/build/host/probes.js.map +1 -0
- package/build/host/processes.d.ts +32 -0
- package/build/host/processes.js +63 -0
- package/build/host/processes.js.map +1 -0
- package/build/host/render.d.ts +84 -0
- package/build/host/render.js +235 -0
- package/build/host/render.js.map +1 -0
- package/build/host/watch.d.ts +90 -0
- package/build/host/watch.js +165 -0
- package/build/host/watch.js.map +1 -0
- package/build/host.d.ts +13 -671
- package/build/host.js +12 -2126
- package/build/http/answers.js +29 -0
- package/build/http/answers.js.map +1 -0
- package/build/http/fake-server.js +61 -0
- package/build/http/fake-server.js.map +1 -0
- package/build/http/outcome.d.ts +115 -0
- package/build/http/outcome.js +325 -0
- package/build/http/outcome.js.map +1 -0
- package/build/http/probes.d.ts +83 -0
- package/build/http/probes.js +232 -0
- package/build/http/probes.js.map +1 -0
- package/build/http/transport.d.ts +165 -0
- package/build/http/transport.js +518 -0
- package/build/http/transport.js.map +1 -0
- package/build/http-client/client.d.ts +121 -0
- package/build/http-client/client.js +330 -0
- package/build/http-client/client.js.map +1 -0
- package/build/http-client/connection.d.ts +78 -0
- package/build/http-client/connection.js +54 -0
- package/build/http-client/connection.js.map +1 -0
- package/build/http-client/machine.d.ts +14 -0
- package/build/http-client/machine.js +109 -0
- package/build/http-client/machine.js.map +1 -0
- package/build/http-client/meets.d.ts +11 -0
- package/build/http-client/meets.js +74 -0
- package/build/http-client/meets.js.map +1 -0
- package/build/http-client/session.d.ts +87 -0
- package/build/http-client/session.js +99 -0
- package/build/http-client/session.js.map +1 -0
- package/build/http-client/starting.d.ts +25 -0
- package/build/http-client/starting.js +65 -0
- package/build/http-client/starting.js.map +1 -0
- package/build/http-client.d.ts +9 -330
- package/build/http-client.js +8 -707
- package/build/http.d.ts +5 -353
- package/build/http.js +3 -1086
- package/build/identity.d.ts +16 -0
- package/build/identity.js +44 -0
- package/build/identity.js.map +1 -0
- package/build/index.d.ts +35 -412
- package/build/index.js +25 -3278
- package/build/knobs.d.ts +1 -0
- package/build/{knobs-BJcz6m01.js → knobs.js} +2 -34
- package/build/knobs.js.map +1 -0
- package/build/latch.d.ts +29 -0
- package/build/latch.js +122 -0
- package/build/latch.js.map +1 -0
- package/build/{rules-HyFCBStQ.d.ts → law.d.ts} +19 -37
- package/build/law.js +149 -0
- package/build/law.js.map +1 -0
- package/build/lifecycle.d.ts +144 -0
- package/build/lifecycle.js +411 -0
- package/build/lifecycle.js.map +1 -0
- package/build/limiter/machine.d.ts +40 -0
- package/build/limiter/machine.js +279 -0
- package/build/limiter/machine.js.map +1 -0
- package/build/{policy-CBcK2XGg.d.ts → limiter/policy.d.ts} +28 -29
- package/build/{policy-UYALkqVq.js → limiter/policy.js} +10 -9
- package/build/limiter/policy.js.map +1 -0
- package/build/{limiter-BNKnwzeY.d.ts → limiter/quota.d.ts} +32 -74
- package/build/{limiter-DrM0PQV9.js → limiter/quota.js} +3 -418
- package/build/limiter/quota.js.map +1 -0
- package/build/{store-DFwFKrsB.d.ts → limiter/store.d.ts} +10 -11
- package/build/limiter/store.js +145 -0
- package/build/limiter/store.js.map +1 -0
- package/build/limiter.d.ts +4 -3
- package/build/limiter.js +4 -2
- package/build/lint.d.ts +152 -0
- package/build/lint.js +576 -0
- package/build/lint.js.map +1 -0
- package/build/logger.d.ts +71 -0
- package/build/logger.js +183 -0
- package/build/logger.js.map +1 -0
- package/build/machines.d.ts +41 -0
- package/build/machines.js +451 -0
- package/build/machines.js.map +1 -0
- package/build/metrics.d.ts +53 -0
- package/build/metrics.js +111 -0
- package/build/metrics.js.map +1 -0
- package/build/mongodb/client.d.ts +131 -0
- package/build/mongodb/client.js +388 -0
- package/build/mongodb/client.js.map +1 -0
- package/build/mongodb/connection.d.ts +112 -0
- package/build/mongodb/connection.js +12 -0
- package/build/mongodb/connection.js.map +1 -0
- package/build/mongodb/machine.d.ts +14 -0
- package/build/mongodb/machine.js +122 -0
- package/build/mongodb/machine.js.map +1 -0
- package/build/mongodb/meets.d.ts +15 -0
- package/build/mongodb/meets.js +103 -0
- package/build/mongodb/meets.js.map +1 -0
- package/build/mongodb/starting.d.ts +29 -0
- package/build/mongodb/starting.js +77 -0
- package/build/mongodb/starting.js.map +1 -0
- package/build/mongodb.d.ts +8 -296
- package/build/mongodb.js +7 -682
- package/build/operation.d.ts +27 -0
- package/build/{operation-DF73RZOI.js → operation.js} +2 -2
- package/build/operation.js.map +1 -0
- package/build/otel.d.ts +2 -2
- package/build/pg/client.d.ts +96 -0
- package/build/pg/client.js +277 -0
- package/build/pg/client.js.map +1 -0
- package/build/pg/connection.d.ts +75 -0
- package/build/pg/connection.js +11 -0
- package/build/pg/connection.js.map +1 -0
- package/build/pg/machine.d.ts +13 -0
- package/build/pg/machine.js +111 -0
- package/build/pg/machine.js.map +1 -0
- package/build/pg/meets.d.ts +14 -0
- package/build/pg/meets.js +83 -0
- package/build/pg/meets.js.map +1 -0
- package/build/pg/starting.d.ts +26 -0
- package/build/pg/starting.js +62 -0
- package/build/pg/starting.js.map +1 -0
- package/build/pg.d.ts +8 -219
- package/build/pg.js +7 -525
- package/build/poller.d.ts +9 -0
- package/build/poller.js +154 -0
- package/build/poller.js.map +1 -0
- package/build/process.d.ts +114 -0
- package/build/process.js +170 -0
- package/build/process.js.map +1 -0
- package/build/profiles.d.ts +68 -0
- package/build/profiles.js +120 -0
- package/build/profiles.js.map +1 -0
- package/build/{queue-DXdw1vLO.d.ts → queue.d.ts} +16 -29
- package/build/queue.js +279 -0
- package/build/queue.js.map +1 -0
- package/build/rabbitmq/client.d.ts +144 -0
- package/build/rabbitmq/client.js +426 -0
- package/build/rabbitmq/client.js.map +1 -0
- package/build/rabbitmq/connection.d.ts +91 -0
- package/build/rabbitmq/connection.js +15 -0
- package/build/rabbitmq/connection.js.map +1 -0
- package/build/rabbitmq/machine.d.ts +13 -0
- package/build/rabbitmq/machine.js +159 -0
- package/build/rabbitmq/machine.js.map +1 -0
- package/build/rabbitmq/meets.d.ts +12 -0
- package/build/rabbitmq/meets.js +96 -0
- package/build/rabbitmq/meets.js.map +1 -0
- package/build/rabbitmq/starting.d.ts +27 -0
- package/build/rabbitmq/starting.js +67 -0
- package/build/rabbitmq/starting.js.map +1 -0
- package/build/rabbitmq.d.ts +8 -283
- package/build/rabbitmq.js +7 -742
- package/build/redis/client.d.ts +86 -0
- package/build/redis/client.js +285 -0
- package/build/redis/client.js.map +1 -0
- package/build/redis/connection.d.ts +57 -0
- package/build/redis/connection.js +11 -0
- package/build/redis/connection.js.map +1 -0
- package/build/redis/limiter.d.ts +43 -0
- package/build/redis/limiter.js +164 -0
- package/build/redis/limiter.js.map +1 -0
- package/build/redis/machine.d.ts +12 -0
- package/build/redis/machine.js +144 -0
- package/build/redis/machine.js.map +1 -0
- package/build/redis/meets.d.ts +11 -0
- package/build/redis/meets.js +75 -0
- package/build/redis/meets.js.map +1 -0
- package/build/redis/operations.d.ts +5 -0
- package/build/redis/operations.js +114 -0
- package/build/redis/operations.js.map +1 -0
- package/build/redis.d.ts +10 -207
- package/build/redis.js +9 -771
- package/build/requests.d.ts +66 -0
- package/build/requests.js +279 -0
- package/build/requests.js.map +1 -0
- package/build/retrier.d.ts +62 -0
- package/build/{retrier-Dnjhvxol.js → retrier.js} +2 -2
- package/build/retrier.js.map +1 -0
- package/build/routes.d.ts +35 -0
- package/build/routes.js +77 -0
- package/build/routes.js.map +1 -0
- package/build/rules.d.ts +19 -2
- package/build/stops.d.ts +51 -0
- package/build/stops.js +86 -0
- package/build/stops.js.map +1 -0
- package/build/testing.d.ts +8 -270
- package/build/testing.js +4 -1027
- package/build/testing.js.map +1 -1
- package/build/timing.d.ts +63 -0
- package/build/{timing-DZl36uXs.js → timing.js} +2 -2
- package/build/timing.js.map +1 -0
- package/build/units.d.ts +17 -0
- package/build/units.js +236 -0
- package/build/units.js.map +1 -0
- package/build/watchdog.d.ts +38 -0
- package/build/watchdog.js +57 -0
- package/build/watchdog.js.map +1 -0
- package/build/websocket/client.d.ts +136 -0
- package/build/websocket/client.js +422 -0
- package/build/websocket/client.js.map +1 -0
- package/build/websocket/connection.d.ts +83 -0
- package/build/websocket/connection.js +57 -0
- package/build/websocket/connection.js.map +1 -0
- package/build/websocket/machine.d.ts +12 -0
- package/build/websocket/machine.js +155 -0
- package/build/websocket/machine.js.map +1 -0
- package/build/websocket/meets.d.ts +14 -0
- package/build/websocket/meets.js +92 -0
- package/build/websocket/meets.js.map +1 -0
- package/build/websocket/starting.d.ts +25 -0
- package/build/websocket/starting.js +69 -0
- package/build/websocket/starting.js.map +1 -0
- package/build/websocket.d.ts +9 -264
- package/build/websocket.js +8 -774
- package/build/worker.d.ts +92 -0
- package/build/worker.js +153 -0
- package/build/worker.js.map +1 -0
- package/package.json +5 -4
- package/build/backoff-QTRGfkEK.d.ts +0 -45
- package/build/clock-DpVgpY4h.d.ts +0 -2
- package/build/grpc.js.map +0 -1
- package/build/host.js.map +0 -1
- package/build/http-client.js.map +0 -1
- package/build/http.js.map +0 -1
- package/build/index.js.map +0 -1
- package/build/knobs-BJcz6m01.js.map +0 -1
- package/build/lifecycle-D0J-zwZF.d.ts +0 -374
- package/build/limiter-DrM0PQV9.js.map +0 -1
- package/build/logger-B6drv9w0.d.ts +0 -488
- package/build/machine-BhVeHr_m.js.map +0 -1
- package/build/mongodb.js.map +0 -1
- package/build/operation-DF73RZOI.js.map +0 -1
- package/build/pg.js.map +0 -1
- package/build/policy-UYALkqVq.js.map +0 -1
- package/build/process-7R9M29wN.js +0 -437
- package/build/process-7R9M29wN.js.map +0 -1
- package/build/rabbitmq.js.map +0 -1
- package/build/redis.js.map +0 -1
- package/build/requests-DD2ALv0_.js +0 -583
- package/build/requests-DD2ALv0_.js.map +0 -1
- package/build/retrier-Dnjhvxol.js.map +0 -1
- package/build/timing-DZl36uXs.js.map +0 -1
- package/build/websocket.js.map +0 -1
package/build/index.js
CHANGED
|
@@ -1,3281 +1,28 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { a as runUnit, c as untilAbort, i as retryOf, l as routePatternProblems, n as boundProblems, o as settleWithin, r as flights, s as sleeper, t as tracker } from "./requests-DD2ALv0_.js";
|
|
8
|
-
import { backoffBand, backoffDelay, backoffProblems } from "./backoff.js";
|
|
1
|
+
import { asDebounce, asWindow, failSpan, heartbeatDetection, recoverSpan } from "./timing.js";
|
|
2
|
+
import { FORMAT } from "./description.js";
|
|
3
|
+
import { mayReissue } from "./operation.js";
|
|
4
|
+
import { service } from "./builder.js";
|
|
5
|
+
import { compress, compression, evaluate, inequalities, symbolsOf, verdictOf } from "./law.js";
|
|
6
|
+
import { DRAIN_RULE_IDS, STOPS, STOP_RULE, lintStop } from "./stops.js";
|
|
9
7
|
import { DRAIN_RULES, O04, O09, O10, O16 } from "./rules.js";
|
|
8
|
+
import { BOUND_RULES, CLOSE_BOUND_RULE, DECLARATION_RULE, LAW_STATISTIC, LIMITER_RULES, NAME_RULE, PLATFORM_MARKS, PROGRESS_RULE, SERVICE_RULES, UNIT_BOUND_RULE, compressionOf, lint, serviceBindings } from "./lint.js";
|
|
9
|
+
import { IDENTITY_TYPES } from "./identity.js";
|
|
10
|
+
import { platformClock, realClock, unrefClock } from "./clock.js";
|
|
10
11
|
import { detectionTime, heartbeat } from "./heartbeat.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { backoffBand, backoffDelay, backoffProblems } from "./backoff.js";
|
|
13
|
+
import { retrier, retry, retryAfterDelay } from "./retrier.js";
|
|
14
|
+
import { keyedQuota, memoryQuotaStore, quotaProblems, quotaRule, quotaStep, takeQuotas } from "./limiter/quota.js";
|
|
15
|
+
import { SLIDING_WINDOW_MACHINE, TOKEN_BUCKET_MACHINE, limiterMachine, limiterStateOf } from "./limiter/machine.js";
|
|
16
|
+
import { capacityOf, decide, freeAt, idleSpan, limitRule, limiterProblems, longestRetryAfter, overCapacity, retryAfterSeconds, unitsOf } from "./limiter/policy.js";
|
|
17
|
+
import { keyedLimiter, memoryStore } from "./limiter/store.js";
|
|
18
|
+
import { DURATION_FRACTIONS, EPISODES_KEPT, redact } from "./metrics.js";
|
|
19
|
+
import { LOG_LEVELS, atLeast, consoleLogger, logEvents } from "./logger.js";
|
|
20
|
+
import { STAGES, cooperativeAbortAt, stageRank } from "./lifecycle.js";
|
|
21
|
+
import { EXIT_SIGNALS, PROCESS_TRIGGERS, UNCAUGHT_ORIGINS, embeddedProcess, isExitCode, isExitSignal, isMessageShape, matchesMessage, nodeProcess, processPortOf } from "./process.js";
|
|
22
|
+
import { daemon, ephemeral, kubernetes, localDev } from "./profiles.js";
|
|
23
|
+
import { watchdog } from "./watchdog.js";
|
|
24
|
+
import { workerLoop } from "./worker.js";
|
|
25
|
+
import { queueDrain, redeliverOnce } from "./queue.js";
|
|
26
|
+
import { EXIT_MACHINE, GATE_MACHINE, LATCH_MACHINE, MACHINES, POLLER_MACHINE, WATCHDOG_MACHINE, WINDOW_MACHINE, capKnob, drainMachine } from "./machines.js";
|
|
14
27
|
import { replay } from "@mark1russell7/algebra/replay";
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* This is the format of a service as data: what `describe()` exports, and nothing starts. The
|
|
18
|
-
* timer algebra and the ordering law read it. Each timer has a unit. Each knob comes from the
|
|
19
|
-
* declaration, because the runtime has no defaults. A bound that nobody gives is "no bound", not
|
|
20
|
-
* zero.
|
|
21
|
-
*/
|
|
22
|
-
const FORMAT = "mark1russell7.service/1";
|
|
23
|
-
// src/metrics.ts
|
|
24
|
-
/** These fractions of the timeout of a poll are the bounds of its histogram. */
|
|
25
|
-
const DURATION_FRACTIONS = [
|
|
26
|
-
.05,
|
|
27
|
-
.1,
|
|
28
|
-
.25,
|
|
29
|
-
.5,
|
|
30
|
-
.75,
|
|
31
|
-
1
|
|
32
|
-
];
|
|
33
|
-
/** The runtime keeps this many episodes of each check. */
|
|
34
|
-
const EPISODES_KEPT = 8;
|
|
35
|
-
/**
|
|
36
|
-
* This function removes the secrets that a cause can hold. These are the user and the password of a
|
|
37
|
-
* URL, and the value of a key, for example `password=`. A driver error can hold a connection string.
|
|
38
|
-
*/
|
|
39
|
-
const redact = (cause) => cause.replace(/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s/@]*@/gi, "$1***@").replace(/\b(password|passwd|pwd|secret|token|api[_-]?key|auth)(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s,;&]+)/gi, "$1$2***");
|
|
40
|
-
/** This function makes a meter. */
|
|
41
|
-
const meter = () => {
|
|
42
|
-
const checks = /* @__PURE__ */ new Map();
|
|
43
|
-
const transitions = /* @__PURE__ */ new Map();
|
|
44
|
-
const add = (check) => {
|
|
45
|
-
if (checks.has(check.name)) return;
|
|
46
|
-
const timeout = valueIn(check.poll.timeout, "ms");
|
|
47
|
-
checks.set(check.name, {
|
|
48
|
-
ok: 0,
|
|
49
|
-
error: 0,
|
|
50
|
-
indeterminate: 0,
|
|
51
|
-
thrown: 0,
|
|
52
|
-
forced: 0,
|
|
53
|
-
le: DURATION_FRACTIONS.map((fraction) => fraction * timeout),
|
|
54
|
-
counts: DURATION_FRACTIONS.map(() => 0),
|
|
55
|
-
timedOut: 0,
|
|
56
|
-
episodes: [],
|
|
57
|
-
open: null
|
|
58
|
-
});
|
|
59
|
-
};
|
|
60
|
-
const close = (counted, end) => {
|
|
61
|
-
if (counted.open === null) return null;
|
|
62
|
-
return {
|
|
63
|
-
start: counted.open.start,
|
|
64
|
-
end,
|
|
65
|
-
failures: counted.open.failures,
|
|
66
|
-
cause: counted.open.cause
|
|
67
|
-
};
|
|
68
|
-
};
|
|
69
|
-
const poll = (check, result) => {
|
|
70
|
-
add(check);
|
|
71
|
-
const counted = checks.get(check.name);
|
|
72
|
-
counted[result.outcome] += 1;
|
|
73
|
-
if (result.thrown) counted.thrown += 1;
|
|
74
|
-
if (result.forced) counted.forced += 1;
|
|
75
|
-
if (result.timedOut) counted.timedOut += 1;
|
|
76
|
-
else {
|
|
77
|
-
const at = counted.le.findIndex((bound) => result.took <= bound);
|
|
78
|
-
const bucket = at === -1 ? counted.counts.length - 1 : at;
|
|
79
|
-
counted.counts[bucket] = (counted.counts[bucket] ?? 0) + 1;
|
|
80
|
-
}
|
|
81
|
-
if (result.outcome === "error") {
|
|
82
|
-
if (counted.open === null) counted.open = {
|
|
83
|
-
start: result.at,
|
|
84
|
-
failures: 0,
|
|
85
|
-
cause: result.cause === null ? null : redact(result.cause)
|
|
86
|
-
};
|
|
87
|
-
counted.open.failures += 1;
|
|
88
|
-
} else if (result.outcome === "ok" && counted.open !== null) {
|
|
89
|
-
counted.episodes.push(close(counted, result.at));
|
|
90
|
-
counted.open = null;
|
|
91
|
-
if (counted.episodes.length > 8) counted.episodes.shift();
|
|
92
|
-
}
|
|
93
|
-
};
|
|
94
|
-
const event = (event) => {
|
|
95
|
-
const key = `${event.machine} ${event.instance} ${event.from} -> ${event.to}`;
|
|
96
|
-
transitions.set(key, (transitions.get(key) ?? 0) + 1);
|
|
97
|
-
};
|
|
98
|
-
const metrics = () => {
|
|
99
|
-
const out = {};
|
|
100
|
-
for (const [name, counted] of checks) {
|
|
101
|
-
const open = close(counted, null);
|
|
102
|
-
out[name] = {
|
|
103
|
-
outcomes: {
|
|
104
|
-
ok: counted.ok,
|
|
105
|
-
error: counted.error,
|
|
106
|
-
indeterminate: counted.indeterminate
|
|
107
|
-
},
|
|
108
|
-
thrown: counted.thrown,
|
|
109
|
-
forced: counted.forced,
|
|
110
|
-
durations: {
|
|
111
|
-
le: [...counted.le],
|
|
112
|
-
counts: [...counted.counts],
|
|
113
|
-
timedOut: counted.timedOut
|
|
114
|
-
},
|
|
115
|
-
episodes: open === null ? [...counted.episodes] : [...counted.episodes, open].slice(-8)
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
return {
|
|
119
|
-
checks: out,
|
|
120
|
-
transitions: Object.fromEntries(transitions)
|
|
121
|
-
};
|
|
122
|
-
};
|
|
123
|
-
return {
|
|
124
|
-
add,
|
|
125
|
-
poll,
|
|
126
|
-
event,
|
|
127
|
-
metrics
|
|
128
|
-
};
|
|
129
|
-
};
|
|
130
|
-
// src/health.ts
|
|
131
|
-
/**
|
|
132
|
-
* This function makes the runtime of a declaration. The service has no defaults of its own: each
|
|
133
|
-
* timing, poll and coupling comes from the description.
|
|
134
|
-
*/
|
|
135
|
-
const healthRuntime = (description, runs, options = {}) => {
|
|
136
|
-
const clock = options.clock ?? realClock;
|
|
137
|
-
const random = options.random ?? Math.random;
|
|
138
|
-
const forced = new Set(options.forceFail ?? []);
|
|
139
|
-
const startedAt = clock.now();
|
|
140
|
-
const budget = valueIn(description.policies.startup, "ms");
|
|
141
|
-
const entries = /* @__PURE__ */ new Map();
|
|
142
|
-
const dependencies = new Set(description.dependencies.map((dependency) => dependency.name));
|
|
143
|
-
const lateDependencies = /* @__PURE__ */ new Set();
|
|
144
|
-
const listeners = /* @__PURE__ */ new Set();
|
|
145
|
-
let started = false;
|
|
146
|
-
let draining = false;
|
|
147
|
-
let stopped = false;
|
|
148
|
-
let gate = "closed";
|
|
149
|
-
const counts = meter();
|
|
150
|
-
const record = (event) => {
|
|
151
|
-
counts.event(event);
|
|
152
|
-
options.record?.(event);
|
|
153
|
-
};
|
|
154
|
-
const tracer = (machine, instance) => (from, to, on) => record({
|
|
155
|
-
at: clock.now(),
|
|
156
|
-
machine,
|
|
157
|
-
instance,
|
|
158
|
-
from,
|
|
159
|
-
to,
|
|
160
|
-
on
|
|
161
|
-
});
|
|
162
|
-
const gateTrace = tracer("gate", description.name);
|
|
163
|
-
const moveGate = (to, on, detail) => {
|
|
164
|
-
const from = gate;
|
|
165
|
-
gate = to;
|
|
166
|
-
record({
|
|
167
|
-
at: clock.now(),
|
|
168
|
-
machine: "gate",
|
|
169
|
-
instance: description.name,
|
|
170
|
-
from,
|
|
171
|
-
to,
|
|
172
|
-
on,
|
|
173
|
-
...detail === void 0 ? {} : { detail }
|
|
174
|
-
});
|
|
175
|
-
};
|
|
176
|
-
let budgetTimer = clock.after(budget, () => {
|
|
177
|
-
budgetTimer = null;
|
|
178
|
-
if (gate !== "open" && gate !== "failed") gateTrace(gate, gate, "startup.over-budget");
|
|
179
|
-
});
|
|
180
|
-
const changed = () => {
|
|
181
|
-
if (listeners.size === 0) return;
|
|
182
|
-
const now = verdict();
|
|
183
|
-
for (const listener of listeners) listener(now);
|
|
184
|
-
};
|
|
185
|
-
const add = (check, given, late) => {
|
|
186
|
-
const run = options.wrapCheck?.(check, given) ?? given;
|
|
187
|
-
const timing = check.gates === null ? null : description.policies[check.gates];
|
|
188
|
-
const made = timing === null ? null : latch(timing, clock, check.gates === "readiness", changed, tracer(timing.type === "window" ? "window" : "latch", check.name));
|
|
189
|
-
const entry = {
|
|
190
|
-
check,
|
|
191
|
-
latch: made,
|
|
192
|
-
poller: poller({
|
|
193
|
-
name: check.name,
|
|
194
|
-
poll: check.poll,
|
|
195
|
-
run,
|
|
196
|
-
clock,
|
|
197
|
-
random,
|
|
198
|
-
forced: () => forced.has(check.name) || check.dependency !== null && forced.has(check.dependency),
|
|
199
|
-
record: (result) => {
|
|
200
|
-
counts.poll(check, result);
|
|
201
|
-
made?.record(result);
|
|
202
|
-
if (result.outcome === "ok" && check.dependency !== null) {
|
|
203
|
-
if (lateDependencies.delete(check.dependency)) changed();
|
|
204
|
-
}
|
|
205
|
-
},
|
|
206
|
-
trace: tracer("poller", check.name)
|
|
207
|
-
})
|
|
208
|
-
};
|
|
209
|
-
entries.set(check.name, entry);
|
|
210
|
-
counts.add(check);
|
|
211
|
-
if (late && !stopped) {
|
|
212
|
-
entry.poller.pollNow();
|
|
213
|
-
entry.poller.start(entry.poller.jittered());
|
|
214
|
-
}
|
|
215
|
-
};
|
|
216
|
-
for (const check of description.checks) {
|
|
217
|
-
const run = runs.get(check.name);
|
|
218
|
-
if (run === void 0) throw new Error(`check ${check.name} has no function: the runtime has no defaults`);
|
|
219
|
-
add(check, run, false);
|
|
220
|
-
}
|
|
221
|
-
const latchesOf = (gateOf, keep = () => true) => [...entries.values()].flatMap((entry) => entry.check.gates === gateOf && entry.latch !== null && keep(entry.check) ? [entry.latch] : []);
|
|
222
|
-
const released = (latches) => latches.every((each) => !each.tripped);
|
|
223
|
-
/** This function tells if a policy fails through a declared coupling from another policy. */
|
|
224
|
-
const coupledOff = (to, base) => description.couplings.some((coupling) => coupling.to === to && !base[coupling.from]);
|
|
225
|
-
const verdict = () => {
|
|
226
|
-
const base = {
|
|
227
|
-
readiness: released(latchesOf("readiness")) && lateDependencies.size === 0,
|
|
228
|
-
liveness: released(latchesOf("liveness"))
|
|
229
|
-
};
|
|
230
|
-
const alive = base.liveness && !coupledOff("liveness", base);
|
|
231
|
-
const open = started && !draining && !coupledOff("readiness", base);
|
|
232
|
-
const ready = open && base.readiness;
|
|
233
|
-
const dependencyReady = (name) => !lateDependencies.has(name) && released(latchesOf("readiness", (check) => check.dependency === name));
|
|
234
|
-
const own = released(latchesOf("readiness", (check) => check.dependency === null));
|
|
235
|
-
const dependenciesOut = {};
|
|
236
|
-
for (const name of dependencies) dependenciesOut[name] = dependencyReady(name);
|
|
237
|
-
const controllers = {};
|
|
238
|
-
for (const controller of description.controllers) controllers[controller.name] = open && own && controller.requires.every(dependencyReady);
|
|
239
|
-
const checks = {};
|
|
240
|
-
for (const [name, entry] of entries) checks[name] = {
|
|
241
|
-
gates: entry.check.gates,
|
|
242
|
-
dependency: entry.check.dependency,
|
|
243
|
-
tripped: entry.latch?.tripped ?? null,
|
|
244
|
-
streak: entry.latch?.streak ?? 0,
|
|
245
|
-
since: entry.latch?.since ?? null,
|
|
246
|
-
last: entry.poller.last
|
|
247
|
-
};
|
|
248
|
-
const elapsed = clock.now() - startedAt;
|
|
249
|
-
return {
|
|
250
|
-
started,
|
|
251
|
-
draining,
|
|
252
|
-
ready,
|
|
253
|
-
alive,
|
|
254
|
-
controllers,
|
|
255
|
-
dependencies: dependenciesOut,
|
|
256
|
-
checks,
|
|
257
|
-
startup: {
|
|
258
|
-
budget,
|
|
259
|
-
elapsed,
|
|
260
|
-
overBudget: !started && elapsed > budget
|
|
261
|
-
}
|
|
262
|
-
};
|
|
263
|
-
};
|
|
264
|
-
const answer = (kind, pass, controller) => ({
|
|
265
|
-
status: pass || kind === "report" ? 200 : 503,
|
|
266
|
-
contentType: "application/health+json",
|
|
267
|
-
body: {
|
|
268
|
-
status: pass ? "pass" : "fail",
|
|
269
|
-
service: description.name,
|
|
270
|
-
probe: kind,
|
|
271
|
-
...controller === void 0 ? {} : { controller },
|
|
272
|
-
...kind === "report" ? { verdict: verdict() } : {}
|
|
273
|
-
}
|
|
274
|
-
});
|
|
275
|
-
return {
|
|
276
|
-
verdict,
|
|
277
|
-
metrics: counts.metrics,
|
|
278
|
-
probe(kind, controller) {
|
|
279
|
-
const now = verdict();
|
|
280
|
-
if (kind === "startup") return answer(kind, now.started);
|
|
281
|
-
if (kind === "liveness") return answer(kind, now.alive);
|
|
282
|
-
if (kind === "report") return answer(kind, now.ready && now.alive);
|
|
283
|
-
if (controller === void 0) return answer(kind, now.ready);
|
|
284
|
-
const known = Object.hasOwn(now.controllers, controller) ? now.controllers[controller] : void 0;
|
|
285
|
-
if (known === void 0) throw new Error(`${description.name} has no controller ${controller}`);
|
|
286
|
-
return answer(kind, known, controller);
|
|
287
|
-
},
|
|
288
|
-
subscribe(listener) {
|
|
289
|
-
listeners.add(listener);
|
|
290
|
-
return () => listeners.delete(listener);
|
|
291
|
-
},
|
|
292
|
-
forceFail(name, on) {
|
|
293
|
-
if (on) forced.add(name);
|
|
294
|
-
else forced.delete(name);
|
|
295
|
-
},
|
|
296
|
-
drain() {
|
|
297
|
-
if (draining) return;
|
|
298
|
-
draining = true;
|
|
299
|
-
changed();
|
|
300
|
-
},
|
|
301
|
-
stop() {
|
|
302
|
-
stopped = true;
|
|
303
|
-
budgetTimer?.();
|
|
304
|
-
budgetTimer = null;
|
|
305
|
-
for (const entry of entries.values()) {
|
|
306
|
-
entry.poller.stop();
|
|
307
|
-
entry.latch?.stop();
|
|
308
|
-
}
|
|
309
|
-
},
|
|
310
|
-
beginStartup() {
|
|
311
|
-
if (gate === "closed") moveGate("starting", "startup.begin");
|
|
312
|
-
},
|
|
313
|
-
failStartup(cause) {
|
|
314
|
-
if (gate === "open" || gate === "failed") return;
|
|
315
|
-
moveGate("failed", "startup.fail", cause);
|
|
316
|
-
budgetTimer?.();
|
|
317
|
-
budgetTimer = null;
|
|
318
|
-
},
|
|
319
|
-
async warmUp() {
|
|
320
|
-
if (started || stopped || gate === "failed" || gate === "sweeping") return;
|
|
321
|
-
moveGate("sweeping", "mark-ready");
|
|
322
|
-
const sweep = clock.now();
|
|
323
|
-
await Promise.all([...entries.values()].map((entry) => entry.poller.pollNow()));
|
|
324
|
-
if (stopped || gate !== "sweeping") return;
|
|
325
|
-
for (const entry of entries.values()) entry.poller.start(entry.poller.jittered() - (clock.now() - sweep));
|
|
326
|
-
started = true;
|
|
327
|
-
budgetTimer?.();
|
|
328
|
-
budgetTimer = null;
|
|
329
|
-
moveGate("open", "sweep.done");
|
|
330
|
-
changed();
|
|
331
|
-
},
|
|
332
|
-
addCheck(check, run) {
|
|
333
|
-
if (entries.has(check.name)) throw new Error(`${description.name} already has a check named ${check.name}`);
|
|
334
|
-
add(check, run, true);
|
|
335
|
-
changed();
|
|
336
|
-
},
|
|
337
|
-
addDependency(name) {
|
|
338
|
-
if (dependencies.has(name)) return;
|
|
339
|
-
dependencies.add(name);
|
|
340
|
-
lateDependencies.add(name);
|
|
341
|
-
changed();
|
|
342
|
-
}
|
|
343
|
-
};
|
|
344
|
-
};
|
|
345
|
-
// src/logger.ts
|
|
346
|
-
/** These are the levels of a log entry, from the least to the most severe. */
|
|
347
|
-
const LOG_LEVELS = [
|
|
348
|
-
"debug",
|
|
349
|
-
"info",
|
|
350
|
-
"warn",
|
|
351
|
-
"error"
|
|
352
|
-
];
|
|
353
|
-
/** This function tells if `level` is at `least` or more severe. */
|
|
354
|
-
const atLeast = (level, least) => LOG_LEVELS.indexOf(level) >= LOG_LEVELS.indexOf(least);
|
|
355
|
-
/** This function gives the fields with the trace ids of the source added. A field of the entry stays. */
|
|
356
|
-
const withTraceIds = (fields, source) => {
|
|
357
|
-
const ids = source();
|
|
358
|
-
if (ids === void 0) return fields;
|
|
359
|
-
return {
|
|
360
|
-
trace_id: ids.trace_id,
|
|
361
|
-
span_id: ids.span_id,
|
|
362
|
-
...fields
|
|
363
|
-
};
|
|
364
|
-
};
|
|
365
|
-
/** This function gives a value of a field that JSON can write: an error as its name, message and stack, and a bigint as text. */
|
|
366
|
-
const jsonValue = (_key, value) => {
|
|
367
|
-
if (typeof value === "bigint") return value.toString();
|
|
368
|
-
if (value instanceof Error) return {
|
|
369
|
-
name: value.name,
|
|
370
|
-
message: value.message,
|
|
371
|
-
...value.stack === void 0 ? {} : { stack: value.stack }
|
|
372
|
-
};
|
|
373
|
-
return value;
|
|
374
|
-
};
|
|
375
|
-
/**
|
|
376
|
-
* This function gives one line of JSON for an entry. A field named `level` or `message` does not
|
|
377
|
-
* replace the level or the message. A field that JSON cannot write, for example a cycle, gives its failure as text.
|
|
378
|
-
*/
|
|
379
|
-
const jsonLine = (level, message, fields) => {
|
|
380
|
-
const { level: _level, message: _message, ...rest } = fields;
|
|
381
|
-
try {
|
|
382
|
-
return JSON.stringify({
|
|
383
|
-
level,
|
|
384
|
-
message,
|
|
385
|
-
...rest
|
|
386
|
-
}, jsonValue);
|
|
387
|
-
} catch (error) {
|
|
388
|
-
return JSON.stringify({
|
|
389
|
-
level,
|
|
390
|
-
message,
|
|
391
|
-
unwritten: error instanceof Error ? error.message : String(error)
|
|
392
|
-
});
|
|
393
|
-
}
|
|
394
|
-
};
|
|
395
|
-
/**
|
|
396
|
-
* This function makes the console logger. It writes each entry at its level or more severe to the
|
|
397
|
-
* method of that level. In the format `text`, it writes the message first and then the fields. In
|
|
398
|
-
* the format `json`, it writes one line of JSON.
|
|
399
|
-
*/
|
|
400
|
-
const consoleLogger = (options = {}) => {
|
|
401
|
-
const least = options.level ?? "info";
|
|
402
|
-
const format = options.format ?? "text";
|
|
403
|
-
const source = options.traceIds;
|
|
404
|
-
if (format !== "text" && format !== "json") throw new Error(`the log format ${JSON.stringify(format)} is not "text" or "json"`);
|
|
405
|
-
return { log(level, message, entryFields) {
|
|
406
|
-
if (!atLeast(level, least)) return;
|
|
407
|
-
const fields = source === void 0 ? entryFields : withTraceIds(entryFields, source);
|
|
408
|
-
const target = options.console ?? globalThis.console;
|
|
409
|
-
if (format === "json") target[level](jsonLine(level, message, fields));
|
|
410
|
-
else if (Object.keys(fields).length === 0) target[level](message);
|
|
411
|
-
else target[level](message, fields);
|
|
412
|
-
} };
|
|
413
|
-
};
|
|
414
|
-
const seconds = (cap) => `${valueIn(cap, "s")} s`;
|
|
415
|
-
/** This function gives the exit code from the detail of an exit event, for example `exit 1`. */
|
|
416
|
-
const exitCodeOf = (detail) => {
|
|
417
|
-
const found = /^exit (\d+)$/.exec(detail ?? "");
|
|
418
|
-
return found === null ? null : Number(found[1]);
|
|
419
|
-
};
|
|
420
|
-
/**
|
|
421
|
-
* This function makes a recorder that writes each lifecycle event to a logger. Thus the logger is
|
|
422
|
-
* one sink of the same stream as the recorder, not a second set of calls. The levels are these:
|
|
423
|
-
* - `error`: each error of the lifecycle, and an exit at once on a repeat signal.
|
|
424
|
-
* - `warn`: a phase at its cap, a failed startup, a startup over its budget, a tripped latch and a blocked event loop.
|
|
425
|
-
* - `info`: each step of the startup, the start and the end of each phase, the exit, and each recovery.
|
|
426
|
-
* - `debug`: each other transition, for example each poll of a check.
|
|
427
|
-
* With the description, the entry of a phase also gives its cap. A failure of the logger does not stop the lifecycle.
|
|
428
|
-
*/
|
|
429
|
-
const logEvents = (logger, description) => {
|
|
430
|
-
const caps = new Map((description?.shutdown.phases ?? []).map((phase) => [phase.name, phase.cap]));
|
|
431
|
-
const startedAt = /* @__PURE__ */ new Map();
|
|
432
|
-
const write = (level, message, fields) => {
|
|
433
|
-
try {
|
|
434
|
-
logger.log(level, message, fields);
|
|
435
|
-
} catch {}
|
|
436
|
-
};
|
|
437
|
-
const phaseEnd = (event, base) => {
|
|
438
|
-
const phase = event.from;
|
|
439
|
-
const cap = caps.get(phase);
|
|
440
|
-
const begun = startedAt.get(phase);
|
|
441
|
-
const reached = event.on === "phase.cap";
|
|
442
|
-
const fields = {
|
|
443
|
-
...base,
|
|
444
|
-
phase,
|
|
445
|
-
outcome: reached ? "cap" : "done",
|
|
446
|
-
...cap === void 0 ? {} : { cap },
|
|
447
|
-
...begun === void 0 ? {} : { took: {
|
|
448
|
-
value: event.at - begun,
|
|
449
|
-
unit: "ms"
|
|
450
|
-
} }
|
|
451
|
-
};
|
|
452
|
-
if (reached) write("warn", `the phase ${phase} reached its cap${cap === void 0 ? "" : ` of ${seconds(cap)}`}`, fields);
|
|
453
|
-
else write("info", `the phase ${phase} ended in its cap`, fields);
|
|
454
|
-
};
|
|
455
|
-
const phaseStart = (event, base) => {
|
|
456
|
-
const phase = event.to;
|
|
457
|
-
const cap = caps.get(phase);
|
|
458
|
-
startedAt.set(phase, event.at);
|
|
459
|
-
write("info", `the phase ${phase} starts${cap === void 0 ? "" : `, with a cap of ${seconds(cap)}`}`, {
|
|
460
|
-
...base,
|
|
461
|
-
phase,
|
|
462
|
-
...cap === void 0 ? {} : { cap }
|
|
463
|
-
});
|
|
464
|
-
};
|
|
465
|
-
return (event) => {
|
|
466
|
-
const { machine, instance, from, to, on, at, detail } = event;
|
|
467
|
-
const base = {
|
|
468
|
-
machine,
|
|
469
|
-
instance,
|
|
470
|
-
from,
|
|
471
|
-
to,
|
|
472
|
-
on,
|
|
473
|
-
at,
|
|
474
|
-
...detail === void 0 ? {} : { detail }
|
|
475
|
-
};
|
|
476
|
-
switch (machine) {
|
|
477
|
-
case "gate":
|
|
478
|
-
if (on === "component.started") return write("info", `${detail ?? "a component"} started`, {
|
|
479
|
-
...base,
|
|
480
|
-
component: detail
|
|
481
|
-
});
|
|
482
|
-
if (on === "startup.begin") return write("info", `the startup of ${instance} begins`, base);
|
|
483
|
-
if (on === "mark-ready") return write("info", "the warm-up sweep begins", base);
|
|
484
|
-
if (on === "sweep.done") return write("info", `${instance} is ready: the startup gate is open`, base);
|
|
485
|
-
if (on === "startup.fail") return write("warn", `the startup of ${instance} failed`, base);
|
|
486
|
-
if (on === "startup.over-budget") return write("warn", `the startup of ${instance} is over its budget`, base);
|
|
487
|
-
break;
|
|
488
|
-
case "exit":
|
|
489
|
-
if (on === "error") return write("error", detail ?? "an error of the lifecycle", base);
|
|
490
|
-
if (to === "exited") {
|
|
491
|
-
const code = exitCodeOf(detail);
|
|
492
|
-
const fields = {
|
|
493
|
-
...base,
|
|
494
|
-
code
|
|
495
|
-
};
|
|
496
|
-
if (on === "exit.repeat") return write("error", `a signal came again, and ${instance} exits at once with the code ${code}`, fields);
|
|
497
|
-
return write("info", `${instance} exited with the code ${code}`, fields);
|
|
498
|
-
}
|
|
499
|
-
if (from === "running" && to === "stopping") return write("info", `the shutdown of ${instance} begins`, {
|
|
500
|
-
...base,
|
|
501
|
-
cause: detail
|
|
502
|
-
});
|
|
503
|
-
if (on === "exit.repeat") return write("info", "a trigger came again while the shutdown runs", base);
|
|
504
|
-
break;
|
|
505
|
-
case "drain":
|
|
506
|
-
if (on === "signal.abort") return write("info", `the signal of ${instance} aborts`, base);
|
|
507
|
-
if (on === "drain.cooperative-abort") return write("info", "the cooperative abort of the drain", base);
|
|
508
|
-
if (on === "phase.done" || on === "phase.cap") phaseEnd(event, base);
|
|
509
|
-
if (to !== "done" && from !== to) phaseStart(event, base);
|
|
510
|
-
else if (to === "done") write("info", `the shutdown of ${instance} ended its phases`, base);
|
|
511
|
-
return;
|
|
512
|
-
case "latch":
|
|
513
|
-
case "window":
|
|
514
|
-
if (to === "tripped" && from !== "tripped") return write("warn", `the check ${instance} tripped its latch`, base);
|
|
515
|
-
if (to === "released" && (from === "tripped" || from === "recovering")) return write("info", `the check ${instance} released its latch`, base);
|
|
516
|
-
break;
|
|
517
|
-
case "watchdog":
|
|
518
|
-
if (to === "blocked" && from !== "blocked") return write("warn", "the event loop is blocked", base);
|
|
519
|
-
if (to === "watching" && from === "blocked") return write("info", "the event loop is no longer blocked", base);
|
|
520
|
-
}
|
|
521
|
-
write("debug", `${machine} ${instance}: ${from} -> ${to} on ${on}`, base);
|
|
522
|
-
};
|
|
523
|
-
};
|
|
524
|
-
// src/lifecycle.ts
|
|
525
|
-
/**
|
|
526
|
-
* These are the stages of the shutdown, in their sequence. A declared phase with one of these
|
|
527
|
-
* names gets the work of the runtime for its stage. A phase with another name gets only the work of
|
|
528
|
-
* its owners.
|
|
529
|
-
* - `fail readiness`: readiness fails at the start of the shutdown. This phase then holds for its
|
|
530
|
-
* cap, while the endpoints of the platform remove the service.
|
|
531
|
-
* - `stop intake`: each transport stops its intake, and new requests get a retryable 503.
|
|
532
|
-
* - `drain`: the transports and the requests in flight end, until the cap. The cooperative abort
|
|
533
|
-
* comes before the end of the cap, by the cooperative window.
|
|
534
|
-
* - `close transports`: each transport closes.
|
|
535
|
-
* - `stop the poller`: each check and watchdog stops. Then the signal of the service aborts.
|
|
536
|
-
* - `tear down dependencies`: each dependency closes, in the reverse sequence of its connection.
|
|
537
|
-
* - `close the health server`: the probes stop.
|
|
538
|
-
* - `flush the output`: the work of the owners of this phase ends, and then the process port
|
|
539
|
-
* writes the buffered output. Without this phase, the process exits without a wait.
|
|
540
|
-
*/
|
|
541
|
-
const STAGES = [
|
|
542
|
-
"fail readiness",
|
|
543
|
-
"stop intake",
|
|
544
|
-
"drain",
|
|
545
|
-
"close transports",
|
|
546
|
-
"stop the poller",
|
|
547
|
-
"tear down dependencies",
|
|
548
|
-
"close the health server",
|
|
549
|
-
"flush the output"
|
|
550
|
-
];
|
|
551
|
-
/** This function gives the place of a stage in the sequence, or -1 for a phase of another name. */
|
|
552
|
-
const stageRank = (name) => STAGES.indexOf(name);
|
|
553
|
-
/** The signal of the service aborts after the poller stops, and before the dependencies close. */
|
|
554
|
-
const ABORT_AFTER = stageRank("stop the poller");
|
|
555
|
-
/**
|
|
556
|
-
* This function gives the time of the cooperative abort, in milliseconds after the drain starts. It
|
|
557
|
-
* is the cap less the cooperative window. With the floor `half`, it is not less than half the cap,
|
|
558
|
-
* and a window without a bound gives half the cap. With the floor `none`, it is not less than 0.
|
|
559
|
-
*/
|
|
560
|
-
const cooperativeAbortAt = (cap, window, floor = "half") => {
|
|
561
|
-
const before = window.value === "infinite" ? Number.POSITIVE_INFINITY : valueIn(window, "ms");
|
|
562
|
-
return Math.max(cap - before, floor === "none" ? 0 : cap / 2);
|
|
563
|
-
};
|
|
564
|
-
/** This function gives the problems of the option `uncaught`: an exit code that is not from 0 to 255. */
|
|
565
|
-
const uncaughtProblems = (uncaught) => {
|
|
566
|
-
const code = uncaught?.exitCode;
|
|
567
|
-
return code === void 0 || isExitCode(code) ? [] : [`the exit code ${code} of an uncaught error is not a whole number from 0 to 255`];
|
|
568
|
-
};
|
|
569
|
-
const message = (error) => error instanceof Error ? error.message : String(error);
|
|
570
|
-
/** This function gives the text of an error with its stack, when it has one. */
|
|
571
|
-
const withStack = (error) => error instanceof Error && typeof error.stack === "string" && error.stack.length > 0 ? error.stack : message(error);
|
|
572
|
-
const UNCAUGHT_NAMES = {
|
|
573
|
-
uncaughtException: "an uncaught exception",
|
|
574
|
-
unhandledRejection: "an unhandled rejection"
|
|
575
|
-
};
|
|
576
|
-
/**
|
|
577
|
-
* This function gives the problems of a declaration that the lifecycle cannot start: a stage with
|
|
578
|
-
* work and no declared phase. The runtime has no defaults, thus a transport without a drain phase
|
|
579
|
-
* has no time to drain.
|
|
580
|
-
*/
|
|
581
|
-
const lifecycleProblems = (description, components) => {
|
|
582
|
-
const declared = new Set(description.shutdown.phases.map((phase) => phase.name));
|
|
583
|
-
const needs = (stage, why) => declared.has(stage) ? [] : [`${description.name} has ${why}, thus it declares the phase "${stage}"`];
|
|
584
|
-
const problems = [];
|
|
585
|
-
for (const transport of description.transports) if (!components.transports.has(transport.name)) problems.push(`transport ${transport.name} has no implementation: the runtime has no defaults`);
|
|
586
|
-
if (components.transports.size > 0 || description.controllers.length > 0) {
|
|
587
|
-
problems.push(...needs("stop intake", "transports or controllers"));
|
|
588
|
-
problems.push(...needs("drain", "transports or controllers"));
|
|
589
|
-
}
|
|
590
|
-
if (components.transports.size > 0) problems.push(...needs("close transports", "transports"));
|
|
591
|
-
if (description.checks.length > 0) problems.push(...needs("stop the poller", "checks"));
|
|
592
|
-
if (components.dependencies.size > 0) problems.push(...needs("tear down dependencies", "dependency handles"));
|
|
593
|
-
if (components.healthServer !== null) problems.push(...needs("close the health server", "a health server"));
|
|
594
|
-
return problems;
|
|
595
|
-
};
|
|
596
|
-
/**
|
|
597
|
-
* This function starts the lifecycle of a service. The startup has a sequence: the health server
|
|
598
|
-
* first, then each dependency, then each transport, and the startup gate last. A trigger during the
|
|
599
|
-
* startup gets its shutdown after the startup ends. The shutdown does the declared phases in their
|
|
600
|
-
* sequence, each one capped. Only the lifecycle ends the process. Its exit code is the code of the
|
|
601
|
-
* exit request, else 1 after an error, else 0.
|
|
602
|
-
*/
|
|
603
|
-
const lifecycle = (description, runtime, components, options = {}, bind = () => {}) => {
|
|
604
|
-
const clock = options.clock ?? realClock;
|
|
605
|
-
const port = options.process ?? nodeProcess;
|
|
606
|
-
const record = (machine, from, to, on, detail) => options.record?.({
|
|
607
|
-
at: clock.now(),
|
|
608
|
-
machine,
|
|
609
|
-
instance: description.name,
|
|
610
|
-
from,
|
|
611
|
-
to,
|
|
612
|
-
on,
|
|
613
|
-
...detail === void 0 ? {} : { detail }
|
|
614
|
-
});
|
|
615
|
-
const service = new AbortController();
|
|
616
|
-
const errors = [];
|
|
617
|
-
const exitCodes = description.shutdown.exitCodes ?? {};
|
|
618
|
-
const repeatPolicy = description.shutdown.onRepeatSignal;
|
|
619
|
-
let state = "starting";
|
|
620
|
-
let exitCode = null;
|
|
621
|
-
let requested;
|
|
622
|
-
let intakeStopped = false;
|
|
623
|
-
let shutdownStarted = false;
|
|
624
|
-
let failedStartup = false;
|
|
625
|
-
let exitState = "running";
|
|
626
|
-
const moveExit = (to, on, detail) => {
|
|
627
|
-
record("exit", exitState, to, on, detail);
|
|
628
|
-
exitState = to;
|
|
629
|
-
};
|
|
630
|
-
const fail = (cause) => {
|
|
631
|
-
errors.push(cause);
|
|
632
|
-
if (exitState !== "exited") record("exit", exitState, exitState, "error", cause);
|
|
633
|
-
};
|
|
634
|
-
const requests = tracker(description.controllers, clock, (controller) => {
|
|
635
|
-
if (state === "starting") return "the service has not started";
|
|
636
|
-
if (state === "exited") return "the service has exited";
|
|
637
|
-
if (state === "draining") return null;
|
|
638
|
-
return runtime.verdict().controllers[controller] === true ? null : `controller ${controller} is not ready`;
|
|
639
|
-
}, components.choices);
|
|
640
|
-
let finish = () => {};
|
|
641
|
-
const done = new Promise((resolve) => {
|
|
642
|
-
finish = resolve;
|
|
643
|
-
});
|
|
644
|
-
let healthListening = false;
|
|
645
|
-
const connected = [];
|
|
646
|
-
const startedTransports = [];
|
|
647
|
-
let startupEnded = () => {};
|
|
648
|
-
const startup = new Promise((resolve) => {
|
|
649
|
-
startupEnded = resolve;
|
|
650
|
-
});
|
|
651
|
-
let offs = [];
|
|
652
|
-
let release = () => {};
|
|
653
|
-
/** This function ends the process with a code, one time, and removes each handler of a trigger and the hold. */
|
|
654
|
-
const end = (code, on) => {
|
|
655
|
-
state = "exited";
|
|
656
|
-
exitCode = code;
|
|
657
|
-
for (const off of offs) off();
|
|
658
|
-
offs = [];
|
|
659
|
-
moveExit("exited", on, `exit ${code}`);
|
|
660
|
-
finish(code);
|
|
661
|
-
release();
|
|
662
|
-
port.exit(code);
|
|
663
|
-
};
|
|
664
|
-
/**
|
|
665
|
-
* This function receives a trigger of the process: the first one starts the shutdown. The cause
|
|
666
|
-
* names the trigger, and a message that stands for a signal tells it.
|
|
667
|
-
*/
|
|
668
|
-
const triggered = (trigger, cause = trigger) => {
|
|
669
|
-
if (exitState === "exited") return;
|
|
670
|
-
if (shutdownStarted || exitState === "stopping") {
|
|
671
|
-
if (isExitSignal(trigger) && repeatPolicy === "exit") {
|
|
672
|
-
fail(`${cause} came again while the shutdown ran, and the declaration exits at once`);
|
|
673
|
-
end(1, "exit.repeat");
|
|
674
|
-
return;
|
|
675
|
-
}
|
|
676
|
-
record("exit", exitState, exitState, "exit.repeat", cause);
|
|
677
|
-
return;
|
|
678
|
-
}
|
|
679
|
-
const code = exitCodes[trigger];
|
|
680
|
-
shutdown(cause, code === void 0 ? {} : { code });
|
|
681
|
-
};
|
|
682
|
-
const messages = description.shutdown.messages ?? [];
|
|
683
|
-
/** This function receives a message of the IPC channel. The first declared shape that it matches gives its trigger. */
|
|
684
|
-
const received = (message) => {
|
|
685
|
-
const found = messages.find((each) => matchesMessage(each.match, message));
|
|
686
|
-
if (found === void 0) return;
|
|
687
|
-
if (found.as === void 0) triggered("ipc-message");
|
|
688
|
-
else triggered(found.as, `${found.as} (a message of the IPC channel)`);
|
|
689
|
-
};
|
|
690
|
-
const uncaught = options.uncaught;
|
|
691
|
-
const uncaughtCode = uncaught?.exitCode !== void 0 && isExitCode(uncaught.exitCode) ? uncaught.exitCode : 1;
|
|
692
|
-
/**
|
|
693
|
-
* This function receives an error that no code caught. A skipped error goes to the logger at
|
|
694
|
-
* `warn`. Each other error is an error of the lifecycle, and it starts the shutdown with its code.
|
|
695
|
-
*/
|
|
696
|
-
const caught = (error, origin) => {
|
|
697
|
-
if (exitState === "exited") return;
|
|
698
|
-
const name = UNCAUGHT_NAMES[origin];
|
|
699
|
-
let skip = false;
|
|
700
|
-
try {
|
|
701
|
-
skip = uncaught?.ignore?.(error, origin) === true;
|
|
702
|
-
} catch {}
|
|
703
|
-
if (skip) {
|
|
704
|
-
try {
|
|
705
|
-
options.logger?.log("warn", `${name} was skipped: ${message(error)}`, {
|
|
706
|
-
origin,
|
|
707
|
-
error: withStack(error)
|
|
708
|
-
});
|
|
709
|
-
} catch {}
|
|
710
|
-
return;
|
|
711
|
-
}
|
|
712
|
-
fail(`${name}: ${withStack(error)}`);
|
|
713
|
-
shutdown(`${name}: ${message(error)}`, { code: uncaughtCode });
|
|
714
|
-
};
|
|
715
|
-
/**
|
|
716
|
-
* This function does the work of one phase, until its cap. It tells if the work ended in time. When
|
|
717
|
-
* work is left at the cap, `excuse` gives the cause of a cap that is not a failure, or null.
|
|
718
|
-
*/
|
|
719
|
-
const runPhase = async (phase, hold, work, excuse = () => null) => {
|
|
720
|
-
const cap = valueIn(phase.cap, "ms");
|
|
721
|
-
const deadline = clock.now() + cap;
|
|
722
|
-
const capped = new AbortController();
|
|
723
|
-
const context = {
|
|
724
|
-
deadline,
|
|
725
|
-
signal: capped.signal,
|
|
726
|
-
clock
|
|
727
|
-
};
|
|
728
|
-
let pending = work.length;
|
|
729
|
-
const all = Promise.all(work.map(async (each) => {
|
|
730
|
-
try {
|
|
731
|
-
await each(context);
|
|
732
|
-
} catch (error) {
|
|
733
|
-
fail(`the phase ${phase.name} failed: ${message(error)}`);
|
|
734
|
-
} finally {
|
|
735
|
-
pending -= 1;
|
|
736
|
-
}
|
|
737
|
-
}));
|
|
738
|
-
let cancelCap = () => {};
|
|
739
|
-
const end = new Promise((resolve) => {
|
|
740
|
-
cancelCap = clock.after(cap, () => resolve("cap"));
|
|
741
|
-
});
|
|
742
|
-
if (await Promise.race([all.then(() => "work"), end]) === "work" && hold) await end;
|
|
743
|
-
const excused = pending > 0 ? excuse(pending) : null;
|
|
744
|
-
cancelCap();
|
|
745
|
-
capped.abort(/* @__PURE__ */ new Error(`the phase ${phase.name} reached its cap`));
|
|
746
|
-
return {
|
|
747
|
-
inTime: pending === 0,
|
|
748
|
-
excused
|
|
749
|
-
};
|
|
750
|
-
};
|
|
751
|
-
async function shutdown(cause = "the service asked for it", request = {}) {
|
|
752
|
-
if (request.code !== void 0 && !isExitCode(request.code)) throw new Error(`${description.name}: the exit code ${request.code} is not a whole number from 0 to 255`);
|
|
753
|
-
if (exitState === "exited") return done;
|
|
754
|
-
if (request.code !== void 0 && requested === void 0) requested = request.code;
|
|
755
|
-
if (shutdownStarted) {
|
|
756
|
-
if (exitState === "stopping") record("exit", exitState, exitState, "exit.repeat", cause);
|
|
757
|
-
return done;
|
|
758
|
-
}
|
|
759
|
-
shutdownStarted = true;
|
|
760
|
-
if (exitState === "running") moveExit("stopping", "exit.request", cause);
|
|
761
|
-
await startup;
|
|
762
|
-
if (exitState === "exited") {
|
|
763
|
-
runtime.stop();
|
|
764
|
-
components.stopPollers();
|
|
765
|
-
return exitCode;
|
|
766
|
-
}
|
|
767
|
-
state = "draining";
|
|
768
|
-
runtime.drain();
|
|
769
|
-
let drainState = "running";
|
|
770
|
-
const moveDrain = (to, on, detail) => {
|
|
771
|
-
record("drain", drainState, to, on, detail);
|
|
772
|
-
drainState = to;
|
|
773
|
-
};
|
|
774
|
-
const abortService = () => {
|
|
775
|
-
if (service.signal.aborted) return;
|
|
776
|
-
service.abort(/* @__PURE__ */ new Error("the service stopped"));
|
|
777
|
-
record("drain", drainState, drainState, "signal.abort");
|
|
778
|
-
};
|
|
779
|
-
const phases = description.shutdown.phases;
|
|
780
|
-
let on = "exit.request";
|
|
781
|
-
let detail;
|
|
782
|
-
for (const phase of phases) {
|
|
783
|
-
if (exitState === "exited") break;
|
|
784
|
-
const rank = stageRank(phase.name);
|
|
785
|
-
moveDrain(phase.name, on, detail);
|
|
786
|
-
if (rank > ABORT_AFTER) abortService();
|
|
787
|
-
const work = [];
|
|
788
|
-
const owned = components.phaseWork.get(phase.name) ?? [];
|
|
789
|
-
if (phase.name === "stop intake") {
|
|
790
|
-
work.push(() => {
|
|
791
|
-
intakeStopped = true;
|
|
792
|
-
requests.stopIntake();
|
|
793
|
-
});
|
|
794
|
-
for (const [, impl] of startedTransports) work.push(() => impl.stopIntake());
|
|
795
|
-
}
|
|
796
|
-
let cancelAbort = () => {};
|
|
797
|
-
let excuse;
|
|
798
|
-
if (phase.name === "drain") {
|
|
799
|
-
const cap = valueIn(phase.cap, "ms");
|
|
800
|
-
const cooperative = new AbortController();
|
|
801
|
-
const fireAt = cooperativeAbortAt(cap, description.shutdown.cooperativeWindow, description.shutdown.cooperativeFloor);
|
|
802
|
-
cancelAbort = clock.after(fireAt, () => {
|
|
803
|
-
cooperative.abort(/* @__PURE__ */ new Error("the cooperative abort of the drain"));
|
|
804
|
-
requests.cooperativeAbort();
|
|
805
|
-
record("drain", drainState, drainState, "drain.cooperative-abort");
|
|
806
|
-
});
|
|
807
|
-
work.push(({ deadline }) => requests.drain(deadline));
|
|
808
|
-
let transportsLeft = 0;
|
|
809
|
-
for (const [, impl] of startedTransports) work.push(async ({ deadline }) => {
|
|
810
|
-
transportsLeft += 1;
|
|
811
|
-
try {
|
|
812
|
-
await impl.drain(deadline, cooperative.signal);
|
|
813
|
-
} finally {
|
|
814
|
-
transportsLeft -= 1;
|
|
815
|
-
}
|
|
816
|
-
});
|
|
817
|
-
excuse = (left) => {
|
|
818
|
-
const holds = lateHolds(life);
|
|
819
|
-
if (left > transportsLeft || transportsLeft > holds.length || holds.some((each) => each.running > 0)) return null;
|
|
820
|
-
return `answers not flushed: ${holds.reduce((sum, each) => sum + each.unflushed, 0)}`;
|
|
821
|
-
};
|
|
822
|
-
}
|
|
823
|
-
if (phase.name === "close transports") for (const [, impl] of startedTransports) work.push(() => impl.close());
|
|
824
|
-
if (phase.name === "stop the poller") work.push(() => {
|
|
825
|
-
runtime.stop();
|
|
826
|
-
components.stopPollers();
|
|
827
|
-
});
|
|
828
|
-
if (phase.name === "tear down dependencies") work.push(async () => {
|
|
829
|
-
for (const [, handle] of [...connected].reverse()) await handle.close();
|
|
830
|
-
});
|
|
831
|
-
if (phase.name === "close the health server" && healthListening && components.healthServer !== null) {
|
|
832
|
-
const server = components.healthServer;
|
|
833
|
-
work.push(() => server.close());
|
|
834
|
-
}
|
|
835
|
-
if (phase.name === "flush the output") work.push(async (context) => {
|
|
836
|
-
await Promise.all(owned.map(async (each) => {
|
|
837
|
-
try {
|
|
838
|
-
await each(context);
|
|
839
|
-
} catch (error) {
|
|
840
|
-
fail(`the phase ${phase.name} failed: ${message(error)}`);
|
|
841
|
-
}
|
|
842
|
-
}));
|
|
843
|
-
await port.flush?.();
|
|
844
|
-
});
|
|
845
|
-
else for (const each of owned) work.push(each);
|
|
846
|
-
const { inTime, excused } = await runPhase(phase, phase.name === "fail readiness", work, excuse);
|
|
847
|
-
cancelAbort();
|
|
848
|
-
if (!inTime && excused === null) fail(`the phase ${phase.name} did not end in its cap of ${valueIn(phase.cap, "s")} s`);
|
|
849
|
-
on = inTime ? "phase.done" : "phase.cap";
|
|
850
|
-
const outsideCount = phase.name === "drain" ? takeOutside(life) : 0;
|
|
851
|
-
const parts = [inTime ? null : excused, outsideCount > 0 ? `streams outside the lifecycle: ${outsideCount}` : null].filter((part) => part !== null);
|
|
852
|
-
detail = parts.length > 0 ? parts.join(", ") : void 0;
|
|
853
|
-
}
|
|
854
|
-
if (exitState !== "exited") moveDrain("done", on, detail);
|
|
855
|
-
abortService();
|
|
856
|
-
runtime.stop();
|
|
857
|
-
components.stopPollers();
|
|
858
|
-
if (!intakeStopped) requests.stopIntake();
|
|
859
|
-
if (exitState === "exited") return exitCode;
|
|
860
|
-
const code = requested !== void 0 && requested !== 0 ? requested : errors.length > 0 ? 1 : 0;
|
|
861
|
-
end(code, "shutdown.done");
|
|
862
|
-
return code;
|
|
863
|
-
}
|
|
864
|
-
const life = {
|
|
865
|
-
health: runtime,
|
|
866
|
-
get state() {
|
|
867
|
-
return state;
|
|
868
|
-
},
|
|
869
|
-
get exitCode() {
|
|
870
|
-
return exitCode;
|
|
871
|
-
},
|
|
872
|
-
get errors() {
|
|
873
|
-
return [...errors];
|
|
874
|
-
},
|
|
875
|
-
started: startup,
|
|
876
|
-
done,
|
|
877
|
-
signal: service.signal,
|
|
878
|
-
shutdown,
|
|
879
|
-
request: (controller, route, work, request) => ownStream(requests.request(controller, route, work, request), life),
|
|
880
|
-
inFlight: (controller) => requests.inFlight(controller),
|
|
881
|
-
keepAlive: () => state === "running" && runtime.verdict().ready
|
|
882
|
-
};
|
|
883
|
-
bind(life);
|
|
884
|
-
release = port.hold?.() ?? (() => {});
|
|
885
|
-
offs = [...EXIT_SIGNALS, ...description.shutdown.triggers ?? []].map((trigger) => port.on(trigger, trigger === "ipc-message" ? received : () => triggered(trigger)));
|
|
886
|
-
if (uncaught !== void 0 && port.onUncaught !== void 0) offs.push(port.onUncaught(caught));
|
|
887
|
-
(async () => {
|
|
888
|
-
const started = (what) => record("gate", "starting", "starting", "component.started", what);
|
|
889
|
-
try {
|
|
890
|
-
runtime.beginStartup();
|
|
891
|
-
if (components.healthServer !== null) {
|
|
892
|
-
await components.healthServer.listen(runtime);
|
|
893
|
-
healthListening = true;
|
|
894
|
-
started("the health server");
|
|
895
|
-
}
|
|
896
|
-
for (const dependency of description.dependencies) {
|
|
897
|
-
const handle = components.dependencies.get(dependency.name);
|
|
898
|
-
if (handle === void 0) continue;
|
|
899
|
-
await handle.connect(service.signal);
|
|
900
|
-
connected.push([dependency.name, handle]);
|
|
901
|
-
started(`dependency ${dependency.name}`);
|
|
902
|
-
}
|
|
903
|
-
for (const transport of description.transports) {
|
|
904
|
-
const impl = components.transports.get(transport.name);
|
|
905
|
-
await impl.start(service.signal, life);
|
|
906
|
-
startedTransports.push([transport.name, impl]);
|
|
907
|
-
started(`${transport.kind} ${transport.name}`);
|
|
908
|
-
}
|
|
909
|
-
await runtime.warmUp();
|
|
910
|
-
if (state === "starting") state = "running";
|
|
911
|
-
} catch (error) {
|
|
912
|
-
const cause = `the startup failed: ${message(error)}`;
|
|
913
|
-
failedStartup = true;
|
|
914
|
-
runtime.failStartup(cause);
|
|
915
|
-
fail(cause);
|
|
916
|
-
if (exitState === "running") moveExit("stopping", "startup.fail", cause);
|
|
917
|
-
else if (exitState === "stopping") record("exit", exitState, exitState, "startup.fail", cause);
|
|
918
|
-
} finally {
|
|
919
|
-
startupEnded();
|
|
920
|
-
}
|
|
921
|
-
})();
|
|
922
|
-
startup.then(() => {
|
|
923
|
-
if (failedStartup && !shutdownStarted && exitState !== "exited") shutdown("the startup failed");
|
|
924
|
-
});
|
|
925
|
-
return life;
|
|
926
|
-
};
|
|
927
|
-
// src/watchdog.ts
|
|
928
|
-
/** This function starts a watchdog on a clock. */
|
|
929
|
-
const watchdog = (clock, spec, trace) => {
|
|
930
|
-
const tick = valueIn(spec.tick, "ms");
|
|
931
|
-
const threshold = valueIn(spec.threshold, "ms");
|
|
932
|
-
if (!(tick > 0)) throw new RangeError(`watchdog ${spec.name} ticks every ${tick} ms, not above zero`);
|
|
933
|
-
let blocked = false;
|
|
934
|
-
let stopped = false;
|
|
935
|
-
let worst = 0;
|
|
936
|
-
let sincePoll = 0;
|
|
937
|
-
let next = null;
|
|
938
|
-
const named = () => stopped ? "stopped" : blocked ? "blocked" : "watching";
|
|
939
|
-
const arm = () => {
|
|
940
|
-
const due = clock.now() + tick;
|
|
941
|
-
next = clock.after(tick, () => {
|
|
942
|
-
next = null;
|
|
943
|
-
if (stopped) return;
|
|
944
|
-
const late = Math.max(0, clock.now() - due);
|
|
945
|
-
worst = Math.max(worst, late);
|
|
946
|
-
sincePoll = Math.max(sincePoll, late);
|
|
947
|
-
const from = named();
|
|
948
|
-
blocked = late > threshold;
|
|
949
|
-
trace?.(from, named(), "tick");
|
|
950
|
-
arm();
|
|
951
|
-
});
|
|
952
|
-
};
|
|
953
|
-
arm();
|
|
954
|
-
return {
|
|
955
|
-
check: () => {
|
|
956
|
-
const failed = blocked || sincePoll > threshold;
|
|
957
|
-
sincePoll = 0;
|
|
958
|
-
return failed ? "error" : "ok";
|
|
959
|
-
},
|
|
960
|
-
get worst() {
|
|
961
|
-
return worst;
|
|
962
|
-
},
|
|
963
|
-
get blocked() {
|
|
964
|
-
return blocked;
|
|
965
|
-
},
|
|
966
|
-
stop() {
|
|
967
|
-
if (stopped) return;
|
|
968
|
-
const from = named();
|
|
969
|
-
stopped = true;
|
|
970
|
-
next?.();
|
|
971
|
-
next = null;
|
|
972
|
-
trace?.(from, "stopped", "stop");
|
|
973
|
-
}
|
|
974
|
-
};
|
|
975
|
-
};
|
|
976
|
-
// src/builder.ts
|
|
977
|
-
const isTime$1 = (amount) => convertible(amount.unit, "s");
|
|
978
|
-
const above = (amount) => amount.value === "infinite" || isTime$1(amount) && valueIn(amount, "s") > 0;
|
|
979
|
-
const whole = (count) => Number.isInteger(count) && count >= 1;
|
|
980
|
-
/** This function copies a check, with its poll and the loops that mark it. */
|
|
981
|
-
const copyCheck = (check) => ({
|
|
982
|
-
...check,
|
|
983
|
-
poll: { ...check.poll },
|
|
984
|
-
...check.marks === void 0 ? {} : { marks: {
|
|
985
|
-
loops: [...check.marks.loops],
|
|
986
|
-
stall: { ...check.marks.stall }
|
|
987
|
-
} }
|
|
988
|
-
});
|
|
989
|
-
/** This function gives the problems of the loops that mark a check. `lint()` compares the stall time with the loops. */
|
|
990
|
-
const marksProblems = (check) => {
|
|
991
|
-
const marks = check.marks;
|
|
992
|
-
if (marks === void 0) return [];
|
|
993
|
-
const problems = [];
|
|
994
|
-
if (check.step !== "progress") problems.push(`check ${check.name} names the loops that mark it, but its step is ${check.step}, not progress`);
|
|
995
|
-
if (marks.loops.length === 0) problems.push(`check ${check.name} names no loop that marks it`);
|
|
996
|
-
if (typeof marks.stall.value !== "number" || !above(marks.stall)) problems.push(`check ${check.name} has a stall time that is not a time above zero`);
|
|
997
|
-
return problems;
|
|
998
|
-
};
|
|
999
|
-
/** This function gives the problems of a timing: a zero interval or count fails decoding. */
|
|
1000
|
-
const timingProblems = (policy, timing) => {
|
|
1001
|
-
if (timing.type === "debounce") {
|
|
1002
|
-
const problems = [];
|
|
1003
|
-
if (!above(timing.interval)) problems.push(`the ${policy} debounce polls at an interval that is not above zero`);
|
|
1004
|
-
if (!whole(timing.failAfter)) problems.push(`the ${policy} debounce fails after ${timing.failAfter} polls, not a whole number above zero`);
|
|
1005
|
-
if (!whole(timing.recoverAfter)) problems.push(`the ${policy} debounce recovers after ${timing.recoverAfter} polls, not a whole number above zero`);
|
|
1006
|
-
return problems;
|
|
1007
|
-
}
|
|
1008
|
-
const negative = (amount) => amount !== void 0 && (!isTime$1(amount) || valueIn(amount, "s") < 0);
|
|
1009
|
-
return negative(timing.failAfter) || negative(timing.recoverAfter) ? [`the ${policy} window is not a time of zero or more`] : [];
|
|
1010
|
-
};
|
|
1011
|
-
/** This function gives the problems of the rules of the shutdown: an unknown name, or an exit code that a process cannot give. */
|
|
1012
|
-
const shutdownProblems = (floor, repeat, triggers, exitCodes, messages) => {
|
|
1013
|
-
const problems = [];
|
|
1014
|
-
if (floor !== void 0 && floor !== "half" && floor !== "none") problems.push(`the cooperative floor ${String(floor)} is not "half" or "none"`);
|
|
1015
|
-
if (repeat !== void 0 && repeat !== "ignore" && repeat !== "exit") problems.push(`the policy of a repeat signal ${String(repeat)} is not "ignore" or "exit"`);
|
|
1016
|
-
for (const trigger of triggers) if (!PROCESS_TRIGGERS.includes(trigger)) problems.push(`the trigger ${String(trigger)} is not a process trigger (${PROCESS_TRIGGERS.join(", ")})`);
|
|
1017
|
-
if (triggers.includes("ipc-message") && messages.length === 0) problems.push(`the trigger ipc-message has no message to match: declare it with svc.trigger("ipc-message", { match })`);
|
|
1018
|
-
for (const message of messages) {
|
|
1019
|
-
if (!isMessageShape(message.match)) problems.push(`the match of a message trigger is not a value of JSON: ${String(message.match)}`);
|
|
1020
|
-
if (message.as !== void 0 && !isExitSignal(message.as)) problems.push(`a message trigger stands for ${String(message.as)}, not an exit signal (${EXIT_SIGNALS.join(", ")})`);
|
|
1021
|
-
}
|
|
1022
|
-
for (const [trigger, code] of Object.entries(exitCodes)) {
|
|
1023
|
-
if (!(EXIT_SIGNALS.includes(trigger) || triggers.includes(trigger))) problems.push(`the exit code of ${trigger} names no exit signal and no declared trigger`);
|
|
1024
|
-
if (code === void 0 || !isExitCode(code)) problems.push(`the exit code of ${trigger} is ${String(code)}, not a whole number from 0 to 255`);
|
|
1025
|
-
}
|
|
1026
|
-
return problems;
|
|
1027
|
-
};
|
|
1028
|
-
/**
|
|
1029
|
-
* This function gives the options with one recorder for the recorder and the logger of the options.
|
|
1030
|
-
* Without a logger, it gives the same options.
|
|
1031
|
-
*/
|
|
1032
|
-
const withLogger = (options, description) => {
|
|
1033
|
-
if (options.logger === void 0) return options;
|
|
1034
|
-
const given = options.record;
|
|
1035
|
-
const log = logEvents(options.logger, description);
|
|
1036
|
-
return {
|
|
1037
|
-
...options,
|
|
1038
|
-
record: (event) => {
|
|
1039
|
-
given?.(event);
|
|
1040
|
-
log(event);
|
|
1041
|
-
}
|
|
1042
|
-
};
|
|
1043
|
-
};
|
|
1044
|
-
/** This function starts the composition of a service. */
|
|
1045
|
-
const service = (name) => {
|
|
1046
|
-
if (name.trim().length === 0) throw new Error("a service needs a name");
|
|
1047
|
-
const transports = [];
|
|
1048
|
-
const dependencies = [];
|
|
1049
|
-
const controllers = [];
|
|
1050
|
-
const choices = /* @__PURE__ */ new Map();
|
|
1051
|
-
const checks = [];
|
|
1052
|
-
const runs = /* @__PURE__ */ new Map();
|
|
1053
|
-
const limiters = [];
|
|
1054
|
-
const couplings = [];
|
|
1055
|
-
const phases = [];
|
|
1056
|
-
let readiness;
|
|
1057
|
-
let liveness;
|
|
1058
|
-
let startup;
|
|
1059
|
-
let cooperative;
|
|
1060
|
-
let floor;
|
|
1061
|
-
let repeat;
|
|
1062
|
-
const triggers = [];
|
|
1063
|
-
const messages = [];
|
|
1064
|
-
const exitCodes = {};
|
|
1065
|
-
let started = false;
|
|
1066
|
-
let partial = false;
|
|
1067
|
-
let runtime;
|
|
1068
|
-
let live;
|
|
1069
|
-
let running;
|
|
1070
|
-
let life;
|
|
1071
|
-
const transportImpls = /* @__PURE__ */ new Map();
|
|
1072
|
-
const handles = /* @__PURE__ */ new Map();
|
|
1073
|
-
const phaseWork = /* @__PURE__ */ new Map();
|
|
1074
|
-
const watchdogs = [];
|
|
1075
|
-
let probes = null;
|
|
1076
|
-
/** This function stops a change to the shape of the service after the start. */
|
|
1077
|
-
const composing = (what) => {
|
|
1078
|
-
if (started) throw new Error(`${name} has started: ${what} is declared before svc.start()`);
|
|
1079
|
-
};
|
|
1080
|
-
const once = (list, entry, what) => {
|
|
1081
|
-
if (list.some((held) => held.name === entry)) throw new Error(`${name} already has a ${what} named ${entry}`);
|
|
1082
|
-
};
|
|
1083
|
-
const transport = (kind) => (entry, knobs, impl) => {
|
|
1084
|
-
composing(`a ${kind}`);
|
|
1085
|
-
once(transports, entry, "transport");
|
|
1086
|
-
transports.push({
|
|
1087
|
-
name: entry,
|
|
1088
|
-
kind,
|
|
1089
|
-
knobs: { ...knobs }
|
|
1090
|
-
});
|
|
1091
|
-
if (impl !== void 0) transportImpls.set(entry, impl);
|
|
1092
|
-
return builder;
|
|
1093
|
-
};
|
|
1094
|
-
/** This function makes the runtime of the declaration: its checks, and a watchdog for each spec. */
|
|
1095
|
-
const makeRuntime = (given) => {
|
|
1096
|
-
const description = builder.describe();
|
|
1097
|
-
const options = withLogger(given, description);
|
|
1098
|
-
const clock = options.clock ?? realClock;
|
|
1099
|
-
const dogs = [];
|
|
1100
|
-
const all = new Map(runs);
|
|
1101
|
-
for (const spec of watchdogs) {
|
|
1102
|
-
const dog = watchdog(clock, spec, (from, to, on) => options.record?.({
|
|
1103
|
-
at: clock.now(),
|
|
1104
|
-
machine: "watchdog",
|
|
1105
|
-
instance: spec.name,
|
|
1106
|
-
from,
|
|
1107
|
-
to,
|
|
1108
|
-
on
|
|
1109
|
-
}));
|
|
1110
|
-
dogs.push(dog);
|
|
1111
|
-
all.set(spec.name, () => dog.check());
|
|
1112
|
-
}
|
|
1113
|
-
const stopPollers = () => {
|
|
1114
|
-
for (const dog of dogs) dog.stop();
|
|
1115
|
-
};
|
|
1116
|
-
let made;
|
|
1117
|
-
try {
|
|
1118
|
-
made = healthRuntime(description, all, options);
|
|
1119
|
-
} catch (error) {
|
|
1120
|
-
stopPollers();
|
|
1121
|
-
throw error;
|
|
1122
|
-
}
|
|
1123
|
-
return {
|
|
1124
|
-
runtime: {
|
|
1125
|
-
...made,
|
|
1126
|
-
stop: () => {
|
|
1127
|
-
made.stop();
|
|
1128
|
-
stopPollers();
|
|
1129
|
-
}
|
|
1130
|
-
},
|
|
1131
|
-
description,
|
|
1132
|
-
stopPollers,
|
|
1133
|
-
options
|
|
1134
|
-
};
|
|
1135
|
-
};
|
|
1136
|
-
const builder = {
|
|
1137
|
-
name,
|
|
1138
|
-
get health() {
|
|
1139
|
-
return live ?? null;
|
|
1140
|
-
},
|
|
1141
|
-
get lifecycle() {
|
|
1142
|
-
return life ?? null;
|
|
1143
|
-
},
|
|
1144
|
-
dependency(entry, spec, handle) {
|
|
1145
|
-
once(dependencies, entry, "dependency");
|
|
1146
|
-
if (handle !== void 0 && !started) handles.set(entry, handle);
|
|
1147
|
-
dependencies.push({
|
|
1148
|
-
name: entry,
|
|
1149
|
-
...spec,
|
|
1150
|
-
knobs: { ...spec.knobs },
|
|
1151
|
-
operations: { ...spec.operations }
|
|
1152
|
-
});
|
|
1153
|
-
if (started) {
|
|
1154
|
-
partial = true;
|
|
1155
|
-
live?.addDependency(entry);
|
|
1156
|
-
}
|
|
1157
|
-
return builder;
|
|
1158
|
-
},
|
|
1159
|
-
server: transport("server"),
|
|
1160
|
-
consumer: transport("consumer"),
|
|
1161
|
-
worker: transport("worker"),
|
|
1162
|
-
controller(entry) {
|
|
1163
|
-
composing("a controller");
|
|
1164
|
-
once(controllers, entry, "controller");
|
|
1165
|
-
const held = {
|
|
1166
|
-
name: entry,
|
|
1167
|
-
requires: [],
|
|
1168
|
-
routes: {},
|
|
1169
|
-
patterns: []
|
|
1170
|
-
};
|
|
1171
|
-
controllers.push(held);
|
|
1172
|
-
const handle = {
|
|
1173
|
-
requires(...needed) {
|
|
1174
|
-
composing(`what ${entry} requires`);
|
|
1175
|
-
held.requires.push(...needed);
|
|
1176
|
-
return handle;
|
|
1177
|
-
},
|
|
1178
|
-
route(served, operation) {
|
|
1179
|
-
composing(`${entry}'s routes`);
|
|
1180
|
-
if (held.routes[served] !== void 0) throw new Error(`${entry} already serves ${served}`);
|
|
1181
|
-
held.routes[served] = { ...operation };
|
|
1182
|
-
return handle;
|
|
1183
|
-
},
|
|
1184
|
-
pattern(pattern, operations, operationOf) {
|
|
1185
|
-
composing(`${entry}'s route patterns`);
|
|
1186
|
-
if (held.patterns.some((each) => each.pattern === pattern)) throw new Error(`${entry} already has the route pattern ${pattern}`);
|
|
1187
|
-
if (operationOf === void 0 && Object.keys(operations).length > 1) throw new Error(`${entry}'s pattern ${pattern} has ${Object.keys(operations).length} operations, thus it needs the function that picks one`);
|
|
1188
|
-
held.patterns.push({
|
|
1189
|
-
pattern,
|
|
1190
|
-
operations: Object.fromEntries(Object.entries(operations).map(([name, operation]) => [name, { ...operation }]))
|
|
1191
|
-
});
|
|
1192
|
-
if (operationOf !== void 0) {
|
|
1193
|
-
const mine = choices.get(entry) ?? /* @__PURE__ */ new Map();
|
|
1194
|
-
mine.set(pattern, operationOf);
|
|
1195
|
-
choices.set(entry, mine);
|
|
1196
|
-
}
|
|
1197
|
-
return handle;
|
|
1198
|
-
}
|
|
1199
|
-
};
|
|
1200
|
-
return handle;
|
|
1201
|
-
},
|
|
1202
|
-
check(check, run) {
|
|
1203
|
-
once(checks, check.name, "check");
|
|
1204
|
-
if (started && run === void 0) throw new Error(`check ${check.name} joins after the start, thus it needs its function`);
|
|
1205
|
-
checks.push(copyCheck(check));
|
|
1206
|
-
if (run !== void 0) runs.set(check.name, run);
|
|
1207
|
-
if (started) {
|
|
1208
|
-
partial = true;
|
|
1209
|
-
live?.addCheck(copyCheck(check), run);
|
|
1210
|
-
}
|
|
1211
|
-
return builder;
|
|
1212
|
-
},
|
|
1213
|
-
limiter(entry, spec) {
|
|
1214
|
-
composing(`the limiter ${entry}`);
|
|
1215
|
-
once(limiters, entry, "limiter");
|
|
1216
|
-
const given = spec ?? {};
|
|
1217
|
-
limiters.push({
|
|
1218
|
-
name: entry,
|
|
1219
|
-
...given,
|
|
1220
|
-
...given.policy === void 0 ? {} : { policy: { ...given.policy } },
|
|
1221
|
-
...given.store === void 0 ? {} : { store: { ...given.store } },
|
|
1222
|
-
...given.evict === void 0 ? {} : { evict: { ...given.evict } }
|
|
1223
|
-
});
|
|
1224
|
-
return builder;
|
|
1225
|
-
},
|
|
1226
|
-
healthServer(server) {
|
|
1227
|
-
composing("the health server");
|
|
1228
|
-
if (probes !== null) throw new Error(`${name} already has a health server`);
|
|
1229
|
-
probes = server;
|
|
1230
|
-
return builder;
|
|
1231
|
-
},
|
|
1232
|
-
onPhase(phase, work) {
|
|
1233
|
-
composing(`work for the phase ${phase}`);
|
|
1234
|
-
phaseWork.set(phase, [...phaseWork.get(phase) ?? [], work]);
|
|
1235
|
-
return builder;
|
|
1236
|
-
},
|
|
1237
|
-
watchdog(spec) {
|
|
1238
|
-
composing(`the watchdog ${spec.name}`);
|
|
1239
|
-
once(checks, spec.name, "check");
|
|
1240
|
-
checks.push({
|
|
1241
|
-
name: spec.name,
|
|
1242
|
-
dependency: null,
|
|
1243
|
-
step: "local-state",
|
|
1244
|
-
ownConnection: true,
|
|
1245
|
-
gates: spec.gates,
|
|
1246
|
-
poll: { ...spec.poll }
|
|
1247
|
-
});
|
|
1248
|
-
watchdogs.push(spec);
|
|
1249
|
-
return builder;
|
|
1250
|
-
},
|
|
1251
|
-
readiness(timing) {
|
|
1252
|
-
composing("readiness");
|
|
1253
|
-
readiness = timing;
|
|
1254
|
-
return builder;
|
|
1255
|
-
},
|
|
1256
|
-
liveness(timing) {
|
|
1257
|
-
composing("liveness");
|
|
1258
|
-
liveness = timing;
|
|
1259
|
-
return builder;
|
|
1260
|
-
},
|
|
1261
|
-
startup(budget) {
|
|
1262
|
-
composing("the startup budget");
|
|
1263
|
-
startup = budget;
|
|
1264
|
-
return builder;
|
|
1265
|
-
},
|
|
1266
|
-
couple(coupling) {
|
|
1267
|
-
composing("a coupling");
|
|
1268
|
-
couplings.push({ ...coupling });
|
|
1269
|
-
return builder;
|
|
1270
|
-
},
|
|
1271
|
-
phase(entry, cap) {
|
|
1272
|
-
composing("a shutdown phase");
|
|
1273
|
-
once(phases, entry, "shutdown phase");
|
|
1274
|
-
phases.push({
|
|
1275
|
-
name: entry,
|
|
1276
|
-
cap
|
|
1277
|
-
});
|
|
1278
|
-
return builder;
|
|
1279
|
-
},
|
|
1280
|
-
cooperativeWindow(window) {
|
|
1281
|
-
composing("the cooperative window");
|
|
1282
|
-
cooperative = window;
|
|
1283
|
-
return builder;
|
|
1284
|
-
},
|
|
1285
|
-
cooperativeFloor(given) {
|
|
1286
|
-
composing("the cooperative floor");
|
|
1287
|
-
floor = given;
|
|
1288
|
-
return builder;
|
|
1289
|
-
},
|
|
1290
|
-
onRepeatSignal(policy) {
|
|
1291
|
-
composing("the policy of a repeat signal");
|
|
1292
|
-
repeat = policy;
|
|
1293
|
-
return builder;
|
|
1294
|
-
},
|
|
1295
|
-
trigger(...given) {
|
|
1296
|
-
composing("a trigger");
|
|
1297
|
-
const [first, message] = given;
|
|
1298
|
-
if (first === "ipc-message" && typeof message === "object" && message !== null) {
|
|
1299
|
-
if (!triggers.includes("ipc-message")) triggers.push("ipc-message");
|
|
1300
|
-
messages.push({ ...message });
|
|
1301
|
-
return builder;
|
|
1302
|
-
}
|
|
1303
|
-
for (const each of given) if (typeof each === "string" && !triggers.includes(each)) triggers.push(each);
|
|
1304
|
-
return builder;
|
|
1305
|
-
},
|
|
1306
|
-
exitCode(trigger, code) {
|
|
1307
|
-
composing(`the exit code of ${trigger}`);
|
|
1308
|
-
exitCodes[trigger] = code;
|
|
1309
|
-
return builder;
|
|
1310
|
-
},
|
|
1311
|
-
profile(given) {
|
|
1312
|
-
composing("a profile");
|
|
1313
|
-
if (given.readiness !== void 0) builder.readiness(given.readiness);
|
|
1314
|
-
if (given.liveness !== void 0) builder.liveness(given.liveness);
|
|
1315
|
-
if (given.startup !== void 0) builder.startup(given.startup);
|
|
1316
|
-
for (const coupling of given.couplings ?? []) builder.couple(coupling);
|
|
1317
|
-
for (const each of given.phases ?? []) builder.phase(each.name, each.cap);
|
|
1318
|
-
if (given.cooperativeWindow !== void 0) builder.cooperativeWindow(given.cooperativeWindow);
|
|
1319
|
-
if (given.cooperativeFloor !== void 0) builder.cooperativeFloor(given.cooperativeFloor);
|
|
1320
|
-
if (given.onRepeatSignal !== void 0) builder.onRepeatSignal(given.onRepeatSignal);
|
|
1321
|
-
if (given.triggers !== void 0) builder.trigger(...given.triggers);
|
|
1322
|
-
for (const message of given.messages ?? []) builder.trigger("ipc-message", message);
|
|
1323
|
-
for (const [trigger, code] of Object.entries(given.exitCodes ?? {})) if (code !== void 0) builder.exitCode(trigger, code);
|
|
1324
|
-
return builder;
|
|
1325
|
-
},
|
|
1326
|
-
describe() {
|
|
1327
|
-
const missing = [];
|
|
1328
|
-
if (readiness === void 0) missing.push("readiness (svc.readiness)");
|
|
1329
|
-
if (liveness === void 0) missing.push("liveness (svc.liveness)");
|
|
1330
|
-
if (startup === void 0) missing.push("a startup budget (svc.startup)");
|
|
1331
|
-
if (phases.length === 0) missing.push("its shutdown phases (svc.phase)");
|
|
1332
|
-
if (cooperative === void 0) missing.push("a cooperative window, or no bound (svc.cooperativeWindow)");
|
|
1333
|
-
if (repeat === void 0) missing.push(`what a repeat signal does, "ignore" or "exit" (svc.onRepeatSignal)`);
|
|
1334
|
-
if (missing.length > 0) throw new Error(`${name} does not declare ${missing.join("; ")}: the runtime has no defaults`);
|
|
1335
|
-
const known = new Set(dependencies.map((entry) => entry.name));
|
|
1336
|
-
const problems = [];
|
|
1337
|
-
const policies = {
|
|
1338
|
-
readiness,
|
|
1339
|
-
liveness
|
|
1340
|
-
};
|
|
1341
|
-
for (const gate of ["readiness", "liveness"]) problems.push(...timingProblems(gate, policies[gate]));
|
|
1342
|
-
for (const check of checks) {
|
|
1343
|
-
if (check.dependency !== null && !known.has(check.dependency)) problems.push(`check ${check.name} exercises ${check.dependency}, which is not a dependency`);
|
|
1344
|
-
if (!isTime$1(check.poll.interval) || !isTime$1(check.poll.timeout)) problems.push(`check ${check.name} polls in a unit that is not a time`);
|
|
1345
|
-
else if (!above(check.poll.interval) || !above(check.poll.timeout)) problems.push(`check ${check.name} polls with an interval or a timeout that is not above zero`);
|
|
1346
|
-
if (!(check.poll.jitter >= 0 && check.poll.jitter < 1)) problems.push(`check ${check.name} has a jitter of ${check.poll.jitter}, not a fraction below 1`);
|
|
1347
|
-
problems.push(...marksProblems(check));
|
|
1348
|
-
const timing = check.gates === null ? null : policies[check.gates];
|
|
1349
|
-
if (timing?.type === "debounce" && isTime$1(check.poll.interval) && above(timing.interval) && valueIn(check.poll.interval, "s") !== valueIn(timing.interval, "s")) problems.push(`check ${check.name} polls every ${valueIn(check.poll.interval, "s")} s, but the ${check.gates} debounce counts polls every ${valueIn(timing.interval, "s")} s`);
|
|
1350
|
-
}
|
|
1351
|
-
for (const controller of controllers) {
|
|
1352
|
-
for (const needed of controller.requires) if (!known.has(needed)) problems.push(`controller ${controller.name} requires ${needed}, which is not a dependency`);
|
|
1353
|
-
for (const [route, operation] of Object.entries(controller.routes)) problems.push(...operationProblems(`controller ${controller.name}`, route, operation));
|
|
1354
|
-
for (const declared of controller.patterns) problems.push(...routePatternProblems(`controller ${controller.name}`, declared));
|
|
1355
|
-
}
|
|
1356
|
-
for (const limiter of limiters) {
|
|
1357
|
-
const { name: entry, ...spec } = limiter;
|
|
1358
|
-
problems.push(...limiterProblems(entry, spec));
|
|
1359
|
-
if (limiter.store?.kind === "shared" && typeof limiter.store.dependency === "string" && !known.has(limiter.store.dependency)) problems.push(`limiter ${entry} keeps its counts in ${limiter.store.dependency}, which is not a dependency`);
|
|
1360
|
-
}
|
|
1361
|
-
for (const dependency of dependencies) for (const [name, operation] of Object.entries(dependency.operations)) problems.push(...operationProblems(`dependency ${dependency.name}`, name, operation));
|
|
1362
|
-
for (const phase of phases) if (!isTime$1(phase.cap)) problems.push(`phase ${phase.name}'s cap is not a time`);
|
|
1363
|
-
problems.push(...shutdownProblems(floor, repeat, triggers, exitCodes, messages));
|
|
1364
|
-
const ranked = phases.map((phase) => phase.name).filter((each) => stageRank(each) >= 0);
|
|
1365
|
-
for (let at = 1; at < ranked.length; at += 1) if (stageRank(ranked[at]) < stageRank(ranked[at - 1])) problems.push(`the phase "${ranked[at]}" comes after "${ranked[at - 1]}", but the stages go ${STAGES.join(", ")}`);
|
|
1366
|
-
if (problems.length > 0) throw new Error(`${name}: ${problems.join("; ")}`);
|
|
1367
|
-
const controllersOut = controllers.map((entry) => ({
|
|
1368
|
-
name: entry.name,
|
|
1369
|
-
requires: [...entry.requires],
|
|
1370
|
-
routes: { ...entry.routes },
|
|
1371
|
-
...entry.patterns.length === 0 ? {} : { patterns: entry.patterns.map((each) => ({
|
|
1372
|
-
pattern: each.pattern,
|
|
1373
|
-
operations: { ...each.operations }
|
|
1374
|
-
})) }
|
|
1375
|
-
}));
|
|
1376
|
-
return {
|
|
1377
|
-
format: FORMAT,
|
|
1378
|
-
name,
|
|
1379
|
-
partial,
|
|
1380
|
-
transports: transports.map((entry) => ({
|
|
1381
|
-
...entry,
|
|
1382
|
-
knobs: { ...entry.knobs }
|
|
1383
|
-
})),
|
|
1384
|
-
dependencies: dependencies.map((entry) => ({
|
|
1385
|
-
...entry,
|
|
1386
|
-
knobs: { ...entry.knobs },
|
|
1387
|
-
operations: { ...entry.operations }
|
|
1388
|
-
})),
|
|
1389
|
-
controllers: controllersOut,
|
|
1390
|
-
checks: checks.map(copyCheck),
|
|
1391
|
-
...limiters.length === 0 ? {} : { limiters: limiters.map((entry) => ({
|
|
1392
|
-
...entry,
|
|
1393
|
-
policy: { ...entry.policy },
|
|
1394
|
-
store: { ...entry.store },
|
|
1395
|
-
evict: { maxKeys: { ...entry.evict.maxKeys } }
|
|
1396
|
-
})) },
|
|
1397
|
-
policies: {
|
|
1398
|
-
readiness,
|
|
1399
|
-
liveness,
|
|
1400
|
-
startup
|
|
1401
|
-
},
|
|
1402
|
-
couplings: couplings.map((entry) => ({ ...entry })),
|
|
1403
|
-
shutdown: {
|
|
1404
|
-
phases: phases.map((entry) => ({ ...entry })),
|
|
1405
|
-
cooperativeWindow: cooperative,
|
|
1406
|
-
...floor === void 0 ? {} : { cooperativeFloor: floor },
|
|
1407
|
-
onRepeatSignal: repeat,
|
|
1408
|
-
...triggers.length === 0 ? {} : { triggers: [...triggers] },
|
|
1409
|
-
...messages.length === 0 ? {} : { messages: messages.map((each) => ({ ...each })) },
|
|
1410
|
-
...Object.keys(exitCodes).length === 0 ? {} : { exitCodes: { ...exitCodes } }
|
|
1411
|
-
}
|
|
1412
|
-
};
|
|
1413
|
-
},
|
|
1414
|
-
start(options = {}) {
|
|
1415
|
-
if (runtime !== void 0) return runtime;
|
|
1416
|
-
if (running !== void 0) return Promise.reject(/* @__PURE__ */ new Error(`${name} runs its lifecycle: use its health`));
|
|
1417
|
-
let made;
|
|
1418
|
-
try {
|
|
1419
|
-
made = makeRuntime(options).runtime;
|
|
1420
|
-
} catch (error) {
|
|
1421
|
-
return Promise.reject(error);
|
|
1422
|
-
}
|
|
1423
|
-
started = true;
|
|
1424
|
-
live = made;
|
|
1425
|
-
runtime = made.warmUp().then(() => made);
|
|
1426
|
-
return runtime;
|
|
1427
|
-
},
|
|
1428
|
-
run(options = {}) {
|
|
1429
|
-
if (running !== void 0) return running;
|
|
1430
|
-
if (runtime !== void 0) return Promise.reject(/* @__PURE__ */ new Error(`${name} started its health only: run() starts the whole lifecycle`));
|
|
1431
|
-
try {
|
|
1432
|
-
const description = builder.describe();
|
|
1433
|
-
const components = {
|
|
1434
|
-
transports: transportImpls,
|
|
1435
|
-
dependencies: handles,
|
|
1436
|
-
healthServer: probes,
|
|
1437
|
-
phaseWork,
|
|
1438
|
-
choices,
|
|
1439
|
-
stopPollers: () => {}
|
|
1440
|
-
};
|
|
1441
|
-
const problems = [...lifecycleProblems(description, components), ...uncaughtProblems(options.uncaught)];
|
|
1442
|
-
if (problems.length > 0) throw new Error(`${name}: ${problems.join("; ")}`);
|
|
1443
|
-
const made = makeRuntime(options);
|
|
1444
|
-
started = true;
|
|
1445
|
-
live = made.runtime;
|
|
1446
|
-
const ran = lifecycle(made.description, made.runtime, {
|
|
1447
|
-
...components,
|
|
1448
|
-
stopPollers: made.stopPollers
|
|
1449
|
-
}, made.options, (given) => {
|
|
1450
|
-
life = given;
|
|
1451
|
-
});
|
|
1452
|
-
running = ran.started.then(() => ran);
|
|
1453
|
-
} catch (error) {
|
|
1454
|
-
return Promise.reject(error);
|
|
1455
|
-
}
|
|
1456
|
-
return running;
|
|
1457
|
-
}
|
|
1458
|
-
};
|
|
1459
|
-
return builder;
|
|
1460
|
-
};
|
|
1461
|
-
// src/law.ts
|
|
1462
|
-
const value = (bound) => bound === "inf" || bound === "off" ? Infinity : Number(bound);
|
|
1463
|
-
const keyOf = (term) => term.at !== void 0 ? `${term.knob}@${term.at}` : term.knob;
|
|
1464
|
-
/** This function gives the bound of a symbol, or undefined when the symbol has no bound. */
|
|
1465
|
-
const boundOf$1 = (bind, key) => {
|
|
1466
|
-
const bound = bind[key];
|
|
1467
|
-
return typeof bound === "number" || bound === "inf" || bound === "off" ? bound : void 0;
|
|
1468
|
-
};
|
|
1469
|
-
/** This function gives the inequalities of a rule: its expr, then the others. */
|
|
1470
|
-
const inequalities = (rule) => [rule.expr, ...rule.expr_more ?? []];
|
|
1471
|
-
/** This function gives the symbols that a rule reads, as `symbol@hop`. */
|
|
1472
|
-
const symbolsOf = (rule) => {
|
|
1473
|
-
const out = [];
|
|
1474
|
-
for (const inequality of inequalities(rule)) for (const term of [...inequality.lhs.terms, ...inequality.rhs.terms]) {
|
|
1475
|
-
const key = keyOf(term);
|
|
1476
|
-
if (!out.includes(key)) out.push(key);
|
|
1477
|
-
}
|
|
1478
|
-
return out;
|
|
1479
|
-
};
|
|
1480
|
-
/** This function gives the value of a side, Σ coef × symbol + const, or null when a symbol has no bound. */
|
|
1481
|
-
const sideOf = (side, bind) => {
|
|
1482
|
-
let total = side.const ?? 0;
|
|
1483
|
-
for (const term of side.terms) {
|
|
1484
|
-
const bound = boundOf$1(bind, keyOf(term));
|
|
1485
|
-
if (bound === void 0) return null;
|
|
1486
|
-
total += term.coef * value(bound);
|
|
1487
|
-
}
|
|
1488
|
-
return total;
|
|
1489
|
-
};
|
|
1490
|
-
/** This function gives the margin of an inequality: from the binding (one for all, or one each), else from the rule, else 0. */
|
|
1491
|
-
const marginOf = (inequality, index, bind) => {
|
|
1492
|
-
if (Array.isArray(bind.margins)) return Number(bind.margins[index] ?? 0);
|
|
1493
|
-
if (typeof bind.margin === "number") return bind.margin;
|
|
1494
|
-
return typeof inequality.margin === "number" ? inequality.margin : 0;
|
|
1495
|
-
};
|
|
1496
|
-
/** This function tells if an inequality reads a symbol with no limit: "inf", "off" or an infinite number. */
|
|
1497
|
-
const readsAbsent = (inequality, bind) => [...inequality.lhs.terms, ...inequality.rhs.terms].some((term) => {
|
|
1498
|
-
const bound = boundOf$1(bind, keyOf(term));
|
|
1499
|
-
return bound !== void 0 && value(bound) === Infinity;
|
|
1500
|
-
});
|
|
1501
|
-
/**
|
|
1502
|
-
* This function evaluates each inequality on one binding. "<" holds when lhs + margin < rhs, and
|
|
1503
|
-
* ">" holds when lhs > rhs + margin. Only the inequalities with all their symbols bound apply.
|
|
1504
|
-
* The others are null. Absent timers can leave no order: the same infinity on each side, or both
|
|
1505
|
-
* signs of infinity on one side. Then the inequality holds only because a timer is absent.
|
|
1506
|
-
*/
|
|
1507
|
-
const evaluate = (rule, bind) => inequalities(rule).map((inequality, index) => {
|
|
1508
|
-
const lhs = sideOf(inequality.lhs, bind);
|
|
1509
|
-
const rhs = sideOf(inequality.rhs, bind);
|
|
1510
|
-
const margin = marginOf(inequality, index, bind);
|
|
1511
|
-
if (lhs === null || rhs === null) return {
|
|
1512
|
-
lhs,
|
|
1513
|
-
rhs,
|
|
1514
|
-
margin,
|
|
1515
|
-
holds: null
|
|
1516
|
-
};
|
|
1517
|
-
if ((Number.isNaN(lhs) || Number.isNaN(rhs) || lhs === rhs && !Number.isFinite(lhs)) && readsAbsent(inequality, bind)) return {
|
|
1518
|
-
lhs,
|
|
1519
|
-
rhs,
|
|
1520
|
-
margin,
|
|
1521
|
-
holds: true
|
|
1522
|
-
};
|
|
1523
|
-
return {
|
|
1524
|
-
lhs,
|
|
1525
|
-
rhs,
|
|
1526
|
-
margin,
|
|
1527
|
-
holds: inequality.op === "<" ? lhs + margin < rhs : inequality.op === "<=" ? lhs + margin <= rhs : inequality.op === ">" ? lhs > rhs + margin : lhs >= rhs + margin
|
|
1528
|
-
};
|
|
1529
|
-
});
|
|
1530
|
-
/**
|
|
1531
|
-
* This function gives the verdict of an evaluation. When an applicable inequality fails, the
|
|
1532
|
-
* verdict is "violated". When all hold only because a timer is absent, the verdict is "unbounded",
|
|
1533
|
-
* because nothing then limits the window. Otherwise the verdict is "holds". When no inequality
|
|
1534
|
-
* applies, the arithmetic cannot decide, and the verdict is "depends".
|
|
1535
|
-
*/
|
|
1536
|
-
const verdictOf = (rule, bind, evaluated) => {
|
|
1537
|
-
const applicable = evaluated.flatMap((entry, index) => entry.holds === null ? [] : [index]);
|
|
1538
|
-
if (applicable.length === 0) return "depends";
|
|
1539
|
-
if (applicable.some((index) => evaluated[index]?.holds === false)) return "violated";
|
|
1540
|
-
const all = inequalities(rule);
|
|
1541
|
-
return applicable.some((index) => readsAbsent(all[index], bind)) ? "unbounded" : "holds";
|
|
1542
|
-
};
|
|
1543
|
-
/** This function gives the mark of a symbol: its mark at its hop, else the mark of the bare symbol. */
|
|
1544
|
-
const markOf = (marks, key) => marks[key] ?? marks[key.split("@")[0] ?? key];
|
|
1545
|
-
const STEP_EPSILON = 1e-9;
|
|
1546
|
-
const scaled = (seconds, k, mark) => {
|
|
1547
|
-
switch (mark.kind) {
|
|
1548
|
-
case "scalable": return seconds * k;
|
|
1549
|
-
case "fixed": return seconds;
|
|
1550
|
-
case "stepped": return Math.max(Math.ceil(seconds * k / mark.step - STEP_EPSILON), 0) * mark.step;
|
|
1551
|
-
case "floored": return Math.max(seconds * k, mark.floor);
|
|
1552
|
-
}
|
|
1553
|
-
};
|
|
1554
|
-
/**
|
|
1555
|
-
* This function scales each timer of a binding by k, by its mark (E10). An absent timer ("inf" or
|
|
1556
|
-
* "off") stays absent. A chain scales each value. The margins of the binding and the constants of
|
|
1557
|
-
* the rules do not scale: they are tolerances, not timers. A symbol with no mark keeps its value.
|
|
1558
|
-
*/
|
|
1559
|
-
const compress = (bind, k, marks) => {
|
|
1560
|
-
if (!Number.isFinite(k) || k <= 0) throw new RangeError(`a compression scales time by a finite factor above 0, not ${k}`);
|
|
1561
|
-
const out = {};
|
|
1562
|
-
const unmarked = [];
|
|
1563
|
-
for (const [key, bound] of Object.entries(bind)) {
|
|
1564
|
-
if (key === "margin" || key === "margins") continue;
|
|
1565
|
-
const mark = markOf(marks, key);
|
|
1566
|
-
if (mark === void 0) {
|
|
1567
|
-
if (bound !== void 0) unmarked.push(key);
|
|
1568
|
-
out[key] = bound;
|
|
1569
|
-
} else if (typeof bound === "number") out[key] = scaled(bound, k, mark);
|
|
1570
|
-
else if (Array.isArray(bound)) out[key] = bound.map((seconds) => scaled(seconds, k, mark));
|
|
1571
|
-
else out[key] = bound;
|
|
1572
|
-
}
|
|
1573
|
-
const margins = {
|
|
1574
|
-
...bind.margin !== void 0 ? { margin: bind.margin } : {},
|
|
1575
|
-
...bind.margins !== void 0 ? { margins: bind.margins } : {}
|
|
1576
|
-
};
|
|
1577
|
-
return {
|
|
1578
|
-
bind: {
|
|
1579
|
-
...out,
|
|
1580
|
-
...margins
|
|
1581
|
-
},
|
|
1582
|
-
unmarked
|
|
1583
|
-
};
|
|
1584
|
-
};
|
|
1585
|
-
/**
|
|
1586
|
-
* This function tells which rules a compression by k changes. A compression keeps each ordering
|
|
1587
|
-
* when no verdict changes. A change in either direction counts: a violation that the compression
|
|
1588
|
-
* hides is as wrong as one that it makes. The unmarked symbols are the symbols that the rules read.
|
|
1589
|
-
*/
|
|
1590
|
-
const compression = (rules, bind, k, marks) => {
|
|
1591
|
-
const compressed = compress(bind, k, marks);
|
|
1592
|
-
const changed = rules.flatMap((rule) => {
|
|
1593
|
-
const before = verdictOf(rule, bind, evaluate(rule, bind));
|
|
1594
|
-
const after = verdictOf(rule, compressed.bind, evaluate(rule, compressed.bind));
|
|
1595
|
-
return before === after ? [] : [{
|
|
1596
|
-
rule: rule.id,
|
|
1597
|
-
before,
|
|
1598
|
-
after
|
|
1599
|
-
}];
|
|
1600
|
-
});
|
|
1601
|
-
const read = new Set(rules.flatMap(symbolsOf));
|
|
1602
|
-
return {
|
|
1603
|
-
changed,
|
|
1604
|
-
unmarked: compressed.unmarked.filter((symbol) => read.has(symbol))
|
|
1605
|
-
};
|
|
1606
|
-
};
|
|
1607
|
-
// src/lint.ts
|
|
1608
|
-
/** The law binds this statistic of a distribution. */
|
|
1609
|
-
const LAW_STATISTIC = "p99";
|
|
1610
|
-
/** These are the identifiers of the rules that each declaration gets. */
|
|
1611
|
-
const SERVICE_RULES = [
|
|
1612
|
-
"O04",
|
|
1613
|
-
"O09",
|
|
1614
|
-
"O10",
|
|
1615
|
-
"O11",
|
|
1616
|
-
"O12",
|
|
1617
|
-
"O13",
|
|
1618
|
-
"O14",
|
|
1619
|
-
"O15",
|
|
1620
|
-
"O16",
|
|
1621
|
-
"O17",
|
|
1622
|
-
"O18",
|
|
1623
|
-
"O19",
|
|
1624
|
-
"O20",
|
|
1625
|
-
"O21",
|
|
1626
|
-
"O22",
|
|
1627
|
-
"O23",
|
|
1628
|
-
"O27",
|
|
1629
|
-
"O29",
|
|
1630
|
-
"O30"
|
|
1631
|
-
];
|
|
1632
|
-
/**
|
|
1633
|
-
* These are the two rules of the law for a keyed limiter, as data. The caller of `lint()` gives
|
|
1634
|
-
* them, as it gives each other rule. Each one applies one time for each limiter.
|
|
1635
|
-
* - O29: the longest `Retry-After` of a limiter is at most the retry window of its caller. Its
|
|
1636
|
-
* second inequality applies the same bound to a new key at the cap of keys.
|
|
1637
|
-
* Otherwise the caller stops before the limit lets its call in, and a refusal becomes a failure.
|
|
1638
|
-
* - O30: the least delay of a caller before it sends a refused call again is at least the longest
|
|
1639
|
-
* `Retry-After` of the policy. Otherwise a caller that does not read the header sends again too
|
|
1640
|
-
* early, and the limiter refuses each early call again.
|
|
1641
|
-
*
|
|
1642
|
-
* The identifiers O29 and O30 are provisional. The private corpus of the law (R3, track B) has 28
|
|
1643
|
-
* rules, O01 to O28. It does not have these two rules yet, thus they take the next two identifiers.
|
|
1644
|
-
*/
|
|
1645
|
-
const LIMITER_RULES = [{
|
|
1646
|
-
id: "O29",
|
|
1647
|
-
rule: "the longest Retry-After of a limiter, at its cap of keys too, is at most the retry window of its caller",
|
|
1648
|
-
window: "the caller stops before the limit lets its call in, thus a refusal becomes a failure",
|
|
1649
|
-
expr: {
|
|
1650
|
-
lhs: { terms: [{
|
|
1651
|
-
knob: "retry_after",
|
|
1652
|
-
coef: 1,
|
|
1653
|
-
at: "limiter"
|
|
1654
|
-
}] },
|
|
1655
|
-
op: "<=",
|
|
1656
|
-
rhs: { terms: [{
|
|
1657
|
-
knob: "client_retry_window",
|
|
1658
|
-
coef: 1,
|
|
1659
|
-
at: "caller"
|
|
1660
|
-
}] },
|
|
1661
|
-
margin: 0
|
|
1662
|
-
},
|
|
1663
|
-
expr_more: [{
|
|
1664
|
-
lhs: { terms: [{
|
|
1665
|
-
knob: "retry_after_at_cap",
|
|
1666
|
-
coef: 1,
|
|
1667
|
-
at: "limiter"
|
|
1668
|
-
}] },
|
|
1669
|
-
op: "<=",
|
|
1670
|
-
rhs: { terms: [{
|
|
1671
|
-
knob: "client_retry_window",
|
|
1672
|
-
coef: 1,
|
|
1673
|
-
at: "caller"
|
|
1674
|
-
}] },
|
|
1675
|
-
margin: 0
|
|
1676
|
-
}]
|
|
1677
|
-
}, {
|
|
1678
|
-
id: "O30",
|
|
1679
|
-
rule: "the least delay of a caller before it sends a refused call again is at least the longest Retry-After of the limiter",
|
|
1680
|
-
window: "a caller that does not read Retry-After sends again too early, and the limiter refuses each early call again",
|
|
1681
|
-
expr: {
|
|
1682
|
-
lhs: { terms: [{
|
|
1683
|
-
knob: "retry_delay",
|
|
1684
|
-
coef: 1,
|
|
1685
|
-
at: "caller"
|
|
1686
|
-
}] },
|
|
1687
|
-
op: ">=",
|
|
1688
|
-
rhs: { terms: [{
|
|
1689
|
-
knob: "retry_after",
|
|
1690
|
-
coef: 1,
|
|
1691
|
-
at: "limiter"
|
|
1692
|
-
}] },
|
|
1693
|
-
margin: 0
|
|
1694
|
-
}
|
|
1695
|
-
}];
|
|
1696
|
-
/** This function gives a finite amount in seconds. */
|
|
1697
|
-
const secondsOf = (amount) => rationals.toNumber(convertExact(rationals.fromNumber(amount.value), amount.unit, "s"));
|
|
1698
|
-
/** This function gives an amount, or no bound, as the law binds it. */
|
|
1699
|
-
const boundOf = (amount) => amount.value === "infinite" ? "inf" : secondsOf(amount);
|
|
1700
|
-
/**
|
|
1701
|
-
* This function gives a value of the scenario as the law binds it, with its epoch. A distribution
|
|
1702
|
-
* binds its p99. A distribution without a p99 binds nothing, thus each rule that reads it depends.
|
|
1703
|
-
*/
|
|
1704
|
-
const claimOf = (value) => {
|
|
1705
|
-
if (!("epoch" in value)) return {
|
|
1706
|
-
bound: boundOf(value),
|
|
1707
|
-
epoch: null
|
|
1708
|
-
};
|
|
1709
|
-
if ("value" in value) return {
|
|
1710
|
-
bound: boundOf(value.value),
|
|
1711
|
-
epoch: value.epoch
|
|
1712
|
-
};
|
|
1713
|
-
const statistic = value.distribution["p99"];
|
|
1714
|
-
return {
|
|
1715
|
-
bound: statistic === void 0 ? void 0 : secondsOf(statistic),
|
|
1716
|
-
epoch: value.epoch
|
|
1717
|
-
};
|
|
1718
|
-
};
|
|
1719
|
-
/** This function binds each value of a part of the scenario, at the symbol that `at` gives, and records its epoch. */
|
|
1720
|
-
const bindScenario = (values, at, bind, epochs) => {
|
|
1721
|
-
for (const [symbol, value] of Object.entries(values)) {
|
|
1722
|
-
const claim = claimOf(value);
|
|
1723
|
-
const key = at(symbol);
|
|
1724
|
-
epochs[key] = claim.epoch;
|
|
1725
|
-
if (claim.bound !== void 0) bind[key] = claim.bound;
|
|
1726
|
-
else delete bind[key];
|
|
1727
|
-
}
|
|
1728
|
-
};
|
|
1729
|
-
/** This function tells if a unit is a time. A unit that the package does not know is not a time. */
|
|
1730
|
-
const isTime = (unit) => {
|
|
1731
|
-
try {
|
|
1732
|
-
return convertible(unit, "s");
|
|
1733
|
-
} catch {
|
|
1734
|
-
return false;
|
|
1735
|
-
}
|
|
1736
|
-
};
|
|
1737
|
-
/** This function gives the largest of some numbers of seconds, or undefined when there are none. */
|
|
1738
|
-
const most = (values) => values.length === 0 ? void 0 : Math.max(...values);
|
|
1739
|
-
/** This function gives what the declaration tells about the service, as symbols of the law, and the source of each. */
|
|
1740
|
-
const serviceBindings = (svc, scenario) => {
|
|
1741
|
-
const bind = {};
|
|
1742
|
-
const from = {};
|
|
1743
|
-
const epochs = {};
|
|
1744
|
-
const set = (symbol, bound, said) => {
|
|
1745
|
-
if (bound === void 0) return;
|
|
1746
|
-
bind[symbol] = bound;
|
|
1747
|
-
from[symbol] = said;
|
|
1748
|
-
};
|
|
1749
|
-
const gating = svc.checks.filter((check) => check.gates !== null);
|
|
1750
|
-
const readinessChecks = gating.filter((check) => check.gates === "readiness");
|
|
1751
|
-
const livenessChecks = gating.filter((check) => check.gates === "liveness");
|
|
1752
|
-
set("check_poll_interval@service", most(gating.map((check) => secondsOf(check.poll.interval))), "the slowest gating check's interval");
|
|
1753
|
-
set("check_timeout@service", most(gating.map((check) => secondsOf(check.poll.timeout))), "the longest gating check's timeout");
|
|
1754
|
-
set("slowest_poll@service", most(gating.map((check) => secondsOf(check.poll.timeout) + secondsOf(check.poll.interval) * (1 + check.poll.jitter))), "a check's timeout plus one jittered interval, at its slowest");
|
|
1755
|
-
const windowOf = (checks, policy) => most(checks.map((check) => secondsOf(failSpan(svc.policies[policy], check.poll, "timeout").max)));
|
|
1756
|
-
const readiness = windowOf(readinessChecks, "readiness");
|
|
1757
|
-
const liveness = windowOf(livenessChecks, "liveness");
|
|
1758
|
-
set("readiness_detection_window@service", readiness, "the readiness latch, failing by timeout");
|
|
1759
|
-
if (readiness !== void 0) set("readiness_detection_window@pod", readiness + secondsOf(scenario.probes.readiness), "the readiness latch, then the kubelet's probe");
|
|
1760
|
-
if (liveness !== void 0) set("liveness_detection_window@pod", liveness + secondsOf(scenario.probes.liveness), "the liveness window, then the kubelet's probe");
|
|
1761
|
-
set("startup_budget@pod", secondsOf(svc.policies.startup), "the declared startup budget");
|
|
1762
|
-
set("process_drain_time@process", svc.shutdown.phases.reduce((total, phase) => total + secondsOf(phase.cap), 0), `the shutdown phases' caps, summed (${svc.shutdown.phases.map((phase) => phase.name).join(", ")})`);
|
|
1763
|
-
set("cooperative_window@process", boundOf(svc.shutdown.cooperativeWindow), "the declared cooperative window");
|
|
1764
|
-
const hold = svc.shutdown.phases.find((phase) => phase.name === "fail readiness");
|
|
1765
|
-
set("readiness_hold@process", hold === void 0 ? 0 : secondsOf(hold.cap), hold === void 0 ? "no fail readiness phase: the intake stops at once after the signal" : "the cap of the fail readiness phase, which holds while the endpoints update");
|
|
1766
|
-
const capOf = (name) => {
|
|
1767
|
-
const phase = svc.shutdown.phases.find((each) => each.name === name);
|
|
1768
|
-
return phase === void 0 ? void 0 : secondsOf(phase.cap);
|
|
1769
|
-
};
|
|
1770
|
-
set("drain_phase_cap@process", capOf("drain"), "the cap of the drain phase alone");
|
|
1771
|
-
set("close_phase_cap@process", capOf("close transports"), "the cap of the close transports phase alone");
|
|
1772
|
-
bindScenario(scenario.platform, (symbol) => symbol, bind, epochs);
|
|
1773
|
-
for (const [symbol, epoch] of Object.entries(epochs)) if (bind[symbol] === void 0) delete from[symbol];
|
|
1774
|
-
else from[symbol] = epoch === null ? "the scenario" : `the scenario, for ${epoch}`;
|
|
1775
|
-
return {
|
|
1776
|
-
bind,
|
|
1777
|
-
from,
|
|
1778
|
-
epochs
|
|
1779
|
-
};
|
|
1780
|
-
};
|
|
1781
|
-
/** This function gives the client knobs of a dependency at @client, and the values of the scenario at its own hops. */
|
|
1782
|
-
const dependencyBindings = (dependency, scenario) => {
|
|
1783
|
-
const bind = {};
|
|
1784
|
-
const epochs = {};
|
|
1785
|
-
for (const [knob, amount] of Object.entries(dependency.knobs)) bind[`${knob}@client`] = boundOf(amount);
|
|
1786
|
-
bindScenario(scenario.dependencies?.[dependency.name] ?? {}, (symbol) => symbol.includes("@") ? symbol : `${symbol}@dependency`, bind, epochs);
|
|
1787
|
-
return {
|
|
1788
|
-
bind,
|
|
1789
|
-
epochs
|
|
1790
|
-
};
|
|
1791
|
-
};
|
|
1792
|
-
/**
|
|
1793
|
-
* These are the hops of the law at which the knobs of a transport bind, by its kind. A worker does
|
|
1794
|
-
* its units as a consumer does, thus both bind at `consumer` (O20). A server binds at `server`, in a
|
|
1795
|
-
* scope of its own. In the scope of a dependency, `@server` is the server of that dependency. Thus
|
|
1796
|
-
* the knobs of a server of the service do not bind there, or O19 pairs the wrong two sides.
|
|
1797
|
-
*/
|
|
1798
|
-
const TRANSPORT_HOPS = {
|
|
1799
|
-
server: "server",
|
|
1800
|
-
consumer: "consumer",
|
|
1801
|
-
worker: "consumer"
|
|
1802
|
-
};
|
|
1803
|
-
/**
|
|
1804
|
-
* This function gives the time knobs of a transport at the hop of its kind, and the values of the
|
|
1805
|
-
* scenario for the transport. These values are of the source of a consumer, or of the clients of a
|
|
1806
|
-
* server. A knob with a hop in its name binds at that hop. A count binds no symbol.
|
|
1807
|
-
*/
|
|
1808
|
-
const transportBindings = (transport, hop, scenario) => {
|
|
1809
|
-
const bind = {};
|
|
1810
|
-
const epochs = {};
|
|
1811
|
-
for (const [knob, amount] of Object.entries(transport.knobs)) {
|
|
1812
|
-
if (!isTime(amount.unit)) continue;
|
|
1813
|
-
bind[knob.includes("@") ? knob : `${knob}@${hop}`] = boundOf(amount);
|
|
1814
|
-
}
|
|
1815
|
-
bindScenario(scenario.transports?.[transport.name] ?? {}, (symbol) => symbol, bind, epochs);
|
|
1816
|
-
return {
|
|
1817
|
-
bind,
|
|
1818
|
-
epochs
|
|
1819
|
-
};
|
|
1820
|
-
};
|
|
1821
|
-
/**
|
|
1822
|
-
* This function gives the symbols of a limiter, and the values of the scenario for its callers.
|
|
1823
|
-
* - `retry_after@limiter` is the longest `Retry-After` of a refusal by the policy, in whole seconds.
|
|
1824
|
-
* It is one interval of the bucket, or one window, as the HTTP edge sends it.
|
|
1825
|
-
* - `retry_after_at_cap@limiter` is the longest `Retry-After` of a new key at the cap of keys. It is
|
|
1826
|
-
* the idle span, `burst / rate` or one window.
|
|
1827
|
-
*/
|
|
1828
|
-
const limiterBindings = (limiter, scenario) => {
|
|
1829
|
-
const bind = {
|
|
1830
|
-
"retry_after@limiter": retryAfterSeconds(longestRetryAfter(limiter.policy)),
|
|
1831
|
-
"retry_after_at_cap@limiter": retryAfterSeconds(idleSpan(limiter.policy))
|
|
1832
|
-
};
|
|
1833
|
-
const epochs = {};
|
|
1834
|
-
bindScenario(scenario.limiters?.[limiter.name] ?? {}, (symbol) => symbol.includes("@") ? symbol : `${symbol}@caller`, bind, epochs);
|
|
1835
|
-
return {
|
|
1836
|
-
bind,
|
|
1837
|
-
epochs
|
|
1838
|
-
};
|
|
1839
|
-
};
|
|
1840
|
-
const SCOPED = /@(client|dependency|server)$/;
|
|
1841
|
-
const SERVER_SCOPED = /@server$/;
|
|
1842
|
-
const TRANSPORT_SCOPED = /@consumer$/;
|
|
1843
|
-
const LIMITER_SCOPED = /@(limiter|caller)$/;
|
|
1844
|
-
/** This identifier is the rule of the findings about a field that the declaration does not give. */
|
|
1845
|
-
const DECLARATION_RULE = "declaration";
|
|
1846
|
-
/**
|
|
1847
|
-
* This function gives a finding for each field of the shutdown that a description does not give.
|
|
1848
|
-
* `describe()` refuses such a declaration. A description from another source, for example the
|
|
1849
|
-
* JSON of 0.4, can lack the field. The verdict is `depends`, and `unbound` names the field.
|
|
1850
|
-
*/
|
|
1851
|
-
const declarationFindings = (svc) => svc.shutdown.onRepeatSignal === void 0 ? [{
|
|
1852
|
-
rule: DECLARATION_RULE,
|
|
1853
|
-
scope: svc.name,
|
|
1854
|
-
verdict: "depends",
|
|
1855
|
-
statement: "the declaration tells what a signal does while the shutdown runs: ignore or exit",
|
|
1856
|
-
window: "a repeat signal has no declared effect",
|
|
1857
|
-
evaluated: [],
|
|
1858
|
-
unbound: ["shutdown.onRepeatSignal"]
|
|
1859
|
-
}] : [];
|
|
1860
|
-
/** This identifier is the rule of the findings for two parts that have one name in the scopes of a rule. */
|
|
1861
|
-
const NAME_RULE = "unique-names";
|
|
1862
|
-
/**
|
|
1863
|
-
* This function gives a finding for each name that two parts share in one set of scopes. A rule
|
|
1864
|
-
* about a client applies to each dependency and each server transport. A rule about a consumer
|
|
1865
|
-
* applies to each consumer and each worker, and a rule about a limiter to each limiter. Two parts
|
|
1866
|
-
* with one name in a set give findings with one scope, thus a reader cannot tell them apart.
|
|
1867
|
-
*/
|
|
1868
|
-
const nameFindings = (svc) => {
|
|
1869
|
-
return [
|
|
1870
|
-
["the dependencies and the server transports", [...svc.dependencies.map((each) => each.name), ...svc.transports.filter((each) => each.kind === "server").map((each) => each.name)]],
|
|
1871
|
-
["the consumers and the workers", svc.transports.filter((each) => each.kind !== "server").map((each) => each.name)],
|
|
1872
|
-
["the limiters", (svc.limiters ?? []).map((each) => each.name)]
|
|
1873
|
-
].flatMap(([set, names]) => [...new Set(names.filter((name, index) => names.indexOf(name) !== index))].map((name) => ({
|
|
1874
|
-
rule: NAME_RULE,
|
|
1875
|
-
scope: name,
|
|
1876
|
-
verdict: "violated",
|
|
1877
|
-
statement: `each of ${set} has a name of its own`,
|
|
1878
|
-
window: `two parts have the name ${name}, thus their findings have one scope`,
|
|
1879
|
-
evaluated: [],
|
|
1880
|
-
unbound: []
|
|
1881
|
-
})));
|
|
1882
|
-
};
|
|
1883
|
-
/** This identifier is the rule of the findings about the stall time of a progress check (`Check.marks`). */
|
|
1884
|
-
const PROGRESS_RULE = "progress";
|
|
1885
|
-
/**
|
|
1886
|
-
* This rule compares the stall time of a progress check with the longest gap between two marks of
|
|
1887
|
-
* the slowest loop that it names. A shorter stall time trips the check while each loop turns on time.
|
|
1888
|
-
*/
|
|
1889
|
-
const PROGRESS_LAW = {
|
|
1890
|
-
id: PROGRESS_RULE,
|
|
1891
|
-
rule: "the stall time of a progress check is at least the longest gap between two marks of the slowest loop that it names",
|
|
1892
|
-
window: "a loop that turns on time can trip the check",
|
|
1893
|
-
expr: {
|
|
1894
|
-
lhs: { terms: [{
|
|
1895
|
-
knob: "stall",
|
|
1896
|
-
coef: 1,
|
|
1897
|
-
at: "check"
|
|
1898
|
-
}] },
|
|
1899
|
-
op: ">=",
|
|
1900
|
-
rhs: { terms: [{
|
|
1901
|
-
knob: "mark_gap",
|
|
1902
|
-
coef: 1,
|
|
1903
|
-
at: "loop"
|
|
1904
|
-
}] },
|
|
1905
|
-
margin: 0
|
|
1906
|
-
}
|
|
1907
|
-
};
|
|
1908
|
-
/** These are the knobs of a worker that bound the gap between two of its marks. */
|
|
1909
|
-
const WORKER_GAP_KNOBS = [
|
|
1910
|
-
"worker_sleep",
|
|
1911
|
-
"unit_processing_time",
|
|
1912
|
-
"retry_base",
|
|
1913
|
-
"retry_cap"
|
|
1914
|
-
];
|
|
1915
|
-
/**
|
|
1916
|
-
* This function gives the longest gap between two marks of each loop with a name. The rule counts
|
|
1917
|
-
* a mark at the end of each turn of the loop, also a turn that fails.
|
|
1918
|
-
* - A check polls on a fixed delay from the start of each poll, and each poll ends at its timeout
|
|
1919
|
-
* at the latest. Thus the gap is `interval × (1 + jitter) + timeout`.
|
|
1920
|
-
* - After an iteration, a lane of a worker waits its sleep, or its backoff after a failure. The
|
|
1921
|
-
* longest backoff is `retry_cap` plus its jitter of up to `retry_base`. The next iteration then
|
|
1922
|
-
* takes up to `unit_processing_time`. Thus the gap is the longer wait plus that bound.
|
|
1923
|
-
*
|
|
1924
|
-
* A sleep or a bound without a limit gives a gap without a limit. An iteration that does not end
|
|
1925
|
-
* at its abort can take more time, and the rule does not bound it.
|
|
1926
|
-
*/
|
|
1927
|
-
const gapsOf = (svc, loop) => {
|
|
1928
|
-
const gaps = svc.checks.filter((check) => check.name === loop).map((check) => ({
|
|
1929
|
-
loop,
|
|
1930
|
-
seconds: secondsOf(check.poll.interval) * (1 + check.poll.jitter) + secondsOf(check.poll.timeout),
|
|
1931
|
-
unbound: []
|
|
1932
|
-
}));
|
|
1933
|
-
for (const worker of svc.transports) {
|
|
1934
|
-
if (worker.kind !== "worker" || worker.name !== loop) continue;
|
|
1935
|
-
const knob = (name) => {
|
|
1936
|
-
const amount = worker.knobs[name];
|
|
1937
|
-
if (amount === void 0 || !isTime(amount.unit)) return void 0;
|
|
1938
|
-
return amount.value === "infinite" ? Infinity : secondsOf(amount);
|
|
1939
|
-
};
|
|
1940
|
-
const unbound = WORKER_GAP_KNOBS.filter((name) => knob(name) === void 0).map((name) => `${loop}.${name}`);
|
|
1941
|
-
if (unbound.length > 0) {
|
|
1942
|
-
gaps.push({
|
|
1943
|
-
loop,
|
|
1944
|
-
seconds: void 0,
|
|
1945
|
-
unbound
|
|
1946
|
-
});
|
|
1947
|
-
continue;
|
|
1948
|
-
}
|
|
1949
|
-
const [sleep, unit, base, cap] = WORKER_GAP_KNOBS.map((name) => knob(name));
|
|
1950
|
-
const backoff = Number.isFinite(base) && Number.isFinite(cap) ? backoffBand({
|
|
1951
|
-
base,
|
|
1952
|
-
factor: 2,
|
|
1953
|
-
cap,
|
|
1954
|
-
jitter: {
|
|
1955
|
-
kind: "add",
|
|
1956
|
-
amount: base
|
|
1957
|
-
}
|
|
1958
|
-
}).high : Infinity;
|
|
1959
|
-
gaps.push({
|
|
1960
|
-
loop,
|
|
1961
|
-
seconds: Math.max(sleep, backoff) + unit,
|
|
1962
|
-
unbound: []
|
|
1963
|
-
});
|
|
1964
|
-
}
|
|
1965
|
-
return gaps;
|
|
1966
|
-
};
|
|
1967
|
-
/**
|
|
1968
|
-
* This function gives the findings of each check that names the loops that mark it. A name that is
|
|
1969
|
-
* not a check or a worker of the service gets a finding of its own, with the verdict `depends`.
|
|
1970
|
-
* Without `marks`, a check gets no finding.
|
|
1971
|
-
*/
|
|
1972
|
-
const progressFindings = (svc) => {
|
|
1973
|
-
const findings = [];
|
|
1974
|
-
const finding = (check, fields) => ({
|
|
1975
|
-
rule: PROGRESS_RULE,
|
|
1976
|
-
scope: check.name,
|
|
1977
|
-
statement: PROGRESS_LAW.rule,
|
|
1978
|
-
window: PROGRESS_LAW.window,
|
|
1979
|
-
...fields
|
|
1980
|
-
});
|
|
1981
|
-
for (const check of svc.checks) {
|
|
1982
|
-
const marks = check.marks;
|
|
1983
|
-
if (marks === void 0) continue;
|
|
1984
|
-
const gaps = [];
|
|
1985
|
-
for (const loop of marks.loops) {
|
|
1986
|
-
const found = gapsOf(svc, loop);
|
|
1987
|
-
if (found.length > 0) gaps.push(...found);
|
|
1988
|
-
else findings.push(finding(check, {
|
|
1989
|
-
verdict: "depends",
|
|
1990
|
-
statement: "each loop that a progress check names is a check or a worker of the service",
|
|
1991
|
-
window: `the check waits for the marks of ${loop}, and the service has no loop with that name`,
|
|
1992
|
-
evaluated: [],
|
|
1993
|
-
unbound: [loop]
|
|
1994
|
-
}));
|
|
1995
|
-
}
|
|
1996
|
-
if (gaps.length === 0) continue;
|
|
1997
|
-
const unbound = [...isTime(marks.stall.unit) ? [] : [`${check.name}.marks.stall`], ...gaps.flatMap((gap) => gap.unbound)];
|
|
1998
|
-
const slowest = gaps.reduce((worst, gap) => (gap.seconds ?? -Infinity) > (worst.seconds ?? -Infinity) ? gap : worst);
|
|
1999
|
-
const bind = {};
|
|
2000
|
-
if (isTime(marks.stall.unit)) bind["stall@check"] = boundOf(marks.stall);
|
|
2001
|
-
if (unbound.length === 0 && slowest.seconds !== void 0) bind["mark_gap@loop"] = Number.isFinite(slowest.seconds) ? slowest.seconds : "inf";
|
|
2002
|
-
const evaluated = evaluate(PROGRESS_LAW, bind);
|
|
2003
|
-
findings.push(finding(check, {
|
|
2004
|
-
verdict: verdictOf(PROGRESS_LAW, bind, evaluated),
|
|
2005
|
-
...unbound.length === 0 ? { statement: `${PROGRESS_LAW.rule} (${slowest.loop})` } : {},
|
|
2006
|
-
evaluated,
|
|
2007
|
-
unbound
|
|
2008
|
-
}));
|
|
2009
|
-
}
|
|
2010
|
-
return findings;
|
|
2011
|
-
};
|
|
2012
|
-
/** This identifier is the rule of the findings about the bound of a unit against the cap of the `drain` phase. */
|
|
2013
|
-
const UNIT_BOUND_RULE = "unit-bound";
|
|
2014
|
-
/** This identifier is the rule of the findings about the close bound against the cap of the `close transports` phase. */
|
|
2015
|
-
const CLOSE_BOUND_RULE = "close-bound";
|
|
2016
|
-
/**
|
|
2017
|
-
* These are the rules of the bounds of a loop. Each one applies one time for each consumer and
|
|
2018
|
-
* each worker (S6 A10, A11). `lint()` gives them, as it gives `PROGRESS_RULE`.
|
|
2019
|
-
* - unit-bound: `unit_processing_time` is less than the cap of the `drain` phase. Otherwise the
|
|
2020
|
-
* drain can cut a unit that ends in its bound. A unit without a bound violates the rule.
|
|
2021
|
-
* - close-bound: `close_bound` is less than the cap of the `close transports` phase. Otherwise the
|
|
2022
|
-
* phase can end before the loop closes.
|
|
2023
|
-
*
|
|
2024
|
-
* The corpus of the law (R3) does not have these rules, thus they have identifiers of the service,
|
|
2025
|
-
* not O-numbers. The symbols `drain_phase_cap@process` and `close_phase_cap@process` are the caps of
|
|
2026
|
-
* one phase. `process_drain_time@process` is the sum of the caps of all the phases. A phase that the
|
|
2027
|
-
* declaration does not give binds no symbol, thus its finding is `depends`.
|
|
2028
|
-
*/
|
|
2029
|
-
const BOUND_RULES = [{
|
|
2030
|
-
id: UNIT_BOUND_RULE,
|
|
2031
|
-
rule: "the bound of one unit of a consumer or a worker is less than the cap of the drain phase",
|
|
2032
|
-
window: "the drain can cut a unit that ends in its bound",
|
|
2033
|
-
expr: {
|
|
2034
|
-
lhs: { terms: [{
|
|
2035
|
-
knob: "unit_processing_time",
|
|
2036
|
-
coef: 1,
|
|
2037
|
-
at: "consumer"
|
|
2038
|
-
}] },
|
|
2039
|
-
op: "<",
|
|
2040
|
-
rhs: { terms: [{
|
|
2041
|
-
knob: "drain_phase_cap",
|
|
2042
|
-
coef: 1,
|
|
2043
|
-
at: "process"
|
|
2044
|
-
}] },
|
|
2045
|
-
margin: 0
|
|
2046
|
-
}
|
|
2047
|
-
}, {
|
|
2048
|
-
id: CLOSE_BOUND_RULE,
|
|
2049
|
-
rule: "the close bound of a consumer or a worker is less than the cap of the close transports phase",
|
|
2050
|
-
window: "the phase can end before the loop closes",
|
|
2051
|
-
expr: {
|
|
2052
|
-
lhs: { terms: [{
|
|
2053
|
-
knob: "close_bound",
|
|
2054
|
-
coef: 1,
|
|
2055
|
-
at: "consumer"
|
|
2056
|
-
}] },
|
|
2057
|
-
op: "<",
|
|
2058
|
-
rhs: { terms: [{
|
|
2059
|
-
knob: "close_phase_cap",
|
|
2060
|
-
coef: 1,
|
|
2061
|
-
at: "process"
|
|
2062
|
-
}] },
|
|
2063
|
-
margin: 0
|
|
2064
|
-
}
|
|
2065
|
-
}];
|
|
2066
|
-
/**
|
|
2067
|
-
* This function gives the scopes of a rule that reads these symbols: the service, each client, each
|
|
2068
|
-
* consumer and worker, or each limiter. A rule that reads a symbol at `server` applies to each client
|
|
2069
|
-
* and to each server transport. In the scope of a server, the scenario gives the client side.
|
|
2070
|
-
*/
|
|
2071
|
-
const scopesOf = (svc, scenario, service, symbols) => {
|
|
2072
|
-
const within = (name, scoped) => {
|
|
2073
|
-
const bind = {
|
|
2074
|
-
...service.bind,
|
|
2075
|
-
...scoped.bind
|
|
2076
|
-
};
|
|
2077
|
-
for (const symbol of Object.keys(scoped.epochs)) if (!(symbol in scoped.bind)) delete bind[symbol];
|
|
2078
|
-
return [
|
|
2079
|
-
name,
|
|
2080
|
-
bind,
|
|
2081
|
-
{
|
|
2082
|
-
...service.epochs,
|
|
2083
|
-
...scoped.epochs
|
|
2084
|
-
}
|
|
2085
|
-
];
|
|
2086
|
-
};
|
|
2087
|
-
const transports = (hop) => svc.transports.filter((transport) => TRANSPORT_HOPS[transport.kind] === hop).map((transport) => within(transport.name, transportBindings(transport, hop, scenario)));
|
|
2088
|
-
return symbols.some((symbol) => LIMITER_SCOPED.test(symbol)) ? (svc.limiters ?? []).map((limiter) => within(limiter.name, limiterBindings(limiter, scenario))) : symbols.some((symbol) => TRANSPORT_SCOPED.test(symbol)) ? transports("consumer") : symbols.some((symbol) => SCOPED.test(symbol)) ? [...svc.dependencies.map((dependency) => within(dependency.name, dependencyBindings(dependency, scenario))), ...symbols.some((symbol) => SERVER_SCOPED.test(symbol)) ? transports("server") : []] : [[
|
|
2089
|
-
svc.name,
|
|
2090
|
-
service.bind,
|
|
2091
|
-
service.epochs
|
|
2092
|
-
]];
|
|
2093
|
-
};
|
|
2094
|
-
/** This function gives the epochs of the symbols of a rule that the scenario binds. */
|
|
2095
|
-
const epochsOf = (symbols, epochs) => Object.fromEntries(symbols.flatMap((symbol) => symbol in epochs ? [[symbol, epochs[symbol] ?? null]] : []));
|
|
2096
|
-
/**
|
|
2097
|
-
* These are the marks of the timers of the platform (E10). Kubernetes takes the probe period, the
|
|
2098
|
-
* preStop sleep and the termination grace in whole seconds, thus each is stepped by 1 s. The
|
|
2099
|
-
* cluster sets the propagation of the endpoints, thus a compression of the service cannot scale it.
|
|
2100
|
-
*/
|
|
2101
|
-
const PLATFORM_MARKS = {
|
|
2102
|
-
"probe_period@pod": {
|
|
2103
|
-
kind: "stepped",
|
|
2104
|
-
step: 1
|
|
2105
|
-
},
|
|
2106
|
-
"prestop_sleep@pod": {
|
|
2107
|
-
kind: "stepped",
|
|
2108
|
-
step: 1
|
|
2109
|
-
},
|
|
2110
|
-
"termination_grace@pod": {
|
|
2111
|
-
kind: "stepped",
|
|
2112
|
-
step: 1
|
|
2113
|
-
},
|
|
2114
|
-
"propagation_bound@cluster": { kind: "fixed" }
|
|
2115
|
-
};
|
|
2116
|
-
/**
|
|
2117
|
-
* This function tells which findings of `lint` a compression by k changes (E10). It scales the
|
|
2118
|
-
* bindings of each scope by the marks. Thus a harness that runs the service k times faster learns
|
|
2119
|
-
* the orderings that its run breaks. The unmarked symbols are the symbols of the rules with no mark.
|
|
2120
|
-
* It reads the rules that `lint` applies with their bindings: the service rules and `BOUND_RULES`.
|
|
2121
|
-
*/
|
|
2122
|
-
const compressionOf = (svc, scenario, rules, k, marks) => {
|
|
2123
|
-
const service = serviceBindings(svc, scenario);
|
|
2124
|
-
const changed = [];
|
|
2125
|
-
const unmarked = /* @__PURE__ */ new Set();
|
|
2126
|
-
for (const rule of [...rules.filter((each) => SERVICE_RULES.includes(each.id)), ...BOUND_RULES]) for (const [scope, bind] of scopesOf(svc, scenario, service, symbolsOf(rule))) {
|
|
2127
|
-
const result = compression([rule], bind, k, marks);
|
|
2128
|
-
for (const change of result.changed) changed.push({
|
|
2129
|
-
...change,
|
|
2130
|
-
scope
|
|
2131
|
-
});
|
|
2132
|
-
for (const symbol of result.unmarked) unmarked.add(symbol);
|
|
2133
|
-
}
|
|
2134
|
-
return {
|
|
2135
|
-
changed,
|
|
2136
|
-
unmarked: [...unmarked]
|
|
2137
|
-
};
|
|
2138
|
-
};
|
|
2139
|
-
/** This function gives the findings of one rule, one for each scope of the rule. */
|
|
2140
|
-
const findingsOf = (svc, scenario, service, rule) => {
|
|
2141
|
-
const symbols = symbolsOf(rule);
|
|
2142
|
-
return scopesOf(svc, scenario, service, symbols).map(([scope, bind, all]) => {
|
|
2143
|
-
const evaluated = evaluate(rule, bind);
|
|
2144
|
-
const epochs = epochsOf(symbols, all);
|
|
2145
|
-
return {
|
|
2146
|
-
rule: rule.id,
|
|
2147
|
-
scope,
|
|
2148
|
-
verdict: verdictOf(rule, bind, evaluated),
|
|
2149
|
-
statement: rule.rule,
|
|
2150
|
-
...rule.window !== void 0 ? { window: rule.window } : {},
|
|
2151
|
-
evaluated,
|
|
2152
|
-
unbound: symbols.filter((symbol) => bind[symbol] === void 0),
|
|
2153
|
-
...Object.values(epochs).some((epoch) => epoch !== null) ? { epochs } : {}
|
|
2154
|
-
};
|
|
2155
|
-
});
|
|
2156
|
-
};
|
|
2157
|
-
/**
|
|
2158
|
-
* This function applies each service rule to a declaration and a scenario. A rule applies one time
|
|
2159
|
-
* for the service, for each client, for each consumer and worker, or for each limiter. It also
|
|
2160
|
-
* gives a finding for each necessary field that the description does not give (`DECLARATION_RULE`).
|
|
2161
|
-
* It gives a finding for each name that two parts share in the scopes of a rule (`NAME_RULE`). Each progress check
|
|
2162
|
-
* that names its loops also gets its findings (`PROGRESS_RULE`). The findings of the bounds of each
|
|
2163
|
-
* consumer and each worker come last (`BOUND_RULES`).
|
|
2164
|
-
*/
|
|
2165
|
-
const lint = (svc, scenario, rules) => {
|
|
2166
|
-
const service = serviceBindings(svc, scenario);
|
|
2167
|
-
return [
|
|
2168
|
-
...declarationFindings(svc),
|
|
2169
|
-
...nameFindings(svc),
|
|
2170
|
-
...progressFindings(svc),
|
|
2171
|
-
...rules.filter((rule) => SERVICE_RULES.includes(rule.id)).flatMap((rule) => findingsOf(svc, scenario, service, rule)),
|
|
2172
|
-
...BOUND_RULES.flatMap((rule) => findingsOf(svc, scenario, service, rule))
|
|
2173
|
-
];
|
|
2174
|
-
};
|
|
2175
|
-
// src/stops.ts
|
|
2176
|
-
/**
|
|
2177
|
-
* These are the identifiers of the rules of the drain, from the data of the entry `./rules`. Each
|
|
2178
|
-
* way that a pod stops applies only its own rules of this set.
|
|
2179
|
-
*/
|
|
2180
|
-
const DRAIN_RULE_IDS = DRAIN_RULES.map((rule) => rule.id);
|
|
2181
|
-
/**
|
|
2182
|
-
* These are the four ways that a pod stops.
|
|
2183
|
-
* - A deletion runs the preStop hook, then SIGTERM. The endpoints drop the pod at its deletion,
|
|
2184
|
-
* thus the preStop sleep and the hold must cover the propagation (O04).
|
|
2185
|
-
* - A liveness restart also runs the preStop hook and SIGTERM, with the grace. The pod stays in its
|
|
2186
|
-
* endpoints until its readiness fails, thus readiness must fail first (O09).
|
|
2187
|
-
* - A crash and an OOM stop run no hook and no drain. The kubelet finds the exit at its relist, and
|
|
2188
|
-
* no rule bounds that time. A client that can reissue a request is the only guard.
|
|
2189
|
-
*/
|
|
2190
|
-
const STOPS = {
|
|
2191
|
-
deletion: {
|
|
2192
|
-
way: "deletion",
|
|
2193
|
-
signal: "SIGTERM",
|
|
2194
|
-
preStop: true,
|
|
2195
|
-
drain: true,
|
|
2196
|
-
leaves: "deletion",
|
|
2197
|
-
rules: [
|
|
2198
|
-
"O04",
|
|
2199
|
-
"O10",
|
|
2200
|
-
"O16"
|
|
2201
|
-
]
|
|
2202
|
-
},
|
|
2203
|
-
"liveness-restart": {
|
|
2204
|
-
way: "liveness-restart",
|
|
2205
|
-
signal: "SIGTERM",
|
|
2206
|
-
preStop: true,
|
|
2207
|
-
drain: true,
|
|
2208
|
-
leaves: "readiness",
|
|
2209
|
-
rules: [
|
|
2210
|
-
"O09",
|
|
2211
|
-
"O10",
|
|
2212
|
-
"O16"
|
|
2213
|
-
]
|
|
2214
|
-
},
|
|
2215
|
-
crash: {
|
|
2216
|
-
way: "crash",
|
|
2217
|
-
signal: null,
|
|
2218
|
-
preStop: false,
|
|
2219
|
-
drain: false,
|
|
2220
|
-
leaves: "exit",
|
|
2221
|
-
rules: []
|
|
2222
|
-
},
|
|
2223
|
-
oom: {
|
|
2224
|
-
way: "oom",
|
|
2225
|
-
signal: "SIGKILL",
|
|
2226
|
-
preStop: false,
|
|
2227
|
-
drain: false,
|
|
2228
|
-
leaves: "exit",
|
|
2229
|
-
rules: []
|
|
2230
|
-
}
|
|
2231
|
-
};
|
|
2232
|
-
/** The identifier of the finding of a way of stopping that runs no drain. */
|
|
2233
|
-
const STOP_RULE = "stop";
|
|
2234
|
-
/**
|
|
2235
|
-
* This function applies the rules to a declaration for one way of stopping. A rule of the drain
|
|
2236
|
-
* applies only when it guards that way. The other rules apply as `lint()` applies them. A way with
|
|
2237
|
-
* no drain also gets a finding of `STOP_RULE` that depends. No rule bounds the time until the
|
|
2238
|
-
* kubelet finds the exit, and the work in flight is lost. Such a way starts no phase, thus it gets no
|
|
2239
|
-
* finding of `BOUND_RULES`.
|
|
2240
|
-
*/
|
|
2241
|
-
const lintStop = (svc, scenario, rules, way) => {
|
|
2242
|
-
const stop = STOPS[way];
|
|
2243
|
-
const applies = rules.filter((rule) => !DRAIN_RULE_IDS.includes(rule.id) || stop.rules.includes(rule.id));
|
|
2244
|
-
const findings = lint(svc, scenario, applies);
|
|
2245
|
-
if (stop.drain) return findings;
|
|
2246
|
-
return [...findings.filter((finding) => !BOUND_RULES.some((rule) => rule.id === finding.rule)), {
|
|
2247
|
-
rule: STOP_RULE,
|
|
2248
|
-
scope: svc.name,
|
|
2249
|
-
verdict: "depends",
|
|
2250
|
-
statement: `a ${way} runs no preStop hook and no drain, and no rule bounds the time until the kubelet finds the exit`,
|
|
2251
|
-
window: "the work in flight is lost, and the endpoints keep the pod until the kubelet finds the exit",
|
|
2252
|
-
evaluated: [],
|
|
2253
|
-
unbound: ["exit_detection@pod"]
|
|
2254
|
-
}];
|
|
2255
|
-
};
|
|
2256
|
-
// src/identity.ts
|
|
2257
|
-
/**
|
|
2258
|
-
* The exported type names that hold or reach a value that service knows by identity. A package
|
|
2259
|
-
* whose public API takes or gives a value of one of these types has service as a peer dependency.
|
|
2260
|
-
* A plain dependency is not sufficient. Then the package and its caller use one copy of service.
|
|
2261
|
-
*
|
|
2262
|
-
* The roots are `Lifecycle` and `RequestStream`, the keys of the maps, and `TransportImpl` and
|
|
2263
|
-
* `RequestHandler`, whose values write to the maps of their own copy. The list is each exported type
|
|
2264
|
-
* that names a root, or names a type of the list, in its declaration. A type can be on the list
|
|
2265
|
-
* although its own use crosses copies safely: for that type, a peer is the safe choice.
|
|
2266
|
-
*
|
|
2267
|
-
* `identity.test.ts` calculates this list from the sources and from the maps, thus the list stays
|
|
2268
|
-
* current.
|
|
2269
|
-
*/
|
|
2270
|
-
const IDENTITY_TYPES = [
|
|
2271
|
-
"Client",
|
|
2272
|
-
"GrpcClient",
|
|
2273
|
-
"HttpClient",
|
|
2274
|
-
"HttpTransport",
|
|
2275
|
-
"IterationContext",
|
|
2276
|
-
"JobContext",
|
|
2277
|
-
"Lifecycle",
|
|
2278
|
-
"MongoClient",
|
|
2279
|
-
"MountContext",
|
|
2280
|
-
"PgClient",
|
|
2281
|
-
"QueueDrain",
|
|
2282
|
-
"QueueDrainOptions",
|
|
2283
|
-
"RabbitClient",
|
|
2284
|
-
"RedisClient",
|
|
2285
|
-
"RedisSender",
|
|
2286
|
-
"RequestHandler",
|
|
2287
|
-
"RequestStream",
|
|
2288
|
-
"ServiceBuilder",
|
|
2289
|
-
"SubjectFactory",
|
|
2290
|
-
"TransportImpl",
|
|
2291
|
-
"TransportSubject",
|
|
2292
|
-
"UpgradeHandler",
|
|
2293
|
-
"WorkerLoop",
|
|
2294
|
-
"WorkerLoopOptions",
|
|
2295
|
-
"WsClient"
|
|
2296
|
-
];
|
|
2297
|
-
// src/profiles.ts
|
|
2298
|
-
const s = (value) => ({
|
|
2299
|
-
value,
|
|
2300
|
-
unit: "s"
|
|
2301
|
-
});
|
|
2302
|
-
const phase = (name, cap) => ({
|
|
2303
|
-
name,
|
|
2304
|
-
cap
|
|
2305
|
-
});
|
|
2306
|
-
/**
|
|
2307
|
-
* This profile is for a service on the computer of a developer. No load balancer routes to it,
|
|
2308
|
-
* thus it has no readiness hold. Its caps sum to 10 s, and a second signal exits at once. It
|
|
2309
|
-
* declares no checks, thus the caller declares readiness, liveness and the startup budget.
|
|
2310
|
-
*/
|
|
2311
|
-
const localDev = {
|
|
2312
|
-
phases: [
|
|
2313
|
-
phase("stop intake", s(1)),
|
|
2314
|
-
phase("drain", s(4)),
|
|
2315
|
-
phase("close transports", s(1)),
|
|
2316
|
-
phase("stop the poller", s(1)),
|
|
2317
|
-
phase("tear down dependencies", s(1)),
|
|
2318
|
-
phase("close the health server", s(1)),
|
|
2319
|
-
phase("flush the output", s(1))
|
|
2320
|
-
],
|
|
2321
|
-
cooperativeWindow: s(2),
|
|
2322
|
-
cooperativeFloor: "half",
|
|
2323
|
-
onRepeatSignal: "exit"
|
|
2324
|
-
};
|
|
2325
|
-
/**
|
|
2326
|
-
* This profile is for a background host on the computer of a developer, under a supervisor or a
|
|
2327
|
-
* command that stops it. It has the caps of `localDev`. A repeat signal has no effect, thus the
|
|
2328
|
-
* host ends its drain. It has no process trigger: a background host has no parent on stdin.
|
|
2329
|
-
*/
|
|
2330
|
-
const daemon = {
|
|
2331
|
-
phases: localDev.phases.map((each) => ({ ...each })),
|
|
2332
|
-
cooperativeWindow: localDev.cooperativeWindow,
|
|
2333
|
-
cooperativeFloor: "half",
|
|
2334
|
-
onRepeatSignal: "ignore"
|
|
2335
|
-
};
|
|
2336
|
-
/**
|
|
2337
|
-
* This profile is for a small process: a command that runs one time, or a child process on stdio.
|
|
2338
|
-
* It has no checks and no probes, and its policies gate nothing. The cooperative abort occurs at the
|
|
2339
|
-
* start of the drain, thus a first Ctrl-C stops the work at once. A second signal exits at once.
|
|
2340
|
-
* The exit codes tell the shell how the process stopped: 130 for SIGINT, 143 for SIGTERM and 131
|
|
2341
|
-
* for SIGQUIT. A stdio child also declares `triggers: ["stdin-close"]`.
|
|
2342
|
-
*/
|
|
2343
|
-
const ephemeral = {
|
|
2344
|
-
readiness: {
|
|
2345
|
-
type: "window",
|
|
2346
|
-
failAfter: s(0)
|
|
2347
|
-
},
|
|
2348
|
-
liveness: {
|
|
2349
|
-
type: "window",
|
|
2350
|
-
failAfter: s(0)
|
|
2351
|
-
},
|
|
2352
|
-
startup: s(10),
|
|
2353
|
-
phases: [
|
|
2354
|
-
phase("stop intake", s(1)),
|
|
2355
|
-
phase("drain", s(2)),
|
|
2356
|
-
phase("close transports", s(1)),
|
|
2357
|
-
phase("stop the poller", s(1)),
|
|
2358
|
-
phase("tear down dependencies", s(2)),
|
|
2359
|
-
phase("flush the output", s(1))
|
|
2360
|
-
],
|
|
2361
|
-
cooperativeWindow: s(2),
|
|
2362
|
-
cooperativeFloor: "none",
|
|
2363
|
-
onRepeatSignal: "exit",
|
|
2364
|
-
exitCodes: {
|
|
2365
|
-
SIGINT: 130,
|
|
2366
|
-
SIGTERM: 143,
|
|
2367
|
-
SIGQUIT: 131
|
|
2368
|
-
}
|
|
2369
|
-
};
|
|
2370
|
-
/** These are the parts of the budget of each stage in the profile `kubernetes`, in thirtieths. */
|
|
2371
|
-
const KUBERNETES_SHARES = [
|
|
2372
|
-
["fail readiness", 2],
|
|
2373
|
-
["stop intake", 5],
|
|
2374
|
-
["drain", 15],
|
|
2375
|
-
["close transports", 2],
|
|
2376
|
-
["stop the poller", 1],
|
|
2377
|
-
["tear down dependencies", 2],
|
|
2378
|
-
["close the health server", 1]
|
|
2379
|
-
];
|
|
2380
|
-
/** This part of the budget is the cooperative window, in thirtieths. */
|
|
2381
|
-
const KUBERNETES_WINDOW = 5;
|
|
2382
|
-
/**
|
|
2383
|
-
* This function gives the profile of a pod in Kubernetes. The budget is the grace less the preStop
|
|
2384
|
-
* sleep: the time from SIGTERM to SIGKILL. Each stage gets a part of the budget, rounded down to
|
|
2385
|
-
* the millisecond.
|
|
2386
|
-
* - The parts in thirtieths are 2, 5, 15, 2, 1, 2 and 1, in the sequence of the stages. Thus 2
|
|
2387
|
-
* thirtieths of the budget stay as a margin before SIGKILL.
|
|
2388
|
-
* - The cooperative window is 5 thirtieths of the budget.
|
|
2389
|
-
* - The kubelet sends SIGTERM one time, thus a repeat signal has no effect.
|
|
2390
|
-
*
|
|
2391
|
-
* At the defaults of Kubernetes (a grace of 30 s and no preStop hook), the caps are 2, 5, 15, 2, 1,
|
|
2392
|
-
* 2 and 1 s. Thus the sum of the caps and the preStop sleep fits in the grace (O10). The window is
|
|
2393
|
-
* shorter than the drain, which is shorter than the budget (O16).
|
|
2394
|
-
*/
|
|
2395
|
-
const kubernetes = (pod) => {
|
|
2396
|
-
const time = (amount, what) => {
|
|
2397
|
-
if (!convertible(amount.unit, "s")) throw new Error(`the ${what} of the pod is not a time`);
|
|
2398
|
-
return valueIn(amount, "ms");
|
|
2399
|
-
};
|
|
2400
|
-
const grace = time(pod.grace, "grace");
|
|
2401
|
-
const preStop = time(pod.preStop, "preStop sleep");
|
|
2402
|
-
if (preStop < 0) throw new Error("the preStop sleep of the pod is less than 0");
|
|
2403
|
-
const budget = grace - preStop;
|
|
2404
|
-
if (!(budget > 0)) throw new Error(`the grace of the pod (${grace} ms) is not longer than its preStop sleep (${preStop} ms)`);
|
|
2405
|
-
const part = (thirtieths) => s(Math.floor(budget * thirtieths / 30) / 1e3);
|
|
2406
|
-
return {
|
|
2407
|
-
phases: KUBERNETES_SHARES.map(([name, thirtieths]) => phase(name, part(thirtieths))),
|
|
2408
|
-
cooperativeWindow: part(KUBERNETES_WINDOW),
|
|
2409
|
-
cooperativeFloor: "half",
|
|
2410
|
-
onRepeatSignal: "ignore"
|
|
2411
|
-
};
|
|
2412
|
-
};
|
|
2413
|
-
// src/worker.ts
|
|
2414
|
-
const OWNER$1 = "workerLoop";
|
|
2415
|
-
/**
|
|
2416
|
-
* This function makes a worker loop as a transport (S6 A10). It has a setup, an iteration and a
|
|
2417
|
-
* teardown, and each lane does its iterations one at a time.
|
|
2418
|
-
* - The start does the setup. A failed setup gets its teardown, and the start throws.
|
|
2419
|
-
* - The first iteration of each lane comes after the startup ends. Each sleep and each backoff is on the clock.
|
|
2420
|
-
* - `stopIntake` starts no new iteration, and it ends each sleep and each backoff.
|
|
2421
|
-
* - `drain` waits for each iteration in flight until the deadline, and gives it the cooperative
|
|
2422
|
-
* abort. A throw after that abort is not a failure.
|
|
2423
|
-
* - `close` aborts each iteration, waits for its end, and does the teardown, all in `close_bound`.
|
|
2424
|
-
*/
|
|
2425
|
-
const workerLoop = (knobs, options) => {
|
|
2426
|
-
const sleep = millisecondsOf(knobs, "worker_sleep", OWNER$1);
|
|
2427
|
-
const bound = millisecondsOf(knobs, "unit_processing_time", OWNER$1);
|
|
2428
|
-
const backoff = retryOf(knobs, OWNER$1);
|
|
2429
|
-
const closeBound = boundedMillisecondsOf(knobs, "close_bound", OWNER$1);
|
|
2430
|
-
const lanes = countOf(knobs, "concurrency", OWNER$1);
|
|
2431
|
-
if (!Number.isFinite(lanes)) throw new Error(`${OWNER$1} needs a concurrency with a bound: each lane is a loop`);
|
|
2432
|
-
const clock = options.clock ?? realClock;
|
|
2433
|
-
const random = options.random ?? Math.random;
|
|
2434
|
-
const running = flights(clock);
|
|
2435
|
-
const rests = sleeper(clock);
|
|
2436
|
-
const intake = new AbortController();
|
|
2437
|
-
const pending = /* @__PURE__ */ new Set();
|
|
2438
|
-
let started = false;
|
|
2439
|
-
let setup = null;
|
|
2440
|
-
let state;
|
|
2441
|
-
let tornDown = null;
|
|
2442
|
-
let closing = null;
|
|
2443
|
-
const report = (error, failures) => {
|
|
2444
|
-
try {
|
|
2445
|
-
options.onError?.(error, failures);
|
|
2446
|
-
} catch {}
|
|
2447
|
-
};
|
|
2448
|
-
/** This function does the teardown one time, after the setup ends, and only when the setup started. */
|
|
2449
|
-
const teardown = () => {
|
|
2450
|
-
if (tornDown !== null) return tornDown;
|
|
2451
|
-
const release = options.teardown;
|
|
2452
|
-
const made = setup;
|
|
2453
|
-
tornDown = made === null || release === void 0 ? Promise.resolve() : made.then((given) => release(given), () => release(void 0));
|
|
2454
|
-
return tornDown;
|
|
2455
|
-
};
|
|
2456
|
-
const keep = (work) => {
|
|
2457
|
-
pending.add(work);
|
|
2458
|
-
const forget = () => {
|
|
2459
|
-
pending.delete(work);
|
|
2460
|
-
};
|
|
2461
|
-
work.then(forget, forget);
|
|
2462
|
-
return work;
|
|
2463
|
-
};
|
|
2464
|
-
/** This function is one lane: an iteration, then its sleep or its backoff, until the intake stops. */
|
|
2465
|
-
const lane = async (index, life) => {
|
|
2466
|
-
let failures = 0;
|
|
2467
|
-
let since = rests.wakes;
|
|
2468
|
-
await rests.sleep(0, since);
|
|
2469
|
-
for (;;) {
|
|
2470
|
-
if (intake.signal.aborted) return;
|
|
2471
|
-
since = rests.wakes;
|
|
2472
|
-
const abort = new AbortController();
|
|
2473
|
-
const remove = running.add(abort);
|
|
2474
|
-
const count = failures;
|
|
2475
|
-
const end = await runUnit({
|
|
2476
|
-
clock,
|
|
2477
|
-
bound,
|
|
2478
|
-
abort,
|
|
2479
|
-
life,
|
|
2480
|
-
request: options.request,
|
|
2481
|
-
unit: index,
|
|
2482
|
-
work: (signal) => keep((async () => options.iteration({
|
|
2483
|
-
signal,
|
|
2484
|
-
intake: intake.signal,
|
|
2485
|
-
state,
|
|
2486
|
-
lane: index,
|
|
2487
|
-
failures: count,
|
|
2488
|
-
life
|
|
2489
|
-
}))())
|
|
2490
|
-
});
|
|
2491
|
-
remove();
|
|
2492
|
-
if (end.kind === "cut") return;
|
|
2493
|
-
if (end.kind === "done") {
|
|
2494
|
-
failures = 0;
|
|
2495
|
-
await rests.sleep(end.value === "more" ? 0 : sleep, end.value === "more" ? rests.wakes : since);
|
|
2496
|
-
continue;
|
|
2497
|
-
}
|
|
2498
|
-
failures += 1;
|
|
2499
|
-
report(end.kind === "failed" ? end.error : /* @__PURE__ */ new Error(`the service refused the iteration: ${end.cause}`), failures);
|
|
2500
|
-
await rests.backoff(backoffDelay(backoff, failures, random));
|
|
2501
|
-
}
|
|
2502
|
-
};
|
|
2503
|
-
return {
|
|
2504
|
-
knobs: { ...knobs },
|
|
2505
|
-
async start(signal, life) {
|
|
2506
|
-
if (closing !== null) throw new Error(`${OWNER$1}: a closed loop does not start again`);
|
|
2507
|
-
if (started) throw new Error(`${OWNER$1}: the loop started already`);
|
|
2508
|
-
started = true;
|
|
2509
|
-
const make = options.setup;
|
|
2510
|
-
setup = make === void 0 ? Promise.resolve(void 0) : new Promise((resolve) => resolve(make(signal)));
|
|
2511
|
-
try {
|
|
2512
|
-
await setup.then((given) => {
|
|
2513
|
-
state = given;
|
|
2514
|
-
});
|
|
2515
|
-
} catch (error) {
|
|
2516
|
-
await settleWithin(clock, closeBound, teardown());
|
|
2517
|
-
throw error;
|
|
2518
|
-
}
|
|
2519
|
-
life.started.then(() => {
|
|
2520
|
-
if (intake.signal.aborted || life.state !== "running") return;
|
|
2521
|
-
for (let index = 0; index < lanes; index += 1) lane(index, life).catch((error) => report(error, 0));
|
|
2522
|
-
});
|
|
2523
|
-
},
|
|
2524
|
-
stopIntake() {
|
|
2525
|
-
if (intake.signal.aborted) return;
|
|
2526
|
-
intake.abort(/* @__PURE__ */ new Error(`${OWNER$1} stopped its intake`));
|
|
2527
|
-
rests.stop();
|
|
2528
|
-
},
|
|
2529
|
-
drain(deadline, signal) {
|
|
2530
|
-
return running.drain(deadline, signal);
|
|
2531
|
-
},
|
|
2532
|
-
inFlight: () => running.size,
|
|
2533
|
-
wake() {
|
|
2534
|
-
if (intake.signal.aborted) return false;
|
|
2535
|
-
rests.wake();
|
|
2536
|
-
return true;
|
|
2537
|
-
},
|
|
2538
|
-
close() {
|
|
2539
|
-
if (closing !== null) return closing;
|
|
2540
|
-
if (!intake.signal.aborted) intake.abort(/* @__PURE__ */ new Error(`${OWNER$1} closed`));
|
|
2541
|
-
rests.stop();
|
|
2542
|
-
running.abort(/* @__PURE__ */ new Error(`${OWNER$1} closed`));
|
|
2543
|
-
running.clear();
|
|
2544
|
-
closing = (async () => {
|
|
2545
|
-
const began = clock.now();
|
|
2546
|
-
const ended = await settleWithin(clock, closeBound, Promise.allSettled([...pending]));
|
|
2547
|
-
const down = await settleWithin(clock, Math.max(0, closeBound - (clock.now() - began)), teardown());
|
|
2548
|
-
if (ended.kind === "bound") throw new Error(`${OWNER$1}: an iteration did not end in the close_bound of ${closeBound} ms after its abort`);
|
|
2549
|
-
if (down.kind === "failed") throw down.error;
|
|
2550
|
-
if (down.kind === "bound") throw new Error(`${OWNER$1}: the teardown did not end in the close_bound of ${closeBound} ms`);
|
|
2551
|
-
})();
|
|
2552
|
-
return closing;
|
|
2553
|
-
},
|
|
2554
|
-
problems(description) {
|
|
2555
|
-
return boundProblems(OWNER$1, description);
|
|
2556
|
-
}
|
|
2557
|
-
};
|
|
2558
|
-
};
|
|
2559
|
-
// src/queue.ts
|
|
2560
|
-
/**
|
|
2561
|
-
* This function gives the redelivery policy of S6 A11. A poison message gets `ack`. Each other job
|
|
2562
|
-
* goes back on the queue after its first failure, one time. After a later failure, `then` gives the
|
|
2563
|
-
* action: the declared policy of the queue.
|
|
2564
|
-
*/
|
|
2565
|
-
const redeliverOnce = (then, poison) => (failure) => {
|
|
2566
|
-
if (poison?.(failure.error, failure.job) === true) return "ack";
|
|
2567
|
-
if (failure.delivery <= 1) return "requeue";
|
|
2568
|
-
return typeof then === "function" ? then(failure) : then;
|
|
2569
|
-
};
|
|
2570
|
-
const OWNER = "queueDrain";
|
|
2571
|
-
const ACTIONS = [
|
|
2572
|
-
"requeue",
|
|
2573
|
-
"reject",
|
|
2574
|
-
"ack"
|
|
2575
|
-
];
|
|
2576
|
-
const STOPPED = Symbol("the intake stopped");
|
|
2577
|
-
const isAsyncIterable = (value) => value !== null && (typeof value === "object" || typeof value === "function") && typeof value[Symbol.asyncIterator] === "function";
|
|
2578
|
-
/**
|
|
2579
|
-
* This function makes a queue drain as a transport (S6 A11). One taker takes the jobs from the
|
|
2580
|
-
* source while a slot is free, and each job operates in its slot.
|
|
2581
|
-
* - A success gets `ack`. A failure gets the action of the redelivery policy.
|
|
2582
|
-
* - `stopIntake` takes no new job, and a job that the drain took and did not start goes back (`requeue`, `unstarted`).
|
|
2583
|
-
* - `drain` waits for each job in flight until the deadline, and gives it the cooperative abort.
|
|
2584
|
-
* - `close` aborts each job, waits for its end, and closes the source, all in `close_bound`. A job
|
|
2585
|
-
* that does not end in the bound gets `reject`.
|
|
2586
|
-
*/
|
|
2587
|
-
const queueDrain = (knobs, options) => {
|
|
2588
|
-
const poll = millisecondsOf(knobs, "queue_poll_interval", OWNER);
|
|
2589
|
-
const bound = millisecondsOf(knobs, "unit_processing_time", OWNER);
|
|
2590
|
-
const backoff = retryOf(knobs, OWNER);
|
|
2591
|
-
const closeBound = boundedMillisecondsOf(knobs, "close_bound", OWNER);
|
|
2592
|
-
const concurrency = countOf(knobs, "concurrency", OWNER);
|
|
2593
|
-
const { settle } = options;
|
|
2594
|
-
const clock = options.clock ?? realClock;
|
|
2595
|
-
const random = options.random ?? Math.random;
|
|
2596
|
-
const running = flights(clock);
|
|
2597
|
-
const rests = sleeper(clock);
|
|
2598
|
-
const intake = new AbortController();
|
|
2599
|
-
const pending = /* @__PURE__ */ new Set();
|
|
2600
|
-
const active = /* @__PURE__ */ new Set();
|
|
2601
|
-
let refusals = 0;
|
|
2602
|
-
let pausedUntil = Number.NEGATIVE_INFINITY;
|
|
2603
|
-
let cancelSource = () => {};
|
|
2604
|
-
let started = false;
|
|
2605
|
-
let closing = null;
|
|
2606
|
-
const report = (error, job) => {
|
|
2607
|
-
try {
|
|
2608
|
-
options.onError?.(error, job);
|
|
2609
|
-
} catch {}
|
|
2610
|
-
};
|
|
2611
|
-
const keep = (work) => {
|
|
2612
|
-
pending.add(work);
|
|
2613
|
-
const forget = () => {
|
|
2614
|
-
pending.delete(work);
|
|
2615
|
-
};
|
|
2616
|
-
work.then(forget, forget);
|
|
2617
|
-
};
|
|
2618
|
-
/** This function tells the queue the end of a job, one time. A throw of the queue is a failure. */
|
|
2619
|
-
const once = async (unit, tell) => {
|
|
2620
|
-
if (unit.settled) return;
|
|
2621
|
-
unit.settled = true;
|
|
2622
|
-
try {
|
|
2623
|
-
await tell();
|
|
2624
|
-
} catch (error) {
|
|
2625
|
-
report(error, unit.job);
|
|
2626
|
-
}
|
|
2627
|
-
};
|
|
2628
|
-
const handBack = (job) => keep((async () => {
|
|
2629
|
-
try {
|
|
2630
|
-
await settle.requeue(job, "unstarted");
|
|
2631
|
-
} catch (error) {
|
|
2632
|
-
report(error, job);
|
|
2633
|
-
}
|
|
2634
|
-
})());
|
|
2635
|
-
/** This function gives the action of the policy for a failed job. A policy that throws, or that gives no action, rejects the job. */
|
|
2636
|
-
const actionOf = (job, error, delivery) => {
|
|
2637
|
-
try {
|
|
2638
|
-
const action = options.redelivery({
|
|
2639
|
-
job,
|
|
2640
|
-
error,
|
|
2641
|
-
delivery
|
|
2642
|
-
});
|
|
2643
|
-
if (ACTIONS.includes(action)) return action;
|
|
2644
|
-
report(/* @__PURE__ */ new Error(`the redelivery policy gave ${String(action)}, not requeue, reject or ack`), job);
|
|
2645
|
-
} catch (thrown) {
|
|
2646
|
-
report(thrown, job);
|
|
2647
|
-
}
|
|
2648
|
-
return "reject";
|
|
2649
|
-
};
|
|
2650
|
-
const settleEnd = (unit, end, delivery) => {
|
|
2651
|
-
const { job } = unit;
|
|
2652
|
-
if (end.kind === "done") return once(unit, () => settle.ack(job));
|
|
2653
|
-
if (end.kind === "refused") {
|
|
2654
|
-
report(/* @__PURE__ */ new Error(`the service refused the job: ${end.cause}`), job);
|
|
2655
|
-
return once(unit, () => settle.requeue(job, "unstarted"));
|
|
2656
|
-
}
|
|
2657
|
-
if (end.kind === "cut") {
|
|
2658
|
-
if (end.retryable) return once(unit, () => settle.requeue(job, "cut"));
|
|
2659
|
-
const cut = new Error(end.cause);
|
|
2660
|
-
return once(unit, () => settle.reject(job, cut));
|
|
2661
|
-
}
|
|
2662
|
-
const { error } = end;
|
|
2663
|
-
report(error, job);
|
|
2664
|
-
const action = actionOf(job, error, delivery);
|
|
2665
|
-
if (action === "requeue") return once(unit, () => settle.requeue(job, "failed"));
|
|
2666
|
-
return once(unit, () => action === "ack" ? settle.ack(job) : settle.reject(job, error));
|
|
2667
|
-
};
|
|
2668
|
-
/** This function starts one job in a slot. The slot stays in flight until the queue has the end of the job. */
|
|
2669
|
-
const runJob = (job, life) => {
|
|
2670
|
-
const unit = {
|
|
2671
|
-
job,
|
|
2672
|
-
settled: false
|
|
2673
|
-
};
|
|
2674
|
-
active.add(unit);
|
|
2675
|
-
const abort = new AbortController();
|
|
2676
|
-
const remove = running.add(abort);
|
|
2677
|
-
let delivery;
|
|
2678
|
-
try {
|
|
2679
|
-
delivery = settle.delivery(job);
|
|
2680
|
-
} catch (error) {
|
|
2681
|
-
report(error, job);
|
|
2682
|
-
delivery = Number.POSITIVE_INFINITY;
|
|
2683
|
-
}
|
|
2684
|
-
keep((async () => {
|
|
2685
|
-
const end = await runUnit({
|
|
2686
|
-
clock,
|
|
2687
|
-
bound,
|
|
2688
|
-
abort,
|
|
2689
|
-
life,
|
|
2690
|
-
request: options.request,
|
|
2691
|
-
unit: job,
|
|
2692
|
-
work: (signal) => options.handle(job, {
|
|
2693
|
-
signal,
|
|
2694
|
-
intake: intake.signal,
|
|
2695
|
-
delivery,
|
|
2696
|
-
life
|
|
2697
|
-
})
|
|
2698
|
-
});
|
|
2699
|
-
if (end.kind === "refused") {
|
|
2700
|
-
refusals += 1;
|
|
2701
|
-
pausedUntil = clock.now() + backoffDelay(backoff, refusals, random);
|
|
2702
|
-
} else refusals = 0;
|
|
2703
|
-
await settleEnd(unit, end, delivery);
|
|
2704
|
-
})().finally(() => {
|
|
2705
|
-
active.delete(unit);
|
|
2706
|
-
remove();
|
|
2707
|
-
}));
|
|
2708
|
-
};
|
|
2709
|
-
/** This function gives the take of the source. */
|
|
2710
|
-
const takeOf = () => {
|
|
2711
|
-
const source = options.source;
|
|
2712
|
-
if (isAsyncIterable(source)) {
|
|
2713
|
-
const iterator = source[Symbol.asyncIterator]();
|
|
2714
|
-
cancelSource = () => {
|
|
2715
|
-
try {
|
|
2716
|
-
Promise.resolve(iterator.return?.()).catch(() => {});
|
|
2717
|
-
} catch {}
|
|
2718
|
-
};
|
|
2719
|
-
return async () => {
|
|
2720
|
-
const step = await iterator.next();
|
|
2721
|
-
return step.done === true ? { kind: "end" } : {
|
|
2722
|
-
kind: "job",
|
|
2723
|
-
job: step.value
|
|
2724
|
-
};
|
|
2725
|
-
};
|
|
2726
|
-
}
|
|
2727
|
-
const pull = source;
|
|
2728
|
-
return () => new Promise((resolve) => resolve(pull(intake.signal))).then((job) => job === null || job === void 0 ? { kind: "none" } : {
|
|
2729
|
-
kind: "job",
|
|
2730
|
-
job
|
|
2731
|
-
});
|
|
2732
|
-
};
|
|
2733
|
-
/** This function is the taker: it takes a job while a slot is free, until the intake stops or the source ends. */
|
|
2734
|
-
const taker = async (life) => {
|
|
2735
|
-
const take = takeOf();
|
|
2736
|
-
let failures = 0;
|
|
2737
|
-
for (;;) {
|
|
2738
|
-
if (intake.signal.aborted) return;
|
|
2739
|
-
if (running.size >= concurrency) {
|
|
2740
|
-
await running.less();
|
|
2741
|
-
continue;
|
|
2742
|
-
}
|
|
2743
|
-
const pause = pausedUntil - clock.now();
|
|
2744
|
-
if (pause > 0) {
|
|
2745
|
-
await rests.backoff(pause);
|
|
2746
|
-
continue;
|
|
2747
|
-
}
|
|
2748
|
-
const since = rests.wakes;
|
|
2749
|
-
const taking = take().catch((error) => ({
|
|
2750
|
-
kind: "error",
|
|
2751
|
-
error
|
|
2752
|
-
}));
|
|
2753
|
-
const first = await untilAbort(taking, intake.signal, STOPPED);
|
|
2754
|
-
if (first === STOPPED || intake.signal.aborted) {
|
|
2755
|
-
if (first === STOPPED) taking.then((late) => {
|
|
2756
|
-
if (late.kind === "job") handBack(late.job);
|
|
2757
|
-
});
|
|
2758
|
-
else if (first.kind === "job") handBack(first.job);
|
|
2759
|
-
return;
|
|
2760
|
-
}
|
|
2761
|
-
if (first.kind === "error") {
|
|
2762
|
-
failures += 1;
|
|
2763
|
-
report(first.error, void 0);
|
|
2764
|
-
await rests.backoff(backoffDelay(backoff, failures, random));
|
|
2765
|
-
continue;
|
|
2766
|
-
}
|
|
2767
|
-
failures = 0;
|
|
2768
|
-
if (first.kind === "end") {
|
|
2769
|
-
report(/* @__PURE__ */ new Error(`the source of ${OWNER} ended while its intake was open: it takes no more jobs`), void 0);
|
|
2770
|
-
return;
|
|
2771
|
-
}
|
|
2772
|
-
if (first.kind === "none") {
|
|
2773
|
-
await rests.sleep(poll, since);
|
|
2774
|
-
continue;
|
|
2775
|
-
}
|
|
2776
|
-
runJob(first.job, life);
|
|
2777
|
-
}
|
|
2778
|
-
};
|
|
2779
|
-
const stopTaking = (cause) => {
|
|
2780
|
-
if (intake.signal.aborted) return;
|
|
2781
|
-
intake.abort(new Error(cause));
|
|
2782
|
-
rests.stop();
|
|
2783
|
-
running.release();
|
|
2784
|
-
cancelSource();
|
|
2785
|
-
};
|
|
2786
|
-
return {
|
|
2787
|
-
knobs: { ...knobs },
|
|
2788
|
-
start(_signal, life) {
|
|
2789
|
-
if (closing !== null) throw new Error(`${OWNER}: a closed drain does not start again`);
|
|
2790
|
-
if (started) throw new Error(`${OWNER}: the drain started already`);
|
|
2791
|
-
started = true;
|
|
2792
|
-
life.started.then(() => {
|
|
2793
|
-
if (intake.signal.aborted || life.state !== "running") return;
|
|
2794
|
-
taker(life).catch((error) => report(error, void 0));
|
|
2795
|
-
});
|
|
2796
|
-
},
|
|
2797
|
-
stopIntake() {
|
|
2798
|
-
stopTaking(`${OWNER} stopped its intake`);
|
|
2799
|
-
},
|
|
2800
|
-
drain(deadline, signal) {
|
|
2801
|
-
return running.drain(deadline, signal);
|
|
2802
|
-
},
|
|
2803
|
-
inFlight: () => running.size,
|
|
2804
|
-
wake() {
|
|
2805
|
-
if (intake.signal.aborted) return false;
|
|
2806
|
-
rests.wake();
|
|
2807
|
-
return true;
|
|
2808
|
-
},
|
|
2809
|
-
close() {
|
|
2810
|
-
if (closing !== null) return closing;
|
|
2811
|
-
stopTaking(`${OWNER} closed`);
|
|
2812
|
-
running.abort(/* @__PURE__ */ new Error(`${OWNER} closed`));
|
|
2813
|
-
running.clear();
|
|
2814
|
-
closing = (async () => {
|
|
2815
|
-
const began = clock.now();
|
|
2816
|
-
const ended = await settleWithin(clock, closeBound, Promise.allSettled([...pending]));
|
|
2817
|
-
if (ended.kind === "bound") for (const unit of [...active]) once(unit, () => settle.reject(unit.job, /* @__PURE__ */ new Error(`${OWNER} closed with the job in flight`)));
|
|
2818
|
-
const shut = options.close;
|
|
2819
|
-
const down = await settleWithin(clock, Math.max(0, closeBound - (clock.now() - began)), shut === void 0 ? Promise.resolve() : (async () => shut())());
|
|
2820
|
-
if (ended.kind === "bound") throw new Error(`${OWNER}: a job did not end in the close_bound of ${closeBound} ms after its abort`);
|
|
2821
|
-
if (down.kind === "failed") throw down.error;
|
|
2822
|
-
if (down.kind === "bound") throw new Error(`${OWNER}: the close of the source did not end in the close_bound of ${closeBound} ms`);
|
|
2823
|
-
})();
|
|
2824
|
-
return closing;
|
|
2825
|
-
},
|
|
2826
|
-
problems(description) {
|
|
2827
|
-
return boundProblems(OWNER, description);
|
|
2828
|
-
}
|
|
2829
|
-
};
|
|
2830
|
-
};
|
|
2831
|
-
// src/machines.ts
|
|
2832
|
-
/**
|
|
2833
|
-
* The machines of the runtime as `mark1russell7.machine/1` documents. The documents are the
|
|
2834
|
-
* specification, and TypeScript implements them. A test records the transitions of the runtime
|
|
2835
|
-
* and replays them against each document (`replay` of `@mark1russell7/algebra`), thus the code and
|
|
2836
|
-
* the documents cannot disagree.
|
|
2837
|
-
*/
|
|
2838
|
-
const SOURCE = [{
|
|
2839
|
-
ref: "packages/service/src (the runtime implements this document)",
|
|
2840
|
-
rung: "implemented"
|
|
2841
|
-
}];
|
|
2842
|
-
const base = (id, title) => ({
|
|
2843
|
-
format: "mark1russell7.machine/1",
|
|
2844
|
-
id: `service.machine.${id}`,
|
|
2845
|
-
title,
|
|
2846
|
-
layer: "self",
|
|
2847
|
-
version: "0.2.0",
|
|
2848
|
-
sources: SOURCE
|
|
2849
|
-
});
|
|
2850
|
-
const t = (id, from, on, to, extra = {}) => ({
|
|
2851
|
-
id,
|
|
2852
|
-
from,
|
|
2853
|
-
on,
|
|
2854
|
-
to,
|
|
2855
|
-
...extra
|
|
2856
|
-
});
|
|
2857
|
-
/**
|
|
2858
|
-
* The startup gate opens one time. The lifecycle moves it from `closed` to `starting` while its
|
|
2859
|
-
* components start. Then one warm-up sweep of each check opens it. A component that does not start
|
|
2860
|
-
* fails it for good.
|
|
2861
|
-
*/
|
|
2862
|
-
const GATE_MACHINE = {
|
|
2863
|
-
...base("gate", "The startup gate"),
|
|
2864
|
-
states: {
|
|
2865
|
-
closed: { initial: true },
|
|
2866
|
-
starting: { note: "the health server, then the dependencies, then the transports" },
|
|
2867
|
-
sweeping: { note: "one bounded warm-up sweep of each check" },
|
|
2868
|
-
open: { final: true },
|
|
2869
|
-
failed: { final: true }
|
|
2870
|
-
},
|
|
2871
|
-
events: {
|
|
2872
|
-
"startup.begin": { kind: "local" },
|
|
2873
|
-
"component.started": { kind: "local" },
|
|
2874
|
-
"mark-ready": { kind: "local" },
|
|
2875
|
-
"sweep.done": { kind: "local" },
|
|
2876
|
-
"startup.fail": { kind: "local" },
|
|
2877
|
-
"startup.over-budget": { kind: "timer" }
|
|
2878
|
-
},
|
|
2879
|
-
knobs: { startup: {
|
|
2880
|
-
key: "policies.startup",
|
|
2881
|
-
scope: "service",
|
|
2882
|
-
unit: "s",
|
|
2883
|
-
default: null
|
|
2884
|
-
} },
|
|
2885
|
-
timers: { startup_budget: {
|
|
2886
|
-
duration: {
|
|
2887
|
-
knob: "startup",
|
|
2888
|
-
unit: "s"
|
|
2889
|
-
},
|
|
2890
|
-
arms: ["enter:closed"],
|
|
2891
|
-
cancels: ["sweep.done", "startup.fail"],
|
|
2892
|
-
fires: "startup.over-budget",
|
|
2893
|
-
note: "the gate says that it is over its budget, and the startup probe of the platform acts on it"
|
|
2894
|
-
} },
|
|
2895
|
-
transitions: [
|
|
2896
|
-
t("begin", "closed", "startup.begin", "starting"),
|
|
2897
|
-
t("component", "starting", "component.started", "starting"),
|
|
2898
|
-
t("mark-ready", ["closed", "starting"], "mark-ready", "sweeping"),
|
|
2899
|
-
t("open", "sweeping", "sweep.done", "open"),
|
|
2900
|
-
t("fail", [
|
|
2901
|
-
"closed",
|
|
2902
|
-
"starting",
|
|
2903
|
-
"sweeping"
|
|
2904
|
-
], "startup.fail", "failed"),
|
|
2905
|
-
t("over-budget-closed", "closed", "startup.over-budget", "closed", { timer: "startup_budget" }),
|
|
2906
|
-
t("over-budget-starting", "starting", "startup.over-budget", "starting", { timer: "startup_budget" }),
|
|
2907
|
-
t("over-budget-sweeping", "sweeping", "startup.over-budget", "sweeping", { timer: "startup_budget" })
|
|
2908
|
-
]
|
|
2909
|
-
};
|
|
2910
|
-
/** A debounce latch counts the polls of its check in sequence. A readiness latch starts tripped. */
|
|
2911
|
-
const LATCH_MACHINE = {
|
|
2912
|
-
...base("latch", "The latch of a policy, as a debounce"),
|
|
2913
|
-
states: {
|
|
2914
|
-
tripped: {
|
|
2915
|
-
initial: true,
|
|
2916
|
-
note: "a readiness latch starts here; a liveness latch starts released"
|
|
2917
|
-
},
|
|
2918
|
-
released: {}
|
|
2919
|
-
},
|
|
2920
|
-
events: {
|
|
2921
|
-
"poll.ok": { kind: "local" },
|
|
2922
|
-
"poll.fail": { kind: "local" }
|
|
2923
|
-
},
|
|
2924
|
-
knobs: {
|
|
2925
|
-
fail_after: {
|
|
2926
|
-
key: "policies.<gate>.failAfter",
|
|
2927
|
-
scope: "check",
|
|
2928
|
-
unit: "count",
|
|
2929
|
-
default: null
|
|
2930
|
-
},
|
|
2931
|
-
recover_after: {
|
|
2932
|
-
key: "policies.<gate>.recoverAfter",
|
|
2933
|
-
scope: "check",
|
|
2934
|
-
unit: "count",
|
|
2935
|
-
default: null
|
|
2936
|
-
}
|
|
2937
|
-
},
|
|
2938
|
-
timers: {},
|
|
2939
|
-
transitions: [
|
|
2940
|
-
t("pass", "released", "poll.ok", "released"),
|
|
2941
|
-
t("fail-count", "released", "poll.fail", "released", { guard: "failures in sequence < fail_after" }),
|
|
2942
|
-
t("trip", "released", "poll.fail", "tripped", { guard: "failures in sequence >= fail_after" }),
|
|
2943
|
-
t("fail", "tripped", "poll.fail", "tripped"),
|
|
2944
|
-
t("pass-count", "tripped", "poll.ok", "tripped", { guard: "passes in sequence < recover_after" }),
|
|
2945
|
-
t("release", "tripped", "poll.ok", "released", { guard: "passes in sequence >= recover_after" })
|
|
2946
|
-
]
|
|
2947
|
-
};
|
|
2948
|
-
/**
|
|
2949
|
-
* A window latch measures time on the monotonic clock from the first failure of a sequence, and
|
|
2950
|
-
* from the first pass after a trip.
|
|
2951
|
-
*/
|
|
2952
|
-
const WINDOW_MACHINE = {
|
|
2953
|
-
...base("window", "The latch of a policy, as a window"),
|
|
2954
|
-
states: {
|
|
2955
|
-
released: {
|
|
2956
|
-
initial: true,
|
|
2957
|
-
note: "a liveness latch starts here; a readiness latch starts tripped"
|
|
2958
|
-
},
|
|
2959
|
-
failing: { note: "the fail window runs" },
|
|
2960
|
-
tripped: {},
|
|
2961
|
-
recovering: { note: "the recover window runs" }
|
|
2962
|
-
},
|
|
2963
|
-
events: {
|
|
2964
|
-
"poll.ok": { kind: "local" },
|
|
2965
|
-
"poll.fail": { kind: "local" },
|
|
2966
|
-
"window.failed": { kind: "timer" },
|
|
2967
|
-
"window.recovered": { kind: "timer" }
|
|
2968
|
-
},
|
|
2969
|
-
knobs: {
|
|
2970
|
-
fail_after: {
|
|
2971
|
-
key: "policies.<gate>.failAfter",
|
|
2972
|
-
scope: "check",
|
|
2973
|
-
unit: "s",
|
|
2974
|
-
default: null
|
|
2975
|
-
},
|
|
2976
|
-
recover_after: {
|
|
2977
|
-
key: "policies.<gate>.recoverAfter",
|
|
2978
|
-
scope: "check",
|
|
2979
|
-
unit: "s",
|
|
2980
|
-
default: null
|
|
2981
|
-
}
|
|
2982
|
-
},
|
|
2983
|
-
timers: {
|
|
2984
|
-
fail_window: {
|
|
2985
|
-
duration: {
|
|
2986
|
-
knob: "fail_after",
|
|
2987
|
-
unit: "s"
|
|
2988
|
-
},
|
|
2989
|
-
arms: ["enter:failing"],
|
|
2990
|
-
cancels: ["poll.ok"],
|
|
2991
|
-
fires: "window.failed"
|
|
2992
|
-
},
|
|
2993
|
-
recover_window: {
|
|
2994
|
-
duration: {
|
|
2995
|
-
knob: "recover_after",
|
|
2996
|
-
unit: "s"
|
|
2997
|
-
},
|
|
2998
|
-
arms: ["enter:recovering"],
|
|
2999
|
-
cancels: ["poll.fail"],
|
|
3000
|
-
fires: "window.recovered"
|
|
3001
|
-
}
|
|
3002
|
-
},
|
|
3003
|
-
transitions: [
|
|
3004
|
-
t("pass", "released", "poll.ok", "released"),
|
|
3005
|
-
t("start-failing", "released", "poll.fail", "failing", { guard: "fail_after > 0" }),
|
|
3006
|
-
t("trip-at-once", "released", "poll.fail", "tripped", { guard: "fail_after = 0" }),
|
|
3007
|
-
t("still-failing", "failing", "poll.fail", "failing"),
|
|
3008
|
-
t("stop-failing", "failing", "poll.ok", "released"),
|
|
3009
|
-
t("trip", "failing", "window.failed", "tripped", { timer: "fail_window" }),
|
|
3010
|
-
t("fail", "tripped", "poll.fail", "tripped"),
|
|
3011
|
-
t("release-at-once", "tripped", "poll.ok", "released", { guard: "recover_after is absent or 0" }),
|
|
3012
|
-
t("start-recovering", "tripped", "poll.ok", "recovering", { guard: "recover_after > 0" }),
|
|
3013
|
-
t("still-recovering", "recovering", "poll.ok", "recovering"),
|
|
3014
|
-
t("stop-recovering", "recovering", "poll.fail", "tripped"),
|
|
3015
|
-
t("release", "recovering", "window.recovered", "released", { timer: "recover_window" })
|
|
3016
|
-
]
|
|
3017
|
-
};
|
|
3018
|
-
/**
|
|
3019
|
-
* A poller starts one check on a jittered loop with a fixed delay. A call that outlives its timeout
|
|
3020
|
-
* stays pending (`hung`), and the poller does not start the check again until the call settles.
|
|
3021
|
-
*/
|
|
3022
|
-
const POLLER_MACHINE = {
|
|
3023
|
-
...base("poller", "The poller of a check"),
|
|
3024
|
-
states: {
|
|
3025
|
-
idle: {
|
|
3026
|
-
initial: true,
|
|
3027
|
-
note: "before the loop starts"
|
|
3028
|
-
},
|
|
3029
|
-
waiting: { note: "the loop timer runs, and no call is pending" },
|
|
3030
|
-
calling: { note: "a call runs, in its timeout" },
|
|
3031
|
-
hung: { note: "a call outlived its timeout, and it is still pending" },
|
|
3032
|
-
stopped: { final: true }
|
|
3033
|
-
},
|
|
3034
|
-
events: {
|
|
3035
|
-
start: { kind: "local" },
|
|
3036
|
-
sweep: { kind: "local" },
|
|
3037
|
-
tick: { kind: "timer" },
|
|
3038
|
-
answer: { kind: "local" },
|
|
3039
|
-
timeout: { kind: "timer" },
|
|
3040
|
-
"late-answer": { kind: "local" },
|
|
3041
|
-
stop: { kind: "local" }
|
|
3042
|
-
},
|
|
3043
|
-
knobs: {
|
|
3044
|
-
interval: {
|
|
3045
|
-
key: "checks.<name>.poll.interval",
|
|
3046
|
-
scope: "check",
|
|
3047
|
-
unit: "s",
|
|
3048
|
-
default: null
|
|
3049
|
-
},
|
|
3050
|
-
timeout: {
|
|
3051
|
-
key: "checks.<name>.poll.timeout",
|
|
3052
|
-
scope: "check",
|
|
3053
|
-
unit: "s",
|
|
3054
|
-
default: null
|
|
3055
|
-
}
|
|
3056
|
-
},
|
|
3057
|
-
timers: {
|
|
3058
|
-
loop: {
|
|
3059
|
-
duration: {
|
|
3060
|
-
knob: "interval",
|
|
3061
|
-
unit: "s"
|
|
3062
|
-
},
|
|
3063
|
-
arms: ["start", "tick"],
|
|
3064
|
-
cancels: ["stop"],
|
|
3065
|
-
fires: "tick",
|
|
3066
|
-
note: "the interval, with its jitter: a fixed delay from the start of each poll"
|
|
3067
|
-
},
|
|
3068
|
-
poll_timeout: {
|
|
3069
|
-
duration: {
|
|
3070
|
-
knob: "timeout",
|
|
3071
|
-
unit: "s"
|
|
3072
|
-
},
|
|
3073
|
-
arms: ["enter:calling"],
|
|
3074
|
-
cancels: ["answer", "stop"],
|
|
3075
|
-
fires: "timeout"
|
|
3076
|
-
}
|
|
3077
|
-
},
|
|
3078
|
-
transitions: [
|
|
3079
|
-
t("sweep", "idle", "sweep", "calling"),
|
|
3080
|
-
t("start", "idle", "start", "waiting"),
|
|
3081
|
-
t("start-while-calling", "calling", "start", "calling", { note: "a late check polls at once, then starts its loop" }),
|
|
3082
|
-
t("start-while-hung", "hung", "start", "hung"),
|
|
3083
|
-
t("tick", "waiting", "tick", "calling", { guard: "the force-fail list does not name the check" }),
|
|
3084
|
-
t("tick-forced", "waiting", "tick", "waiting", { guard: "the force-fail list names the check" }),
|
|
3085
|
-
t("tick-pending", "calling", "tick", "calling", { note: "the poll fails at once: its call is pending" }),
|
|
3086
|
-
t("tick-hung", "hung", "tick", "hung", { note: "the poll fails at once: its call is pending" }),
|
|
3087
|
-
t("answer-loop", "calling", "answer", "waiting", { guard: "the loop runs" }),
|
|
3088
|
-
t("answer-sweep", "calling", "answer", "idle", { guard: "the loop has not started" }),
|
|
3089
|
-
t("timeout", "calling", "timeout", "hung", { timer: "poll_timeout" }),
|
|
3090
|
-
t("late-answer-loop", "hung", "late-answer", "waiting", { guard: "the loop runs" }),
|
|
3091
|
-
t("late-answer-sweep", "hung", "late-answer", "idle", { guard: "the loop has not started" }),
|
|
3092
|
-
t("stop", [
|
|
3093
|
-
"idle",
|
|
3094
|
-
"waiting",
|
|
3095
|
-
"calling",
|
|
3096
|
-
"hung"
|
|
3097
|
-
], "stop", "stopped"),
|
|
3098
|
-
t("settle-after-stop", "stopped", [
|
|
3099
|
-
"answer",
|
|
3100
|
-
"late-answer",
|
|
3101
|
-
"timeout"
|
|
3102
|
-
], "stopped")
|
|
3103
|
-
]
|
|
3104
|
-
};
|
|
3105
|
-
/**
|
|
3106
|
-
* The exit manager is the only part that ends the process. The first exit request starts the
|
|
3107
|
-
* shutdown. A later request has no effect, but a repeat signal exits at once when the declaration
|
|
3108
|
-
* says `onRepeatSignal: "exit"`. The exit code is the code of the exit request, else 1 after an
|
|
3109
|
-
* error, else 0.
|
|
3110
|
-
*/
|
|
3111
|
-
const EXIT_MACHINE = {
|
|
3112
|
-
...base("exit", "The exit manager"),
|
|
3113
|
-
states: {
|
|
3114
|
-
running: { initial: true },
|
|
3115
|
-
stopping: {},
|
|
3116
|
-
exited: { final: true }
|
|
3117
|
-
},
|
|
3118
|
-
events: {
|
|
3119
|
-
"exit.request": {
|
|
3120
|
-
kind: "app",
|
|
3121
|
-
note: "SIGINT, SIGTERM, SIGQUIT, a declared process trigger or shutdown()"
|
|
3122
|
-
},
|
|
3123
|
-
"exit.repeat": {
|
|
3124
|
-
kind: "app",
|
|
3125
|
-
note: "a trigger or shutdown() while the shutdown runs"
|
|
3126
|
-
},
|
|
3127
|
-
"startup.fail": { kind: "local" },
|
|
3128
|
-
error: { kind: "local" },
|
|
3129
|
-
"shutdown.done": { kind: "local" }
|
|
3130
|
-
},
|
|
3131
|
-
knobs: { on_repeat_signal: {
|
|
3132
|
-
key: "shutdown.onRepeatSignal",
|
|
3133
|
-
scope: "service",
|
|
3134
|
-
unit: null,
|
|
3135
|
-
default: null
|
|
3136
|
-
} },
|
|
3137
|
-
timers: {},
|
|
3138
|
-
transitions: [
|
|
3139
|
-
t("request", "running", "exit.request", "stopping"),
|
|
3140
|
-
t("startup-fail", "running", "startup.fail", "stopping"),
|
|
3141
|
-
t("error-running", "running", "error", "running"),
|
|
3142
|
-
t("repeat", "stopping", "exit.repeat", "stopping", { guard: "a process trigger or shutdown() repeats, or a signal repeats and on_repeat_signal is not exit" }),
|
|
3143
|
-
t("repeat-exit", "stopping", "exit.repeat", "exited", {
|
|
3144
|
-
guard: "a signal repeats and on_repeat_signal is exit",
|
|
3145
|
-
note: "the process exits at once with the code 1"
|
|
3146
|
-
}),
|
|
3147
|
-
t("startup-fail-stopping", "stopping", "startup.fail", "stopping"),
|
|
3148
|
-
t("error", "stopping", "error", "stopping"),
|
|
3149
|
-
t("exit", "stopping", "shutdown.done", "exited", { note: "the exit code is the code of the exit request, else 1 after an error, else 0" })
|
|
3150
|
-
]
|
|
3151
|
-
};
|
|
3152
|
-
/** A watchdog measures how late its own tick comes, and a late tick shows a blocked event loop. */
|
|
3153
|
-
const WATCHDOG_MACHINE = {
|
|
3154
|
-
...base("watchdog", "The watchdog of the event loop"),
|
|
3155
|
-
states: {
|
|
3156
|
-
watching: { initial: true },
|
|
3157
|
-
blocked: {},
|
|
3158
|
-
stopped: { final: true }
|
|
3159
|
-
},
|
|
3160
|
-
events: {
|
|
3161
|
-
tick: { kind: "timer" },
|
|
3162
|
-
stop: { kind: "local" }
|
|
3163
|
-
},
|
|
3164
|
-
knobs: {
|
|
3165
|
-
tick: {
|
|
3166
|
-
key: "watchdog.tick",
|
|
3167
|
-
scope: "process",
|
|
3168
|
-
unit: "s",
|
|
3169
|
-
default: null
|
|
3170
|
-
},
|
|
3171
|
-
threshold: {
|
|
3172
|
-
key: "watchdog.threshold",
|
|
3173
|
-
scope: "process",
|
|
3174
|
-
unit: "s",
|
|
3175
|
-
default: null
|
|
3176
|
-
}
|
|
3177
|
-
},
|
|
3178
|
-
timers: { tick: {
|
|
3179
|
-
duration: {
|
|
3180
|
-
knob: "tick",
|
|
3181
|
-
unit: "s"
|
|
3182
|
-
},
|
|
3183
|
-
arms: ["enter:watching", "tick"],
|
|
3184
|
-
cancels: ["stop"],
|
|
3185
|
-
fires: "tick"
|
|
3186
|
-
} },
|
|
3187
|
-
transitions: [
|
|
3188
|
-
t("on-time", "watching", "tick", "watching", { guard: "lateness <= threshold" }),
|
|
3189
|
-
t("block", "watching", "tick", "blocked", { guard: "lateness > threshold" }),
|
|
3190
|
-
t("still-blocked", "blocked", "tick", "blocked", { guard: "lateness > threshold" }),
|
|
3191
|
-
t("unblock", "blocked", "tick", "watching", { guard: "lateness <= threshold" }),
|
|
3192
|
-
t("stop", ["watching", "blocked"], "stop", "stopped")
|
|
3193
|
-
]
|
|
3194
|
-
};
|
|
3195
|
-
/** This function gives the name of a knob for the cap of a phase: `drain` gives `drain_cap`. */
|
|
3196
|
-
const capKnob = (phase) => `${phase.replace(/[^A-Za-z0-9]+/g, "_")}_cap`;
|
|
3197
|
-
/**
|
|
3198
|
-
* This function gives the drain of a declaration as a machine. It has one state for each declared
|
|
3199
|
-
* phase, in its sequence, and each state has a timer at its cap. The drain also has the cooperative abort: the
|
|
3200
|
-
* cap of the drain less the cooperative window. With the floor `half`, it is half that cap at the
|
|
3201
|
-
* least. With the floor `none`, it is 0 at the least.
|
|
3202
|
-
*/
|
|
3203
|
-
const drainMachine = (description) => {
|
|
3204
|
-
const phases = description.shutdown.phases;
|
|
3205
|
-
const states = { running: { initial: true } };
|
|
3206
|
-
for (const phase of phases) states[phase.name] = {};
|
|
3207
|
-
states["done"] = { final: true };
|
|
3208
|
-
const knobs = Object.fromEntries([...phases.map((phase) => [capKnob(phase.name), {
|
|
3209
|
-
key: `shutdown.phases.${phase.name}.cap`,
|
|
3210
|
-
scope: "service",
|
|
3211
|
-
unit: "s",
|
|
3212
|
-
default: null
|
|
3213
|
-
}]), ["cooperative_window", {
|
|
3214
|
-
key: "shutdown.cooperativeWindow",
|
|
3215
|
-
scope: "service",
|
|
3216
|
-
unit: "s",
|
|
3217
|
-
default: null
|
|
3218
|
-
}]]);
|
|
3219
|
-
const timers = {};
|
|
3220
|
-
for (const phase of phases) timers[`${capKnob(phase.name)}_timer`] = {
|
|
3221
|
-
duration: {
|
|
3222
|
-
knob: capKnob(phase.name),
|
|
3223
|
-
unit: "s"
|
|
3224
|
-
},
|
|
3225
|
-
arms: [`enter:${phase.name}`],
|
|
3226
|
-
cancels: ["phase.done"],
|
|
3227
|
-
fires: "phase.cap"
|
|
3228
|
-
};
|
|
3229
|
-
const floor = description.shutdown.cooperativeFloor === "none" ? "0" : "drain_cap / 2";
|
|
3230
|
-
if (phases.some((phase) => phase.name === "drain")) timers["cooperative_abort"] = {
|
|
3231
|
-
duration: {
|
|
3232
|
-
expr: `max(drain_cap - cooperative_window, ${floor})`,
|
|
3233
|
-
unit: "s"
|
|
3234
|
-
},
|
|
3235
|
-
arms: ["enter:drain"],
|
|
3236
|
-
cancels: ["phase.done", "phase.cap"],
|
|
3237
|
-
fires: "drain.cooperative-abort"
|
|
3238
|
-
};
|
|
3239
|
-
const transitions = [];
|
|
3240
|
-
const order = [
|
|
3241
|
-
"running",
|
|
3242
|
-
...phases.map((phase) => phase.name),
|
|
3243
|
-
"done"
|
|
3244
|
-
];
|
|
3245
|
-
for (let at = 0; at + 1 < order.length; at += 1) {
|
|
3246
|
-
const from = order[at];
|
|
3247
|
-
const to = order[at + 1];
|
|
3248
|
-
transitions.push(at === 0 ? t(`begin`, from, "exit.request", to) : t(`after-${from}`, from, ["phase.done", "phase.cap"], to, {
|
|
3249
|
-
timer: `${capKnob(from)}_timer`,
|
|
3250
|
-
note: "phase.cap: the work of the phase did not end in its cap, and the exit code is 1"
|
|
3251
|
-
}));
|
|
3252
|
-
}
|
|
3253
|
-
if (phases.some((phase) => phase.name === "drain")) transitions.push(t("cooperative-abort", "drain", "drain.cooperative-abort", "drain", { timer: "cooperative_abort" }));
|
|
3254
|
-
for (const state of [...phases.map((phase) => phase.name), "done"]) transitions.push(t(`signal-abort-${state}`, state, "signal.abort", state));
|
|
3255
|
-
return {
|
|
3256
|
-
...base("drain", `The drain of ${description.name}`),
|
|
3257
|
-
states,
|
|
3258
|
-
events: {
|
|
3259
|
-
"exit.request": { kind: "app" },
|
|
3260
|
-
"phase.done": { kind: "local" },
|
|
3261
|
-
"phase.cap": { kind: "timer" },
|
|
3262
|
-
"drain.cooperative-abort": { kind: "timer" },
|
|
3263
|
-
"signal.abort": { kind: "local" }
|
|
3264
|
-
},
|
|
3265
|
-
knobs,
|
|
3266
|
-
timers,
|
|
3267
|
-
transitions
|
|
3268
|
-
};
|
|
3269
|
-
};
|
|
3270
|
-
/** These are the static machines of the runtime, by the name that their events carry. */
|
|
3271
|
-
const MACHINES = {
|
|
3272
|
-
gate: GATE_MACHINE,
|
|
3273
|
-
latch: LATCH_MACHINE,
|
|
3274
|
-
window: WINDOW_MACHINE,
|
|
3275
|
-
poller: POLLER_MACHINE,
|
|
3276
|
-
exit: EXIT_MACHINE,
|
|
3277
|
-
watchdog: WATCHDOG_MACHINE
|
|
3278
|
-
};
|
|
3279
|
-
export { BOUND_RULES, CLOSE_BOUND_RULE, DECLARATION_RULE, DRAIN_RULES, DRAIN_RULE_IDS, DURATION_FRACTIONS, EPISODES_KEPT, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, IDENTITY_TYPES, LATCH_MACHINE, LAW_STATISTIC, LIMITER_RULES, LOG_LEVELS, MACHINES, NAME_RULE, O04, O09, O10, O16, PLATFORM_MARKS, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, STOPS, STOP_RULE, TOKEN_BUCKET_MACHINE, UNCAUGHT_ORIGINS, UNIT_BOUND_RULE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, atLeast, backoffBand, backoffDelay, backoffProblems, capKnob, capacityOf, compress, compression, compressionOf, consoleLogger, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, keyedQuota, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, lintStop, localDev, logEvents, longestRetryAfter, matchesMessage, mayReissue, memoryQuotaStore, memoryStore, nodeProcess, overCapacity, platformClock, processPortOf, queueDrain, quotaProblems, quotaRule, quotaStep, realClock, recoverSpan, redact, redeliverOnce, replay, retrier, retry, retryAfterDelay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, takeQuotas, unitsOf, verdictOf, watchdog, workerLoop };
|
|
3280
|
-
|
|
3281
|
-
//# sourceMappingURL=index.js.map
|
|
28
|
+
export { BOUND_RULES, CLOSE_BOUND_RULE, DECLARATION_RULE, DRAIN_RULES, DRAIN_RULE_IDS, DURATION_FRACTIONS, EPISODES_KEPT, EXIT_MACHINE, EXIT_SIGNALS, FORMAT, GATE_MACHINE, IDENTITY_TYPES, LATCH_MACHINE, LAW_STATISTIC, LIMITER_RULES, LOG_LEVELS, MACHINES, NAME_RULE, O04, O09, O10, O16, PLATFORM_MARKS, POLLER_MACHINE, PROCESS_TRIGGERS, PROGRESS_RULE, SERVICE_RULES, SLIDING_WINDOW_MACHINE, STAGES, STOPS, STOP_RULE, TOKEN_BUCKET_MACHINE, UNCAUGHT_ORIGINS, UNIT_BOUND_RULE, WATCHDOG_MACHINE, WINDOW_MACHINE, asDebounce, asWindow, atLeast, backoffBand, backoffDelay, backoffProblems, capKnob, capacityOf, compress, compression, compressionOf, consoleLogger, cooperativeAbortAt, daemon, decide, detectionTime, drainMachine, embeddedProcess, ephemeral, evaluate, failSpan, freeAt, heartbeat, heartbeatDetection, idleSpan, inequalities, isExitCode, isExitSignal, isMessageShape, keyedLimiter, keyedQuota, kubernetes, limitRule, limiterMachine, limiterProblems, limiterStateOf, lint, lintStop, localDev, logEvents, longestRetryAfter, matchesMessage, mayReissue, memoryQuotaStore, memoryStore, nodeProcess, overCapacity, platformClock, processPortOf, queueDrain, quotaProblems, quotaRule, quotaStep, realClock, recoverSpan, redact, redeliverOnce, replay, retrier, retry, retryAfterDelay, retryAfterSeconds, service, serviceBindings, stageRank, symbolsOf, takeQuotas, unitsOf, unrefClock, verdictOf, watchdog, workerLoop };
|