@mark1russell7/service 0.8.10 → 0.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -2
- package/build/host.d.ts +108 -3
- package/build/host.js +298 -7
- package/build/host.js.map +1 -1
- package/build/index.js +1 -267
- package/build/index.js.map +1 -1
- package/build/{ports-DZoipkSD.d.ts → ports-B1ayTL-j.d.ts} +13 -1
- package/build/process-7R9M29wN.js +437 -0
- package/build/process-7R9M29wN.js.map +1 -0
- package/build/testing.d.ts +4 -1
- package/build/testing.js +15 -3
- package/build/testing.js.map +1 -1
- package/package.json +2 -2
- package/build/process-Oe0Abbsm.js +0 -170
- package/build/process-Oe0Abbsm.js.map +0 -1
package/README.md
CHANGED
|
@@ -608,6 +608,8 @@ Each effect goes through a port (`HostPorts`): the commands, the files, the proc
|
|
|
608
608
|
| `writeDiscovery`, `readDiscovery`, `removeDiscovery` | The discovery file: the process ID, the port, the address and the start time |
|
|
609
609
|
| `stopTree(pid, ports)` | The stop of a process tree by its process ID |
|
|
610
610
|
| `probeOnce`, `parseEnvFile`, `rotatingLog` | A ready probe, an env file and a log file with rotation |
|
|
611
|
+
| `watchTargets(spec, ports, { onChange })` | The outside watcher: the health of each target, from a probe on its own loop |
|
|
612
|
+
| `watchProblems` | The problems of a watch spec |
|
|
611
613
|
|
|
612
614
|
- **The task.** On Windows, `installService` writes the XML of a scheduled task and gives it to `schtasks /Create`. The task starts at logon, at boot or each day, as the user, without a window. It starts the program again after a failure.
|
|
613
615
|
- **The unit.** On Linux, `installService` writes a systemd user unit in `$XDG_CONFIG_HOME/systemd/user` and enables it. A daily start also gets a timer. Other platforms get `unsupported`, and nothing changes.
|
|
@@ -617,7 +619,7 @@ Each effect goes through a port (`HostPorts`): the commands, the files, the proc
|
|
|
617
619
|
- **The outcomes.** Each operation gives data, not a throw: `done`, `failed` with each command and its output, `refused` with the problems of the spec, or `unsupported`.
|
|
618
620
|
- **The lock.** The lock file has the process ID and its start time. Thus a new process with the same ID does not hold the lock. A lock whose process stopped is stale, and the next take removes it.
|
|
619
621
|
- **The discovery file.** A client reads the port and the address of a running instance from it. `readDiscovery` tells `running`, `absent`, `stale` or `malformed`, and it does not throw.
|
|
620
|
-
- **The sequence of a group.** A program starts when each program of its `after` is ready. A ready probe is an HTTP status, a TCP connection or a command that gives 0. The times of a probe are quantities.
|
|
622
|
+
- **The sequence of a group.** A program starts when each program of its `after` is ready. A ready probe is an HTTP status, a TCP connection or a command that gives 0. An HTTP probe can also examine the times in its JSON answer (`fresh`). The times of a probe are quantities.
|
|
621
623
|
- **A program that stops.** It starts again after the backoff of its `retrier`. The count of tries goes back to 0 after the program is ready for `resetAfter`. A program also waits until its `after` programs are ready again.
|
|
622
624
|
- **A program that is not ready in its time.** The supervisor stops it after `within`, and starts it again after the backoff. The state of the program tells the reason.
|
|
623
625
|
- **A program that hangs.** A hung program does not exit, thus only a probe finds it. With `alive`, the supervisor uses a liveness probe each `interval` after the program is ready. After `failures` failures in sequence, it stops the program in stages and starts it again after the backoff. A pass sets the count to 0. Without `alive`, only an exit starts a program again.
|
|
@@ -679,6 +681,43 @@ console.log(outcome.kind === "failed" ? outcome.problem : outcome.kind);
|
|
|
679
681
|
|
|
680
682
|
- **The stop of the group.** The phase `tear down dependencies` caps `group.stop()`. Give that phase a cap above the sum of the graces of the programs.
|
|
681
683
|
|
|
684
|
+
### The outside watcher
|
|
685
|
+
|
|
686
|
+
A heartbeat from a program tells only that the program sends. `watchTargets` probes the targets from a second computer. Thus it also finds a program that answers while its work stopped. It also finds a failure of the provider of the heartbeat.
|
|
687
|
+
|
|
688
|
+
- **The targets.** Each target has a probe, an interval, a timeout and a jitter. Each target polls on its own loop, on the clock of the ports.
|
|
689
|
+
- **The verdict.** The default is a debounce: 3 failed polls in sequence make a target down, and 2 passed polls make it up. A window measures the time of the failures.
|
|
690
|
+
- **The health.** A target is `unknown` at the start. Its first pass makes it `up`, and its verdict makes it `down`. `onChange` gets each change, with the reason of the latest failed poll.
|
|
691
|
+
- **The freshness bounds.** With `fresh`, an HTTP probe also reads its answer as JSON. Each bound names a time with a JSON Pointer (RFC 6901), and it gives the largest age.
|
|
692
|
+
- **The times.** A time is epoch milliseconds or an ISO 8601 text. A missing time fails. The age is from the wall clock of the watcher, or from the time of the target at `now`.
|
|
693
|
+
- **The size.** The probe reads up to 1 MiB of the answer. A larger answer fails.
|
|
694
|
+
- **The alerts.** The watcher sends no message. The `onChange` handler sends the alert on a path that does not go through the target.
|
|
695
|
+
|
|
696
|
+
```ts
|
|
697
|
+
// watch.js: the outside watcher of the stack, on a second computer
|
|
698
|
+
import { nodeHostPorts, watchTargets } from "@mark1russell7/service/host";
|
|
699
|
+
|
|
700
|
+
const s = (value: number) => ({ value, unit: "s" });
|
|
701
|
+
const watcher = watchTargets(
|
|
702
|
+
{
|
|
703
|
+
targets: [
|
|
704
|
+
{ name: "api", probe: { kind: "http", url: "http://stack.internal:8080/liveness" }, interval: s(30), timeout: s(5) },
|
|
705
|
+
{
|
|
706
|
+
name: "pipeline",
|
|
707
|
+
// the newest test delivery is 26 h old or younger
|
|
708
|
+
probe: { kind: "http", url: "http://stack.internal:8080/freshness", fresh: [{ pointer: "/delivery/lastTestSendAt", within: { value: 26, unit: "h" } }] },
|
|
709
|
+
interval: { value: 5, unit: "min" },
|
|
710
|
+
timeout: s(10),
|
|
711
|
+
verdict: { type: "debounce", failAfter: 2, recoverAfter: 1 },
|
|
712
|
+
},
|
|
713
|
+
],
|
|
714
|
+
},
|
|
715
|
+
nodeHostPorts(),
|
|
716
|
+
// notify: the path of the alert, outside the stack
|
|
717
|
+
{ onChange: (change) => notify(`${change.target} is ${change.to}: ${change.why ?? "it passes again"}`) },
|
|
718
|
+
);
|
|
719
|
+
```
|
|
720
|
+
|
|
682
721
|
## The testing subpath
|
|
683
722
|
|
|
684
723
|
`@mark1russell7/service/testing` has the seams that a test of a service needs:
|
|
@@ -687,7 +726,7 @@ console.log(outcome.kind === "failed" ? outcome.problem : outcome.kind);
|
|
|
687
726
|
- `fakeProcess({ flush, clock })` gives a process port that takes signals and process triggers from a test (`signal(name)`, `trigger(name)`). It keeps each exit code in place of an exit. `flush` is its flush of the output, and `log` gives the time of each flush and each exit. `holds` is the count of holds of the process, and `heldAtExit` gives that count at each exit. `message(value)` sends a message of the IPC channel.
|
|
688
727
|
- `seeded(seed)` gives a seeded random number generator for the jitter (`start({ random })`).
|
|
689
728
|
- `scriptedCheck(behavior)` gives a check whose behavior a test sets: `"ok"`, `"error"`, `"indeterminate"`, `"throw"`, `"timeout"` or `"hang"`.
|
|
690
|
-
- `quietServer()` starts a TCP echo server that can go quiet. Its connections stay open while it is quiet, as with a frozen peer.
|
|
729
|
+
- `quietServer()` starts a TCP echo server that can go quiet. Its connections stay open while it is quiet, as with a frozen peer. `close()` resolves after the close of each connection, thus `connections` is 0 at that time.
|
|
691
730
|
- `detachedLifecycle()` gives a lifecycle with no service, for the start of a transport in a test.
|
|
692
731
|
- `fakeHost(options)` gives the ports of the host entry on a fake clock, a fake file system and fake processes. It records each command and each signal. A test ends a child with `exit(code)`, and `polite` makes each child obey a staged stop.
|
|
693
732
|
- `transportConformance(make, options)` generates the conformance test of a transport. `make` gives a subject: the transport, a way to offer new work and a way to hold work in flight. The cases check that `stopIntake` refuses new work, and that the count is correct and comes from events. They also check that `drain` holds to its deadline and gives the cooperative abort, and that `close` is idempotent and bounded. The fake clock makes the cases fast, and the real clock measures them. The option `streams` adds six cases of the streams (S6 D4). Then the subject also gives `call`, which sends the work of the kit through `life.request`. A unary call gives one value and its status. A stream stays in flight until its end, and a failure after its first item is an error line, not a status. The drain waits for an open stream, and its cooperative abort and its cut end the stream with an end that the caller gets.
|
package/build/host.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* of programs that start in sequence. Each effect of the system goes through `HostPorts`.
|
|
5
5
|
* A test gives `fakeHost()` of `./testing`, and a program gives `nodeHostPorts()`.
|
|
6
6
|
*/
|
|
7
|
-
import { a as FilePort, c as
|
|
7
|
+
import { a as FilePort, c as NetPort, d as SpawnRequest, f as StopSignal, i as CommandResult, l as ProcessMark, n as CommandOptions, o as HostPorts, p as joinPath, r as CommandPort, s as HttpAnswer, t as ChildHandle, u as ProcessesPort } from "./ports-B1ayTL-j.js";
|
|
8
8
|
import { t as Backoff } from "./backoff-QTRGfkEK.js";
|
|
9
9
|
import { Y as MessageShape, c as Logger } from "./logger-B6drv9w0.js";
|
|
10
10
|
import { n as Clock } from "./clock-DpVgpY4h.js";
|
|
@@ -353,9 +353,23 @@ export declare const stopService: (spec: ServiceSpec, ports: HostPorts, options?
|
|
|
353
353
|
* have gives `installed: false`. A malformed or stale discovery file gives its read, and not a throw.
|
|
354
354
|
*/
|
|
355
355
|
export declare const serviceStatus: (spec: ServiceSpec, ports: HostPorts, options?: InstanceOptions) => Promise<ServiceStatus>;
|
|
356
|
+
/**
|
|
357
|
+
* A freshness bound reads one time in the JSON answer of an http probe. The time is epoch
|
|
358
|
+
* milliseconds or an ISO 8601 text. The probe fails when the time is older than `within`, or when
|
|
359
|
+
* the answer does not have it.
|
|
360
|
+
*/
|
|
361
|
+
interface FreshnessBound {
|
|
362
|
+
/** This JSON Pointer (RFC 6901) names the time in the answer, for example `/delivery/lastTestSendAt`. */
|
|
363
|
+
readonly pointer: string;
|
|
364
|
+
/** This time is the largest age that passes. */
|
|
365
|
+
readonly within: Quantity;
|
|
366
|
+
}
|
|
356
367
|
/**
|
|
357
368
|
* A ready probe tells when a program is ready.
|
|
358
|
-
* - `http`: a GET request to the address gives the status, 200 by default.
|
|
369
|
+
* - `http`: a GET request to the address gives the status, 200 by default. With `fresh`, the answer
|
|
370
|
+
* is also JSON, and each bound of `fresh` passes. The age of a time is measured from `now`: a
|
|
371
|
+
* JSON Pointer to the time of the target in the answer. Without `now`, the wall clock of the host
|
|
372
|
+
* gives the time.
|
|
359
373
|
* - `tcp`: a TCP connection to the port opens. The default host is `127.0.0.1`.
|
|
360
374
|
* - `command`: a command gives the exit code 0.
|
|
361
375
|
*/
|
|
@@ -363,6 +377,8 @@ type ReadyProbe = {
|
|
|
363
377
|
readonly kind: "http";
|
|
364
378
|
readonly url: string;
|
|
365
379
|
readonly status?: number | undefined;
|
|
380
|
+
readonly fresh?: readonly FreshnessBound[] | undefined;
|
|
381
|
+
readonly now?: string | undefined;
|
|
366
382
|
} | {
|
|
367
383
|
readonly kind: "tcp";
|
|
368
384
|
readonly port: number;
|
|
@@ -380,10 +396,99 @@ type ProbeResult = {
|
|
|
380
396
|
readonly ok: false;
|
|
381
397
|
readonly why: string;
|
|
382
398
|
};
|
|
399
|
+
/** This size is the largest answer that a probe with freshness bounds reads: 1 MiB. */
|
|
400
|
+
export declare const PROBE_BODY_LIMIT = 1048576;
|
|
383
401
|
/** This function gives the problems of a probe: each value that a probe cannot use. */
|
|
384
402
|
export declare const probeProblems: (probe: ReadyProbe) => string[];
|
|
385
403
|
/** This function uses a probe one time, in a timeout of milliseconds. It does not throw. */
|
|
386
404
|
export declare const probeOnce: (probe: ReadyProbe, timeout: number, ports: HostPorts) => Promise<ProbeResult>;
|
|
405
|
+
/**
|
|
406
|
+
* A verdict tells when a target goes down and when it comes up again.
|
|
407
|
+
* - `debounce`: `failAfter` failed polls in sequence make it down, and `recoverAfter` passed polls in sequence make it up.
|
|
408
|
+
* - `window`: failures for the time `failAfter` make it down. Passes for the time `recoverAfter` make it up. Without
|
|
409
|
+
* `recoverAfter`, the first pass makes it up.
|
|
410
|
+
*/
|
|
411
|
+
type TargetVerdict = {
|
|
412
|
+
readonly type: "debounce";
|
|
413
|
+
readonly failAfter: number;
|
|
414
|
+
readonly recoverAfter: number;
|
|
415
|
+
} | {
|
|
416
|
+
readonly type: "window";
|
|
417
|
+
readonly failAfter: Quantity;
|
|
418
|
+
readonly recoverAfter?: Quantity | undefined;
|
|
419
|
+
};
|
|
420
|
+
/** A watch target is one thing that a watcher probes on its own loop. */
|
|
421
|
+
interface WatchTarget {
|
|
422
|
+
/** This name is unique in the spec. A change names its target with it. */
|
|
423
|
+
readonly name: string;
|
|
424
|
+
readonly probe: ReadyProbe;
|
|
425
|
+
/** This time is between the starts of two polls. */
|
|
426
|
+
readonly interval: Quantity;
|
|
427
|
+
/** This time is the most that one poll waits. It is not above the interval. */
|
|
428
|
+
readonly timeout: Quantity;
|
|
429
|
+
/** This fraction moves each interval by up to ± its part, thus two watchers do not poll in step. The default is 0.1. */
|
|
430
|
+
readonly jitter?: number | undefined;
|
|
431
|
+
/** The default is a debounce: 3 failures make it down, and 2 passes make it up. */
|
|
432
|
+
readonly verdict?: TargetVerdict | undefined;
|
|
433
|
+
}
|
|
434
|
+
/** A watch spec lists the targets of one watcher. */
|
|
435
|
+
interface WatchSpec {
|
|
436
|
+
readonly targets: readonly WatchTarget[];
|
|
437
|
+
}
|
|
438
|
+
/** The health of a target is `unknown` until its first verdict. */
|
|
439
|
+
type TargetHealth = "unknown" | "up" | "down";
|
|
440
|
+
/** A target change is one transition of the health of a target. */
|
|
441
|
+
interface TargetChange {
|
|
442
|
+
readonly target: string;
|
|
443
|
+
readonly from: TargetHealth;
|
|
444
|
+
readonly to: "up" | "down";
|
|
445
|
+
/** This time is the wall time of the change, in milliseconds after the epoch. */
|
|
446
|
+
readonly at: number;
|
|
447
|
+
/** This text tells why the latest poll failed, for a change to down. It is null for a change to up. */
|
|
448
|
+
readonly why: string | null;
|
|
449
|
+
}
|
|
450
|
+
/** A target state is what a watcher knows of one target at this time. */
|
|
451
|
+
interface TargetState {
|
|
452
|
+
readonly name: string;
|
|
453
|
+
readonly health: TargetHealth;
|
|
454
|
+
/** This time is the wall time of the latest change of the health, or of the start. */
|
|
455
|
+
readonly since: number;
|
|
456
|
+
/** This result is the latest poll, or null before the first poll. */
|
|
457
|
+
readonly last: ProbeResult | null;
|
|
458
|
+
/** This time is the wall time of the latest poll, or null. */
|
|
459
|
+
readonly lastAt: number | null;
|
|
460
|
+
/** This count is the failed polls in sequence. */
|
|
461
|
+
readonly failures: number;
|
|
462
|
+
}
|
|
463
|
+
interface WatchOptions {
|
|
464
|
+
/** This function gets each change of the health of a target. A throw in it does not stop the watcher. */
|
|
465
|
+
readonly onChange: (change: TargetChange) => void;
|
|
466
|
+
/** This function gives a number in [0, 1) for the jitter. The default is `Math.random`. */
|
|
467
|
+
readonly random?: (() => number) | undefined;
|
|
468
|
+
/** This function gets each transition of the poller and the latch of each target, with the name of the target. */
|
|
469
|
+
readonly trace?: ((target: string, from: string, to: string, on: string) => void) | undefined;
|
|
470
|
+
}
|
|
471
|
+
/** A target watch is a running watcher. */
|
|
472
|
+
interface TargetWatch {
|
|
473
|
+
/** This method gives the state of each target, in the sequence of the spec. */
|
|
474
|
+
states(): TargetState[];
|
|
475
|
+
/** This method polls each target one time at once, outside its loop, and gives the states after the polls. */
|
|
476
|
+
pollNow(): Promise<TargetState[]>;
|
|
477
|
+
/** This method stops each loop. A stopped watcher gives no change. */
|
|
478
|
+
stop(): void;
|
|
479
|
+
}
|
|
480
|
+
/** This verdict is the default: 3 failures in sequence make a target down, and 2 passes make it up. */
|
|
481
|
+
export declare const DEFAULT_VERDICT: TargetVerdict;
|
|
482
|
+
/** This function gives the problems of a watch spec: each value that the watcher cannot use. */
|
|
483
|
+
export declare const watchProblems: (spec: WatchSpec) => string[];
|
|
484
|
+
/**
|
|
485
|
+
* This function starts a watcher of targets: an outside prober. Each target has its own loop of polls, on the
|
|
486
|
+
* clock of the ports, with a jitter. A verdict turns the polls into the health of the target: a debounce or a
|
|
487
|
+
* window. A target is `unknown` at the start. Its first pass makes it up, and its verdict makes it down. Each change
|
|
488
|
+
* goes to `onChange`, with the reason of the latest failed poll. The watcher throws at the start when the spec has a
|
|
489
|
+
* problem.
|
|
490
|
+
*/
|
|
491
|
+
export declare const watchTargets: (spec: WatchSpec, ports: HostPorts, options: WatchOptions) => TargetWatch;
|
|
387
492
|
/** An env file is the variables of a file of environment variables, and the problem of each line that it skipped. */
|
|
388
493
|
interface EnvFile {
|
|
389
494
|
readonly env: Readonly<Record<string, string>>;
|
|
@@ -569,4 +674,4 @@ interface NodeHostOptions {
|
|
|
569
674
|
* Only a program uses it. A test gives `fakeHost()` of `./testing`, thus it starts no process and installs no task.
|
|
570
675
|
*/
|
|
571
676
|
export declare const nodeHostPorts: (options?: NodeHostOptions) => HostPorts;
|
|
572
|
-
export { type AliveSpec, type ChildHandle, type CommandOptions, type CommandPort, type CommandRecord, type CommandResult, type Discovery, type DiscoveryFields, type DiscoveryRead, type EnvFile, type FilePort, type GroupOptions, type GroupSpec, type GroupSupervisor, type HostPorts, type InstallOptions, type InstanceLock, type InstanceOptions, type LockOptions, type LockOwner, type LockResult, type LockState, type LogSpec, type NetPort, type NodeHostOptions, type ProbeResult, type ProcessMark, type ProcessesPort, type ProgramPhase, type ProgramRestart, type ProgramSpec, type ProgramState, type ReadDiscoveryOptions, type ReadyProbe, type ReadySpec, type RestartPolicy, type RotatingLog, type ServiceOutcome, type ServicePaths, type ServiceSpec, type ServiceStart, type ServiceStatus, type SpawnRequest, type StopSignal, type StopSpec, type TaskContext, type TreeStopOutcome, joinPath };
|
|
677
|
+
export { type AliveSpec, type ChildHandle, type CommandOptions, type CommandPort, type CommandRecord, type CommandResult, type Discovery, type DiscoveryFields, type DiscoveryRead, type EnvFile, type FilePort, type FreshnessBound, type GroupOptions, type GroupSpec, type GroupSupervisor, type HostPorts, type HttpAnswer, type InstallOptions, type InstanceLock, type InstanceOptions, type LockOptions, type LockOwner, type LockResult, type LockState, type LogSpec, type NetPort, type NodeHostOptions, type ProbeResult, type ProcessMark, type ProcessesPort, type ProgramPhase, type ProgramRestart, type ProgramSpec, type ProgramState, type ReadDiscoveryOptions, type ReadyProbe, type ReadySpec, type RestartPolicy, type RotatingLog, type ServiceOutcome, type ServicePaths, type ServiceSpec, type ServiceStart, type ServiceStatus, type SpawnRequest, type StopSignal, type StopSpec, type TargetChange, type TargetHealth, type TargetState, type TargetVerdict, type TargetWatch, type TaskContext, type TreeStopOutcome, type WatchOptions, type WatchSpec, type WatchTarget, joinPath };
|
package/build/host.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { d as poller, f as latch, s as isMessageShape } from "./process-7R9M29wN.js";
|
|
1
2
|
import { backoffProblems } from "./backoff.js";
|
|
2
|
-
import { s as isMessageShape } from "./process-Oe0Abbsm.js";
|
|
3
3
|
import { t as retrier } from "./retrier-BObZq10-.js";
|
|
4
4
|
import { realClock } from "./clock.js";
|
|
5
|
-
import { convertible, isQuantity, valueIn } from "@mark1russell7/quantity";
|
|
5
|
+
import { convertible, formatDuration, isQuantity, valueIn } from "@mark1russell7/quantity";
|
|
6
6
|
import { connect } from "node:net";
|
|
7
7
|
import { escapeAttribute } from "@mark1russell7/text";
|
|
8
8
|
import { execFile, spawn } from "node:child_process";
|
|
@@ -931,23 +931,110 @@ const serviceStatus = async (spec, ports, options = {}) => {
|
|
|
931
931
|
};
|
|
932
932
|
};
|
|
933
933
|
// src/host/probes.ts
|
|
934
|
+
/** This size is the largest answer that a probe with freshness bounds reads: 1 MiB. */
|
|
935
|
+
const PROBE_BODY_LIMIT = 1048576;
|
|
936
|
+
const isPointer = (pointer) => typeof pointer === "string" && (pointer === "" || pointer.startsWith("/") && !/~[^01]|~$/.test(pointer));
|
|
934
937
|
/** This function gives the problems of a probe: each value that a probe cannot use. */
|
|
935
938
|
const probeProblems = (probe) => {
|
|
936
939
|
if (probe.kind === "http") {
|
|
937
940
|
const problems = [];
|
|
938
941
|
if (!/^https?:\/\//.test(probe.url)) problems.push(`the address ${JSON.stringify(probe.url)} is not an http or https address`);
|
|
939
942
|
if (probe.status !== void 0 && !(Number.isInteger(probe.status) && probe.status >= 100 && probe.status <= 599)) problems.push(`the status ${probe.status} is not an HTTP status`);
|
|
943
|
+
if (probe.now !== void 0 && !isPointer(probe.now)) problems.push(`the now ${JSON.stringify(probe.now)} is not a JSON Pointer`);
|
|
944
|
+
if (probe.now !== void 0 && probe.fresh === void 0) problems.push("the probe has a now, but no fresh list");
|
|
945
|
+
if (probe.fresh !== void 0 && probe.fresh.length === 0) problems.push("the fresh list is empty");
|
|
946
|
+
for (const bound of probe.fresh ?? []) {
|
|
947
|
+
if (!isPointer(bound.pointer)) problems.push(`the freshness pointer ${JSON.stringify(bound.pointer)} is not a JSON Pointer`);
|
|
948
|
+
if (!(isQuantity(bound.within) && convertible(bound.within.unit, "ms") && valueIn(bound.within, "ms") >= 0)) problems.push(`the freshness bound of ${JSON.stringify(bound.pointer)} is not a time of zero or more`);
|
|
949
|
+
}
|
|
940
950
|
return problems;
|
|
941
951
|
}
|
|
942
952
|
if (probe.kind === "tcp") return Number.isInteger(probe.port) && probe.port >= 1 && probe.port <= 65535 ? [] : [`the port ${probe.port} is not a port from 1 to 65535`];
|
|
943
953
|
if (probe.kind === "command") return probe.program === "" ? ["the program of the command probe is empty"] : [];
|
|
944
954
|
return [`the probe kind ${JSON.stringify(probe.kind)} is not http, tcp or command`];
|
|
945
955
|
};
|
|
956
|
+
/** This function gives the value at a JSON Pointer (RFC 6901), or undefined when the document does not have it. */
|
|
957
|
+
const pointAt = (document, pointer) => {
|
|
958
|
+
if (pointer === "") return document;
|
|
959
|
+
let at = document;
|
|
960
|
+
for (const token of pointer.slice(1).split("/")) {
|
|
961
|
+
const key = token.replaceAll("~1", "/").replaceAll("~0", "~");
|
|
962
|
+
if (Array.isArray(at)) {
|
|
963
|
+
if (!/^(0|[1-9]\d*)$/.test(key) || Number(key) >= at.length) return void 0;
|
|
964
|
+
at = at[Number(key)];
|
|
965
|
+
} else if (at !== null && typeof at === "object" && Object.hasOwn(at, key)) at = at[key];
|
|
966
|
+
else return void 0;
|
|
967
|
+
}
|
|
968
|
+
return at;
|
|
969
|
+
};
|
|
970
|
+
/** This function gives the time of a value in epoch milliseconds: a finite number, or an ISO 8601 text. */
|
|
971
|
+
const timeOf = (value) => {
|
|
972
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
973
|
+
if (typeof value === "string") {
|
|
974
|
+
const time = Date.parse(value);
|
|
975
|
+
return Number.isNaN(time) ? null : time;
|
|
976
|
+
}
|
|
977
|
+
return null;
|
|
978
|
+
};
|
|
979
|
+
/** This function examines the JSON answer of an http probe against its freshness bounds. */
|
|
980
|
+
const freshness = (url, body, bounds, nowPointer, wall) => {
|
|
981
|
+
let document;
|
|
982
|
+
try {
|
|
983
|
+
document = JSON.parse(body);
|
|
984
|
+
} catch {
|
|
985
|
+
return {
|
|
986
|
+
ok: false,
|
|
987
|
+
why: `${url} did not give JSON`
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
const now = nowPointer === void 0 ? wall : timeOf(pointAt(document, nowPointer));
|
|
991
|
+
if (now === null) return {
|
|
992
|
+
ok: false,
|
|
993
|
+
why: `${url} has no time at ${nowPointer}`
|
|
994
|
+
};
|
|
995
|
+
const stale = [];
|
|
996
|
+
for (const bound of bounds) {
|
|
997
|
+
const time = timeOf(pointAt(document, bound.pointer));
|
|
998
|
+
if (time === null) {
|
|
999
|
+
stale.push(`${bound.pointer} has no time`);
|
|
1000
|
+
continue;
|
|
1001
|
+
}
|
|
1002
|
+
const age = Math.max(0, now - time);
|
|
1003
|
+
if (age > valueIn(bound.within, "ms")) stale.push(`${bound.pointer} is ${formatDuration({
|
|
1004
|
+
value: age,
|
|
1005
|
+
unit: "ms"
|
|
1006
|
+
})} old, above ${formatDuration(bound.within)}`);
|
|
1007
|
+
}
|
|
1008
|
+
return stale.length === 0 ? { ok: true } : {
|
|
1009
|
+
ok: false,
|
|
1010
|
+
why: `${url}: ${stale.join("; ")}`
|
|
1011
|
+
};
|
|
1012
|
+
};
|
|
946
1013
|
/** This function uses a probe one time, in a timeout of milliseconds. It does not throw. */
|
|
947
1014
|
const probeOnce = async (probe, timeout, ports) => {
|
|
948
1015
|
try {
|
|
949
1016
|
if (probe.kind === "http") {
|
|
950
1017
|
const want = probe.status ?? 200;
|
|
1018
|
+
if (probe.fresh !== void 0) {
|
|
1019
|
+
if (ports.net.httpText === void 0) return {
|
|
1020
|
+
ok: false,
|
|
1021
|
+
why: "the net port cannot read an answer, thus it cannot examine freshness"
|
|
1022
|
+
};
|
|
1023
|
+
const answer = await ports.net.httpText(probe.url, timeout, PROBE_BODY_LIMIT);
|
|
1024
|
+
if (answer === null) return {
|
|
1025
|
+
ok: false,
|
|
1026
|
+
why: `${probe.url} gave no answer in ${timeout} ms`
|
|
1027
|
+
};
|
|
1028
|
+
if (answer.status !== want) return {
|
|
1029
|
+
ok: false,
|
|
1030
|
+
why: `${probe.url} gave ${answer.status}, not ${want}`
|
|
1031
|
+
};
|
|
1032
|
+
if (!answer.complete) return {
|
|
1033
|
+
ok: false,
|
|
1034
|
+
why: `the answer of ${probe.url} is above ${PROBE_BODY_LIMIT} bytes`
|
|
1035
|
+
};
|
|
1036
|
+
return freshness(probe.url, answer.body, probe.fresh, probe.now, ports.wall());
|
|
1037
|
+
}
|
|
951
1038
|
const status = await ports.net.httpStatus(probe.url, timeout);
|
|
952
1039
|
if (status === want) return { ok: true };
|
|
953
1040
|
return {
|
|
@@ -978,6 +1065,163 @@ const probeOnce = async (probe, timeout, ports) => {
|
|
|
978
1065
|
};
|
|
979
1066
|
}
|
|
980
1067
|
};
|
|
1068
|
+
// src/host/watch.ts
|
|
1069
|
+
/** This verdict is the default: 3 failures in sequence make a target down, and 2 passes make it up. */
|
|
1070
|
+
const DEFAULT_VERDICT = {
|
|
1071
|
+
type: "debounce",
|
|
1072
|
+
failAfter: 3,
|
|
1073
|
+
recoverAfter: 2
|
|
1074
|
+
};
|
|
1075
|
+
const DEFAULT_JITTER = .1;
|
|
1076
|
+
const ms$1 = (amount) => valueIn(amount, "ms");
|
|
1077
|
+
const isTimeAtLeast$1 = (value, floor, strict) => isTime(value) && (strict ? ms$1(value) > floor : ms$1(value) >= floor);
|
|
1078
|
+
const whole = (value) => Number.isInteger(value) && value >= 1;
|
|
1079
|
+
/** This function gives the problems of a watch spec: each value that the watcher cannot use. */
|
|
1080
|
+
const watchProblems = (spec) => {
|
|
1081
|
+
const problems = [];
|
|
1082
|
+
if (spec.targets.length === 0) problems.push("the spec has no target");
|
|
1083
|
+
const names = /* @__PURE__ */ new Set();
|
|
1084
|
+
for (const target of spec.targets) {
|
|
1085
|
+
const of = `the target ${JSON.stringify(target.name)}`;
|
|
1086
|
+
if (target.name === "") problems.push("a target name is empty");
|
|
1087
|
+
if (names.has(target.name)) problems.push(`${of} is in the spec two times`);
|
|
1088
|
+
names.add(target.name);
|
|
1089
|
+
problems.push(...probeProblems(target.probe).map((problem) => `${of}: ${problem}`));
|
|
1090
|
+
if (!isTimeAtLeast$1(target.interval, 0, true)) problems.push(`${of}: the interval is not a time above zero`);
|
|
1091
|
+
if (!isTimeAtLeast$1(target.timeout, 0, true)) problems.push(`${of}: the timeout is not a time above zero`);
|
|
1092
|
+
else if (isTimeAtLeast$1(target.interval, 0, true) && ms$1(target.timeout) > ms$1(target.interval)) problems.push(`${of}: the timeout is above the interval`);
|
|
1093
|
+
const jitter = target.jitter ?? DEFAULT_JITTER;
|
|
1094
|
+
if (!(Number.isFinite(jitter) && jitter >= 0 && jitter < 1)) problems.push(`${of}: the jitter ${jitter} is not a fraction from 0 to below 1`);
|
|
1095
|
+
const verdict = target.verdict ?? DEFAULT_VERDICT;
|
|
1096
|
+
if (verdict.type === "debounce") {
|
|
1097
|
+
if (!whole(verdict.failAfter)) problems.push(`${of}: the verdict fails after ${verdict.failAfter} polls, not a whole number above zero`);
|
|
1098
|
+
if (!whole(verdict.recoverAfter)) problems.push(`${of}: the verdict recovers after ${verdict.recoverAfter} polls, not a whole number above zero`);
|
|
1099
|
+
} else if (verdict.type === "window") {
|
|
1100
|
+
if (!isTimeAtLeast$1(verdict.failAfter, 0, false)) problems.push(`${of}: the failAfter of the window is not a time of zero or more`);
|
|
1101
|
+
if (verdict.recoverAfter !== void 0 && !isTimeAtLeast$1(verdict.recoverAfter, 0, false)) problems.push(`${of}: the recoverAfter of the window is not a time of zero or more`);
|
|
1102
|
+
} else problems.push(`${of}: the verdict type ${JSON.stringify(verdict.type)} is not debounce or window`);
|
|
1103
|
+
}
|
|
1104
|
+
return problems;
|
|
1105
|
+
};
|
|
1106
|
+
const timingOf = (target) => {
|
|
1107
|
+
const verdict = target.verdict ?? DEFAULT_VERDICT;
|
|
1108
|
+
return verdict.type === "debounce" ? {
|
|
1109
|
+
type: "debounce",
|
|
1110
|
+
interval: target.interval,
|
|
1111
|
+
failAfter: verdict.failAfter,
|
|
1112
|
+
recoverAfter: verdict.recoverAfter
|
|
1113
|
+
} : {
|
|
1114
|
+
type: "window",
|
|
1115
|
+
failAfter: verdict.failAfter,
|
|
1116
|
+
recoverAfter: verdict.recoverAfter
|
|
1117
|
+
};
|
|
1118
|
+
};
|
|
1119
|
+
/**
|
|
1120
|
+
* This function starts a watcher of targets: an outside prober. Each target has its own loop of polls, on the
|
|
1121
|
+
* clock of the ports, with a jitter. A verdict turns the polls into the health of the target: a debounce or a
|
|
1122
|
+
* window. A target is `unknown` at the start. Its first pass makes it up, and its verdict makes it down. Each change
|
|
1123
|
+
* goes to `onChange`, with the reason of the latest failed poll. The watcher throws at the start when the spec has a
|
|
1124
|
+
* problem.
|
|
1125
|
+
*/
|
|
1126
|
+
const watchTargets = (spec, ports, options) => {
|
|
1127
|
+
const problems = watchProblems(spec);
|
|
1128
|
+
if (problems.length > 0) throw new Error(`the watch spec has problems: ${problems.join("; ")}`);
|
|
1129
|
+
const random = options.random ?? Math.random;
|
|
1130
|
+
let stopped = false;
|
|
1131
|
+
const emit = (unit, to) => {
|
|
1132
|
+
if (unit.health === to) return;
|
|
1133
|
+
const change = {
|
|
1134
|
+
target: unit.target.name,
|
|
1135
|
+
from: unit.health,
|
|
1136
|
+
to,
|
|
1137
|
+
at: ports.wall(),
|
|
1138
|
+
why: to === "down" ? unit.why : null
|
|
1139
|
+
};
|
|
1140
|
+
unit.health = to;
|
|
1141
|
+
unit.since = change.at;
|
|
1142
|
+
if (stopped) return;
|
|
1143
|
+
try {
|
|
1144
|
+
options.onChange(change);
|
|
1145
|
+
} catch (error) {
|
|
1146
|
+
ports.logger?.log("warn", `the change handler of the watch failed: ${error instanceof Error ? error.message : String(error)}`, { target: change.target });
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
const watched = spec.targets.map((target) => {
|
|
1150
|
+
const unit = {
|
|
1151
|
+
target,
|
|
1152
|
+
health: "unknown",
|
|
1153
|
+
since: ports.wall(),
|
|
1154
|
+
last: null,
|
|
1155
|
+
lastAt: null,
|
|
1156
|
+
failures: 0,
|
|
1157
|
+
why: null
|
|
1158
|
+
};
|
|
1159
|
+
const trace = options.trace === void 0 ? void 0 : (from, to, on) => options.trace?.(target.name, from, to, on);
|
|
1160
|
+
const timeout = ms$1(target.timeout);
|
|
1161
|
+
let probed = null;
|
|
1162
|
+
const gate = latch(timingOf(target), ports.clock, false, () => emit(unit, gate.tripped ? "down" : "up"), trace);
|
|
1163
|
+
const record = (result) => {
|
|
1164
|
+
const own = probed;
|
|
1165
|
+
probed = null;
|
|
1166
|
+
const seen = result.outcome === "ok" ? { ok: true } : {
|
|
1167
|
+
ok: false,
|
|
1168
|
+
why: !result.timedOut && own !== null && !own.ok ? own.why : result.cause ?? "the poll failed"
|
|
1169
|
+
};
|
|
1170
|
+
unit.last = seen;
|
|
1171
|
+
unit.lastAt = ports.wall();
|
|
1172
|
+
unit.failures = seen.ok ? 0 : unit.failures + 1;
|
|
1173
|
+
if (!seen.ok) unit.why = seen.why;
|
|
1174
|
+
gate.record(result);
|
|
1175
|
+
if (seen.ok && unit.health === "unknown" && !gate.tripped) emit(unit, "up");
|
|
1176
|
+
};
|
|
1177
|
+
return {
|
|
1178
|
+
unit,
|
|
1179
|
+
gate,
|
|
1180
|
+
loop: poller({
|
|
1181
|
+
name: target.name,
|
|
1182
|
+
poll: {
|
|
1183
|
+
interval: target.interval,
|
|
1184
|
+
jitter: target.jitter ?? DEFAULT_JITTER,
|
|
1185
|
+
timeout: target.timeout
|
|
1186
|
+
},
|
|
1187
|
+
run: async () => {
|
|
1188
|
+
const result = await probeOnce(target.probe, timeout, ports);
|
|
1189
|
+
probed = result;
|
|
1190
|
+
return result.ok ? "ok" : "error";
|
|
1191
|
+
},
|
|
1192
|
+
clock: ports.clock,
|
|
1193
|
+
random,
|
|
1194
|
+
forced: () => false,
|
|
1195
|
+
record,
|
|
1196
|
+
trace
|
|
1197
|
+
})
|
|
1198
|
+
};
|
|
1199
|
+
});
|
|
1200
|
+
const states = () => watched.map(({ unit }) => ({
|
|
1201
|
+
name: unit.target.name,
|
|
1202
|
+
health: unit.health,
|
|
1203
|
+
since: unit.since,
|
|
1204
|
+
last: unit.last,
|
|
1205
|
+
lastAt: unit.lastAt,
|
|
1206
|
+
failures: unit.failures
|
|
1207
|
+
}));
|
|
1208
|
+
for (const { loop } of watched) loop.start(0);
|
|
1209
|
+
return {
|
|
1210
|
+
states,
|
|
1211
|
+
async pollNow() {
|
|
1212
|
+
await Promise.all(watched.map(({ loop }) => loop.pollNow()));
|
|
1213
|
+
return states();
|
|
1214
|
+
},
|
|
1215
|
+
stop() {
|
|
1216
|
+
if (stopped) return;
|
|
1217
|
+
stopped = true;
|
|
1218
|
+
for (const { gate, loop } of watched) {
|
|
1219
|
+
loop.stop();
|
|
1220
|
+
gate.stop();
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
};
|
|
1224
|
+
};
|
|
981
1225
|
// src/host/env.ts
|
|
982
1226
|
const KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
983
1227
|
/** This function reads a value in double quotes: `\n`, `\t`, `\"` and `\\` are escapes. */
|
|
@@ -1776,19 +2020,66 @@ const nodeProcesses = {
|
|
|
1776
2020
|
signalGroup: (pid, signal) => process.platform !== "win32" && sendSignal(-pid, signal),
|
|
1777
2021
|
spawn: nodeSpawn
|
|
1778
2022
|
};
|
|
2023
|
+
/**
|
|
2024
|
+
* This function sends a GET request that does not follow a redirect. The timeout also stops the
|
|
2025
|
+
* read of the body. The status read and the text read use it, thus they have one timeout.
|
|
2026
|
+
*/
|
|
2027
|
+
const get = (url, timeout) => fetch(url, {
|
|
2028
|
+
signal: AbortSignal.timeout(timeout),
|
|
2029
|
+
redirect: "manual"
|
|
2030
|
+
});
|
|
2031
|
+
/**
|
|
2032
|
+
* This function reads the body of an answer up to `limit` bytes, and gives the status and the text.
|
|
2033
|
+
* At the limit, it cancels the body, and `complete` is false. The decoder keeps the bytes of a
|
|
2034
|
+
* character that two parts of the body split. The unit tests give it answers with no network.
|
|
2035
|
+
*/
|
|
2036
|
+
const answerText = async (response, limit) => {
|
|
2037
|
+
const reader = response.body?.getReader();
|
|
2038
|
+
if (reader === void 0) return {
|
|
2039
|
+
status: response.status,
|
|
2040
|
+
body: "",
|
|
2041
|
+
complete: true
|
|
2042
|
+
};
|
|
2043
|
+
const decoder = new TextDecoder();
|
|
2044
|
+
let body = "";
|
|
2045
|
+
let size = 0;
|
|
2046
|
+
for (;;) {
|
|
2047
|
+
const read = await reader.read();
|
|
2048
|
+
if (read.done) return {
|
|
2049
|
+
status: response.status,
|
|
2050
|
+
body: body + decoder.decode(),
|
|
2051
|
+
complete: true
|
|
2052
|
+
};
|
|
2053
|
+
if (size + read.value.byteLength > limit) {
|
|
2054
|
+
body += decoder.decode(read.value.subarray(0, limit - size));
|
|
2055
|
+
await reader.cancel();
|
|
2056
|
+
return {
|
|
2057
|
+
status: response.status,
|
|
2058
|
+
body,
|
|
2059
|
+
complete: false
|
|
2060
|
+
};
|
|
2061
|
+
}
|
|
2062
|
+
body += decoder.decode(read.value, { stream: true });
|
|
2063
|
+
size += read.value.byteLength;
|
|
2064
|
+
}
|
|
2065
|
+
};
|
|
1779
2066
|
const nodeNet = {
|
|
1780
2067
|
async httpStatus(url, timeout) {
|
|
1781
2068
|
try {
|
|
1782
|
-
const response = await
|
|
1783
|
-
signal: AbortSignal.timeout(timeout),
|
|
1784
|
-
redirect: "manual"
|
|
1785
|
-
});
|
|
2069
|
+
const response = await get(url, timeout);
|
|
1786
2070
|
await response.body?.cancel();
|
|
1787
2071
|
return response.status;
|
|
1788
2072
|
} catch {
|
|
1789
2073
|
return null;
|
|
1790
2074
|
}
|
|
1791
2075
|
},
|
|
2076
|
+
async httpText(url, timeout, limit) {
|
|
2077
|
+
try {
|
|
2078
|
+
return await answerText(await get(url, timeout), limit);
|
|
2079
|
+
} catch {
|
|
2080
|
+
return null;
|
|
2081
|
+
}
|
|
2082
|
+
},
|
|
1792
2083
|
tcpConnect: (host, port, timeout) => new Promise((resolve) => {
|
|
1793
2084
|
const socket = connect({
|
|
1794
2085
|
host,
|
|
@@ -1831,6 +2122,6 @@ const nodeHostPorts = (options = {}) => {
|
|
|
1831
2122
|
...options.logger === void 0 ? {} : { logger: options.logger }
|
|
1832
2123
|
};
|
|
1833
2124
|
};
|
|
1834
|
-
export { DEFAULT_RESTART, START_MARGIN_MS, acquireInstanceLock, discoveryOf, groupProblems, inspectInstanceLock, installService, joinPath, lockText, nodeHostPorts, parseDiscovery, parseEnvFile, parseLockOwner, probeOnce, probeProblems, readDiscovery, removeDiscovery, renderSystemdTimer, renderSystemdUnit, renderTaskXml, rotatingLog, sameProcess, servicePaths, serviceProblems, serviceStatus, startOrder, startService, stopService, stopTree, superviseGroup, systemdArg, uninstallService, utf16le, windowsArg, writeDiscovery };
|
|
2125
|
+
export { DEFAULT_RESTART, DEFAULT_VERDICT, PROBE_BODY_LIMIT, START_MARGIN_MS, acquireInstanceLock, discoveryOf, groupProblems, inspectInstanceLock, installService, joinPath, lockText, nodeHostPorts, parseDiscovery, parseEnvFile, parseLockOwner, probeOnce, probeProblems, readDiscovery, removeDiscovery, renderSystemdTimer, renderSystemdUnit, renderTaskXml, rotatingLog, sameProcess, servicePaths, serviceProblems, serviceStatus, startOrder, startService, stopService, stopTree, superviseGroup, systemdArg, uninstallService, utf16le, watchProblems, watchTargets, windowsArg, writeDiscovery };
|
|
1835
2126
|
|
|
1836
2127
|
//# sourceMappingURL=host.js.map
|