@aywengo/mercury-fleet 0.0.1-bootstrap
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/LICENSE +21 -0
- package/README.md +144 -0
- package/dist/auth.js +89 -0
- package/dist/bindings.js +201 -0
- package/dist/child.js +137 -0
- package/dist/cli.js +371 -0
- package/dist/config.js +80 -0
- package/dist/credentials.js +93 -0
- package/dist/db.js +162 -0
- package/dist/dispatch.js +176 -0
- package/dist/events.js +114 -0
- package/dist/http.js +91 -0
- package/dist/interact.js +88 -0
- package/dist/logger.js +50 -0
- package/dist/metrics.js +226 -0
- package/dist/probe.js +224 -0
- package/dist/prober.js +94 -0
- package/dist/redact.js +72 -0
- package/dist/registry.js +197 -0
- package/dist/routing.js +219 -0
- package/dist/server.js +540 -0
- package/dist/stream.js +132 -0
- package/dist/sweep.js +183 -0
- package/dist/version.js +7 -0
- package/package.json +25 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to **Mercury Fleet** are recorded here. The host product
|
|
4
|
+
has its own [`../CHANGELOG.md`](../CHANGELOG.md).
|
|
5
|
+
|
|
6
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
7
|
+
and this project uses [Semantic Versioning](https://semver.org/).
|
|
8
|
+
|
|
9
|
+
## [Unreleased]
|
|
10
|
+
|
|
11
|
+
## [0.1.0] - not yet published
|
|
12
|
+
|
|
13
|
+
First published Fleet release, **pending**: `@aywengo/mercury-fleet` is still not on the npm
|
|
14
|
+
registry, so nothing here is installable yet. Fleet runs from a source checkout with
|
|
15
|
+
`npm run fleet`. See [the release notes](../docs/releases/fleet/0.1.0.md) for what publishing
|
|
16
|
+
still requires.
|
|
17
|
+
|
|
18
|
+
The one packaging change since `0.1.0-rc1` matters more than it looks: the package now ships
|
|
19
|
+
compiled JavaScript in `dist/` with `bin` pointing at `dist/cli.js`. It previously shipped
|
|
20
|
+
TypeScript sources with `bin` pointing at `cli.ts`, and Node refuses to strip types from a file
|
|
21
|
+
under `node_modules`, so the package would have installed cleanly and failed on its first command.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Ship `dist/*.js` instead of `*.ts`, so the installed package runs. `bin` now points at
|
|
26
|
+
`dist/cli.js`; `files` now ships `dist/`.
|
|
27
|
+
- The package description names what Fleet actually does. It said "Phase 0, no dispatch" long
|
|
28
|
+
after dispatch, routing, reconciliation, event mirroring and the metrics rollup had landed, and
|
|
29
|
+
that string is what npm renders on the package page.
|
|
30
|
+
|
|
31
|
+
## [0.1.0-rc1] - 2026-09-06
|
|
32
|
+
|
|
33
|
+
Recorded here as a version but **never published**: `@aywengo/mercury-fleet`
|
|
34
|
+
is not on the npm registry (404), there is no `fleet-v0.1.0-rc1` tag and no GitHub Release for it, so
|
|
35
|
+
`npm install -g @aywengo/mercury-fleet` does not work. Fleet can only be run from a source checkout
|
|
36
|
+
(`npm run fleet`). See [`../docs/releases/fleet/0.1.0-rc1.md`](../docs/releases/fleet/0.1.0-rc1.md).
|
|
37
|
+
|
|
38
|
+
### Added
|
|
39
|
+
|
|
40
|
+
- Host registry and probe (`fleet hosts`, `fleet probe --watch`).
|
|
41
|
+
- `fleet serve` HTTP API and systemd unit.
|
|
42
|
+
- Credential file (mode `0600`); names travel on the command line, never secrets.
|
|
43
|
+
- `GET /healthz` reports `{ ok, ts, product: "fleet", version }`.
|
|
44
|
+
- `fleet --version` prints `mercury-fleet <version>`.
|
|
45
|
+
- Run dispatch: submission records a binding that survives the process that made it, and the same
|
|
46
|
+
client token never produces a second child Run.
|
|
47
|
+
- The reconciliation sweep. A child Fleet cannot reach yields `UNKNOWN`, never `FAILED`.
|
|
48
|
+
- Event mirroring, metadata-only by default, with paging that resumes where it stopped.
|
|
49
|
+
- Routing across registered hosts, and forwarding of input, cancel and retry.
|
|
50
|
+
- `GET /metrics`, a Prometheus rollup that merges hosts into one HELP/TYPE block per metric.
|
|
51
|
+
|
|
52
|
+
### Not included
|
|
53
|
+
|
|
54
|
+
- Moving a Run between hosts. One Run, one host, for life.
|
|
55
|
+
- `mercuryctl` (separate, unreleased product).
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Roman Melnyk
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Fleet
|
|
2
|
+
|
|
3
|
+
Fleet manages several independent Mercury instances from one place. It is a federation layer: it talks to
|
|
4
|
+
each Mercury over its public HTTP API and never touches a Mercury database or imports Mercury code.
|
|
5
|
+
|
|
6
|
+
Design: [`docs/fleet-design.md`](../docs/fleet-design.md). All six build phases are shipped: registry and
|
|
7
|
+
probe, dispatch, reconciliation, event aggregation, routing, interaction, and the metrics rollup.
|
|
8
|
+
|
|
9
|
+
Two surfaces, deliberately different:
|
|
10
|
+
|
|
11
|
+
- **`fleet` (this CLI)** — the host registry, probing, and credential inspection. Local, interactive, no
|
|
12
|
+
network service of its own.
|
|
13
|
+
- **`fleet serve` (the service)** — everything that has to outlive the person who started it: submitting
|
|
14
|
+
Runs, routing them, reconciling their state, aggregating events, and the Prometheus rollup. See
|
|
15
|
+
[Service](#service) below.
|
|
16
|
+
|
|
17
|
+
Version `FLEET_VERSION` lives in [`version.ts`](version.ts) and must match
|
|
18
|
+
[`package.json`](package.json). Changelog: [`CHANGELOG.md`](CHANGELOG.md).
|
|
19
|
+
`fleet --version` prints `mercury-fleet <version>`. `GET /healthz` includes
|
|
20
|
+
`product: "fleet"` and that same version.
|
|
21
|
+
|
|
22
|
+
## Quick start
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
# 1. Child credentials live in a 0600 file, referenced by name. Never a command-line argument: argv is
|
|
26
|
+
# world-readable through ps, and Fleet holds a credential for every Mercury it can reach.
|
|
27
|
+
mkdir -p ~/.fleet && chmod 700 ~/.fleet
|
|
28
|
+
cat > ~/.fleet/credentials.json <<'JSON'
|
|
29
|
+
{ "mac-studio": "<token from that host's MERCURY_API_TOKENS>" }
|
|
30
|
+
JSON
|
|
31
|
+
chmod 600 ~/.fleet/credentials.json
|
|
32
|
+
|
|
33
|
+
# 2. Register hosts and look at them.
|
|
34
|
+
npm run fleet -- hosts add mac-studio --url https://studio.lan:3000 --credential mac-studio
|
|
35
|
+
npm run fleet -- hosts list --live
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
ID STATE SEEN WORKERS RUNS QUEUE AGENTS URL
|
|
40
|
+
mac-studio up 0s 1 2 0 5 https://studio.lan:3000
|
|
41
|
+
box-lan-2 auth-fail 0s 0 0 3 - http://box2.lan:3000
|
|
42
|
+
box-lan-2: HTTP 401 from /api/agents: the host is reachable but this credential was rejected.
|
|
43
|
+
box-lan-3 down 4m - - - - http://box3.lan:3000
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Commands
|
|
47
|
+
|
|
48
|
+
| Command | Does |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `fleet hosts add <id> --url <base> --credential <ref> [--label k=v] [--path <abs>] [--disabled]` | Register a host. `--credential` names a ref; the secret is never an argument. |
|
|
51
|
+
| `fleet hosts list [--json] [--live]` | Show hosts with their last probe. `--live` probes first. |
|
|
52
|
+
| `fleet hosts probe [<id>] [--json]` | Probe now. Exits non-zero if the host is unusable, so it composes in a readiness check. |
|
|
53
|
+
| `fleet hosts enable\|disable <id>` | Include or exclude a host from sweeps. |
|
|
54
|
+
| `fleet hosts rm <id>` | Forget a host and its cached probe. |
|
|
55
|
+
| `fleet probe --watch` | Sweep every enabled host on `FLEET_PROBE_INTERVAL_MS` until interrupted. |
|
|
56
|
+
| `fleet credentials list` | Credential **names** only. Values are never printed by any command. |
|
|
57
|
+
|
|
58
|
+
## What each state means
|
|
59
|
+
|
|
60
|
+
The states are deliberately not collapsed into up/down. Each one sends the operator somewhere different,
|
|
61
|
+
and a probe that reports "down" for a host that is healthy but rejecting our token wastes the operator's
|
|
62
|
+
time on the wrong machine.
|
|
63
|
+
|
|
64
|
+
| State | Means | Fix |
|
|
65
|
+
| --- | --- | --- |
|
|
66
|
+
| `up` | Reachable, queue configured, credential accepted. | — |
|
|
67
|
+
| `auth-fail` | Host is fine. Our credential is not. | Check the ref in the credential file. |
|
|
68
|
+
| `no-worker` | API answers, but its queue is not configured, so it executes nothing. | Start that host's worker. |
|
|
69
|
+
| `down` | Nothing listening. | Host, port, or `MERCURY_BIND_HOST`. |
|
|
70
|
+
| `timeout` | Something answered too slowly to trust. | Load, or a half-dead process. |
|
|
71
|
+
| `not-mercury` | A server is there; it is not a Mercury API. | Wrong port or URL. |
|
|
72
|
+
| `http-error` | Unexpected status from a Mercury endpoint. | Check that host's logs. |
|
|
73
|
+
| `never-probed` | Registered, not swept yet. | `fleet hosts probe`. |
|
|
74
|
+
|
|
75
|
+
## Configuration
|
|
76
|
+
|
|
77
|
+
| Variable | Default | Meaning |
|
|
78
|
+
| --- | --- | --- |
|
|
79
|
+
| `FLEET_DB` | `fleet.db` | Fleet's own SQLite database, separate from every Mercury's. |
|
|
80
|
+
| `FLEET_CREDENTIALS_FILE` | `~/.fleet/credentials.json` | Must be mode `0600`; Fleet refuses otherwise. |
|
|
81
|
+
| `FLEET_PROBE_INTERVAL_MS` | `15000` | Sweep interval for `probe --watch`. |
|
|
82
|
+
| `FLEET_PROBE_TIMEOUT_MS` | `5000` | Per-request timeout. A hung host must not stall the sweep. |
|
|
83
|
+
| `FLEET_ALLOW_INSECURE_CREDENTIALS` | unset | `1` bypasses the mode check. For filesystems that cannot do `0600`. |
|
|
84
|
+
|
|
85
|
+
> The default is laptop-shaped on purpose, for development. Fleet runs as a **service**
|
|
86
|
+
> ([`docs/fleet-design.md` §15](../docs/fleet-design.md#15-fleet-as-a-service)), and a hardened unit sets
|
|
87
|
+
> `ProtectHome=true`, which cannot read anything under a home directory. A service deployment sets
|
|
88
|
+
> `FLEET_CREDENTIALS_FILE=/etc/fleet/credentials.json` explicitly instead of relying on this default.
|
|
89
|
+
|
|
90
|
+
## Service
|
|
91
|
+
|
|
92
|
+
`fleet serve` runs the HTTP API. It binds `127.0.0.1:3100` by default.
|
|
93
|
+
|
|
94
|
+
| Variable | Default | Meaning |
|
|
95
|
+
| --- | --- | --- |
|
|
96
|
+
| `FLEET_BIND_HOST` | `127.0.0.1` | Bind address. |
|
|
97
|
+
| `FLEET_PORT` | `3100` | Listen port. |
|
|
98
|
+
| `FLEET_API_TOKENS` | unset | `token:owner[:hosts]`, comma-separated. `hosts` scopes a caller to a subset. |
|
|
99
|
+
| `FLEET_ADMIN_TOKEN` | unset | A caller that may see and change every host. |
|
|
100
|
+
| `FLEET_TLS_CERT` / `FLEET_TLS_KEY` | unset | Both or neither. Required to bind beyond loopback. |
|
|
101
|
+
| `FLEET_SWEEP_INTERVAL_MS` | `10000` | How often bindings are reconciled against every host. |
|
|
102
|
+
| `FLEET_STREAM_POLL_MS` | `1000` | Poll interval behind the aggregated Run stream. |
|
|
103
|
+
| `FLEET_REPO_URLS_FILE` | unset | `localPath` → git URL map used by the router. |
|
|
104
|
+
|
|
105
|
+
**It refuses to start in an unsafe configuration**, rather than serving and leaving discovery to an audit:
|
|
106
|
+
binding beyond loopback without TLS, half a TLS pair, or no caller tokens at all each fail at startup with
|
|
107
|
+
the reason. A Fleet token reaches every Mercury in the fleet, so plaintext on a shared network is not a
|
|
108
|
+
deployment someone should arrive at by omission.
|
|
109
|
+
|
|
110
|
+
| Endpoint | Does |
|
|
111
|
+
| --- | --- |
|
|
112
|
+
| `GET /healthz` | Liveness, with `product: "fleet"` and the version. Unauthenticated. |
|
|
113
|
+
| `GET /metrics` | Prometheus rollup across hosts, every series relabelled `host="<hostId>"`. |
|
|
114
|
+
| `GET\|POST /fleet/hosts` | List and register hosts. |
|
|
115
|
+
| `POST /fleet/hosts/:id/enable`, `.../disable`, `DELETE /fleet/hosts/:id` | Include or exclude a host from sweeps. |
|
|
116
|
+
| `POST /fleet/hosts/:id/probe` | Probe one host now. |
|
|
117
|
+
| `POST /fleet/runs` | Submit a Run. Name a `host`, or omit it and let the router choose. |
|
|
118
|
+
| `GET /fleet/runs` | Every Run across the fleet, one merged view. |
|
|
119
|
+
| `GET /fleet/runs/:id` | One Run, with its binding and current child state. |
|
|
120
|
+
| `GET /fleet/runs/:id/events`, `.../stream` | Aggregated history and SSE for a fleet Run. |
|
|
121
|
+
| `POST /fleet/runs/:id/input`, `.../cancel`, `.../retry` | Answer, cancel, or retry through Fleet. |
|
|
122
|
+
| `POST /fleet/probe` | Sweep every enabled host. |
|
|
123
|
+
|
|
124
|
+
Changing the registry — adding, removing, enabling or disabling a host, and sweeping — requires the admin
|
|
125
|
+
token. Reads and Run submission do not, but are scoped: a caller limited to a subset of hosts cannot route
|
|
126
|
+
work onto a hidden host, read another host's Run, or learn another host's queue depth from `/metrics`.
|
|
127
|
+
Routing failures name every host considered and why each was excluded, because "no host matched" without
|
|
128
|
+
reasons is an hour of guessing.
|
|
129
|
+
|
|
130
|
+
## Two rules this directory is built around
|
|
131
|
+
|
|
132
|
+
**Nothing here imports from `src/`.** Fleet speaks HTTP so that it can drive a Mercury it did not build.
|
|
133
|
+
`fleet/test/coupling.test.ts` enforces this, and includes tests proving the guard can actually fire.
|
|
134
|
+
|
|
135
|
+
**`hosts` is truth; everything else is cache.** Probe results live in their own table, so deleting them
|
|
136
|
+
costs one sweep. That split is what makes a Fleet crash cheap: the registry survives, and nothing Fleet
|
|
137
|
+
holds can orphan a Run on a machine nobody is watching.
|
|
138
|
+
|
|
139
|
+
## Development
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
npm run test:fleet # 187 tests, no network beyond localhost
|
|
143
|
+
npm run typecheck # covers fleet/ as well as src/
|
|
144
|
+
```
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caller authentication and the per-caller host allowlist (docs/fleet-design.md sections 9 and 15.3).
|
|
3
|
+
*
|
|
4
|
+
* Two credential boundaries exist and must not be conflated:
|
|
5
|
+
*
|
|
6
|
+
* caller -> Fleet tokens in FLEET_API_TOKENS. Never forwarded to a child.
|
|
7
|
+
* Fleet -> child credential_ref entries in the credential file.
|
|
8
|
+
*
|
|
9
|
+
* The allowlist is authorisation, not a filter. A caller permitted only for `box-lan-2` gets 403 when it
|
|
10
|
+
* names another host; it does not get a silently narrowed choice, and it does not get to learn whether the
|
|
11
|
+
* other host exists. Without this, one leaked Fleet token is every Mercury on the LAN, which is the whole
|
|
12
|
+
* reason section 9 exists.
|
|
13
|
+
*/
|
|
14
|
+
import { timingSafeEqual } from 'node:crypto';
|
|
15
|
+
/**
|
|
16
|
+
* Parse FLEET_API_TOKENS.
|
|
17
|
+
*
|
|
18
|
+
* Format: `token:owner[:hosts]`, entries separated by commas, where hosts is `*` or `host1+host2`.
|
|
19
|
+
*
|
|
20
|
+
* Omitting the hosts field grants NOTHING rather than everything. The permissive default would be the
|
|
21
|
+
* convenient one, and it is the one that turns a single leaked token into fleet-wide access, so the safe
|
|
22
|
+
* reading is the default and an operator has to type `:*` to widen it.
|
|
23
|
+
*/
|
|
24
|
+
export function parseCallerTokens(raw) {
|
|
25
|
+
const map = new Map();
|
|
26
|
+
for (const entry of (raw ?? '').split(',')) {
|
|
27
|
+
const trimmed = entry.trim();
|
|
28
|
+
if (!trimmed)
|
|
29
|
+
continue;
|
|
30
|
+
const parts = trimmed.split(':');
|
|
31
|
+
const [token, owner, hosts] = parts;
|
|
32
|
+
if (!token || !owner)
|
|
33
|
+
continue;
|
|
34
|
+
if (parts.length > 3)
|
|
35
|
+
continue; // malformed rather than guessed at
|
|
36
|
+
let allowed;
|
|
37
|
+
if (hosts === undefined || hosts === '') {
|
|
38
|
+
allowed = [];
|
|
39
|
+
}
|
|
40
|
+
else if (hosts === '*') {
|
|
41
|
+
allowed = '*';
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
const list = hosts.split('+').map((h) => h.trim()).filter(Boolean);
|
|
45
|
+
allowed = list.length ? list : [];
|
|
46
|
+
}
|
|
47
|
+
map.set(token, { ownerId: owner, isAdmin: false, allowedHosts: allowed });
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
resolve(token) {
|
|
51
|
+
return map.get(token) ?? null;
|
|
52
|
+
},
|
|
53
|
+
owners() {
|
|
54
|
+
return [...map.values()].map((c) => c.ownerId).sort();
|
|
55
|
+
},
|
|
56
|
+
unrestrictedOwners() {
|
|
57
|
+
return [...map.values()].filter((c) => c.allowedHosts === '*').map((c) => c.ownerId).sort();
|
|
58
|
+
},
|
|
59
|
+
secrets() {
|
|
60
|
+
return [...map.keys()];
|
|
61
|
+
},
|
|
62
|
+
get size() {
|
|
63
|
+
return map.size;
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Compare a bearer token against the admin token without leaking content through timing.
|
|
69
|
+
*
|
|
70
|
+
* A length mismatch returns false immediately. That discloses the length of the configured secret and
|
|
71
|
+
* nothing about its content; padding to a common length would risk a comparison that reports equality on a
|
|
72
|
+
* shared prefix, which is the worse failure.
|
|
73
|
+
*/
|
|
74
|
+
export function isAdminToken(adminToken, token) {
|
|
75
|
+
if (!adminToken)
|
|
76
|
+
return false;
|
|
77
|
+
const a = Buffer.from(adminToken, 'utf8');
|
|
78
|
+
const b = Buffer.from(token, 'utf8');
|
|
79
|
+
if (a.length !== b.length)
|
|
80
|
+
return false;
|
|
81
|
+
return timingSafeEqual(a, b);
|
|
82
|
+
}
|
|
83
|
+
export function hostAllowed(caller, hostId) {
|
|
84
|
+
if (caller.isAdmin)
|
|
85
|
+
return true;
|
|
86
|
+
if (caller.allowedHosts === '*')
|
|
87
|
+
return true;
|
|
88
|
+
return caller.allowedHosts.includes(hostId);
|
|
89
|
+
}
|
package/dist/bindings.js
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The binding table: which host owns which Run.
|
|
3
|
+
*
|
|
4
|
+
* This is Fleet's second piece of truth, and the one the design singles out. Losing `hosts` means Fleet does
|
|
5
|
+
* not know what to talk to; losing `fleet_runs` means Runs are executing on machines nobody can name. So the
|
|
6
|
+
* operations here are written so that a binding exists BEFORE Fleet asks a child for anything, and the
|
|
7
|
+
* child's id is filled in afterwards.
|
|
8
|
+
*/
|
|
9
|
+
/** Status recorded for a Run whose child could not be reached. Distinct from every real child status. */
|
|
10
|
+
export const UNKNOWN = 'UNKNOWN';
|
|
11
|
+
function toBinding(row) {
|
|
12
|
+
let requested = {};
|
|
13
|
+
try {
|
|
14
|
+
requested = JSON.parse(row.requested);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
// A corrupt payload must not make the binding unreadable. The binding is the thing that matters; the
|
|
18
|
+
// echo of the request is a convenience.
|
|
19
|
+
requested = { _unparseable: true };
|
|
20
|
+
}
|
|
21
|
+
return {
|
|
22
|
+
fleetRunId: row.fleet_run_id,
|
|
23
|
+
hostId: row.host_id,
|
|
24
|
+
ownerId: row.owner_id,
|
|
25
|
+
childRunId: row.child_run_id,
|
|
26
|
+
clientToken: row.client_token,
|
|
27
|
+
requested,
|
|
28
|
+
createdAt: row.created_at,
|
|
29
|
+
boundAt: row.bound_at,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function toState(row) {
|
|
33
|
+
return {
|
|
34
|
+
fleetRunId: row.fleet_run_id,
|
|
35
|
+
status: row.status,
|
|
36
|
+
cursor: Number(row.cursor),
|
|
37
|
+
lastSeenAt: row.last_seen_at,
|
|
38
|
+
lastError: row.last_error,
|
|
39
|
+
eventsDrained: Number(row.events_drained ?? 0) === 1,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export class BindingStore {
|
|
43
|
+
db;
|
|
44
|
+
constructor(db) {
|
|
45
|
+
this.db = db;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Record the intent to dispatch, before contacting the child.
|
|
49
|
+
*
|
|
50
|
+
* `requested` is stored as received so a later retry after a crash re-sends exactly the same payload; a
|
|
51
|
+
* payload that drifted between attempts would defeat the child's idempotency key.
|
|
52
|
+
*/
|
|
53
|
+
createPending(input) {
|
|
54
|
+
this.db
|
|
55
|
+
.prepare(`INSERT INTO fleet_runs (fleet_run_id, host_id, owner_id, child_run_id, client_token, requested, created_at)
|
|
56
|
+
VALUES (?, ?, ?, NULL, ?, ?, ?)`)
|
|
57
|
+
.run(input.fleetRunId, input.hostId, input.ownerId, input.clientToken ?? null, JSON.stringify(input.requested), new Date().toISOString());
|
|
58
|
+
return this.get(input.fleetRunId);
|
|
59
|
+
}
|
|
60
|
+
get(fleetRunId) {
|
|
61
|
+
const row = this.db.prepare('SELECT * FROM fleet_runs WHERE fleet_run_id = ?').get(fleetRunId);
|
|
62
|
+
return row ? toBinding(row) : null;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Look up by idempotency token WITHIN one owner. Unscoped lookup was a leak: two callers reusing the same
|
|
66
|
+
* memorable token collided, and the second received the first one's run id, host and status.
|
|
67
|
+
*/
|
|
68
|
+
findByClientToken(ownerId, token) {
|
|
69
|
+
const row = this.db
|
|
70
|
+
.prepare('SELECT * FROM fleet_runs WHERE owner_id = ? AND client_token = ?')
|
|
71
|
+
.get(ownerId, token);
|
|
72
|
+
return row ? toBinding(row) : null;
|
|
73
|
+
}
|
|
74
|
+
bind(fleetRunId, childRunId) {
|
|
75
|
+
this.db
|
|
76
|
+
.prepare('UPDATE fleet_runs SET child_run_id = ?, bound_at = ? WHERE fleet_run_id = ?')
|
|
77
|
+
.run(childRunId, new Date().toISOString(), fleetRunId);
|
|
78
|
+
return this.get(fleetRunId);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Drop a binding when the child definitively refused.
|
|
82
|
+
*
|
|
83
|
+
* Only reachable for a 4xx, where no Run was created. A transport failure or a 5xx must leave the binding
|
|
84
|
+
* in place with child_run_id NULL, because the Run may exist and a deleted binding is how it becomes an
|
|
85
|
+
* orphan.
|
|
86
|
+
*/
|
|
87
|
+
discard(fleetRunId) {
|
|
88
|
+
this.db.prepare('DELETE FROM fleet_runs WHERE fleet_run_id = ?').run(fleetRunId);
|
|
89
|
+
}
|
|
90
|
+
/** Bindings whose child answer was never recorded: the crash-recovery worklist. */
|
|
91
|
+
pending() {
|
|
92
|
+
const rows = this.db
|
|
93
|
+
.prepare('SELECT * FROM fleet_runs WHERE child_run_id IS NULL ORDER BY created_at')
|
|
94
|
+
.all();
|
|
95
|
+
return rows.map(toBinding);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Bindings scoped to the hosts the caller may see. An empty allowlist yields no rows rather than all of
|
|
99
|
+
* them -- the same fail-closed reading the caller allowlist uses.
|
|
100
|
+
*/
|
|
101
|
+
/**
|
|
102
|
+
* Bindings scoped to the hosts the caller may see. An empty allowlist yields no rows rather than all of
|
|
103
|
+
* them -- the same fail-closed reading the caller allowlist uses.
|
|
104
|
+
*
|
|
105
|
+
* run_state is joined rather than fetched per row. The previous shape issued one extra SELECT per binding,
|
|
106
|
+
* so a list of N Runs cost N+1 queries on the endpoint operators hit most often.
|
|
107
|
+
*/
|
|
108
|
+
list(hostIds) {
|
|
109
|
+
// Fail closed on an empty allowlist. node:sqlite happens to accept `IN ()` and return no rows, so the
|
|
110
|
+
// query would also come back empty without this line -- but that is one driver's behaviour, not a
|
|
111
|
+
// guarantee worth depending on for an authorization decision. The order matters too: '*' is a string of
|
|
112
|
+
// length 1, so an emptiness test written first would never catch the wildcard.
|
|
113
|
+
if (hostIds !== '*' && hostIds.length === 0)
|
|
114
|
+
return [];
|
|
115
|
+
const where = hostIds === '*' ? '' : `WHERE fleet_runs.host_id IN (${hostIds.map(() => '?').join(',')})`;
|
|
116
|
+
const args = hostIds === '*' ? [] : hostIds;
|
|
117
|
+
const rows = this.db
|
|
118
|
+
.prepare(`SELECT fleet_runs.*,
|
|
119
|
+
run_state.status AS state_status, run_state.cursor AS state_cursor,
|
|
120
|
+
run_state.last_seen_at AS state_last_seen_at, run_state.last_error AS state_last_error,
|
|
121
|
+
run_state.events_drained AS state_events_drained
|
|
122
|
+
FROM fleet_runs
|
|
123
|
+
LEFT JOIN run_state ON run_state.fleet_run_id = fleet_runs.fleet_run_id
|
|
124
|
+
${where}
|
|
125
|
+
ORDER BY fleet_runs.created_at DESC`)
|
|
126
|
+
.all(...args);
|
|
127
|
+
return rows.map((row) => ({
|
|
128
|
+
...toBinding(row),
|
|
129
|
+
state: row.state_status === null || row.state_status === undefined
|
|
130
|
+
? null
|
|
131
|
+
: {
|
|
132
|
+
fleetRunId: row.fleet_run_id, status: row.state_status, cursor: Number(row.state_cursor ?? 0),
|
|
133
|
+
lastSeenAt: row.state_last_seen_at ?? null, lastError: row.state_last_error ?? null,
|
|
134
|
+
eventsDrained: Number(row.state_events_drained ?? 0) === 1,
|
|
135
|
+
},
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
recordState(input) {
|
|
139
|
+
this.db
|
|
140
|
+
.prepare(`INSERT INTO run_state (fleet_run_id, status, cursor, last_seen_at, last_error)
|
|
141
|
+
VALUES (?, ?, ?, ?, ?)
|
|
142
|
+
ON CONFLICT(fleet_run_id) DO UPDATE SET
|
|
143
|
+
status = excluded.status, cursor = excluded.cursor,
|
|
144
|
+
last_seen_at = excluded.last_seen_at, last_error = excluded.last_error`)
|
|
145
|
+
.run(input.fleetRunId, input.status, input.cursor, input.lastSeenAt, input.lastError);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Point a binding at a new child Run and drop the mirrored window with it.
|
|
149
|
+
*
|
|
150
|
+
* Retry creates a NEW child Run, and the mirror is keyed on the child's own (run, sequence) pair -- so a
|
|
151
|
+
* second child Run would collide with the first one's sequences rather than continue them. Clearing is the
|
|
152
|
+
* honest option: the old events still live on the child under the old id, and the new Run's log is rebuilt
|
|
153
|
+
* from sequence 1. Doing neither would serve an interleaving of two Runs' events as if they were one.
|
|
154
|
+
*/
|
|
155
|
+
rebind(fleetRunId, childRunId) {
|
|
156
|
+
const now = new Date().toISOString();
|
|
157
|
+
// One transaction. Three separate statements here could leave a binding pointing at a new child Run while
|
|
158
|
+
// the mirror still holds the OLD Run's sequences -- which is worse than either half failing, because the
|
|
159
|
+
// two windows are indistinguishable in the table: both are (run, sequence) rows starting at 1.
|
|
160
|
+
this.db.exec('BEGIN');
|
|
161
|
+
try {
|
|
162
|
+
this.db.prepare('DELETE FROM fleet_events WHERE fleet_run_id = ?').run(fleetRunId);
|
|
163
|
+
this.db
|
|
164
|
+
.prepare('UPDATE fleet_runs SET child_run_id = ?, bound_at = ? WHERE fleet_run_id = ?')
|
|
165
|
+
.run(childRunId, now, fleetRunId);
|
|
166
|
+
this.db
|
|
167
|
+
.prepare('UPDATE run_state SET cursor = 0, events_drained = 0 WHERE fleet_run_id = ?')
|
|
168
|
+
.run(fleetRunId);
|
|
169
|
+
this.db.exec('COMMIT');
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
this.db.exec('ROLLBACK');
|
|
173
|
+
throw err;
|
|
174
|
+
}
|
|
175
|
+
return this.get(fleetRunId);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Record how far a Run's event log has been read, touching nothing else.
|
|
179
|
+
*
|
|
180
|
+
* Status and staleness are deliberately absent: mirroring and reconciliation are separate concerns with
|
|
181
|
+
* separate failure modes, and a successful event read implies nothing about the Run's state. Going through
|
|
182
|
+
* recordState() would overwrite a status reconciliation had just decided.
|
|
183
|
+
*
|
|
184
|
+
* `drained` reports what the last pass saw rather than being one-way. A terminal Run that still owes a log
|
|
185
|
+
* is re-read regardless of this value, so an occasional false costs one extra read while a false `true`
|
|
186
|
+
* would abandon a log forever -- which is why mirrorEvents distinguishes the two.
|
|
187
|
+
*/
|
|
188
|
+
setCursor(fleetRunId, cursor, drained) {
|
|
189
|
+
this.db
|
|
190
|
+
.prepare(`INSERT INTO run_state (fleet_run_id, status, cursor, last_seen_at, last_error, events_drained)
|
|
191
|
+
VALUES (?, ?, ?, NULL, NULL, ?)
|
|
192
|
+
ON CONFLICT(fleet_run_id) DO UPDATE SET
|
|
193
|
+
cursor = excluded.cursor,
|
|
194
|
+
events_drained = excluded.events_drained`)
|
|
195
|
+
.run(fleetRunId, UNKNOWN, cursor, drained ? 1 : 0);
|
|
196
|
+
}
|
|
197
|
+
state(fleetRunId) {
|
|
198
|
+
const row = this.db.prepare('SELECT * FROM run_state WHERE fleet_run_id = ?').get(fleetRunId);
|
|
199
|
+
return row ? toState(row) : null;
|
|
200
|
+
}
|
|
201
|
+
}
|
package/dist/child.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slice of a child Mercury's HTTP API that Fleet uses.
|
|
3
|
+
*
|
|
4
|
+
* Only the enumerated endpoints (design section 9: Fleet must not proxy arbitrary paths). A caller can never
|
|
5
|
+
* widen this surface; it is fixed here, so a child growing an internal route does not hand Fleet a new way to
|
|
6
|
+
* be pointed at it.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Describe a transport failure usefully.
|
|
10
|
+
*
|
|
11
|
+
* undici wraps everything as `fetch failed` and puts the actual reason in `cause`, so reporting only the
|
|
12
|
+
* message tells an operator nothing about why a host is unreachable. The distinction they need -- refused,
|
|
13
|
+
* timed out, name not resolving, TLS -- lives in the cause.
|
|
14
|
+
*/
|
|
15
|
+
function describeTransportError(err) {
|
|
16
|
+
const e = err;
|
|
17
|
+
const detail = e.cause?.code ?? e.cause?.message;
|
|
18
|
+
return `${e.message}${detail ? `: ${detail}` : ''}`.slice(0, 200);
|
|
19
|
+
}
|
|
20
|
+
async function call(opts, url, init) {
|
|
21
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
22
|
+
let res;
|
|
23
|
+
try {
|
|
24
|
+
res = await doFetch(url, { ...init, signal: AbortSignal.timeout(opts.timeoutMs) });
|
|
25
|
+
}
|
|
26
|
+
catch (err) {
|
|
27
|
+
// Transport failure after a POST means the request may have been fully processed. The caller must not
|
|
28
|
+
// conclude anything from this except "ask again later".
|
|
29
|
+
return { kind: 'unknown', reason: describeTransportError(err) };
|
|
30
|
+
}
|
|
31
|
+
if (res.status >= 500) {
|
|
32
|
+
return { kind: 'unknown', reason: `HTTP ${res.status} from child` };
|
|
33
|
+
}
|
|
34
|
+
if (res.status >= 400) {
|
|
35
|
+
let detail = '';
|
|
36
|
+
try {
|
|
37
|
+
const body = await res.json();
|
|
38
|
+
detail = body.error ?? '';
|
|
39
|
+
}
|
|
40
|
+
catch { /* body was not JSON; the status still stands */ }
|
|
41
|
+
return { kind: 'rejected', status: res.status, detail: detail.slice(0, 300) };
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
return { kind: 'ok', value: await res.json() };
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
// A 2xx with an unreadable body is genuinely ambiguous for a create: the Run may exist.
|
|
48
|
+
return { kind: 'unknown', reason: `child response was not JSON: ${err.message}`.slice(0, 200) };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Same classification as `call`, but the success body is text.
|
|
53
|
+
*
|
|
54
|
+
* Kept separate rather than parameterised because the failure meanings differ: a 2xx with an unreadable JSON
|
|
55
|
+
* body after a POST is ambiguous about whether a Run was created, whereas a body that cannot be read here is
|
|
56
|
+
* simply a scrape that produced nothing usable.
|
|
57
|
+
*/
|
|
58
|
+
async function callText(opts, url, init) {
|
|
59
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
60
|
+
let res;
|
|
61
|
+
try {
|
|
62
|
+
res = await doFetch(url, { ...init, signal: AbortSignal.timeout(opts.timeoutMs) });
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
return { kind: 'unknown', reason: describeTransportError(err) };
|
|
66
|
+
}
|
|
67
|
+
if (res.status >= 500)
|
|
68
|
+
return { kind: 'unknown', reason: `HTTP ${res.status} from child` };
|
|
69
|
+
if (res.status >= 400)
|
|
70
|
+
return { kind: 'rejected', status: res.status, detail: `HTTP ${res.status}` };
|
|
71
|
+
try {
|
|
72
|
+
return { kind: 'ok', value: await res.text() };
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
return { kind: 'unknown', reason: `child metrics body unreadable: ${err.message}`.slice(0, 200) };
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
export function createChildClient(opts) {
|
|
79
|
+
const headers = (token) => ({
|
|
80
|
+
authorization: `Bearer ${token}`, accept: 'application/json',
|
|
81
|
+
});
|
|
82
|
+
return {
|
|
83
|
+
async createRun(host, payload, idempotencyKey) {
|
|
84
|
+
return call(opts, `${host.baseUrl}/api/runs`, {
|
|
85
|
+
method: 'POST',
|
|
86
|
+
headers: { ...headers(host.token), 'content-type': 'application/json', 'idempotency-key': idempotencyKey },
|
|
87
|
+
body: JSON.stringify(payload),
|
|
88
|
+
});
|
|
89
|
+
},
|
|
90
|
+
async getRun(host, runId) {
|
|
91
|
+
// The id comes from Fleet's own binding table, never from a caller, so this cannot be aimed at an
|
|
92
|
+
// arbitrary path on a child.
|
|
93
|
+
const safe = encodeURIComponent(runId);
|
|
94
|
+
const res = await call(opts, `${host.baseUrl}/api/runs/${safe}`, { method: 'GET', headers: headers(host.token) });
|
|
95
|
+
if (res.kind !== 'ok')
|
|
96
|
+
return res;
|
|
97
|
+
// Unwrap here rather than at every call site. Reading `.status` off the envelope yields undefined,
|
|
98
|
+
// which then reaches a SQLite bind as a 500 -- which is exactly what this line used to do, because
|
|
99
|
+
// the fake child in the tests had the same wrong shape as this client.
|
|
100
|
+
if (!res.value || typeof res.value.run !== 'object' || res.value.run === null) {
|
|
101
|
+
return { kind: 'unknown', reason: 'child run response had no run object' };
|
|
102
|
+
}
|
|
103
|
+
return { kind: 'ok', value: res.value.run };
|
|
104
|
+
},
|
|
105
|
+
async submitInput(host, runId, input) {
|
|
106
|
+
const safe = encodeURIComponent(runId);
|
|
107
|
+
return call(opts, `${host.baseUrl}/api/runs/${safe}/input`, {
|
|
108
|
+
method: 'POST', headers: { ...headers(host.token), 'content-type': 'application/json' },
|
|
109
|
+
body: JSON.stringify({ input }),
|
|
110
|
+
});
|
|
111
|
+
},
|
|
112
|
+
async cancelRun(host, runId) {
|
|
113
|
+
const safe = encodeURIComponent(runId);
|
|
114
|
+
return call(opts, `${host.baseUrl}/api/runs/${safe}/cancel`, { method: 'POST', headers: headers(host.token) });
|
|
115
|
+
},
|
|
116
|
+
async retryRun(host, runId) {
|
|
117
|
+
const safe = encodeURIComponent(runId);
|
|
118
|
+
return call(opts, `${host.baseUrl}/api/runs/${safe}/retry`, { method: 'POST', headers: headers(host.token) });
|
|
119
|
+
},
|
|
120
|
+
async getMetrics(host) {
|
|
121
|
+
// Text, not JSON: the value is returned verbatim rather than parsed here, so an unexpected metric family
|
|
122
|
+
// reaches the merge layer intact and can be reported as dropped instead of vanishing.
|
|
123
|
+
return callText(opts, `${host.baseUrl}/metrics`, {
|
|
124
|
+
method: 'GET',
|
|
125
|
+
// The shared helper asks for JSON because every other endpoint returns it. Asking a metrics endpoint
|
|
126
|
+
// for JSON is a latent trap: harmless while the child ignores Accept, wrong the day it honours it.
|
|
127
|
+
headers: { ...headers(host.token), accept: 'text/plain' },
|
|
128
|
+
});
|
|
129
|
+
},
|
|
130
|
+
async getEvents(host, runId, after, limit) {
|
|
131
|
+
const safe = encodeURIComponent(runId);
|
|
132
|
+
const url = `${host.baseUrl}/api/runs/${safe}/events?after=${encodeURIComponent(String(after))}`
|
|
133
|
+
+ `&limit=${encodeURIComponent(String(limit))}`;
|
|
134
|
+
return call(opts, url, { method: 'GET', headers: headers(host.token) });
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|