@ours.network/fleet 0.10.2 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/dist/application/capabilities.d.ts +6 -0
- package/dist/application/capabilities.js +37 -0
- package/dist/application/errors.d.ts +31 -0
- package/dist/application/errors.js +51 -0
- package/dist/application/fleet-query-service.d.ts +31 -0
- package/dist/application/fleet-query-service.js +180 -0
- package/dist/application/log-service.d.ts +28 -0
- package/dist/application/log-service.js +146 -0
- package/dist/application/role-command-service.d.ts +37 -0
- package/dist/application/role-command-service.js +82 -0
- package/dist/application/role-creation-service.d.ts +142 -0
- package/dist/application/role-creation-service.js +374 -0
- package/dist/application/role-repository.d.ts +20 -0
- package/dist/application/role-repository.js +168 -0
- package/dist/application/session-control.d.ts +55 -0
- package/dist/application/session-control.js +115 -0
- package/dist/application/types.d.ts +156 -0
- package/dist/application/types.js +1 -0
- package/dist/cli.js +141 -0
- package/dist/config.d.ts +7 -2
- package/dist/config.js +18 -4
- package/dist/creation.d.ts +11 -0
- package/dist/creation.js +22 -5
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +34 -0
- package/dist/index.d.ts +10 -1
- package/dist/index.js +9 -1
- package/dist/runner.js +10 -2
- package/dist/session/acp.js +5 -1
- package/dist/session/control.d.ts +4 -2
- package/dist/session/control.js +45 -13
- package/dist/spawn.d.ts +20 -2
- package/dist/spawn.js +94 -24
- package/dist/supervisor/launchd.js +17 -0
- package/dist/supervisor/none.js +17 -0
- package/dist/supervisor/systemd.js +4 -0
- package/dist/supervisor/types.d.ts +6 -0
- package/dist/tmux.d.ts +2 -0
- package/dist/tmux.js +8 -0
- package/dist/web/audit.d.ts +22 -0
- package/dist/web/audit.js +54 -0
- package/dist/web/auth.d.ts +61 -0
- package/dist/web/auth.js +186 -0
- package/dist/web/control.d.ts +14 -0
- package/dist/web/control.js +110 -0
- package/dist/web/device-store.d.ts +27 -0
- package/dist/web/device-store.js +155 -0
- package/dist/web/events.d.ts +15 -0
- package/dist/web/events.js +34 -0
- package/dist/web/lock.d.ts +5 -0
- package/dist/web/lock.js +69 -0
- package/dist/web/runtime.d.ts +12 -0
- package/dist/web/runtime.js +170 -0
- package/dist/web/server.d.ts +35 -0
- package/dist/web/server.js +261 -0
- package/dist/web/service.d.ts +42 -0
- package/dist/web/service.js +180 -0
- package/dist/web/terminal/bridge.d.ts +27 -0
- package/dist/web/terminal/bridge.js +317 -0
- package/dist/web-app/assets/TerminalView-DcImdrI1.js +9 -0
- package/dist/web-app/assets/index-BokQN1Ao.js +9 -0
- package/dist/web-app/assets/index-lAXzaOZM.css +1 -0
- package/dist/web-app/icons/ours-fleet-maskable.svg +4 -0
- package/dist/web-app/icons/ours-fleet.svg +4 -0
- package/dist/web-app/index.html +17 -0
- package/dist/web-app/manifest.webmanifest +15 -0
- package/dist/web-app/offline.html +18 -0
- package/dist/web-app/sw.js +51 -0
- package/package.json +26 -3
package/README.md
CHANGED
|
@@ -128,6 +128,72 @@ ours-fleet spawn Coder --harness codex --model gpt-5.4 \
|
|
|
128
128
|
# wake owner separately in fleet.yaml with monitor.mode: fleet|native.
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
## Local web console
|
|
132
|
+
|
|
133
|
+
The interactive console is packaged with `@ours.network/fleet` and runs only on
|
|
134
|
+
IPv4 loopback:
|
|
135
|
+
|
|
136
|
+
```sh
|
|
137
|
+
npm run build
|
|
138
|
+
ours-fleet web
|
|
139
|
+
# choose a free port for an isolated test:
|
|
140
|
+
ours-fleet web serve --port 0 --no-open
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The normal `ours-fleet web` command installs or updates an owner-level native
|
|
144
|
+
service, starts it on stable `127.0.0.1:49271`, and opens/re-pairs the browser.
|
|
145
|
+
Linux uses a systemd user unit and macOS uses a LaunchAgent; both restart after
|
|
146
|
+
process failures and send logs to the native supervisor. Service management is
|
|
147
|
+
explicit through `ours-fleet web install|start|stop|restart|status|uninstall`.
|
|
148
|
+
Use `ours-fleet web serve` for a foreground development process. Linux login
|
|
149
|
+
persistence requires linger; the installer reports when it is missing but
|
|
150
|
+
never enables it or requests root privileges.
|
|
151
|
+
|
|
152
|
+
The normal command opens a one-time five-minute fragment directly in the local
|
|
153
|
+
browser; it does not print a reusable secret. If the browser needs pairing
|
|
154
|
+
again while the server is running, use `ours-fleet web open`. The fragment is
|
|
155
|
+
exchanged once and removed from history. A paired browser receives a rotating,
|
|
156
|
+
30-day `HttpOnly; SameSite=Strict` device credential and can return after a
|
|
157
|
+
session idle timeout or server restart. Use **Sign out** to revoke the current
|
|
158
|
+
browser, or `ours-fleet web revoke-all` to revoke every trusted browser and
|
|
159
|
+
active web session.
|
|
160
|
+
|
|
161
|
+
Only a domain-separated SHA-256 device-secret hash and bounded timestamps are
|
|
162
|
+
stored in the owner-private fleet state directory (`0700` directory, `0600`
|
|
163
|
+
atomic file). The re-pair and revoke controls use an owner-private Unix socket;
|
|
164
|
+
local processes running as the same OS user are therefore inside the trust
|
|
165
|
+
boundary. Keep the console local: it has no `--host`, proxy, TLS, or
|
|
166
|
+
remote-access mode.
|
|
167
|
+
|
|
168
|
+
The console is installable as a standalone PWA. Its service worker caches only
|
|
169
|
+
the data-free offline page and successful content-hashed JavaScript/CSS assets.
|
|
170
|
+
HTML stays network-first and no API, authentication, bootstrap, device,
|
|
171
|
+
WebSocket, terminal, event, audit, log, session, query-bearing, or error response
|
|
172
|
+
is cached. When the local daemon is unavailable, the PWA shows an explicit
|
|
173
|
+
offline shell and no stale fleet state.
|
|
174
|
+
|
|
175
|
+
The console provides evidence-separated inventory and status, ACP activity and
|
|
176
|
+
permission controls, redacted logs, typed text send, confirmed lifecycle
|
|
177
|
+
actions, transactional permanent/temporary creation, and a shared tmux browser
|
|
178
|
+
terminal. Identity is fixed to the role name. Creation uses the current
|
|
179
|
+
authenticated identity existence check and reports verified, missing, or
|
|
180
|
+
unknown evidence; a newly launched harness follows its generated first-boot
|
|
181
|
+
instructions to choose or create and bind the identity. The console never
|
|
182
|
+
claims that the host created an identity and never deletes one.
|
|
183
|
+
`node-pty` is optional: if its native module cannot load, ACP and all
|
|
184
|
+
non-terminal features remain available and tmux Terminal is disabled with a
|
|
185
|
+
diagnostic.
|
|
186
|
+
|
|
187
|
+
Security boundaries:
|
|
188
|
+
|
|
189
|
+
- exact runtime `Host` and `Origin`, CSRF, one-time WebSocket tickets, and
|
|
190
|
+
loopback binding are enforced server-side;
|
|
191
|
+
- cwd values must resolve beneath configured local roots;
|
|
192
|
+
- terminal bytes are intentionally unredacted and are never copied into audit
|
|
193
|
+
records; normal logs are bounded and redacted;
|
|
194
|
+
- tests use temporary fleet homes, fake supervisors/identity providers, and
|
|
195
|
+
isolated tmux sockets—never active role state.
|
|
196
|
+
|
|
131
197
|
Permanent spawns are written to `~/fleet.d/<Name>.yaml` — your hand-written
|
|
132
198
|
`~/fleet.yaml` is **never** machine-edited. `ours-fleet rm <Name>` unspawns.
|
|
133
199
|
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { RoleRecord, RoleCapabilities, RoleStatus } from './types.js';
|
|
2
|
+
export interface CapabilityContext {
|
|
3
|
+
terminalPtyAvailable?: boolean;
|
|
4
|
+
controlProtocolVersion?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function roleCapabilities(role: RoleRecord, status: RoleStatus, context?: CapabilityContext): RoleCapabilities;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export function roleCapabilities(role, status, context = {}) {
|
|
2
|
+
const backend = role.configuredBackend ?? (role.detectedBackend === 'acp' || role.detectedBackend === 'tmux' ? role.detectedBackend : undefined);
|
|
3
|
+
const online = status.session.reachability === 'online';
|
|
4
|
+
const acp = backend === 'acp';
|
|
5
|
+
const tmux = backend === 'tmux';
|
|
6
|
+
const protocolVersion = context.controlProtocolVersion ?? 1;
|
|
7
|
+
const inactive = status.overall === 'offline'
|
|
8
|
+
|| (!online && status.supervisor.liveness === 'stopped');
|
|
9
|
+
const terminalAvailable = tmux && online && context.terminalPtyAvailable === true;
|
|
10
|
+
const terminalReason = terminalAvailable ? undefined
|
|
11
|
+
: !tmux ? 'wrong_backend'
|
|
12
|
+
: !online ? 'offline'
|
|
13
|
+
: 'pty_unavailable';
|
|
14
|
+
return {
|
|
15
|
+
protocolVersion,
|
|
16
|
+
inventory: true,
|
|
17
|
+
status: true,
|
|
18
|
+
output: {
|
|
19
|
+
recent: acp || tmux, stream: (acp && protocolVersion >= 2) || terminalAvailable,
|
|
20
|
+
structured: acp, replayCursor: acp && protocolVersion >= 2,
|
|
21
|
+
},
|
|
22
|
+
input: { text: online && (acp || tmux), rawKeys: terminalAvailable, interrupt: online && acp, steering: false },
|
|
23
|
+
permissions: { observe: acp, respond: acp && online },
|
|
24
|
+
terminal: {
|
|
25
|
+
available: terminalAvailable, reason: terminalReason,
|
|
26
|
+
multiViewer: terminalAvailable, writerLease: terminalAvailable,
|
|
27
|
+
},
|
|
28
|
+
lifecycle: {
|
|
29
|
+
start: role.lifetime === 'permanent' && status.supervisor.liveness !== 'running',
|
|
30
|
+
stop: role.lifetime === 'permanent' && !inactive && status.supervisor.liveness !== 'stopped',
|
|
31
|
+
restartResume: role.lifetime === 'permanent' && !inactive,
|
|
32
|
+
restartFresh: role.lifetime === 'permanent' && !inactive,
|
|
33
|
+
remove: false,
|
|
34
|
+
},
|
|
35
|
+
logs: { tail: true, follow: status.supervisor.backend !== 'none', cursor: status.supervisor.backend !== 'none' },
|
|
36
|
+
};
|
|
37
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type FleetErrorCode = 'role_not_found' | 'capability_unavailable' | 'invalid_request' | 'offline' | 'control_unavailable' | 'timeout' | 'rejected' | 'backend_failure' | 'conflict' | 'stale_state' | 'prerequisite_unavailable' | 'idempotency_conflict' | 'rollback_incomplete' | 'launched_unconfirmed' | 'unauthorized' | 'forbidden' | 'rate_limited' | 'incompatible_version' | 'internal';
|
|
2
|
+
export interface FleetErrorShape {
|
|
3
|
+
code: FleetErrorCode;
|
|
4
|
+
message: string;
|
|
5
|
+
provesOffline: boolean;
|
|
6
|
+
retryable: boolean;
|
|
7
|
+
requestId: string;
|
|
8
|
+
details?: Record<string, string | number | boolean>;
|
|
9
|
+
}
|
|
10
|
+
export declare class FleetError extends Error implements FleetErrorShape {
|
|
11
|
+
readonly code: FleetErrorCode;
|
|
12
|
+
readonly options: {
|
|
13
|
+
provesOffline?: boolean;
|
|
14
|
+
retryable?: boolean;
|
|
15
|
+
requestId?: string;
|
|
16
|
+
details?: Record<string, string | number | boolean>;
|
|
17
|
+
};
|
|
18
|
+
readonly requestId: string;
|
|
19
|
+
constructor(code: FleetErrorCode, message: string, options?: {
|
|
20
|
+
provesOffline?: boolean;
|
|
21
|
+
retryable?: boolean;
|
|
22
|
+
requestId?: string;
|
|
23
|
+
details?: Record<string, string | number | boolean>;
|
|
24
|
+
});
|
|
25
|
+
get provesOffline(): boolean;
|
|
26
|
+
get retryable(): boolean;
|
|
27
|
+
get details(): Record<string, string | number | boolean> | undefined;
|
|
28
|
+
toJSON(): FleetErrorShape;
|
|
29
|
+
}
|
|
30
|
+
export declare function safeLine(input: string, max?: number): string;
|
|
31
|
+
export declare function normalizeError(error: unknown, requestId?: string): FleetError;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { SessionControlError } from '../session/types.js';
|
|
3
|
+
import { CreationConflictError } from '../creation.js';
|
|
4
|
+
const CONTROL_CODES = {
|
|
5
|
+
offline: 'offline',
|
|
6
|
+
'control-unavailable': 'control_unavailable',
|
|
7
|
+
timeout: 'timeout',
|
|
8
|
+
rejected: 'rejected',
|
|
9
|
+
backend: 'backend_failure',
|
|
10
|
+
};
|
|
11
|
+
export class FleetError extends Error {
|
|
12
|
+
code;
|
|
13
|
+
options;
|
|
14
|
+
requestId;
|
|
15
|
+
constructor(code, message, options = {}) {
|
|
16
|
+
super(safeLine(message));
|
|
17
|
+
this.code = code;
|
|
18
|
+
this.options = options;
|
|
19
|
+
this.name = 'FleetError';
|
|
20
|
+
this.requestId = options.requestId ?? randomUUID();
|
|
21
|
+
}
|
|
22
|
+
get provesOffline() { return this.options.provesOffline ?? this.code === 'offline'; }
|
|
23
|
+
get retryable() {
|
|
24
|
+
return this.options.retryable
|
|
25
|
+
?? ['timeout', 'control_unavailable', 'backend_failure', 'launched_unconfirmed'].includes(this.code);
|
|
26
|
+
}
|
|
27
|
+
get details() { return this.options.details; }
|
|
28
|
+
toJSON() {
|
|
29
|
+
return {
|
|
30
|
+
code: this.code, message: this.message, provesOffline: this.provesOffline,
|
|
31
|
+
retryable: this.retryable, requestId: this.requestId, details: this.details,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export function safeLine(input, max = 512) {
|
|
36
|
+
return input.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').replace(/\s+/g, ' ').trim().slice(0, max);
|
|
37
|
+
}
|
|
38
|
+
export function normalizeError(error, requestId) {
|
|
39
|
+
if (error instanceof FleetError)
|
|
40
|
+
return error;
|
|
41
|
+
if (error instanceof CreationConflictError)
|
|
42
|
+
return new FleetError('conflict', error.message, { requestId, retryable: true });
|
|
43
|
+
if (error instanceof SessionControlError) {
|
|
44
|
+
const code = CONTROL_CODES[error.kind];
|
|
45
|
+
return new FleetError(code, error.message, {
|
|
46
|
+
requestId, provesOffline: error.kind === 'offline',
|
|
47
|
+
retryable: error.kind === 'timeout' || error.kind === 'control-unavailable' || error.kind === 'backend',
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return new FleetError('internal', error instanceof Error ? error.message : String(error), { requestId });
|
|
51
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { SupervisorBackend } from '../supervisor/types.js';
|
|
2
|
+
import { controlRequest } from '../session/control.js';
|
|
3
|
+
import { Tmux } from '../tmux.js';
|
|
4
|
+
import { type CapabilityContext } from './capabilities.js';
|
|
5
|
+
import { RoleRepository } from './role-repository.js';
|
|
6
|
+
import type { RoleCapabilities, RoleRecord, RoleStatus } from './types.js';
|
|
7
|
+
export interface FleetQueryOptions {
|
|
8
|
+
repository: RoleRepository;
|
|
9
|
+
supervisor: SupervisorBackend;
|
|
10
|
+
tmux?: Tmux;
|
|
11
|
+
control?: typeof controlRequest;
|
|
12
|
+
capabilityContext?: CapabilityContext;
|
|
13
|
+
}
|
|
14
|
+
export declare class FleetQueryService {
|
|
15
|
+
private readonly options;
|
|
16
|
+
private readonly tmux;
|
|
17
|
+
private readonly control;
|
|
18
|
+
constructor(options: FleetQueryOptions);
|
|
19
|
+
list(): Promise<Array<{
|
|
20
|
+
role: RoleRecord;
|
|
21
|
+
status: RoleStatus;
|
|
22
|
+
capabilities: RoleCapabilities;
|
|
23
|
+
}>>;
|
|
24
|
+
detail(id: string): Promise<{
|
|
25
|
+
role: RoleRecord;
|
|
26
|
+
status: RoleStatus;
|
|
27
|
+
capabilities: RoleCapabilities;
|
|
28
|
+
}>;
|
|
29
|
+
status(role: RoleRecord): Promise<RoleStatus>;
|
|
30
|
+
private session;
|
|
31
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { lstatSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { controlRequest } from '../session/control.js';
|
|
4
|
+
import { SessionControlError } from '../session/types.js';
|
|
5
|
+
import { readExitRecord, readRestartLedger } from '../runner.js';
|
|
6
|
+
import { Tmux } from '../tmux.js';
|
|
7
|
+
import { roleCapabilities } from './capabilities.js';
|
|
8
|
+
import { FleetError } from './errors.js';
|
|
9
|
+
const clean = (value, max = 512) => value.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').trim().slice(0, max);
|
|
10
|
+
function boundedText(path, max = 16 * 1024) {
|
|
11
|
+
try {
|
|
12
|
+
const stat = lstatSync(path);
|
|
13
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > max)
|
|
14
|
+
return undefined;
|
|
15
|
+
return readFileSync(path, 'utf8');
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function readMonitor(dir, now = Date.now()) {
|
|
22
|
+
const owner = clean(boundedText(join(dir, '.monitor-owner'), 64) ?? '');
|
|
23
|
+
const raw = clean(boundedText(join(dir, '.monitor-status')) ?? '');
|
|
24
|
+
const statAt = (() => {
|
|
25
|
+
try {
|
|
26
|
+
return lstatSync(join(dir, '.monitor-status')).mtime.toISOString();
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
})();
|
|
32
|
+
const lower = raw.toLowerCase();
|
|
33
|
+
const health = /fail|error/.test(lower) ? 'failed'
|
|
34
|
+
: /degrad|api error/.test(lower) ? 'degraded'
|
|
35
|
+
: raw ? 'armed' : 'unknown';
|
|
36
|
+
const stale = statAt ? now - Date.parse(statAt) > 60_000 : true;
|
|
37
|
+
return {
|
|
38
|
+
mode: owner === 'fleet' || owner === 'native' ? owner : 'unknown',
|
|
39
|
+
health, updatedAt: statAt, detail: raw || undefined, stale,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function readIsolation(dir) {
|
|
43
|
+
const text = boundedText(join(dir, '.isolation-degraded'));
|
|
44
|
+
return text ? { degraded: true, detail: clean(text) } : { degraded: false };
|
|
45
|
+
}
|
|
46
|
+
function sessionOverall(supervisor, session, restart, monitor, isolation, problems) {
|
|
47
|
+
if (restart.circuit === 'open' || monitor.health === 'failed' || monitor.health === 'degraded'
|
|
48
|
+
|| isolation.degraded || problems.some(problem => problem.severity === 'error')
|
|
49
|
+
|| session.readiness === 'failed')
|
|
50
|
+
return 'attention';
|
|
51
|
+
// A reachable ACP/tmux session is the user's live interaction surface. Its
|
|
52
|
+
// evidence remains authoritative even when a service manager no longer owns
|
|
53
|
+
// the process (for example, a detached ACP session with an inactive unit).
|
|
54
|
+
if (session.reachability === 'online'
|
|
55
|
+
&& (session.readiness === 'running' || session.readiness === 'awaiting_permission'))
|
|
56
|
+
return 'busy';
|
|
57
|
+
if (session.reachability === 'online' && session.readiness === 'idle')
|
|
58
|
+
return 'ready';
|
|
59
|
+
if (session.reachability === 'offline'
|
|
60
|
+
&& (session.evidence === 'authoritative' || supervisor.liveness === 'stopped'))
|
|
61
|
+
return 'offline';
|
|
62
|
+
return 'unknown';
|
|
63
|
+
}
|
|
64
|
+
export class FleetQueryService {
|
|
65
|
+
options;
|
|
66
|
+
tmux;
|
|
67
|
+
control;
|
|
68
|
+
constructor(options) {
|
|
69
|
+
this.options = options;
|
|
70
|
+
this.tmux = options.tmux ?? new Tmux();
|
|
71
|
+
this.control = options.control ?? controlRequest;
|
|
72
|
+
}
|
|
73
|
+
async list() {
|
|
74
|
+
const roles = await this.options.repository.list();
|
|
75
|
+
return Promise.all(roles.map(async (role) => {
|
|
76
|
+
const status = await this.status(role);
|
|
77
|
+
return { role, status, capabilities: roleCapabilities(role, status, this.options.capabilityContext) };
|
|
78
|
+
}));
|
|
79
|
+
}
|
|
80
|
+
async detail(id) {
|
|
81
|
+
const role = await this.options.repository.get(id);
|
|
82
|
+
if (!role)
|
|
83
|
+
throw new FleetError('role_not_found', `no such role '${id}'`, { provesOffline: false });
|
|
84
|
+
const status = await this.status(role);
|
|
85
|
+
return { role, status, capabilities: roleCapabilities(role, status, this.options.capabilityContext) };
|
|
86
|
+
}
|
|
87
|
+
async status(role) {
|
|
88
|
+
const observedAt = new Date().toISOString();
|
|
89
|
+
const live = await this.options.supervisor.liveness(role.id)
|
|
90
|
+
.catch(error => ({ state: 'unknown', detail: error.message }));
|
|
91
|
+
const dir = this.options.repository.stateDir(role);
|
|
92
|
+
const restart = dir ? readRestartLedger(dir) : {
|
|
93
|
+
circuit: 'closed', consecutiveImmediateFailures: 0, nextDelayMs: 0,
|
|
94
|
+
lastReason: '', openedAt: undefined,
|
|
95
|
+
};
|
|
96
|
+
const monitor = dir ? readMonitor(dir) : {
|
|
97
|
+
mode: 'unknown', health: 'unknown', stale: true,
|
|
98
|
+
};
|
|
99
|
+
const isolation = dir ? readIsolation(dir) : { degraded: false };
|
|
100
|
+
const lastExit = dir ? readExitRecord(join(dir, '.exit-status')) ?? undefined : undefined;
|
|
101
|
+
const session = await this.session(role, dir, live.state);
|
|
102
|
+
const problems = [...role.problems];
|
|
103
|
+
if (live.state === 'running' && session.reachability !== 'online')
|
|
104
|
+
problems.push({
|
|
105
|
+
code: 'supervisor_session_disagreement', severity: 'warning',
|
|
106
|
+
detail: 'supervisor is live but the session is not reachable',
|
|
107
|
+
});
|
|
108
|
+
if (role.lifetime === 'permanent' && live.state === 'stopped'
|
|
109
|
+
&& session.reachability === 'online')
|
|
110
|
+
problems.push({
|
|
111
|
+
code: 'session_supervisor_disagreement', severity: 'warning',
|
|
112
|
+
detail: 'session is live while its permanent supervisor service is inactive',
|
|
113
|
+
});
|
|
114
|
+
const supervisor = {
|
|
115
|
+
backend: this.options.supervisor.id, liveness: live.state,
|
|
116
|
+
nativeState: live.detail.split(/\s/)[0], detail: clean(live.detail),
|
|
117
|
+
};
|
|
118
|
+
return {
|
|
119
|
+
roleId: role.id, observedAt,
|
|
120
|
+
overall: sessionOverall(supervisor, session, restart, monitor, isolation, problems),
|
|
121
|
+
supervisor, session,
|
|
122
|
+
restart: {
|
|
123
|
+
circuit: restart.circuit,
|
|
124
|
+
consecutiveImmediateFailures: restart.consecutiveImmediateFailures,
|
|
125
|
+
nextDelayMs: restart.nextDelayMs,
|
|
126
|
+
lastReason: restart.lastReason || undefined,
|
|
127
|
+
openedAt: restart.openedAt,
|
|
128
|
+
},
|
|
129
|
+
monitor, lastExit, isolation, problems,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
async session(role, dir, supervisor) {
|
|
133
|
+
const intended = role.configuredBackend;
|
|
134
|
+
if (intended === 'acp' && dir) {
|
|
135
|
+
try {
|
|
136
|
+
const response = await this.control(dir, { command: 'snapshot' }, 2_000);
|
|
137
|
+
if (!response.ok)
|
|
138
|
+
throw new SessionControlError(response.kind ?? 'backend', response.error ?? 'snapshot failed');
|
|
139
|
+
const snapshot = response.result;
|
|
140
|
+
return {
|
|
141
|
+
backend: 'acp', reachability: snapshot.alive ? 'online' : 'offline',
|
|
142
|
+
readiness: snapshot.readiness, evidence: 'authoritative',
|
|
143
|
+
sessionId: snapshot.sessionId, lastError: snapshot.lastError,
|
|
144
|
+
pendingPermissionId: snapshot.pendingPermissionId,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
const failure = error instanceof SessionControlError ? error.kind : 'backend';
|
|
149
|
+
const offline = failure === 'offline';
|
|
150
|
+
return {
|
|
151
|
+
backend: 'acp',
|
|
152
|
+
reachability: offline ? 'offline'
|
|
153
|
+
: failure === 'control-unavailable' ? 'unavailable' : 'unknown',
|
|
154
|
+
readiness: offline ? 'failed' : 'unknown',
|
|
155
|
+
evidence: 'authoritative', lastError: clean(error.message),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (intended === 'tmux' || role.detectedBackend === 'tmux') {
|
|
160
|
+
try {
|
|
161
|
+
const has = await this.tmux.has(role.id);
|
|
162
|
+
return {
|
|
163
|
+
backend: 'tmux', reachability: has ? 'online' : supervisor === 'stopped' ? 'offline' : 'unknown',
|
|
164
|
+
readiness: has ? 'idle' : supervisor === 'running' ? 'starting' : 'failed',
|
|
165
|
+
evidence: 'inferred',
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
return {
|
|
170
|
+
backend: 'tmux', reachability: 'unknown', readiness: 'unknown',
|
|
171
|
+
evidence: 'inferred', lastError: clean(error.message),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
backend: 'unknown', reachability: supervisor === 'stopped' ? 'offline' : 'unknown',
|
|
177
|
+
readiness: supervisor === 'stopped' ? 'failed' : 'unknown', evidence: 'inferred',
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Exec } from '../exec.js';
|
|
2
|
+
import type { SupervisorBackend } from '../supervisor/types.js';
|
|
3
|
+
import type { LogPage, LogRecord } from './types.js';
|
|
4
|
+
export declare function redactLogLine(input: string, patterns?: RegExp[]): {
|
|
5
|
+
text: string;
|
|
6
|
+
redactionApplied: boolean;
|
|
7
|
+
};
|
|
8
|
+
export interface RoleLogSource {
|
|
9
|
+
tail(limit: number, cursor?: string): Promise<LogPage>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Presentation-only compaction for noisy historical monitor transport hiccups.
|
|
13
|
+
* Exact reason and adjacency are part of the key: a different or intervening
|
|
14
|
+
* record always starts a new run. Source journal/file records are untouched.
|
|
15
|
+
*/
|
|
16
|
+
export declare function compactMonitorStreamHiccups(records: LogRecord[]): LogRecord[];
|
|
17
|
+
export declare class StructuredLogService {
|
|
18
|
+
private readonly backend;
|
|
19
|
+
private readonly exec;
|
|
20
|
+
private readonly cursorKey;
|
|
21
|
+
constructor(backend: SupervisorBackend, exec?: Exec);
|
|
22
|
+
source(roleId: string): RoleLogSource;
|
|
23
|
+
private tail;
|
|
24
|
+
private cursor;
|
|
25
|
+
private validCursor;
|
|
26
|
+
}
|
|
27
|
+
/** Bounded launchd file reader used by tests and diagnostics. */
|
|
28
|
+
export declare function readKnownRoleLog(roleId: string, limit?: number): LogPage;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { lstatSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { createHmac, randomBytes } from 'node:crypto';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { realExec } from '../exec.js';
|
|
5
|
+
import { logsRoot } from '../paths.js';
|
|
6
|
+
import { unitFor } from '../supervisor/systemd.js';
|
|
7
|
+
import { ROLE_NAME_RE } from '../config.js';
|
|
8
|
+
import { FleetError } from './errors.js';
|
|
9
|
+
const STANDARD_REDACTIONS = [
|
|
10
|
+
/\b(authorization\s*:\s*(?:bearer|basic)\s+)[^\s]+/gi,
|
|
11
|
+
/\b((?:api[_-]?key|token|secret|password)\s*[=:]\s*)[^\s,;]+/gi,
|
|
12
|
+
/-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+PRIVATE KEY-----/g,
|
|
13
|
+
/\bours_[A-Za-z0-9_-]{16,}\b/g,
|
|
14
|
+
];
|
|
15
|
+
export function redactLogLine(input, patterns = []) {
|
|
16
|
+
let text = input.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').slice(0, 16 * 1024);
|
|
17
|
+
const before = text;
|
|
18
|
+
for (const pattern of [...STANDARD_REDACTIONS, ...patterns.slice(0, 20)])
|
|
19
|
+
text = text.replace(pattern, (_match, prefix) => `${prefix ?? ''}[REDACTED]`);
|
|
20
|
+
return { text, redactionApplied: text !== before };
|
|
21
|
+
}
|
|
22
|
+
const MONITOR_STREAM_HICCUP = /^monitor degraded:\s*stream hiccup\s*\((.+)\)\s*$/;
|
|
23
|
+
/**
|
|
24
|
+
* Presentation-only compaction for noisy historical monitor transport hiccups.
|
|
25
|
+
* Exact reason and adjacency are part of the key: a different or intervening
|
|
26
|
+
* record always starts a new run. Source journal/file records are untouched.
|
|
27
|
+
*/
|
|
28
|
+
export function compactMonitorStreamHiccups(records) {
|
|
29
|
+
const result = [];
|
|
30
|
+
for (let index = 0; index < records.length;) {
|
|
31
|
+
const first = records[index];
|
|
32
|
+
const match = MONITOR_STREAM_HICCUP.exec(first.text);
|
|
33
|
+
if (!match) {
|
|
34
|
+
result.push(first);
|
|
35
|
+
index++;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const reason = match[1];
|
|
39
|
+
let end = index + 1;
|
|
40
|
+
while (end < records.length) {
|
|
41
|
+
const candidate = MONITOR_STREAM_HICCUP.exec(records[end].text);
|
|
42
|
+
if (!candidate || candidate[1] !== reason)
|
|
43
|
+
break;
|
|
44
|
+
end++;
|
|
45
|
+
}
|
|
46
|
+
if (end - index === 1) {
|
|
47
|
+
result.push(first);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
const last = records[end - 1];
|
|
51
|
+
result.push({
|
|
52
|
+
at: first.at,
|
|
53
|
+
text: 'monitor degraded: stream hiccup',
|
|
54
|
+
cursor: last.cursor,
|
|
55
|
+
redactionApplied: records.slice(index, end).some(record => record.redactionApplied),
|
|
56
|
+
compacted: {
|
|
57
|
+
kind: 'monitor_stream_hiccup', reason, count: end - index,
|
|
58
|
+
firstAt: first.at, lastAt: last.at,
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
index = end;
|
|
63
|
+
}
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
export class StructuredLogService {
|
|
67
|
+
backend;
|
|
68
|
+
exec;
|
|
69
|
+
cursorKey = randomBytes(32);
|
|
70
|
+
constructor(backend, exec = realExec) {
|
|
71
|
+
this.backend = backend;
|
|
72
|
+
this.exec = exec;
|
|
73
|
+
}
|
|
74
|
+
source(roleId) {
|
|
75
|
+
if (!ROLE_NAME_RE.test(roleId))
|
|
76
|
+
throw new FleetError('invalid_request', 'invalid role id');
|
|
77
|
+
return { tail: (limit, cursor) => this.tail(roleId, limit, cursor) };
|
|
78
|
+
}
|
|
79
|
+
async tail(roleId, requested, cursor) {
|
|
80
|
+
const limit = Math.min(Math.max(requested, 1), 1_000);
|
|
81
|
+
const args = this.backend.id === 'systemd'
|
|
82
|
+
? {
|
|
83
|
+
cmd: 'journalctl',
|
|
84
|
+
args: ['--user', '-u', unitFor(roleId), '-o', 'json', '--no-pager', '-n', String(limit)],
|
|
85
|
+
}
|
|
86
|
+
: this.backend.logsArgs(roleId, false);
|
|
87
|
+
if (!['journalctl', 'tail', 'tmux'].includes(args.cmd))
|
|
88
|
+
throw new FleetError('capability_unavailable', 'log backend is unsupported');
|
|
89
|
+
const result = await this.exec(args.cmd, args.args);
|
|
90
|
+
if (result.code !== 0)
|
|
91
|
+
throw new FleetError('backend_failure', `log query failed with exit ${result.code}`);
|
|
92
|
+
const lines = result.stdout.split(/\r?\n/).filter(Boolean).slice(-limit);
|
|
93
|
+
const parsed = lines.map((line, index) => {
|
|
94
|
+
let text = line;
|
|
95
|
+
let at;
|
|
96
|
+
if (this.backend.id === 'systemd') {
|
|
97
|
+
try {
|
|
98
|
+
const json = JSON.parse(line);
|
|
99
|
+
text = typeof json.MESSAGE === 'string' ? json.MESSAGE : line;
|
|
100
|
+
if (typeof json.__REALTIME_TIMESTAMP === 'string') {
|
|
101
|
+
const micros = Number(json.__REALTIME_TIMESTAMP);
|
|
102
|
+
if (Number.isFinite(micros))
|
|
103
|
+
at = new Date(micros / 1000).toISOString();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
catch { /* preserve the bounded raw line */ }
|
|
107
|
+
}
|
|
108
|
+
const redacted = redactLogLine(text);
|
|
109
|
+
return {
|
|
110
|
+
at, ...redacted,
|
|
111
|
+
cursor: this.cursor(`${roleId}:${index}:${line.length}`),
|
|
112
|
+
};
|
|
113
|
+
});
|
|
114
|
+
const records = compactMonitorStreamHiccups(parsed);
|
|
115
|
+
return {
|
|
116
|
+
records, nextCursor: records.at(-1)?.cursor,
|
|
117
|
+
truncated: lines.length >= limit || Boolean(cursor && !this.validCursor(cursor)),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
cursor(value) {
|
|
121
|
+
const body = Buffer.from(value).toString('base64url');
|
|
122
|
+
const tag = createHmac('sha256', this.cursorKey).update(body).digest('base64url').slice(0, 16);
|
|
123
|
+
return `${body}.${tag}`;
|
|
124
|
+
}
|
|
125
|
+
validCursor(cursor) {
|
|
126
|
+
const [body, tag] = cursor.split('.');
|
|
127
|
+
return Boolean(body && tag && this.cursor(Buffer.from(body, 'base64url').toString()).split('.')[1] === tag);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/** Bounded launchd file reader used by tests and diagnostics. */
|
|
131
|
+
export function readKnownRoleLog(roleId, limit = 200) {
|
|
132
|
+
if (!ROLE_NAME_RE.test(roleId))
|
|
133
|
+
throw new FleetError('invalid_request', 'invalid role id');
|
|
134
|
+
const path = join(logsRoot(), `${roleId}.log`);
|
|
135
|
+
try {
|
|
136
|
+
const stat = lstatSync(path);
|
|
137
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 8 * 1024 * 1024)
|
|
138
|
+
throw new Error('unsafe or oversized log file');
|
|
139
|
+
const records = readFileSync(path, 'utf8').split(/\r?\n/).filter(Boolean).slice(-limit)
|
|
140
|
+
.map(line => ({ ...redactLogLine(line) }));
|
|
141
|
+
return { records, truncated: stat.size > 512 * 1024 };
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
throw new FleetError('backend_failure', `could not read role log: ${error.message}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type OpsDeps } from '../ops.js';
|
|
2
|
+
import { FleetError } from './errors.js';
|
|
3
|
+
import { RoleRepository } from './role-repository.js';
|
|
4
|
+
import type { RoleStatus } from './types.js';
|
|
5
|
+
export type LifecycleAction = 'start' | 'stop' | 'restart_resume' | 'restart_fresh';
|
|
6
|
+
export interface CommandReceipt {
|
|
7
|
+
actionId: string;
|
|
8
|
+
roleId: string;
|
|
9
|
+
action: LifecycleAction;
|
|
10
|
+
acceptedAt: string;
|
|
11
|
+
completedAt?: string;
|
|
12
|
+
state: 'accepted' | 'running' | 'succeeded' | 'failed' | 'uncertain';
|
|
13
|
+
error?: ReturnType<FleetError['toJSON']>;
|
|
14
|
+
postcondition?: Pick<RoleStatus, 'overall' | 'observedAt'>;
|
|
15
|
+
}
|
|
16
|
+
export interface RoleCommandOptions {
|
|
17
|
+
repository: RoleRepository;
|
|
18
|
+
ops: OpsDeps;
|
|
19
|
+
configPath?: string;
|
|
20
|
+
status(roleId: string): Promise<RoleStatus>;
|
|
21
|
+
onProgress?: (receipt: CommandReceipt) => void;
|
|
22
|
+
}
|
|
23
|
+
export declare class RoleCommandService {
|
|
24
|
+
private readonly options;
|
|
25
|
+
private readonly receipts;
|
|
26
|
+
private readonly locks;
|
|
27
|
+
constructor(options: RoleCommandOptions);
|
|
28
|
+
execute(input: {
|
|
29
|
+
roleId: string;
|
|
30
|
+
action: LifecycleAction;
|
|
31
|
+
actionId?: string;
|
|
32
|
+
confirmation?: string;
|
|
33
|
+
}): Promise<CommandReceipt>;
|
|
34
|
+
get(actionId: string): CommandReceipt | undefined;
|
|
35
|
+
private run;
|
|
36
|
+
private serial;
|
|
37
|
+
}
|