@abelxiaoxing/cadence 2.0.25 → 2.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/docs/conversation-mode.md +1 -1
- package/docs/remote-access.md +1 -1
- package/docs/service-management.md +62 -128
- package/package.json +3 -2
- package/src/cadence-mode.ts +18 -7
- package/src/native-build.ts +1 -1
- package/src/native-client.ts +1 -1
- package/src/operator-cli.mjs +80 -8
- package/src/service/checkout-lock.ts +7 -0
- package/src/service/coordination.ts +56 -364
- package/src/service/liveness.ts +238 -0
- package/src/service-cli.mjs +1582 -2828
- package/src/service-host.mjs +16984 -15885
- package/web/dist/assets/cadence.css +1 -1
- package/web/dist/assets/cadence.js +16 -16
- package/web/dist/manifest.json +7 -7
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/** Evidence that a lock holder can no longer act. Startup self-healing reclaims
|
|
2
|
+
* only what this module proves dead; it never guesses from age or a PID alone. */
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import { hostname, uptime } from "node:os";
|
|
6
|
+
|
|
7
|
+
/** Written into every new lock so a later process can prove its holder gone. */
|
|
8
|
+
export interface ProcessMark {
|
|
9
|
+
pid: number;
|
|
10
|
+
host: string;
|
|
11
|
+
/** Kernel boot identity; a different one proves every old process exited. */
|
|
12
|
+
boot?: string | null;
|
|
13
|
+
/** Platform start identity of `pid`; a different one proves PID reuse. */
|
|
14
|
+
started?: string | null;
|
|
15
|
+
}
|
|
16
|
+
export type HolderState =
|
|
17
|
+
/** The recorded process is running (or cannot be told apart from it). */
|
|
18
|
+
| "live"
|
|
19
|
+
/** Proven gone: different boot, absent PID or a reused PID. */
|
|
20
|
+
| "dead"
|
|
21
|
+
/** Recorded on another host; nothing here can observe it. */
|
|
22
|
+
| "foreign"
|
|
23
|
+
/** The PID exists but neither its identity nor its absence can be shown. */
|
|
24
|
+
| "unknown";
|
|
25
|
+
export interface Holder {
|
|
26
|
+
state: HolderState;
|
|
27
|
+
/** The holder belonged to an earlier boot of this machine. */
|
|
28
|
+
reboot: boolean;
|
|
29
|
+
}
|
|
30
|
+
/** Legacy locks carry no boot identity. Their file must predate this boot by
|
|
31
|
+
* this much before the boot time alone counts as proof (clock steps at boot). */
|
|
32
|
+
export const LEGACY_BOOT_MARGIN_MS = 10 * 60 * 1000;
|
|
33
|
+
/** A live PID that started this long after its lock was written is not the
|
|
34
|
+
* process that wrote it. */
|
|
35
|
+
const REUSE_SLACK_MS = 5000;
|
|
36
|
+
const LINUX_TICKS_PER_SECOND = 100;
|
|
37
|
+
|
|
38
|
+
let cachedBoot: string | null | undefined;
|
|
39
|
+
/** Stable for the life of this kernel boot; null where the platform has none. */
|
|
40
|
+
export function bootId(): string | null {
|
|
41
|
+
if (cachedBoot !== undefined) return cachedBoot;
|
|
42
|
+
cachedBoot = null;
|
|
43
|
+
try {
|
|
44
|
+
if (process.platform === "linux") {
|
|
45
|
+
const id = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
46
|
+
if (/^[0-9a-f-]{36}$/.test(id)) cachedBoot = id;
|
|
47
|
+
} else if (process.platform === "darwin") {
|
|
48
|
+
const r = spawnSync("sysctl", ["-n", "kern.bootsessionuuid"], {
|
|
49
|
+
encoding: "utf8",
|
|
50
|
+
timeout: 2000,
|
|
51
|
+
});
|
|
52
|
+
const id = r.status === 0 ? r.stdout.trim() : "";
|
|
53
|
+
if (/^[0-9A-Fa-f-]{36}$/.test(id)) cachedBoot = id;
|
|
54
|
+
}
|
|
55
|
+
} catch {
|
|
56
|
+
cachedBoot = null;
|
|
57
|
+
}
|
|
58
|
+
return cachedBoot;
|
|
59
|
+
}
|
|
60
|
+
/** Wall-clock boot time in ms. */
|
|
61
|
+
export function bootTime(): number {
|
|
62
|
+
if (process.platform === "linux")
|
|
63
|
+
try {
|
|
64
|
+
const line = /^btime (\d+)$/m.exec(readFileSync("/proc/stat", "utf8"));
|
|
65
|
+
if (line) return Number(line[1]) * 1000;
|
|
66
|
+
} catch {
|
|
67
|
+
/* fall back below */
|
|
68
|
+
}
|
|
69
|
+
return Date.now() - uptime() * 1000;
|
|
70
|
+
}
|
|
71
|
+
function linuxStat(pid: number): string[] | null {
|
|
72
|
+
try {
|
|
73
|
+
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
74
|
+
// The command name may contain spaces and parentheses; fields follow the last ')'.
|
|
75
|
+
return stat
|
|
76
|
+
.slice(stat.lastIndexOf(")") + 2)
|
|
77
|
+
.trim()
|
|
78
|
+
.split(" ");
|
|
79
|
+
} catch {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Clock-independent start identity of a running process, or null. */
|
|
84
|
+
export function processStart(pid: number): string | null {
|
|
85
|
+
if (!Number.isSafeInteger(pid) || pid < 1) return null;
|
|
86
|
+
if (process.platform === "linux") {
|
|
87
|
+
const fields = linuxStat(pid);
|
|
88
|
+
// Field 22 (starttime) is the 20th after the command name.
|
|
89
|
+
const ticks = fields?.[19];
|
|
90
|
+
return ticks && /^\d+$/.test(ticks) ? `ticks:${ticks}` : null;
|
|
91
|
+
}
|
|
92
|
+
if (process.platform === "darwin") {
|
|
93
|
+
const r = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], {
|
|
94
|
+
encoding: "utf8",
|
|
95
|
+
timeout: 2000,
|
|
96
|
+
env: { ...process.env, LC_ALL: "C" },
|
|
97
|
+
});
|
|
98
|
+
const value = r.status === 0 ? r.stdout.trim() : "";
|
|
99
|
+
return value ? `lstart:${value}` : null;
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
/** Approximate wall-clock start in ms, for legacy marks without `started`. */
|
|
104
|
+
function processStartMs(pid: number): number | null {
|
|
105
|
+
if (process.platform === "linux") {
|
|
106
|
+
const ticks = linuxStat(pid)?.[19];
|
|
107
|
+
return ticks && /^\d+$/.test(ticks)
|
|
108
|
+
? bootTime() + (Number(ticks) * 1000) / LINUX_TICKS_PER_SECOND
|
|
109
|
+
: null;
|
|
110
|
+
}
|
|
111
|
+
const start = processStart(pid);
|
|
112
|
+
if (start?.startsWith("lstart:")) {
|
|
113
|
+
const ms = Date.parse(start.slice("lstart:".length));
|
|
114
|
+
return Number.isFinite(ms) ? ms : null;
|
|
115
|
+
}
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
let cachedSelf: string | null | undefined;
|
|
119
|
+
/** This process's mark for a new lock. */
|
|
120
|
+
export function selfMark(): ProcessMark {
|
|
121
|
+
cachedSelf ??= processStart(process.pid);
|
|
122
|
+
return {
|
|
123
|
+
pid: process.pid,
|
|
124
|
+
host: hostname(),
|
|
125
|
+
boot: bootId(),
|
|
126
|
+
started: cachedSelf,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function probe(pid: number): "alive" | "absent" | "error" {
|
|
130
|
+
try {
|
|
131
|
+
process.kill(pid, 0);
|
|
132
|
+
return "alive";
|
|
133
|
+
} catch (error) {
|
|
134
|
+
const code =
|
|
135
|
+
error instanceof Error && "code" in error ? error.code : undefined;
|
|
136
|
+
// EPERM: the PID belongs to another account. Cadence state is private to
|
|
137
|
+
// one account, so that process cannot be the holder that wrote it.
|
|
138
|
+
return code === "ESRCH" || code === "EPERM" ? "absent" : "error";
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** Classify the writer of a lock. `writtenAt` is the lock file's mtime. */
|
|
142
|
+
export function holderState(mark: ProcessMark, writtenAt?: number): Holder {
|
|
143
|
+
if (mark.host !== hostname()) return { state: "foreign", reboot: false };
|
|
144
|
+
const boot = bootId();
|
|
145
|
+
if (mark.boot && boot && mark.boot !== boot)
|
|
146
|
+
return { state: "dead", reboot: true };
|
|
147
|
+
const legacyEarlierBoot =
|
|
148
|
+
!mark.boot &&
|
|
149
|
+
writtenAt !== undefined &&
|
|
150
|
+
writtenAt < bootTime() - LEGACY_BOOT_MARGIN_MS;
|
|
151
|
+
if (mark.pid === process.pid) {
|
|
152
|
+
// Our own PID: either this process wrote it, or an earlier process with
|
|
153
|
+
// the same number did (PIDs restart after a reboot, and a container's
|
|
154
|
+
// first process always has the same one).
|
|
155
|
+
const self = selfMark().started;
|
|
156
|
+
if (mark.started && self)
|
|
157
|
+
return mark.started === self ? live() : dead(legacyEarlierBoot);
|
|
158
|
+
const ownStart = Date.now() - process.uptime() * 1000;
|
|
159
|
+
return writtenAt !== undefined && writtenAt < ownStart - REUSE_SLACK_MS
|
|
160
|
+
? dead(legacyEarlierBoot)
|
|
161
|
+
: live();
|
|
162
|
+
}
|
|
163
|
+
const seen = probe(mark.pid);
|
|
164
|
+
if (seen === "absent") return { state: "dead", reboot: legacyEarlierBoot };
|
|
165
|
+
if (seen === "error") return { state: "unknown", reboot: false };
|
|
166
|
+
if (mark.started) {
|
|
167
|
+
const now = processStart(mark.pid);
|
|
168
|
+
if (now) return now === mark.started ? live() : dead(legacyEarlierBoot);
|
|
169
|
+
// The process may have exited between the probe and this read.
|
|
170
|
+
return probe(mark.pid) === "absent"
|
|
171
|
+
? dead(legacyEarlierBoot)
|
|
172
|
+
: { state: "unknown", reboot: false };
|
|
173
|
+
}
|
|
174
|
+
// Legacy mark: the holder must have started before it wrote its lock.
|
|
175
|
+
const startedAt = processStartMs(mark.pid);
|
|
176
|
+
if (startedAt !== null && writtenAt !== undefined)
|
|
177
|
+
return startedAt > writtenAt + REUSE_SLACK_MS
|
|
178
|
+
? dead(legacyEarlierBoot)
|
|
179
|
+
: live();
|
|
180
|
+
if (legacyEarlierBoot) return dead(true);
|
|
181
|
+
return startedAt === null ? { state: "unknown", reboot: false } : live();
|
|
182
|
+
}
|
|
183
|
+
function live(): Holder {
|
|
184
|
+
return { state: "live", reboot: false };
|
|
185
|
+
}
|
|
186
|
+
function dead(reboot: boolean): Holder {
|
|
187
|
+
return { state: "dead", reboot };
|
|
188
|
+
}
|
|
189
|
+
/** mtime (ms) from a PrivateFiles stamp (`...:ctimeMs:mtimeMs`). */
|
|
190
|
+
export function stampTime(stamp: string): number | undefined {
|
|
191
|
+
const value = Number(stamp.slice(stamp.lastIndexOf(":") + 1));
|
|
192
|
+
return Number.isFinite(value) ? value : undefined;
|
|
193
|
+
}
|
|
194
|
+
/** Optional mark fields when decoding an existing lock. */
|
|
195
|
+
export function decodeMarkFields(o: Record<string, unknown>): {
|
|
196
|
+
boot: string | null;
|
|
197
|
+
started: string | null;
|
|
198
|
+
} {
|
|
199
|
+
const field = (value: unknown) =>
|
|
200
|
+
typeof value === "string" && value.length <= 128 ? value : null;
|
|
201
|
+
return { boot: field(o.boot), started: field(o.started) };
|
|
202
|
+
}
|
|
203
|
+
/** Ask every process in a group to stop, then force it. Resolves true once
|
|
204
|
+
* the group is empty; false if it is still present after the wait. */
|
|
205
|
+
export async function stopProcessGroup(
|
|
206
|
+
pgid: number,
|
|
207
|
+
waitMs = 5000,
|
|
208
|
+
): Promise<boolean> {
|
|
209
|
+
const gone = () => {
|
|
210
|
+
try {
|
|
211
|
+
process.kill(process.platform === "win32" ? pgid : -pgid, 0);
|
|
212
|
+
return false;
|
|
213
|
+
} catch (error) {
|
|
214
|
+
return (
|
|
215
|
+
error instanceof Error && "code" in error && error.code === "ESRCH"
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
const send = (signal: NodeJS.Signals) => {
|
|
220
|
+
try {
|
|
221
|
+
if (process.platform === "win32") process.kill(pgid, signal);
|
|
222
|
+
else process.kill(-pgid, signal);
|
|
223
|
+
} catch {
|
|
224
|
+
/* already gone, or the next probe reports it */
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
if (gone()) return true;
|
|
228
|
+
send("SIGTERM");
|
|
229
|
+
const deadline = Date.now() + waitMs;
|
|
230
|
+
while (Date.now() < deadline) {
|
|
231
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
232
|
+
if (gone()) return true;
|
|
233
|
+
if (Date.now() > deadline - waitMs / 2) send("SIGKILL");
|
|
234
|
+
}
|
|
235
|
+
send("SIGKILL");
|
|
236
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
237
|
+
return gone();
|
|
238
|
+
}
|