@logicseek/hello-cc 0.1.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -0
- package/README.md +26 -4
- package/README.zh-CN.md +22 -3
- package/bin/hcc.mjs +3 -5622
- package/docs/README.md +8 -0
- package/docs/README.zh-CN.md +6 -0
- package/docs/architecture.md +29 -18
- package/docs/architecture.zh-CN.md +20 -10
- package/docs/commands.md +16 -5
- package/docs/commands.zh-CN.md +16 -5
- package/docs/current-state-review-2026-08.md +341 -0
- package/docs/defect-review-2026-07.md +382 -0
- package/docs/guide.md +53 -5
- package/docs/guide.zh-CN.md +46 -3
- package/docs/superpowers/plans/2026-08-03-runtime-safety-plan.md +294 -0
- package/docs/superpowers/plans/2026-08-03-v1-data-identity-plan.md +400 -0
- package/docs/superpowers/plans/2026-08-03-v1-release-verification-plan.md +284 -0
- package/docs/superpowers/plans/2026-08-03-web-api-hardening-plan.md +303 -0
- package/docs/superpowers/plans/2026-08-04-runtime-buffer-gc-two-phase.md +71 -0
- package/docs/superpowers/specs/2026-08-03-runtime-safety-design.md +117 -0
- package/docs/superpowers/specs/2026-08-03-v1-data-identity-design.md +120 -0
- package/docs/superpowers/specs/2026-08-03-v1-release-verification-design.md +117 -0
- package/docs/superpowers/specs/2026-08-03-web-api-hardening-design.md +104 -0
- package/lib/cli/app.mjs +642 -0
- package/lib/cli/commands/coordination.mjs +247 -0
- package/lib/cli/commands/doctor.mjs +81 -0
- package/lib/cli/commands/gc.mjs +852 -0
- package/lib/cli/commands/hook.mjs +205 -0
- package/lib/cli/commands/install.mjs +294 -0
- package/lib/cli/commands/lifecycle.mjs +101 -0
- package/lib/cli/commands/lock.mjs +197 -0
- package/lib/cli/commands/misc.mjs +61 -0
- package/lib/cli/commands/msg.mjs +152 -0
- package/lib/cli/commands/peer.mjs +233 -0
- package/lib/cli/commands/query.mjs +96 -0
- package/lib/cli/commands/run.mjs +463 -0
- package/lib/cli/commands/task.mjs +405 -0
- package/lib/cli/commands/team.mjs +131 -0
- package/lib/cli/commands/tmux.mjs +838 -0
- package/lib/cli/commands/up.mjs +135 -0
- package/lib/cli-args.mjs +16 -2
- package/lib/coordination-state.mjs +112 -51
- package/lib/core/coordination/automation.mjs +5 -8
- package/lib/core/coordination/clock-safety.mjs +323 -0
- package/lib/core/coordination/event-retention.mjs +17 -0
- package/lib/core/coordination/gc-plan.mjs +614 -0
- package/lib/core/coordination/lease-renewal.mjs +102 -0
- package/lib/core/coordination/lock-evidence.mjs +77 -0
- package/lib/core/coordination/optimistic-evidence.mjs +26 -0
- package/lib/core/coordination/tasks.mjs +156 -36
- package/lib/core/peers/evidence-runtime.mjs +312 -0
- package/lib/core/peers/evidence.mjs +103 -0
- package/lib/core/peers/hook-owner.mjs +32 -0
- package/lib/core/peers/liveness.mjs +44 -4
- package/lib/core/peers/peer-helpers.mjs +121 -0
- package/lib/core/peers/session.mjs +8 -2
- package/lib/core/peers/tmux-safety.mjs +290 -0
- package/lib/core/sessions/launch.mjs +15 -1
- package/lib/db/connection.mjs +104 -0
- package/lib/db/events.mjs +34 -0
- package/lib/db/migration-backup.mjs +225 -0
- package/lib/db/schema.mjs +102 -5
- package/lib/db/stores/peers.mjs +34 -0
- package/lib/integrations/peers/identity.mjs +62 -16
- package/lib/integrations/shims/script.mjs +139 -15
- package/lib/process/identity.mjs +192 -0
- package/lib/process/pty-lifecycle.mjs +149 -0
- package/lib/runtime/buffer-directory-lease.mjs +47 -0
- package/lib/runtime/buffer-evidence.mjs +336 -0
- package/lib/runtime/buffer-gc-protocol.mjs +167 -0
- package/lib/runtime/buffer-gc.mjs +270 -0
- package/lib/runtime/client.mjs +61 -8
- package/lib/runtime/fatal-shutdown.mjs +54 -0
- package/lib/runtime/project-path.mjs +139 -0
- package/lib/runtime/projects.mjs +320 -40
- package/lib/runtime/state.mjs +232 -29
- package/lib/shared/cleanup-error.mjs +54 -0
- package/lib/shared/clock-grace.mjs +106 -0
- package/lib/shared/errors.mjs +9 -0
- package/lib/shared/file-lock.mjs +370 -0
- package/lib/shared/json-file.mjs +6 -2
- package/lib/shared/redact.mjs +60 -0
- package/lib/shared/socket-lock-worker.mjs +201 -0
- package/lib/terminal/tmux.mjs +78 -4
- package/lib/tmux.mjs +4 -0
- package/lib/ui/help.mjs +53 -15
- package/lib/web/api-version.mjs +42 -0
- package/lib/web/auto-attach.mjs +185 -0
- package/lib/web/buffer-gc-runtime.mjs +175 -0
- package/lib/web/cookie-auth.mjs +119 -0
- package/lib/web/external-sessions.mjs +254 -0
- package/lib/web/http-routes.mjs +542 -0
- package/lib/web/http.mjs +143 -7
- package/lib/web/liveness-reaper.mjs +145 -0
- package/lib/web/peer-actions.mjs +132 -23
- package/lib/web/project-contexts.mjs +178 -0
- package/lib/web/pty-sessions.mjs +203 -0
- package/lib/web/runtime-main.mjs +714 -0
- package/lib/web/runtime.mjs +91 -36
- package/lib/web/session-serialize.mjs +0 -0
- package/lib/web/startup.mjs +333 -0
- package/lib/web/tls.mjs +365 -0
- package/lib/web/tmux-sessions.mjs +1361 -0
- package/lib/web/tmux-stream.mjs +166 -0
- package/lib/web/ui-template.mjs +152 -20
- package/package.json +5 -2
- package/scripts/check-free-identifiers.mjs +290 -0
- package/scripts/regression.mjs +5646 -262
package/lib/web/http.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
|
+
import { timingSafeEqual } from 'node:crypto';
|
|
2
3
|
import { CliError } from '../shared/errors.mjs';
|
|
3
4
|
|
|
4
5
|
function readRequestBody(req, limit = 1024 * 1024) {
|
|
@@ -19,13 +20,39 @@ function readRequestBody(req, limit = 1024 * 1024) {
|
|
|
19
20
|
export async function readJsonRequest(req) {
|
|
20
21
|
const body = await readRequestBody(req);
|
|
21
22
|
if (!body.trim()) return {};
|
|
22
|
-
|
|
23
|
+
try {
|
|
24
|
+
return JSON.parse(body);
|
|
25
|
+
} catch (err) {
|
|
26
|
+
if (err instanceof SyntaxError) {
|
|
27
|
+
throw new CliError('BAD_REQUEST', 'Invalid JSON request body');
|
|
28
|
+
}
|
|
29
|
+
throw err;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function contentSecurityPolicy(nonce) {
|
|
34
|
+
const value = String(nonce || '');
|
|
35
|
+
if (!/^[A-Za-z0-9_-]{16,}$/.test(value)) {
|
|
36
|
+
throw new CliError('BAD_ARGS', 'A valid CSP nonce is required');
|
|
37
|
+
}
|
|
38
|
+
return "default-src 'self'; " +
|
|
39
|
+
`script-src 'self' 'nonce-${value}'; ` +
|
|
40
|
+
"style-src 'self' 'unsafe-inline'; " +
|
|
41
|
+
"img-src 'self' data:; " +
|
|
42
|
+
"connect-src 'self' ws: wss:; " +
|
|
43
|
+
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'";
|
|
23
44
|
}
|
|
24
45
|
|
|
25
|
-
export function sendHttp(res, status, contentType, body) {
|
|
46
|
+
export function sendHttp(res, status, contentType, body, options = {}) {
|
|
26
47
|
res.writeHead(status, {
|
|
27
48
|
'Content-Type': contentType,
|
|
28
|
-
'Cache-Control': 'no-store'
|
|
49
|
+
'Cache-Control': 'no-store',
|
|
50
|
+
'Referrer-Policy': 'no-referrer',
|
|
51
|
+
'X-Content-Type-Options': 'nosniff',
|
|
52
|
+
'X-Frame-Options': 'DENY',
|
|
53
|
+
'Content-Security-Policy': options.nonce
|
|
54
|
+
? contentSecurityPolicy(options.nonce)
|
|
55
|
+
: "frame-ancestors 'none'"
|
|
29
56
|
});
|
|
30
57
|
res.end(body);
|
|
31
58
|
}
|
|
@@ -39,7 +66,11 @@ export function sendFile(res, filePath, contentType) {
|
|
|
39
66
|
const body = fs.readFileSync(filePath);
|
|
40
67
|
res.writeHead(200, {
|
|
41
68
|
'Content-Type': contentType,
|
|
42
|
-
'Cache-Control': 'no-store'
|
|
69
|
+
'Cache-Control': 'no-store',
|
|
70
|
+
'Referrer-Policy': 'no-referrer',
|
|
71
|
+
'X-Content-Type-Options': 'nosniff',
|
|
72
|
+
'X-Frame-Options': 'DENY',
|
|
73
|
+
'Content-Security-Policy': "frame-ancestors 'none'"
|
|
43
74
|
});
|
|
44
75
|
res.end(body);
|
|
45
76
|
} catch {
|
|
@@ -47,9 +78,114 @@ export function sendFile(res, filePath, contentType) {
|
|
|
47
78
|
}
|
|
48
79
|
}
|
|
49
80
|
|
|
81
|
+
export function isLoopbackRemote(req) {
|
|
82
|
+
const addr = req?.socket?.remoteAddress || '';
|
|
83
|
+
return addr === '::1' || addr.startsWith('127.') || addr.startsWith('::ffff:127.');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function tokenMatches(provided, expected) {
|
|
87
|
+
if (!provided || !expected || provided.length !== expected.length) return false;
|
|
88
|
+
try {
|
|
89
|
+
return timingSafeEqual(Buffer.from(provided), Buffer.from(expected));
|
|
90
|
+
} catch {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function firstForwardedValue(value) {
|
|
96
|
+
const raw = Array.isArray(value) ? value[0] : value;
|
|
97
|
+
return String(raw || '').split(',')[0].trim();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// PROXY-2: browsers omit the default port in the Origin serialization while the
|
|
101
|
+
// Host header may carry it; normalize both sides to hostname + effective port.
|
|
102
|
+
function normalizedOriginAuthority(hostValue, protocol) {
|
|
103
|
+
const raw = String(hostValue || '');
|
|
104
|
+
if (!raw || raw !== raw.trim()) return null;
|
|
105
|
+
try {
|
|
106
|
+
const parsed = new URL(`${protocol}//${raw}`);
|
|
107
|
+
if (parsed.username || parsed.password || parsed.pathname !== '/' || parsed.search || parsed.hash) return null;
|
|
108
|
+
return {
|
|
109
|
+
hostname: parsed.hostname.toLowerCase(),
|
|
110
|
+
port: parsed.port || (protocol === 'https:' ? '443' : '80')
|
|
111
|
+
};
|
|
112
|
+
} catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function normalizedProxyOrigin(value) {
|
|
118
|
+
try {
|
|
119
|
+
const parsed = new URL(String(value || ''));
|
|
120
|
+
if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password ||
|
|
121
|
+
parsed.pathname !== '/' || parsed.search || parsed.hash) return null;
|
|
122
|
+
const authority = normalizedOriginAuthority(parsed.host, parsed.protocol);
|
|
123
|
+
return authority ? { protocol: parsed.protocol, authority } : null;
|
|
124
|
+
} catch {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function trustedProxyOrigin(req, options) {
|
|
130
|
+
if (!options?.trustProxy || !isLoopbackRemote(req)) return null;
|
|
131
|
+
const expected = normalizedProxyOrigin(options.proxyOrigin);
|
|
132
|
+
if (!expected) return null;
|
|
133
|
+
const forwardedProto = firstForwardedValue(req?.headers?.['x-forwarded-proto']).toLowerCase();
|
|
134
|
+
const forwardedHost = firstForwardedValue(req?.headers?.['x-forwarded-host']);
|
|
135
|
+
if (`${forwardedProto}:` !== expected.protocol) return null;
|
|
136
|
+
const authority = normalizedOriginAuthority(forwardedHost, expected.protocol);
|
|
137
|
+
if (!authority || authority.hostname !== expected.authority.hostname ||
|
|
138
|
+
authority.port !== expected.authority.port) return null;
|
|
139
|
+
return expected;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function requestMatchesProxyOrigin(req, options = {}) {
|
|
143
|
+
return Boolean(trustedProxyOrigin(req, options));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function requestIsSecure(req, options = {}) {
|
|
147
|
+
const trustProxy = options === true || options?.trustProxy === true;
|
|
148
|
+
if (trustProxy) return trustedProxyOrigin(req, options)?.protocol === 'https:';
|
|
149
|
+
return Boolean(req?.socket?.encrypted);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function requestOriginMatches(req, options = {}) {
|
|
153
|
+
const trustProxy = options === true || options?.trustProxy === true;
|
|
154
|
+
const origin = req?.headers?.origin || '';
|
|
155
|
+
const proxyOrigin = trustProxy ? trustedProxyOrigin(req, options) : null;
|
|
156
|
+
if (trustProxy && !proxyOrigin) return false;
|
|
157
|
+
const host = trustProxy ? new URL(options.proxyOrigin).host : req?.headers?.host || '';
|
|
158
|
+
if (!origin || !host) return false;
|
|
159
|
+
try {
|
|
160
|
+
const parsed = new URL(origin);
|
|
161
|
+
const expectedProtocol = proxyOrigin?.protocol || (req?.socket?.encrypted ? 'https:' : 'http:');
|
|
162
|
+
if (parsed.username || parsed.password || parsed.pathname !== '/' || parsed.search || parsed.hash) return false;
|
|
163
|
+
const originAuthority = {
|
|
164
|
+
hostname: parsed.hostname.toLowerCase(),
|
|
165
|
+
port: parsed.port || (expectedProtocol === 'https:' ? '443' : '80')
|
|
166
|
+
};
|
|
167
|
+
const hostAuthority = normalizedOriginAuthority(host, expectedProtocol);
|
|
168
|
+
return parsed.protocol === expectedProtocol &&
|
|
169
|
+
hostAuthority !== null &&
|
|
170
|
+
originAuthority.hostname === hostAuthority.hostname && originAuthority.port === hostAuthority.port;
|
|
171
|
+
} catch {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
50
176
|
export function authOk(url, req, token) {
|
|
51
|
-
|
|
52
|
-
|
|
177
|
+
// No token configured: only trust loopback clients. A tokenless runtime is
|
|
178
|
+
// refused on non-loopback binds at startup, so this is defense-in-depth.
|
|
179
|
+
if (!token) return isLoopbackRemote(req);
|
|
180
|
+
// v1-token-query-csrf-bypass: the query token is accepted ONLY on the cookie
|
|
181
|
+
// exchange paths (the console page and the login endpoint). Everywhere else
|
|
182
|
+
// — including all API routes and the WS upgrade — a leaked ?token= URL must
|
|
183
|
+
// not authenticate writes, so a captured console URL cannot be replayed
|
|
184
|
+
// cross-site. API clients authenticate with Authorization: Bearer.
|
|
185
|
+
const pathname = url.pathname || '';
|
|
186
|
+
const allowQueryToken = pathname === '/' || pathname === '/login';
|
|
187
|
+
const queryToken = allowQueryToken ? (url.searchParams.get('token') || '') : '';
|
|
53
188
|
const auth = req.headers.authorization || '';
|
|
54
|
-
|
|
189
|
+
const bearer = auth.startsWith('Bearer ') ? auth.slice(7) : '';
|
|
190
|
+
return tokenMatches(queryToken, token) || tokenMatches(bearer, token);
|
|
55
191
|
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// Clock-aware liveness reaper, extracted from lib/web/runtime-main.mjs.
|
|
2
|
+
// Backstop for peers that died without an exit signal: wall-vs-monotonic
|
|
3
|
+
// drift detection, per-project stale candidates, and evidence-checked
|
|
4
|
+
// mutation through the shared clock-safety observer.
|
|
5
|
+
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import { performance } from 'node:perf_hooks';
|
|
9
|
+
import { CLOCK_GRACE_SEC, classifyClockDrift, clockGraceSuppressed, readClockGraceUntil } from '../shared/clock-grace.mjs';
|
|
10
|
+
import { peerEvidenceAllowsReap } from '../core/peers/evidence.mjs';
|
|
11
|
+
|
|
12
|
+
export function createLivenessReaper(deps) {
|
|
13
|
+
const {
|
|
14
|
+
ctx, projectContexts, sessions,
|
|
15
|
+
sessionsForProject, connectWebProject,
|
|
16
|
+
now, addEvent,
|
|
17
|
+
peerEvidenceFromDb, mutatePeerWithEvidence, observeClockSafetyOrThrow,
|
|
18
|
+
UNKNOWN_EVIDENCE_GRACE_SEC, redactedLogText, sameResolvedPath
|
|
19
|
+
} = deps;
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
// ── Liveness reaper ───────────────────────────────────────────────────────
|
|
23
|
+
// Backstop for peers that died without an exit signal (kill -9, crash, a
|
|
24
|
+
// provider Stop hook that never fired). Confirmed-dead owners are reaped
|
|
25
|
+
// immediately; unknown evidence must remain stale through the shared grace.
|
|
26
|
+
// This only detaches DB state and never authorizes tmux destruction.
|
|
27
|
+
let reaperInFlight = false;
|
|
28
|
+
// Compare wall time with a monotonic clock so an event-loop stall or machine
|
|
29
|
+
// sleep does not look like a wall-clock step. Only their drift is safety
|
|
30
|
+
// relevant; on a jump, the shared observer evaluates current owner evidence.
|
|
31
|
+
let lastClockProbe = { wallMs: Date.now(), monotonicMs: performance.now() };
|
|
32
|
+
function detectClockJump() {
|
|
33
|
+
const current = { wallMs: Date.now(), monotonicMs: performance.now() };
|
|
34
|
+
const wallDeltaMs = current.wallMs - lastClockProbe.wallMs;
|
|
35
|
+
const monotonicDeltaMs = current.monotonicMs - lastClockProbe.monotonicMs;
|
|
36
|
+
lastClockProbe = current;
|
|
37
|
+
const jump = classifyClockDrift({ wallDeltaMs, monotonicDeltaMs });
|
|
38
|
+
if (jump) {
|
|
39
|
+
console.error(redactedLogText(`[${new Date().toISOString()}] wall-clock ${jump.kind} drift detected (${jump.driftMs >= 0 ? '+' : ''}${Math.round(jump.driftMs / 1000)}s); evaluating ownership evidence with a ${CLOCK_GRACE_SEC}s unknown-evidence grace window`));
|
|
40
|
+
}
|
|
41
|
+
return jump;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function runtimeProjectContexts() {
|
|
45
|
+
const contexts = new Map([[path.resolve(ctx.root), ctx]]);
|
|
46
|
+
for (const projectCtx of projectContexts.values()) {
|
|
47
|
+
if (!sameResolvedPath(projectCtx.root, ctx.root)) {
|
|
48
|
+
try {
|
|
49
|
+
if (!fs.statSync(projectCtx.root).isDirectory() || !fs.statSync(projectCtx.dbPath).isFile()) continue;
|
|
50
|
+
} catch {
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
contexts.set(path.resolve(projectCtx.root), projectCtx);
|
|
55
|
+
}
|
|
56
|
+
return [...contexts.values()];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function runClockAwareReaper() {
|
|
60
|
+
if (reaperInFlight) return;
|
|
61
|
+
reaperInFlight = true;
|
|
62
|
+
try {
|
|
63
|
+
// Probe exactly once per scheduler tick. A clock jump is machine-wide;
|
|
64
|
+
// each project feeds that signal and its own evidence snapshot through
|
|
65
|
+
// the shared observer rather than maintaining runtime-only grace state.
|
|
66
|
+
const jump = detectClockJump();
|
|
67
|
+
for (const projectCtx of runtimeProjectContexts()) {
|
|
68
|
+
let db = null;
|
|
69
|
+
try {
|
|
70
|
+
db = connectWebProject(projectCtx, { migrateRegistered: false, create: false });
|
|
71
|
+
const t = now();
|
|
72
|
+
const managedPeerIds = new Set();
|
|
73
|
+
for (const session of sessionsForProject(projectCtx)) {
|
|
74
|
+
if (session.status !== 'running') continue;
|
|
75
|
+
const managedId = session.peerId || session.id;
|
|
76
|
+
if (managedId && peerEvidenceFromDb(db, projectCtx, managedId).state === 'live') {
|
|
77
|
+
managedPeerIds.add(managedId);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const rows = db.prepare(`
|
|
81
|
+
SELECT p.id, p.status, p.pid, p.pid_start_token, p.pid_command_hash, p.last_seen_at,
|
|
82
|
+
b.transport, b.runtime_target
|
|
83
|
+
FROM peers p
|
|
84
|
+
LEFT JOIN peer_bindings b ON b.peer = p.id
|
|
85
|
+
WHERE p.status IN ('running', 'working', 'busy')
|
|
86
|
+
AND p.last_seen_at < ?
|
|
87
|
+
`).all(t - UNKNOWN_EVIDENCE_GRACE_SEC);
|
|
88
|
+
const candidates = rows.map((row) => {
|
|
89
|
+
const evidence = peerEvidenceFromDb(db, projectCtx, row.id);
|
|
90
|
+
return {
|
|
91
|
+
boundary: Number(row.last_seen_at) + UNKNOWN_EVIDENCE_GRACE_SEC,
|
|
92
|
+
evidence: evidence.state,
|
|
93
|
+
owner: row.id
|
|
94
|
+
};
|
|
95
|
+
});
|
|
96
|
+
observeClockSafetyOrThrow(db, {
|
|
97
|
+
operation: 'ownership',
|
|
98
|
+
candidates,
|
|
99
|
+
nowSec: t,
|
|
100
|
+
clockJump: jump
|
|
101
|
+
});
|
|
102
|
+
// Unknown evidence is the only reason a jump opens blanket grace.
|
|
103
|
+
// Probes above completed before the observer's write transaction.
|
|
104
|
+
if (clockGraceSuppressed(t, readClockGraceUntil(db))) continue;
|
|
105
|
+
for (const row of rows) {
|
|
106
|
+
if (managedPeerIds.has(row.id)) continue;
|
|
107
|
+
mutatePeerWithEvidence(db, projectCtx, row.id, (subject, evidence) => {
|
|
108
|
+
db.prepare('UPDATE peers SET status = ? WHERE id = ?').run('exited', row.id);
|
|
109
|
+
db.prepare('UPDATE peer_bindings SET runtime_target = NULL, updated_at = ? WHERE peer = ?').run(t, row.id);
|
|
110
|
+
try {
|
|
111
|
+
addEvent(db, 'peer.reaped', 'web-runtime', null, {
|
|
112
|
+
peer: row.id,
|
|
113
|
+
pid: subject.peer.pid,
|
|
114
|
+
reason: evidence.reason
|
|
115
|
+
});
|
|
116
|
+
} catch {}
|
|
117
|
+
}, {
|
|
118
|
+
acceptEvidence: (currentEvidence, subject) => peerEvidenceAllowsReap(currentEvidence, {
|
|
119
|
+
nowSec: now(),
|
|
120
|
+
lastSeenAt: Number(subject.peer.last_seen_at),
|
|
121
|
+
staleAfterSec: UNKNOWN_EVIDENCE_GRACE_SEC,
|
|
122
|
+
graceUntil: readClockGraceUntil(db)
|
|
123
|
+
}),
|
|
124
|
+
beforeMutate: ({ subject, evidence: currentEvidence }) => peerEvidenceAllowsReap(currentEvidence, {
|
|
125
|
+
nowSec: now(),
|
|
126
|
+
lastSeenAt: Number(subject.peer.last_seen_at),
|
|
127
|
+
staleAfterSec: UNKNOWN_EVIDENCE_GRACE_SEC,
|
|
128
|
+
graceUntil: readClockGraceUntil(db)
|
|
129
|
+
})
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
} catch (err) {
|
|
133
|
+
console.error(redactedLogText(`[${new Date().toISOString()}] liveness reaper failed for ${projectCtx.root}: ${err?.message || err}`));
|
|
134
|
+
} finally {
|
|
135
|
+
try { db?.close(); } catch {}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
} finally {
|
|
139
|
+
reaperInFlight = false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const reaperPoller = setInterval(runClockAwareReaper, 30000);
|
|
143
|
+
|
|
144
|
+
return { reaperPoller, runtimeProjectContexts, runClockAwareReaper, detectClockJump };
|
|
145
|
+
}
|
package/lib/web/peer-actions.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import process from 'node:process';
|
|
3
|
-
import { intOpt, required } from '../cli-args.mjs';
|
|
3
|
+
import { intOpt, positiveSafeIntOpt, required } from '../cli-args.mjs';
|
|
4
4
|
import { CliError } from '../shared/errors.mjs';
|
|
5
5
|
import {
|
|
6
6
|
lockLabel,
|
|
@@ -9,6 +9,19 @@ import {
|
|
|
9
9
|
scopedLockResource
|
|
10
10
|
} from '../core/coordination/locks.mjs';
|
|
11
11
|
import { normalizeStateResources } from '../ui/state-render.mjs';
|
|
12
|
+
import {
|
|
13
|
+
clockGraceSuppressed
|
|
14
|
+
} from '../shared/clock-grace.mjs';
|
|
15
|
+
import {
|
|
16
|
+
captureLockAcquireSubject,
|
|
17
|
+
captureLockRenewalSubject,
|
|
18
|
+
clockCandidatesFromLocks,
|
|
19
|
+
observeLockOwnerEvidence,
|
|
20
|
+
sameLockAcquireSubject
|
|
21
|
+
} from '../core/coordination/lock-evidence.mjs';
|
|
22
|
+
import { runOptimisticEvidenceMutation } from '../core/coordination/optimistic-evidence.mjs';
|
|
23
|
+
import { clockSafetyUnavailable } from '../core/coordination/clock-safety.mjs';
|
|
24
|
+
import { leaseDeadline, renewOwnedLocks } from '../core/coordination/lease-renewal.mjs';
|
|
12
25
|
|
|
13
26
|
const DEFAULT_ACTIVE_PEER_TTL = 600;
|
|
14
27
|
const DEFAULT_LOCK_TTL = 900;
|
|
@@ -36,6 +49,12 @@ export function createWebPeerActions(deps = {}) {
|
|
|
36
49
|
const defaultLockTtl = Number(deps.defaultLockTtl ?? DEFAULT_LOCK_TTL);
|
|
37
50
|
const detectBranch = requireFn(deps.detectBranch, 'detectBranch');
|
|
38
51
|
const now = typeof deps.now === 'function' ? deps.now : defaultNow;
|
|
52
|
+
const observeClockSafetyInTransaction = requireFn(
|
|
53
|
+
deps.observeClockSafetyInTransaction,
|
|
54
|
+
'observeClockSafetyInTransaction'
|
|
55
|
+
);
|
|
56
|
+
const observePeerEvidence = typeof deps.observePeerEvidence === 'function' ? deps.observePeerEvidence : null;
|
|
57
|
+
const peerEvidenceFromDb = requireFn(deps.peerEvidenceFromDb, 'peerEvidenceFromDb');
|
|
39
58
|
const positiveIntOpt = requireFn(deps.positiveIntOpt, 'positiveIntOpt');
|
|
40
59
|
const queryInbox = requireFn(deps.queryInbox, 'queryInbox');
|
|
41
60
|
const statusSnapshot = requireFn(deps.statusSnapshot, 'statusSnapshot');
|
|
@@ -45,6 +64,18 @@ export function createWebPeerActions(deps = {}) {
|
|
|
45
64
|
const tx = requireFn(deps.tx, 'tx');
|
|
46
65
|
const upsertPeer = requireFn(deps.upsertPeer, 'upsertPeer');
|
|
47
66
|
|
|
67
|
+
function prepareClockObservation(db, subject, evidenceByOwner) {
|
|
68
|
+
try {
|
|
69
|
+
return observeClockSafetyInTransaction(db, {
|
|
70
|
+
operation: 'ownership',
|
|
71
|
+
candidates: clockCandidatesFromLocks(subject, evidenceByOwner),
|
|
72
|
+
nowSec: subject.observedAt
|
|
73
|
+
});
|
|
74
|
+
} catch (error) {
|
|
75
|
+
throw clockSafetyUnavailable(error);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
48
79
|
function webMutationPeer(peer, input = {}) {
|
|
49
80
|
const actorPeer = String(input.actorPeer || input.actor_peer || '').trim();
|
|
50
81
|
if (!actorPeer) {
|
|
@@ -93,19 +124,53 @@ export function createWebPeerActions(deps = {}) {
|
|
|
93
124
|
|
|
94
125
|
function webPeerHeartbeat(projectCtx, peer, input = {}) {
|
|
95
126
|
webMutationPeer(peer, input);
|
|
96
|
-
const
|
|
127
|
+
const ttlOverride = input.ttl === undefined
|
|
128
|
+
? null
|
|
129
|
+
: positiveSafeIntOpt(input, 'ttl', defaultLockTtl);
|
|
97
130
|
const status = input.status || null;
|
|
98
131
|
const renewLocks = input['renew-locks'] !== false && input.renew_locks !== false;
|
|
99
|
-
const db = connect(projectCtx);
|
|
100
132
|
const t = now();
|
|
133
|
+
if (ttlOverride !== null) leaseDeadline(t, ttlOverride);
|
|
134
|
+
const db = connect(projectCtx);
|
|
101
135
|
let renewed = 0;
|
|
102
136
|
try {
|
|
103
137
|
touchPeer(db, peer, status);
|
|
104
138
|
if (renewLocks) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
139
|
+
try {
|
|
140
|
+
renewed = runOptimisticEvidenceMutation(db, {
|
|
141
|
+
capture: (subjectDb) => {
|
|
142
|
+
try {
|
|
143
|
+
return captureLockRenewalSubject(subjectDb, { owner: peer, now: t });
|
|
144
|
+
} catch (error) {
|
|
145
|
+
throw clockSafetyUnavailable(error);
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
observe: (subject) => new Map([[
|
|
149
|
+
peer,
|
|
150
|
+
subject.peer
|
|
151
|
+
? (observePeerEvidence
|
|
152
|
+
? observePeerEvidence(projectCtx, subject.peer, subject.binding)
|
|
153
|
+
: peerEvidenceFromDb(db, projectCtx, peer))
|
|
154
|
+
: { state: 'unknown', reason: 'peer_missing' }
|
|
155
|
+
]]),
|
|
156
|
+
same: (left, right) => JSON.stringify(left) === JSON.stringify(right),
|
|
157
|
+
beforeMutate: (subject, evidenceByOwner) =>
|
|
158
|
+
prepareClockObservation(db, subject, evidenceByOwner),
|
|
159
|
+
changedMessage: `Lock subjects changed while renewing locks for ${peer}; retry`,
|
|
160
|
+
mutate: (subject, evidenceByOwner) => {
|
|
161
|
+
const evidenceLive = evidenceByOwner.get(peer)?.state === 'live';
|
|
162
|
+
return renewOwnedLocks(db, {
|
|
163
|
+
owner: peer,
|
|
164
|
+
nowSec: t,
|
|
165
|
+
ttlOverride,
|
|
166
|
+
includeExpired: evidenceLive
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
});
|
|
170
|
+
} catch (error) {
|
|
171
|
+
if (error instanceof CliError) throw error;
|
|
172
|
+
throw clockSafetyUnavailable(error);
|
|
173
|
+
}
|
|
109
174
|
}
|
|
110
175
|
addEvent(db, 'peer.heartbeat', peer, null, webAuditPayload(peer, input, { status, renewed }));
|
|
111
176
|
} finally {
|
|
@@ -149,12 +214,20 @@ export function createWebPeerActions(deps = {}) {
|
|
|
149
214
|
const id = intOpt(input, 'id', intOpt({ id: input.task }, 'id'));
|
|
150
215
|
if (!id) throw new CliError('BAD_REQUEST', 'task id required');
|
|
151
216
|
const reason = required(input, 'reason');
|
|
152
|
-
const policy = input.policy || '
|
|
217
|
+
const policy = input.force ? 'any' : (input.policy || 'blocked-or-stale');
|
|
153
218
|
const staleAfter = positiveIntOpt(input, 'stale-after', intOpt(input, 'stale_after', activePeerTtl), { max: 86400 * 30 });
|
|
154
219
|
const db = connect(projectCtx);
|
|
155
220
|
try {
|
|
156
221
|
touchPeer(db, peer, 'working');
|
|
157
|
-
const task = takeOverTaskForPeer(db, peer, id, {
|
|
222
|
+
const task = takeOverTaskForPeer(db, peer, id, {
|
|
223
|
+
reason,
|
|
224
|
+
policy,
|
|
225
|
+
staleAfter,
|
|
226
|
+
source: 'web',
|
|
227
|
+
ownerEvidenceFor: (owner, _row, ownerRow, binding) => observePeerEvidence
|
|
228
|
+
? observePeerEvidence(projectCtx, ownerRow, binding)
|
|
229
|
+
: peerEvidenceFromDb(db, projectCtx, owner)
|
|
230
|
+
});
|
|
158
231
|
return { peer, task, summary: `took over task #${task.id}: ${task.title}` };
|
|
159
232
|
} finally {
|
|
160
233
|
db.close();
|
|
@@ -164,46 +237,82 @@ export function createWebPeerActions(deps = {}) {
|
|
|
164
237
|
function webPeerLockAcquire(projectCtx, peer, input = {}) {
|
|
165
238
|
webMutationPeer(peer, input);
|
|
166
239
|
const requested = scopedLockResource(required(input, 'resource'), input.scope);
|
|
167
|
-
const ttl =
|
|
240
|
+
const ttl = positiveSafeIntOpt(input, 'ttl', defaultLockTtl);
|
|
168
241
|
const taskId = intOpt(input, 'task', intOpt(input, 'task_id', null));
|
|
169
242
|
const reason = input.reason || '';
|
|
243
|
+
const acquisitionNow = now();
|
|
244
|
+
leaseDeadline(acquisitionNow, ttl);
|
|
170
245
|
const db = connect(projectCtx);
|
|
171
246
|
try {
|
|
172
247
|
touchPeer(db, peer, 'working');
|
|
173
|
-
const
|
|
174
|
-
|
|
248
|
+
const outcome = runOptimisticEvidenceMutation(db, {
|
|
249
|
+
capture: (subjectDb) => {
|
|
250
|
+
try {
|
|
251
|
+
return captureLockAcquireSubject(subjectDb, {
|
|
252
|
+
taskId,
|
|
253
|
+
requested,
|
|
254
|
+
now: acquisitionNow
|
|
255
|
+
});
|
|
256
|
+
} catch (error) {
|
|
257
|
+
throw clockSafetyUnavailable(error);
|
|
258
|
+
}
|
|
259
|
+
},
|
|
260
|
+
observe: (subject) => observeLockOwnerEvidence(subject, (row, binding) => observePeerEvidence
|
|
261
|
+
? observePeerEvidence(projectCtx, row, binding)
|
|
262
|
+
: peerEvidenceFromDb(db, projectCtx, row.id)),
|
|
263
|
+
same: sameLockAcquireSubject,
|
|
264
|
+
beforeMutate: (subject, evidenceByOwner) =>
|
|
265
|
+
prepareClockObservation(db, subject, evidenceByOwner),
|
|
266
|
+
changedMessage: `Lock subjects changed while acquiring ${lockLabel(requested)}; retry`,
|
|
267
|
+
mutate: (subject, evidenceByOwner, clockObservation) => {
|
|
268
|
+
const t = subject.observedAt;
|
|
175
269
|
if (taskId) {
|
|
176
|
-
const task =
|
|
270
|
+
const task = subject.task;
|
|
177
271
|
if (!task) throw new CliError('NOT_FOUND', `Task #${taskId} does not exist`);
|
|
178
272
|
assertTaskOwnerForMutation(db, peer, task, 'lock-acquire');
|
|
179
273
|
}
|
|
180
|
-
|
|
181
|
-
|
|
274
|
+
// During clock grace, retained lock rows still conflict — but NOT
|
|
275
|
+
// EXPIRED locks of verified-dead owners (CS-04): a dead owner's expired
|
|
276
|
+
// lock must not block acquisition while the GC that would delete the row
|
|
277
|
+
// is itself deferred by the same grace window.
|
|
278
|
+
const graceActive = clockGraceSuppressed(t, clockObservation.graceUntil);
|
|
279
|
+
const conflict = subject.locks.find((row) => {
|
|
280
|
+
if (row.owner === peer || !locksConflict(row, requested)) return false;
|
|
281
|
+
const ownerEvidence = evidenceByOwner.get(row.owner)?.state;
|
|
282
|
+
if (graceActive && Number(row.expires_at) <= t && ownerEvidence === 'dead') return false;
|
|
283
|
+
return graceActive || Number(row.expires_at) > t || ownerEvidence === 'live';
|
|
284
|
+
});
|
|
182
285
|
if (conflict) {
|
|
183
|
-
|
|
286
|
+
return { error: new CliError('LOCK_HELD', `Resource ${lockLabel(requested)} conflicts with lock ${lockLabel(conflict)} held by ${conflict.owner}`, {
|
|
184
287
|
resource: requested.base_resource,
|
|
185
288
|
scope: requested.scope,
|
|
186
289
|
lock_resource: conflict.resource,
|
|
187
290
|
lock_scope: lockScope(conflict),
|
|
188
291
|
owner: conflict.owner,
|
|
189
292
|
expires_at: iso(conflict.expires_at)
|
|
190
|
-
});
|
|
293
|
+
}) };
|
|
191
294
|
}
|
|
192
|
-
const existing =
|
|
295
|
+
const existing = subject.locks.find((row) => row.resource === requested.resource) || null;
|
|
296
|
+
const expiresAt = leaseDeadline(t, ttl);
|
|
193
297
|
db.prepare(`
|
|
194
|
-
INSERT INTO locks(resource, base_resource, scope, owner, task_id, reason, expires_at, created_at)
|
|
195
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
298
|
+
INSERT INTO locks(resource, base_resource, scope, owner, task_id, reason, expires_at, created_at, ttl_sec)
|
|
299
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
196
300
|
ON CONFLICT(resource) DO UPDATE SET
|
|
197
301
|
base_resource = excluded.base_resource,
|
|
198
302
|
scope = excluded.scope,
|
|
199
303
|
owner = excluded.owner,
|
|
200
304
|
task_id = excluded.task_id,
|
|
201
305
|
reason = excluded.reason,
|
|
202
|
-
expires_at = excluded.expires_at
|
|
203
|
-
|
|
306
|
+
expires_at = excluded.expires_at,
|
|
307
|
+
created_at = excluded.created_at,
|
|
308
|
+
ttl_sec = excluded.ttl_sec
|
|
309
|
+
`).run(requested.resource, requested.base_resource, requested.scope, peer, taskId, reason, expiresAt, t, ttl);
|
|
204
310
|
addEvent(db, 'lock.acquired', peer, taskId, webAuditPayload(peer, input, { resource: requested.base_resource, lock_resource: requested.resource, scope: requested.scope, ttl, previous_owner: existing ? existing.owner : null }));
|
|
205
|
-
return db.prepare('SELECT * FROM locks WHERE resource = ?').get(requested.resource);
|
|
311
|
+
return { lock: db.prepare('SELECT * FROM locks WHERE resource = ?').get(requested.resource) };
|
|
312
|
+
}
|
|
206
313
|
});
|
|
314
|
+
if (outcome.error) throw outcome.error;
|
|
315
|
+
const lock = outcome.lock;
|
|
207
316
|
return { peer, lock, summary: `locked ${lockLabel(lock)} by ${lock.owner} until ${iso(lock.expires_at)}` };
|
|
208
317
|
} finally {
|
|
209
318
|
db.close();
|