@staix/agent-hub 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +2 -2
- package/docs/agent-notes/daemon.md +2 -0
- package/docs/agent-notes/local-worker.md +3 -1
- package/docs/agent-notes/tasks.md +3 -1
- package/docs/agent-notes/tests.md +1 -0
- package/docs/design/task-board.md +1 -1
- package/docs/dogfood.md +6 -3
- package/docs/operations.md +73 -11
- package/docs/quickstart.md +11 -3
- package/docs/sdlc.md +1 -1
- package/docs/security.md +17 -2
- package/docs/verification/2026-11-03-0.29.0.md +59 -0
- package/docs/verification/2026-11-03-control-write-stall-windows.json +241 -0
- package/docs/verification/2026-11-03-control-write-stall-windows.md +20 -0
- package/docs/verification/2026-11-03-live-board-latency.json +556 -0
- package/docs/verification/2026-11-03-live-board-latency.md +64 -0
- package/docs/verified.json +13 -13
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +524 -18
- package/src/cli/help.ts +1 -1
- package/src/cli/output.ts +5 -0
- package/src/cli/recovery-package.ts +9 -0
- package/src/hub/board.ts +3 -2
- package/src/hub/control-client.ts +16 -1
- package/src/hub/control-write-trace.ts +409 -0
- package/src/hub/daemon.ts +171 -47
- package/src/hub/hub-tools.ts +1 -1
- package/src/hub/routing.ts +9 -2
- package/src/hub/runtime-capture-prune.ts +4 -1
- package/src/hub/sdlc-release-authorization.ts +13 -4
- package/src/hub/sdlc-release.ts +23 -10
- package/src/hub/sdlc.ts +15 -2
- package/src/hub/task-worktree.ts +331 -0
- package/src/hub/tasks.ts +162 -17
- package/src/local/deny.ts +49 -0
- package/src/local/tools.ts +4 -2
- package/src/memory/brief.ts +5 -0
- package/src/memory/client.ts +5 -2
package/src/cli/help.ts
CHANGED
|
@@ -88,7 +88,7 @@ const SECTIONS: [string, [usage: string, description: string][]][] = [
|
|
|
88
88
|
["Tasks and review", [
|
|
89
89
|
["ahub board import github --label <label> [--milestone <milestone>] [--repo <owner/repo>]", "person or configured conductor: read bounded open issues into deduplicated Backlog; never writes GitHub"],
|
|
90
90
|
["ahub board [state | --ready] [--columns] [--release <name> | --release-id <id>] [--handoffs] [--json] [--full] [--color=auto|always|never]", "the task board, one line per task at a terminal width; --full: whole titles and SDLC advice; --ready: proposed tasks with nothing left to wait for; --handoffs: what this project handed to other projects and the last state each reported"],
|
|
91
|
-
["ahub task propose [--class <c> | <class>] <title...> [--owner <peer>] [--path <p>]... [--after <id>]... [--urgent] [--needs-network] [--needs-worktree] [--backlog] [--detail <text>]", "put a task on the board; --needs-network keeps it away from a peer whose shell has none; --needs-worktree keeps it away from a sandboxed peer
|
|
91
|
+
["ahub task propose [--class <c> | <class>] <title...> [--owner <peer>] [--path <p>]... [--after <id>]... [--urgent] [--needs-network] [--needs-worktree] [--backlog] [--detail <text>]", "put a task on the board; --needs-network keeps it away from a peer whose shell has none; --needs-worktree keeps it away from a sandboxed peer unless the hub can make the task worktree for it (it makes it, commits and pushes at done); a first word that names a class is taken as the class; while a task named by --after is not approved yet, --owner reserves the task for that peer, offered to it first once it is ready; otherwise --owner assigns it now"],
|
|
92
92
|
["ahub task handoff --project <root|id> [<class>] <title...> [--path <p>]... [--issue <n>] [--task <n>] [--detail <text>] [--json] [--color=auto|always|never]", "person only: register work that belongs to ANOTHER project that runs agent-hub on that project's board (an ownerless proposal its own roles take from there); the target must be registered and running, nothing is queued; a feature or behavior change first needs a spec issue there (--issue); its state changes show in ahub board --handoffs (an agent that hands off gets them as task messages)"],
|
|
93
93
|
["ahub backlog list [--json] [--full] [--color=auto|always|never] | add <class> <title...> | plan <id> <revision> <fields-json> <reason...> | pin <id> <revision> true|false <reason...> | undo <id> <revision> [--json] [--color=auto|always|never]", "person only: plan work with a reason; {} records a note-only revision, fields.sdlc sets backlog repo/issue identity and clears old spec acceptance; preserve pins/frozen scope and refuse stale edits; list is by rank (--json: raw listing)"],
|
|
94
94
|
["ahub release new <name> <goal> <order> | list | show <name> | plan <name> <revision> <dates-json> | activate|freeze|ship|drop <name> <revision> [--json] [--color=auto|always|never]", "person only, except show (any agent, public metadata): planned UTC millisecond dates and phased scope; ship records a time after approval, never tags or publishes"],
|
package/src/cli/output.ts
CHANGED
|
@@ -14,6 +14,7 @@ import type { SpecsStatus } from "../hub/specs-task.ts";
|
|
|
14
14
|
|
|
15
15
|
export interface DoctorCheck { section: "Tools" | "Hub" | "Config" | "Models" | "Memory"; level: "ok" | "warn" | "fail" | "unknown"; name: string; detail: string }
|
|
16
16
|
export interface StatusOutput {
|
|
17
|
+
controlWriteTiming?: { active: number; slowRequests: number; blockedLoopSamples: number; droppedActive: number; droppedRecent: number; droppedLogs: number; droppedSpans?: number; droppedMetadata?: number; expiredRequests?: number };
|
|
17
18
|
specs?: SpecsStatus & { missingBlockHint?: boolean };
|
|
18
19
|
queueBackpressure?: QueueBackpressureStatus;
|
|
19
20
|
pendingRestart?: readonly { key: string; reason: string }[];
|
|
@@ -213,6 +214,10 @@ export function renderStatus(data: StatusOutput, columns?: number, now = Date.no
|
|
|
213
214
|
if (data.roleInfo) data = { ...data, peers: Object.fromEntries(Object.entries(data.peers).map(([peer, value]) => [peer, { ...value, ...data.roleInfo?.[peer] }])) };
|
|
214
215
|
const out = lines(`agent-hub${data.version ? ` ${data.version}` : ""} pid ${data.pid ?? "unknown"} control 127.0.0.1:${data.controlPort ?? "unknown"}`, columns, "info");
|
|
215
216
|
if (data.cwd) out.push(...lines(data.cwd, columns));
|
|
217
|
+
const timing = data.controlWriteTiming;
|
|
218
|
+
if (full && timing && Object.values(timing).every(value => Number.isSafeInteger(value) && value >= 0)) {
|
|
219
|
+
out.push(...lines(`control writes: active ${timing.active}, slow ${timing.slowRequests}, loop delays ${timing.blockedLoopSamples}, dropped ${timing.droppedActive + timing.droppedLogs + (timing.droppedSpans ?? 0) + (timing.droppedMetadata ?? 0)}, expired ${timing.expiredRequests ?? 0}; timings in hub.log`, columns, timing.slowRequests || timing.blockedLoopSamples ? "attention" : undefined));
|
|
220
|
+
}
|
|
216
221
|
const specs = data.specs;
|
|
217
222
|
if (specs && [specs.specCount, specs.documents, specs.errors, specs.stale].every(value => Number.isSafeInteger(value) && value >= 0)
|
|
218
223
|
&& (specs.specCount > 0 || specs.documents > 0 || specs.errors > 0)) {
|
|
@@ -127,6 +127,9 @@ export function verifyRecoveryTarget(kind: "restart" | "upgrade", root: string,
|
|
|
127
127
|
return verifyPackage(root, version, recoveryPackageOptions(kind, root));
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
+
/** #1120: the documentation checker that `ahub sdlc authorize-release` and release readiness spawn. A capture without it can never record release consent. */
|
|
131
|
+
export const RELEASE_CHECKER = "scripts/check-docs.mjs";
|
|
132
|
+
// ponytail: the checker is copied and kept by prune, but is not in packageDigest. Widening the digest would make every release staged before this change refuse as "cached release changed". Ceiling: a tampered checker in a capture is not detected by the digest. Upgrade path: fold it into packageDigest with a digest-version bump once no pre-#1120 capture or cached release remains.
|
|
130
133
|
const RUNTIME_ASSETS = ["src", "plugins", "templates", ".claude-plugin", "package.json", "README.md", "LICENSE", "CHANGELOG.md"] as const;
|
|
131
134
|
function runtimeDirectory(directory: string): void {
|
|
132
135
|
let info;
|
|
@@ -159,6 +162,12 @@ function preserveRuntimeAssets(sourceRoot: string, destination: string, expected
|
|
|
159
162
|
mkdirSync(temporary, { mode: 0o700 });
|
|
160
163
|
try {
|
|
161
164
|
for (const name of RUNTIME_ASSETS) cpSync(join(sourceRoot, name), join(temporary, name), { recursive: true, dereference: false });
|
|
165
|
+
// Absent in a source that predates the checker (a registry package before #1120 ships it): the capture then reports the checker as unavailable, never a stale document.
|
|
166
|
+
if (existsSync(join(sourceRoot, RELEASE_CHECKER))) {
|
|
167
|
+
if (!lstatSync(join(sourceRoot, RELEASE_CHECKER)).isFile()) throw new Error(`${RELEASE_CHECKER} must be a regular file to be preserved in a runtime capture; replace it with a regular file`);
|
|
168
|
+
mkdirSync(join(temporary, dirname(RELEASE_CHECKER)), { recursive: true });
|
|
169
|
+
cpSync(join(sourceRoot, RELEASE_CHECKER), join(temporary, RELEASE_CHECKER), { dereference: false });
|
|
170
|
+
}
|
|
162
171
|
preserveRuntimeDependencies(sourceRoot, temporary);
|
|
163
172
|
if (digest(temporary, true) !== expectedDigest || digest(sourceRoot) !== expectedDigest)
|
|
164
173
|
throw new Error("source package changed while being preserved");
|
package/src/hub/board.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { ReviewerPoolReason } from "./reviewer-pool.ts";
|
|
|
11
11
|
import type { GitHubSource } from "./github-import.ts";
|
|
12
12
|
import type { Database } from "./db-schema.ts";
|
|
13
13
|
import { openHubDatabase, advanceHubDbSchema } from "./db-schema.ts";
|
|
14
|
+
import { controlWriteSpan } from "./control-write-trace.ts";
|
|
14
15
|
import { planningPatch, type PlanningFields, type Release, type ReleaseState } from "./planning.ts";
|
|
15
16
|
import { HUB, USER, type PeerId } from "./envelope.ts";
|
|
16
17
|
import type { SweepRecord } from "./task-sweep.ts";
|
|
@@ -431,11 +432,11 @@ export class Board {
|
|
|
431
432
|
|
|
432
433
|
/** Keep the generation and business write atomic without changing Bun's reported row count. */
|
|
433
434
|
private taskWrite<T extends { changes: number }>(write: () => T): T {
|
|
434
|
-
return this.db.transaction(() => {
|
|
435
|
+
return controlWriteSpan("board-write", () => this.db.transaction(() => {
|
|
435
436
|
const result = { ...write() };
|
|
436
437
|
if (result.changes) this.db.run("UPDATE board_status_revision SET revision=revision+1 WHERE id=1");
|
|
437
438
|
return result;
|
|
438
|
-
})();
|
|
439
|
+
})());
|
|
439
440
|
}
|
|
440
441
|
|
|
441
442
|
statusRevision(): number {
|
|
@@ -4,6 +4,13 @@ import { projectContext } from "./project.ts";
|
|
|
4
4
|
import { newToolRequestId, TOOL_WRITE_FEATURE, toolWriteOperation, toolWriteUnknownReply, toolWriteRefusedReply } from "./tool-request.ts";
|
|
5
5
|
import { externalAdapterMetadata, externalInstanceCatalog, type ExternalInstanceCatalog, type ExternalAdapterMetadata } from "../adapters/external-metadata.ts";
|
|
6
6
|
import { negotiateProtocol, protocolWindow, type ProtocolWindow } from "./protocol-window.ts";
|
|
7
|
+
import { ControlWriteTelemetry, controlWriteMetadata, controlWriteSpan } from "./control-write-trace.ts";
|
|
8
|
+
|
|
9
|
+
// The client has no authoritative task privacy view. Keep kind/timing only, never its args or task id.
|
|
10
|
+
const clientWriteTelemetry = new ControlWriteTelemetry({ emit: line => {
|
|
11
|
+
if (process.env.AHUB_CONTROL_WRITE_TIMING === "1") console.error(`[agent-hub] ${line}`);
|
|
12
|
+
} });
|
|
13
|
+
const publicClientTiming = () => true;
|
|
7
14
|
|
|
8
15
|
export function stateDirFor(cwd: string): string {
|
|
9
16
|
return projectContext(cwd).stateDir;
|
|
@@ -221,6 +228,14 @@ export class ControlClient {
|
|
|
221
228
|
|
|
222
229
|
/** `timeoutMs`: for calls a person waits on. The hub has its own budget; this one only makes sure the CLI never hangs. */
|
|
223
230
|
request(msg: Record<string, unknown>, timeoutMs = 30_000, signal?: AbortSignal): Promise<any> {
|
|
231
|
+
if (process.env.AHUB_CONTROL_WRITE_TIMING !== "1") return this.requestBody(msg, timeoutMs, signal);
|
|
232
|
+
const metadata = controlWriteMetadata(msg);
|
|
233
|
+
if (!metadata) return this.requestBody(msg, timeoutMs, signal);
|
|
234
|
+
return clientWriteTelemetry.run({ kind: metadata.kind }, publicClientTiming, () => controlWriteSpan("client-wait", () =>
|
|
235
|
+
this.requestBody(msg, timeoutMs, signal)));
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
private requestBody(msg: Record<string, unknown>, timeoutMs: number, signal?: AbortSignal): Promise<any> {
|
|
224
239
|
if (this.ws.readyState !== WebSocket.OPEN || this.closing && msg.t !== "request_cancel") return Promise.resolve({ ok: false, error: "hub connection is not open" });
|
|
225
240
|
const invocation = msg.t === "task" && msg.op === "hub_invoke";
|
|
226
241
|
const operation = toolWriteOperation(msg), write = operation !== undefined;
|
|
@@ -266,7 +281,7 @@ export class ControlClient {
|
|
|
266
281
|
signal?.addEventListener("abort", aborted, { once: true });
|
|
267
282
|
if (signal?.aborted) { aborted(); return; }
|
|
268
283
|
timer = timeoutMs ? setTimeout(() => abandon(`no answer from the hub within ${Math.round(timeoutMs / 1000)} s`), timeoutMs) : undefined;
|
|
269
|
-
try { this.ws.send(JSON.stringify({ ...wire, rid })); }
|
|
284
|
+
try { controlWriteSpan("client-send", () => this.ws.send(JSON.stringify({ ...wire, rid }))); }
|
|
270
285
|
catch { abandon("hub request could not be sent"); }
|
|
271
286
|
});
|
|
272
287
|
}
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
|
|
3
|
+
export const CONTROL_WRITE_KINDS = [
|
|
4
|
+
"hub_task_propose", "hub_review", "hub_task_accept", "hub_task_done", "hub_task_decline", "hub_task_cancel",
|
|
5
|
+
"hub_task_assign", "task_assign", "task_assign_checked", "task_unassign", "hub_delivery_done", "background",
|
|
6
|
+
] as const;
|
|
7
|
+
export type ControlWriteKind = typeof CONTROL_WRITE_KINDS[number];
|
|
8
|
+
|
|
9
|
+
export const CONTROL_WRITE_PHASES = [
|
|
10
|
+
"client-send", "client-wait", "daemon-dispatch", "daemon-reply",
|
|
11
|
+
"board-write", "queue-hook", "status-hook", "memory", "external", "snapshot",
|
|
12
|
+
] as const;
|
|
13
|
+
export type ControlWritePhase = typeof CONTROL_WRITE_PHASES[number];
|
|
14
|
+
|
|
15
|
+
export const CONTROL_WRITE_SLOW_MS = 1_000;
|
|
16
|
+
export const CONTROL_WRITE_LOOP_DELAY_MS = 250;
|
|
17
|
+
export const CONTROL_WRITE_SAMPLE_MS = 100;
|
|
18
|
+
export const CONTROL_WRITE_MAX_LINE_BYTES = 2_048;
|
|
19
|
+
export const CONTROL_WRITE_MAX_DURATION_MS = 86_400_000;
|
|
20
|
+
export const CONTROL_WRITE_REQUEST_MAX_AGE_MS = 5 * 60_000;
|
|
21
|
+
|
|
22
|
+
const MAX_ACTIVE_REQUESTS = 128;
|
|
23
|
+
const MAX_ACTIVE_SPANS = 256;
|
|
24
|
+
const MAX_RECENT_SPANS = 512;
|
|
25
|
+
const MAX_LOOP_SPANS = 8;
|
|
26
|
+
const MIN_SYNC_PREFIX_MS = 10;
|
|
27
|
+
const MAX_EMITS_PER_MINUTE = 60;
|
|
28
|
+
const RATE_LIMIT_MS = 1_000;
|
|
29
|
+
const RATE_KEYS = ["slow", "loop"] as const;
|
|
30
|
+
|
|
31
|
+
export interface ControlWriteMetadata { kind: ControlWriteKind; task?: number }
|
|
32
|
+
export type ControlWritePrivacy = () => boolean;
|
|
33
|
+
export type ControlWriteSink = (line: string) => void;
|
|
34
|
+
export interface ControlWriteTelemetryOptions {
|
|
35
|
+
emit: ControlWriteSink;
|
|
36
|
+
now?: () => number;
|
|
37
|
+
maxActiveRequests?: number;
|
|
38
|
+
maxActiveSpans?: number;
|
|
39
|
+
maxRecentSpans?: number;
|
|
40
|
+
rateLimitMs?: number;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface PhaseTotal { phase: ControlWritePhase; durationMs: number }
|
|
44
|
+
interface RequestTrace {
|
|
45
|
+
telemetry: ControlWriteTelemetry;
|
|
46
|
+
metadata: ControlWriteMetadata;
|
|
47
|
+
privacyCurrent: ControlWritePrivacy;
|
|
48
|
+
logPrivacy: ControlWritePrivacy;
|
|
49
|
+
startedAt: number;
|
|
50
|
+
phases: Map<ControlWritePhase, number>;
|
|
51
|
+
phaseOrder: ControlWritePhase[];
|
|
52
|
+
settled: boolean;
|
|
53
|
+
privacySuppressed: boolean;
|
|
54
|
+
}
|
|
55
|
+
interface SpanTrace {
|
|
56
|
+
id?: number;
|
|
57
|
+
request: RequestTrace;
|
|
58
|
+
phase: ControlWritePhase;
|
|
59
|
+
startedAt: number;
|
|
60
|
+
async: boolean;
|
|
61
|
+
finished: boolean;
|
|
62
|
+
prefixRecorded: boolean;
|
|
63
|
+
}
|
|
64
|
+
interface SpanInterval {
|
|
65
|
+
kind: ControlWriteKind;
|
|
66
|
+
task?: number;
|
|
67
|
+
phase: ControlWritePhase;
|
|
68
|
+
startedAt: number;
|
|
69
|
+
endedAt?: number;
|
|
70
|
+
async: boolean;
|
|
71
|
+
privacyCurrent: ControlWritePrivacy;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const currentTrace = new AsyncLocalStorage<RequestTrace>();
|
|
75
|
+
|
|
76
|
+
function isObject(value: unknown): value is Record<string, unknown> {
|
|
77
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function validTaskId(value: unknown): value is number {
|
|
81
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function numericTaskId(value: unknown): number | undefined {
|
|
85
|
+
return validTaskId(value) ? value : undefined;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function ownValue(value: Record<string, unknown>, key: string): unknown {
|
|
89
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
90
|
+
return descriptor && Object.hasOwn(descriptor, "value") ? descriptor.value : undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const OPERATIONS: Readonly<Record<string, ControlWriteKind>> = {
|
|
94
|
+
hub_task_propose: "hub_task_propose", hub_review: "hub_review", hub_task_accept: "hub_task_accept",
|
|
95
|
+
hub_task_done: "hub_task_done", hub_task_decline: "hub_task_decline", hub_task_cancel: "hub_task_cancel",
|
|
96
|
+
hub_task_assign: "hub_task_assign", task_assign: "task_assign", task_assign_checked: "task_assign_checked", task_unassign: "task_unassign",
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/** The control protocol contributes only a closed operation name and, when present, a positive task id. */
|
|
100
|
+
export function controlWriteMetadata(message: unknown): ControlWriteMetadata | undefined {
|
|
101
|
+
if (!isObject(message)) return undefined;
|
|
102
|
+
try {
|
|
103
|
+
let kind: ControlWriteKind | undefined;
|
|
104
|
+
let args: Record<string, unknown> | undefined;
|
|
105
|
+
const type = ownValue(message, "t"), operation = ownValue(message, "op");
|
|
106
|
+
if (type === "delivery_complete") kind = "hub_delivery_done";
|
|
107
|
+
else if (type === "task" && typeof operation === "string") {
|
|
108
|
+
kind = Object.hasOwn(OPERATIONS, operation) ? OPERATIONS[operation] : undefined;
|
|
109
|
+
const rawArgs = ownValue(message, "args");
|
|
110
|
+
args = isObject(rawArgs) ? rawArgs : undefined;
|
|
111
|
+
}
|
|
112
|
+
if (!kind) return undefined;
|
|
113
|
+
const task = numericTaskId(args ? ownValue(args, "id") : undefined);
|
|
114
|
+
return task === undefined ? { kind } : { kind, task };
|
|
115
|
+
} catch { return undefined; }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function isPhase(value: string): value is ControlWritePhase {
|
|
119
|
+
return (CONTROL_WRITE_PHASES as readonly string[]).includes(value);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function isThenable(value: unknown): value is PromiseLike<unknown> {
|
|
123
|
+
try { return !!value && (typeof value === "object" || typeof value === "function") && typeof (value as { then?: unknown }).then === "function"; }
|
|
124
|
+
catch { return false; }
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Add a bounded phase to the current request. With no request context, this is a no-op. */
|
|
128
|
+
export function controlWriteSpan<T>(phase: ControlWritePhase, fn: () => T): T {
|
|
129
|
+
const request = currentTrace.getStore();
|
|
130
|
+
if (!request || request.settled || request.privacySuppressed || !isPhase(phase)) return fn();
|
|
131
|
+
const span = request.telemetry.beginSpan(request, phase);
|
|
132
|
+
let result: T;
|
|
133
|
+
try { result = fn(); }
|
|
134
|
+
catch (error) { request.telemetry.finishSpan(span); throw error; }
|
|
135
|
+
if (!isThenable(result)) { request.telemetry.finishSpan(span); return result; }
|
|
136
|
+
span.async = true;
|
|
137
|
+
request.telemetry.recordSyncPrefix(span);
|
|
138
|
+
return Promise.resolve(result).then(
|
|
139
|
+
value => { request.telemetry.finishSpan(span); return value; },
|
|
140
|
+
error => { request.telemetry.finishSpan(span); throw error; },
|
|
141
|
+
) as T;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Bounded per-daemon control write and event-loop timing. Logs contain metadata, never request payloads. */
|
|
145
|
+
export class ControlWriteTelemetry {
|
|
146
|
+
private readonly nowFn: () => number;
|
|
147
|
+
private readonly maxActiveRequests: number;
|
|
148
|
+
private readonly maxActiveSpans: number;
|
|
149
|
+
private readonly maxRecentSpans: number;
|
|
150
|
+
private readonly rateLimitMs: number;
|
|
151
|
+
private readonly requests = new Map<number, RequestTrace>();
|
|
152
|
+
private readonly spans = new Map<number, SpanTrace>();
|
|
153
|
+
private readonly recent: SpanInterval[] = [];
|
|
154
|
+
private readonly emitted: number[] = [];
|
|
155
|
+
private readonly lastEmit = new Map<typeof RATE_KEYS[number], number>();
|
|
156
|
+
private nextRequestId = 1;
|
|
157
|
+
private nextSpanId = 1;
|
|
158
|
+
private timer: ReturnType<typeof setInterval> | undefined;
|
|
159
|
+
private nextSampleAt: number | undefined;
|
|
160
|
+
private lastNow = 0;
|
|
161
|
+
private stopped = false;
|
|
162
|
+
private completedRequests = 0;
|
|
163
|
+
private expiredRequests = 0;
|
|
164
|
+
private slowRequests = 0;
|
|
165
|
+
private blockedLoopSamples = 0;
|
|
166
|
+
private droppedActive = 0;
|
|
167
|
+
private droppedSpans = 0;
|
|
168
|
+
private droppedRecent = 0;
|
|
169
|
+
private droppedLogs = 0;
|
|
170
|
+
private droppedMetadata = 0;
|
|
171
|
+
private clockFailures = 0;
|
|
172
|
+
|
|
173
|
+
constructor(private readonly options: ControlWriteTelemetryOptions) {
|
|
174
|
+
this.nowFn = options.now ?? (() => performance.now());
|
|
175
|
+
this.maxActiveRequests = options.maxActiveRequests ?? MAX_ACTIVE_REQUESTS;
|
|
176
|
+
this.maxActiveSpans = options.maxActiveSpans ?? MAX_ACTIVE_SPANS;
|
|
177
|
+
this.maxRecentSpans = options.maxRecentSpans ?? MAX_RECENT_SPANS;
|
|
178
|
+
this.rateLimitMs = options.rateLimitMs ?? RATE_LIMIT_MS;
|
|
179
|
+
if (![this.maxActiveRequests, this.maxActiveSpans, this.maxRecentSpans].every(value => Number.isSafeInteger(value) && value > 0)
|
|
180
|
+
|| !Number.isSafeInteger(this.rateLimitMs) || this.rateLimitMs < 0 || this.rateLimitMs > RATE_LIMIT_MS
|
|
181
|
+
|| this.maxActiveRequests > MAX_ACTIVE_REQUESTS || this.maxActiveSpans > MAX_ACTIVE_SPANS || this.maxRecentSpans > MAX_RECENT_SPANS) throw new Error("invalid control write trace bounds");
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Wrap a synchronous or asynchronous operation without changing its result or exception. */
|
|
185
|
+
run<T>(metadata: ControlWriteMetadata, privacyCurrent: ControlWritePrivacy, fn: () => T, inputPublic?: ControlWritePrivacy): T {
|
|
186
|
+
if (this.stopped) return fn();
|
|
187
|
+
const safeMetadata = this.safeMetadata(metadata);
|
|
188
|
+
if (!safeMetadata || typeof privacyCurrent !== "function") {
|
|
189
|
+
this.droppedMetadata++;
|
|
190
|
+
return fn();
|
|
191
|
+
}
|
|
192
|
+
const now = this.readNow();
|
|
193
|
+
this.expireRequests(now);
|
|
194
|
+
if (this.requests.size >= this.maxActiveRequests) { this.droppedActive++; return fn(); }
|
|
195
|
+
if (!this.privateNow(privacyCurrent)) return fn();
|
|
196
|
+
const inputVisible = !inputPublic || this.privateNow(inputPublic);
|
|
197
|
+
inputPublic = undefined; // The admission-only callback may carry raw input; never retain it in a trace.
|
|
198
|
+
if (!inputVisible) return fn();
|
|
199
|
+
const request: RequestTrace = {
|
|
200
|
+
telemetry: this, metadata: safeMetadata, privacyCurrent,
|
|
201
|
+
startedAt: now, phases: new Map(), phaseOrder: [], settled: false, privacySuppressed: false,
|
|
202
|
+
logPrivacy: () => false,
|
|
203
|
+
};
|
|
204
|
+
request.logPrivacy = () => !request.privacySuppressed && this.privateNow(request.privacyCurrent);
|
|
205
|
+
const id = this.nextRequestId++;
|
|
206
|
+
this.requests.set(id, request);
|
|
207
|
+
let result: T;
|
|
208
|
+
try { result = currentTrace.run(request, fn); }
|
|
209
|
+
catch (error) { this.finishRequest(id, request, "threw"); throw error; }
|
|
210
|
+
if (!isThenable(result)) { this.finishRequest(id, request, "returned"); return result; }
|
|
211
|
+
return Promise.resolve(result).then(
|
|
212
|
+
value => { this.finishRequest(id, request, "returned"); return value; },
|
|
213
|
+
error => { this.finishRequest(id, request, "threw"); throw error; },
|
|
214
|
+
) as T;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Observe a daemon boundary; work outside a control request is labelled as background. */
|
|
218
|
+
observe<T>(phase: ControlWritePhase, privacyCurrent: ControlWritePrivacy, fn: () => T, inputPublic?: ControlWritePrivacy): T {
|
|
219
|
+
if (this.stopped) return fn();
|
|
220
|
+
const current = currentTrace.getStore();
|
|
221
|
+
if (current?.telemetry === this && !current.settled) {
|
|
222
|
+
if (current.privacySuppressed) return fn();
|
|
223
|
+
const inputVisible = this.privateNow(privacyCurrent) && (!inputPublic || this.privateNow(inputPublic));
|
|
224
|
+
inputPublic = undefined;
|
|
225
|
+
if (!inputVisible) current.privacySuppressed = true;
|
|
226
|
+
return controlWriteSpan(phase, fn);
|
|
227
|
+
}
|
|
228
|
+
return this.run({ kind: "background" }, privacyCurrent, () => controlWriteSpan(phase, fn), inputPublic);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Start one unref'd event-loop delay probe for the daemon. */
|
|
232
|
+
start(): void {
|
|
233
|
+
if (this.stopped || this.timer) return;
|
|
234
|
+
this.nextSampleAt = this.readNow() + CONTROL_WRITE_SAMPLE_MS;
|
|
235
|
+
this.timer = setInterval(() => this.sample(), CONTROL_WRITE_SAMPLE_MS);
|
|
236
|
+
this.timer.unref?.();
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Stop the probe and suppress later diagnostics from this daemon instance. */
|
|
240
|
+
stop(): void {
|
|
241
|
+
this.stopped = true;
|
|
242
|
+
if (this.timer) clearInterval(this.timer);
|
|
243
|
+
this.timer = undefined;
|
|
244
|
+
this.nextSampleAt = undefined;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Deterministic probe seam used by tests; a missing cause remains explicitly unknown. */
|
|
248
|
+
sample(): void {
|
|
249
|
+
if (this.stopped) return;
|
|
250
|
+
const now = this.readNow(), expected = this.nextSampleAt ?? now;
|
|
251
|
+
this.expireRequests(now);
|
|
252
|
+
this.nextSampleAt = now + CONTROL_WRITE_SAMPLE_MS;
|
|
253
|
+
const delayMs = Math.max(0, now - expected);
|
|
254
|
+
if (delayMs < CONTROL_WRITE_LOOP_DELAY_MS) return;
|
|
255
|
+
this.blockedLoopSamples++;
|
|
256
|
+
const from = expected;
|
|
257
|
+
const observed: { kind: ControlWriteKind; task?: number; phase: ControlWritePhase; durationMs: number; open?: true; privacyCurrent: ControlWritePrivacy }[] = [];
|
|
258
|
+
for (const interval of this.recent) {
|
|
259
|
+
if (interval.async || interval.endedAt === undefined || interval.startedAt >= now || interval.endedAt < from) continue;
|
|
260
|
+
observed.push({ kind: interval.kind, ...(interval.task === undefined ? {} : { task: interval.task }), phase: interval.phase, durationMs: this.roundMs(interval.endedAt - interval.startedAt), privacyCurrent: interval.privacyCurrent });
|
|
261
|
+
}
|
|
262
|
+
for (const span of this.spans.values()) {
|
|
263
|
+
if (!span.async || span.startedAt >= now) continue;
|
|
264
|
+
observed.push({ kind: span.request.metadata.kind, ...(span.request.metadata.task === undefined ? {} : { task: span.request.metadata.task }), phase: span.phase, durationMs: this.roundMs(now - span.startedAt), open: true,
|
|
265
|
+
privacyCurrent: span.request.logPrivacy });
|
|
266
|
+
}
|
|
267
|
+
const unique = new Map<string, (typeof observed)[number]>();
|
|
268
|
+
for (const row of observed.sort((a, b) => b.durationMs - a.durationMs)) {
|
|
269
|
+
const key = `${row.kind}:${row.task ?? ""}:${row.phase}:${row.open === true}`;
|
|
270
|
+
if (!unique.has(key)) unique.set(key, row);
|
|
271
|
+
}
|
|
272
|
+
const selected = [...unique.values()].slice(0, MAX_LOOP_SPANS);
|
|
273
|
+
const spans = selected.map(({ kind, task, phase, durationMs, open }) => ({ kind, ...(task === undefined ? {} : { task }), phase, durationMs, ...(open ? { open: true as const } : {}) }));
|
|
274
|
+
const guards = [...new Set(selected.map(row => row.privacyCurrent))];
|
|
275
|
+
const attribution = selected.some(row => row.open !== true) ? "sync_overlap_observed" : selected.length ? "open_async_wait" : "unknown";
|
|
276
|
+
this.emit("loop", {
|
|
277
|
+
type: "control_write_loop_delay", delayMs: this.roundMs(delayMs), attribution, spans,
|
|
278
|
+
}, () => guards.every(guard => this.privateNow(guard)));
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Flat numeric counters for the authenticated full-status projection. */
|
|
282
|
+
status(): Record<string, number> {
|
|
283
|
+
return {
|
|
284
|
+
active: this.requests.size,
|
|
285
|
+
activeSpans: this.spans.size,
|
|
286
|
+
completedRequests: this.completedRequests,
|
|
287
|
+
expiredRequests: this.expiredRequests,
|
|
288
|
+
slowRequests: this.slowRequests,
|
|
289
|
+
blockedLoopSamples: this.blockedLoopSamples,
|
|
290
|
+
recentSpans: this.recent.length,
|
|
291
|
+
droppedActive: this.droppedActive,
|
|
292
|
+
droppedSpans: this.droppedSpans,
|
|
293
|
+
droppedRecent: this.droppedRecent,
|
|
294
|
+
droppedLogs: this.droppedLogs,
|
|
295
|
+
droppedMetadata: this.droppedMetadata,
|
|
296
|
+
clockFailures: this.clockFailures,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
beginSpan(request: RequestTrace, phase: ControlWritePhase): SpanTrace {
|
|
301
|
+
const span: SpanTrace = { request, phase, startedAt: this.readNow(), async: false, finished: false, prefixRecorded: false };
|
|
302
|
+
if (this.spans.size >= this.maxActiveSpans) { this.droppedSpans++; return span; }
|
|
303
|
+
span.id = this.nextSpanId++;
|
|
304
|
+
this.spans.set(span.id, span);
|
|
305
|
+
return span;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
finishSpan(span: SpanTrace): void {
|
|
309
|
+
if (span.finished) return;
|
|
310
|
+
span.finished = true;
|
|
311
|
+
if (span.request.settled) return;
|
|
312
|
+
const endedAt = this.readNow();
|
|
313
|
+
if (span.id !== undefined && this.spans.get(span.id) === span) this.spans.delete(span.id);
|
|
314
|
+
const durationMs = Math.max(0, endedAt - span.startedAt);
|
|
315
|
+
const request = span.request;
|
|
316
|
+
const prior = request.phases.get(span.phase) ?? 0;
|
|
317
|
+
if (!request.phaseOrder.includes(span.phase)) request.phaseOrder.push(span.phase);
|
|
318
|
+
request.phases.set(span.phase, prior + durationMs);
|
|
319
|
+
if (!span.async && !span.prefixRecorded) this.rememberSyncInterval(span, endedAt);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Preserve a long synchronous prelude inside an async function for the delayed loop probe. */
|
|
323
|
+
recordSyncPrefix(span: SpanTrace): void {
|
|
324
|
+
if (span.finished || span.prefixRecorded || span.request.settled) return;
|
|
325
|
+
const endedAt = this.readNow();
|
|
326
|
+
if (endedAt - span.startedAt < MIN_SYNC_PREFIX_MS) return;
|
|
327
|
+
span.prefixRecorded = true;
|
|
328
|
+
this.rememberSyncInterval(span, endedAt);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
private rememberSyncInterval(span: SpanTrace, endedAt: number): void {
|
|
332
|
+
const request = span.request;
|
|
333
|
+
this.recent.push({ kind: request.metadata.kind, ...(request.metadata.task === undefined ? {} : { task: request.metadata.task }), phase: span.phase, startedAt: span.startedAt, endedAt, async: false, privacyCurrent: request.logPrivacy });
|
|
334
|
+
while (this.recent.length > this.maxRecentSpans) { this.recent.shift(); this.droppedRecent++; }
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
private finishRequest(id: number, request: RequestTrace, outcome: "returned" | "threw"): void {
|
|
338
|
+
if (!this.requests.delete(id)) return;
|
|
339
|
+
request.settled = true;
|
|
340
|
+
for (const [spanId, span] of this.spans) if (span.request === request) this.spans.delete(spanId);
|
|
341
|
+
this.completedRequests++;
|
|
342
|
+
const durationMs = this.roundMs(this.readNow() - request.startedAt);
|
|
343
|
+
if (durationMs < CONTROL_WRITE_SLOW_MS) return;
|
|
344
|
+
this.slowRequests++;
|
|
345
|
+
const phases: PhaseTotal[] = request.phaseOrder.map(phase => ({ phase, durationMs: this.roundMs(request.phases.get(phase) ?? 0) }));
|
|
346
|
+
this.emit("slow", {
|
|
347
|
+
type: "control_write_slow", kind: request.metadata.kind, ...(request.metadata.task === undefined ? {} : { task: request.metadata.task }),
|
|
348
|
+
durationMs, outcome, phases: phases.slice(0, MAX_LOOP_SPANS), ...(phases.length > MAX_LOOP_SPANS ? { truncated: true } : {}),
|
|
349
|
+
}, request.logPrivacy);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** Retire diagnostics only; the operation and its eventual result remain owned by the caller. */
|
|
353
|
+
private expireRequests(now: number): void {
|
|
354
|
+
for (const [id, request] of this.requests) {
|
|
355
|
+
if (now - request.startedAt < CONTROL_WRITE_REQUEST_MAX_AGE_MS) continue;
|
|
356
|
+
request.settled = true;
|
|
357
|
+
request.privacySuppressed = true; // Recent intervals must not attribute an expired request either.
|
|
358
|
+
this.requests.delete(id);
|
|
359
|
+
this.expiredRequests++;
|
|
360
|
+
}
|
|
361
|
+
for (const [id, span] of this.spans) if (span.request.settled) this.spans.delete(id);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
private emit(key: typeof RATE_KEYS[number], value: Record<string, unknown>, privacyCurrent: ControlWritePrivacy): void {
|
|
365
|
+
if (this.stopped || !this.privateNow(privacyCurrent)) return;
|
|
366
|
+
const now = this.readNow(), last = this.lastEmit.get(key);
|
|
367
|
+
if (last !== undefined && now - last < this.rateLimitMs) { this.droppedLogs++; return; }
|
|
368
|
+
while (this.emitted.length && now - this.emitted[0]! >= 60_000) this.emitted.shift();
|
|
369
|
+
if (this.emitted.length >= MAX_EMITS_PER_MINUTE) { this.droppedLogs++; return; }
|
|
370
|
+
let line: string;
|
|
371
|
+
try { line = JSON.stringify(value); } catch { this.droppedLogs++; return; }
|
|
372
|
+
if (Buffer.byteLength(line) > CONTROL_WRITE_MAX_LINE_BYTES) {
|
|
373
|
+
const minimal = value.type === "control_write_loop_delay"
|
|
374
|
+
? { type: "control_write_loop_delay", delayMs: typeof value.delayMs === "number" ? value.delayMs : 0, attribution: "unknown", spans: [], truncated: true }
|
|
375
|
+
: { type: "control_write_slow", kind: value.kind, ...(value.task === undefined ? {} : { task: value.task }), durationMs: value.durationMs, outcome: value.outcome, phases: [], truncated: true };
|
|
376
|
+
line = JSON.stringify(minimal);
|
|
377
|
+
if (Buffer.byteLength(line) > CONTROL_WRITE_MAX_LINE_BYTES) { this.droppedLogs++; return; }
|
|
378
|
+
}
|
|
379
|
+
this.lastEmit.set(key, now);
|
|
380
|
+
this.emitted.push(now);
|
|
381
|
+
try { this.options.emit(line); } catch { this.droppedLogs++; }
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
private privateNow(guard: ControlWritePrivacy): boolean {
|
|
385
|
+
try { return guard() === true; } catch { return false; }
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
private safeMetadata(metadata: unknown): ControlWriteMetadata | undefined {
|
|
389
|
+
if (!isObject(metadata)) return undefined;
|
|
390
|
+
try {
|
|
391
|
+
const kind = ownValue(metadata, "kind"), task = ownValue(metadata, "task");
|
|
392
|
+
if (typeof kind !== "string" || !(CONTROL_WRITE_KINDS as readonly string[]).includes(kind) || task !== undefined && !validTaskId(task)) return undefined;
|
|
393
|
+
return { kind: kind as ControlWriteKind, ...(task === undefined ? {} : { task }) };
|
|
394
|
+
} catch { return undefined; }
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
private readNow(): number {
|
|
398
|
+
let value: number;
|
|
399
|
+
try { value = this.nowFn(); } catch { this.clockFailures++; return this.lastNow; }
|
|
400
|
+
if (!Number.isFinite(value) || value < this.lastNow) { this.clockFailures++; return this.lastNow; }
|
|
401
|
+
this.lastNow = value;
|
|
402
|
+
return value;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
private roundMs(value: number): number {
|
|
406
|
+
const bounded = Number.isFinite(value) ? Math.min(CONTROL_WRITE_MAX_DURATION_MS, Math.max(0, value)) : CONTROL_WRITE_MAX_DURATION_MS;
|
|
407
|
+
return Math.round(bounded * 10) / 10;
|
|
408
|
+
}
|
|
409
|
+
}
|