dsh-decision-layer 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +5 -3
- package/README.md +5 -3
- package/docs/en/DETAIL.md +1 -0
- package/docs/zh/DETAIL.md +1 -0
- package/lib/client.js +297 -3
- package/lib/host/backend.d.ts +3 -2
- package/lib/host/backend.js +61 -43
- package/lib/host/checkpoint-review.d.ts +27 -0
- package/lib/host/checkpoint-review.js +334 -0
- package/lib/host/config-store.js +21 -1
- package/lib/host/danger-gate.js +8 -6
- package/lib/host/evaluation-trace.d.ts +8 -0
- package/lib/host/evaluation-trace.js +70 -0
- package/lib/host/execution-observer.d.ts +26 -0
- package/lib/host/execution-observer.js +168 -0
- package/lib/host/index.d.ts +1 -0
- package/lib/host/index.js +61 -16
- package/lib/host/log-analyze.d.ts +18 -0
- package/lib/host/log-analyze.js +63 -6
- package/lib/host/log-row.d.ts +3 -0
- package/lib/host/log-row.js +66 -0
- package/lib/host/narrowing.js +11 -9
- package/lib/host/self-check.js +10 -5
- package/lib/host/session-state.d.ts +3 -1
- package/lib/host/session-state.js +30 -46
- package/lib/host/task-completion.js +6 -4
- package/lib/host/types.d.ts +43 -3
- package/package.json +1 -1
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { isObject } from "./types.js";
|
|
3
|
+
const numeric = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
4
|
+
function measuredEvaluator(evaluate, now = () => performance.now()) {
|
|
5
|
+
return async (input, options) => {
|
|
6
|
+
const start = now();
|
|
7
|
+
const telemetry = { requestId: randomUUID(), latencyMs: 0 };
|
|
8
|
+
try {
|
|
9
|
+
telemetry.requestBytes = Buffer.byteLength(JSON.stringify(input));
|
|
10
|
+
} catch {
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
const result = await evaluate(input, options);
|
|
14
|
+
telemetry.latencyMs = Math.max(0, now() - start);
|
|
15
|
+
telemetry.model = result.model;
|
|
16
|
+
if (result.usage) {
|
|
17
|
+
if (numeric(result.usage.input_tokens)) telemetry.inputTokens = result.usage.input_tokens;
|
|
18
|
+
if (numeric(result.usage.output_tokens)) telemetry.outputTokens = result.usage.output_tokens;
|
|
19
|
+
}
|
|
20
|
+
return { ...result, telemetry: { ...telemetry, ...result.telemetry, latencyMs: telemetry.latencyMs } };
|
|
21
|
+
} catch (error) {
|
|
22
|
+
telemetry.latencyMs = Math.max(0, now() - start);
|
|
23
|
+
if (error instanceof Error) {
|
|
24
|
+
const existing = error.telemetry;
|
|
25
|
+
Object.assign(error, { telemetry: { ...telemetry, ...existing, latencyMs: telemetry.latencyMs } });
|
|
26
|
+
}
|
|
27
|
+
throw error;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function createEvaluationTrace(evaluate, now = () => performance.now()) {
|
|
32
|
+
const requestId = randomUUID();
|
|
33
|
+
let started;
|
|
34
|
+
let ended = 0;
|
|
35
|
+
const samples = [];
|
|
36
|
+
const record = (value) => {
|
|
37
|
+
const sample = isObject(value) && isObject(value.telemetry) ? value.telemetry : {};
|
|
38
|
+
samples.push(sample);
|
|
39
|
+
ended = now();
|
|
40
|
+
};
|
|
41
|
+
const traced = async (input, options) => {
|
|
42
|
+
started ??= now();
|
|
43
|
+
try {
|
|
44
|
+
const result = await evaluate(input, options);
|
|
45
|
+
record(result);
|
|
46
|
+
return result;
|
|
47
|
+
} catch (error) {
|
|
48
|
+
record(error);
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
return {
|
|
53
|
+
evaluate: traced,
|
|
54
|
+
detail() {
|
|
55
|
+
if (!samples.length || started === void 0) return {};
|
|
56
|
+
const detail = { requestId, latencyMs: Math.max(0, ended - started) };
|
|
57
|
+
for (const key of ["requestBytes", "backendAttempts", "inputTokens", "outputTokens"]) {
|
|
58
|
+
if (samples.every((sample) => numeric(sample[key]))) detail[key] = samples.reduce((sum, sample) => sum + sample[key], 0);
|
|
59
|
+
}
|
|
60
|
+
const models = new Set(samples.map((sample) => sample.model).filter((model) => typeof model === "string"));
|
|
61
|
+
if (models.size === 1) detail.model = [...models][0];
|
|
62
|
+
else if (models.size > 1) detail.model = "mixed";
|
|
63
|
+
return detail;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export {
|
|
68
|
+
createEvaluationTrace,
|
|
69
|
+
measuredEvaluator
|
|
70
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export type ExecutionEvidence = {
|
|
2
|
+
seq: number;
|
|
3
|
+
step: number;
|
|
4
|
+
tool: string;
|
|
5
|
+
outcome: 'ok' | 'error' | 'unknown';
|
|
6
|
+
command?: string;
|
|
7
|
+
path?: string;
|
|
8
|
+
result?: string;
|
|
9
|
+
};
|
|
10
|
+
export type ExecutionSnapshot = {
|
|
11
|
+
throughSeq: number;
|
|
12
|
+
step: number;
|
|
13
|
+
evidenceSteps: number;
|
|
14
|
+
evidence: ExecutionEvidence[];
|
|
15
|
+
unresolvedFailures: number;
|
|
16
|
+
artifactGeneration: number;
|
|
17
|
+
verificationGeneration?: number;
|
|
18
|
+
verificationAfterChange: boolean;
|
|
19
|
+
fingerprint: string;
|
|
20
|
+
failureFingerprint: string;
|
|
21
|
+
};
|
|
22
|
+
/** Pure, bounded per-turn collector. It neither reads history nor invokes host APIs. */
|
|
23
|
+
export declare function createExecutionObserver(turn: number): {
|
|
24
|
+
snapshot: () => ExecutionSnapshot;
|
|
25
|
+
observe(event: unknown): ExecutionSnapshot | undefined;
|
|
26
|
+
};
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { decisionText } from "./decision-context.js";
|
|
3
|
+
import { redactCredentials, stripInjected } from "./excerpt.js";
|
|
4
|
+
import { isObject } from "./types.js";
|
|
5
|
+
const SHELL = /* @__PURE__ */ new Set(["bash", "shell", "pwsh"]);
|
|
6
|
+
const WRITES = /* @__PURE__ */ new Set(["write", "edit", "file_write", "write_file", "str_replace_editor", "apply_patch"]);
|
|
7
|
+
const RECOVERY = /* @__PURE__ */ new Set(["TOOL_OUTCOME_UNKNOWN", "TOOL_NOT_STARTED", "ABORTED_BEFORE_DISPATCH", "ABORTED"]);
|
|
8
|
+
const MAX_CALLS = 128;
|
|
9
|
+
const MAX_STRATEGIES = 64;
|
|
10
|
+
const OPAQUE = /[A-Za-z0-9_+-]{24,}/g;
|
|
11
|
+
const PERSISTENT_SIGNATURE = /\[Command finished with exit code|\[shell exited|\[Command timed out or OOM|persistent bash shell was reset/i;
|
|
12
|
+
const digest = (value) => createHash("sha256").update(value).digest("hex");
|
|
13
|
+
const integer = (value) => Number.isSafeInteger(value) && value >= 0;
|
|
14
|
+
const bounded = (value, bytes) => decisionText(typeof value === "string" ? value.replace(OPAQUE, "[redacted]") : "", bytes);
|
|
15
|
+
const label = (value, bytes) => decisionText(value, bytes);
|
|
16
|
+
const STATUS_LINE = /^\[(?:exit code: -?\d+|Command finished with exit code -?\d+|shell exited(?:: code -?\d+)?|Command timed out or OOM|timed out after [^\]\n]*|stopped: [^\]\n]*|killed by signal: [^\]\n]*|shell killed by signal: [^\]\n]*|sandbox: [^\]\n]*|still running after [^\]\n]*)\]$/i;
|
|
17
|
+
function shellStatus(text) {
|
|
18
|
+
const lines = text.slice(-4096).split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
19
|
+
for (let index = lines.length - 1; index >= 0; index--) {
|
|
20
|
+
const line = lines[index];
|
|
21
|
+
if (!STATUS_LINE.test(line)) continue;
|
|
22
|
+
if (/^\[still running after/i.test(line)) return { backgrounded: true, explicit: true };
|
|
23
|
+
const code = /^\[(?:exit code: (-?\d+)|Command finished with exit code (-?\d+)|shell exited: code (-?\d+))\]$/i.exec(line);
|
|
24
|
+
if (code) return { exit: Number(code[1] ?? code[2] ?? code[3]), explicit: true };
|
|
25
|
+
if (/^\[shell exited\]$/i.test(line)) return { explicit: true };
|
|
26
|
+
return { terminated: true, explicit: true };
|
|
27
|
+
}
|
|
28
|
+
return { exit: 0 };
|
|
29
|
+
}
|
|
30
|
+
function argumentsOf(value) {
|
|
31
|
+
if (typeof value === "string") {
|
|
32
|
+
if (value.length > 32 * 1024) return {};
|
|
33
|
+
try {
|
|
34
|
+
const parsed = JSON.parse(value);
|
|
35
|
+
return isObject(parsed) ? parsed : {};
|
|
36
|
+
} catch {
|
|
37
|
+
return {};
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return isObject(value) ? value : {};
|
|
41
|
+
}
|
|
42
|
+
function modifies(command) {
|
|
43
|
+
return /(?:\b(?:sed|perl)\b[^\n]*\s-[^\s]*i|\b(?:rm|mv|cp|touch|mkdir|tee)\s|\bgit\s+(?:apply|reset|checkout|restore|clean)\b|(?:^|[^>])>{1,2}\s*\S|\bnpm\s+(?:install|uninstall)\b)/i.test(command);
|
|
44
|
+
}
|
|
45
|
+
function verifies(command) {
|
|
46
|
+
if (/[\n;|&<>`]|\$\(/.test(command)) return false;
|
|
47
|
+
return /^(?:(?:npm|pnpm|yarn)\s+(?:test\b|(?:run\s+)?(?:check|typecheck|lint|build|test(?::[\w-]+)?)\b)|(?:pytest|vitest|jest)\b|node\s+--test\b|(?:cargo|go)\s+test\b)/i.test(command.trim());
|
|
48
|
+
}
|
|
49
|
+
function createExecutionObserver(turn) {
|
|
50
|
+
let throughSeq = -1;
|
|
51
|
+
let latestStep = 0;
|
|
52
|
+
let evidenceSteps = 0;
|
|
53
|
+
let dirtyStep;
|
|
54
|
+
let artifactGeneration = 0;
|
|
55
|
+
let verificationGeneration;
|
|
56
|
+
const calls = /* @__PURE__ */ new Map();
|
|
57
|
+
const failures = /* @__PURE__ */ new Map();
|
|
58
|
+
const explicitShells = /* @__PURE__ */ new Set();
|
|
59
|
+
const evidence = [];
|
|
60
|
+
const snapshot = () => {
|
|
61
|
+
const selected = evidence.map((item) => ({ ...item }));
|
|
62
|
+
const strategies = [...failures].sort(([a], [b]) => a.localeCompare(b));
|
|
63
|
+
return {
|
|
64
|
+
throughSeq,
|
|
65
|
+
step: latestStep,
|
|
66
|
+
evidenceSteps,
|
|
67
|
+
evidence: selected,
|
|
68
|
+
unresolvedFailures: Math.max(0, ...strategies.map(([, count]) => count)),
|
|
69
|
+
artifactGeneration,
|
|
70
|
+
...verificationGeneration === void 0 ? {} : { verificationGeneration },
|
|
71
|
+
verificationAfterChange: verificationGeneration !== void 0 && verificationGeneration < artifactGeneration,
|
|
72
|
+
fingerprint: digest(JSON.stringify([selected, artifactGeneration, verificationGeneration, strategies])),
|
|
73
|
+
failureFingerprint: digest(JSON.stringify(strategies))
|
|
74
|
+
};
|
|
75
|
+
};
|
|
76
|
+
return {
|
|
77
|
+
snapshot,
|
|
78
|
+
observe(event) {
|
|
79
|
+
if (!isObject(event) || !integer(event.seq) || event.seq <= throughSeq || typeof event.type !== "string" || !isObject(event.data)) return;
|
|
80
|
+
throughSeq = event.seq;
|
|
81
|
+
const data = event.data;
|
|
82
|
+
if (data.turn !== turn || !integer(data.step)) return;
|
|
83
|
+
latestStep = data.step;
|
|
84
|
+
if (event.type === "tool/call") {
|
|
85
|
+
if (typeof data.callId !== "string" || !data.callId || data.callId.length > 256 || typeof data.name !== "string") return;
|
|
86
|
+
const rawTool = data.name.slice(0, 256);
|
|
87
|
+
const tool = label(rawTool, 96);
|
|
88
|
+
if (!tool) return;
|
|
89
|
+
const args = argumentsOf(data.arguments);
|
|
90
|
+
const shell = SHELL.has(rawTool);
|
|
91
|
+
const commandValue = args.command ?? args.cmd;
|
|
92
|
+
const commandTruncated = shell && typeof commandValue === "string" && commandValue.length > 32 * 1024;
|
|
93
|
+
const rawCommand = shell && typeof commandValue === "string" ? commandValue.slice(0, 32 * 1024) : "";
|
|
94
|
+
const backgrounded = shell && args.run_in_background === true;
|
|
95
|
+
const command = bounded(rawCommand, 256);
|
|
96
|
+
const pathValue = args.file_path ?? args.path ?? args.filePath;
|
|
97
|
+
const rawPath = typeof pathValue === "string" ? pathValue.slice(0, 32 * 1024) : "";
|
|
98
|
+
const path = bounded(rawPath, 160);
|
|
99
|
+
const strategy = digest(JSON.stringify([rawTool, rawCommand, rawPath]));
|
|
100
|
+
calls.set(data.callId, {
|
|
101
|
+
rawTool,
|
|
102
|
+
tool,
|
|
103
|
+
step: data.step,
|
|
104
|
+
command,
|
|
105
|
+
path,
|
|
106
|
+
strategy,
|
|
107
|
+
shell,
|
|
108
|
+
backgrounded,
|
|
109
|
+
modifies: WRITES.has(rawTool) || shell && (commandTruncated || modifies(rawCommand)),
|
|
110
|
+
verifies: shell && !commandTruncated && verifies(rawCommand) && !modifies(rawCommand)
|
|
111
|
+
});
|
|
112
|
+
if (calls.size > MAX_CALLS) calls.delete(calls.keys().next().value);
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (event.type === "tool/result") {
|
|
116
|
+
if (event.surfaceOp !== void 0 && event.surfaceOp !== "append" && !(isObject(event.surfaceOp) && event.surfaceOp.op === "append")) return;
|
|
117
|
+
const message = isObject(data.message) ? data.message : void 0;
|
|
118
|
+
if (!message || message.role !== "tool") return;
|
|
119
|
+
const source = isObject(message.source) ? message.source : void 0;
|
|
120
|
+
if (source && source.kind !== "tool") return;
|
|
121
|
+
if (source?.callId !== void 0 && message.toolCallId !== void 0 && source.callId !== message.toolCallId) return;
|
|
122
|
+
const id = typeof message.toolCallId === "string" ? message.toolCallId : source?.callId;
|
|
123
|
+
const call = typeof id === "string" ? calls.get(id) : void 0;
|
|
124
|
+
if (!call || call.step !== data.step) return;
|
|
125
|
+
calls.delete(id);
|
|
126
|
+
const error = isObject(data.error) ? data.error : void 0;
|
|
127
|
+
const synthetic = typeof error?.code === "string" && RECOVERY.has(error.code);
|
|
128
|
+
const content = call.shell && Array.isArray(message.content) ? message.content.filter(isObject).filter((block) => block.type === "text" && typeof block.text === "string").slice(0, 16).map((block) => {
|
|
129
|
+
const text = redactCredentials(stripInjected(block.text.replace(OPAQUE, "[redacted]")));
|
|
130
|
+
return text.length <= 32 * 1024 ? text : `${text.slice(0, 24 * 1024)}
|
|
131
|
+
[output omitted]
|
|
132
|
+
${text.slice(-8 * 1024)}`;
|
|
133
|
+
}).join("\n") : "";
|
|
134
|
+
const status = call.shell ? call.backgrounded ? { backgrounded: true } : shellStatus(content) : void 0;
|
|
135
|
+
if (call.shell && status?.explicit && PERSISTENT_SIGNATURE.test(content)) {
|
|
136
|
+
explicitShells.add(call.rawTool);
|
|
137
|
+
if (explicitShells.size > 16) explicitShells.delete(explicitShells.values().next().value);
|
|
138
|
+
}
|
|
139
|
+
const paused = Boolean(call.shell && status && !status.explicit && !call.backgrounded && explicitShells.has(call.rawTool));
|
|
140
|
+
const outcome = synthetic ? "unknown" : message.isError === true ? "error" : !status ? "ok" : status.backgrounded ? "unknown" : paused ? "unknown" : status.terminated ? "error" : status.exit === void 0 ? "unknown" : status.exit === 0 ? "ok" : "error";
|
|
141
|
+
if (outcome === "error") {
|
|
142
|
+
failures.set(call.strategy, Math.min((failures.get(call.strategy) ?? 0) + 1, 1e3));
|
|
143
|
+
if (failures.size > MAX_STRATEGIES) failures.delete(failures.keys().next().value);
|
|
144
|
+
} else if (outcome === "ok") failures.delete(call.strategy);
|
|
145
|
+
if (call.modifies && !synthetic) artifactGeneration++;
|
|
146
|
+
if (call.verifies && outcome === "ok" && !paused && status?.exit === 0) verificationGeneration = artifactGeneration;
|
|
147
|
+
const item = { seq: event.seq, step: data.step, tool: call.tool, outcome };
|
|
148
|
+
if (call.command) item.command = call.command;
|
|
149
|
+
if (call.path) item.path = call.path;
|
|
150
|
+
if (content) item.result = bounded(content, 256);
|
|
151
|
+
evidence.push(item);
|
|
152
|
+
while (evidence.length > 12 || Buffer.byteLength(JSON.stringify(evidence)) > 4096) evidence.shift();
|
|
153
|
+
if (!synthetic) dirtyStep = data.step;
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
if (event.type === "step/end" && dirtyStep === data.step) {
|
|
157
|
+
evidenceSteps++;
|
|
158
|
+
dirtyStep = void 0;
|
|
159
|
+
calls.clear();
|
|
160
|
+
return snapshot();
|
|
161
|
+
}
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
export {
|
|
167
|
+
createExecutionObserver
|
|
168
|
+
};
|
package/lib/host/index.d.ts
CHANGED
package/lib/host/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
2
3
|
import { createUserMessage } from "@deepseek-ai/dsh-llm";
|
|
3
4
|
import { resolveConfig, featureEnabled } from "./config-store.js";
|
|
4
5
|
import { createBackend } from "./backend.js";
|
|
@@ -12,6 +13,7 @@ import { createLoopGuard } from "./loop-guard.js";
|
|
|
12
13
|
import { createLogStore } from "./log-store.js";
|
|
13
14
|
import { createStateStore } from "./state-store.js";
|
|
14
15
|
import { createDecisionContext } from "./decision-context.js";
|
|
16
|
+
import { createCheckpointReview } from "./checkpoint-review.js";
|
|
15
17
|
import { analyzeSessionLogs, reconstructSessionSeed, listSessionDecisions } from "./log-analyze.js";
|
|
16
18
|
import { isObject } from "./types.js";
|
|
17
19
|
const name = "dsh-decision-layer";
|
|
@@ -46,11 +48,16 @@ function agentTools(ctx, agent) {
|
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
async function apply(ctx, options = {}) {
|
|
49
|
-
const path = options.configPath;
|
|
51
|
+
const path = options.configPath ?? (options.storageDir ? join(options.storageDir, "config.json") : void 0);
|
|
52
|
+
const logDir = options.storageDir ? join(options.storageDir, "logs") : void 0;
|
|
50
53
|
const backend = createBackend({ config: () => resolveConfig({ path }) });
|
|
51
54
|
let logStore;
|
|
52
|
-
const stateStore = createStateStore();
|
|
55
|
+
const stateStore = createStateStore({ dir: options.storageDir ? join(options.storageDir, "state") : void 0 });
|
|
56
|
+
let closeCheckpoint;
|
|
53
57
|
const sessions = createSessionState({
|
|
58
|
+
onEnabled: (id, enabled) => {
|
|
59
|
+
if (!enabled) closeCheckpoint?.(id);
|
|
60
|
+
},
|
|
54
61
|
onLog: (entry) => logStore?.append(entry),
|
|
55
62
|
// Restore a resumed session from its durable snapshot; if it has none yet but
|
|
56
63
|
// has persisted JSONL decision history (e.g. a session that predates this
|
|
@@ -58,13 +65,12 @@ async function apply(ctx, options = {}) {
|
|
|
58
65
|
// logs instead. Logs are read across all ports and filtered by the globally
|
|
59
66
|
// unique session id, so a changed instance port never orphans the history.
|
|
60
67
|
loadState: async (id) => await stateStore.load(id) ?? reconstructSessionSeed(id, {
|
|
68
|
+
dir: logDir,
|
|
61
69
|
trendConfig: (await resolveConfig({ path })).checkSettings
|
|
62
70
|
}).catch(() => void 0),
|
|
63
71
|
persistState: (id, state) => stateStore.save(id, state)
|
|
64
72
|
});
|
|
65
|
-
if (typeof ctx.effect === "function") ctx.effect(() => () =>
|
|
66
|
-
void stateStore.dispose();
|
|
67
|
-
});
|
|
73
|
+
if (typeof ctx.effect === "function") ctx.effect(() => () => stateStore.dispose());
|
|
68
74
|
const steeredIds = /* @__PURE__ */ new Map();
|
|
69
75
|
const rememberSteer = (message) => {
|
|
70
76
|
const id = isObject(message) && typeof message.id === "string" ? message.id : void 0;
|
|
@@ -79,6 +85,14 @@ async function apply(ctx, options = {}) {
|
|
|
79
85
|
return agent.steer(message);
|
|
80
86
|
};
|
|
81
87
|
const decisionContext = createDecisionContext({ ignored: (id) => steeredIds.has(id) });
|
|
88
|
+
const checkpoints = createCheckpointReview({
|
|
89
|
+
evaluate: (input, request) => backend.evaluate(input, request),
|
|
90
|
+
sessions,
|
|
91
|
+
settings: async () => (await resolveConfig({ path })).checkpointSettings
|
|
92
|
+
});
|
|
93
|
+
closeCheckpoint = (id) => checkpoints.close(id);
|
|
94
|
+
if (typeof ctx.effect === "function") ctx.effect(() => () => checkpoints.dispose());
|
|
95
|
+
const checkpointRequests = /* @__PURE__ */ new WeakMap();
|
|
82
96
|
const gate = createDangerGate({
|
|
83
97
|
evaluate: (input, request) => backend.evaluate(input, request),
|
|
84
98
|
sessions,
|
|
@@ -186,6 +200,19 @@ async function apply(ctx, options = {}) {
|
|
|
186
200
|
if (typeof id === "string") await sessions.ensureLoaded(id);
|
|
187
201
|
if (typeof id === "string" && Number.isInteger(turn)) {
|
|
188
202
|
const entered = decisionContext.enter(agent, { turn, messages: payload.messages });
|
|
203
|
+
if (agent) {
|
|
204
|
+
try {
|
|
205
|
+
if (entered.snapshot) checkpointRequests.set(agent, { turn, request: entered.snapshot.context.request });
|
|
206
|
+
const cached = checkpointRequests.get(agent);
|
|
207
|
+
checkpoints.enter(agent, {
|
|
208
|
+
turn,
|
|
209
|
+
request: cached && cached.turn === turn ? cached.request : "",
|
|
210
|
+
changed: entered.changed,
|
|
211
|
+
signal: payload.signal
|
|
212
|
+
});
|
|
213
|
+
} catch {
|
|
214
|
+
}
|
|
215
|
+
}
|
|
189
216
|
const previous = narrowedTurns.get(id);
|
|
190
217
|
const turnChanged = !previous || previous.turn !== turn;
|
|
191
218
|
if (turnChanged || entered.changed) {
|
|
@@ -227,13 +254,26 @@ async function apply(ctx, options = {}) {
|
|
|
227
254
|
};
|
|
228
255
|
if (typeof ctx.effect === "function") ctx.effect(installNarrowingHook);
|
|
229
256
|
else installNarrowingHook();
|
|
230
|
-
const
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
257
|
+
const installSessionLifecycle = () => {
|
|
258
|
+
const offEvents = ctx.on("session/event", (subject, event) => {
|
|
259
|
+
try {
|
|
260
|
+
checkpoints.observe(subject, event);
|
|
261
|
+
} catch {
|
|
262
|
+
}
|
|
263
|
+
if (!isObject(event) || event.type !== "turn/end") return;
|
|
264
|
+
const id = isObject(subject) && typeof subject.id === "string" ? subject.id : void 0;
|
|
265
|
+
if (id !== void 0) releaseNarrowing(id);
|
|
266
|
+
});
|
|
267
|
+
const offDisposed = ctx.on("session/disposed", (subject) => {
|
|
268
|
+
if (isObject(subject) && typeof subject.id === "string") checkpoints.close(subject.id);
|
|
269
|
+
});
|
|
270
|
+
return () => {
|
|
271
|
+
offEvents?.();
|
|
272
|
+
offDisposed?.();
|
|
273
|
+
};
|
|
274
|
+
};
|
|
275
|
+
if (typeof ctx.effect === "function") ctx.effect(installSessionLifecycle);
|
|
276
|
+
else installSessionLifecycle();
|
|
237
277
|
}
|
|
238
278
|
const rawSkill = await readFile(new URL("../../skills/decision-layer/SKILL.md", import.meta.url), "utf8");
|
|
239
279
|
const content = rawSkill.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "").trim();
|
|
@@ -248,7 +288,11 @@ async function apply(ctx, options = {}) {
|
|
|
248
288
|
}, additionalProperties: false },
|
|
249
289
|
output,
|
|
250
290
|
timeoutMs: 25e3,
|
|
251
|
-
|
|
291
|
+
// Local measurement metadata never crosses the model-visible tool boundary.
|
|
292
|
+
execute: async (args, exec) => {
|
|
293
|
+
const result = await backend.evaluate(args, { signal: exec.signal });
|
|
294
|
+
return { model: result.model, answers: result.answers, ...result.usage ? { usage: result.usage } : {} };
|
|
295
|
+
}
|
|
252
296
|
});
|
|
253
297
|
ctx.tools.register({
|
|
254
298
|
name: "decision_check_connection",
|
|
@@ -264,19 +308,20 @@ async function apply(ctx, options = {}) {
|
|
|
264
308
|
}
|
|
265
309
|
});
|
|
266
310
|
ctx.inject(["webServer"], (web) => {
|
|
267
|
-
logStore = createLogStore({ port: web.webServer.port });
|
|
311
|
+
logStore = createLogStore({ port: web.webServer.port, dir: logDir });
|
|
268
312
|
const port = web.webServer.port;
|
|
269
313
|
const analyze = async (sessionId) => analyzeSessionLogs(sessionId, {
|
|
270
314
|
port,
|
|
315
|
+
dir: logDir,
|
|
271
316
|
trendConfig: (await resolveConfig({ path })).checkSettings
|
|
272
317
|
});
|
|
273
|
-
const listRows = (sessionId, { page, pageSize, lowOnly } = {}) => listSessionDecisions(sessionId, { port, page, pageSize, lowOnly });
|
|
318
|
+
const listRows = (sessionId, { page, pageSize, lowOnly } = {}) => listSessionDecisions(sessionId, { port, dir: logDir, page, pageSize, lowOnly });
|
|
274
319
|
const unregister = createManagerRoutes({ path, sessions, backend, logStore, analyze, listRows, bindHost: web.webServer.host }).map((route) => web.webServer.register(route));
|
|
275
320
|
return () => {
|
|
276
321
|
unregister.forEach((dispose) => dispose());
|
|
277
322
|
const store = logStore;
|
|
278
323
|
logStore = void 0;
|
|
279
|
-
|
|
324
|
+
return store?.dispose();
|
|
280
325
|
};
|
|
281
326
|
});
|
|
282
327
|
}
|
|
@@ -50,6 +50,24 @@ export type Profile = {
|
|
|
50
50
|
error: number;
|
|
51
51
|
steered: number;
|
|
52
52
|
};
|
|
53
|
+
checkpoint: {
|
|
54
|
+
attempts: number;
|
|
55
|
+
concern: number;
|
|
56
|
+
uncertain: number;
|
|
57
|
+
error: number;
|
|
58
|
+
skipped: number;
|
|
59
|
+
};
|
|
60
|
+
telemetry: {
|
|
61
|
+
measured: number;
|
|
62
|
+
latencyMsSum: number;
|
|
63
|
+
latencyMsP50: number | null;
|
|
64
|
+
latencyMsP95: number | null;
|
|
65
|
+
inputTokens: number;
|
|
66
|
+
outputTokens: number;
|
|
67
|
+
usageMeasured: number;
|
|
68
|
+
backendAttempts: number;
|
|
69
|
+
transportMeasured: number;
|
|
70
|
+
};
|
|
53
71
|
};
|
|
54
72
|
export declare function buildProfile(decisions: ReadonlyMap<string, PersistedRecord> | Iterable<PersistedRecord>): Profile;
|
|
55
73
|
export declare function backtestTrend(checkRecords: Iterable<unknown>, { trendMinConfidence, trendRun, trendSevereRun, lowScoreThreshold }?: TrendConfig): TrendBacktest;
|
package/lib/host/log-analyze.js
CHANGED
|
@@ -2,7 +2,8 @@ import { createReadableStream } from "./log-read.js";
|
|
|
2
2
|
import { advanceRun, severityOf } from "./quality-trend.js";
|
|
3
3
|
import { narrowApplied } from "./narrowing.js";
|
|
4
4
|
import { isObject } from "./types.js";
|
|
5
|
-
|
|
5
|
+
import { sanitizeLogRow } from "./log-row.js";
|
|
6
|
+
const DECISION_KINDS = /* @__PURE__ */ new Set(["gate", "check", "narrow", "complete", "checkpoint"]);
|
|
6
7
|
const atOf = (record) => {
|
|
7
8
|
const value = record.at;
|
|
8
9
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
@@ -22,7 +23,8 @@ function foldSession(records) {
|
|
|
22
23
|
continue;
|
|
23
24
|
}
|
|
24
25
|
if (DECISION_KINDS.has(record.kind) && typeof record.id === "string") {
|
|
25
|
-
|
|
26
|
+
const row = sanitizeLogRow(record);
|
|
27
|
+
if (row) decisions.set(record.id, row);
|
|
26
28
|
}
|
|
27
29
|
}
|
|
28
30
|
return { decisions, annotations };
|
|
@@ -39,14 +41,43 @@ function emptyProfile() {
|
|
|
39
41
|
gate: { attempts: 0, allow: 0, ask: 0, deny: 0, error: 0 },
|
|
40
42
|
check: { attempts: 0, ok: 0, low: 0, error: 0, scoreSum: 0, scoreCount: 0, confidence: { lt40: 0, lt60: 0, lt80: 0, gte80: 0 } },
|
|
41
43
|
narrow: { attempts: 0, applied: 0, error: 0, droppedSum: 0, tooManyCandidates: 0 },
|
|
42
|
-
complete: { attempts: 0, satisfied: 0, unsatisfied: 0, insufficient: 0, error: 0, steered: 0 }
|
|
44
|
+
complete: { attempts: 0, satisfied: 0, unsatisfied: 0, insufficient: 0, error: 0, steered: 0 },
|
|
45
|
+
checkpoint: { attempts: 0, concern: 0, uncertain: 0, error: 0, skipped: 0 },
|
|
46
|
+
telemetry: {
|
|
47
|
+
measured: 0,
|
|
48
|
+
latencyMsSum: 0,
|
|
49
|
+
latencyMsP50: null,
|
|
50
|
+
latencyMsP95: null,
|
|
51
|
+
inputTokens: 0,
|
|
52
|
+
outputTokens: 0,
|
|
53
|
+
usageMeasured: 0,
|
|
54
|
+
backendAttempts: 0,
|
|
55
|
+
transportMeasured: 0
|
|
56
|
+
}
|
|
43
57
|
};
|
|
44
58
|
}
|
|
45
59
|
function buildProfile(decisions) {
|
|
46
60
|
const profile = emptyProfile();
|
|
47
61
|
const rows = decisions instanceof Map ? [...decisions.values()] : [...decisions];
|
|
62
|
+
const latencies = [];
|
|
48
63
|
for (const record of rows) {
|
|
49
64
|
const kind = record.kind;
|
|
65
|
+
if (DECISION_KINDS.has(kind) && record.outcome !== "skipped") {
|
|
66
|
+
if (typeof record.latencyMs === "number" && Number.isFinite(record.latencyMs) && record.latencyMs >= 0) {
|
|
67
|
+
profile.telemetry.measured++;
|
|
68
|
+
profile.telemetry.latencyMsSum += record.latencyMs;
|
|
69
|
+
latencies.push(record.latencyMs);
|
|
70
|
+
}
|
|
71
|
+
if (Number.isSafeInteger(record.inputTokens) && record.inputTokens >= 0 && Number.isSafeInteger(record.outputTokens) && record.outputTokens >= 0) {
|
|
72
|
+
profile.telemetry.usageMeasured++;
|
|
73
|
+
profile.telemetry.inputTokens += record.inputTokens;
|
|
74
|
+
profile.telemetry.outputTokens += record.outputTokens;
|
|
75
|
+
}
|
|
76
|
+
if (Number.isSafeInteger(record.backendAttempts) && record.backendAttempts >= 0) {
|
|
77
|
+
profile.telemetry.transportMeasured++;
|
|
78
|
+
profile.telemetry.backendAttempts += record.backendAttempts;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
50
81
|
switch (kind) {
|
|
51
82
|
case "gate": {
|
|
52
83
|
const bucket = profile.gate;
|
|
@@ -90,10 +121,26 @@ function buildProfile(decisions) {
|
|
|
90
121
|
if (record.steered === true) bucket.steered++;
|
|
91
122
|
break;
|
|
92
123
|
}
|
|
124
|
+
case "checkpoint": {
|
|
125
|
+
const bucket = profile.checkpoint;
|
|
126
|
+
if (record.outcome === "skipped") bucket.skipped++;
|
|
127
|
+
else {
|
|
128
|
+
bucket.attempts++;
|
|
129
|
+
if (record.outcome === "error") bucket.error++;
|
|
130
|
+
else if (record.outcome === "concern") bucket.concern++;
|
|
131
|
+
else if (record.outcome === "uncertain") bucket.uncertain++;
|
|
132
|
+
}
|
|
133
|
+
break;
|
|
134
|
+
}
|
|
93
135
|
default:
|
|
94
136
|
break;
|
|
95
137
|
}
|
|
96
138
|
}
|
|
139
|
+
latencies.sort((a, b) => a - b);
|
|
140
|
+
if (latencies.length) {
|
|
141
|
+
profile.telemetry.latencyMsP50 = latencies[Math.ceil(latencies.length * 0.5) - 1];
|
|
142
|
+
profile.telemetry.latencyMsP95 = latencies[Math.ceil(latencies.length * 0.95) - 1];
|
|
143
|
+
}
|
|
97
144
|
return profile;
|
|
98
145
|
}
|
|
99
146
|
function backtestTrend(checkRecords, { trendMinConfidence = 0.6, trendRun = 3, trendSevereRun = 5, lowScoreThreshold = 1 } = {}) {
|
|
@@ -158,16 +205,26 @@ function buildSeed(decisions, { maxLog = 50, trendConfig = {} } = {}) {
|
|
|
158
205
|
check: { attempts: 0, failures: 0, low: 0, run: 0, severity: "normal" },
|
|
159
206
|
narrow: { attempts: 0, failures: 0, applied: 0, dropped: 0 },
|
|
160
207
|
complete: { attempts: 0, failures: 0, unsatisfied: 0, satisfied: 0, insufficient: 0, steered: 0 },
|
|
208
|
+
checkpoint: { attempts: 0, failures: 0, concern: 0, uncertain: 0, skipped: 0 },
|
|
161
209
|
log: []
|
|
162
210
|
};
|
|
163
211
|
const minConfidence = typeof trendConfig.trendMinConfidence === "number" ? trendConfig.trendMinConfidence : 0.6;
|
|
164
212
|
const thresholds = { trendRun: trendConfig.trendRun ?? 3, trendSevereRun: trendConfig.trendSevereRun ?? 5 };
|
|
165
213
|
let run = 0;
|
|
166
214
|
for (const record of ordered) {
|
|
167
|
-
|
|
215
|
+
const checkpoint = record.kind === "checkpoint";
|
|
216
|
+
if (!checkpoint && record.outcome !== "skipped") seed.attempts++;
|
|
168
217
|
const isError = record.outcome === "error";
|
|
169
|
-
if (isError) seed.failures++;
|
|
170
|
-
if (
|
|
218
|
+
if (!checkpoint && isError) seed.failures++;
|
|
219
|
+
if (checkpoint) {
|
|
220
|
+
if (record.outcome === "skipped") seed.checkpoint.skipped++;
|
|
221
|
+
else {
|
|
222
|
+
seed.checkpoint.attempts++;
|
|
223
|
+
if (isError) seed.checkpoint.failures++;
|
|
224
|
+
else if (record.outcome === "concern") seed.checkpoint.concern++;
|
|
225
|
+
else if (record.outcome === "uncertain") seed.checkpoint.uncertain++;
|
|
226
|
+
}
|
|
227
|
+
} else if (record.kind === "gate") {
|
|
171
228
|
seed.gate.attempts++;
|
|
172
229
|
if (isError) seed.gate.failures++;
|
|
173
230
|
if (record.outcome === "allow") seed.gate.allow++;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { redactCredentials } from "./excerpt.js";
|
|
2
|
+
import { isObject } from "./types.js";
|
|
3
|
+
const KINDS = /* @__PURE__ */ new Set(["gate", "check", "narrow", "complete", "checkpoint"]);
|
|
4
|
+
const TRIGGERS = /* @__PURE__ */ new Set(["heartbeat", "unresolved_failure", "verification_after_change"]);
|
|
5
|
+
const INTEGER_FIELDS = [
|
|
6
|
+
"turn",
|
|
7
|
+
"step",
|
|
8
|
+
"dropped",
|
|
9
|
+
"kept",
|
|
10
|
+
"candidates",
|
|
11
|
+
"conditions",
|
|
12
|
+
"satisfied",
|
|
13
|
+
"unsatisfied",
|
|
14
|
+
"insufficient",
|
|
15
|
+
"throughSeq",
|
|
16
|
+
"evidenceCount",
|
|
17
|
+
"inputTokens",
|
|
18
|
+
"outputTokens",
|
|
19
|
+
"requestBytes",
|
|
20
|
+
"backendAttempts"
|
|
21
|
+
];
|
|
22
|
+
function sanitizeLogRow(value) {
|
|
23
|
+
if (!isObject(value) || typeof value.kind !== "string" || !KINDS.has(value.kind)) return void 0;
|
|
24
|
+
const row = { kind: value.kind };
|
|
25
|
+
for (const [key, limit] of [
|
|
26
|
+
["id", 160],
|
|
27
|
+
["tool", 128],
|
|
28
|
+
["suggestion", 16],
|
|
29
|
+
["action", 32],
|
|
30
|
+
["reason", 64],
|
|
31
|
+
["subject", 120],
|
|
32
|
+
["task", 512],
|
|
33
|
+
["command", 512],
|
|
34
|
+
["path", 512],
|
|
35
|
+
["mode", 16],
|
|
36
|
+
["requestId", 160],
|
|
37
|
+
["callId", 160],
|
|
38
|
+
["model", 128]
|
|
39
|
+
]) {
|
|
40
|
+
const text = value[key];
|
|
41
|
+
if (typeof text === "string" && text.trim()) row[key] = redactCredentials(text).replace(/\s+/g, " ").trim().slice(0, limit);
|
|
42
|
+
}
|
|
43
|
+
for (const key of INTEGER_FIELDS) {
|
|
44
|
+
const n = value[key];
|
|
45
|
+
if (Number.isSafeInteger(n) && n >= 0) row[key] = n;
|
|
46
|
+
}
|
|
47
|
+
for (const key of ["at", "latencyMs", "score", "confidence"]) {
|
|
48
|
+
const n = value[key];
|
|
49
|
+
if (typeof n === "number" && Number.isFinite(n) && n >= 0 && (key !== "confidence" || n <= 1)) {
|
|
50
|
+
row[key] = key === "at" ? n : Math.round(n * 100) / 100;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
row.outcome = typeof value.outcome === "string" ? value.outcome.slice(0, 32) : "";
|
|
54
|
+
if (typeof value.applied === "boolean") row.applied = value.applied;
|
|
55
|
+
if (typeof value.steered === "boolean") row.steered = value.steered;
|
|
56
|
+
if (value.lowConfidence === true) row.lowConfidence = true;
|
|
57
|
+
if (typeof value.trigger === "string" && TRIGGERS.has(value.trigger)) row.trigger = value.trigger;
|
|
58
|
+
if (Array.isArray(value.tools)) {
|
|
59
|
+
const names = value.tools.filter((name) => typeof name === "string" && Boolean(name.trim())).slice(0, 64).map((name) => redactCredentials(name.trim()).slice(0, 128));
|
|
60
|
+
if (names.length) row.tools = names;
|
|
61
|
+
}
|
|
62
|
+
return row;
|
|
63
|
+
}
|
|
64
|
+
export {
|
|
65
|
+
sanitizeLogRow
|
|
66
|
+
};
|