@kweaver-ai/kweaver-sdk 0.7.4 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/README.zh.md +18 -0
- package/dist/api/agent-observability.d.ts +51 -0
- package/dist/api/agent-observability.js +108 -0
- package/dist/api/conversations.d.ts +4 -8
- package/dist/api/conversations.js +16 -58
- package/dist/api/datasources.d.ts +2 -20
- package/dist/api/datasources.js +7 -123
- package/dist/api/trace.d.ts +44 -0
- package/dist/api/trace.js +81 -0
- package/dist/api/vega.d.ts +53 -0
- package/dist/api/vega.js +144 -0
- package/dist/cli.js +5 -0
- package/dist/commands/bkn-ops.js +12 -6
- package/dist/commands/bkn-utils.d.ts +9 -0
- package/dist/commands/bkn-utils.js +17 -0
- package/dist/commands/ds.js +7 -2
- package/dist/commands/trace.d.ts +14 -0
- package/dist/commands/trace.js +168 -0
- package/dist/resources/datasources.js +2 -1
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.js +15 -0
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.yaml +16 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.js +44 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/register.d.ts +1 -0
- package/dist/trace-core/diagnose/builtin-rules/register.js +11 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.js +29 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.js +45 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.js +38 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.yaml +16 -0
- package/dist/trace-core/diagnose/index.d.ts +9 -0
- package/dist/trace-core/diagnose/index.js +104 -0
- package/dist/trace-core/diagnose/predicate-registry.d.ts +7 -0
- package/dist/trace-core/diagnose/predicate-registry.js +30 -0
- package/dist/trace-core/diagnose/report-assembler.d.ts +12 -0
- package/dist/trace-core/diagnose/report-assembler.js +90 -0
- package/dist/trace-core/diagnose/rule-loader.d.ts +11 -0
- package/dist/trace-core/diagnose/rule-loader.js +86 -0
- package/dist/trace-core/diagnose/schemas.d.ts +109 -0
- package/dist/trace-core/diagnose/schemas.js +94 -0
- package/dist/trace-core/diagnose/signal-probe.d.ts +5 -0
- package/dist/trace-core/diagnose/signal-probe.js +21 -0
- package/dist/trace-core/diagnose/synthesizer-template.d.ts +2 -0
- package/dist/trace-core/diagnose/synthesizer-template.js +49 -0
- package/dist/trace-core/diagnose/trace-shaper.d.ts +3 -0
- package/dist/trace-core/diagnose/trace-shaper.js +72 -0
- package/dist/trace-core/diagnose/types.d.ts +124 -0
- package/dist/trace-core/diagnose/types.js +1 -0
- package/package.json +14 -4
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
function resultCount(s) {
|
|
2
|
+
const v = s.attributes["gen_ai.retrieval.result_count"];
|
|
3
|
+
return typeof v === "number" ? v : null;
|
|
4
|
+
}
|
|
5
|
+
export const predicate = (trace) => {
|
|
6
|
+
const ordered = trace.spans
|
|
7
|
+
.slice()
|
|
8
|
+
.sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
9
|
+
const hits = [];
|
|
10
|
+
for (let i = 0; i < ordered.length; i++) {
|
|
11
|
+
const s = ordered[i];
|
|
12
|
+
if (s.kind !== "retrieval")
|
|
13
|
+
continue;
|
|
14
|
+
if (resultCount(s) !== 0)
|
|
15
|
+
continue;
|
|
16
|
+
const next = ordered[i + 1];
|
|
17
|
+
if (!next)
|
|
18
|
+
continue;
|
|
19
|
+
if (next.kind === "llm") {
|
|
20
|
+
hits.push({
|
|
21
|
+
evidenceSpans: [s.spanId, next.spanId],
|
|
22
|
+
excerpt: `retrieval returned 0 results; next step was LLM generation with no fallback path`,
|
|
23
|
+
bindings: {},
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
// retrieval (retry/rewrite) or tool (alt source) → no hit
|
|
27
|
+
}
|
|
28
|
+
return hits;
|
|
29
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: retrieval_empty_no_fallback
|
|
3
|
+
severity: medium
|
|
4
|
+
symptom: empty_retrieval_result_no_fallback_path
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: cascading_error
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "when retrieval returns 0 results, branch to query rewrite, alternate source, or explicit 'no answer' before generating"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "if(retrieval.result_count == 0): next_step in [retry, rewrite, alt_source, no_answer]"
|
|
14
|
+
predicate: builtin:retrieval_empty_no_fallback
|
|
15
|
+
params: {}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
function getPrompt(s) {
|
|
2
|
+
const v = s.attributes["gen_ai.prompt"] ?? s.attributes["llm.prompt"];
|
|
3
|
+
return typeof v === "string" ? v : "";
|
|
4
|
+
}
|
|
5
|
+
function getErrorMessage(s) {
|
|
6
|
+
const v = s.attributes["error.message"];
|
|
7
|
+
return typeof v === "string" ? v : "";
|
|
8
|
+
}
|
|
9
|
+
function getToolName(s) {
|
|
10
|
+
const v = s.attributes["gen_ai.tool.name"];
|
|
11
|
+
return typeof v === "string" ? v : s.name;
|
|
12
|
+
}
|
|
13
|
+
export const predicate = (trace) => {
|
|
14
|
+
const allSpans = trace.spans
|
|
15
|
+
.slice()
|
|
16
|
+
.sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
17
|
+
const hits = [];
|
|
18
|
+
for (let i = 0; i < allSpans.length; i++) {
|
|
19
|
+
const s = allSpans[i];
|
|
20
|
+
if (s.kind !== "tool" || s.status !== "error")
|
|
21
|
+
continue;
|
|
22
|
+
const errMsg = getErrorMessage(s);
|
|
23
|
+
const toolName = getToolName(s);
|
|
24
|
+
// find next LLM span
|
|
25
|
+
let next;
|
|
26
|
+
for (let j = i + 1; j < allSpans.length; j++) {
|
|
27
|
+
if (allSpans[j].kind === "llm") {
|
|
28
|
+
next = allSpans[j];
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (!next)
|
|
33
|
+
continue;
|
|
34
|
+
const prompt = getPrompt(next).toLowerCase();
|
|
35
|
+
const errInPrompt = errMsg.length > 0 && prompt.includes(errMsg.toLowerCase());
|
|
36
|
+
if (!errInPrompt) {
|
|
37
|
+
hits.push({
|
|
38
|
+
evidenceSpans: [s.spanId, next.spanId],
|
|
39
|
+
excerpt: `tool '${toolName}' errored ('${errMsg}') but next LLM prompt did not propagate the error`,
|
|
40
|
+
bindings: { tool_name: toolName, error_message: errMsg },
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return hits;
|
|
45
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: tool_error_swallowed
|
|
3
|
+
severity: high
|
|
4
|
+
symptom: tool_error_not_propagated_to_next_prompt
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: cascading_error
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "after tool '{{tool_name}}' errors, include error.message in the next LLM prompt or take a recovery branch"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "next_llm_prompt_after({{tool_name}}_error).contains(error.message)"
|
|
14
|
+
predicate: builtin:tool_error_swallowed
|
|
15
|
+
params: {}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
const STATE_KEY = "gen_ai.conversation.state";
|
|
2
|
+
function toolName(s) {
|
|
3
|
+
const v = s.attributes["gen_ai.tool.name"];
|
|
4
|
+
return typeof v === "string" ? v : s.name;
|
|
5
|
+
}
|
|
6
|
+
function deepEqual(a, b) {
|
|
7
|
+
return JSON.stringify(a) === JSON.stringify(b); // PR-A: simple JSON compare; sufficient for tool args
|
|
8
|
+
}
|
|
9
|
+
export const predicate = (trace, params) => {
|
|
10
|
+
const minConsecutive = params.min_consecutive ?? 3;
|
|
11
|
+
const tools = (trace.byKind.get("tool") ?? []).slice().sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
12
|
+
const hits = [];
|
|
13
|
+
let i = 0;
|
|
14
|
+
while (i < tools.length) {
|
|
15
|
+
const start = tools[i];
|
|
16
|
+
const startName = toolName(start);
|
|
17
|
+
const startArgs = start.attributes["gen_ai.tool.args"];
|
|
18
|
+
const startState = start.attributes[STATE_KEY];
|
|
19
|
+
let j = i + 1;
|
|
20
|
+
while (j < tools.length &&
|
|
21
|
+
toolName(tools[j]) === startName &&
|
|
22
|
+
deepEqual(tools[j].attributes["gen_ai.tool.args"], startArgs) &&
|
|
23
|
+
// state unchanged across the run (or both undefined)
|
|
24
|
+
(tools[j].attributes[STATE_KEY] === startState || (startState === undefined && tools[j].attributes[STATE_KEY] === undefined)))
|
|
25
|
+
j++;
|
|
26
|
+
const runLen = j - i;
|
|
27
|
+
if (runLen >= minConsecutive) {
|
|
28
|
+
const evidenceSpans = tools.slice(i, j).map((s) => s.spanId);
|
|
29
|
+
hits.push({
|
|
30
|
+
evidenceSpans,
|
|
31
|
+
excerpt: `tool '${startName}' called ${runLen} times consecutively with identical args; conversation state unchanged`,
|
|
32
|
+
bindings: { tool_name: startName, loop_count: runLen, max_count: minConsecutive - 1 },
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
i = j;
|
|
36
|
+
}
|
|
37
|
+
return hits;
|
|
38
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: tool_loop_no_state_change
|
|
3
|
+
severity: high
|
|
4
|
+
symptom: repeated_tool_call_without_state_change
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: retry_loop
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "add stop condition after {{loop_count}} equivalent failed retrievals of '{{tool_name}}'"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "tool_call_count({{tool_name}}) <= {{max_count}}"
|
|
14
|
+
predicate: builtin:tool_loop_no_state_change
|
|
15
|
+
params:
|
|
16
|
+
min_consecutive: 3
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { RuleLoadError } from "./rule-loader.js";
|
|
2
|
+
import { RuleProbeError } from "./signal-probe.js";
|
|
3
|
+
import type { DiagnoseOpts, Report } from "./types.js";
|
|
4
|
+
import "./builtin-rules/register.js";
|
|
5
|
+
export declare class TraceNotFoundError extends Error {
|
|
6
|
+
constructor(conversationId: string);
|
|
7
|
+
}
|
|
8
|
+
export declare function diagnose(conversationId: string, opts: DiagnoseOpts): Promise<Report>;
|
|
9
|
+
export { TraceNotFoundError as DiagnoseTraceNotFound, RuleLoadError, RuleProbeError };
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import yaml from "js-yaml";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { getSpansByConversationId } from "../../api/trace.js";
|
|
6
|
+
import { assembleTraceTree } from "./trace-shaper.js";
|
|
7
|
+
import { loadRules, RuleLoadError } from "./rule-loader.js";
|
|
8
|
+
import { runRules, RuleProbeError } from "./signal-probe.js";
|
|
9
|
+
import { templateSynthesize } from "./synthesizer-template.js";
|
|
10
|
+
import { assembleReport, reportToYamlObject } from "./report-assembler.js";
|
|
11
|
+
import "./builtin-rules/register.js"; // side effect: registers all builtin predicates
|
|
12
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const BUILTIN_DIR = path.join(__dirname, "builtin-rules");
|
|
14
|
+
export class TraceNotFoundError extends Error {
|
|
15
|
+
constructor(conversationId) {
|
|
16
|
+
super(`no spans found for conversation: ${conversationId}`);
|
|
17
|
+
this.name = "TraceNotFoundError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export async function diagnose(conversationId, opts) {
|
|
21
|
+
// PR-A: opts.noLlm, opts.agentProvider, opts.timeoutMs are reserved for PR-B's
|
|
22
|
+
// agent / rubric path; they are accepted by the interface but not consumed here.
|
|
23
|
+
const cwdRulesDir = opts.rulesDir ?? path.join(process.cwd(), "diagnosis-rules");
|
|
24
|
+
const fetched = await getSpansByConversationId({
|
|
25
|
+
baseUrl: opts.baseUrl,
|
|
26
|
+
token: opts.token,
|
|
27
|
+
businessDomain: opts.businessDomain,
|
|
28
|
+
conversationId,
|
|
29
|
+
});
|
|
30
|
+
const rawSpans = fetched.spans;
|
|
31
|
+
if (rawSpans.length === 0)
|
|
32
|
+
throw new TraceNotFoundError(conversationId);
|
|
33
|
+
// A conversation may produce multiple OTel traces (one per turn). PR-A
|
|
34
|
+
// diagnose is single-trace: pick the first observed traceId; warn on extras.
|
|
35
|
+
const observedTraceIds = fetched.traceIds.length > 0
|
|
36
|
+
? fetched.traceIds
|
|
37
|
+
: [...new Set(rawSpans.map((s) => s.traceId).filter((t) => Boolean(t)))];
|
|
38
|
+
const primaryTraceId = observedTraceIds[0] ?? conversationId;
|
|
39
|
+
if (observedTraceIds.length > 1) {
|
|
40
|
+
process.stderr.write(`warning: conversation ${conversationId} has ${observedTraceIds.length} traces; diagnosing the first (${primaryTraceId})\n`);
|
|
41
|
+
}
|
|
42
|
+
const spansForPrimary = observedTraceIds.length > 0
|
|
43
|
+
? rawSpans.filter((s) => !s.traceId || s.traceId === primaryTraceId)
|
|
44
|
+
: rawSpans;
|
|
45
|
+
const tree = assembleTraceTree(primaryTraceId, spansForPrimary);
|
|
46
|
+
const rules = await loadRules({
|
|
47
|
+
builtinDir: BUILTIN_DIR,
|
|
48
|
+
cwdRulesDir,
|
|
49
|
+
extraRulesDir: null,
|
|
50
|
+
noBuiltin: opts.noBuiltin,
|
|
51
|
+
});
|
|
52
|
+
const hits = await runRules(rules, tree);
|
|
53
|
+
const version = await cliVersion();
|
|
54
|
+
// Build provisional findings list to feed the synthesizer.
|
|
55
|
+
const provisionalReport = assembleReport({
|
|
56
|
+
traceId: primaryTraceId,
|
|
57
|
+
agentId: extractAgentId(tree),
|
|
58
|
+
tenant: extractTenant(tree),
|
|
59
|
+
cliVersion: version,
|
|
60
|
+
rules,
|
|
61
|
+
hits,
|
|
62
|
+
summary: { headline: "", primaryRootCause: null, fixPriority: [], crossFindingLinks: [] },
|
|
63
|
+
});
|
|
64
|
+
const summary = templateSynthesize(provisionalReport.findings);
|
|
65
|
+
const report = { ...provisionalReport, summary };
|
|
66
|
+
if (opts.out !== null) {
|
|
67
|
+
await fs.mkdir(path.dirname(opts.out), { recursive: true });
|
|
68
|
+
await fs.writeFile(opts.out, yaml.dump(reportToYamlObject(report)), "utf8");
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
process.stdout.write(yaml.dump(reportToYamlObject(report)));
|
|
72
|
+
}
|
|
73
|
+
if (report.findings.length === 0) {
|
|
74
|
+
process.stderr.write("no findings\n");
|
|
75
|
+
}
|
|
76
|
+
return report;
|
|
77
|
+
}
|
|
78
|
+
function extractAgentId(tree) {
|
|
79
|
+
for (const s of tree.spans) {
|
|
80
|
+
const v = s.attributes["gen_ai.agent.id"];
|
|
81
|
+
if (typeof v === "string")
|
|
82
|
+
return v;
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
function extractTenant(tree) {
|
|
87
|
+
for (const s of tree.spans) {
|
|
88
|
+
const v = s.attributes["tenant"];
|
|
89
|
+
if (typeof v === "string")
|
|
90
|
+
return v;
|
|
91
|
+
}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
async function cliVersion() {
|
|
95
|
+
try {
|
|
96
|
+
const pkgPath = path.join(__dirname, "..", "..", "..", "package.json");
|
|
97
|
+
const txt = await fs.readFile(pkgPath, "utf8");
|
|
98
|
+
return JSON.parse(txt).version ?? "0.0.0";
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return "0.0.0";
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
export { TraceNotFoundError as DiagnoseTraceNotFound, RuleLoadError, RuleProbeError };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Predicate } from "./types.js";
|
|
2
|
+
export declare class PredicateNotFoundError extends Error {
|
|
3
|
+
constructor(name: string);
|
|
4
|
+
}
|
|
5
|
+
export declare function registerPredicate(name: string, fn: Predicate): void;
|
|
6
|
+
export declare function resolvePredicate(ref: string): Predicate;
|
|
7
|
+
export declare function clearRegistry(): void;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export class PredicateNotFoundError extends Error {
|
|
2
|
+
constructor(name) {
|
|
3
|
+
super(`predicate not registered: ${name}`);
|
|
4
|
+
this.name = "PredicateNotFoundError";
|
|
5
|
+
}
|
|
6
|
+
}
|
|
7
|
+
const REGISTRY = new Map();
|
|
8
|
+
export function registerPredicate(name, fn) {
|
|
9
|
+
if (REGISTRY.has(name)) {
|
|
10
|
+
throw new Error(`predicate already registered: ${name}`);
|
|
11
|
+
}
|
|
12
|
+
REGISTRY.set(name, fn);
|
|
13
|
+
}
|
|
14
|
+
export function resolvePredicate(ref) {
|
|
15
|
+
const m = ref.match(/^([a-z-]+):(.+)$/);
|
|
16
|
+
if (!m)
|
|
17
|
+
throw new Error(`malformed predicate ref: ${ref}`);
|
|
18
|
+
const [, scheme, name] = m;
|
|
19
|
+
if (scheme !== "builtin") {
|
|
20
|
+
throw new Error(`unsupported predicate scheme: ${scheme} (only 'builtin:' is allowed in PR-A)`);
|
|
21
|
+
}
|
|
22
|
+
const fn = REGISTRY.get(name);
|
|
23
|
+
if (!fn)
|
|
24
|
+
throw new PredicateNotFoundError(name);
|
|
25
|
+
return fn;
|
|
26
|
+
}
|
|
27
|
+
// Test-only escape hatch.
|
|
28
|
+
export function clearRegistry() {
|
|
29
|
+
REGISTRY.clear();
|
|
30
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Hit, Report, Rule, Summary } from "./types.js";
|
|
2
|
+
export interface AssembleReportOpts {
|
|
3
|
+
traceId: string;
|
|
4
|
+
agentId: string | null;
|
|
5
|
+
tenant: string | null;
|
|
6
|
+
cliVersion: string;
|
|
7
|
+
rules: Rule[];
|
|
8
|
+
hits: Map<string, Hit[]>;
|
|
9
|
+
summary: Summary;
|
|
10
|
+
}
|
|
11
|
+
export declare function assembleReport(opts: AssembleReportOpts): Report;
|
|
12
|
+
export declare function reportToYamlObject(r: Report): unknown;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
function renderTemplate(tpl, bindings) {
|
|
2
|
+
return tpl.replace(/{{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*}}/g, (_, key) => {
|
|
3
|
+
const v = bindings[key];
|
|
4
|
+
return v === undefined ? `{{${key}}}` : String(v);
|
|
5
|
+
});
|
|
6
|
+
}
|
|
7
|
+
export function assembleReport(opts) {
|
|
8
|
+
const findings = [];
|
|
9
|
+
for (const rule of opts.rules) {
|
|
10
|
+
const ruleHits = opts.hits.get(rule.id) ?? [];
|
|
11
|
+
for (const hit of ruleHits) {
|
|
12
|
+
findings.push({
|
|
13
|
+
ruleId: rule.id,
|
|
14
|
+
judgmentKind: "symbolic",
|
|
15
|
+
severity: rule.severity,
|
|
16
|
+
symptom: rule.symptom,
|
|
17
|
+
likelyCause: rule.symptom, // PR-A: no LLM, so we mirror symptom; PR-B agent overrides this
|
|
18
|
+
evidence: { spans: hit.evidenceSpans, excerpt: hit.excerpt },
|
|
19
|
+
suggestedFix: {
|
|
20
|
+
target: rule.suggestedFix.target,
|
|
21
|
+
change: renderTemplate(rule.suggestedFix.changeTemplate, hit.bindings),
|
|
22
|
+
},
|
|
23
|
+
confidence: "low",
|
|
24
|
+
verifyWith: {
|
|
25
|
+
suggestedEvalCase: {
|
|
26
|
+
queryId: null, // PR-A: no query extraction yet (deferred per spec)
|
|
27
|
+
query: null,
|
|
28
|
+
assertions: rule.verifyWith.assertionTemplates.map((t) => renderTemplate(t, hit.bindings)),
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
schemaVersion: "trace-diagnose-report/v1",
|
|
36
|
+
trace: { traceId: opts.traceId, agentId: opts.agentId, tenant: opts.tenant },
|
|
37
|
+
run: {
|
|
38
|
+
diagnosedAt: new Date().toISOString(),
|
|
39
|
+
cliVersion: opts.cliVersion,
|
|
40
|
+
mode: "symbolic-only",
|
|
41
|
+
rulesApplied: opts.rules.map((r) => r.id),
|
|
42
|
+
rulesSkipped: [],
|
|
43
|
+
synthesizerMode: "template",
|
|
44
|
+
},
|
|
45
|
+
summary: opts.summary,
|
|
46
|
+
findings,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// Convert internal camelCase Report to the snake_case shape used by ReportSchema (and by yaml output).
|
|
50
|
+
export function reportToYamlObject(r) {
|
|
51
|
+
return {
|
|
52
|
+
schema_version: r.schemaVersion,
|
|
53
|
+
trace: { trace_id: r.trace.traceId, agent_id: r.trace.agentId, tenant: r.trace.tenant },
|
|
54
|
+
run: {
|
|
55
|
+
diagnosed_at: r.run.diagnosedAt,
|
|
56
|
+
cli_version: r.run.cliVersion,
|
|
57
|
+
mode: r.run.mode,
|
|
58
|
+
rules_applied: r.run.rulesApplied,
|
|
59
|
+
rules_skipped: r.run.rulesSkipped.map((s) => ({ rule_id: s.ruleId, reason: s.reason })),
|
|
60
|
+
synthesizer_mode: r.run.synthesizerMode,
|
|
61
|
+
},
|
|
62
|
+
summary: {
|
|
63
|
+
headline: r.summary.headline,
|
|
64
|
+
primary_root_cause: r.summary.primaryRootCause === null ? null : {
|
|
65
|
+
finding_ids: r.summary.primaryRootCause.findingIds,
|
|
66
|
+
description: r.summary.primaryRootCause.description,
|
|
67
|
+
target_for_fix: r.summary.primaryRootCause.targetForFix,
|
|
68
|
+
},
|
|
69
|
+
fix_priority: r.summary.fixPriority.map((p) => ({ finding_id: p.findingId, reason: p.reason })),
|
|
70
|
+
cross_finding_links: r.summary.crossFindingLinks.map((l) => ({ finding_ids: l.findingIds, relation: l.relation })),
|
|
71
|
+
},
|
|
72
|
+
findings: r.findings.map((f) => ({
|
|
73
|
+
rule_id: f.ruleId,
|
|
74
|
+
judgment_kind: f.judgmentKind,
|
|
75
|
+
severity: f.severity,
|
|
76
|
+
symptom: f.symptom,
|
|
77
|
+
likely_cause: f.likelyCause,
|
|
78
|
+
evidence: { spans: f.evidence.spans, excerpt: f.evidence.excerpt },
|
|
79
|
+
suggested_fix: { target: f.suggestedFix.target, change: f.suggestedFix.change },
|
|
80
|
+
confidence: f.confidence,
|
|
81
|
+
verify_with: {
|
|
82
|
+
suggested_eval_case: {
|
|
83
|
+
query_id: f.verifyWith.suggestedEvalCase.queryId,
|
|
84
|
+
query: f.verifyWith.suggestedEvalCase.query,
|
|
85
|
+
assertions: f.verifyWith.suggestedEvalCase.assertions,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
})),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Rule } from "./types.js";
|
|
2
|
+
export declare class RuleLoadError extends Error {
|
|
3
|
+
constructor(message: string);
|
|
4
|
+
}
|
|
5
|
+
export interface LoadRulesOpts {
|
|
6
|
+
builtinDir: string | null;
|
|
7
|
+
cwdRulesDir: string | null;
|
|
8
|
+
extraRulesDir: string | null;
|
|
9
|
+
noBuiltin: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function loadRules(opts: LoadRulesOpts): Promise<Rule[]>;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import yaml from "js-yaml";
|
|
4
|
+
import { RuleSchema } from "./schemas.js";
|
|
5
|
+
import { resolvePredicate } from "./predicate-registry.js";
|
|
6
|
+
export class RuleLoadError extends Error {
|
|
7
|
+
constructor(message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = "RuleLoadError";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
async function listYamls(dir) {
|
|
13
|
+
try {
|
|
14
|
+
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
15
|
+
return entries
|
|
16
|
+
.filter((e) => e.isFile() && (e.name.endsWith(".yaml") || e.name.endsWith(".yml")))
|
|
17
|
+
.map((e) => path.join(dir, e.name));
|
|
18
|
+
}
|
|
19
|
+
catch (e) {
|
|
20
|
+
const err = e;
|
|
21
|
+
if (err.code === "ENOENT")
|
|
22
|
+
return [];
|
|
23
|
+
throw err;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
async function parseOne(filePath) {
|
|
27
|
+
const raw = await fs.readFile(filePath, "utf8");
|
|
28
|
+
let parsed;
|
|
29
|
+
try {
|
|
30
|
+
parsed = yaml.load(raw);
|
|
31
|
+
}
|
|
32
|
+
catch (e) {
|
|
33
|
+
throw new RuleLoadError(`yaml parse error in ${filePath}: ${e.message}`);
|
|
34
|
+
}
|
|
35
|
+
const result = RuleSchema.safeParse(parsed);
|
|
36
|
+
if (!result.success) {
|
|
37
|
+
throw new RuleLoadError(`schema validation failed for ${filePath}: ${result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')}`);
|
|
38
|
+
}
|
|
39
|
+
const r = result.data;
|
|
40
|
+
if (!r.predicate) {
|
|
41
|
+
throw new RuleLoadError(`PR-A only supports symbolic rules; ${filePath} has no predicate`);
|
|
42
|
+
}
|
|
43
|
+
// resolvePredicate throws PredicateNotFoundError; rewrap for uniform caller experience.
|
|
44
|
+
try {
|
|
45
|
+
resolvePredicate(r.predicate);
|
|
46
|
+
}
|
|
47
|
+
catch (e) {
|
|
48
|
+
throw new RuleLoadError(`${filePath}: ${e.message}`);
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
schemaVersion: r.schema_version,
|
|
52
|
+
id: r.id,
|
|
53
|
+
severity: r.severity,
|
|
54
|
+
symptom: r.symptom,
|
|
55
|
+
taxonomy: { signalsAxis: r.taxonomy.signals_axis, msClass: r.taxonomy.ms_class },
|
|
56
|
+
suggestedFix: { target: r.suggested_fix.target, changeTemplate: r.suggested_fix.change_template },
|
|
57
|
+
verifyWith: { assertionTemplates: r.verify_with.assertion_templates },
|
|
58
|
+
predicateRef: r.predicate,
|
|
59
|
+
params: r.params,
|
|
60
|
+
sourcePath: filePath,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export async function loadRules(opts) {
|
|
64
|
+
const dirs = [];
|
|
65
|
+
if (opts.builtinDir && !opts.noBuiltin)
|
|
66
|
+
dirs.push(opts.builtinDir);
|
|
67
|
+
if (opts.cwdRulesDir)
|
|
68
|
+
dirs.push(opts.cwdRulesDir);
|
|
69
|
+
if (opts.extraRulesDir)
|
|
70
|
+
dirs.push(opts.extraRulesDir);
|
|
71
|
+
const seenIds = new Map(); // id → first path
|
|
72
|
+
const rules = [];
|
|
73
|
+
for (const dir of dirs) {
|
|
74
|
+
const yamls = await listYamls(dir);
|
|
75
|
+
for (const f of yamls) {
|
|
76
|
+
const r = await parseOne(f);
|
|
77
|
+
const prev = seenIds.get(r.id);
|
|
78
|
+
if (prev) {
|
|
79
|
+
throw new RuleLoadError(`rule id conflict for '${r.id}': defined in both ${prev} and ${f}`);
|
|
80
|
+
}
|
|
81
|
+
seenIds.set(r.id, f);
|
|
82
|
+
rules.push(r);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return rules;
|
|
86
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const RuleSchema: z.ZodObject<{
|
|
3
|
+
schema_version: z.ZodLiteral<"diagnosis-rule/v1">;
|
|
4
|
+
id: z.ZodString;
|
|
5
|
+
severity: z.ZodEnum<{
|
|
6
|
+
low: "low";
|
|
7
|
+
medium: "medium";
|
|
8
|
+
high: "high";
|
|
9
|
+
}>;
|
|
10
|
+
symptom: z.ZodString;
|
|
11
|
+
taxonomy: z.ZodObject<{
|
|
12
|
+
signals_axis: z.ZodEnum<{
|
|
13
|
+
interaction: "interaction";
|
|
14
|
+
execution: "execution";
|
|
15
|
+
environment: "environment";
|
|
16
|
+
}>;
|
|
17
|
+
ms_class: z.ZodEnum<{
|
|
18
|
+
retry_loop: "retry_loop";
|
|
19
|
+
tool_misuse: "tool_misuse";
|
|
20
|
+
context_loss: "context_loss";
|
|
21
|
+
goal_drift: "goal_drift";
|
|
22
|
+
cascading_error: "cascading_error";
|
|
23
|
+
silent_quality_degradation: "silent_quality_degradation";
|
|
24
|
+
}>;
|
|
25
|
+
}, z.core.$strip>;
|
|
26
|
+
suggested_fix: z.ZodObject<{
|
|
27
|
+
target: z.ZodString;
|
|
28
|
+
change_template: z.ZodString;
|
|
29
|
+
}, z.core.$strip>;
|
|
30
|
+
verify_with: z.ZodObject<{
|
|
31
|
+
assertion_templates: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
32
|
+
}, z.core.$strip>;
|
|
33
|
+
predicate: z.ZodOptional<z.ZodString>;
|
|
34
|
+
rubric: z.ZodOptional<z.ZodUnknown>;
|
|
35
|
+
params: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
36
|
+
}, z.core.$strip>;
|
|
37
|
+
export type RuleYaml = z.infer<typeof RuleSchema>;
|
|
38
|
+
export declare const ReportSchema: z.ZodObject<{
|
|
39
|
+
schema_version: z.ZodLiteral<"trace-diagnose-report/v1">;
|
|
40
|
+
trace: z.ZodObject<{
|
|
41
|
+
trace_id: z.ZodString;
|
|
42
|
+
agent_id: z.ZodNullable<z.ZodString>;
|
|
43
|
+
tenant: z.ZodNullable<z.ZodString>;
|
|
44
|
+
}, z.core.$strip>;
|
|
45
|
+
run: z.ZodObject<{
|
|
46
|
+
diagnosed_at: z.ZodString;
|
|
47
|
+
cli_version: z.ZodString;
|
|
48
|
+
mode: z.ZodEnum<{
|
|
49
|
+
"symbolic-only": "symbolic-only";
|
|
50
|
+
"rubric-only": "rubric-only";
|
|
51
|
+
hybrid: "hybrid";
|
|
52
|
+
}>;
|
|
53
|
+
rules_applied: z.ZodArray<z.ZodString>;
|
|
54
|
+
rules_skipped: z.ZodArray<z.ZodObject<{
|
|
55
|
+
rule_id: z.ZodString;
|
|
56
|
+
reason: z.ZodString;
|
|
57
|
+
}, z.core.$strip>>;
|
|
58
|
+
synthesizer_mode: z.ZodEnum<{
|
|
59
|
+
agent: "agent";
|
|
60
|
+
template: "template";
|
|
61
|
+
}>;
|
|
62
|
+
}, z.core.$strip>;
|
|
63
|
+
summary: z.ZodObject<{
|
|
64
|
+
headline: z.ZodString;
|
|
65
|
+
primary_root_cause: z.ZodNullable<z.ZodObject<{
|
|
66
|
+
finding_ids: z.ZodArray<z.ZodNumber>;
|
|
67
|
+
description: z.ZodString;
|
|
68
|
+
target_for_fix: z.ZodString;
|
|
69
|
+
}, z.core.$strip>>;
|
|
70
|
+
fix_priority: z.ZodArray<z.ZodObject<{
|
|
71
|
+
finding_id: z.ZodNumber;
|
|
72
|
+
reason: z.ZodString;
|
|
73
|
+
}, z.core.$strip>>;
|
|
74
|
+
cross_finding_links: z.ZodArray<z.ZodObject<{
|
|
75
|
+
finding_ids: z.ZodArray<z.ZodNumber>;
|
|
76
|
+
relation: z.ZodString;
|
|
77
|
+
}, z.core.$strip>>;
|
|
78
|
+
}, z.core.$strip>;
|
|
79
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
80
|
+
rule_id: z.ZodString;
|
|
81
|
+
judgment_kind: z.ZodEnum<{
|
|
82
|
+
symbolic: "symbolic";
|
|
83
|
+
}>;
|
|
84
|
+
severity: z.ZodEnum<{
|
|
85
|
+
low: "low";
|
|
86
|
+
medium: "medium";
|
|
87
|
+
high: "high";
|
|
88
|
+
}>;
|
|
89
|
+
symptom: z.ZodString;
|
|
90
|
+
likely_cause: z.ZodString;
|
|
91
|
+
evidence: z.ZodObject<{
|
|
92
|
+
spans: z.ZodArray<z.ZodString>;
|
|
93
|
+
excerpt: z.ZodString;
|
|
94
|
+
}, z.core.$strip>;
|
|
95
|
+
suggested_fix: z.ZodObject<{
|
|
96
|
+
target: z.ZodString;
|
|
97
|
+
change: z.ZodString;
|
|
98
|
+
}, z.core.$strip>;
|
|
99
|
+
confidence: z.ZodLiteral<"low">;
|
|
100
|
+
verify_with: z.ZodObject<{
|
|
101
|
+
suggested_eval_case: z.ZodObject<{
|
|
102
|
+
query_id: z.ZodNullable<z.ZodString>;
|
|
103
|
+
query: z.ZodNullable<z.ZodString>;
|
|
104
|
+
assertions: z.ZodArray<z.ZodString>;
|
|
105
|
+
}, z.core.$strip>;
|
|
106
|
+
}, z.core.$strip>;
|
|
107
|
+
}, z.core.$strip>>;
|
|
108
|
+
}, z.core.$strip>;
|
|
109
|
+
export type ReportYaml = z.infer<typeof ReportSchema>;
|