@kweaver-ai/kweaver-sdk 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -52
- package/README.zh.md +41 -46
- package/dist/agent-providers/index.d.ts +7 -0
- package/dist/agent-providers/index.js +5 -0
- package/dist/agent-providers/prompt-template.d.ts +62 -0
- package/dist/agent-providers/prompt-template.js +105 -0
- package/dist/agent-providers/prompts/rubric-judge-v1.prompt.md +51 -0
- package/dist/agent-providers/prompts/within-trace-synthesizer-v1.prompt.md +60 -0
- package/dist/agent-providers/providers/claude-code-subprocess.d.ts +74 -0
- package/dist/agent-providers/providers/claude-code-subprocess.js +259 -0
- package/dist/agent-providers/providers/stub.d.ts +47 -0
- package/dist/agent-providers/providers/stub.js +77 -0
- package/dist/agent-providers/registry.d.ts +45 -0
- package/dist/agent-providers/registry.js +77 -0
- package/dist/agent-providers/types.d.ts +91 -0
- package/dist/agent-providers/types.js +25 -0
- package/dist/api/agent-chat.js +8 -6
- package/dist/api/context-loader.d.ts +1 -0
- package/dist/api/resources.d.ts +94 -0
- package/dist/api/resources.js +166 -0
- package/dist/api/semantic-search.d.ts +5 -0
- package/dist/api/semantic-search.js +5 -0
- package/dist/api/skills.d.ts +75 -2
- package/dist/api/skills.js +108 -12
- package/dist/api/trace.d.ts +5 -0
- package/dist/api/trace.js +4 -0
- package/dist/cli.js +109 -15
- package/dist/client.d.ts +3 -3
- package/dist/client.js +5 -5
- package/dist/commands/agent/mode.d.ts +6 -0
- package/dist/commands/agent/mode.js +75 -0
- package/dist/commands/agent-members.js +27 -11
- package/dist/commands/agent.js +469 -286
- package/dist/commands/auth.js +184 -71
- package/dist/commands/bkn-metric.js +37 -16
- package/dist/commands/bkn-ops.js +164 -86
- package/dist/commands/bkn-query.js +99 -31
- package/dist/commands/bkn-schema.d.ts +3 -3
- package/dist/commands/bkn-schema.js +127 -86
- package/dist/commands/bkn.js +153 -114
- package/dist/commands/call.js +23 -13
- package/dist/commands/config.js +22 -12
- package/dist/commands/context-loader.js +625 -49
- package/dist/commands/dataflow.js +14 -6
- package/dist/commands/ds.js +52 -30
- package/dist/commands/explore.js +18 -15
- package/dist/commands/model.js +53 -42
- package/dist/commands/resource.d.ts +1 -0
- package/dist/commands/{dataview.js → resource.js} +62 -84
- package/dist/commands/skill.d.ts +21 -1
- package/dist/commands/skill.js +567 -43
- package/dist/commands/token.js +11 -0
- package/dist/commands/tool.js +46 -29
- package/dist/commands/toolbox.js +31 -15
- package/dist/commands/trace.d.ts +26 -1
- package/dist/commands/trace.js +515 -15
- package/dist/commands/vega.js +466 -250
- package/dist/help/format.d.ts +65 -0
- package/dist/help/format.js +141 -0
- package/dist/index.d.ts +5 -5
- package/dist/index.js +3 -3
- package/dist/resources/bkn.d.ts +5 -0
- package/dist/resources/bkn.js +5 -0
- package/dist/resources/{dataviews.d.ts → resources.d.ts} +10 -11
- package/dist/resources/{dataviews.js → resources.js} +12 -13
- package/dist/resources/skills.d.ts +17 -1
- package/dist/resources/skills.js +32 -1
- package/dist/trace-ai/diagnose/agent-binding.d.ts +67 -0
- package/dist/trace-ai/diagnose/agent-binding.js +257 -0
- package/dist/trace-ai/diagnose/builtin-rules/tool-retry-intent-mismatch.yaml +68 -0
- package/dist/trace-ai/diagnose/index.d.ts +32 -0
- package/dist/trace-ai/diagnose/index.js +246 -0
- package/dist/trace-ai/diagnose/output-schema-converter.d.ts +24 -0
- package/dist/trace-ai/diagnose/output-schema-converter.js +81 -0
- package/dist/trace-ai/diagnose/query-extractor.d.ts +14 -0
- package/dist/trace-ai/diagnose/query-extractor.js +45 -0
- package/dist/trace-ai/diagnose/report-assembler.d.ts +31 -0
- package/dist/{trace-core → trace-ai}/diagnose/report-assembler.js +19 -9
- package/dist/trace-ai/diagnose/report-markdown.d.ts +18 -0
- package/dist/trace-ai/diagnose/report-markdown.js +192 -0
- package/dist/{trace-core → trace-ai}/diagnose/rule-loader.js +42 -8
- package/dist/{trace-core → trace-ai}/diagnose/schemas.d.ts +77 -2
- package/dist/trace-ai/diagnose/schemas.js +154 -0
- package/dist/trace-ai/diagnose/signal-probe.d.ts +17 -0
- package/dist/trace-ai/diagnose/signal-probe.js +39 -0
- package/dist/trace-ai/diagnose/synthesizer-agent.d.ts +40 -0
- package/dist/trace-ai/diagnose/synthesizer-agent.js +158 -0
- package/dist/{trace-core → trace-ai}/diagnose/trace-shaper.js +1 -0
- package/dist/{trace-core → trace-ai}/diagnose/types.d.ts +55 -6
- package/dist/trace-ai/eval-set/assertion-evaluator.d.ts +29 -0
- package/dist/trace-ai/eval-set/assertion-evaluator.js +100 -0
- package/dist/trace-ai/eval-set/builder.d.ts +36 -0
- package/dist/trace-ai/eval-set/builder.js +126 -0
- package/dist/trace-ai/eval-set/index.d.ts +15 -0
- package/dist/trace-ai/eval-set/index.js +10 -0
- package/dist/trace-ai/eval-set/output-writer.d.ts +27 -0
- package/dist/trace-ai/eval-set/output-writer.js +126 -0
- package/dist/trace-ai/eval-set/query-picker.d.ts +37 -0
- package/dist/trace-ai/eval-set/query-picker.js +147 -0
- package/dist/trace-ai/eval-set/redactor.d.ts +42 -0
- package/dist/trace-ai/eval-set/redactor.js +133 -0
- package/dist/trace-ai/eval-set/rubric-templates/answer-match-reference.prompt.md +19 -0
- package/dist/trace-ai/eval-set/schemas.d.ts +136 -0
- package/dist/trace-ai/eval-set/schemas.js +130 -0
- package/dist/trace-ai/eval-set/semantic-match-provider.d.ts +33 -0
- package/dist/trace-ai/eval-set/semantic-match-provider.js +51 -0
- package/dist/trace-ai/eval-set/test-runner.d.ts +34 -0
- package/dist/trace-ai/eval-set/test-runner.js +153 -0
- package/dist/trace-ai/eval-set/types.d.ts +46 -0
- package/dist/trace-ai/eval-set/types.js +8 -0
- package/dist/trace-ai/exp/bundle-writer.d.ts +10 -0
- package/dist/trace-ai/exp/bundle-writer.js +54 -0
- package/dist/trace-ai/exp/claude-binary.d.ts +5 -0
- package/dist/trace-ai/exp/claude-binary.js +30 -0
- package/dist/trace-ai/exp/coordinator.d.ts +45 -0
- package/dist/trace-ai/exp/coordinator.js +203 -0
- package/dist/trace-ai/exp/eval-runner.d.ts +14 -0
- package/dist/trace-ai/exp/eval-runner.js +47 -0
- package/dist/trace-ai/exp/exp-store/abort-signal.d.ts +3 -0
- package/dist/trace-ai/exp/exp-store/abort-signal.js +27 -0
- package/dist/trace-ai/exp/exp-store/candidate-lineage-yaml.d.ts +4 -0
- package/dist/trace-ai/exp/exp-store/candidate-lineage-yaml.js +37 -0
- package/dist/trace-ai/exp/exp-store/events-jsonl.d.ts +17 -0
- package/dist/trace-ai/exp/exp-store/events-jsonl.js +60 -0
- package/dist/trace-ai/exp/exp-store/exp-registry.d.ts +6 -0
- package/dist/trace-ai/exp/exp-store/exp-registry.js +41 -0
- package/dist/trace-ai/exp/exp-store/index.d.ts +46 -0
- package/dist/trace-ai/exp/exp-store/index.js +59 -0
- package/dist/trace-ai/exp/exp-store/lock.d.ts +3 -0
- package/dist/trace-ai/exp/exp-store/lock.js +73 -0
- package/dist/trace-ai/exp/exp-store/mission-md.d.ts +3 -0
- package/dist/trace-ai/exp/exp-store/mission-md.js +37 -0
- package/dist/trace-ai/exp/exp-store/readme-template.d.ts +5 -0
- package/dist/trace-ai/exp/exp-store/readme-template.js +25 -0
- package/dist/trace-ai/exp/exp-store/round-yaml.d.ts +3 -0
- package/dist/trace-ai/exp/exp-store/round-yaml.js +33 -0
- package/dist/trace-ai/exp/index.d.ts +8 -0
- package/dist/trace-ai/exp/index.js +238 -0
- package/dist/trace-ai/exp/info.d.ts +35 -0
- package/dist/trace-ai/exp/info.js +120 -0
- package/dist/trace-ai/exp/patch/agent-config.d.ts +1 -0
- package/dist/trace-ai/exp/patch/agent-config.js +26 -0
- package/dist/trace-ai/exp/patch/index.d.ts +2 -0
- package/dist/trace-ai/exp/patch/index.js +13 -0
- package/dist/trace-ai/exp/patch/skill.d.ts +1 -0
- package/dist/trace-ai/exp/patch/skill.js +24 -0
- package/dist/trace-ai/exp/providers/synthesizer-client.d.ts +14 -0
- package/dist/trace-ai/exp/providers/synthesizer-client.js +39 -0
- package/dist/trace-ai/exp/providers/triage-client.d.ts +19 -0
- package/dist/trace-ai/exp/providers/triage-client.js +51 -0
- package/dist/trace-ai/exp/schemas.d.ts +147 -0
- package/dist/trace-ai/exp/schemas.js +50 -0
- package/dist/trace-ai/exp/scoring.d.ts +2 -0
- package/dist/trace-ai/exp/scoring.js +46 -0
- package/dist/trace-ai/scan/aggregator.d.ts +20 -0
- package/dist/trace-ai/scan/aggregator.js +26 -0
- package/dist/trace-ai/scan/artifacts/paths.d.ts +12 -0
- package/dist/trace-ai/scan/artifacts/paths.js +18 -0
- package/dist/trace-ai/scan/artifacts/writer.d.ts +67 -0
- package/dist/trace-ai/scan/artifacts/writer.js +96 -0
- package/dist/trace-ai/scan/batched-rubric.d.ts +55 -0
- package/dist/trace-ai/scan/batched-rubric.js +159 -0
- package/dist/trace-ai/scan/cross-trace-synthesizer.d.ts +24 -0
- package/dist/trace-ai/scan/cross-trace-synthesizer.js +93 -0
- package/dist/trace-ai/scan/index.d.ts +31 -0
- package/dist/trace-ai/scan/index.js +390 -0
- package/dist/trace-ai/scan/prompts/builtin/cross-trace-synthesizer-v1.prompt.md +44 -0
- package/dist/trace-ai/scan/prompts/builtin/rubric-judge-batch-v1.prompt.md +44 -0
- package/dist/trace-ai/scan/runner.d.ts +25 -0
- package/dist/trace-ai/scan/runner.js +42 -0
- package/dist/trace-ai/scan/sampler.d.ts +18 -0
- package/dist/trace-ai/scan/sampler.js +81 -0
- package/dist/trace-ai/scan/scan-summary-markdown.d.ts +2 -0
- package/dist/trace-ai/scan/scan-summary-markdown.js +71 -0
- package/dist/trace-ai/scan/scan-summary-schema.d.ts +73 -0
- package/dist/trace-ai/scan/scan-summary-schema.js +61 -0
- package/dist/trace-ai/scan/single-agent-validator.d.ts +23 -0
- package/dist/trace-ai/scan/single-agent-validator.js +42 -0
- package/dist/trace-ai/scan/traces-list-parser.d.ts +15 -0
- package/dist/trace-ai/scan/traces-list-parser.js +46 -0
- package/package.json +2 -2
- package/dist/api/dataviews.d.ts +0 -117
- package/dist/api/dataviews.js +0 -265
- package/dist/commands/dataview.d.ts +0 -8
- package/dist/trace-core/diagnose/index.d.ts +0 -9
- package/dist/trace-core/diagnose/index.js +0 -104
- package/dist/trace-core/diagnose/report-assembler.d.ts +0 -12
- package/dist/trace-core/diagnose/schemas.js +0 -94
- package/dist/trace-core/diagnose/signal-probe.d.ts +0 -5
- package/dist/trace-core/diagnose/signal-probe.js +0 -21
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/excessive-tool-calls-per-turn.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/excessive-tool-calls-per-turn.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/excessive-tool-calls-per-turn.yaml +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/llm-response-truncated-no-continue.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/llm-response-truncated-no-continue.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/llm-response-truncated-no-continue.yaml +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/register.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/register.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/retrieval-empty-no-fallback.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/retrieval-empty-no-fallback.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/retrieval-empty-no-fallback.yaml +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-error-swallowed.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-error-swallowed.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-error-swallowed.yaml +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-loop-no-state-change.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-loop-no-state-change.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/builtin-rules/tool-loop-no-state-change.yaml +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/predicate-registry.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/predicate-registry.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/rule-loader.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/synthesizer-template.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/synthesizer-template.js +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/trace-shaper.d.ts +0 -0
- /package/dist/{trace-core → trace-ai}/diagnose/types.js +0 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a rubric YAML's `output_schema` (a JSON-Schema-ish blob) into a
|
|
3
|
+
* zod schema the agent provider validates LLM responses against.
|
|
4
|
+
*
|
|
5
|
+
* We don't pull in a full JSON-Schema-to-Zod converter — rubric YAMLs use
|
|
6
|
+
* a deliberately narrow subset: `type: object` with `required[]` and
|
|
7
|
+
* `properties{type, enum, items}`. Anything richer is rejected at load
|
|
8
|
+
* time so authors don't accidentally rely on full JSON Schema semantics
|
|
9
|
+
* we haven't implemented.
|
|
10
|
+
*
|
|
11
|
+
* Supported per-property `type` values: `string`, `number`, `boolean`,
|
|
12
|
+
* `array` (homogeneous items by `items.type`), `object` (recursive).
|
|
13
|
+
* `enum` (string-only) is supported on `string` properties.
|
|
14
|
+
*
|
|
15
|
+
* Unsupported / rejected at conversion time: `type: integer` (use number),
|
|
16
|
+
* `anyOf`/`oneOf`, `$ref`, `additionalProperties: false`, `format`.
|
|
17
|
+
*/
|
|
18
|
+
import { z } from "zod";
|
|
19
|
+
export class OutputSchemaConversionError extends Error {
|
|
20
|
+
path;
|
|
21
|
+
constructor(message, path) {
|
|
22
|
+
super(`${message} (at ${path})`);
|
|
23
|
+
this.path = path;
|
|
24
|
+
this.name = "OutputSchemaConversionError";
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
function convertProp(spec, path) {
|
|
28
|
+
const t = spec.type;
|
|
29
|
+
if (typeof t !== "string") {
|
|
30
|
+
throw new OutputSchemaConversionError(`property is missing 'type' string`, path);
|
|
31
|
+
}
|
|
32
|
+
switch (t) {
|
|
33
|
+
case "string": {
|
|
34
|
+
if (Array.isArray(spec.enum)) {
|
|
35
|
+
if (spec.enum.length === 0) {
|
|
36
|
+
throw new OutputSchemaConversionError(`empty enum`, path);
|
|
37
|
+
}
|
|
38
|
+
for (const v of spec.enum) {
|
|
39
|
+
if (typeof v !== "string") {
|
|
40
|
+
throw new OutputSchemaConversionError(`enum supports string values only`, path);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return z.enum(spec.enum);
|
|
44
|
+
}
|
|
45
|
+
return z.string();
|
|
46
|
+
}
|
|
47
|
+
case "number": return z.number();
|
|
48
|
+
case "boolean": return z.boolean();
|
|
49
|
+
case "array": {
|
|
50
|
+
const items = spec.items;
|
|
51
|
+
if (!items) {
|
|
52
|
+
throw new OutputSchemaConversionError(`array property requires 'items'`, path);
|
|
53
|
+
}
|
|
54
|
+
return z.array(convertProp(items, `${path}.items`));
|
|
55
|
+
}
|
|
56
|
+
case "object": {
|
|
57
|
+
const subProps = spec.properties ?? {};
|
|
58
|
+
const subRequired = spec.required ?? [];
|
|
59
|
+
return buildObject(subProps, subRequired, path);
|
|
60
|
+
}
|
|
61
|
+
default:
|
|
62
|
+
throw new OutputSchemaConversionError(`unsupported type '${t}'`, path);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function buildObject(properties, required, path) {
|
|
66
|
+
const shape = {};
|
|
67
|
+
const requiredSet = new Set(required);
|
|
68
|
+
for (const [key, spec] of Object.entries(properties)) {
|
|
69
|
+
const sub = convertProp(spec, `${path}.${key}`);
|
|
70
|
+
shape[key] = requiredSet.has(key) ? sub : sub.optional();
|
|
71
|
+
}
|
|
72
|
+
for (const req of required) {
|
|
73
|
+
if (!(req in properties)) {
|
|
74
|
+
throw new OutputSchemaConversionError(`required key '${req}' is not present in properties`, path);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return z.object(shape);
|
|
78
|
+
}
|
|
79
|
+
export function rubricOutputToZod(rubric) {
|
|
80
|
+
return buildObject(rubric.output_schema.properties, rubric.output_schema.required, "output_schema");
|
|
81
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { TraceTree } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Extract the most recent user-role message from a trace's input.messages.
|
|
4
|
+
*
|
|
5
|
+
* Scans spans for `gen_ai.input.messages` (a JSON-stringified array of
|
|
6
|
+
* {role, content}), checking two locations in order:
|
|
7
|
+
* 1. span.events[*].attributes — emitted by dolphin otel_listener as the
|
|
8
|
+
* "gen_ai.client.inference.operation.details" event (primary path)
|
|
9
|
+
* 2. span.attributes — fallback for runtimes that promote the
|
|
10
|
+
* field directly onto the span
|
|
11
|
+
*
|
|
12
|
+
* Returns the last `role === "user"` message content, or null if not found.
|
|
13
|
+
*/
|
|
14
|
+
export declare function extractUserQueryFromTrace(tree: TraceTree): string | null;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract the most recent user-role message from a trace's input.messages.
|
|
3
|
+
*
|
|
4
|
+
* Scans spans for `gen_ai.input.messages` (a JSON-stringified array of
|
|
5
|
+
* {role, content}), checking two locations in order:
|
|
6
|
+
* 1. span.events[*].attributes — emitted by dolphin otel_listener as the
|
|
7
|
+
* "gen_ai.client.inference.operation.details" event (primary path)
|
|
8
|
+
* 2. span.attributes — fallback for runtimes that promote the
|
|
9
|
+
* field directly onto the span
|
|
10
|
+
*
|
|
11
|
+
* Returns the last `role === "user"` message content, or null if not found.
|
|
12
|
+
*/
|
|
13
|
+
export function extractUserQueryFromTrace(tree) {
|
|
14
|
+
for (const span of tree.spans) {
|
|
15
|
+
const candidates = [];
|
|
16
|
+
// Primary: event attributes (dolphin otel_listener path)
|
|
17
|
+
for (const ev of span.events ?? []) {
|
|
18
|
+
const v = ev.attributes?.["gen_ai.input.messages"];
|
|
19
|
+
if (typeof v === "string")
|
|
20
|
+
candidates.push(v);
|
|
21
|
+
}
|
|
22
|
+
// Fallback: span attributes
|
|
23
|
+
const spanAttr = span.attributes?.["gen_ai.input.messages"];
|
|
24
|
+
if (typeof spanAttr === "string")
|
|
25
|
+
candidates.push(spanAttr);
|
|
26
|
+
for (const raw of candidates) {
|
|
27
|
+
let parsed;
|
|
28
|
+
try {
|
|
29
|
+
parsed = JSON.parse(raw);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (!Array.isArray(parsed))
|
|
35
|
+
continue;
|
|
36
|
+
for (let i = parsed.length - 1; i >= 0; i--) {
|
|
37
|
+
const m = parsed[i];
|
|
38
|
+
if (m?.role === "user" && typeof m.content === "string" && m.content.length > 0) {
|
|
39
|
+
return m.content;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Finding, Hit, Report, Rule, Summary } from "./types.js";
|
|
2
|
+
export interface AssembleReportOpts {
|
|
3
|
+
traceId: string;
|
|
4
|
+
agentId: string | null;
|
|
5
|
+
tenant: string | null;
|
|
6
|
+
cliVersion: string;
|
|
7
|
+
rules: Rule[];
|
|
8
|
+
hits: Map<string, Hit[]>;
|
|
9
|
+
/** Additional pre-built findings (rubric judgments come from agent-binding). */
|
|
10
|
+
extraFindings?: Finding[];
|
|
11
|
+
summary: Summary;
|
|
12
|
+
/** Run mode. Default `symbolic-only` for backward compat. */
|
|
13
|
+
mode?: 'symbolic-only' | 'rubric-only' | 'hybrid';
|
|
14
|
+
/** Rubric rules skipped due to --no-llm / unavailable provider / etc. */
|
|
15
|
+
rulesSkipped?: {
|
|
16
|
+
ruleId: string;
|
|
17
|
+
reason: string;
|
|
18
|
+
}[];
|
|
19
|
+
/** Stage-3 synthesizer that produced `summary`. */
|
|
20
|
+
synthesizerMode?: 'template' | 'agent';
|
|
21
|
+
/** User query extracted from trace input.messages (2026-05-13). */
|
|
22
|
+
userQuery?: string | null;
|
|
23
|
+
/** Conversation/query ID for suggested_eval_case correlation (2026-05-13). */
|
|
24
|
+
queryId?: string | null;
|
|
25
|
+
}
|
|
26
|
+
/** Build symbolic-pillar findings from rule+hit pairs.
|
|
27
|
+
* Exported so callers (e.g. tests, index.ts) can compose findings from
|
|
28
|
+
* multiple sources before handing them to a custom summary path. */
|
|
29
|
+
export declare function symbolicHitsToFindings(rules: Rule[], hits: Map<string, Hit[]>, userQuery?: string | null, queryId?: string | null): Finding[];
|
|
30
|
+
export declare function assembleReport(opts: AssembleReportOpts): Report;
|
|
31
|
+
export declare function reportToYamlObject(r: Report): unknown;
|
|
@@ -4,17 +4,22 @@ function renderTemplate(tpl, bindings) {
|
|
|
4
4
|
return v === undefined ? `{{${key}}}` : String(v);
|
|
5
5
|
});
|
|
6
6
|
}
|
|
7
|
-
|
|
7
|
+
/** Build symbolic-pillar findings from rule+hit pairs.
|
|
8
|
+
* Exported so callers (e.g. tests, index.ts) can compose findings from
|
|
9
|
+
* multiple sources before handing them to a custom summary path. */
|
|
10
|
+
export function symbolicHitsToFindings(rules, hits, userQuery = null, queryId = null) {
|
|
8
11
|
const findings = [];
|
|
9
|
-
for (const rule of
|
|
10
|
-
|
|
12
|
+
for (const rule of rules) {
|
|
13
|
+
if (rule.predicateRef === null)
|
|
14
|
+
continue;
|
|
15
|
+
const ruleHits = hits.get(rule.id) ?? [];
|
|
11
16
|
for (const hit of ruleHits) {
|
|
12
17
|
findings.push({
|
|
13
18
|
ruleId: rule.id,
|
|
14
19
|
judgmentKind: "symbolic",
|
|
15
20
|
severity: rule.severity,
|
|
16
21
|
symptom: rule.symptom,
|
|
17
|
-
likelyCause: rule.symptom, //
|
|
22
|
+
likelyCause: rule.symptom, // symbolic: no LLM, so mirror symptom; rubric agent overrides
|
|
18
23
|
evidence: { spans: hit.evidenceSpans, excerpt: hit.excerpt },
|
|
19
24
|
suggestedFix: {
|
|
20
25
|
target: rule.suggestedFix.target,
|
|
@@ -23,24 +28,29 @@ export function assembleReport(opts) {
|
|
|
23
28
|
confidence: "low",
|
|
24
29
|
verifyWith: {
|
|
25
30
|
suggestedEvalCase: {
|
|
26
|
-
queryId
|
|
27
|
-
query:
|
|
31
|
+
queryId,
|
|
32
|
+
query: userQuery,
|
|
28
33
|
assertions: rule.verifyWith.assertionTemplates.map((t) => renderTemplate(t, hit.bindings)),
|
|
29
34
|
},
|
|
30
35
|
},
|
|
31
36
|
});
|
|
32
37
|
}
|
|
33
38
|
}
|
|
39
|
+
return findings;
|
|
40
|
+
}
|
|
41
|
+
export function assembleReport(opts) {
|
|
42
|
+
const symbolicFindings = symbolicHitsToFindings(opts.rules, opts.hits, opts.userQuery ?? null, opts.queryId ?? null);
|
|
43
|
+
const findings = [...symbolicFindings, ...(opts.extraFindings ?? [])];
|
|
34
44
|
return {
|
|
35
45
|
schemaVersion: "trace-diagnose-report/v1",
|
|
36
46
|
trace: { traceId: opts.traceId, agentId: opts.agentId, tenant: opts.tenant },
|
|
37
47
|
run: {
|
|
38
48
|
diagnosedAt: new Date().toISOString(),
|
|
39
49
|
cliVersion: opts.cliVersion,
|
|
40
|
-
mode: "symbolic-only",
|
|
50
|
+
mode: opts.mode ?? "symbolic-only",
|
|
41
51
|
rulesApplied: opts.rules.map((r) => r.id),
|
|
42
|
-
rulesSkipped: [],
|
|
43
|
-
synthesizerMode: "template",
|
|
52
|
+
rulesSkipped: opts.rulesSkipped ?? [],
|
|
53
|
+
synthesizerMode: opts.synthesizerMode ?? "template",
|
|
44
54
|
},
|
|
45
55
|
summary: opts.summary,
|
|
46
56
|
findings,
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Report } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Optional context the md renderer uses to build runnable verification
|
|
4
|
+
* commands. None of these are in the yaml schema (which stays v1-locked and
|
|
5
|
+
* CLI-agnostic) — they live only in the markdown view so users who paste the
|
|
6
|
+
* md into a ticket / PR have copy-pasteable shell commands without needing to
|
|
7
|
+
* remember the trace's conversation context.
|
|
8
|
+
*/
|
|
9
|
+
export interface MarkdownRenderOpts {
|
|
10
|
+
/** The conversation_id passed to `kweaver trace diagnose`. Used to render
|
|
11
|
+
* the "re-run diagnosis" command. When undefined, that command is rendered
|
|
12
|
+
* with a `<conversation_id>` placeholder. */
|
|
13
|
+
conversationId?: string;
|
|
14
|
+
/** Business domain (`-bd` flag). When undefined, commands omit the flag and
|
|
15
|
+
* inherit kweaver's default (`bd_public`). */
|
|
16
|
+
businessDomain?: string;
|
|
17
|
+
}
|
|
18
|
+
export declare function renderReportMarkdown(r: Report, opts?: MarkdownRenderOpts): string;
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// Human-readable markdown view of a trace-diagnose report.
|
|
2
|
+
//
|
|
3
|
+
// The YAML report (see `report-assembler.reportToYamlObject`) is the source of
|
|
4
|
+
// truth; this file is a pure projection. Persisted alongside the yaml when
|
|
5
|
+
// `--format=both`. Markdown was chosen over a stdout pretty-print because
|
|
6
|
+
// reports are commonly pasted into tickets / PRs / wikis where ephemeral
|
|
7
|
+
// terminal output would be lost.
|
|
8
|
+
//
|
|
9
|
+
// Structure (inverted-pyramid: most actionable first):
|
|
10
|
+
// 1. Title + one-line meta
|
|
11
|
+
// 2. Summary — headline (+ primary root cause if any)
|
|
12
|
+
// 3. Fix priority table (omitted when empty)
|
|
13
|
+
// 4. Findings — one section per finding, excerpt as a blockquote
|
|
14
|
+
// 5. Cross-finding links (omitted when empty)
|
|
15
|
+
// 6. How to verify — kweaver CLI commands the reader can paste to
|
|
16
|
+
// independently re-confirm the report's claims against the live trace.
|
|
17
|
+
// Sourced from Report fields + the caller-supplied conversation_id /
|
|
18
|
+
// business_domain (which are not part of the yaml schema — yaml stays
|
|
19
|
+
// CLI-agnostic, markdown is the CLI-aware view).
|
|
20
|
+
// 7. Run — mode / synthesizer / rules applied & skipped (reference)
|
|
21
|
+
export function renderReportMarkdown(r, opts = {}) {
|
|
22
|
+
const lines = [];
|
|
23
|
+
const shortId = r.trace.traceId.length > 16 ? `${r.trace.traceId.slice(0, 16)}…` : r.trace.traceId;
|
|
24
|
+
lines.push(`# Trace Diagnose Report — \`${shortId}\``);
|
|
25
|
+
lines.push("");
|
|
26
|
+
lines.push(`> trace \`${r.trace.traceId}\` · agent \`${r.trace.agentId ?? "—"}\` · tenant \`${r.trace.tenant ?? "—"}\` · diagnosed ${r.run.diagnosedAt} · cli \`${r.run.cliVersion}\``);
|
|
27
|
+
lines.push("");
|
|
28
|
+
// ── Summary ──────────────────────────────────────────────────────────────
|
|
29
|
+
lines.push("## Summary");
|
|
30
|
+
lines.push("");
|
|
31
|
+
lines.push(`**${r.summary.headline}**`);
|
|
32
|
+
lines.push("");
|
|
33
|
+
if (r.summary.primaryRootCause !== null) {
|
|
34
|
+
const rc = r.summary.primaryRootCause;
|
|
35
|
+
const fids = rc.findingIds.map((i) => `#${i}`).join(", ");
|
|
36
|
+
lines.push(`Primary root cause spans findings ${fids} — target for fix: \`${rc.targetForFix}\`.`);
|
|
37
|
+
lines.push("");
|
|
38
|
+
lines.push(`> ${escapeBlockquote(rc.description)}`);
|
|
39
|
+
lines.push("");
|
|
40
|
+
}
|
|
41
|
+
// ── Fix priority ─────────────────────────────────────────────────────────
|
|
42
|
+
if (r.summary.fixPriority.length > 0) {
|
|
43
|
+
lines.push("## Fix priority");
|
|
44
|
+
lines.push("");
|
|
45
|
+
lines.push("| Order | Finding | Rule | Reason |");
|
|
46
|
+
lines.push("|---|---|---|---|");
|
|
47
|
+
r.summary.fixPriority.forEach((p, idx) => {
|
|
48
|
+
const f = r.findings[p.findingId];
|
|
49
|
+
const ruleCell = f ? `\`${f.ruleId}\` [${f.severity}/${f.judgmentKind}]` : `(unknown #${p.findingId})`;
|
|
50
|
+
lines.push(`| ${idx + 1} | #${p.findingId} | ${ruleCell} | ${escapeTableCell(p.reason)} |`);
|
|
51
|
+
});
|
|
52
|
+
lines.push("");
|
|
53
|
+
}
|
|
54
|
+
// ── Findings ─────────────────────────────────────────────────────────────
|
|
55
|
+
lines.push(`## Findings (${r.findings.length})`);
|
|
56
|
+
lines.push("");
|
|
57
|
+
if (r.findings.length === 0) {
|
|
58
|
+
lines.push(`_No findings were emitted by any of the ${r.run.rulesApplied.length} applied rules._`);
|
|
59
|
+
lines.push("");
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
r.findings.forEach((f, idx) => renderFinding(lines, f, idx));
|
|
63
|
+
}
|
|
64
|
+
// ── Cross-finding links ──────────────────────────────────────────────────
|
|
65
|
+
if (r.summary.crossFindingLinks.length > 0) {
|
|
66
|
+
lines.push("## Cross-finding links");
|
|
67
|
+
lines.push("");
|
|
68
|
+
for (const link of r.summary.crossFindingLinks) {
|
|
69
|
+
const ids = link.findingIds.map((i) => `#${i}`).join(" ↔ ");
|
|
70
|
+
lines.push(`- ${ids} — ${link.relation}`);
|
|
71
|
+
}
|
|
72
|
+
lines.push("");
|
|
73
|
+
}
|
|
74
|
+
// ── How to verify ────────────────────────────────────────────────────────
|
|
75
|
+
renderVerificationSection(lines, r, opts);
|
|
76
|
+
// ── Run reference ────────────────────────────────────────────────────────
|
|
77
|
+
lines.push("## Run");
|
|
78
|
+
lines.push("");
|
|
79
|
+
lines.push(`- **mode**: \`${r.run.mode}\` · **synthesizer**: \`${r.run.synthesizerMode}\` · **rules**: ${r.run.rulesApplied.length} applied, ${r.run.rulesSkipped.length} skipped`);
|
|
80
|
+
lines.push(`- **applied**: ${r.run.rulesApplied.map((id) => `\`${id}\``).join(", ")}`);
|
|
81
|
+
if (r.run.rulesSkipped.length > 0) {
|
|
82
|
+
lines.push("- **skipped**:");
|
|
83
|
+
for (const s of r.run.rulesSkipped) {
|
|
84
|
+
lines.push(` - \`${s.ruleId}\` — ${s.reason}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
lines.push("");
|
|
88
|
+
return lines.join("\n");
|
|
89
|
+
}
|
|
90
|
+
function renderFinding(lines, f, idx) {
|
|
91
|
+
lines.push(`### #${idx} \`${f.ruleId}\` — [${f.severity}/${f.judgmentKind}]`);
|
|
92
|
+
lines.push("");
|
|
93
|
+
if (f.evidence.excerpt.trim().length > 0) {
|
|
94
|
+
for (const ln of f.evidence.excerpt.trim().split(/\r?\n/)) {
|
|
95
|
+
lines.push(`> ${ln}`);
|
|
96
|
+
}
|
|
97
|
+
lines.push("");
|
|
98
|
+
}
|
|
99
|
+
const meta = [];
|
|
100
|
+
meta.push(`- **symptom**: ${f.symptom}`);
|
|
101
|
+
meta.push(`- **likely cause**: ${f.likelyCause}`);
|
|
102
|
+
meta.push(`- **confidence**: ${f.confidence}`);
|
|
103
|
+
if (f.evidence.spans.length > 0) {
|
|
104
|
+
meta.push(`- **evidence spans**: ${f.evidence.spans.map((s) => `\`${s}\``).join(", ")}`);
|
|
105
|
+
}
|
|
106
|
+
meta.push(`- **suggested fix** → \`${f.suggestedFix.target}\`: ${f.suggestedFix.change}`);
|
|
107
|
+
if (f.verifyWith.suggestedEvalCase.assertions.length > 0) {
|
|
108
|
+
meta.push(`- **verify with**:`);
|
|
109
|
+
for (const a of f.verifyWith.suggestedEvalCase.assertions) {
|
|
110
|
+
meta.push(` - ${a}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
for (const m of meta)
|
|
114
|
+
lines.push(m);
|
|
115
|
+
lines.push("");
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Render kweaver CLI verification commands so a reader can independently
|
|
119
|
+
* re-confirm the diagnosis against the live trace. Sections:
|
|
120
|
+
* 1. Re-fetch the raw spans (proves the trace data the report was built
|
|
121
|
+
* from still matches what observability returns)
|
|
122
|
+
* 2. Re-diagnose with --no-llm (reproducibility check — same symbolic
|
|
123
|
+
* findings should fire deterministically; rules out claude-side flake)
|
|
124
|
+
* 3. Inspect suspect spans per finding (only when findings.length > 0)
|
|
125
|
+
* 4. Check recurrence across the agent's other conversations
|
|
126
|
+
*
|
|
127
|
+
* The commands intentionally omit auth flags (--token / --base-url) — the
|
|
128
|
+
* reader is expected to have `kweaver auth` already configured or to be
|
|
129
|
+
* working in the same shell session that produced this report.
|
|
130
|
+
*/
|
|
131
|
+
function renderVerificationSection(lines, r, opts) {
|
|
132
|
+
const bdFlag = opts.businessDomain ? ` -bd ${opts.businessDomain}` : "";
|
|
133
|
+
const convId = opts.conversationId ?? "<conversation_id>";
|
|
134
|
+
lines.push("## How to verify");
|
|
135
|
+
lines.push("");
|
|
136
|
+
lines.push("Paste these into a shell to independently re-confirm the report against the live trace.");
|
|
137
|
+
lines.push("");
|
|
138
|
+
// 1. Re-fetch raw spans for the trace.
|
|
139
|
+
lines.push("### 1. Re-fetch the raw trace");
|
|
140
|
+
lines.push("");
|
|
141
|
+
lines.push("```bash");
|
|
142
|
+
lines.push(`kweaver call -X POST '/api/agent-observability/v1/traces/_search' \\`);
|
|
143
|
+
lines.push(` -d '{"query":{"term":{"traceId":"${r.trace.traceId}"}}}'${bdFlag} \\`);
|
|
144
|
+
lines.push(` | jq '.hits.hits[]._source | {spanId, name, kind: .attributes."gen_ai.operation.name", status: .status.code}'`);
|
|
145
|
+
lines.push("```");
|
|
146
|
+
lines.push("");
|
|
147
|
+
// 2. Re-run diagnosis deterministically.
|
|
148
|
+
lines.push("### 2. Re-run diagnosis (reproducibility check)");
|
|
149
|
+
lines.push("");
|
|
150
|
+
lines.push("```bash");
|
|
151
|
+
lines.push(`kweaver trace diagnose ${convId} --no-llm --out /tmp/verify.yaml${bdFlag}`);
|
|
152
|
+
lines.push("# then diff against this report's yaml — symbolic findings should match exactly");
|
|
153
|
+
lines.push("```");
|
|
154
|
+
lines.push("");
|
|
155
|
+
// 3. Inspect suspect spans per finding.
|
|
156
|
+
if (r.findings.length > 0) {
|
|
157
|
+
lines.push("### 3. Inspect the suspect spans");
|
|
158
|
+
lines.push("");
|
|
159
|
+
r.findings.forEach((f, idx) => {
|
|
160
|
+
if (f.evidence.spans.length === 0)
|
|
161
|
+
return;
|
|
162
|
+
const spanList = f.evidence.spans.map((s) => `"${s}"`).join(", ");
|
|
163
|
+
lines.push(`Finding #${idx} (\`${f.ruleId}\`):`);
|
|
164
|
+
lines.push("");
|
|
165
|
+
lines.push("```bash");
|
|
166
|
+
lines.push(`kweaver call -X POST '/api/agent-observability/v1/traces/_search' \\`);
|
|
167
|
+
lines.push(` -d '{"query":{"terms":{"spanId":[${spanList}]}}}'${bdFlag} \\`);
|
|
168
|
+
lines.push(` | jq '.hits.hits[]._source.attributes'`);
|
|
169
|
+
lines.push("```");
|
|
170
|
+
lines.push("");
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
// 4. Recurrence check.
|
|
174
|
+
if (r.trace.agentId !== null) {
|
|
175
|
+
const sectionNum = r.findings.length > 0 ? 4 : 3;
|
|
176
|
+
lines.push(`### ${sectionNum}. Check whether this pattern recurs for the agent`);
|
|
177
|
+
lines.push("");
|
|
178
|
+
lines.push("```bash");
|
|
179
|
+
lines.push(`kweaver agent sessions ${r.trace.agentId} --limit 20${bdFlag}`);
|
|
180
|
+
lines.push("# sample a few conversation_ids from the list, re-diagnose each, count rule hits");
|
|
181
|
+
lines.push("```");
|
|
182
|
+
lines.push("");
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
function escapeTableCell(s) {
|
|
186
|
+
// Pipes and newlines break GFM tables; collapse newlines and escape `|`.
|
|
187
|
+
return s.replace(/\r?\n/g, " ").replace(/\|/g, "\\|");
|
|
188
|
+
}
|
|
189
|
+
function escapeBlockquote(s) {
|
|
190
|
+
// Blockquote-safe; just collapse newlines so the whole description sits in one line.
|
|
191
|
+
return s.replace(/\r?\n+/g, " ").trim();
|
|
192
|
+
}
|
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import yaml from "js-yaml";
|
|
4
4
|
import { RuleSchema } from "./schemas.js";
|
|
5
5
|
import { resolvePredicate } from "./predicate-registry.js";
|
|
6
|
+
import { rubricOutputToZod, OutputSchemaConversionError } from "./output-schema-converter.js";
|
|
6
7
|
export class RuleLoadError extends Error {
|
|
7
8
|
constructor(message) {
|
|
8
9
|
super(message);
|
|
@@ -37,15 +38,47 @@ async function parseOne(filePath) {
|
|
|
37
38
|
throw new RuleLoadError(`schema validation failed for ${filePath}: ${result.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')}`);
|
|
38
39
|
}
|
|
39
40
|
const r = result.data;
|
|
40
|
-
|
|
41
|
-
|
|
41
|
+
let predicateRef = null;
|
|
42
|
+
let rubric = null;
|
|
43
|
+
if (r.predicate) {
|
|
44
|
+
// resolvePredicate throws PredicateNotFoundError; rewrap for uniform caller experience.
|
|
45
|
+
try {
|
|
46
|
+
resolvePredicate(r.predicate);
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
throw new RuleLoadError(`${filePath}: ${e.message}`);
|
|
50
|
+
}
|
|
51
|
+
predicateRef = r.predicate;
|
|
42
52
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
53
|
+
else if (r.rubric) {
|
|
54
|
+
// Compile output_schema → zod at load time so authors see schema errors
|
|
55
|
+
// up-front via `trace diagnose rules validate <path>`, not at LLM call time.
|
|
56
|
+
let outputZodSchema;
|
|
57
|
+
try {
|
|
58
|
+
outputZodSchema = rubricOutputToZod(r.rubric);
|
|
59
|
+
}
|
|
60
|
+
catch (e) {
|
|
61
|
+
if (e instanceof OutputSchemaConversionError) {
|
|
62
|
+
throw new RuleLoadError(`${filePath}: rubric.output_schema: ${e.message}`);
|
|
63
|
+
}
|
|
64
|
+
throw e;
|
|
65
|
+
}
|
|
66
|
+
rubric = {
|
|
67
|
+
judgeQuestion: r.rubric.judge_question,
|
|
68
|
+
inputs: r.rubric.inputs.map((i) => ({ kind: i.kind, source: i.source })),
|
|
69
|
+
outputSchemaRaw: r.rubric.output_schema,
|
|
70
|
+
outputZodSchema,
|
|
71
|
+
agentBinding: {
|
|
72
|
+
provider: r.rubric.agent_binding.provider,
|
|
73
|
+
promptTemplateRef: r.rubric.agent_binding.prompt_template_ref,
|
|
74
|
+
},
|
|
75
|
+
gatesOn: r.rubric.gates_on,
|
|
76
|
+
};
|
|
46
77
|
}
|
|
47
|
-
|
|
48
|
-
|
|
78
|
+
else {
|
|
79
|
+
// RuleSchema's XOR refinement should have already caught this; keep an
|
|
80
|
+
// explicit branch so the failure mode is obvious if schemas drift.
|
|
81
|
+
throw new RuleLoadError(`${filePath}: rule has neither predicate nor rubric`);
|
|
49
82
|
}
|
|
50
83
|
return {
|
|
51
84
|
schemaVersion: r.schema_version,
|
|
@@ -55,7 +88,8 @@ async function parseOne(filePath) {
|
|
|
55
88
|
taxonomy: { signalsAxis: r.taxonomy.signals_axis, msClass: r.taxonomy.ms_class },
|
|
56
89
|
suggestedFix: { target: r.suggested_fix.target, changeTemplate: r.suggested_fix.change_template },
|
|
57
90
|
verifyWith: { assertionTemplates: r.verify_with.assertion_templates },
|
|
58
|
-
predicateRef
|
|
91
|
+
predicateRef,
|
|
92
|
+
rubric,
|
|
59
93
|
params: r.params,
|
|
60
94
|
sourcePath: filePath,
|
|
61
95
|
};
|
|
@@ -1,4 +1,38 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Rubric input source descriptor. The supported source prefixes are
|
|
4
|
+
* resolved by `diagnose/agent-binding.ts` against the in-memory TraceTree:
|
|
5
|
+
*
|
|
6
|
+
* - `extract_from_root_attr:<dot.path>` → root span attribute by name
|
|
7
|
+
* - `filter_by_kind:[kind1,kind2,...]` → ordered span subset by kind
|
|
8
|
+
* - `literal:<json>` → constant blob (debug / fixtures)
|
|
9
|
+
*
|
|
10
|
+
* Authors describe **which slice of the trace** the agent needs as context;
|
|
11
|
+
* the binding does the actual extraction so rule YAML stays declarative.
|
|
12
|
+
*/
|
|
13
|
+
declare const RubricInputSchema: z.ZodObject<{
|
|
14
|
+
kind: z.ZodString;
|
|
15
|
+
source: z.ZodString;
|
|
16
|
+
}, z.core.$strip>;
|
|
17
|
+
declare const RubricSchema: z.ZodObject<{
|
|
18
|
+
judge_question: z.ZodString;
|
|
19
|
+
inputs: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
20
|
+
kind: z.ZodString;
|
|
21
|
+
source: z.ZodString;
|
|
22
|
+
}, z.core.$strip>>>;
|
|
23
|
+
output_schema: z.ZodObject<{
|
|
24
|
+
type: z.ZodLiteral<"object">;
|
|
25
|
+
required: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
26
|
+
properties: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
27
|
+
}, z.core.$strip>;
|
|
28
|
+
agent_binding: z.ZodObject<{
|
|
29
|
+
provider: z.ZodString;
|
|
30
|
+
prompt_template_ref: z.ZodString;
|
|
31
|
+
}, z.core.$strip>;
|
|
32
|
+
gates_on: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
33
|
+
}, z.core.$strip>;
|
|
34
|
+
export type RubricYaml = z.infer<typeof RubricSchema>;
|
|
35
|
+
export type RubricInputYaml = z.infer<typeof RubricInputSchema>;
|
|
2
36
|
export declare const RuleSchema: z.ZodObject<{
|
|
3
37
|
schema_version: z.ZodLiteral<"diagnosis-rule/v1">;
|
|
4
38
|
id: z.ZodString;
|
|
@@ -31,7 +65,23 @@ export declare const RuleSchema: z.ZodObject<{
|
|
|
31
65
|
assertion_templates: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
32
66
|
}, z.core.$strip>;
|
|
33
67
|
predicate: z.ZodOptional<z.ZodString>;
|
|
34
|
-
rubric: z.ZodOptional<z.
|
|
68
|
+
rubric: z.ZodOptional<z.ZodObject<{
|
|
69
|
+
judge_question: z.ZodString;
|
|
70
|
+
inputs: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
71
|
+
kind: z.ZodString;
|
|
72
|
+
source: z.ZodString;
|
|
73
|
+
}, z.core.$strip>>>;
|
|
74
|
+
output_schema: z.ZodObject<{
|
|
75
|
+
type: z.ZodLiteral<"object">;
|
|
76
|
+
required: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
77
|
+
properties: z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
78
|
+
}, z.core.$strip>;
|
|
79
|
+
agent_binding: z.ZodObject<{
|
|
80
|
+
provider: z.ZodString;
|
|
81
|
+
prompt_template_ref: z.ZodString;
|
|
82
|
+
}, z.core.$strip>;
|
|
83
|
+
gates_on: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
84
|
+
}, z.core.$strip>>;
|
|
35
85
|
params: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
36
86
|
}, z.core.$strip>;
|
|
37
87
|
export type RuleYaml = z.infer<typeof RuleSchema>;
|
|
@@ -80,6 +130,7 @@ export declare const ReportSchema: z.ZodObject<{
|
|
|
80
130
|
rule_id: z.ZodString;
|
|
81
131
|
judgment_kind: z.ZodEnum<{
|
|
82
132
|
symbolic: "symbolic";
|
|
133
|
+
rubric: "rubric";
|
|
83
134
|
}>;
|
|
84
135
|
severity: z.ZodEnum<{
|
|
85
136
|
low: "low";
|
|
@@ -96,7 +147,11 @@ export declare const ReportSchema: z.ZodObject<{
|
|
|
96
147
|
target: z.ZodString;
|
|
97
148
|
change: z.ZodString;
|
|
98
149
|
}, z.core.$strip>;
|
|
99
|
-
confidence: z.
|
|
150
|
+
confidence: z.ZodEnum<{
|
|
151
|
+
low: "low";
|
|
152
|
+
medium: "medium";
|
|
153
|
+
high: "high";
|
|
154
|
+
}>;
|
|
100
155
|
verify_with: z.ZodObject<{
|
|
101
156
|
suggested_eval_case: z.ZodObject<{
|
|
102
157
|
query_id: z.ZodNullable<z.ZodString>;
|
|
@@ -107,3 +162,23 @@ export declare const ReportSchema: z.ZodObject<{
|
|
|
107
162
|
}, z.core.$strip>>;
|
|
108
163
|
}, z.core.$strip>;
|
|
109
164
|
export type ReportYaml = z.infer<typeof ReportSchema>;
|
|
165
|
+
/** The Summary section in isolation — exported so the agent synthesizer
|
|
166
|
+
* can validate its LLM output against the same shape the report uses. */
|
|
167
|
+
export declare const SummaryOutputSchema: z.ZodObject<{
|
|
168
|
+
headline: z.ZodString;
|
|
169
|
+
primary_root_cause: z.ZodNullable<z.ZodObject<{
|
|
170
|
+
finding_ids: z.ZodArray<z.ZodNumber>;
|
|
171
|
+
description: z.ZodString;
|
|
172
|
+
target_for_fix: z.ZodString;
|
|
173
|
+
}, z.core.$strip>>;
|
|
174
|
+
fix_priority: z.ZodArray<z.ZodObject<{
|
|
175
|
+
finding_id: z.ZodNumber;
|
|
176
|
+
reason: z.ZodString;
|
|
177
|
+
}, z.core.$strip>>;
|
|
178
|
+
cross_finding_links: z.ZodArray<z.ZodObject<{
|
|
179
|
+
finding_ids: z.ZodArray<z.ZodNumber>;
|
|
180
|
+
relation: z.ZodString;
|
|
181
|
+
}, z.core.$strip>>;
|
|
182
|
+
}, z.core.$strip>;
|
|
183
|
+
export type SummaryOutput = z.infer<typeof SummaryOutputSchema>;
|
|
184
|
+
export {};
|