@kweaver-ai/kweaver-sdk 0.7.3 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/README.zh.md +44 -0
- package/bin/kweaver.js +12 -11
- package/dist/api/agent-observability.d.ts +51 -0
- package/dist/api/agent-observability.js +108 -0
- package/dist/api/bkn-backend.d.ts +1 -0
- package/dist/api/bkn-backend.js +1 -1
- package/dist/api/bkn-metrics.d.ts +59 -0
- package/dist/api/bkn-metrics.js +129 -0
- package/dist/api/conversations.d.ts +43 -2
- package/dist/api/conversations.js +77 -23
- package/dist/api/datasources.d.ts +2 -20
- package/dist/api/datasources.js +7 -86
- package/dist/api/model-invocation.d.ts +58 -0
- package/dist/api/model-invocation.js +203 -0
- package/dist/api/models.d.ts +79 -0
- package/dist/api/models.js +183 -0
- package/dist/api/ontology-query-metrics.d.ts +14 -0
- package/dist/api/ontology-query-metrics.js +30 -0
- package/dist/api/trace.d.ts +44 -0
- package/dist/api/trace.js +81 -0
- package/dist/api/vega.d.ts +53 -0
- package/dist/api/vega.js +144 -0
- package/dist/bundled-model-templates.d.ts +17 -0
- package/dist/bundled-model-templates.js +24 -0
- package/dist/cli.js +15 -0
- package/dist/client.d.ts +3 -0
- package/dist/client.js +5 -0
- package/dist/commands/agent.d.ts +7 -1
- package/dist/commands/agent.js +75 -21
- package/dist/commands/bkn-metric.d.ts +1 -0
- package/dist/commands/bkn-metric.js +406 -0
- package/dist/commands/bkn-ops.js +28 -16
- package/dist/commands/bkn-utils.d.ts +38 -0
- package/dist/commands/bkn-utils.js +54 -0
- package/dist/commands/bkn.js +4 -0
- package/dist/commands/ds.js +14 -3
- package/dist/commands/explore-chat.js +2 -2
- package/dist/commands/model.d.ts +72 -0
- package/dist/commands/model.js +1315 -0
- package/dist/commands/trace.d.ts +14 -0
- package/dist/commands/trace.js +168 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +5 -0
- package/dist/resources/datasources.js +2 -1
- package/dist/resources/models.d.ts +40 -0
- package/dist/resources/models.js +88 -0
- package/dist/templates/model/llm-basic.json +13 -0
- package/dist/templates/model/manifest.json +16 -0
- package/dist/templates/model/small-basic.json +6 -0
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.js +15 -0
- package/dist/trace-core/diagnose/builtin-rules/excessive-tool-calls-per-turn.yaml +16 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.js +44 -0
- package/dist/trace-core/diagnose/builtin-rules/llm-response-truncated-no-continue.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/register.d.ts +1 -0
- package/dist/trace-core/diagnose/builtin-rules/register.js +11 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.js +29 -0
- package/dist/trace-core/diagnose/builtin-rules/retrieval-empty-no-fallback.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.js +45 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-error-swallowed.yaml +15 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.d.ts +2 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.js +38 -0
- package/dist/trace-core/diagnose/builtin-rules/tool-loop-no-state-change.yaml +16 -0
- package/dist/trace-core/diagnose/index.d.ts +9 -0
- package/dist/trace-core/diagnose/index.js +104 -0
- package/dist/trace-core/diagnose/predicate-registry.d.ts +7 -0
- package/dist/trace-core/diagnose/predicate-registry.js +30 -0
- package/dist/trace-core/diagnose/report-assembler.d.ts +12 -0
- package/dist/trace-core/diagnose/report-assembler.js +90 -0
- package/dist/trace-core/diagnose/rule-loader.d.ts +11 -0
- package/dist/trace-core/diagnose/rule-loader.js +86 -0
- package/dist/trace-core/diagnose/schemas.d.ts +109 -0
- package/dist/trace-core/diagnose/schemas.js +94 -0
- package/dist/trace-core/diagnose/signal-probe.d.ts +5 -0
- package/dist/trace-core/diagnose/signal-probe.js +21 -0
- package/dist/trace-core/diagnose/synthesizer-template.d.ts +2 -0
- package/dist/trace-core/diagnose/synthesizer-template.js +49 -0
- package/dist/trace-core/diagnose/trace-shaper.d.ts +3 -0
- package/dist/trace-core/diagnose/trace-shaper.js +72 -0
- package/dist/trace-core/diagnose/types.d.ts +124 -0
- package/dist/trace-core/diagnose/types.js +1 -0
- package/dist/utils/trace-views.d.ts +44 -0
- package/dist/utils/trace-views.js +425 -0
- package/package.json +15 -5
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface ParsedTraceArgs {
|
|
2
|
+
subcommand: "diagnose" | "rules-validate" | "help";
|
|
3
|
+
conversationId?: string;
|
|
4
|
+
rulePath?: string;
|
|
5
|
+
out: string | null;
|
|
6
|
+
rulesDir: string | null;
|
|
7
|
+
noBuiltin: boolean;
|
|
8
|
+
noLlm: boolean;
|
|
9
|
+
baseUrl: string | null;
|
|
10
|
+
token: string | null;
|
|
11
|
+
businessDomain: string | null;
|
|
12
|
+
}
|
|
13
|
+
export declare function parseTraceArgs(argv: string[]): ParsedTraceArgs;
|
|
14
|
+
export declare function runTraceCommand(rest: string[]): Promise<number>;
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import yargs from "yargs";
|
|
2
|
+
import { diagnose, TraceNotFoundError } from "../trace-core/diagnose/index.js";
|
|
3
|
+
import { RuleLoadError } from "../trace-core/diagnose/rule-loader.js";
|
|
4
|
+
import { RuleProbeError } from "../trace-core/diagnose/signal-probe.js";
|
|
5
|
+
import { RuleSchema } from "../trace-core/diagnose/schemas.js";
|
|
6
|
+
import { ensureValidToken } from "../auth/oauth.js";
|
|
7
|
+
import yaml from "js-yaml";
|
|
8
|
+
import fs from "node:fs/promises";
|
|
9
|
+
export function parseTraceArgs(argv) {
|
|
10
|
+
if (argv.length === 0) {
|
|
11
|
+
return defaults("help");
|
|
12
|
+
}
|
|
13
|
+
const head = argv[0];
|
|
14
|
+
if (head !== "diagnose") {
|
|
15
|
+
return defaults("help");
|
|
16
|
+
}
|
|
17
|
+
if (argv[1] === "rules" && argv[2] === "validate") {
|
|
18
|
+
return { ...defaults("rules-validate"), rulePath: argv[3] };
|
|
19
|
+
}
|
|
20
|
+
// diagnose <traceId> [flags...]
|
|
21
|
+
const parsed = yargs(argv.slice(1))
|
|
22
|
+
.option("out", { type: "string", default: undefined })
|
|
23
|
+
.option("rules", { type: "string", default: undefined })
|
|
24
|
+
.option("builtin", { type: "boolean", default: true }) // --no-builtin sets this to false
|
|
25
|
+
.option("llm", { type: "boolean", default: false }) // PR-A: forced false (--no-llm)
|
|
26
|
+
.option("token", { type: "string" })
|
|
27
|
+
.option("base-url", { type: "string" })
|
|
28
|
+
.option("business-domain", { alias: "bd", type: "string" })
|
|
29
|
+
.help(false)
|
|
30
|
+
.parseSync();
|
|
31
|
+
return {
|
|
32
|
+
subcommand: "diagnose",
|
|
33
|
+
conversationId: String(parsed._[0] ?? ""),
|
|
34
|
+
out: parsed.out ?? null,
|
|
35
|
+
rulesDir: parsed.rules ?? null,
|
|
36
|
+
noBuiltin: !parsed.builtin,
|
|
37
|
+
noLlm: !parsed.llm,
|
|
38
|
+
baseUrl: parsed.baseUrl ?? null,
|
|
39
|
+
token: parsed.token ?? null,
|
|
40
|
+
businessDomain: parsed.businessDomain ?? null,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function defaults(sub) {
|
|
44
|
+
return {
|
|
45
|
+
subcommand: sub,
|
|
46
|
+
out: null,
|
|
47
|
+
rulesDir: null,
|
|
48
|
+
noBuiltin: false,
|
|
49
|
+
noLlm: true,
|
|
50
|
+
baseUrl: null,
|
|
51
|
+
token: null,
|
|
52
|
+
businessDomain: null,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function printHelp() {
|
|
56
|
+
process.stdout.write(`kweaver trace — trace diagnosis commands
|
|
57
|
+
|
|
58
|
+
Subcommands:
|
|
59
|
+
trace diagnose <conversation_id> Diagnose the trace produced by a conversation; emit YAML report
|
|
60
|
+
(the id is the conversation_id returned by 'agent chat' /
|
|
61
|
+
'agent sessions'; spans are fetched from agent-observability)
|
|
62
|
+
--out <file> Write report to file (default: stdout)
|
|
63
|
+
--rules <dir> Override <cwd>/diagnosis-rules/
|
|
64
|
+
--no-builtin Disable the 5 builtin baseline rules
|
|
65
|
+
--no-llm PR-A: always on; PR-B will allow disabling
|
|
66
|
+
|
|
67
|
+
trace diagnose rules validate <rule.yaml> Validate a rule yaml file (exit 0 ok, 6 fail)
|
|
68
|
+
|
|
69
|
+
Auth flags (any subcommand): --token, --base-url, --business-domain (-bd).
|
|
70
|
+
`);
|
|
71
|
+
}
|
|
72
|
+
export async function runTraceCommand(rest) {
|
|
73
|
+
const args = parseTraceArgs(rest);
|
|
74
|
+
if (args.subcommand === "help") {
|
|
75
|
+
printHelp();
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
if (args.subcommand === "rules-validate") {
|
|
79
|
+
return await runRulesValidate(args.rulePath ?? "");
|
|
80
|
+
}
|
|
81
|
+
// diagnose
|
|
82
|
+
if (!args.conversationId) {
|
|
83
|
+
process.stderr.write("error: missing <conversation_id>\n");
|
|
84
|
+
return 2;
|
|
85
|
+
}
|
|
86
|
+
let baseUrl = args.baseUrl ?? process.env.KWEAVER_BASE_URL ?? "";
|
|
87
|
+
let token = args.token ?? process.env.KWEAVER_TOKEN ?? "";
|
|
88
|
+
const bd = args.businessDomain ?? process.env.KWEAVER_BUSINESS_DOMAIN ?? "bd_public";
|
|
89
|
+
// Fall back to the active platform from `~/.kweaver/` (same as agent trace),
|
|
90
|
+
// so users don't need to pass --base-url / --token explicitly. Tokens are
|
|
91
|
+
// auto-refreshed for OAuth platforms; "__NO_AUTH__" is returned for no-auth.
|
|
92
|
+
if (!baseUrl || !token) {
|
|
93
|
+
try {
|
|
94
|
+
const t = await ensureValidToken();
|
|
95
|
+
if (!baseUrl)
|
|
96
|
+
baseUrl = t.baseUrl;
|
|
97
|
+
if (!token)
|
|
98
|
+
token = t.accessToken;
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
process.stderr.write(`error: missing --base-url / --token, and no active platform in ~/.kweaver/ — ${e.message}\n`);
|
|
102
|
+
return 5;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (!baseUrl || !token) {
|
|
106
|
+
process.stderr.write("error: missing --base-url / --token (or KWEAVER_BASE_URL / KWEAVER_TOKEN env)\n");
|
|
107
|
+
return 5;
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
await diagnose(args.conversationId, {
|
|
111
|
+
out: args.out,
|
|
112
|
+
rulesDir: args.rulesDir,
|
|
113
|
+
noBuiltin: args.noBuiltin,
|
|
114
|
+
noLlm: true,
|
|
115
|
+
agentProvider: null,
|
|
116
|
+
timeoutMs: 60000,
|
|
117
|
+
baseUrl,
|
|
118
|
+
token,
|
|
119
|
+
businessDomain: bd,
|
|
120
|
+
});
|
|
121
|
+
return 0;
|
|
122
|
+
}
|
|
123
|
+
catch (e) {
|
|
124
|
+
if (e instanceof TraceNotFoundError) {
|
|
125
|
+
process.stderr.write(`error: ${e.message}; check time window / tenant\n`);
|
|
126
|
+
return 4;
|
|
127
|
+
}
|
|
128
|
+
if (e instanceof RuleLoadError) {
|
|
129
|
+
process.stderr.write(`error: ${e.message}\n`);
|
|
130
|
+
return 6;
|
|
131
|
+
}
|
|
132
|
+
if (e instanceof RuleProbeError) {
|
|
133
|
+
process.stderr.write(`error: ${e.message}\n`);
|
|
134
|
+
return 6;
|
|
135
|
+
}
|
|
136
|
+
process.stderr.write(`error: ${e.message}\n`);
|
|
137
|
+
return 1;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function runRulesValidate(rulePath) {
|
|
141
|
+
if (!rulePath) {
|
|
142
|
+
process.stderr.write("error: missing <rule.yaml> path\n");
|
|
143
|
+
return 2;
|
|
144
|
+
}
|
|
145
|
+
let raw;
|
|
146
|
+
try {
|
|
147
|
+
raw = await fs.readFile(rulePath, "utf8");
|
|
148
|
+
}
|
|
149
|
+
catch (e) {
|
|
150
|
+
process.stderr.write(`error: cannot read ${rulePath}: ${e.message}\n`);
|
|
151
|
+
return 6;
|
|
152
|
+
}
|
|
153
|
+
let parsed;
|
|
154
|
+
try {
|
|
155
|
+
parsed = yaml.load(raw);
|
|
156
|
+
}
|
|
157
|
+
catch (e) {
|
|
158
|
+
process.stderr.write(`error: yaml parse error: ${e.message}\n`);
|
|
159
|
+
return 6;
|
|
160
|
+
}
|
|
161
|
+
const result = RuleSchema.safeParse(parsed);
|
|
162
|
+
if (!result.success) {
|
|
163
|
+
process.stderr.write(`error: schema validation failed:\n${result.error.issues.map((i) => ` - ${i.path.join(".")}: ${i.message}`).join("\n")}\n`);
|
|
164
|
+
return 6;
|
|
165
|
+
}
|
|
166
|
+
process.stdout.write(`ok: ${rulePath} validates against diagnosis-rule/v1\n`);
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -29,6 +29,10 @@ export type { ListKnowledgeNetworksOptions, GetKnowledgeNetworkOptions, CreateKn
|
|
|
29
29
|
export { listKnowledgeNetworks, getKnowledgeNetwork, createKnowledgeNetwork, updateKnowledgeNetwork, deleteKnowledgeNetwork, listObjectTypes, listRelationTypes, listActionTypes, } from "./api/knowledge-networks.js";
|
|
30
30
|
export type { OntologyQueryBaseOptions, ObjectTypeQueryOptions, ObjectTypePropertiesOptions, SubgraphOptions, ActionTypeQueryOptions, ActionTypeExecuteOptions, ActionExecutionGetOptions, ActionLogsListOptions, ActionLogGetOptions, ActionLogCancelOptions, } from "./api/ontology-query.js";
|
|
31
31
|
export { objectTypeQuery, objectTypeProperties, subgraph, actionTypeQuery, actionTypeExecute, actionExecutionGet, actionLogsList, actionLogGet, actionLogCancel, } from "./api/ontology-query.js";
|
|
32
|
+
export type { MetricQueryDataOptions, MetricDryRunOptions } from "./api/ontology-query-metrics.js";
|
|
33
|
+
export { metricQueryData, metricDryRun } from "./api/ontology-query-metrics.js";
|
|
34
|
+
export type { ListMetricsOptions, CreateMetricsOptions, SearchMetricsOptions, ValidateMetricsOptions, GetMetricOptions, UpdateMetricOptions, DeleteMetricOptions, GetMetricsByIdsOptions, DeleteMetricsByIdsOptions, } from "./api/bkn-metrics.js";
|
|
35
|
+
export { listMetrics, createMetrics, searchMetrics, validateMetrics, getMetric, updateMetric, deleteMetric, getMetrics, deleteMetrics, } from "./api/bkn-metrics.js";
|
|
32
36
|
export type { SendChatRequestOptions, SendChatRequestStreamCallbacks, ChatResult, ProgressItem, AgentInfo, } from "./api/agent-chat.js";
|
|
33
37
|
export { sendChatRequest, sendChatRequestStream, fetchAgentInfo, buildChatUrl, buildAgentInfoUrl, extractText, } from "./api/agent-chat.js";
|
|
34
38
|
export type { ListAgentsOptions, GetAgentOptions, GetAgentByKeyOptions, CreateAgentOptions, UpdateAgentOptions, DeleteAgentOptions, PublishAgentOptions, UnpublishAgentOptions, } from "./api/agent-list.js";
|
|
@@ -52,11 +56,16 @@ export { ContextLoaderResource } from "./resources/context-loader.js";
|
|
|
52
56
|
export { SkillsResource } from "./resources/skills.js";
|
|
53
57
|
export { ToolboxesResource } from "./resources/toolboxes.js";
|
|
54
58
|
export type { InvokeToolArgs } from "./resources/toolboxes.js";
|
|
59
|
+
export { ModelsResource, LlmModelsSubresource, SmallModelsSubresource, ModelInvocationSubresource, } from "./resources/models.js";
|
|
55
60
|
export type { SkillStatus, SkillSummary, SkillInfo, SkillFileSummary, SkillContentIndex, SkillFileReadResult, RegisterSkillResult, DeleteSkillResult, UpdateSkillStatusResult, SkillListResult, ListSkillsOptions, ListSkillMarketOptions, GetSkillOptions, RegisterSkillContentOptions, RegisterSkillZipOptions, UpdateSkillStatusOptions, ReadSkillFileOptions, DownloadSkillOptions, DownloadedSkillArchive, } from "./api/skills.js";
|
|
56
61
|
export { listSkills, listSkillMarket, getSkill, deleteSkill, updateSkillStatus, registerSkillContent, registerSkillZip, getSkillContentIndex, fetchSkillContent, readSkillFile, fetchSkillFile, downloadSkill, installSkillArchive, } from "./api/skills.js";
|
|
57
62
|
export type { ViewField, DataView, CreateDataViewOptions, GetDataViewOptions, ListDataViewsOptions, DeleteDataViewOptions, FindDataViewOptions, QueryDataViewOptions, DataViewQueryResult, } from "./api/dataviews.js";
|
|
58
63
|
export { parseDataView, createDataView, getDataView, listDataViews, deleteDataView, findDataView, queryDataView, } from "./api/dataviews.js";
|
|
59
64
|
export { DataViewsResource } from "./resources/dataviews.js";
|
|
65
|
+
export type { MfManagerBaseOptions, ListLlmModelsOptions, GetLlmModelOptions, AddLlmModelOptions, EditLlmModelOptions, DeleteLlmModelsOptions, TestLlmModelOptions, ListSmallModelsOptions, GetSmallModelOptions, AddSmallModelOptions, EditSmallModelOptions, DeleteSmallModelsOptions, TestSmallModelOptions, } from "./api/models.js";
|
|
66
|
+
export { MF_MODEL_MANAGER_PATH_PREFIX, assertSmallModelConfigAdapterExclusive, assertSmallModelEditBody, listLlmModels, getLlmModel, addLlmModel, editLlmModel, deleteLlmModels, testLlmModel, listSmallModels, getSmallModel, addSmallModel, editSmallModel, deleteSmallModels, testSmallModel, } from "./api/models.js";
|
|
67
|
+
export type { MfApiBaseOptions, ChatMessage, ModelChatCompletionsOptions, ModelChatResult, ModelEmbeddingOptions, ModelRerankOptions, } from "./api/model-invocation.js";
|
|
68
|
+
export { MF_MODEL_API_PATH_PREFIX, consumeOpenAiSseText, modelChatCompletions, modelEmbedding, modelEmbeddings, modelRerank, } from "./api/model-invocation.js";
|
|
60
69
|
export type { BusinessDomain, ListBusinessDomainsOptions } from "./api/business-domains.js";
|
|
61
70
|
export { listBusinessDomains } from "./api/business-domains.js";
|
|
62
71
|
export type { CreateToolboxOptions, DeleteToolboxOptions, SetToolboxStatusOptions, UploadToolOptions, SetToolStatusesOptions, ListToolboxesOptions, ListToolsOptions, InvokeToolOptions, } from "./api/toolboxes.js";
|
package/dist/index.js
CHANGED
|
@@ -27,6 +27,8 @@
|
|
|
27
27
|
*/
|
|
28
28
|
export { listKnowledgeNetworks, getKnowledgeNetwork, createKnowledgeNetwork, updateKnowledgeNetwork, deleteKnowledgeNetwork, listObjectTypes, listRelationTypes, listActionTypes, } from "./api/knowledge-networks.js";
|
|
29
29
|
export { objectTypeQuery, objectTypeProperties, subgraph, actionTypeQuery, actionTypeExecute, actionExecutionGet, actionLogsList, actionLogGet, actionLogCancel, } from "./api/ontology-query.js";
|
|
30
|
+
export { metricQueryData, metricDryRun } from "./api/ontology-query-metrics.js";
|
|
31
|
+
export { listMetrics, createMetrics, searchMetrics, validateMetrics, getMetric, updateMetric, deleteMetric, getMetrics, deleteMetrics, } from "./api/bkn-metrics.js";
|
|
30
32
|
export { sendChatRequest, sendChatRequestStream, fetchAgentInfo, buildChatUrl, buildAgentInfoUrl, extractText, } from "./api/agent-chat.js";
|
|
31
33
|
export { listAgents, getAgent, getAgentByKey, createAgent, updateAgent, deleteAgent, publishAgent, unpublishAgent, } from "./api/agent-list.js";
|
|
32
34
|
export { listConversations, listMessages } from "./api/conversations.js";
|
|
@@ -41,9 +43,12 @@ export { ConversationsResource } from "./resources/conversations.js";
|
|
|
41
43
|
export { ContextLoaderResource } from "./resources/context-loader.js";
|
|
42
44
|
export { SkillsResource } from "./resources/skills.js";
|
|
43
45
|
export { ToolboxesResource } from "./resources/toolboxes.js";
|
|
46
|
+
export { ModelsResource, LlmModelsSubresource, SmallModelsSubresource, ModelInvocationSubresource, } from "./resources/models.js";
|
|
44
47
|
export { listSkills, listSkillMarket, getSkill, deleteSkill, updateSkillStatus, registerSkillContent, registerSkillZip, getSkillContentIndex, fetchSkillContent, readSkillFile, fetchSkillFile, downloadSkill, installSkillArchive, } from "./api/skills.js";
|
|
45
48
|
export { parseDataView, createDataView, getDataView, listDataViews, deleteDataView, findDataView, queryDataView, } from "./api/dataviews.js";
|
|
46
49
|
export { DataViewsResource } from "./resources/dataviews.js";
|
|
50
|
+
export { MF_MODEL_MANAGER_PATH_PREFIX, assertSmallModelConfigAdapterExclusive, assertSmallModelEditBody, listLlmModels, getLlmModel, addLlmModel, editLlmModel, deleteLlmModels, testLlmModel, listSmallModels, getSmallModel, addSmallModel, editSmallModel, deleteSmallModels, testSmallModel, } from "./api/models.js";
|
|
51
|
+
export { MF_MODEL_API_PATH_PREFIX, consumeOpenAiSseText, modelChatCompletions, modelEmbedding, modelEmbeddings, modelRerank, } from "./api/model-invocation.js";
|
|
47
52
|
export { listBusinessDomains } from "./api/business-domains.js";
|
|
48
53
|
export { createToolbox, deleteToolbox, setToolboxStatus, uploadTool, setToolStatuses, listToolboxes, listTools, executeTool, debugTool, } from "./api/toolboxes.js";
|
|
49
54
|
// ── HTTP utilities ────────────────────────────────────────────────────────────
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { testDatasource, createDatasource, listDatasources, getDatasource, deleteDatasource, listTables,
|
|
1
|
+
import { testDatasource, createDatasource, listDatasources, getDatasource, deleteDatasource, listTables, } from "../api/datasources.js";
|
|
2
|
+
import { listTablesWithColumns, scanMetadata } from "../api/vega.js";
|
|
2
3
|
export class DataSourcesResource {
|
|
3
4
|
ctx;
|
|
4
5
|
constructor(ctx) {
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { ClientContext } from "../client.js";
|
|
2
|
+
import { type ListLlmModelsOptions, type ListSmallModelsOptions, type MfManagerBaseOptions } from "../api/models.js";
|
|
3
|
+
import { type MfApiBaseOptions, type ModelChatCompletionsOptions, type ModelEmbeddingOptions, type ModelRerankOptions } from "../api/model-invocation.js";
|
|
4
|
+
type Base = MfManagerBaseOptions;
|
|
5
|
+
export declare class LlmModelsSubresource {
|
|
6
|
+
private readonly ctx;
|
|
7
|
+
constructor(ctx: ClientContext);
|
|
8
|
+
list(opts: Omit<ListLlmModelsOptions, keyof Base> & Partial<MfManagerBaseOptions>): Promise<unknown>;
|
|
9
|
+
get(modelId: string, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
10
|
+
add(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
11
|
+
edit(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
12
|
+
delete(modelIds: string[], opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
13
|
+
test(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
14
|
+
}
|
|
15
|
+
export declare class SmallModelsSubresource {
|
|
16
|
+
private readonly ctx;
|
|
17
|
+
constructor(ctx: ClientContext);
|
|
18
|
+
list(opts: Omit<ListSmallModelsOptions, keyof Base> & Partial<MfManagerBaseOptions>): Promise<unknown>;
|
|
19
|
+
get(modelId: string, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
20
|
+
add(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
21
|
+
edit(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
22
|
+
delete(modelIds: string[], opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
23
|
+
test(body: Record<string, unknown>, opts?: Partial<Pick<MfManagerBaseOptions, "mfManagerBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
24
|
+
}
|
|
25
|
+
export declare class ModelInvocationSubresource {
|
|
26
|
+
private readonly ctx;
|
|
27
|
+
constructor(ctx: ClientContext);
|
|
28
|
+
chat(opts: Omit<ModelChatCompletionsOptions, "baseUrl" | "accessToken" | "businessDomain"> & Partial<Pick<MfApiBaseOptions, "mfApiBaseUrl" | "businessDomain">>): Promise<import("../api/model-invocation.js").ModelChatResult>;
|
|
29
|
+
embedding(opts: Omit<ModelEmbeddingOptions, "baseUrl" | "accessToken" | "businessDomain"> & Partial<Pick<MfApiBaseOptions, "mfApiBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
30
|
+
embeddings(opts: Omit<ModelEmbeddingOptions, "baseUrl" | "accessToken" | "businessDomain"> & Partial<Pick<MfApiBaseOptions, "mfApiBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
31
|
+
rerank(opts: Omit<ModelRerankOptions, "baseUrl" | "accessToken" | "businessDomain"> & Partial<Pick<MfApiBaseOptions, "mfApiBaseUrl" | "businessDomain">>): Promise<unknown>;
|
|
32
|
+
}
|
|
33
|
+
/** Model factory: mf-model-manager (CRUD) + mf-model-api (invoke chat / embedding / rerank). */
|
|
34
|
+
export declare class ModelsResource {
|
|
35
|
+
readonly llm: LlmModelsSubresource;
|
|
36
|
+
readonly small: SmallModelsSubresource;
|
|
37
|
+
readonly invocation: ModelInvocationSubresource;
|
|
38
|
+
constructor(ctx: ClientContext);
|
|
39
|
+
}
|
|
40
|
+
export {};
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { addLlmModel, addSmallModel, deleteLlmModels, deleteSmallModels, editLlmModel, editSmallModel, getLlmModel, getSmallModel, listLlmModels, listSmallModels, testLlmModel, testSmallModel, } from "../api/models.js";
|
|
2
|
+
import { modelChatCompletions, modelEmbedding, modelEmbeddings, modelRerank, } from "../api/model-invocation.js";
|
|
3
|
+
export class LlmModelsSubresource {
|
|
4
|
+
ctx;
|
|
5
|
+
constructor(ctx) {
|
|
6
|
+
this.ctx = ctx;
|
|
7
|
+
}
|
|
8
|
+
list(opts) {
|
|
9
|
+
return listLlmModels({ ...this.ctx.base(), ...opts });
|
|
10
|
+
}
|
|
11
|
+
get(modelId, opts = {}) {
|
|
12
|
+
return getLlmModel({ ...this.ctx.base(), modelId, ...opts });
|
|
13
|
+
}
|
|
14
|
+
add(body, opts = {}) {
|
|
15
|
+
return addLlmModel({ ...this.ctx.base(), body, ...opts });
|
|
16
|
+
}
|
|
17
|
+
edit(body, opts = {}) {
|
|
18
|
+
return editLlmModel({ ...this.ctx.base(), body, ...opts });
|
|
19
|
+
}
|
|
20
|
+
delete(modelIds, opts = {}) {
|
|
21
|
+
return deleteLlmModels({ ...this.ctx.base(), modelIds, ...opts });
|
|
22
|
+
}
|
|
23
|
+
test(body, opts = {}) {
|
|
24
|
+
return testLlmModel({ ...this.ctx.base(), body, ...opts });
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export class SmallModelsSubresource {
|
|
28
|
+
ctx;
|
|
29
|
+
constructor(ctx) {
|
|
30
|
+
this.ctx = ctx;
|
|
31
|
+
}
|
|
32
|
+
list(opts) {
|
|
33
|
+
return listSmallModels({ ...this.ctx.base(), ...opts });
|
|
34
|
+
}
|
|
35
|
+
get(modelId, opts = {}) {
|
|
36
|
+
return getSmallModel({ ...this.ctx.base(), modelId, ...opts });
|
|
37
|
+
}
|
|
38
|
+
add(body, opts = {}) {
|
|
39
|
+
return addSmallModel({ ...this.ctx.base(), body, ...opts });
|
|
40
|
+
}
|
|
41
|
+
edit(body, opts = {}) {
|
|
42
|
+
return editSmallModel({ ...this.ctx.base(), body, ...opts });
|
|
43
|
+
}
|
|
44
|
+
delete(modelIds, opts = {}) {
|
|
45
|
+
return deleteSmallModels({ ...this.ctx.base(), modelIds, ...opts });
|
|
46
|
+
}
|
|
47
|
+
test(body, opts = {}) {
|
|
48
|
+
return testSmallModel({ ...this.ctx.base(), body, ...opts });
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export class ModelInvocationSubresource {
|
|
52
|
+
ctx;
|
|
53
|
+
constructor(ctx) {
|
|
54
|
+
this.ctx = ctx;
|
|
55
|
+
}
|
|
56
|
+
chat(opts) {
|
|
57
|
+
const { baseUrl, accessToken, businessDomain } = this.ctx.base();
|
|
58
|
+
return modelChatCompletions({
|
|
59
|
+
baseUrl,
|
|
60
|
+
accessToken,
|
|
61
|
+
businessDomain,
|
|
62
|
+
...opts,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
embedding(opts) {
|
|
66
|
+
const { baseUrl, accessToken, businessDomain } = this.ctx.base();
|
|
67
|
+
return modelEmbedding({ baseUrl, accessToken, businessDomain, ...opts });
|
|
68
|
+
}
|
|
69
|
+
embeddings(opts) {
|
|
70
|
+
const { baseUrl, accessToken, businessDomain } = this.ctx.base();
|
|
71
|
+
return modelEmbeddings({ baseUrl, accessToken, businessDomain, ...opts });
|
|
72
|
+
}
|
|
73
|
+
rerank(opts) {
|
|
74
|
+
const { baseUrl, accessToken, businessDomain } = this.ctx.base();
|
|
75
|
+
return modelRerank({ baseUrl, accessToken, businessDomain, ...opts });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Model factory: mf-model-manager (CRUD) + mf-model-api (invoke chat / embedding / rerank). */
|
|
79
|
+
export class ModelsResource {
|
|
80
|
+
llm;
|
|
81
|
+
small;
|
|
82
|
+
invocation;
|
|
83
|
+
constructor(ctx) {
|
|
84
|
+
this.llm = new LlmModelsSubresource(ctx);
|
|
85
|
+
this.small = new SmallModelsSubresource(ctx);
|
|
86
|
+
this.invocation = new ModelInvocationSubresource(ctx);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__template_note": "Shape aligned with mf-model-manager POST /llm/add: nested model_config (required). model_series must be one of: tome, qwen, openai, internlm, deepseek, qianxun, claude, chatglm, llama, others, baidu, baidu_tianchen. model_type: llm | rlm | vu. Optional: model_parameters (int>0), quota (bool). openai series: api_key required by backend validation. baidu/baidu_tianchen need extra keys inside model_config per vendor.",
|
|
3
|
+
"model_name": "example-llm",
|
|
4
|
+
"model_series": "others",
|
|
5
|
+
"model_type": "llm",
|
|
6
|
+
"max_model_len": 8192,
|
|
7
|
+
"model_config": {
|
|
8
|
+
"api_url": "https://example.com/v1/chat/completions",
|
|
9
|
+
"api_model": "example-deployment",
|
|
10
|
+
"api_key": ""
|
|
11
|
+
},
|
|
12
|
+
"quota": false
|
|
13
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"llm": [
|
|
3
|
+
{
|
|
4
|
+
"id": "basic",
|
|
5
|
+
"file": "llm-basic.json",
|
|
6
|
+
"summary": "Minimal POST /llm/add body — model_config nested + model_series / max_model_len / model_type"
|
|
7
|
+
}
|
|
8
|
+
],
|
|
9
|
+
"small": [
|
|
10
|
+
{
|
|
11
|
+
"id": "basic",
|
|
12
|
+
"file": "small-basic.json",
|
|
13
|
+
"summary": "Minimal model_config stub for embedding/reranker registration — extend or merge with CLI flags"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"__template_note": "Fragment for small-model model_config only (POST /small-model/add). Backend requires api_url + api_model inside model_config. Root fields model_name, model_type (embedding|reranker), batch_size, and for embedding also max_tokens + embedding_dim are supplied via CLI flags or separate JSON root — not repeated here.",
|
|
3
|
+
"api_url": "https://example.com/v1/embeddings",
|
|
4
|
+
"api_model": "example-embedding-model",
|
|
5
|
+
"api_key": ""
|
|
6
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// PR-A approximation: counts tool calls across the entire trace, not per user turn.
|
|
2
|
+
// Real per-turn scoping requires turn segmentation by gen_ai.conversation.id round trips,
|
|
3
|
+
// which is deferred to PR-B (where the synthesizer can also use turn boundaries for narratives).
|
|
4
|
+
// For single-turn traces (the common case in PR-A) this approximation matches the rule semantics.
|
|
5
|
+
export const predicate = (trace, params) => {
|
|
6
|
+
const max = params.max_tool_calls_per_turn ?? 10;
|
|
7
|
+
const tools = trace.byKind.get("tool") ?? [];
|
|
8
|
+
if (tools.length <= max)
|
|
9
|
+
return [];
|
|
10
|
+
return [{
|
|
11
|
+
evidenceSpans: tools.map((t) => t.spanId),
|
|
12
|
+
excerpt: `tool calls per turn exceeded threshold: ${tools.length} > ${max}`,
|
|
13
|
+
bindings: { count: tools.length, max_calls: max },
|
|
14
|
+
}];
|
|
15
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: excessive_tool_calls_per_turn
|
|
3
|
+
severity: medium
|
|
4
|
+
symptom: excessive_tool_calls_per_user_turn
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: tool_misuse
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "constrain plan to at most {{max_calls}} tool calls per user turn; observed {{count}}"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "tool_call_count_per_turn <= {{max_calls}}"
|
|
14
|
+
predicate: builtin:excessive_tool_calls_per_turn
|
|
15
|
+
params:
|
|
16
|
+
max_tool_calls_per_turn: 10
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
function finishReason(s) {
|
|
2
|
+
// OTel GenAI 1.x emits an array (`finish_reasons`); older spans / fixtures
|
|
3
|
+
// use the singular string form. Accept both; first non-empty entry wins.
|
|
4
|
+
const arr = s.attributes["gen_ai.response.finish_reasons"];
|
|
5
|
+
if (Array.isArray(arr)) {
|
|
6
|
+
for (const r of arr) {
|
|
7
|
+
if (typeof r === "string" && r.length > 0)
|
|
8
|
+
return r;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
const a = s.attributes["gen_ai.response.finish_reason"] ?? s.attributes["llm.finish_reason"];
|
|
12
|
+
return typeof a === "string" ? a : "";
|
|
13
|
+
}
|
|
14
|
+
function conversationId(s) {
|
|
15
|
+
const v = s.attributes["gen_ai.conversation.id"];
|
|
16
|
+
return typeof v === "string" ? v : "";
|
|
17
|
+
}
|
|
18
|
+
export const predicate = (trace) => {
|
|
19
|
+
const llms = (trace.byKind.get("llm") ?? [])
|
|
20
|
+
.slice()
|
|
21
|
+
.sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
22
|
+
const hits = [];
|
|
23
|
+
for (let i = 0; i < llms.length; i++) {
|
|
24
|
+
const s = llms[i];
|
|
25
|
+
if (finishReason(s) !== "length")
|
|
26
|
+
continue;
|
|
27
|
+
const convId = conversationId(s);
|
|
28
|
+
let hasContinuation = false;
|
|
29
|
+
for (let j = i + 1; j < llms.length; j++) {
|
|
30
|
+
if (conversationId(llms[j]) === convId) {
|
|
31
|
+
hasContinuation = true;
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (!hasContinuation) {
|
|
36
|
+
hits.push({
|
|
37
|
+
evidenceSpans: [s.spanId],
|
|
38
|
+
excerpt: `LLM response truncated (finish_reason=length) with no continuation span in conversation '${convId}'`,
|
|
39
|
+
bindings: { conversation_id: convId },
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return hits;
|
|
44
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: llm_response_truncated_no_continue
|
|
3
|
+
severity: medium
|
|
4
|
+
symptom: llm_output_truncated_with_no_continuation
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: context_loss
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "after finish_reason=length, send a continuation request or split the task earlier"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "if(llm.finish_reason == 'length'): next_step in [continuation, split_task]"
|
|
14
|
+
predicate: builtin:llm_response_truncated_no_continue
|
|
15
|
+
params: {}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { registerPredicate } from "../predicate-registry.js";
|
|
2
|
+
import { predicate as toolLoopNoStateChange } from "./tool-loop-no-state-change.js";
|
|
3
|
+
import { predicate as toolErrorSwallowed } from "./tool-error-swallowed.js";
|
|
4
|
+
import { predicate as retrievalEmptyNoFallback } from "./retrieval-empty-no-fallback.js";
|
|
5
|
+
import { predicate as llmResponseTruncatedNoContinue } from "./llm-response-truncated-no-continue.js";
|
|
6
|
+
import { predicate as excessiveToolCallsPerTurn } from "./excessive-tool-calls-per-turn.js";
|
|
7
|
+
registerPredicate("tool_loop_no_state_change", toolLoopNoStateChange);
|
|
8
|
+
registerPredicate("tool_error_swallowed", toolErrorSwallowed);
|
|
9
|
+
registerPredicate("retrieval_empty_no_fallback", retrievalEmptyNoFallback);
|
|
10
|
+
registerPredicate("llm_response_truncated_no_continue", llmResponseTruncatedNoContinue);
|
|
11
|
+
registerPredicate("excessive_tool_calls_per_turn", excessiveToolCallsPerTurn);
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
function resultCount(s) {
|
|
2
|
+
const v = s.attributes["gen_ai.retrieval.result_count"];
|
|
3
|
+
return typeof v === "number" ? v : null;
|
|
4
|
+
}
|
|
5
|
+
export const predicate = (trace) => {
|
|
6
|
+
const ordered = trace.spans
|
|
7
|
+
.slice()
|
|
8
|
+
.sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
9
|
+
const hits = [];
|
|
10
|
+
for (let i = 0; i < ordered.length; i++) {
|
|
11
|
+
const s = ordered[i];
|
|
12
|
+
if (s.kind !== "retrieval")
|
|
13
|
+
continue;
|
|
14
|
+
if (resultCount(s) !== 0)
|
|
15
|
+
continue;
|
|
16
|
+
const next = ordered[i + 1];
|
|
17
|
+
if (!next)
|
|
18
|
+
continue;
|
|
19
|
+
if (next.kind === "llm") {
|
|
20
|
+
hits.push({
|
|
21
|
+
evidenceSpans: [s.spanId, next.spanId],
|
|
22
|
+
excerpt: `retrieval returned 0 results; next step was LLM generation with no fallback path`,
|
|
23
|
+
bindings: {},
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
// retrieval (retry/rewrite) or tool (alt source) → no hit
|
|
27
|
+
}
|
|
28
|
+
return hits;
|
|
29
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
schema_version: diagnosis-rule/v1
|
|
2
|
+
id: retrieval_empty_no_fallback
|
|
3
|
+
severity: medium
|
|
4
|
+
symptom: empty_retrieval_result_no_fallback_path
|
|
5
|
+
taxonomy:
|
|
6
|
+
signals_axis: execution
|
|
7
|
+
ms_class: cascading_error
|
|
8
|
+
suggested_fix:
|
|
9
|
+
target: decision_agent.prompt
|
|
10
|
+
change_template: "when retrieval returns 0 results, branch to query rewrite, alternate source, or explicit 'no answer' before generating"
|
|
11
|
+
verify_with:
|
|
12
|
+
assertion_templates:
|
|
13
|
+
- "if(retrieval.result_count == 0): next_step in [retry, rewrite, alt_source, no_answer]"
|
|
14
|
+
predicate: builtin:retrieval_empty_no_fallback
|
|
15
|
+
params: {}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
function getPrompt(s) {
|
|
2
|
+
const v = s.attributes["gen_ai.prompt"] ?? s.attributes["llm.prompt"];
|
|
3
|
+
return typeof v === "string" ? v : "";
|
|
4
|
+
}
|
|
5
|
+
function getErrorMessage(s) {
|
|
6
|
+
const v = s.attributes["error.message"];
|
|
7
|
+
return typeof v === "string" ? v : "";
|
|
8
|
+
}
|
|
9
|
+
function getToolName(s) {
|
|
10
|
+
const v = s.attributes["gen_ai.tool.name"];
|
|
11
|
+
return typeof v === "string" ? v : s.name;
|
|
12
|
+
}
|
|
13
|
+
export const predicate = (trace) => {
|
|
14
|
+
const allSpans = trace.spans
|
|
15
|
+
.slice()
|
|
16
|
+
.sort((a, b) => Number(BigInt(a.startTimeUnixNano) - BigInt(b.startTimeUnixNano)));
|
|
17
|
+
const hits = [];
|
|
18
|
+
for (let i = 0; i < allSpans.length; i++) {
|
|
19
|
+
const s = allSpans[i];
|
|
20
|
+
if (s.kind !== "tool" || s.status !== "error")
|
|
21
|
+
continue;
|
|
22
|
+
const errMsg = getErrorMessage(s);
|
|
23
|
+
const toolName = getToolName(s);
|
|
24
|
+
// find next LLM span
|
|
25
|
+
let next;
|
|
26
|
+
for (let j = i + 1; j < allSpans.length; j++) {
|
|
27
|
+
if (allSpans[j].kind === "llm") {
|
|
28
|
+
next = allSpans[j];
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (!next)
|
|
33
|
+
continue;
|
|
34
|
+
const prompt = getPrompt(next).toLowerCase();
|
|
35
|
+
const errInPrompt = errMsg.length > 0 && prompt.includes(errMsg.toLowerCase());
|
|
36
|
+
if (!errInPrompt) {
|
|
37
|
+
hits.push({
|
|
38
|
+
evidenceSpans: [s.spanId, next.spanId],
|
|
39
|
+
excerpt: `tool '${toolName}' errored ('${errMsg}') but next LLM prompt did not propagate the error`,
|
|
40
|
+
bindings: { tool_name: toolName, error_message: errMsg },
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return hits;
|
|
45
|
+
};
|