oh-my-knowledge 0.0.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +537 -332
- package/dist/src/analysis/coverage-analyzer.d.ts +57 -0
- package/dist/src/analysis/coverage-analyzer.d.ts.map +1 -0
- package/dist/src/analysis/coverage-analyzer.js +262 -0
- package/dist/src/analysis/coverage-analyzer.js.map +1 -0
- package/dist/src/analysis/gap-analyzer.d.ts +114 -0
- package/dist/src/analysis/gap-analyzer.d.ts.map +1 -0
- package/dist/src/analysis/gap-analyzer.js +439 -0
- package/dist/src/analysis/gap-analyzer.js.map +1 -0
- package/dist/src/analysis/hedging-classifier.d.ts +34 -0
- package/dist/src/analysis/hedging-classifier.d.ts.map +1 -0
- package/dist/src/analysis/hedging-classifier.js +144 -0
- package/dist/src/analysis/hedging-classifier.js.map +1 -0
- package/dist/src/analysis/report-diagnostics.d.ts +9 -0
- package/dist/src/analysis/report-diagnostics.d.ts.map +1 -0
- package/dist/src/analysis/report-diagnostics.js +631 -0
- package/dist/src/analysis/report-diagnostics.js.map +1 -0
- package/dist/src/authoring/evolver.d.ts +64 -0
- package/dist/src/authoring/evolver.d.ts.map +1 -0
- package/dist/src/authoring/evolver.js +275 -0
- package/dist/src/authoring/evolver.js.map +1 -0
- package/dist/src/authoring/generator.d.ts +13 -0
- package/dist/src/authoring/generator.d.ts.map +1 -0
- package/dist/src/authoring/generator.js +60 -0
- package/dist/src/authoring/generator.js.map +1 -0
- package/dist/src/cli.d.ts +3 -0
- package/dist/src/cli.d.ts.map +1 -0
- package/dist/src/cli.js +1004 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/eval-core/cache.d.ts +11 -0
- package/dist/src/eval-core/cache.d.ts.map +1 -0
- package/dist/src/eval-core/cache.js +53 -0
- package/dist/src/eval-core/cache.js.map +1 -0
- package/dist/src/eval-core/ci-gates.d.ts +17 -0
- package/dist/src/eval-core/ci-gates.d.ts.map +1 -0
- package/dist/src/eval-core/ci-gates.js +42 -0
- package/dist/src/eval-core/ci-gates.js.map +1 -0
- package/dist/src/eval-core/dependency-checker.d.ts +37 -0
- package/dist/src/eval-core/dependency-checker.d.ts.map +1 -0
- package/dist/src/eval-core/dependency-checker.js +280 -0
- package/dist/src/eval-core/dependency-checker.js.map +1 -0
- package/dist/src/eval-core/evaluation-execution.d.ts +26 -0
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-execution.js +196 -0
- package/dist/src/eval-core/evaluation-execution.js.map +1 -0
- package/dist/src/eval-core/evaluation-job.d.ts +48 -0
- package/dist/src/eval-core/evaluation-job.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-job.js +112 -0
- package/dist/src/eval-core/evaluation-job.js.map +1 -0
- package/dist/src/eval-core/evaluation-reporting.d.ts +28 -0
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -0
- package/dist/src/eval-core/evaluation-reporting.js +123 -0
- package/dist/src/eval-core/evaluation-reporting.js.map +1 -0
- package/dist/src/eval-core/execution-strategy.d.ts +11 -0
- package/dist/src/eval-core/execution-strategy.d.ts.map +1 -0
- package/dist/src/eval-core/execution-strategy.js +121 -0
- package/dist/src/eval-core/execution-strategy.js.map +1 -0
- package/dist/src/eval-core/fact-checker.d.ts +25 -0
- package/dist/src/eval-core/fact-checker.d.ts.map +1 -0
- package/dist/src/eval-core/fact-checker.js +66 -0
- package/dist/src/eval-core/fact-checker.js.map +1 -0
- package/dist/src/eval-core/schema.d.ts +30 -0
- package/dist/src/eval-core/schema.d.ts.map +1 -0
- package/dist/src/eval-core/schema.js +195 -0
- package/dist/src/eval-core/schema.js.map +1 -0
- package/dist/src/eval-core/statistics.d.ts +86 -0
- package/dist/src/eval-core/statistics.d.ts.map +1 -0
- package/dist/src/eval-core/statistics.js +210 -0
- package/dist/src/eval-core/statistics.js.map +1 -0
- package/dist/src/eval-core/task-planner.d.ts +4 -0
- package/dist/src/eval-core/task-planner.d.ts.map +1 -0
- package/dist/src/eval-core/task-planner.js +33 -0
- package/dist/src/eval-core/task-planner.js.map +1 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +133 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.js +169 -0
- package/dist/src/eval-workflows/each-evaluation-workflow.js.map +1 -0
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +38 -0
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -0
- package/dist/src/eval-workflows/evaluation-pipeline.js +214 -0
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -0
- package/dist/src/eval-workflows/evaluation-preparation.d.ts +64 -0
- package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -0
- package/dist/src/eval-workflows/evaluation-preparation.js +90 -0
- package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -0
- package/dist/src/eval-workflows/run-evaluation.d.ts +105 -0
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -0
- package/dist/src/eval-workflows/run-evaluation.js +264 -0
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -0
- package/dist/src/executors/anthropic-api.d.ts +3 -0
- package/dist/src/executors/anthropic-api.d.ts.map +1 -0
- package/dist/src/executors/anthropic-api.js +43 -0
- package/dist/src/executors/anthropic-api.js.map +1 -0
- package/dist/src/executors/claude-cli.d.ts +3 -0
- package/dist/src/executors/claude-cli.d.ts.map +1 -0
- package/dist/src/executors/claude-cli.js +103 -0
- package/dist/src/executors/claude-cli.js.map +1 -0
- package/dist/src/executors/claude-sdk-trace.d.ts +10 -0
- package/dist/src/executors/claude-sdk-trace.d.ts.map +1 -0
- package/dist/src/executors/claude-sdk-trace.js +100 -0
- package/dist/src/executors/claude-sdk-trace.js.map +1 -0
- package/dist/src/executors/claude-sdk.d.ts +3 -0
- package/dist/src/executors/claude-sdk.d.ts.map +1 -0
- package/dist/src/executors/claude-sdk.js +160 -0
- package/dist/src/executors/claude-sdk.js.map +1 -0
- package/dist/src/executors/gemini.d.ts +3 -0
- package/dist/src/executors/gemini.d.ts.map +1 -0
- package/dist/src/executors/gemini.js +78 -0
- package/dist/src/executors/gemini.js.map +1 -0
- package/dist/src/executors/index.d.ts +7 -0
- package/dist/src/executors/index.d.ts.map +1 -0
- package/dist/src/executors/index.js +22 -0
- package/dist/src/executors/index.js.map +1 -0
- package/dist/src/executors/openai-api.d.ts +3 -0
- package/dist/src/executors/openai-api.d.ts.map +1 -0
- package/dist/src/executors/openai-api.js +40 -0
- package/dist/src/executors/openai-api.js.map +1 -0
- package/dist/src/executors/openai-cli.d.ts +3 -0
- package/dist/src/executors/openai-cli.d.ts.map +1 -0
- package/dist/src/executors/openai-cli.js +60 -0
- package/dist/src/executors/openai-cli.js.map +1 -0
- package/dist/src/executors/script.d.ts +3 -0
- package/dist/src/executors/script.d.ts.map +1 -0
- package/dist/src/executors/script.js +63 -0
- package/dist/src/executors/script.js.map +1 -0
- package/dist/src/executors/shared.d.ts +117 -0
- package/dist/src/executors/shared.d.ts.map +1 -0
- package/dist/src/executors/shared.js +49 -0
- package/dist/src/executors/shared.js.map +1 -0
- package/dist/src/grading/assertions.d.ts +18 -0
- package/dist/src/grading/assertions.d.ts.map +1 -0
- package/dist/src/grading/assertions.js +239 -0
- package/dist/src/grading/assertions.js.map +1 -0
- package/dist/src/grading/index.d.ts +26 -0
- package/dist/src/grading/index.d.ts.map +1 -0
- package/dist/src/grading/index.js +98 -0
- package/dist/src/grading/index.js.map +1 -0
- package/dist/src/grading/judge.d.ts +13 -0
- package/dist/src/grading/judge.d.ts.map +1 -0
- package/dist/src/grading/judge.js +98 -0
- package/dist/src/grading/judge.js.map +1 -0
- package/dist/src/grading/layered-scores.d.ts +13 -0
- package/dist/src/grading/layered-scores.d.ts.map +1 -0
- package/dist/src/grading/layered-scores.js +62 -0
- package/dist/src/grading/layered-scores.js.map +1 -0
- package/dist/src/inputs/eval-config.d.ts +13 -0
- package/dist/src/inputs/eval-config.d.ts.map +1 -0
- package/dist/src/inputs/eval-config.js +136 -0
- package/dist/src/inputs/eval-config.js.map +1 -0
- package/dist/src/inputs/load-samples.d.ts +16 -0
- package/dist/src/inputs/load-samples.d.ts.map +1 -0
- package/dist/src/inputs/load-samples.js +51 -0
- package/dist/src/inputs/load-samples.js.map +1 -0
- package/dist/src/inputs/mcp-resolver.d.ts +50 -0
- package/dist/src/inputs/mcp-resolver.d.ts.map +1 -0
- package/dist/src/inputs/mcp-resolver.js +307 -0
- package/dist/src/inputs/mcp-resolver.js.map +1 -0
- package/dist/src/inputs/skill-loader.d.ts +18 -0
- package/dist/src/inputs/skill-loader.d.ts.map +1 -0
- package/dist/src/inputs/skill-loader.js +222 -0
- package/dist/src/inputs/skill-loader.js.map +1 -0
- package/dist/src/inputs/url-fetcher.d.ts +19 -0
- package/dist/src/inputs/url-fetcher.d.ts.map +1 -0
- package/dist/src/inputs/url-fetcher.js +241 -0
- package/dist/src/inputs/url-fetcher.js.map +1 -0
- package/dist/src/observability/production-analyzer.d.ts +61 -0
- package/dist/src/observability/production-analyzer.d.ts.map +1 -0
- package/dist/src/observability/production-analyzer.js +183 -0
- package/dist/src/observability/production-analyzer.js.map +1 -0
- package/dist/src/observability/trace-adapter.d.ts +75 -0
- package/dist/src/observability/trace-adapter.d.ts.map +1 -0
- package/dist/src/observability/trace-adapter.js +341 -0
- package/dist/src/observability/trace-adapter.js.map +1 -0
- package/dist/src/renderer/html-renderer.d.ts +9 -0
- package/dist/src/renderer/html-renderer.d.ts.map +1 -0
- package/dist/src/renderer/html-renderer.js +338 -0
- package/dist/src/renderer/html-renderer.js.map +1 -0
- package/dist/src/renderer/layout.d.ts +13 -0
- package/dist/src/renderer/layout.d.ts.map +1 -0
- package/dist/src/renderer/layout.js +476 -0
- package/dist/src/renderer/layout.js.map +1 -0
- package/dist/src/renderer/skill-health-renderer.d.ts +20 -0
- package/dist/src/renderer/skill-health-renderer.d.ts.map +1 -0
- package/dist/src/renderer/skill-health-renderer.js +218 -0
- package/dist/src/renderer/skill-health-renderer.js.map +1 -0
- package/dist/src/renderer/summary.d.ts +21 -0
- package/dist/src/renderer/summary.d.ts.map +1 -0
- package/dist/src/renderer/summary.js +1072 -0
- package/dist/src/renderer/summary.js.map +1 -0
- package/dist/src/renderer/table.d.ts +3 -0
- package/dist/src/renderer/table.d.ts.map +1 -0
- package/dist/src/renderer/table.js +134 -0
- package/dist/src/renderer/table.js.map +1 -0
- package/dist/src/renderer/trends.d.ts +6 -0
- package/dist/src/renderer/trends.d.ts.map +1 -0
- package/dist/src/renderer/trends.js +127 -0
- package/dist/src/renderer/trends.js.map +1 -0
- package/dist/src/server/job-store.d.ts +4 -0
- package/dist/src/server/job-store.d.ts.map +1 -0
- package/dist/src/server/job-store.js +68 -0
- package/dist/src/server/job-store.js.map +1 -0
- package/dist/src/server/report-server.d.ts +16 -0
- package/dist/src/server/report-server.d.ts.map +1 -0
- package/dist/src/server/report-server.js +196 -0
- package/dist/src/server/report-server.js.map +1 -0
- package/dist/src/server/report-store.d.ts +44 -0
- package/dist/src/server/report-store.d.ts.map +1 -0
- package/dist/src/server/report-store.js +190 -0
- package/dist/src/server/report-store.js.map +1 -0
- package/dist/src/types.d.ts +553 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +2 -0
- package/dist/src/types.js.map +1 -0
- package/package.json +44 -3
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL content fetcher for eval-samples.
|
|
3
|
+
*
|
|
4
|
+
* Scans sample prompts/contexts for URLs, fetches their content,
|
|
5
|
+
* and inlines the text back into the sample fields.
|
|
6
|
+
*
|
|
7
|
+
* If a URL requires authentication, ensure the CLI environment can access it
|
|
8
|
+
* (e.g., via VPN, proxy, cookies) before running the evaluation.
|
|
9
|
+
*/
|
|
10
|
+
// Match URLs using RFC 3986 allowed characters (whitelist approach).
|
|
11
|
+
// This avoids accidentally consuming CJK characters or punctuation adjacent to URLs.
|
|
12
|
+
const URL_REGEX = /https?:\/\/[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+/g;
|
|
13
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
14
|
+
// RFC 2606 / IANA reserved placeholder domains — these are documentation-only
|
|
15
|
+
// and should never be fetched (they don't resolve to real services).
|
|
16
|
+
const PLACEHOLDER_DOMAIN_REGEX = /^https?:\/\/(?:[^/]+\.)?(?:example\.(?:com|org|net)|localhost|test|invalid|example)(?::\d+)?(?:[/?#]|$)/i;
|
|
17
|
+
export function isPlaceholderUrl(url) {
|
|
18
|
+
return PLACEHOLDER_DOMAIN_REGEX.test(url);
|
|
19
|
+
}
|
|
20
|
+
function asNetworkError(err) {
|
|
21
|
+
return typeof err === 'object' && err !== null ? err : {};
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Resolve all URLs found in sample prompts and contexts.
|
|
25
|
+
* Mutates samples in-place: each URL is replaced with URL + fetched content.
|
|
26
|
+
*
|
|
27
|
+
* @param skipUrls - URLs already resolved (e.g. by MCP), will be skipped
|
|
28
|
+
*/
|
|
29
|
+
export async function resolveUrls(samples, skipUrls) {
|
|
30
|
+
// 1. Collect all unique URLs across all samples, tracking which sample they belong to
|
|
31
|
+
const urlMap = new Map(); // url -> [{sample, field}]
|
|
32
|
+
for (const sample of samples) {
|
|
33
|
+
for (const field of ['prompt', 'context']) {
|
|
34
|
+
const text = sample[field];
|
|
35
|
+
if (!text)
|
|
36
|
+
continue;
|
|
37
|
+
const matches = text.match(URL_REGEX);
|
|
38
|
+
if (!matches)
|
|
39
|
+
continue;
|
|
40
|
+
for (const url of matches) {
|
|
41
|
+
if (skipUrls?.has(url))
|
|
42
|
+
continue;
|
|
43
|
+
if (isPlaceholderUrl(url))
|
|
44
|
+
continue; // RFC 2606 placeholder domains are documentation-only
|
|
45
|
+
if (!urlMap.has(url))
|
|
46
|
+
urlMap.set(url, []);
|
|
47
|
+
urlMap.get(url).push({ sample, field });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (urlMap.size === 0)
|
|
52
|
+
return;
|
|
53
|
+
// 2. Fetch all unique URLs concurrently
|
|
54
|
+
const fetched = new Map(); // url -> { ok, content, error }
|
|
55
|
+
const entries = [...urlMap.keys()];
|
|
56
|
+
process.stderr.write(`ℹ 检测到 ${entries.length} 个 URL,正在获取内容:\n`);
|
|
57
|
+
for (const url of entries) {
|
|
58
|
+
process.stderr.write(` - ${url}\n`);
|
|
59
|
+
}
|
|
60
|
+
const results = await Promise.allSettled(entries.map((url) => fetchUrl(url)));
|
|
61
|
+
for (let i = 0; i < entries.length; i++) {
|
|
62
|
+
const url = entries[i];
|
|
63
|
+
const result = results[i];
|
|
64
|
+
if (result.status === 'fulfilled') {
|
|
65
|
+
fetched.set(url, result.value);
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
fetched.set(url, { ok: false, error: result.reason.message || 'unknown error' });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// 3. Warn about failures, inline successes
|
|
72
|
+
for (const [url, result] of fetched) {
|
|
73
|
+
if (!result.ok) {
|
|
74
|
+
const affectedSamples = urlMap.get(url).map((r) => r.sample.sample_id);
|
|
75
|
+
const unique = [...new Set(affectedSamples)];
|
|
76
|
+
process.stderr.write(`\n${formatError(url, result, unique)}\n`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const failCount = [...fetched.values()].filter((r) => !r.ok).length;
|
|
80
|
+
if (failCount > 0) {
|
|
81
|
+
process.stderr.write(`\n⚠ ${failCount} 个 URL 抓取失败,将使用原始 URL 继续评测\n\n`);
|
|
82
|
+
}
|
|
83
|
+
// 4. Inline fetched content into samples (only successful ones)
|
|
84
|
+
for (const [url, result] of fetched) {
|
|
85
|
+
if (!result.ok)
|
|
86
|
+
continue;
|
|
87
|
+
const replacement = `${url}\n\n---\n${result.content}\n---`;
|
|
88
|
+
for (const { sample, field } of urlMap.get(url)) {
|
|
89
|
+
const current = sample[field];
|
|
90
|
+
if (typeof current === 'string') {
|
|
91
|
+
sample[field] = current.replace(url, replacement);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Fetch a single URL and return its text content.
|
|
98
|
+
*/
|
|
99
|
+
async function fetchUrl(url) {
|
|
100
|
+
let response;
|
|
101
|
+
try {
|
|
102
|
+
response = await fetch(url, {
|
|
103
|
+
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
|
104
|
+
headers: {
|
|
105
|
+
'User-Agent': 'oh-my-knowledge/url-fetcher',
|
|
106
|
+
'Accept': 'text/html, text/plain, */*',
|
|
107
|
+
},
|
|
108
|
+
redirect: 'follow',
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
catch (err) {
|
|
112
|
+
return {
|
|
113
|
+
ok: false,
|
|
114
|
+
statusCode: 0,
|
|
115
|
+
error: networkErrorMessage(err),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
if (!response.ok) {
|
|
119
|
+
return {
|
|
120
|
+
ok: false,
|
|
121
|
+
statusCode: response.status,
|
|
122
|
+
error: `${response.status} ${response.statusText}`,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
const contentType = response.headers.get('content-type') || '';
|
|
126
|
+
const raw = await response.text();
|
|
127
|
+
const content = contentType.includes('text/html')
|
|
128
|
+
? htmlToText(raw)
|
|
129
|
+
: raw.trim();
|
|
130
|
+
// Detect login/redirect pages that return 200 but no real content
|
|
131
|
+
const loginHint = detectLoginPage(content);
|
|
132
|
+
if (loginHint) {
|
|
133
|
+
return {
|
|
134
|
+
ok: false,
|
|
135
|
+
statusCode: 200,
|
|
136
|
+
error: `页面返回了登录/跳转页而非实际内容 (${loginHint})`,
|
|
137
|
+
isLoginPage: true,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
return { ok: true, content };
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Strip HTML tags to extract readable text.
|
|
144
|
+
*/
|
|
145
|
+
function htmlToText(html) {
|
|
146
|
+
return html
|
|
147
|
+
// Remove script/style blocks
|
|
148
|
+
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
|
149
|
+
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
|
150
|
+
// Replace block-level tags with newlines
|
|
151
|
+
.replace(/<\/(p|div|h[1-6]|li|tr|blockquote)>/gi, '\n')
|
|
152
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
153
|
+
// Remove all remaining tags
|
|
154
|
+
.replace(/<[^>]+>/g, '')
|
|
155
|
+
// Decode common HTML entities
|
|
156
|
+
.replace(/&/g, '&')
|
|
157
|
+
.replace(/</g, '<')
|
|
158
|
+
.replace(/>/g, '>')
|
|
159
|
+
.replace(/"/g, '"')
|
|
160
|
+
.replace(/'/g, "'")
|
|
161
|
+
.replace(/ /g, ' ')
|
|
162
|
+
// Clean up whitespace
|
|
163
|
+
.replace(/[ \t]+/g, ' ')
|
|
164
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
165
|
+
.trim();
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Detect login/auth redirect pages that return HTTP 200 but no real content.
|
|
169
|
+
* Returns a hint string if detected, null otherwise.
|
|
170
|
+
*/
|
|
171
|
+
function detectLoginPage(content) {
|
|
172
|
+
if (content.length < 50)
|
|
173
|
+
return '内容过短';
|
|
174
|
+
const lower = content.toLowerCase();
|
|
175
|
+
const loginPatterns = [
|
|
176
|
+
'login', 'sign in', 'sign_in', 'sso', 'redirect_uri',
|
|
177
|
+
'oauth', 'unauthorized', 'cas/login',
|
|
178
|
+
];
|
|
179
|
+
for (const pattern of loginPatterns) {
|
|
180
|
+
if (lower.includes(pattern) && content.length < 500) {
|
|
181
|
+
return pattern;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Map network errors to user-friendly messages.
|
|
188
|
+
*/
|
|
189
|
+
function networkErrorMessage(err) {
|
|
190
|
+
const details = asNetworkError(err);
|
|
191
|
+
const code = details.cause?.code || details.code || '';
|
|
192
|
+
if (code === 'ENOTFOUND')
|
|
193
|
+
return '域名无法解析,请检查 URL 是否正确';
|
|
194
|
+
if (code === 'ECONNREFUSED')
|
|
195
|
+
return '连接被拒绝,请检查网络或代理配置';
|
|
196
|
+
if (code === 'ECONNRESET')
|
|
197
|
+
return '连接被重置,请检查网络';
|
|
198
|
+
if (details.name === 'TimeoutError' || code === 'ABORT_ERR')
|
|
199
|
+
return `请求超时 (${FETCH_TIMEOUT_MS / 1000}s)`;
|
|
200
|
+
return details.message || 'unknown network error';
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Format a fetch error with actionable solution.
|
|
204
|
+
*/
|
|
205
|
+
function formatError(url, result, sampleIds) {
|
|
206
|
+
const sampleLabel = sampleIds.map((id) => `"${id}"`).join(', ');
|
|
207
|
+
const lines = [
|
|
208
|
+
`✗ sample ${sampleLabel}: URL 无法访问`,
|
|
209
|
+
` URL: ${url}`,
|
|
210
|
+
];
|
|
211
|
+
if (result.statusCode) {
|
|
212
|
+
lines.push(` 状态: ${result.error}`);
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
lines.push(` 错误: ${result.error}`);
|
|
216
|
+
}
|
|
217
|
+
lines.push(` 解决方案: ${getSolution(result.statusCode, result)}`);
|
|
218
|
+
return lines.join('\n');
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Map HTTP status code to actionable solution text.
|
|
222
|
+
*/
|
|
223
|
+
function getSolution(statusCode, result) {
|
|
224
|
+
if (result?.isLoginPage) {
|
|
225
|
+
return '该链接需要认证,请确保命令行环境可正常访问(如配置 VPN/代理/cookie),或将文档内容复制到 eval-samples 的 context 字段中';
|
|
226
|
+
}
|
|
227
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
228
|
+
return '该链接需要认证,请确保命令行环境可正常访问,或将文档内容复制到 eval-samples 的 context 字段中';
|
|
229
|
+
}
|
|
230
|
+
if (statusCode === 404) {
|
|
231
|
+
return '页面不存在,请检查链接是否正确';
|
|
232
|
+
}
|
|
233
|
+
if (statusCode && statusCode >= 500) {
|
|
234
|
+
return '服务端错误,请稍后重试';
|
|
235
|
+
}
|
|
236
|
+
if (!statusCode) {
|
|
237
|
+
return '无法连接目标服务,请检查网络或代理配置,或将文档内容复制到 eval-samples 的 context 字段中';
|
|
238
|
+
}
|
|
239
|
+
return '请将文档内容复制到 eval-samples 的 context 字段中';
|
|
240
|
+
}
|
|
241
|
+
//# sourceMappingURL=url-fetcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-fetcher.js","sourceRoot":"","sources":["../../../src/inputs/url-fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,qEAAqE;AACrE,qFAAqF;AACrF,MAAM,SAAS,GAAG,mDAAmD,CAAC;AACtE,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,8EAA8E;AAC9E,qEAAqE;AACrE,MAAM,wBAAwB,GAAG,0GAA0G,CAAC;AAE5I,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,OAAO,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5C,CAAC;AAiBD,SAAS,cAAc,CAAC,GAAY;IAClC,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAuB,CAAC,CAAC,CAAC,EAAE,CAAC;AAChF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAiB,EAAE,QAAsB;IACzE,sFAAsF;IACtF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoD,CAAC,CAAC,2BAA2B;IACvG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAuB,CAAC;YACjD,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACjC,IAAI,gBAAgB,CAAC,GAAG,CAAC;oBAAE,SAAS,CAAC,sDAAsD;gBAC3F,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO;IAE9B,wCAAwC;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC,CAAC,gCAAgC;IAChF,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAEnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC;IAChE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CACpC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAG,MAAM,CAAC,MAAgB,CAAC,OAAO,IAAI,eAAe,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC;YAC7C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,gCAAgC,CAAC,CAAC;IACzE,CAAC;IAED,gEAAgE;IAChE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAS;QACzB,MAAM,WAAW,GAAG,GAAG,GAAG,YAAY,MAAM,CAAC,OAAO,OAAO,CAAC;QAC5D,KAAK,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAW;IACjC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC7C,OAAO,EAAE;gBACP,YAAY,EAAE,6BAA6B;gBAC3C,QAAQ,EAAE,4BAA4B;aACvC;YACD,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,UAAU,EAAE,CAAC;YACb,KAAK,EAAE,mBAAmB,CAAC,GAAG,CAAC;SAChC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,KAAK,EAAE,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAElC,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC/C,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;QACjB,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEf,kEAAkE;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,SAAS,EAAE,CAAC;QACd,OAAO;YACL,EAAE,EAAE,KAAK;YACT,UAAU,EAAE,GAAG;YACf,KAAK,EAAE,sBAAsB,SAAS,GAAG;YACzC,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI;QACT,6BAA6B;SAC5B,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC;QACzC,yCAAyC;SACxC,OAAO,CAAC,uCAAuC,EAAE,IAAI,CAAC;SACtD,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;QAC9B,4BAA4B;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,8BAA8B;SAC7B,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;QACxB,sBAAsB;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,OAAe;IACtC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,MAAM,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,aAAa,GAAG;QACpB,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc;QACpD,OAAO,EAAE,cAAc,EAAE,WAAW;KACrC,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACpD,OAAO,OAAO,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,GAAY;IACvC,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,IAAI,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;IACvD,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,qBAAqB,CAAC;IACvD,IAAI,IAAI,KAAK,cAAc;QAAE,OAAO,kBAAkB,CAAC;IACvD,IAAI,IAAI,KAAK,YAAY;QAAE,OAAO,aAAa,CAAC;IAChD,IAAI,OAAO,CAAC,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,SAAS,gBAAgB,GAAG,IAAI,IAAI,CAAC;IACzG,OAAO,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,MAAmB,EAAE,SAAmB;IACxE,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG;QACZ,YAAY,WAAW,YAAY;QACnC,UAAU,GAAG,EAAE;KAChB,CAAC;IAEF,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACtC,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAW,WAAW,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAChE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,UAA8B,EAAE,MAAmB;IACtE,IAAI,MAAM,EAAE,WAAW,EAAE,CAAC;QACxB,OAAO,+EAA+E,CAAC;IACzF,CAAC;IACD,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAC7C,OAAO,4DAA4D,CAAC;IACtE,CAAC;IACD,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IACD,IAAI,UAAU,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;QACpC,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,0DAA0D,CAAC;IACpE,CAAC;IACD,OAAO,sCAAsC,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Production analyzer (v0.18 skill-health).
|
|
3
|
+
*
|
|
4
|
+
* 接 trace-adapter 输出 → 按 skill 维度聚合 coverage + gap,产出 SkillHealthReport。
|
|
5
|
+
* 复用 analysis/coverage-analyzer + analysis/gap-analyzer,跳过对照组逻辑。
|
|
6
|
+
*
|
|
7
|
+
* 分析流水线:
|
|
8
|
+
* 1. ccTracesToResultEntries(path) → segments + ResultEntry[]
|
|
9
|
+
* 2. 时间窗 / skill 白名单过滤
|
|
10
|
+
* 3. 按 skill name (variant key) 分别 computeCoverage + computeGapReport
|
|
11
|
+
* 4. 聚合 overall 指标 + 健康度色带
|
|
12
|
+
*/
|
|
13
|
+
import { type CoverageReport } from '../analysis/coverage-analyzer.js';
|
|
14
|
+
import type { GapReport } from '../types.js';
|
|
15
|
+
import { type CcSession, type SkillSegment } from './trace-adapter.js';
|
|
16
|
+
export interface SkillHealth {
|
|
17
|
+
skillName: string;
|
|
18
|
+
segmentCount: number;
|
|
19
|
+
toolCallCount: number;
|
|
20
|
+
toolFailureCount: number;
|
|
21
|
+
coverage: CoverageReport | null;
|
|
22
|
+
gap: GapReport;
|
|
23
|
+
}
|
|
24
|
+
export interface SkillHealthReport {
|
|
25
|
+
meta: {
|
|
26
|
+
tracePath: string;
|
|
27
|
+
kbPath: string | null;
|
|
28
|
+
sessionCount: number;
|
|
29
|
+
segmentCount: number;
|
|
30
|
+
messageCount: number;
|
|
31
|
+
toolCallCount: number;
|
|
32
|
+
toolFailureRate: number;
|
|
33
|
+
timeRange: {
|
|
34
|
+
from: string;
|
|
35
|
+
to: string;
|
|
36
|
+
};
|
|
37
|
+
generatedAt: string;
|
|
38
|
+
};
|
|
39
|
+
bySkill: Record<string, SkillHealth>;
|
|
40
|
+
overall: {
|
|
41
|
+
gapRate: number;
|
|
42
|
+
weightedGapRate: number;
|
|
43
|
+
healthBand: 'green' | 'yellow' | 'red';
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export interface AnalyzeOptions {
|
|
47
|
+
kbRoot?: string | null;
|
|
48
|
+
from?: string;
|
|
49
|
+
to?: string;
|
|
50
|
+
skills?: string[];
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* 主入口:从 cc trace 目录生成 SkillHealthReport。
|
|
54
|
+
*/
|
|
55
|
+
export declare function computeSkillHealthReport(tracePath: string, opts?: AnalyzeOptions): SkillHealthReport;
|
|
56
|
+
/**
|
|
57
|
+
* 便利入口:直接从已准备好的 segments(跳过 loadCcSessions)算 report。
|
|
58
|
+
* 用于测试 / 已手工组装过 segments 的场景。
|
|
59
|
+
*/
|
|
60
|
+
export declare function computeSkillHealthFromSegments(segments: SkillSegment[], sessions: CcSession[], tracePath: string, opts?: AnalyzeOptions): SkillHealthReport;
|
|
61
|
+
//# sourceMappingURL=production-analyzer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"production-analyzer.d.ts","sourceRoot":"","sources":["../../../src/observability/production-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAwC,KAAK,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAE7G,OAAO,KAAK,EAAE,SAAS,EAAe,MAAM,aAAa,CAAC;AAC1D,OAAO,EAGL,KAAK,SAAS,EACd,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,GAAG,EAAE,SAAS,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE;QACJ,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,SAAS,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,OAAO,EAAE;QACP,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;KACxC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AA2CD;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,iBAAiB,CAqExG;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,YAAY,EAAE,EACxB,QAAQ,EAAE,SAAS,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,cAAmB,GACxB,iBAAiB,CAOnB"}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Production analyzer (v0.18 skill-health).
|
|
3
|
+
*
|
|
4
|
+
* 接 trace-adapter 输出 → 按 skill 维度聚合 coverage + gap,产出 SkillHealthReport。
|
|
5
|
+
* 复用 analysis/coverage-analyzer + analysis/gap-analyzer,跳过对照组逻辑。
|
|
6
|
+
*
|
|
7
|
+
* 分析流水线:
|
|
8
|
+
* 1. ccTracesToResultEntries(path) → segments + ResultEntry[]
|
|
9
|
+
* 2. 时间窗 / skill 白名单过滤
|
|
10
|
+
* 3. 按 skill name (variant key) 分别 computeCoverage + computeGapReport
|
|
11
|
+
* 4. 聚合 overall 指标 + 健康度色带
|
|
12
|
+
*/
|
|
13
|
+
import { buildKnowledgeIndex, computeCoverage } from '../analysis/coverage-analyzer.js';
|
|
14
|
+
import { computeGapReport } from '../analysis/gap-analyzer.js';
|
|
15
|
+
import { ccTracesToResultEntries, segmentsToResultEntries, } from './trace-adapter.js';
|
|
16
|
+
/**
|
|
17
|
+
* 从时间戳字符串比较。ISO8601 字典序即时序,直接用字符串比较。
|
|
18
|
+
*/
|
|
19
|
+
function timestampLt(a, b) {
|
|
20
|
+
return a < b;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* 判断 segment 是否落在时间窗内(闭区间)。
|
|
24
|
+
*/
|
|
25
|
+
function withinTimeWindow(seg, from, to) {
|
|
26
|
+
if (from && timestampLt(seg.startTimestamp, from))
|
|
27
|
+
return false;
|
|
28
|
+
if (to && timestampLt(to, seg.startTimestamp))
|
|
29
|
+
return false;
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* overall 健康度色带。阈值对齐 bench ci --max-gap-rate 经验值(spec §五)。
|
|
34
|
+
*/
|
|
35
|
+
function healthBandOf(weightedGapRate) {
|
|
36
|
+
if (weightedGapRate >= 0.3)
|
|
37
|
+
return 'red';
|
|
38
|
+
if (weightedGapRate >= 0.1)
|
|
39
|
+
return 'yellow';
|
|
40
|
+
return 'green';
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* 推断 KB root: 没传 --kb 时,取第一个 assistant record 的 cwd。
|
|
44
|
+
* 如果跨多个 cwd,取第一个并 warn。
|
|
45
|
+
*/
|
|
46
|
+
function inferKbRoot(sessions) {
|
|
47
|
+
const cwds = new Set();
|
|
48
|
+
for (const s of sessions) {
|
|
49
|
+
if (s.cwd)
|
|
50
|
+
cwds.add(s.cwd);
|
|
51
|
+
}
|
|
52
|
+
if (cwds.size === 0)
|
|
53
|
+
return null;
|
|
54
|
+
if (cwds.size > 1) {
|
|
55
|
+
process.stderr.write(`[omk] analyze: trace spans ${cwds.size} projects, coverage defaults to first cwd. Pass --kb explicitly for cross-project analysis.\n`);
|
|
56
|
+
}
|
|
57
|
+
return cwds.values().next().value ?? null;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* 主入口:从 cc trace 目录生成 SkillHealthReport。
|
|
61
|
+
*/
|
|
62
|
+
export function computeSkillHealthReport(tracePath, opts = {}) {
|
|
63
|
+
const { sessions, segments } = ccTracesToResultEntries(tracePath);
|
|
64
|
+
// 时间窗 + skill 白名单过滤
|
|
65
|
+
let filtered = segments.filter((s) => withinTimeWindow(s, opts.from, opts.to));
|
|
66
|
+
if (opts.skills?.length) {
|
|
67
|
+
const allow = new Set(opts.skills);
|
|
68
|
+
filtered = filtered.filter((s) => allow.has(s.skillName));
|
|
69
|
+
}
|
|
70
|
+
const filteredEntries = segmentsToResultEntries(filtered);
|
|
71
|
+
// 推断 KB root
|
|
72
|
+
const kbRoot = opts.kbRoot ?? inferKbRoot(sessions);
|
|
73
|
+
const index = kbRoot ? buildKnowledgeIndex(kbRoot) : null;
|
|
74
|
+
// 按 skill 分组聚合
|
|
75
|
+
const skillNames = [...new Set(filtered.map((s) => s.skillName))];
|
|
76
|
+
const bySkill = {};
|
|
77
|
+
for (const skill of skillNames) {
|
|
78
|
+
const skillSegs = filtered.filter((s) => s.skillName === skill);
|
|
79
|
+
const coverage = index ? computeCoverage(filteredEntries, skill, index, kbRoot) : null;
|
|
80
|
+
const gap = computeGapReport(filteredEntries, skill);
|
|
81
|
+
// 挂 trace 源作水印(spec §六)
|
|
82
|
+
gap.testSetPath = tracePath;
|
|
83
|
+
bySkill[skill] = {
|
|
84
|
+
skillName: skill,
|
|
85
|
+
segmentCount: skillSegs.length,
|
|
86
|
+
toolCallCount: skillSegs.reduce((a, s) => a + s.metrics.numToolCalls, 0),
|
|
87
|
+
toolFailureCount: skillSegs.reduce((a, s) => a + s.metrics.numToolFailures, 0),
|
|
88
|
+
coverage,
|
|
89
|
+
gap,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
// Overall 聚合(加权平均,权重 = 每个 skill 的 segment 数)
|
|
93
|
+
const totalSegments = filtered.length;
|
|
94
|
+
const totalGap = Object.values(bySkill).reduce((a, h) => a + h.gap.samplesWithGap, 0);
|
|
95
|
+
const totalWeighted = Object.values(bySkill).reduce((a, h) => a + h.gap.weightedGapRate * h.gap.sampleCount, 0);
|
|
96
|
+
const gapRate = totalSegments > 0 ? Number((totalGap / totalSegments).toFixed(4)) : 0;
|
|
97
|
+
const weightedGapRate = totalSegments > 0 ? Number((totalWeighted / totalSegments).toFixed(4)) : 0;
|
|
98
|
+
// meta
|
|
99
|
+
const totalToolCalls = filtered.reduce((a, s) => a + s.metrics.numToolCalls, 0);
|
|
100
|
+
const totalFailures = filtered.reduce((a, s) => a + s.metrics.numToolFailures, 0);
|
|
101
|
+
const timeRange = filtered.length > 0
|
|
102
|
+
? {
|
|
103
|
+
from: filtered.reduce((m, s) => (timestampLt(s.startTimestamp, m) ? s.startTimestamp : m), filtered[0].startTimestamp),
|
|
104
|
+
to: filtered.reduce((m, s) => (timestampLt(m, s.endTimestamp) ? s.endTimestamp : m), filtered[0].endTimestamp),
|
|
105
|
+
}
|
|
106
|
+
: { from: '', to: '' };
|
|
107
|
+
return {
|
|
108
|
+
meta: {
|
|
109
|
+
tracePath,
|
|
110
|
+
kbPath: kbRoot,
|
|
111
|
+
sessionCount: sessions.length,
|
|
112
|
+
segmentCount: totalSegments,
|
|
113
|
+
messageCount: sessions.reduce((a, s) => a + s.records.length, 0),
|
|
114
|
+
toolCallCount: totalToolCalls,
|
|
115
|
+
toolFailureRate: totalToolCalls > 0 ? Number((totalFailures / totalToolCalls).toFixed(4)) : 0,
|
|
116
|
+
timeRange,
|
|
117
|
+
generatedAt: new Date().toISOString(),
|
|
118
|
+
},
|
|
119
|
+
bySkill,
|
|
120
|
+
overall: { gapRate, weightedGapRate, healthBand: healthBandOf(weightedGapRate) },
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* 便利入口:直接从已准备好的 segments(跳过 loadCcSessions)算 report。
|
|
125
|
+
* 用于测试 / 已手工组装过 segments 的场景。
|
|
126
|
+
*/
|
|
127
|
+
export function computeSkillHealthFromSegments(segments, sessions, tracePath, opts = {}) {
|
|
128
|
+
const filteredSegs = segments.filter((s) => withinTimeWindow(s, opts.from, opts.to));
|
|
129
|
+
const finalSegs = opts.skills?.length
|
|
130
|
+
? filteredSegs.filter((s) => opts.skills.includes(s.skillName))
|
|
131
|
+
: filteredSegs;
|
|
132
|
+
const finalEntries = segmentsToResultEntries(finalSegs);
|
|
133
|
+
return buildReport(finalSegs, finalEntries, sessions, tracePath, opts);
|
|
134
|
+
}
|
|
135
|
+
function buildReport(segments, entries, sessions, tracePath, opts) {
|
|
136
|
+
const kbRoot = opts.kbRoot ?? inferKbRoot(sessions);
|
|
137
|
+
const index = kbRoot ? buildKnowledgeIndex(kbRoot) : null;
|
|
138
|
+
const skillNames = [...new Set(segments.map((s) => s.skillName))];
|
|
139
|
+
const bySkill = {};
|
|
140
|
+
for (const skill of skillNames) {
|
|
141
|
+
const skillSegs = segments.filter((s) => s.skillName === skill);
|
|
142
|
+
const coverage = index ? computeCoverage(entries, skill, index, kbRoot) : null;
|
|
143
|
+
const gap = computeGapReport(entries, skill);
|
|
144
|
+
gap.testSetPath = tracePath;
|
|
145
|
+
bySkill[skill] = {
|
|
146
|
+
skillName: skill,
|
|
147
|
+
segmentCount: skillSegs.length,
|
|
148
|
+
toolCallCount: skillSegs.reduce((a, s) => a + s.metrics.numToolCalls, 0),
|
|
149
|
+
toolFailureCount: skillSegs.reduce((a, s) => a + s.metrics.numToolFailures, 0),
|
|
150
|
+
coverage,
|
|
151
|
+
gap,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
const totalSegments = segments.length;
|
|
155
|
+
const totalGap = Object.values(bySkill).reduce((a, h) => a + h.gap.samplesWithGap, 0);
|
|
156
|
+
const totalWeighted = Object.values(bySkill).reduce((a, h) => a + h.gap.weightedGapRate * h.gap.sampleCount, 0);
|
|
157
|
+
const gapRate = totalSegments > 0 ? Number((totalGap / totalSegments).toFixed(4)) : 0;
|
|
158
|
+
const weightedGapRate = totalSegments > 0 ? Number((totalWeighted / totalSegments).toFixed(4)) : 0;
|
|
159
|
+
const totalToolCalls = segments.reduce((a, s) => a + s.metrics.numToolCalls, 0);
|
|
160
|
+
const totalFailures = segments.reduce((a, s) => a + s.metrics.numToolFailures, 0);
|
|
161
|
+
const timeRange = segments.length > 0
|
|
162
|
+
? {
|
|
163
|
+
from: segments.reduce((m, s) => (timestampLt(s.startTimestamp, m) ? s.startTimestamp : m), segments[0].startTimestamp),
|
|
164
|
+
to: segments.reduce((m, s) => (timestampLt(m, s.endTimestamp) ? s.endTimestamp : m), segments[0].endTimestamp),
|
|
165
|
+
}
|
|
166
|
+
: { from: '', to: '' };
|
|
167
|
+
return {
|
|
168
|
+
meta: {
|
|
169
|
+
tracePath,
|
|
170
|
+
kbPath: kbRoot,
|
|
171
|
+
sessionCount: sessions.length,
|
|
172
|
+
segmentCount: totalSegments,
|
|
173
|
+
messageCount: sessions.reduce((a, s) => a + s.records.length, 0),
|
|
174
|
+
toolCallCount: totalToolCalls,
|
|
175
|
+
toolFailureRate: totalToolCalls > 0 ? Number((totalFailures / totalToolCalls).toFixed(4)) : 0,
|
|
176
|
+
timeRange,
|
|
177
|
+
generatedAt: new Date().toISOString(),
|
|
178
|
+
},
|
|
179
|
+
bySkill,
|
|
180
|
+
overall: { gapRate, weightedGapRate, healthBand: healthBandOf(weightedGapRate) },
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
//# sourceMappingURL=production-analyzer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"production-analyzer.js","sourceRoot":"","sources":["../../../src/observability/production-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAuB,MAAM,kCAAkC,CAAC;AAC7G,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAE/D,OAAO,EACL,uBAAuB,EACvB,uBAAuB,GAGxB,MAAM,oBAAoB,CAAC;AAsC5B;;GAEG;AACH,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS;IACvC,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAiB,EAAE,IAAa,EAAE,EAAW;IACrE,IAAI,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,EAAE,IAAI,WAAW,CAAC,EAAE,EAAE,GAAG,CAAC,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,eAAuB;IAC3C,IAAI,eAAe,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,eAAe,IAAI,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC5C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,QAAqB;IACxC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,GAAG;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,IAAI,CAAC,IAAI,+FAA+F,CAAC,CAAC;IAC/J,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAAiB,EAAE,OAAuB,EAAE;IACnF,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAElE,oBAAoB;IACpB,IAAI,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,eAAe,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAE1D,aAAa;IACb,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1D,eAAe;IACf,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAgC,EAAE,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACvF,MAAM,GAAG,GAAG,gBAAgB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACrD,wBAAwB;QACxB,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,GAAG;YACf,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,SAAS,CAAC,MAAM;YAC9B,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YACxE,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YAC9E,QAAQ;YACR,GAAG;SACJ,CAAC;IACJ,CAAC;IAED,6CAA6C;IAC7C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACtF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,EACvD,CAAC,CACF,CAAC;IACF,MAAM,OAAO,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,eAAe,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnG,OAAO;IACP,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;IAChF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnC,CAAC,CAAC;YACE,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACtH,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;SAC/G;QACH,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;IAEzB,OAAO;QACL,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,QAAQ,CAAC,MAAM;YAC7B,YAAY,EAAE,aAAa;YAC3B,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAChE,aAAa,EAAE,cAAc;YAC7B,eAAe,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7F,SAAS;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC;QACD,OAAO;QACP,OAAO,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,CAAC,eAAe,CAAC,EAAE;KACjF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,QAAwB,EACxB,QAAqB,EACrB,SAAiB,EACjB,OAAuB,EAAE;IAEzB,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM;QACnC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,CAAC,YAAY,CAAC;IACjB,MAAM,YAAY,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;IACxD,OAAO,WAAW,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,WAAW,CAClB,QAAwB,EACxB,OAAsB,EACtB,QAAqB,EACrB,SAAiB,EACjB,IAAoB;IAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1D,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAgC,EAAE,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/E,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC7C,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,GAAG;YACf,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,SAAS,CAAC,MAAM;YAC9B,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YACxE,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YAC9E,QAAQ;YACR,GAAG;SACJ,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACtF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,EACvD,CAAC,CACF,CAAC;IACF,MAAM,OAAO,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,eAAe,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;IAChF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnC,CAAC,CAAC;YACE,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;YACtH,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;SAC/G;QACH,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;IAEzB,OAAO;QACL,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,QAAQ,CAAC,MAAM;YAC7B,YAAY,EAAE,aAAa;YAC3B,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAChE,aAAa,EAAE,cAAc;YAC7B,eAAe,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7F,SAAS;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC;QACD,OAAO;QACP,OAAO,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,CAAC,eAAe,CAAC,EAAE;KACjF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cc session JSONL → omk ResultEntry adapter (v0.18 skill-health)。
|
|
3
|
+
*
|
|
4
|
+
* 核心职责:
|
|
5
|
+
* 1. 读 cc session JSONL 文件 / 目录
|
|
6
|
+
* 2. 按 session 拆分、按 skill 信号切段(见 docs/skill-health-spec.md §四)
|
|
7
|
+
* 3. 每段输出为一个 ResultEntry, variant key = skill 名
|
|
8
|
+
*
|
|
9
|
+
* skill 归属信号(spec §四):
|
|
10
|
+
* - 信号 1: tool_use name="Skill" 的 input.skill
|
|
11
|
+
* - 信号 2: user message 里的 <command-name>/X</command-name>
|
|
12
|
+
* - 无任何信号 → 'general'(裸 cc 对话,不丢弃)
|
|
13
|
+
*
|
|
14
|
+
* v0.18 不处理:
|
|
15
|
+
* - Skill tool 嵌套调用(扁平切段)
|
|
16
|
+
* - 信号 3 (Read .claude/skills/X/SKILL.md,留 v0.19)
|
|
17
|
+
* - 版本对齐(全部归到 skill 名下)
|
|
18
|
+
*/
|
|
19
|
+
import type { ResultEntry, ToolCallInfo, TurnInfo } from '../types.js';
|
|
20
|
+
export interface CcSession {
|
|
21
|
+
sessionId: string;
|
|
22
|
+
sourcePath: string;
|
|
23
|
+
records: unknown[];
|
|
24
|
+
cwd?: string;
|
|
25
|
+
gitBranch?: string;
|
|
26
|
+
startTimestamp?: string;
|
|
27
|
+
endTimestamp?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface SkillSegment {
|
|
30
|
+
skillName: string;
|
|
31
|
+
sessionId: string;
|
|
32
|
+
segmentIndex: number;
|
|
33
|
+
startTimestamp: string;
|
|
34
|
+
endTimestamp: string;
|
|
35
|
+
cwd?: string;
|
|
36
|
+
turns: TurnInfo[];
|
|
37
|
+
toolCalls: ToolCallInfo[];
|
|
38
|
+
metrics: {
|
|
39
|
+
durationMs: number;
|
|
40
|
+
inputTokens: number;
|
|
41
|
+
outputTokens: number;
|
|
42
|
+
cacheReadTokens: number;
|
|
43
|
+
cacheCreationTokens: number;
|
|
44
|
+
numTurns: number;
|
|
45
|
+
numToolCalls: number;
|
|
46
|
+
numToolFailures: number;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* 加载一个目录(或单个 JSONL 文件)下的所有 cc session。目录递归只一层(cc 的实际布局)。
|
|
51
|
+
*/
|
|
52
|
+
export declare function loadCcSessions(path: string): CcSession[];
|
|
53
|
+
/**
|
|
54
|
+
* 扫描 session records, 按 skill 信号把 tool calls 切成多段。
|
|
55
|
+
* 一个 session 可能产生 1-N 个 SkillSegment。
|
|
56
|
+
*/
|
|
57
|
+
export declare function segmentBySkill(session: CcSession): SkillSegment[];
|
|
58
|
+
/**
|
|
59
|
+
* SkillSegment[] → ResultEntry[] (omk 内部分析路径的标准输入)。
|
|
60
|
+
*
|
|
61
|
+
* 映射规则(详见 docs/skill-health-spec.md):
|
|
62
|
+
* - 每 segment 一个 ResultEntry
|
|
63
|
+
* - sample_id = `${sessionId}:${segmentIndex}`
|
|
64
|
+
* - variant key = skill 名(复用 omk 的 variant 维度作为 skill 分组维度)
|
|
65
|
+
*/
|
|
66
|
+
export declare function segmentsToResultEntries(segments: SkillSegment[]): ResultEntry[];
|
|
67
|
+
/**
|
|
68
|
+
* 一站式:目录 → ResultEntry[]。
|
|
69
|
+
*/
|
|
70
|
+
export declare function ccTracesToResultEntries(path: string): {
|
|
71
|
+
entries: ResultEntry[];
|
|
72
|
+
sessions: CcSession[];
|
|
73
|
+
segments: SkillSegment[];
|
|
74
|
+
};
|
|
75
|
+
//# sourceMappingURL=trace-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trace-adapter.d.ts","sourceRoot":"","sources":["../../../src/observability/trace-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAiB,MAAM,aAAa,CAAC;AA+DtF,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IAInB,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAID;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CASxD;AA8FD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,GAAG,YAAY,EAAE,CA+GjE;AAkDD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,WAAW,EAAE,CAO/E;AAgCD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAA;CAAE,CAKjI"}
|