oh-my-knowledge 0.47.0 → 0.49.0

package/dist/observability/skill-health-analyzer.js

@@ -6,25 +6,51 @@
  * 定位是"真实使用 trace 的 skill 维度观察",不是通用 APM / 生产监控。
  *
  * 分析流水线:
- *   1. ccTracesToResultEntries(path) → segments + ResultEntry[]
+ *   1. tracesToResultEntries(path) → segments + ResultEntry[]
  *   2. 时间窗 / skill 白名单过滤
  *   3. 按 skill name (variant key) 分别 computeCoverage + computeGapReport
  *   4. 聚合 overall 指标 + 健康度色带
  */
 import { buildKnowledgeIndex, computeCoverage } from '../analysis/coverage-analyzer.js';
 import { computeGapReport } from '../analysis/gap-analyzer.js';
-import { ccTracesToResultEntries, segmentsToResultEntries, } from './trace-adapter.js';
-/**
- * 从时间戳字符串比较。ISO8601 字典序即时序,直接用字符串比较。
- */
+import { segmentsToResultEntries, skillSegmentTimestampObserved, tracesToResultEntries, } from './trace-adapter.js';
+import { legacyCcSessionToTraceSession } from './trace-source.js';
+import { createTraceSessionIndex } from './trace-session-index.js';
+import { setOwnRecordValue, sumRecordCounts } from '../shared/record-count.js';
+import { checkedSumTokenCounts } from '../shared/token-usage.js';
 function timestampLt(a, b) {
-    return a < b;
+    const left = Date.parse(a);
+    const right = Date.parse(b);
+    return Number.isFinite(left) && Number.isFinite(right) ? left < right : a < b;
+}
+function sumCounts(items, select) {
+    let total = 0;
+    for (const item of items) {
+        total = sumRecordCounts(total, select(item));
+    }
+    return total;
+}
+function sumNonNegativeFinite(items, select) {
+    let total = 0;
+    for (const item of items) {
+        const value = select(item);
+        if (!Number.isFinite(value) || value < 0) {
+            throw new TypeError(`Health metric must be a non-negative finite number, got ${String(value)}`);
+        }
+        total += value;
+        if (!Number.isFinite(total)) {
+            throw new RangeError('Health metric sum exceeds the finite number range');
+        }
+    }
+    return total;
 }
 /**
  * 判断 segment 是否落在时间窗内(闭区间)。
  */
 function withinTimeWindow(seg, from, to) {
-    if (from && timestampLt(seg.startTimestamp, from))
+    if ((from || to) && !skillSegmentTimestampObserved(seg))
+        return false;
+    if (from && timestampLt(seg.endTimestamp, from))
         return false;
     if (to && timestampLt(to, seg.startTimestamp))
         return false;
@@ -44,8 +70,10 @@ export function healthBandOf(weightedGapRate) {
 /**
  * 按 per-skill 失败率判定执行稳定性。阈值见 SkillHealth.stability。
  */
-function stabilityOf(toolFailureRate) {
-    if (toolFailureRate >= 0.4)
+export function toolStabilityOf(toolFailureRate, comparableToolCalls, totalToolCalls) {
+    if (totalToolCalls > 0 && comparableToolCalls === 0)
+        return 'unknown';
+    if (toolFailureRate >= 0.4 && comparableToolCalls >= 5)
         return 'very-unstable';
     if (toolFailureRate >= 0.2)
         return 'unstable';
@@ -68,38 +96,40 @@ export function confidenceOf(segmentCount) {
  * 聚合一组 segment 的 tokens / duration / turns. 平均值按 segment 数(非 toolCall 数)算。
  */
 function aggregateUsage(skillSegs) {
-    let inputTokens = 0;
-    let outputTokens = 0;
-    let cacheReadTokens = 0;
-    let cacheCreationTokens = 0;
-    let durationMs = 0;
-    let numTurns = 0;
-    for (const s of skillSegs) {
-        inputTokens += s.metrics.inputTokens ?? 0;
-        outputTokens += s.metrics.outputTokens ?? 0;
-        cacheReadTokens += s.metrics.cacheReadTokens ?? 0;
-        cacheCreationTokens += s.metrics.cacheCreationTokens ?? 0;
-        durationMs += s.metrics.durationMs ?? 0;
-        numTurns += s.metrics.numTurns ?? 0;
-    }
-    const totalTokens = inputTokens + outputTokens + cacheReadTokens + cacheCreationTokens;
-    const n = skillSegs.length || 1;
-    return {
+    const observed = skillSegs.filter((segment) => segment.metrics.tokenUsageObserved);
+    const inputTokens = checkedSumTokenCounts(...observed.map((s) => s.metrics.inputTokens));
+    const outputTokens = checkedSumTokenCounts(...observed.map((s) => s.metrics.outputTokens));
+    const cacheReadTokens = checkedSumTokenCounts(...observed.map((s) => s.metrics.cacheReadTokens));
+    const cacheCreationTokens = checkedSumTokenCounts(...observed.map((s) => s.metrics.cacheCreationTokens));
+    const totalTokens = checkedSumTokenCounts(inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens);
+    const tokenAggregateValid = [
         inputTokens,
         outputTokens,
         cacheReadTokens,
         cacheCreationTokens,
         totalTokens,
+    ].every((value) => value !== undefined);
+    const tokenObservedSegmentCount = tokenAggregateValid ? observed.length : 0;
+    const durationMs = sumNonNegativeFinite(skillSegs, (segment) => segment.metrics.durationMs);
+    const numTurns = sumCounts(skillSegs, (segment) => segment.metrics.numTurns);
+    const n = skillSegs.length || 1;
+    const tokenDivisor = tokenObservedSegmentCount || 1;
+    return {
+        inputTokens: inputTokens ?? 0,
+        outputTokens: outputTokens ?? 0,
+        cacheReadTokens: cacheReadTokens ?? 0,
+        cacheCreationTokens: cacheCreationTokens ?? 0,
+        totalTokens: totalTokens ?? 0,
+        tokenObservedSegmentCount,
+        tokenCoverage: skillSegs.length > 0
+            ? Number((tokenObservedSegmentCount / skillSegs.length).toFixed(4))
+            : 1,
         durationMs,
         numTurns,
-        avgTokensPerSegment: Math.round(totalTokens / n),
+        avgTokensPerSegment: Math.round((totalTokens ?? 0) / tokenDivisor),
         avgDurationMsPerSegment: Math.round(durationMs / n),
     };
 }
-/**
- * 推断 KB root: 没传 --kb 时,取第一个 assistant record 的 cwd。
- * 如果跨多个 cwd,取第一个并 warn。
- */
 function inferKbRoot(sessions) {
     const cwds = new Set();
     for (const s of sessions) {
@@ -114,90 +144,40 @@ function inferKbRoot(sessions) {
     return cwds.values().next().value ?? null;
 }
 /**
- * 主入口:从 cc trace 目录生成 SkillHealthReport。
+ * 主入口：从受支持的 trace 输入生成 SkillHealthReport。
  */
 export function computeSkillHealthReport(tracePath, opts = {}) {
-    const { sessions, segments } = ccTracesToResultEntries(tracePath);
-    // 时间窗 + skill 白名单过滤
-    let filtered = segments.filter((s) => withinTimeWindow(s, opts.from, opts.to));
-    if (opts.skills?.length) {
-        const allow = new Set(opts.skills);
-        filtered = filtered.filter((s) => allow.has(s.skillName));
-    }
-    const filteredEntries = segmentsToResultEntries(filtered);
-    // 推断 KB root
-    const kbRoot = opts.kbRoot ?? inferKbRoot(sessions);
-    const index = kbRoot ? buildKnowledgeIndex(kbRoot) : null;
-    // 按 skill 分组聚合
-    const skillNames = [...new Set(filtered.map((s) => s.skillName))];
-    const bySkill = {};
-    for (const skill of skillNames) {
-        const skillSegs = filtered.filter((s) => s.skillName === skill);
-        const coverage = index ? computeCoverage(filteredEntries, skill, index, kbRoot) : null;
-        const gap = computeGapReport(filteredEntries, skill);
-        // 挂 trace 源作水印(spec §六)
-        gap.testSetPath = tracePath;
-        const skillToolCalls = skillSegs.reduce((a, s) => a + s.metrics.numToolCalls, 0);
-        const skillFailures = skillSegs.reduce((a, s) => a + s.metrics.numToolFailures, 0);
-        const toolFailureRate = skillToolCalls > 0 ? Number((skillFailures / skillToolCalls).toFixed(4)) : 0;
-        bySkill[skill] = {
-            skillName: skill,
-            segmentCount: skillSegs.length,
-            toolCallCount: skillToolCalls,
-            toolFailureCount: skillFailures,
-            toolFailureRate,
-            stability: stabilityOf(toolFailureRate),
-            confidence: confidenceOf(skillSegs.length),
-            usage: aggregateUsage(skillSegs),
-            coverage,
-            gap,
-        };
-    }
-    // Overall 聚合(加权平均,权重 = 每个 skill 的 segment 数)
-    const totalSegments = filtered.length;
-    const totalGap = Object.values(bySkill).reduce((a, h) => a + h.gap.samplesWithGap, 0);
-    const totalWeighted = Object.values(bySkill).reduce((a, h) => a + h.gap.weightedGapRate * h.gap.sampleCount, 0);
-    const gapRate = totalSegments > 0 ? Number((totalGap / totalSegments).toFixed(4)) : 0;
-    const weightedGapRate = totalSegments > 0 ? Number((totalWeighted / totalSegments).toFixed(4)) : 0;
-    // meta
-    const totalToolCalls = filtered.reduce((a, s) => a + s.metrics.numToolCalls, 0);
-    const totalFailures = filtered.reduce((a, s) => a + s.metrics.numToolFailures, 0);
-    const timeRange = filtered.length > 0
-        ? {
-            from: filtered.reduce((m, s) => (timestampLt(s.startTimestamp, m) ? s.startTimestamp : m), filtered[0].startTimestamp),
-            to: filtered.reduce((m, s) => (timestampLt(m, s.endTimestamp) ? s.endTimestamp : m), filtered[0].endTimestamp),
-        }
-        : { from: '', to: '' };
-    return {
-        kind: 'observe-health',
-        meta: {
-            tracePath,
-            kbPath: kbRoot,
-            sessionCount: sessions.length,
-            segmentCount: totalSegments,
-            messageCount: sessions.reduce((a, s) => a + s.records.length, 0),
-            toolCallCount: totalToolCalls,
-            toolFailureRate: totalToolCalls > 0 ? Number((totalFailures / totalToolCalls).toFixed(4)) : 0,
-            timeRange,
-            generatedAt: new Date().toISOString(),
-        },
-        bySkill,
-        overall: { gapRate, weightedGapRate, healthBand: healthBandOf(weightedGapRate), confidence: confidenceOf(totalSegments) },
-    };
+    const { sessions, segments, ingestion } = tracesToResultEntries(tracePath);
+    return computeSkillHealthFromSegments(segments, sessions, tracePath, opts, ingestion);
 }
 /**
  * 便利入口:直接从已准备好的 segments(跳过 loadCcSessions)算 report。
  * 用于测试 / 已手工组装过 segments 的场景。
  */
-export function computeSkillHealthFromSegments(segments, sessions, tracePath, opts = {}) {
-    const filteredSegs = segments.filter((s) => withinTimeWindow(s, opts.from, opts.to));
-    const finalSegs = opts.skills?.length
-        ? filteredSegs.filter((s) => opts.skills.includes(s.skillName))
-        : filteredSegs;
+export function computeSkillHealthFromSegments(segments, sessions, tracePath, opts = {}, ingestion) {
+    const candidateSegs = segments.filter((segment) => segment.skillName !== 'general');
+    const scopedCandidateSegs = opts.skills?.length
+        ? candidateSegs.filter((segment) => opts.skills.includes(segment.skillName))
+        : candidateSegs;
+    const excludedUntimestampedSegmentCount = opts.from || opts.to
+        ? scopedCandidateSegs.filter((segment) => !skillSegmentTimestampObserved(segment)).length
+        : 0;
+    const finalSegs = scopedCandidateSegs.filter((segment) => withinTimeWindow(segment, opts.from, opts.to));
     const finalEntries = segmentsToResultEntries(finalSegs);
-    return buildReport(finalSegs, finalEntries, sessions, tracePath, opts);
+    return buildReport(finalSegs, finalEntries, sessionsForSegments(sessions, finalSegs), tracePath, opts, ingestion, excludedUntimestampedSegmentCount);
+}
+function sessionsForSegments(sessions, segments) {
+    if (segments.length === 0)
+        return [];
+    const traceSessions = sessions.map((session) => 'events' in session ? session : legacyCcSessionToTraceSession(session));
+    const index = createTraceSessionIndex(traceSessions);
+    const selectedTraceIds = new Set(segments.flatMap((segment) => {
+        const session = index.resolve(segment);
+        return session ? [session.traceId] : [];
+    }));
+    return sessions.filter((_, position) => selectedTraceIds.has(traceSessions[position].traceId));
 }
-function buildReport(segments, entries, sessions, tracePath, opts) {
+function buildReport(segments, entries, sessions, tracePath, opts, ingestion, excludedUntimestampedSegmentCount = 0) {
     const kbRoot = opts.kbRoot ?? inferKbRoot(sessions);
     const index = kbRoot ? buildKnowledgeIndex(kbRoot) : null;
     const skillNames = [...new Set(segments.map((s) => s.skillName))];
@@ -207,33 +187,53 @@ function buildReport(segments, entries, sessions, tracePath, opts) {
         const coverage = index ? computeCoverage(entries, skill, index, kbRoot) : null;
         const gap = computeGapReport(entries, skill);
         gap.testSetPath = tracePath;
-        const skillToolCalls = skillSegs.reduce((a, s) => a + s.metrics.numToolCalls, 0);
-        const skillFailures = skillSegs.reduce((a, s) => a + s.metrics.numToolFailures, 0);
-        const toolFailureRate = skillToolCalls > 0 ? Number((skillFailures / skillToolCalls).toFixed(4)) : 0;
-        bySkill[skill] = {
+        const skillToolCalls = sumCounts(skillSegs, (segment) => segment.metrics.numToolCalls);
+        const skillFailures = sumCounts(skillSegs, (segment) => segment.metrics.numToolFailures);
+        const skillCancelled = sumCounts(skillSegs, (segment) => segment.metrics.numToolCancelled ?? 0);
+        const skillUnknown = sumCounts(skillSegs, (segment) => segment.metrics.numToolUnknown ?? 0);
+        const skillResolved = Math.max(0, skillToolCalls - skillUnknown);
+        const skillComparable = Math.max(0, skillResolved - skillCancelled);
+        const toolFailureRate = skillComparable > 0 ? Number((skillFailures / skillComparable).toFixed(4)) : 0;
+        setOwnRecordValue(bySkill, skill, {
             skillName: skill,
             segmentCount: skillSegs.length,
             toolCallCount: skillToolCalls,
             toolFailureCount: skillFailures,
+            toolCancelledCount: skillCancelled,
+            toolUnknownCount: skillUnknown,
+            toolResolvedCount: skillResolved,
+            toolOutcomeCoverage: skillToolCalls > 0
+                ? Number((skillResolved / skillToolCalls).toFixed(4))
+                : 1,
             toolFailureRate,
-            stability: stabilityOf(toolFailureRate),
+            stability: toolStabilityOf(toolFailureRate, skillComparable, skillToolCalls),
             confidence: confidenceOf(skillSegs.length),
             usage: aggregateUsage(skillSegs),
             coverage,
             gap,
-        };
+        });
     }
     const totalSegments = segments.length;
-    const totalGap = Object.values(bySkill).reduce((a, h) => a + h.gap.samplesWithGap, 0);
-    const totalWeighted = Object.values(bySkill).reduce((a, h) => a + h.gap.weightedGapRate * h.gap.sampleCount, 0);
+    const healthRows = Object.values(bySkill);
+    const totalGap = sumCounts(healthRows, (health) => health.gap.samplesWithGap);
+    const totalWeighted = sumNonNegativeFinite(healthRows, (health) => health.gap.weightedGapRate * health.gap.sampleCount);
     const gapRate = totalSegments > 0 ? Number((totalGap / totalSegments).toFixed(4)) : 0;
     const weightedGapRate = totalSegments > 0 ? Number((totalWeighted / totalSegments).toFixed(4)) : 0;
-    const totalToolCalls = segments.reduce((a, s) => a + s.metrics.numToolCalls, 0);
-    const totalFailures = segments.reduce((a, s) => a + s.metrics.numToolFailures, 0);
-    const timeRange = segments.length > 0
+    const totalToolCalls = sumCounts(segments, (segment) => segment.metrics.numToolCalls);
+    const totalFailures = sumCounts(segments, (segment) => segment.metrics.numToolFailures);
+    const totalCancelled = sumCounts(segments, (segment) => segment.metrics.numToolCancelled ?? 0);
+    const totalUnknown = sumCounts(segments, (segment) => segment.metrics.numToolUnknown ?? 0);
+    const totalResolved = Math.max(0, totalToolCalls - totalUnknown);
+    const totalComparable = Math.max(0, totalResolved - totalCancelled);
+    const timestampedSegments = segments.filter(skillSegmentTimestampObserved);
+    const timeRange = timestampedSegments.length > 0
         ? {
-            from: segments.reduce((m, s) => (timestampLt(s.startTimestamp, m) ? s.startTimestamp : m), segments[0].startTimestamp),
-            to: segments.reduce((m, s) => (timestampLt(m, s.endTimestamp) ? s.endTimestamp : m), segments[0].endTimestamp),
+            from: timestampedSegments.reduce((minimum, segment) => timestampLt(segment.startTimestamp, minimum)
+                ? segment.startTimestamp
+                : minimum, timestampedSegments[0].startTimestamp),
+            to: timestampedSegments.reduce((maximum, segment) => timestampLt(maximum, segment.endTimestamp)
+                ? segment.endTimestamp
+                : maximum, timestampedSegments[0].endTimestamp),
         }
         : { from: '', to: '' };
     return {
@@ -243,13 +243,86 @@ function buildReport(segments, entries, sessions, tracePath, opts) {
             kbPath: kbRoot,
             sessionCount: sessions.length,
             segmentCount: totalSegments,
-            messageCount: sessions.reduce((a, s) => a + s.records.length, 0),
+            messageCount: scopedSessionMessageCount(sessions, segments),
+            timestampedSegmentCount: timestampedSegments.length,
+            timestampCoverage: totalSegments > 0
+                ? Number((timestampedSegments.length / totalSegments).toFixed(4))
+                : 1,
+            excludedUntimestampedSegmentCount,
             toolCallCount: totalToolCalls,
-            toolFailureRate: totalToolCalls > 0 ? Number((totalFailures / totalToolCalls).toFixed(4)) : 0,
+            toolCancelledCount: totalCancelled,
+            toolUnknownCount: totalUnknown,
+            toolResolvedCount: totalResolved,
+            toolOutcomeCoverage: totalToolCalls > 0
+                ? Number((totalResolved / totalToolCalls).toFixed(4))
+                : 1,
+            toolFailureRate: totalComparable > 0 ? Number((totalFailures / totalComparable).toFixed(4)) : 0,
             timeRange,
             generatedAt: new Date().toISOString(),
+            ...(ingestion ? { ingestion } : {}),
         },
         bySkill,
         overall: { gapRate, weightedGapRate, healthBand: healthBandOf(weightedGapRate), confidence: confidenceOf(totalSegments) },
     };
 }
+function scopedSessionMessageCount(sessions, segments) {
+    const traceSessions = sessions.map((session) => 'events' in session ? session : legacyCcSessionToTraceSession(session));
+    const index = createTraceSessionIndex(traceSessions);
+    const rangesByTraceId = new Map();
+    const unboundedTraceIds = new Set();
+    for (const segment of segments) {
+        const session = index.resolve(segment);
+        if (!session)
+            continue;
+        if (segment.startRecordIndex === undefined
+            || segment.endRecordIndex === undefined) {
+            unboundedTraceIds.add(session.traceId);
+            continue;
+        }
+        const ranges = rangesByTraceId.get(session.traceId) ?? [];
+        ranges.push({
+            start: segment.startRecordIndex,
+            end: segment.endRecordIndex,
+        });
+        rangesByTraceId.set(session.traceId, ranges);
+    }
+    let total = 0;
+    for (const [position, session] of sessions.entries()) {
+        if (!('events' in session)) {
+            total = sumRecordCounts(total, sessionMessageCount(session));
+            continue;
+        }
+        const traceId = traceSessions[position].traceId;
+        const ranges = unboundedTraceIds.has(traceId)
+            ? undefined
+            : rangesByTraceId.get(traceId);
+        const count = !ranges
+            ? sessionMessageCount(session)
+            : session.events.filter((event) => event.eventKind === 'message'
+                && ranges.some((range) => event.sourceIndex >= range.start && event.sourceIndex <= range.end)).length;
+        total = sumRecordCounts(total, count);
+    }
+    return total;
+}
+function sessionMessageCount(session) {
+    if ('events' in session) {
+        return session.events.filter((event) => event.eventKind === 'message').length;
+    }
+    return session.records.filter((record) => {
+        if (!record || typeof record !== 'object')
+            return false;
+        const typed = record;
+        if (typed.type !== 'user' && typed.type !== 'assistant')
+            return false;
+        const content = typed.message?.content;
+        if (typeof content === 'string')
+            return content.trim().length > 0;
+        if (!Array.isArray(content))
+            return false;
+        return content.some((part) => part
+            && typeof part === 'object'
+            && part.type === 'text'
+            && typeof part.text === 'string'
+            && Boolean(part.text.trim()));
+    }).length;
+}

package/dist/observability/skill-health-report.d.ts

@@ -0,0 +1,10 @@
+import type { SkillHealthReport } from './skill-health-analyzer.js';
+/**
+ * Parse persisted observe-health data at the storage boundary.
+ *
+ * Legacy reports may omit additive fields such as usage, confidence and
+ * tool-outcome coverage. Those fields are reconstructed from authoritative
+ * counts. Contradictory counts/rates are rejected; derived labels are
+ * recomputed because their threshold formula can evolve between releases.
+ */
+export declare function parseSkillHealthReport(value: unknown): SkillHealthReport | null;