oh-my-knowledge 0.48.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -18
- package/README.zh.md +55 -23
- package/dist/analysis/coverage-analyzer.d.ts +1 -0
- package/dist/analysis/coverage-analyzer.js +125 -62
- package/dist/analysis/failure-clusterer.js +2 -1
- package/dist/analysis/gap-analyzer.d.ts +2 -2
- package/dist/analysis/gap-analyzer.js +13 -3
- package/dist/analysis/hedging-classifier.d.ts +2 -2
- package/dist/analysis/hedging-classifier.js +3 -4
- package/dist/analysis/report-diagnostics.js +9 -7
- package/dist/analysis/sample-diagnostics.js +6 -6
- package/dist/artifact-graph/doctor.js +15 -7
- package/dist/assets/agent-skills/omk/SKILL.md +27 -7
- package/dist/assets/agent-skills/omk/references/commands.md +18 -17
- package/dist/authoring/evolver.d.ts +10 -6
- package/dist/authoring/evolver.js +496 -83
- package/dist/authoring/generator.d.ts +3 -3
- package/dist/authoring/generator.js +5 -10
- package/dist/authoring/sample-fixer.d.ts +8 -6
- package/dist/authoring/sample-fixer.js +76 -5
- package/dist/cli/commands/doctor.js +31 -14
- package/dist/cli/commands/eval/index.d.ts +3 -0
- package/dist/cli/commands/eval/index.js +163 -17
- package/dist/cli/commands/evolve.d.ts +4 -4
- package/dist/cli/commands/evolve.js +27 -13
- package/dist/cli/commands/init.js +16 -3
- package/dist/cli/commands/observe/inbox.js +28 -21
- package/dist/cli/commands/observe/index.js +20 -11
- package/dist/cli/commands/observe/ingest.d.ts +3 -0
- package/dist/cli/commands/observe/ingest.js +30 -2
- package/dist/cli/commands/sample.d.ts +6 -3
- package/dist/cli/commands/sample.js +72 -68
- package/dist/cli/lib/codex-model-hint.d.ts +9 -0
- package/dist/cli/lib/codex-model-hint.js +45 -0
- package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
- package/dist/cli/lib/generation-failure-hint.js +61 -0
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +4 -0
- package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/gen.js +38 -6
- package/dist/cli/lib/i18n-dict/help.js +6 -6
- package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/init.js +13 -9
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +34 -2
- package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
- package/dist/cli/lib/llm-failure-classifier.js +8 -0
- package/dist/cli/lib/parse-run-config.d.ts +6 -5
- package/dist/cli/lib/parse-run-config.js +16 -9
- package/dist/cli/lib/runtime-defaults.d.ts +21 -0
- package/dist/cli/lib/runtime-defaults.js +79 -0
- package/dist/diagnosis/observe-mapper.js +14 -15
- package/dist/diagnosis/observe-producer.js +3 -1
- package/dist/diagnosis/studio-projection.js +14 -7
- package/dist/diagnosis/types.d.ts +2 -0
- package/dist/diagnosis/types.js +12 -0
- package/dist/doctor/endpoint-rule.js +2 -1
- package/dist/eval-core/artifact-file-names.js +18 -1
- package/dist/eval-core/artifact-index.d.ts +7 -11
- package/dist/eval-core/artifact-index.js +139 -80
- package/dist/eval-core/cache.d.ts +12 -3
- package/dist/eval-core/cache.js +89 -29
- package/dist/eval-core/comparability.js +10 -6
- package/dist/eval-core/evaluation-execution.d.ts +2 -1
- package/dist/eval-core/evaluation-execution.js +122 -37
- package/dist/eval-core/evaluation-job.d.ts +4 -1
- package/dist/eval-core/evaluation-job.js +4 -1
- package/dist/eval-core/evaluation-reporting.d.ts +15 -13
- package/dist/eval-core/evaluation-reporting.js +54 -52
- package/dist/eval-core/execution-strategy.d.ts +2 -0
- package/dist/eval-core/execution-strategy.js +11 -9
- package/dist/eval-core/fact-checker.js +15 -7
- package/dist/eval-core/holdout.js +3 -2
- package/dist/eval-core/judge-independence.d.ts +2 -2
- package/dist/eval-core/mock-hook.cjs +23 -6
- package/dist/eval-core/mocks-runtime.js +30 -8
- package/dist/eval-core/report-document.d.ts +12 -0
- package/dist/eval-core/report-document.js +1151 -0
- package/dist/eval-core/report-extensions.d.ts +4 -0
- package/dist/eval-core/report-extensions.js +500 -0
- package/dist/eval-core/report-file-migration.js +7 -2
- package/dist/eval-core/resume-compatibility.d.ts +31 -0
- package/dist/eval-core/resume-compatibility.js +141 -0
- package/dist/eval-core/sample-fingerprint.d.ts +12 -0
- package/dist/eval-core/sample-fingerprint.js +193 -0
- package/dist/eval-core/schema.js +86 -31
- package/dist/eval-core/verdict.d.ts +8 -4
- package/dist/eval-core/verdict.js +24 -10
- package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
- package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
- package/dist/eval-workflows/evaluation-pipeline.js +12 -7
- package/dist/eval-workflows/run-evaluation.d.ts +9 -7
- package/dist/eval-workflows/run-evaluation.js +79 -51
- package/dist/executors/anthropic-api.js +65 -9
- package/dist/executors/claude-cli.js +16 -79
- package/dist/executors/claude-protocol.d.ts +28 -0
- package/dist/executors/claude-protocol.js +180 -0
- package/dist/executors/claude-sdk-trace.js +56 -28
- package/dist/executors/claude-sdk.d.ts +1 -0
- package/dist/executors/claude-sdk.js +39 -93
- package/dist/executors/codex-cli-trace.js +166 -31
- package/dist/executors/codex-cli.d.ts +6 -8
- package/dist/executors/codex-cli.js +49 -151
- package/dist/executors/codex-protocol.d.ts +24 -0
- package/dist/executors/codex-protocol.js +234 -0
- package/dist/executors/codex-sdk.js +68 -120
- package/dist/executors/gemini.js +88 -13
- package/dist/executors/index.d.ts +2 -3
- package/dist/executors/index.js +5 -3
- package/dist/executors/openai-api.js +70 -9
- package/dist/executors/runtime-fingerprint.js +88 -11
- package/dist/executors/script-command.d.ts +8 -0
- package/dist/executors/script-command.js +87 -0
- package/dist/executors/script.js +202 -29
- package/dist/executors/shared.d.ts +35 -3
- package/dist/executors/shared.js +113 -15
- package/dist/grading/assertions.d.ts +1 -1
- package/dist/grading/assertions.js +19 -9
- package/dist/grading/diagnostic.d.ts +9 -2
- package/dist/grading/diagnostic.js +25 -2
- package/dist/grading/index.js +10 -4
- package/dist/grading/judge.js +19 -6
- package/dist/grading/layered-scores.d.ts +2 -3
- package/dist/grading/layered-scores.js +2 -3
- package/dist/inputs/load-samples.d.ts +1 -2
- package/dist/inputs/load-samples.js +23 -1
- package/dist/inputs/mcp-resolver.js +6 -3
- package/dist/inputs/sample-document.d.ts +11 -0
- package/dist/inputs/sample-document.js +96 -0
- package/dist/managed/evidence.d.ts +1 -0
- package/dist/managed/evidence.js +1 -1
- package/dist/managed/store.js +200 -91
- package/dist/observability/codex-trace-adapter.d.ts +5 -0
- package/dist/observability/codex-trace-adapter.js +850 -0
- package/dist/observability/experience.d.ts +32 -6
- package/dist/observability/experience.js +2695 -459
- package/dist/observability/feedback-matchers.js +16 -1
- package/dist/observability/inbox-view-model.d.ts +1 -1
- package/dist/observability/inbox-view-model.js +19 -14
- package/dist/observability/inbox.d.ts +7 -1
- package/dist/observability/inbox.js +632 -124
- package/dist/observability/problem-patterns.js +2 -0
- package/dist/observability/review-state.d.ts +6 -0
- package/dist/observability/review-state.js +235 -63
- package/dist/observability/skill-chain-advisories.js +1 -1
- package/dist/observability/skill-chain.js +17 -4
- package/dist/observability/skill-health-analyzer.d.ts +32 -7
- package/dist/observability/skill-health-analyzer.js +194 -121
- package/dist/observability/skill-health-report.d.ts +10 -0
- package/dist/observability/skill-health-report.js +620 -0
- package/dist/observability/soft-standards/constants.d.ts +0 -1
- package/dist/observability/soft-standards/constants.js +0 -1
- package/dist/observability/soft-standards/index.d.ts +1 -1
- package/dist/observability/soft-standards/index.js +1 -1
- package/dist/observability/soft-standards/llm-extractor.js +8 -10
- package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
- package/dist/observability/soft-standards/skill-standards-store.js +59 -18
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/observability/trace-adapter.d.ts +12 -7
- package/dist/observability/trace-adapter.js +11 -9
- package/dist/observability/trace-attribution.d.ts +13 -5
- package/dist/observability/trace-attribution.js +315 -21
- package/dist/observability/trace-ingestion.d.ts +9 -0
- package/dist/observability/trace-ingestion.js +80 -0
- package/dist/observability/trace-ir.d.ts +113 -0
- package/dist/observability/trace-ir.js +87 -0
- package/dist/observability/trace-segmenter.d.ts +19 -6
- package/dist/observability/trace-segmenter.js +377 -196
- package/dist/observability/trace-session-index.d.ts +19 -0
- package/dist/observability/trace-session-index.js +68 -0
- package/dist/observability/trace-source.d.ts +12 -4
- package/dist/observability/trace-source.js +939 -215
- package/dist/renderer/html-renderer.js +37 -6
- package/dist/renderer/icons.js +3 -0
- package/dist/renderer/observation-inbox-renderer.js +208 -90
- package/dist/renderer/skill-detail-renderer.js +452 -109
- package/dist/renderer/skill-health-renderer.js +69 -12
- package/dist/renderer/summary.js +28 -7
- package/dist/renderer/table.js +21 -4
- package/dist/renderer/test-view.d.ts +1 -0
- package/dist/renderer/test-view.js +44 -9
- package/dist/server/indexed-report-store.js +14 -18
- package/dist/server/job-store.js +64 -26
- package/dist/server/report-server.js +190 -78
- package/dist/server/report-store.js +57 -80
- package/dist/server/skill-index.js +143 -49
- package/dist/server/skill-insights.js +44 -5
- package/dist/shared/artifact-graph.d.ts +3 -0
- package/dist/shared/artifact-graph.js +224 -0
- package/dist/shared/assertion-types.d.ts +8 -0
- package/dist/shared/assertion-types.js +46 -0
- package/dist/shared/atomic-json.d.ts +8 -0
- package/dist/shared/atomic-json.js +33 -0
- package/dist/shared/diagnosis-schema.d.ts +9 -0
- package/dist/shared/diagnosis-schema.js +181 -0
- package/dist/shared/doctor-report.d.ts +3 -0
- package/dist/shared/doctor-report.js +103 -0
- package/dist/shared/evaluation-job.d.ts +6 -0
- package/dist/shared/evaluation-job.js +217 -0
- package/dist/shared/executor-result.d.ts +17 -0
- package/dist/shared/executor-result.js +221 -0
- package/dist/shared/file-lock.d.ts +12 -0
- package/dist/shared/file-lock.js +129 -0
- package/dist/shared/json-value.d.ts +5 -0
- package/dist/shared/json-value.js +36 -0
- package/dist/shared/keyed-mutex.d.ts +7 -0
- package/dist/shared/keyed-mutex.js +24 -0
- package/dist/shared/record-count.d.ts +8 -0
- package/dist/shared/record-count.js +43 -0
- package/dist/shared/sample-contract.d.ts +3 -0
- package/dist/shared/sample-contract.js +332 -0
- package/dist/shared/timestamp.d.ts +6 -0
- package/dist/shared/timestamp.js +64 -0
- package/dist/shared/token-usage.d.ts +19 -0
- package/dist/shared/token-usage.js +50 -0
- package/dist/shared/tool-call-status.d.ts +8 -0
- package/dist/shared/tool-call-status.js +28 -0
- package/dist/shared/tool-identity.d.ts +21 -0
- package/dist/shared/tool-identity.js +84 -0
- package/dist/shared/tool-search.js +73 -16
- package/dist/shared/trace-projection.d.ts +5 -0
- package/dist/shared/trace-projection.js +20 -0
- package/dist/shared/trace-source-kind.d.ts +3 -0
- package/dist/shared/trace-source-kind.js +12 -0
- package/dist/types/diagnosis.d.ts +2 -0
- package/dist/types/eval.d.ts +4 -0
- package/dist/types/executor.d.ts +32 -5
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/judge.d.ts +2 -0
- package/dist/types/observability.d.ts +116 -9
- package/dist/types/report.d.ts +58 -6
- package/dist/types/skill-index.d.ts +7 -0
- package/dist/types/trace.d.ts +2 -0
- package/dist/types/trace.js +1 -0
- package/package.json +9 -5
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync
|
|
2
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { OMK_HOME } from '../eval-core/default-dirs.js';
|
|
5
|
-
import { isReportFileName, reportFilePath } from '../eval-core/artifact-file-names.js';
|
|
5
|
+
import { isReportFileName, randomRunToken, reportFilePath } from '../eval-core/artifact-file-names.js';
|
|
6
6
|
import { migrateLegacyReportFiles } from '../eval-core/report-file-migration.js';
|
|
7
7
|
import { extractGapSignalsFromTrace } from '../analysis/gap-analyzer.js';
|
|
8
|
-
import {
|
|
8
|
+
import { loadTraceSessions, tracesToResultEntries, skillSegmentTimestampObserved, } from './trace-adapter.js';
|
|
9
|
+
import { normalizeTraceTimestamp } from './trace-ir.js';
|
|
9
10
|
import { isSearchToolCall, toolCallQuery } from '../shared/tool-search.js';
|
|
11
|
+
import { isToolCallFailure, isToolCallSuccess } from '../shared/tool-call-status.js';
|
|
12
|
+
import { isTraceSourceKind as isObservationSourceKind } from '../shared/trace-source-kind.js';
|
|
13
|
+
import { incrementRecordCount, ownRecordValue, setOwnRecordValue, sumRecordCounts, } from '../shared/record-count.js';
|
|
10
14
|
import { durationMsBetween } from '../shared/time.js';
|
|
11
|
-
import { buildObservationExperienceReport, normalizeObservationExperienceReport, } from './experience.js';
|
|
15
|
+
import { buildObservationExperienceReport, compactObservationExperienceReport, normalizeObservationExperienceReport, } from './experience.js';
|
|
16
|
+
import { parseDiagnosisBundle } from '../shared/diagnosis-schema.js';
|
|
17
|
+
import { parseTraceIngestionSummary } from './trace-ingestion.js';
|
|
18
|
+
import { createTraceSessionIndex, traceSessionRefIdentity } from './trace-session-index.js';
|
|
19
|
+
import { isInstalledSkillAssetPath } from './trace-attribution.js';
|
|
20
|
+
import { writeJsonFileAtomic } from '../shared/atomic-json.js';
|
|
12
21
|
// observe inbox(观测收件箱)产物根目录。导出名沿用 *_OBSERVATIONS_DIR 以少动 importer,
|
|
13
22
|
// 但落盘目录已统一到 observe-inbox 词根(命令 omk observe inbox / kind observe-inbox)。
|
|
14
23
|
// 项目级 .omk/observe-inbox 优先、全局兜底 —— 这套 project/global 归属是既有正常行为,本次只改名不改归属。
|
|
@@ -16,6 +25,7 @@ export const DEFAULT_PROJECT_OBSERVATIONS_DIR = join(process.cwd(), '.omk', 'obs
|
|
|
16
25
|
export const DEFAULT_GLOBAL_OBSERVATIONS_DIR = join(OMK_HOME, 'observe-inbox');
|
|
17
26
|
export const DEFAULT_OBSERVATIONS_DIR = DEFAULT_PROJECT_OBSERVATIONS_DIR;
|
|
18
27
|
const OBSERVATION_INBOX_SCHEMA_VERSION = 2;
|
|
28
|
+
const UNOBSERVED_TIMESTAMP = '1970-01-01T00:00:00.000Z';
|
|
19
29
|
function hashString(input) {
|
|
20
30
|
return createHash('sha256').update(input).digest('hex').slice(0, 16);
|
|
21
31
|
}
|
|
@@ -39,12 +49,16 @@ export function normalizeObservationKeyInput(value) {
|
|
|
39
49
|
.replace(/\/+/g, '/')
|
|
40
50
|
.replace(/\/$/, ''));
|
|
41
51
|
}
|
|
52
|
+
/** Legacy migration only. New reports take sourceKind from Trace IR. */
|
|
42
53
|
export function inferObservationSourceKind(sourceTrace) {
|
|
43
|
-
|
|
54
|
+
const normalized = sourceTrace.replaceAll('\\', '/').toLowerCase();
|
|
55
|
+
if (/(?:^|\/)\.?openclaw(?:\/|$)/.test(normalized))
|
|
44
56
|
return 'openclaw';
|
|
45
|
-
if (
|
|
57
|
+
if (/(?:^|\/)\.?codex\/sessions(?:\/|$)/.test(normalized))
|
|
58
|
+
return 'codex';
|
|
59
|
+
if (/(?:^|\/)\.?claude(?:\/|$)/.test(normalized))
|
|
46
60
|
return 'claude';
|
|
47
|
-
if (
|
|
61
|
+
if (normalized.endsWith('.log'))
|
|
48
62
|
return 'markdown_log';
|
|
49
63
|
return 'unknown';
|
|
50
64
|
}
|
|
@@ -75,7 +89,7 @@ function snippet(value, max = 240) {
|
|
|
75
89
|
return normalized ? normalized.slice(0, max) : undefined;
|
|
76
90
|
}
|
|
77
91
|
function isSuccessfulSearch(tc) {
|
|
78
|
-
if (!isSearchToolCall(tc) || tc
|
|
92
|
+
if (!isSearchToolCall(tc) || !isToolCallSuccess(tc))
|
|
79
93
|
return false;
|
|
80
94
|
const out = snippet(tc.output, 1000) ?? '';
|
|
81
95
|
return out !== '' && !/No matches found/i.test(out);
|
|
@@ -108,7 +122,7 @@ function failedSearchSubtype(tc, allCalls, index, skillName) {
|
|
|
108
122
|
return 'not_found';
|
|
109
123
|
if (/exceeds maximum allowed tokens|maximum allowed tokens|token limit|timed out|timeout/i.test(out))
|
|
110
124
|
return 'tool_limit';
|
|
111
|
-
if (tc
|
|
125
|
+
if (isToolCallFailure(tc))
|
|
112
126
|
return 'tool_failure';
|
|
113
127
|
const laterSuccess = allCalls.slice(index + 1).some((later) => isSuccessfulSearch(later) && isSameTopicSearch(tc, later));
|
|
114
128
|
return laterSuccess ? 'exploratory_miss' : 'hard_miss';
|
|
@@ -117,12 +131,7 @@ function isTransientPath(value) {
|
|
|
117
131
|
return /(?:^|\s)(?:\/private)?\/tmp\/|(?:^|\s)\/var\/folders\/|figma_[\w.-]+\.(?:png|jpg|jpeg|webp)\b/i.test(value);
|
|
118
132
|
}
|
|
119
133
|
function isSkillAssetPath(value, skillName) {
|
|
120
|
-
|
|
121
|
-
return false;
|
|
122
|
-
return value.includes(`/.claude/skills/${skillName}/`)
|
|
123
|
-
|| value.includes(`.claude/skills/${skillName}/`)
|
|
124
|
-
|| value.includes(`/.openclaw/workspace/skills/${skillName}/`)
|
|
125
|
-
|| value.includes(`.openclaw/workspace/skills/${skillName}/`);
|
|
134
|
+
return isInstalledSkillAssetPath(value, skillName);
|
|
126
135
|
}
|
|
127
136
|
function topicTokens(value) {
|
|
128
137
|
return new Set(normalizeObservationKeyInput(value)
|
|
@@ -205,12 +214,13 @@ function severityFor(signalType, subtype, confidence) {
|
|
|
205
214
|
}
|
|
206
215
|
function buildSessionTimeRanges(sessions) {
|
|
207
216
|
return sessions.map((session) => ({
|
|
208
|
-
sessionId: session.
|
|
209
|
-
|
|
217
|
+
sessionId: session.runId,
|
|
218
|
+
traceId: session.traceId,
|
|
219
|
+
sessionGroupId: session.rootRunId,
|
|
210
220
|
sourceTrace: session.sourcePath,
|
|
211
|
-
sourceKind: session.sourceKind
|
|
212
|
-
traceRole: session.
|
|
213
|
-
traceLabel: session.
|
|
221
|
+
sourceKind: session.sourceKind,
|
|
222
|
+
traceRole: session.role,
|
|
223
|
+
traceLabel: session.label,
|
|
214
224
|
cwd: session.cwd,
|
|
215
225
|
startTimestamp: session.startTimestamp,
|
|
216
226
|
endTimestamp: session.endTimestamp,
|
|
@@ -302,7 +312,8 @@ function itemsFromSegment(segment) {
|
|
|
302
312
|
// attributionConfidence 仅 0.3,没有 skill 改进价值,直接过滤。
|
|
303
313
|
if (segment.skillName === 'general')
|
|
304
314
|
return [];
|
|
305
|
-
const
|
|
315
|
+
const evidenceStreamId = traceSessionRefIdentity(segment);
|
|
316
|
+
const sampleId = `${evidenceStreamId}:${segment.segmentIndex}`;
|
|
306
317
|
const signals = extractGapSignalsFromTrace({
|
|
307
318
|
sampleId,
|
|
308
319
|
turns: segment.turns,
|
|
@@ -339,6 +350,7 @@ function itemsFromSegment(segment) {
|
|
|
339
350
|
outputSnippet: snippet(tc?.output),
|
|
340
351
|
messageIndex: tc?.messageIndex,
|
|
341
352
|
messageUuid: tc?.messageUuid,
|
|
353
|
+
callInstanceId: tc?.callInstanceId,
|
|
342
354
|
toolUseId: tc?.toolUseId,
|
|
343
355
|
segmentTimestamp: tc?.timestamp ?? segment.startTimestamp,
|
|
344
356
|
};
|
|
@@ -359,11 +371,12 @@ function itemsFromSegment(segment) {
|
|
|
359
371
|
const confidence = confidenceForSubtype(subtype, signal);
|
|
360
372
|
const severity = severityFor(signalType, subtype, confidence);
|
|
361
373
|
const item = {
|
|
362
|
-
id: hashString([
|
|
374
|
+
id: hashString([evidenceStreamId, segment.segmentIndex, signal.type, subtype, JSON.stringify(evidence)].join('\u0000')),
|
|
363
375
|
skillName: segment.skillName,
|
|
364
376
|
artifactVersion: 'unknown',
|
|
365
377
|
cwd: segment.cwd,
|
|
366
378
|
sessionId: segment.sessionId,
|
|
379
|
+
traceId: evidenceStreamId,
|
|
367
380
|
sourceTrace: '',
|
|
368
381
|
sourceKind: 'unknown',
|
|
369
382
|
signalType,
|
|
@@ -376,7 +389,9 @@ function itemsFromSegment(segment) {
|
|
|
376
389
|
firstSeen: segment.startTimestamp,
|
|
377
390
|
lastSeen: segment.endTimestamp,
|
|
378
391
|
occurrences: 1,
|
|
392
|
+
timestampedOccurrences: skillSegmentTimestampObserved(segment) ? 1 : 0,
|
|
379
393
|
recentSessionIds: [segment.sessionId],
|
|
394
|
+
recentTraceIds: [evidenceStreamId],
|
|
380
395
|
representativeEvidence: [evidence],
|
|
381
396
|
};
|
|
382
397
|
items.push(item);
|
|
@@ -386,7 +401,7 @@ function itemsFromSegment(segment) {
|
|
|
386
401
|
function skillSessionCountKey(segment) {
|
|
387
402
|
if (segment.traceRole && segment.traceRole !== 'standalone')
|
|
388
403
|
return segment.sessionId;
|
|
389
|
-
return segment
|
|
404
|
+
return traceSessionRefIdentity(segment);
|
|
390
405
|
}
|
|
391
406
|
/**
|
|
392
407
|
* 不再附带 diagnostics 字段 —— observability 不再反向驱动 diagnosis。
|
|
@@ -394,7 +409,8 @@ function skillSessionCountKey(segment) {
|
|
|
394
409
|
* `buildObserveDiagnosticsFromReport(report)` 写入 `report.diagnostics`。
|
|
395
410
|
*/
|
|
396
411
|
export function buildObservationInboxReport(tracePath, options = {}) {
|
|
397
|
-
const { sessions, segments } =
|
|
412
|
+
const { sessions, segments, ingestion } = tracesToResultEntries(tracePath);
|
|
413
|
+
const skillSegments = segments.filter((segment) => segment.skillName !== 'general');
|
|
398
414
|
const generatedAt = new Date().toISOString();
|
|
399
415
|
const sessionTimeRanges = buildSessionTimeRanges(sessions);
|
|
400
416
|
const sessionTimeRange = buildOverallSessionTimeRange(sessionTimeRanges);
|
|
@@ -402,41 +418,89 @@ export function buildObservationInboxReport(tracePath, options = {}) {
|
|
|
402
418
|
const skillInvocationLastSeen = {};
|
|
403
419
|
const skillToolCallCounts = {};
|
|
404
420
|
const sessionsBySkill = new Map();
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
421
|
+
let timestampedSegmentCount = 0;
|
|
422
|
+
for (const segment of skillSegments) {
|
|
423
|
+
incrementRecordCount(skillInvocationCounts, segment.skillName);
|
|
424
|
+
if (skillSegmentTimestampObserved(segment)) {
|
|
425
|
+
timestampedSegmentCount += 1;
|
|
426
|
+
const previousLastSeen = ownRecordValue(skillInvocationLastSeen, segment.skillName);
|
|
427
|
+
if (!previousLastSeen || segment.endTimestamp > previousLastSeen) {
|
|
428
|
+
setOwnRecordValue(skillInvocationLastSeen, segment.skillName, segment.endTimestamp);
|
|
429
|
+
}
|
|
411
430
|
}
|
|
412
|
-
const toolCounts = skillToolCallCounts
|
|
431
|
+
const toolCounts = ownRecordValue(skillToolCallCounts, segment.skillName) ?? {};
|
|
413
432
|
for (const toolCall of segment.toolCalls) {
|
|
414
|
-
toolCounts
|
|
433
|
+
incrementRecordCount(toolCounts, toolCall.tool);
|
|
415
434
|
}
|
|
416
|
-
skillToolCallCounts
|
|
435
|
+
setOwnRecordValue(skillToolCallCounts, segment.skillName, toolCounts);
|
|
417
436
|
const set = sessionsBySkill.get(segment.skillName) ?? new Set();
|
|
418
437
|
set.add(skillSessionCountKey(segment));
|
|
419
438
|
sessionsBySkill.set(segment.skillName, set);
|
|
420
439
|
}
|
|
421
440
|
const skillSessionCounts = Object.fromEntries(Array.from(sessionsBySkill.entries()).map(([skill, sessionIds]) => [skill, sessionIds.size]));
|
|
441
|
+
const sessionIndex = createTraceSessionIndex(sessions);
|
|
442
|
+
const messageRefsByTraceId = new Map();
|
|
443
|
+
const messageRefsForSegment = (segment) => {
|
|
444
|
+
const session = sessionIndex.resolve(segment);
|
|
445
|
+
if (!session)
|
|
446
|
+
return undefined;
|
|
447
|
+
if (messageRefsByTraceId.has(session.traceId)) {
|
|
448
|
+
return messageRefsByTraceId.get(session.traceId);
|
|
449
|
+
}
|
|
450
|
+
const messages = messageRefsFromEvents(session.events);
|
|
451
|
+
messageRefsByTraceId.set(session.traceId, messages);
|
|
452
|
+
return messages;
|
|
453
|
+
};
|
|
422
454
|
const aggregationState = createInboxAggregationState();
|
|
423
|
-
|
|
455
|
+
const occurrenceItems = [];
|
|
456
|
+
for (const segment of skillSegments) {
|
|
424
457
|
const sourceTrace = segment.sourceTrace ?? tracePath;
|
|
458
|
+
const resolvedSession = sessionIndex.resolve(segment);
|
|
459
|
+
const traceId = segment.traceId
|
|
460
|
+
?? resolvedSession?.traceId
|
|
461
|
+
?? traceSessionRefIdentity(segment);
|
|
462
|
+
const sourceKind = segment.sourceKind ?? resolvedSession?.sourceKind ?? 'unknown';
|
|
425
463
|
const segmentItems = itemsFromSegment(segment).map((item) => {
|
|
464
|
+
const evidence = {
|
|
465
|
+
...item.evidence,
|
|
466
|
+
traceId,
|
|
467
|
+
sessionId: segment.sessionId,
|
|
468
|
+
sourceTrace,
|
|
469
|
+
sourceKind,
|
|
470
|
+
};
|
|
426
471
|
const withSource = {
|
|
427
472
|
...item,
|
|
473
|
+
traceId,
|
|
428
474
|
sourceTrace,
|
|
429
|
-
sourceKind
|
|
475
|
+
sourceKind,
|
|
476
|
+
evidence,
|
|
477
|
+
recentTraceIds: [traceId],
|
|
478
|
+
representativeEvidence: [evidence],
|
|
430
479
|
};
|
|
431
480
|
return {
|
|
432
481
|
...withSource,
|
|
433
|
-
messageWindow: buildObservationMessageWindow(withSource),
|
|
482
|
+
messageWindow: buildObservationMessageWindow(withSource, 3, messageRefsForSegment(segment)),
|
|
434
483
|
};
|
|
435
484
|
});
|
|
485
|
+
occurrenceItems.push(...segmentItems);
|
|
436
486
|
addInboxItemsToState(aggregationState, segmentItems);
|
|
437
487
|
}
|
|
438
488
|
const items = finishInboxAggregation(aggregationState);
|
|
439
|
-
|
|
489
|
+
// Experience 归因必须基于每次实际发生的 signal。收件箱聚合会跨 session
|
|
490
|
+
// 合并同类项,firstSeen/lastSeen 和 sourceTrace 已不足以反推原 invocation。
|
|
491
|
+
// occurrence 仍引用聚合 item 的稳定 ID,保证 experience 引用可在 report.items
|
|
492
|
+
// 中解析,同时保留本次发生的 trace/session/timestamp 作为归因事实。
|
|
493
|
+
const experienceItems = occurrenceItems.map((item) => ({
|
|
494
|
+
...item,
|
|
495
|
+
id: aggregationState.byKey.get(keyFor(item))?.id ?? item.id,
|
|
496
|
+
}));
|
|
497
|
+
const experience = buildObservationExperienceReport({
|
|
498
|
+
sessions,
|
|
499
|
+
segments: skillSegments,
|
|
500
|
+
items: experienceItems,
|
|
501
|
+
generatedAt,
|
|
502
|
+
reviewState: options.reviewState,
|
|
503
|
+
});
|
|
440
504
|
const report = {
|
|
441
505
|
kind: 'observe-inbox',
|
|
442
506
|
schemaVersion: OBSERVATION_INBOX_SCHEMA_VERSION,
|
|
@@ -446,12 +510,17 @@ export function buildObservationInboxReport(tracePath, options = {}) {
|
|
|
446
510
|
sessionCount: sessions.length,
|
|
447
511
|
sessionTimeRange,
|
|
448
512
|
sessionTimeRanges,
|
|
449
|
-
|
|
513
|
+
ingestion,
|
|
514
|
+
segmentCount: skillSegments.length,
|
|
450
515
|
itemCount: items.length,
|
|
451
516
|
skillInvocationCounts,
|
|
452
517
|
skillSessionCounts,
|
|
453
518
|
skillInvocationLastSeen,
|
|
454
519
|
skillToolCallCounts,
|
|
520
|
+
timestampedSegmentCount,
|
|
521
|
+
timestampCoverage: skillSegments.length > 0
|
|
522
|
+
? timestampedSegmentCount / skillSegments.length
|
|
523
|
+
: 0,
|
|
455
524
|
},
|
|
456
525
|
items,
|
|
457
526
|
experience,
|
|
@@ -462,44 +531,132 @@ function createInboxAggregationState() {
|
|
|
462
531
|
return {
|
|
463
532
|
byKey: new Map(),
|
|
464
533
|
sessionLastSeenByKey: new Map(),
|
|
534
|
+
traceLastSeenByKey: new Map(),
|
|
535
|
+
primaryOccurrenceIdByKey: new Map(),
|
|
465
536
|
};
|
|
466
537
|
}
|
|
467
538
|
function addInboxItemsToState(state, items) {
|
|
468
539
|
for (const item of items) {
|
|
469
540
|
const key = keyFor(item);
|
|
470
541
|
const sessionLastSeen = state.sessionLastSeenByKey.get(key) ?? new Map();
|
|
542
|
+
const traceLastSeen = state.traceLastSeenByKey.get(key) ?? new Map();
|
|
543
|
+
const traceIdentity = item.traceId
|
|
544
|
+
?? item.evidence.traceId
|
|
545
|
+
?? `${item.sourceTrace}\u0000${item.sessionId}`;
|
|
471
546
|
const previousSessionLastSeen = sessionLastSeen.get(item.sessionId);
|
|
472
547
|
if (!previousSessionLastSeen || item.lastSeen > previousSessionLastSeen) {
|
|
473
548
|
sessionLastSeen.set(item.sessionId, item.lastSeen);
|
|
474
549
|
}
|
|
475
550
|
state.sessionLastSeenByKey.set(key, sessionLastSeen);
|
|
551
|
+
const previousTraceLastSeen = traceLastSeen.get(traceIdentity);
|
|
552
|
+
if (!previousTraceLastSeen || item.lastSeen > previousTraceLastSeen) {
|
|
553
|
+
traceLastSeen.set(traceIdentity, item.lastSeen);
|
|
554
|
+
}
|
|
555
|
+
state.traceLastSeenByKey.set(key, traceLastSeen);
|
|
476
556
|
const existing = state.byKey.get(key);
|
|
477
557
|
if (!existing) {
|
|
478
|
-
state.
|
|
558
|
+
state.primaryOccurrenceIdByKey.set(key, item.id);
|
|
559
|
+
state.byKey.set(key, {
|
|
560
|
+
...item,
|
|
561
|
+
id: hashString(`aggregate\u0000${key}`),
|
|
562
|
+
timestampedOccurrences: timestampedOccurrencesOf(item),
|
|
563
|
+
representativeEvidence: [item.evidence],
|
|
564
|
+
recentSessionIds: [item.sessionId],
|
|
565
|
+
recentTraceIds: [traceIdentity],
|
|
566
|
+
});
|
|
479
567
|
continue;
|
|
480
568
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
if (
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
569
|
+
const existingTimestampedOccurrences = timestampedOccurrencesOf(existing);
|
|
570
|
+
const itemTimestampedOccurrences = timestampedOccurrencesOf(item);
|
|
571
|
+
existing.occurrences = sumRecordCounts(existing.occurrences, item.occurrences);
|
|
572
|
+
if (itemTimestampedOccurrences > 0) {
|
|
573
|
+
if (existingTimestampedOccurrences === 0 || item.firstSeen < existing.firstSeen) {
|
|
574
|
+
existing.firstSeen = item.firstSeen;
|
|
575
|
+
}
|
|
576
|
+
if (existingTimestampedOccurrences === 0 || item.lastSeen > existing.lastSeen) {
|
|
577
|
+
existing.lastSeen = item.lastSeen;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
existing.timestampedOccurrences = sumRecordCounts(existingTimestampedOccurrences, itemTimestampedOccurrences);
|
|
581
|
+
const shouldReplaceEvidence = severityRank(item.severity) > severityRank(existing.severity)
|
|
582
|
+
|| (item.severity === existing.severity
|
|
583
|
+
&& (item.confidence > existing.confidence
|
|
584
|
+
|| (item.confidence === existing.confidence && item.lastSeen > existing.lastSeen)
|
|
585
|
+
|| (item.confidence === existing.confidence
|
|
586
|
+
&& item.lastSeen === existing.lastSeen
|
|
587
|
+
&& item.id.localeCompare(state.primaryOccurrenceIdByKey.get(key) ?? '') < 0)));
|
|
588
|
+
if (shouldReplaceEvidence) {
|
|
589
|
+
state.primaryOccurrenceIdByKey.set(key, item.id);
|
|
490
590
|
existing.severity = item.severity;
|
|
491
|
-
existing.severityReasonCode = item.severityReasonCode;
|
|
492
591
|
existing.evidence = item.evidence;
|
|
493
|
-
existing.
|
|
592
|
+
existing.sessionId = item.sessionId;
|
|
593
|
+
existing.sourceTrace = item.sourceTrace;
|
|
594
|
+
existing.sourceKind = item.sourceKind;
|
|
595
|
+
replaceOptionalProperty(existing, 'severityReasonCode', item.severityReasonCode);
|
|
596
|
+
replaceOptionalProperty(existing, 'messageWindow', item.messageWindow);
|
|
597
|
+
replaceOptionalProperty(existing, 'traceId', item.traceId);
|
|
598
|
+
replaceOptionalProperty(existing, 'cwd', item.cwd);
|
|
494
599
|
}
|
|
600
|
+
existing.confidence = Math.max(existing.confidence, item.confidence);
|
|
601
|
+
existing.attributionConfidence = Math.max(existing.attributionConfidence, item.attributionConfidence);
|
|
495
602
|
existing.recentSessionIds = Array.from(sessionLastSeen.entries())
|
|
496
|
-
.sort((a, b) => b[1].localeCompare(a[1]))
|
|
603
|
+
.sort((a, b) => b[1].localeCompare(a[1]) || a[0].localeCompare(b[0]))
|
|
497
604
|
.map(([sessionId]) => sessionId)
|
|
498
605
|
.slice(0, 3);
|
|
499
|
-
existing.
|
|
500
|
-
|
|
606
|
+
existing.recentTraceIds = Array.from(traceLastSeen.entries())
|
|
607
|
+
.sort((a, b) => b[1].localeCompare(a[1]) || a[0].localeCompare(b[0]))
|
|
608
|
+
.map(([identity]) => identity)
|
|
609
|
+
.slice(0, 3);
|
|
610
|
+
existing.representativeEvidence = stableRepresentativeEvidence(existing.evidence, [...existing.representativeEvidence, item.evidence]);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
function timestampedOccurrencesOf(item) {
|
|
614
|
+
return item.timestampedOccurrences
|
|
615
|
+
?? (item.firstSeen === UNOBSERVED_TIMESTAMP && item.lastSeen === UNOBSERVED_TIMESTAMP
|
|
616
|
+
? 0
|
|
617
|
+
: item.occurrences);
|
|
618
|
+
}
|
|
619
|
+
function replaceOptionalProperty(target, key, value) {
|
|
620
|
+
if (value === undefined) {
|
|
621
|
+
delete target[key];
|
|
622
|
+
}
|
|
623
|
+
else {
|
|
624
|
+
target[key] = value;
|
|
501
625
|
}
|
|
502
626
|
}
|
|
627
|
+
function stableRepresentativeEvidence(primary, values) {
|
|
628
|
+
const primaryKey = observationEvidenceIdentity(primary);
|
|
629
|
+
const uniqueEvidence = new Map();
|
|
630
|
+
for (const value of values) {
|
|
631
|
+
uniqueEvidence.set(observationEvidenceIdentity(value), value);
|
|
632
|
+
}
|
|
633
|
+
uniqueEvidence.set(primaryKey, primary);
|
|
634
|
+
const others = Array.from(uniqueEvidence.entries())
|
|
635
|
+
.filter(([key]) => key !== primaryKey)
|
|
636
|
+
.sort((a, b) => (b[1].segmentTimestamp ?? '').localeCompare(a[1].segmentTimestamp ?? '')
|
|
637
|
+
|| a[0].localeCompare(b[0]))
|
|
638
|
+
.map(([, value]) => value);
|
|
639
|
+
return [primary, ...others].slice(0, 50);
|
|
640
|
+
}
|
|
641
|
+
function observationEvidenceIdentity(value) {
|
|
642
|
+
return JSON.stringify([
|
|
643
|
+
value.traceId ?? '',
|
|
644
|
+
value.sessionId ?? '',
|
|
645
|
+
value.sourceTrace ?? '',
|
|
646
|
+
value.sourceKind ?? '',
|
|
647
|
+
value.tool ?? '',
|
|
648
|
+
value.query ?? '',
|
|
649
|
+
value.path ?? '',
|
|
650
|
+
value.outputSnippet ?? '',
|
|
651
|
+
value.assistantSnippet ?? '',
|
|
652
|
+
value.markerToken ?? '',
|
|
653
|
+
value.messageIndex ?? null,
|
|
654
|
+
value.messageUuid ?? '',
|
|
655
|
+
value.callInstanceId ?? '',
|
|
656
|
+
value.toolUseId ?? '',
|
|
657
|
+
value.segmentTimestamp ?? '',
|
|
658
|
+
]);
|
|
659
|
+
}
|
|
503
660
|
function finishInboxAggregation(state) {
|
|
504
661
|
return Array.from(state.byKey.values()).sort(compareInboxItems);
|
|
505
662
|
}
|
|
@@ -524,21 +681,37 @@ function compareInboxItems(a, b) {
|
|
|
524
681
|
const confidence = b.confidence - a.confidence;
|
|
525
682
|
if (confidence !== 0)
|
|
526
683
|
return confidence;
|
|
684
|
+
const timestampEvidence = Number(timestampedOccurrencesOf(b) > 0) - Number(timestampedOccurrencesOf(a) > 0);
|
|
685
|
+
if (timestampEvidence !== 0)
|
|
686
|
+
return timestampEvidence;
|
|
527
687
|
const lastSeen = b.lastSeen.localeCompare(a.lastSeen);
|
|
528
688
|
if (lastSeen !== 0)
|
|
529
689
|
return lastSeen;
|
|
530
690
|
return b.occurrences - a.occurrences;
|
|
531
691
|
}
|
|
532
692
|
export function saveObservationInboxReport(report, outDir = DEFAULT_OBSERVATIONS_DIR) {
|
|
693
|
+
const compact = compactObservationInboxReport(report);
|
|
694
|
+
if (!normalizeObservationInboxReport(compact)) {
|
|
695
|
+
throw new Error('拒绝写入无法回读的 observe inbox 报告。');
|
|
696
|
+
}
|
|
533
697
|
mkdirSync(outDir, { recursive: true });
|
|
534
698
|
migrateLegacyReportFiles(outDir, 'observe-inbox');
|
|
535
|
-
//
|
|
699
|
+
// 保留毫秒并追加随机段;即使同一毫秒生成两份 report,也不能静默互相覆盖。
|
|
536
700
|
// 例: '2026-05-07T12:00:00.999Z' → '2026-05-07T12-00-00-999'
|
|
537
701
|
const stamp = report.meta.generatedAt.replace(/[:.]/g, '-').replace(/Z$/, '');
|
|
538
|
-
const path = reportFilePath(outDir, stamp);
|
|
539
|
-
|
|
702
|
+
const path = reportFilePath(outDir, `${stamp}-${randomRunToken()}`);
|
|
703
|
+
writeJsonFileAtomic(path, compact);
|
|
540
704
|
return path;
|
|
541
705
|
}
|
|
706
|
+
export function compactObservationInboxReport(report) {
|
|
707
|
+
const { experience, ...persisted } = report;
|
|
708
|
+
return experience
|
|
709
|
+
? {
|
|
710
|
+
...persisted,
|
|
711
|
+
experience: compactObservationExperienceReport(experience),
|
|
712
|
+
}
|
|
713
|
+
: persisted;
|
|
714
|
+
}
|
|
542
715
|
export function loadObservationInboxReports(dir = DEFAULT_OBSERVATIONS_DIR) {
|
|
543
716
|
if (!existsSync(dir)) {
|
|
544
717
|
if (dir === DEFAULT_PROJECT_OBSERVATIONS_DIR && existsSync(DEFAULT_GLOBAL_OBSERVATIONS_DIR)) {
|
|
@@ -555,12 +728,9 @@ export function loadObservationInboxReports(dir = DEFAULT_OBSERVATIONS_DIR) {
|
|
|
555
728
|
if (!report)
|
|
556
729
|
return null;
|
|
557
730
|
report.items = report.items.map((item) => {
|
|
558
|
-
const sourceKind = item.sourceKind;
|
|
559
731
|
return {
|
|
560
732
|
...item,
|
|
561
|
-
sourceKind: sourceKind
|
|
562
|
-
? 'openclaw'
|
|
563
|
-
: (item.sourceKind ?? inferObservationSourceKind(item.sourceTrace)),
|
|
733
|
+
sourceKind: item.sourceKind ?? inferObservationSourceKind(item.sourceTrace),
|
|
564
734
|
severityReasonCode: item.severityReasonCode ?? severityReasonCodeFor(item),
|
|
565
735
|
severityReason: undefined,
|
|
566
736
|
};
|
|
@@ -588,20 +758,358 @@ function normalizeObservationInboxReport(value) {
|
|
|
588
758
|
return null;
|
|
589
759
|
if (report.schemaVersion !== OBSERVATION_INBOX_SCHEMA_VERSION)
|
|
590
760
|
return null;
|
|
591
|
-
if (!report.meta ||
|
|
761
|
+
if (!isObservationInboxMeta(report.meta) || !Array.isArray(report.items))
|
|
592
762
|
return null;
|
|
763
|
+
if (!report.items.every(isObservationInboxItem))
|
|
764
|
+
return null;
|
|
765
|
+
if (report.meta.itemCount !== report.items.length)
|
|
766
|
+
return null;
|
|
767
|
+
const items = report.items.map((item) => ({
|
|
768
|
+
...item,
|
|
769
|
+
timestampedOccurrences: timestampedOccurrencesOf(item),
|
|
770
|
+
}));
|
|
593
771
|
const experience = report.experience === undefined
|
|
594
772
|
? undefined
|
|
595
|
-
: normalizeObservationExperienceReport(report.experience)
|
|
773
|
+
: normalizeObservationExperienceReport(report.experience);
|
|
774
|
+
if (report.experience !== undefined && !experience)
|
|
775
|
+
return null;
|
|
776
|
+
const diagnostics = report.diagnostics === undefined
|
|
777
|
+
? undefined
|
|
778
|
+
: parseDiagnosisBundle(report.diagnostics);
|
|
779
|
+
if (report.diagnostics !== undefined && !diagnostics)
|
|
780
|
+
return null;
|
|
781
|
+
if (!observationInboxReferencesAreConsistent(report.meta, items, experience ?? undefined))
|
|
782
|
+
return null;
|
|
596
783
|
return {
|
|
597
784
|
kind: 'observe-inbox',
|
|
598
785
|
schemaVersion: OBSERVATION_INBOX_SCHEMA_VERSION,
|
|
599
786
|
meta: report.meta,
|
|
600
|
-
items
|
|
787
|
+
items,
|
|
601
788
|
...(experience ? { experience } : {}),
|
|
602
|
-
...(
|
|
789
|
+
...(diagnostics ? { diagnostics } : {}),
|
|
603
790
|
};
|
|
604
791
|
}
|
|
792
|
+
function observationInboxReferencesAreConsistent(meta, items, experience) {
|
|
793
|
+
const itemIds = new Set(items.map((item) => item.id));
|
|
794
|
+
if (itemIds.size !== items.length)
|
|
795
|
+
return false;
|
|
796
|
+
if (items.some((item) => new Set(item.recentSessionIds).size !== item.recentSessionIds.length
|
|
797
|
+
|| (item.recentTraceIds !== undefined
|
|
798
|
+
&& new Set(item.recentTraceIds).size !== item.recentTraceIds.length)
|
|
799
|
+
|| item.occurrences < item.recentSessionIds.length
|
|
800
|
+
|| item.occurrences < (item.recentTraceIds?.length ?? 0)
|
|
801
|
+
|| timestampedOccurrencesOf(item) > item.occurrences
|
|
802
|
+
|| (item.traceId !== undefined
|
|
803
|
+
&& (item.evidence.traceId !== item.traceId
|
|
804
|
+
|| !item.recentTraceIds?.includes(item.traceId)))
|
|
805
|
+
|| (item.evidence.sourceTrace !== undefined
|
|
806
|
+
&& item.evidence.sourceTrace !== item.sourceTrace)
|
|
807
|
+
|| (item.evidence.sessionId !== undefined
|
|
808
|
+
&& item.evidence.sessionId !== item.sessionId)
|
|
809
|
+
|| (item.evidence.sourceKind !== undefined
|
|
810
|
+
&& item.evidence.sourceKind !== item.sourceKind)
|
|
811
|
+
|| item.representativeEvidence.length === 0
|
|
812
|
+
|| observationEvidenceIdentity(item.representativeEvidence[0])
|
|
813
|
+
!== observationEvidenceIdentity(item.evidence)
|
|
814
|
+
|| new Set(item.representativeEvidence.map(observationEvidenceIdentity)).size
|
|
815
|
+
!== item.representativeEvidence.length))
|
|
816
|
+
return false;
|
|
817
|
+
if (meta.sessionCount !== undefined
|
|
818
|
+
&& meta.sessionTimeRanges !== undefined
|
|
819
|
+
&& meta.sessionCount !== meta.sessionTimeRanges.length)
|
|
820
|
+
return false;
|
|
821
|
+
if (meta.sessionTimeRanges !== undefined
|
|
822
|
+
&& (new Set(meta.sessionTimeRanges.map((range) => range.traceId ?? `${range.sourceTrace}\u0000${range.sessionId}`)).size
|
|
823
|
+
!== meta.sessionTimeRanges.length
|
|
824
|
+
|| !overallSessionTimeRangeMatches(meta.sessionTimeRange, meta.sessionTimeRanges)))
|
|
825
|
+
return false;
|
|
826
|
+
if (!experience)
|
|
827
|
+
return true;
|
|
828
|
+
if (meta.generatedAt !== experience.generatedAt
|
|
829
|
+
|| meta.segmentCount !== experience.invocations.length
|
|
830
|
+
||
|
|
831
|
+
experience.invocations.some((invocation) => invocation.relatedObservationIds.some((id) => !itemIds.has(id))))
|
|
832
|
+
return false;
|
|
833
|
+
const invocationCounts = Object.fromEntries(experience.skills.map((skill) => [skill.skillName, skill.invocationCount]));
|
|
834
|
+
const sessionCounts = Object.fromEntries(experience.skills.map((skill) => [skill.skillName, skill.sessionCount]));
|
|
835
|
+
const lastSeen = Object.fromEntries(experience.skills
|
|
836
|
+
.filter((skill) => (skill.timestampedInvocationCount
|
|
837
|
+
?? (skill.firstSeen === UNOBSERVED_TIMESTAMP ? 0 : skill.invocationCount)) > 0)
|
|
838
|
+
.map((skill) => [skill.skillName, skill.lastSeen]));
|
|
839
|
+
const toolCounts = Object.fromEntries(experience.skills.map((skill) => [skill.skillName, skill.toolCounts]));
|
|
840
|
+
return (meta.skillInvocationCounts === undefined
|
|
841
|
+
|| inboxRecordsEqual(meta.skillInvocationCounts, invocationCounts)) && (meta.skillSessionCounts === undefined
|
|
842
|
+
|| inboxRecordsEqual(meta.skillSessionCounts, sessionCounts)) && (meta.skillInvocationLastSeen === undefined
|
|
843
|
+
|| inboxRecordsEqual(meta.skillInvocationLastSeen, lastSeen)) && (meta.skillToolCallCounts === undefined
|
|
844
|
+
|| (inboxRecordKeysEqual(meta.skillToolCallCounts, toolCounts)
|
|
845
|
+
&& Object.keys(toolCounts).every((skillName) => inboxRecordsEqual(meta.skillToolCallCounts?.[skillName] ?? {}, toolCounts[skillName]))));
|
|
846
|
+
}
|
|
847
|
+
function overallSessionTimeRangeMatches(actual, ranges) {
|
|
848
|
+
if (!actual)
|
|
849
|
+
return true;
|
|
850
|
+
const expected = buildOverallSessionTimeRange(ranges);
|
|
851
|
+
if (!expected)
|
|
852
|
+
return false;
|
|
853
|
+
return actual.from === expected.from
|
|
854
|
+
&& actual.to === expected.to
|
|
855
|
+
&& actual.durationMs === expected.durationMs;
|
|
856
|
+
}
|
|
857
|
+
function inboxRecordsEqual(left, right) {
|
|
858
|
+
return inboxRecordKeysEqual(left, right)
|
|
859
|
+
&& Object.keys(left).every((key) => left[key] === right[key]);
|
|
860
|
+
}
|
|
861
|
+
function inboxRecordKeysEqual(left, right) {
|
|
862
|
+
const leftKeys = Object.keys(left).sort();
|
|
863
|
+
const rightKeys = Object.keys(right).sort();
|
|
864
|
+
return leftKeys.length === rightKeys.length
|
|
865
|
+
&& leftKeys.every((key, index) => key === rightKeys[index]);
|
|
866
|
+
}
|
|
867
|
+
function isObservationInboxMeta(value) {
|
|
868
|
+
if (!isInboxRecord(value)
|
|
869
|
+
|| typeof value.tracePath !== 'string'
|
|
870
|
+
|| !isInboxTimestamp(value.generatedAt)
|
|
871
|
+
|| (value.sessionCount !== undefined && !isInboxCount(value.sessionCount))
|
|
872
|
+
|| !isInboxCount(value.segmentCount)
|
|
873
|
+
|| !isInboxCount(value.itemCount)
|
|
874
|
+
|| (value.ingestion !== undefined
|
|
875
|
+
&& parseTraceIngestionSummary(value.ingestion) === null)
|
|
876
|
+
|| (value.skillInvocationCounts !== undefined && !isInboxCountRecord(value.skillInvocationCounts))
|
|
877
|
+
|| (value.skillSessionCounts !== undefined && !isInboxCountRecord(value.skillSessionCounts))
|
|
878
|
+
|| (value.timestampedSegmentCount !== undefined
|
|
879
|
+
&& (!isInboxCount(value.timestampedSegmentCount)
|
|
880
|
+
|| value.timestampedSegmentCount > value.segmentCount))
|
|
881
|
+
|| (value.timestampCoverage !== undefined
|
|
882
|
+
&& (!isInboxRate(value.timestampCoverage)
|
|
883
|
+
|| value.timestampedSegmentCount === undefined
|
|
884
|
+
|| !inboxRatesEqual(value.timestampCoverage, (value.segmentCount > 0
|
|
885
|
+
? value.timestampedSegmentCount / value.segmentCount
|
|
886
|
+
: 0))))
|
|
887
|
+
|| (value.skillInvocationLastSeen !== undefined
|
|
888
|
+
&& !isInboxTimestampRecord(value.skillInvocationLastSeen))
|
|
889
|
+
|| (value.skillToolCallCounts !== undefined
|
|
890
|
+
&& (!isInboxRecord(value.skillToolCallCounts)
|
|
891
|
+
|| !Object.values(value.skillToolCallCounts).every(isInboxCountRecord))))
|
|
892
|
+
return false;
|
|
893
|
+
if (value.sessionTimeRange !== undefined
|
|
894
|
+
&& (!isInboxRecord(value.sessionTimeRange)
|
|
895
|
+
|| !isInboxTimestampRangeOrEmpty(value.sessionTimeRange.from, value.sessionTimeRange.to)
|
|
896
|
+
|| !isInboxDurationConsistent(value.sessionTimeRange.from, value.sessionTimeRange.to, value.sessionTimeRange.durationMs)))
|
|
897
|
+
return false;
|
|
898
|
+
if (value.skillInvocationCounts !== undefined) {
|
|
899
|
+
const invocationCounts = value.skillInvocationCounts;
|
|
900
|
+
const invocationTotal = safeInboxCountSum(Object.values(invocationCounts));
|
|
901
|
+
if (invocationTotal !== value.segmentCount)
|
|
902
|
+
return false;
|
|
903
|
+
if (value.skillSessionCounts !== undefined
|
|
904
|
+
&& Object.entries(value.skillSessionCounts).some(([skillName, count]) => count > (invocationCounts[skillName] ?? 0)))
|
|
905
|
+
return false;
|
|
906
|
+
if (value.skillInvocationLastSeen !== undefined
|
|
907
|
+
&& Object.keys(value.skillInvocationLastSeen).some((skillName) => !Object.hasOwn(invocationCounts, skillName)))
|
|
908
|
+
return false;
|
|
909
|
+
if (value.skillToolCallCounts !== undefined
|
|
910
|
+
&& Object.keys(value.skillToolCallCounts).some((skillName) => !Object.hasOwn(invocationCounts, skillName)))
|
|
911
|
+
return false;
|
|
912
|
+
}
|
|
913
|
+
return value.sessionTimeRanges === undefined
|
|
914
|
+
|| (Array.isArray(value.sessionTimeRanges)
|
|
915
|
+
&& value.sessionTimeRanges.every(isObservationSessionTimeRange));
|
|
916
|
+
}
|
|
917
|
+
function isObservationSessionTimeRange(value) {
|
|
918
|
+
return isInboxRecord(value)
|
|
919
|
+
&& typeof value.sessionId === 'string'
|
|
920
|
+
&& (value.traceId === undefined || typeof value.traceId === 'string')
|
|
921
|
+
&& (value.sessionGroupId === undefined || typeof value.sessionGroupId === 'string')
|
|
922
|
+
&& typeof value.sourceTrace === 'string'
|
|
923
|
+
&& isObservationSourceKind(value.sourceKind)
|
|
924
|
+
&& (value.traceRole === undefined
|
|
925
|
+
|| value.traceRole === 'standalone'
|
|
926
|
+
|| value.traceRole === 'main'
|
|
927
|
+
|| value.traceRole === 'subagent')
|
|
928
|
+
&& (value.traceLabel === undefined || typeof value.traceLabel === 'string')
|
|
929
|
+
&& (value.cwd === undefined || typeof value.cwd === 'string')
|
|
930
|
+
&& isInboxOptionalTimestamp(value.startTimestamp)
|
|
931
|
+
&& isInboxOptionalTimestamp(value.endTimestamp)
|
|
932
|
+
&& isInboxDurationConsistent(value.startTimestamp, value.endTimestamp, value.durationMs);
|
|
933
|
+
}
|
|
934
|
+
function isObservationInboxItem(value) {
|
|
935
|
+
if (!isInboxRecord(value)
|
|
936
|
+
|| typeof value.id !== 'string'
|
|
937
|
+
|| typeof value.skillName !== 'string'
|
|
938
|
+
|| typeof value.artifactVersion !== 'string'
|
|
939
|
+
|| (value.artifactHash !== undefined && typeof value.artifactHash !== 'string')
|
|
940
|
+
|| (value.cwd !== undefined && typeof value.cwd !== 'string')
|
|
941
|
+
|| typeof value.sessionId !== 'string'
|
|
942
|
+
|| (value.traceId !== undefined && typeof value.traceId !== 'string')
|
|
943
|
+
|| typeof value.sourceTrace !== 'string'
|
|
944
|
+
|| (value.sourceKind !== undefined
|
|
945
|
+
&& !isObservationSourceKind(value.sourceKind))
|
|
946
|
+
|| !isObservationSignalType(value.signalType)
|
|
947
|
+
|| !isObservationSignalSubtype(value.signalSubtype)
|
|
948
|
+
|| !isInboxRate(value.confidence)
|
|
949
|
+
|| !isInboxRate(value.attributionConfidence)
|
|
950
|
+
|| !isObservationSeverity(value.severity)
|
|
951
|
+
|| (value.severityReasonCode !== undefined
|
|
952
|
+
&& !isObservationSeverityReasonCode(value.severityReasonCode))
|
|
953
|
+
|| (value.severityReason !== undefined && typeof value.severityReason !== 'string')
|
|
954
|
+
|| !isObservationEvidence(value.evidence)
|
|
955
|
+
|| !isInboxTimestampRange(value.firstSeen, value.lastSeen)
|
|
956
|
+
|| !isInboxCount(value.occurrences)
|
|
957
|
+
|| value.occurrences === 0
|
|
958
|
+
|| (value.timestampedOccurrences !== undefined
|
|
959
|
+
&& (!isInboxCount(value.timestampedOccurrences)
|
|
960
|
+
|| value.timestampedOccurrences > value.occurrences))
|
|
961
|
+
|| !isInboxStringArray(value.recentSessionIds)
|
|
962
|
+
|| (value.recentTraceIds !== undefined
|
|
963
|
+
&& !isInboxStringArray(value.recentTraceIds))
|
|
964
|
+
|| !Array.isArray(value.representativeEvidence)
|
|
965
|
+
|| !value.representativeEvidence.every(isObservationEvidence))
|
|
966
|
+
return false;
|
|
967
|
+
const timestampedOccurrences = value.timestampedOccurrences === undefined
|
|
968
|
+
? (value.firstSeen === UNOBSERVED_TIMESTAMP && value.lastSeen === UNOBSERVED_TIMESTAMP
|
|
969
|
+
? 0
|
|
970
|
+
: value.occurrences)
|
|
971
|
+
: value.timestampedOccurrences;
|
|
972
|
+
if (timestampedOccurrences === 0
|
|
973
|
+
? value.firstSeen !== UNOBSERVED_TIMESTAMP || value.lastSeen !== UNOBSERVED_TIMESTAMP
|
|
974
|
+
: value.firstSeen === UNOBSERVED_TIMESTAMP || value.lastSeen === UNOBSERVED_TIMESTAMP)
|
|
975
|
+
return false;
|
|
976
|
+
return value.messageWindow === undefined || isObservationMessageWindow(value.messageWindow);
|
|
977
|
+
}
|
|
978
|
+
function isObservationEvidence(value) {
|
|
979
|
+
if (!isInboxRecord(value))
|
|
980
|
+
return false;
|
|
981
|
+
const strings = [
|
|
982
|
+
value.traceId,
|
|
983
|
+
value.sessionId,
|
|
984
|
+
value.sourceTrace,
|
|
985
|
+
value.tool,
|
|
986
|
+
value.query,
|
|
987
|
+
value.path,
|
|
988
|
+
value.outputSnippet,
|
|
989
|
+
value.assistantSnippet,
|
|
990
|
+
value.markerToken,
|
|
991
|
+
value.messageUuid,
|
|
992
|
+
value.callInstanceId,
|
|
993
|
+
value.toolUseId,
|
|
994
|
+
value.segmentTimestamp,
|
|
995
|
+
];
|
|
996
|
+
return strings.slice(0, -1).every((field) => field === undefined || typeof field === 'string')
|
|
997
|
+
&& (value.sourceKind === undefined || isObservationSourceKind(value.sourceKind))
|
|
998
|
+
&& isInboxOptionalTimestamp(value.segmentTimestamp)
|
|
999
|
+
&& (value.messageIndex === undefined
|
|
1000
|
+
|| isInboxCount(value.messageIndex));
|
|
1001
|
+
}
|
|
1002
|
+
function isObservationMessageWindow(value) {
|
|
1003
|
+
return isInboxRecord(value)
|
|
1004
|
+
&& Array.isArray(value.before)
|
|
1005
|
+
&& value.before.every(isObservationMessageRef)
|
|
1006
|
+
&& Array.isArray(value.event)
|
|
1007
|
+
&& value.event.every(isObservationMessageRef)
|
|
1008
|
+
&& Array.isArray(value.after)
|
|
1009
|
+
&& value.after.every(isObservationMessageRef)
|
|
1010
|
+
&& (value.resolutionAfter === 'resolved'
|
|
1011
|
+
|| value.resolutionAfter === 'unresolved'
|
|
1012
|
+
|| value.resolutionAfter === 'unknown');
|
|
1013
|
+
}
|
|
1014
|
+
function isObservationMessageRef(value) {
|
|
1015
|
+
return isInboxRecord(value)
|
|
1016
|
+
&& (value.role === 'user'
|
|
1017
|
+
|| value.role === 'assistant'
|
|
1018
|
+
|| value.role === 'other')
|
|
1019
|
+
&& typeof value.snippet === 'string'
|
|
1020
|
+
&& isInboxCount(value.messageIndex)
|
|
1021
|
+
&& (value.uuid === undefined || typeof value.uuid === 'string')
|
|
1022
|
+
&& isInboxOptionalTimestamp(value.timestamp);
|
|
1023
|
+
}
|
|
1024
|
+
function isObservationSignalType(value) {
|
|
1025
|
+
return value === 'failed_search'
|
|
1026
|
+
|| value === 'repeated_failure'
|
|
1027
|
+
|| value === 'hedging'
|
|
1028
|
+
|| value === 'explicit_marker';
|
|
1029
|
+
}
|
|
1030
|
+
function isObservationSignalSubtype(value) {
|
|
1031
|
+
return value === 'hard_miss'
|
|
1032
|
+
|| value === 'repeated_failure'
|
|
1033
|
+
|| value === 'exploratory_miss'
|
|
1034
|
+
|| value === 'tool_error'
|
|
1035
|
+
|| value === 'permission_error'
|
|
1036
|
+
|| value === 'bash_probe'
|
|
1037
|
+
|| value === 'not_found'
|
|
1038
|
+
|| value === 'transient_file_missing'
|
|
1039
|
+
|| value === 'skill_asset_read_failed'
|
|
1040
|
+
|| value === 'permission_denied'
|
|
1041
|
+
|| value === 'tool_limit'
|
|
1042
|
+
|| value === 'tool_failure'
|
|
1043
|
+
|| value === 'regex_only'
|
|
1044
|
+
|| value === 'llm_classified'
|
|
1045
|
+
|| value === 'marker';
|
|
1046
|
+
}
|
|
1047
|
+
function isObservationSeverity(value) {
|
|
1048
|
+
return value === 'high' || value === 'medium' || value === 'low' || value === 'noise';
|
|
1049
|
+
}
|
|
1050
|
+
function isObservationSeverityReasonCode(value) {
|
|
1051
|
+
return value === 'knowledge_gap_suspected'
|
|
1052
|
+
|| value === 'repeated_failure_suspected'
|
|
1053
|
+
|| value === 'explicit_gap_marker'
|
|
1054
|
+
|| value === 'exploratory_probe'
|
|
1055
|
+
|| value === 'skill_asset_unavailable'
|
|
1056
|
+
|| value === 'soft_hedging_signal'
|
|
1057
|
+
|| value === 'tool_or_runtime_noise';
|
|
1058
|
+
}
|
|
1059
|
+
function isInboxRecord(value) {
|
|
1060
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
1061
|
+
}
|
|
1062
|
+
function isInboxCount(value) {
|
|
1063
|
+
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
1064
|
+
}
|
|
1065
|
+
function isInboxRate(value) {
|
|
1066
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
1067
|
+
}
|
|
1068
|
+
function inboxRatesEqual(left, right) {
|
|
1069
|
+
return Math.abs(left - right) <= 1e-6;
|
|
1070
|
+
}
|
|
1071
|
+
function safeInboxCountSum(values) {
|
|
1072
|
+
try {
|
|
1073
|
+
return sumRecordCounts(...values);
|
|
1074
|
+
}
|
|
1075
|
+
catch {
|
|
1076
|
+
return undefined;
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
function isInboxTimestamp(value) {
|
|
1080
|
+
return typeof value === 'string' && normalizeTraceTimestamp(value) !== undefined;
|
|
1081
|
+
}
|
|
1082
|
+
function isInboxOptionalTimestamp(value) {
|
|
1083
|
+
return value === undefined || isInboxTimestamp(value);
|
|
1084
|
+
}
|
|
1085
|
+
function isInboxTimestampRange(start, end) {
|
|
1086
|
+
return isInboxTimestamp(start)
|
|
1087
|
+
&& isInboxTimestamp(end)
|
|
1088
|
+
&& Date.parse(start) <= Date.parse(end);
|
|
1089
|
+
}
|
|
1090
|
+
function isInboxTimestampRangeOrEmpty(start, end) {
|
|
1091
|
+
return start === '' && end === '' || isInboxTimestampRange(start, end);
|
|
1092
|
+
}
|
|
1093
|
+
function isInboxDurationConsistent(start, end, duration) {
|
|
1094
|
+
if (duration !== undefined && !isInboxCount(duration))
|
|
1095
|
+
return false;
|
|
1096
|
+
if (start === undefined || end === undefined || start === '' || end === '') {
|
|
1097
|
+
return duration === undefined;
|
|
1098
|
+
}
|
|
1099
|
+
return typeof start === 'string'
|
|
1100
|
+
&& typeof end === 'string'
|
|
1101
|
+
&& isInboxTimestampRange(start, end)
|
|
1102
|
+
&& duration === durationMsBetween(start, end);
|
|
1103
|
+
}
|
|
1104
|
+
function isInboxStringArray(value) {
|
|
1105
|
+
return Array.isArray(value) && value.every((item) => typeof item === 'string');
|
|
1106
|
+
}
|
|
1107
|
+
function isInboxCountRecord(value) {
|
|
1108
|
+
return isInboxRecord(value) && Object.values(value).every(isInboxCount);
|
|
1109
|
+
}
|
|
1110
|
+
function isInboxTimestampRecord(value) {
|
|
1111
|
+
return isInboxRecord(value) && Object.values(value).every(isInboxTimestamp);
|
|
1112
|
+
}
|
|
605
1113
|
export function loadLatestObservationInboxReport(dir = DEFAULT_OBSERVATIONS_DIR) {
|
|
606
1114
|
const reports = loadObservationInboxReports(dir);
|
|
607
1115
|
if (reports.length === 0)
|
|
@@ -619,13 +1127,13 @@ export function queryObservationInbox(dir = DEFAULT_OBSERVATIONS_DIR) {
|
|
|
619
1127
|
export function summarizeObservationInboxBySkill(items, reports = []) {
|
|
620
1128
|
const invocationCounts = reports.reduce((acc, report) => {
|
|
621
1129
|
for (const [skill, count] of Object.entries(report.meta.skillInvocationCounts ?? {})) {
|
|
622
|
-
acc
|
|
1130
|
+
incrementRecordCount(acc, skill, count);
|
|
623
1131
|
}
|
|
624
1132
|
return acc;
|
|
625
1133
|
}, {});
|
|
626
1134
|
const sessionCounts = reports.reduce((acc, report) => {
|
|
627
1135
|
for (const [skill, count] of Object.entries(report.meta.skillSessionCounts ?? {})) {
|
|
628
|
-
acc
|
|
1136
|
+
incrementRecordCount(acc, skill, count);
|
|
629
1137
|
}
|
|
630
1138
|
return acc;
|
|
631
1139
|
}, {});
|
|
@@ -640,14 +1148,17 @@ export function summarizeObservationInboxBySkill(items, reports = []) {
|
|
|
640
1148
|
const group = bySkill.get(skillName) ?? [];
|
|
641
1149
|
return {
|
|
642
1150
|
skillName,
|
|
643
|
-
invocationCount: invocationCounts[skillName]
|
|
1151
|
+
invocationCount: invocationCounts[skillName]
|
|
1152
|
+
?? sumRecordCounts(...group.map((item) => item.occurrences)),
|
|
644
1153
|
sessionCount: sessionCounts[skillName] ?? new Set(group.flatMap((item) => item.recentSessionIds)).size,
|
|
645
1154
|
observationCount: group.length,
|
|
646
1155
|
highCount: group.filter((item) => item.severity === 'high').length,
|
|
647
1156
|
mediumCount: group.filter((item) => item.severity === 'medium').length,
|
|
648
1157
|
lowCount: group.filter((item) => item.severity === 'low').length,
|
|
649
1158
|
noiseCount: group.filter((item) => item.severity === 'noise').length,
|
|
650
|
-
latestSeen: group
|
|
1159
|
+
latestSeen: group
|
|
1160
|
+
.filter((item) => timestampedOccurrencesOf(item) > 0)
|
|
1161
|
+
.reduce((latest, item) => item.lastSeen > latest ? item.lastSeen : latest, ''),
|
|
651
1162
|
};
|
|
652
1163
|
}).sort((a, b) => {
|
|
653
1164
|
const riskA = a.highCount * 100 + a.mediumCount * 10 + a.lowCount;
|
|
@@ -665,33 +1176,45 @@ export function findObservationInboxItem(id, dir = DEFAULT_OBSERVATIONS_DIR) {
|
|
|
665
1176
|
}
|
|
666
1177
|
return aggregateInboxItems(reports.flatMap((report) => report.items)).find((item) => item.id === id) ?? null;
|
|
667
1178
|
}
|
|
668
|
-
export function buildObservationMessageWindow(item, radius = 3) {
|
|
1179
|
+
export function buildObservationMessageWindow(item, radius = 3, preloadedMessages) {
|
|
669
1180
|
if (!item.sourceTrace.endsWith('.jsonl'))
|
|
670
1181
|
return undefined;
|
|
671
1182
|
const index = item.evidence.messageIndex;
|
|
672
|
-
if (typeof index !== 'number'
|
|
1183
|
+
if (typeof index !== 'number'
|
|
1184
|
+
|| index < 0
|
|
1185
|
+
|| (preloadedMessages === undefined && !existsSync(item.sourceTrace)))
|
|
673
1186
|
return undefined;
|
|
674
|
-
const messages = readJsonlMessageRefs(item.sourceTrace);
|
|
1187
|
+
const messages = preloadedMessages ?? readJsonlMessageRefs(item.sourceTrace);
|
|
675
1188
|
if (messages.length === 0)
|
|
676
1189
|
return undefined;
|
|
677
|
-
const position = messages.findIndex((message) =>
|
|
1190
|
+
const position = messages.findIndex((message) => {
|
|
1191
|
+
if (message.messageIndex !== index)
|
|
1192
|
+
return false;
|
|
1193
|
+
if (item.evidence.toolUseId)
|
|
1194
|
+
return message.snippet.includes(item.evidence.toolUseId);
|
|
1195
|
+
if (item.evidence.messageUuid)
|
|
1196
|
+
return message.uuid === item.evidence.messageUuid;
|
|
1197
|
+
return true;
|
|
1198
|
+
});
|
|
678
1199
|
if (position < 0)
|
|
679
1200
|
return undefined;
|
|
680
1201
|
const before = messages.slice(Math.max(0, position - radius), position);
|
|
681
1202
|
const event = messages.slice(position, position + 1);
|
|
682
1203
|
let after = messages.slice(position + 1, Math.min(messages.length, position + radius + 1));
|
|
683
1204
|
if (item.evidence.toolUseId) {
|
|
684
|
-
const result = messages.find((message) =>
|
|
685
|
-
|
|
1205
|
+
const result = messages.find((message, messagePosition) => messagePosition !== position
|
|
1206
|
+
&& message.messageIndex >= index
|
|
1207
|
+
&& message.snippet.includes(item.evidence.toolUseId));
|
|
1208
|
+
if (result && !event.includes(result)) {
|
|
686
1209
|
event.push(result);
|
|
687
|
-
after = after.filter((message) => message
|
|
1210
|
+
after = after.filter((message) => message !== result);
|
|
688
1211
|
}
|
|
689
1212
|
}
|
|
690
1213
|
return {
|
|
691
1214
|
before,
|
|
692
1215
|
event,
|
|
693
1216
|
after,
|
|
694
|
-
resolutionAfter: inferResolutionAfter(messages.slice(
|
|
1217
|
+
resolutionAfter: inferResolutionAfter(messages.slice(position + 1), item.signalType),
|
|
695
1218
|
};
|
|
696
1219
|
}
|
|
697
1220
|
export function formatObservationShow(item) {
|
|
@@ -717,55 +1240,39 @@ export function formatObservationShow(item) {
|
|
|
717
1240
|
return lines.join('\n');
|
|
718
1241
|
}
|
|
719
1242
|
function readJsonlMessageRefs(path) {
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
1243
|
+
let events;
|
|
1244
|
+
try {
|
|
1245
|
+
events = loadTraceSessions(path)[0]?.events ?? [];
|
|
1246
|
+
}
|
|
1247
|
+
catch {
|
|
1248
|
+
return [];
|
|
1249
|
+
}
|
|
1250
|
+
return messageRefsFromEvents(events);
|
|
1251
|
+
}
|
|
1252
|
+
function messageRefsFromEvents(events) {
|
|
1253
|
+
return events.flatMap((event) => {
|
|
1254
|
+
const common = {
|
|
1255
|
+
messageIndex: event.sourceIndex,
|
|
1256
|
+
uuid: event.sourceEventId ?? event.eventId,
|
|
1257
|
+
timestamp: event.timestamp,
|
|
1258
|
+
};
|
|
1259
|
+
if (event.eventKind === 'message') {
|
|
1260
|
+
const text = snippet(event.text, 500);
|
|
1261
|
+
const role = event.role === 'assistant'
|
|
1262
|
+
? 'assistant'
|
|
1263
|
+
: event.role === 'user' ? 'user' : 'other';
|
|
1264
|
+
return text ? [{ ...common, role, snippet: text }] : [];
|
|
740
1265
|
}
|
|
741
|
-
|
|
742
|
-
|
|
1266
|
+
if (event.eventKind === 'tool_call') {
|
|
1267
|
+
const text = snippet(`tool_use ${event.tool.displayName ?? event.tool.name} ${event.callId} ${JSON.stringify(event.input)}`, 500);
|
|
1268
|
+
return text ? [{ ...common, role: 'assistant', snippet: text }] : [];
|
|
743
1269
|
}
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
if (typeof content === 'string')
|
|
751
|
-
return content;
|
|
752
|
-
if (Array.isArray(content)) {
|
|
753
|
-
return content.map((part) => {
|
|
754
|
-
if (!part || typeof part !== 'object')
|
|
755
|
-
return '';
|
|
756
|
-
const obj = part;
|
|
757
|
-
if (obj.type === 'text')
|
|
758
|
-
return String(obj.text ?? '');
|
|
759
|
-
if (obj.type === 'tool_use')
|
|
760
|
-
return `tool_use ${String(obj.name ?? '')} ${String(obj.id ?? '')} ${JSON.stringify(obj.input ?? {})}`;
|
|
761
|
-
if (obj.type === 'tool_result')
|
|
762
|
-
return `tool_result ${String(obj.tool_use_id ?? '')} ${String(obj.content ?? '')}`;
|
|
763
|
-
if (obj.type === 'thinking')
|
|
764
|
-
return '';
|
|
765
|
-
return JSON.stringify(obj);
|
|
766
|
-
}).filter(Boolean).join('\n');
|
|
767
|
-
}
|
|
768
|
-
return JSON.stringify(record);
|
|
1270
|
+
if (event.eventKind === 'tool_result') {
|
|
1271
|
+
const text = snippet(`tool_result ${event.callId} ${event.output}`, 500);
|
|
1272
|
+
return text ? [{ ...common, role: 'other', snippet: text }] : [];
|
|
1273
|
+
}
|
|
1274
|
+
return [];
|
|
1275
|
+
});
|
|
769
1276
|
}
|
|
770
1277
|
function inferResolutionAfter(messages, signalType) {
|
|
771
1278
|
if (signalType !== 'failed_search' && signalType !== 'repeated_failure')
|
|
@@ -786,7 +1293,8 @@ function formatMessageRefs(messages) {
|
|
|
786
1293
|
export function selectExploreInboxItems(items, limit, includeNoise = false, random = Math.random) {
|
|
787
1294
|
const candidates = items
|
|
788
1295
|
.filter((item) => item.severity === 'medium' || item.severity === 'low' || (includeNoise && item.severity === 'noise'))
|
|
789
|
-
.sort((a, b) => b
|
|
1296
|
+
.sort((a, b) => Number(timestampedOccurrencesOf(b) > 0) - Number(timestampedOccurrencesOf(a) > 0)
|
|
1297
|
+
|| b.lastSeen.localeCompare(a.lastSeen))
|
|
790
1298
|
.slice(0, 50);
|
|
791
1299
|
const shuffled = [...candidates];
|
|
792
1300
|
for (let i = shuffled.length - 1; i > 0; i--) {
|