oh-my-knowledge 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -26
- package/README.zh.md +198 -26
- package/dist/src/authoring/evolver.d.ts +26 -2
- package/dist/src/authoring/evolver.d.ts.map +1 -1
- package/dist/src/authoring/evolver.js +410 -49
- package/dist/src/authoring/evolver.js.map +1 -1
- package/dist/src/authoring/generator.d.ts +5 -2
- package/dist/src/authoring/generator.d.ts.map +1 -1
- package/dist/src/authoring/generator.js +22 -6
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/authoring/sample-fixer.d.ts +10 -4
- package/dist/src/authoring/sample-fixer.d.ts.map +1 -1
- package/dist/src/authoring/sample-fixer.js +153 -126
- package/dist/src/authoring/sample-fixer.js.map +1 -1
- package/dist/src/cli/commands/doctor.d.ts +25 -1
- package/dist/src/cli/commands/doctor.d.ts.map +1 -1
- package/dist/src/cli/commands/doctor.js +233 -124
- package/dist/src/cli/commands/doctor.js.map +1 -1
- package/dist/src/cli/commands/eval/gold/compare.d.ts +17 -0
- package/dist/src/cli/commands/eval/gold/compare.d.ts.map +1 -0
- package/dist/src/cli/commands/eval/gold/compare.js +91 -0
- package/dist/src/cli/commands/eval/gold/compare.js.map +1 -0
- package/dist/src/cli/commands/eval/gold/index.d.ts +9 -0
- package/dist/src/cli/commands/eval/gold/index.d.ts.map +1 -0
- package/dist/src/cli/commands/eval/gold/index.js +34 -0
- package/dist/src/cli/commands/eval/gold/index.js.map +1 -0
- package/dist/src/cli/commands/eval/gold/init.d.ts +11 -0
- package/dist/src/cli/commands/eval/gold/init.d.ts.map +1 -0
- package/dist/src/cli/commands/eval/gold/init.js +51 -0
- package/dist/src/cli/commands/eval/gold/init.js.map +1 -0
- package/dist/src/cli/commands/eval/gold/validate.d.ts +12 -0
- package/dist/src/cli/commands/eval/gold/validate.d.ts.map +1 -0
- package/dist/src/cli/commands/eval/gold/validate.js +46 -0
- package/dist/src/cli/commands/eval/gold/validate.js.map +1 -0
- package/dist/src/cli/commands/eval/index.d.ts +54 -0
- package/dist/src/cli/commands/eval/index.d.ts.map +1 -0
- package/dist/src/cli/commands/{eval-runner.js → eval/index.js} +216 -50
- package/dist/src/cli/commands/eval/index.js.map +1 -0
- package/dist/src/cli/commands/evolve.d.ts +36 -1
- package/dist/src/cli/commands/evolve.d.ts.map +1 -1
- package/dist/src/cli/commands/evolve.js +191 -47
- package/dist/src/cli/commands/evolve.js.map +1 -1
- package/dist/src/cli/commands/init.d.ts +15 -1
- package/dist/src/cli/commands/init.d.ts.map +1 -1
- package/dist/src/cli/commands/init.js +70 -28
- package/dist/src/cli/commands/init.js.map +1 -1
- package/dist/src/cli/commands/observe/inbox.d.ts +23 -0
- package/dist/src/cli/commands/observe/inbox.d.ts.map +1 -0
- package/dist/src/cli/commands/observe/inbox.js +258 -0
- package/dist/src/cli/commands/observe/inbox.js.map +1 -0
- package/dist/src/cli/commands/observe/index.d.ts +22 -0
- package/dist/src/cli/commands/observe/index.d.ts.map +1 -0
- package/dist/src/cli/commands/observe/index.js +118 -0
- package/dist/src/cli/commands/observe/index.js.map +1 -0
- package/dist/src/cli/commands/observe/ingest.d.ts +13 -0
- package/dist/src/cli/commands/observe/ingest.d.ts.map +1 -0
- package/dist/src/cli/commands/observe/ingest.js +69 -0
- package/dist/src/cli/commands/observe/ingest.js.map +1 -0
- package/dist/src/cli/commands/observe/show.d.ts +13 -0
- package/dist/src/cli/commands/observe/show.d.ts.map +1 -0
- package/dist/src/cli/commands/observe/show.js +49 -0
- package/dist/src/cli/commands/observe/show.js.map +1 -0
- package/dist/src/cli/commands/sample.d.ts +28 -5
- package/dist/src/cli/commands/sample.d.ts.map +1 -1
- package/dist/src/cli/commands/sample.js +242 -169
- package/dist/src/cli/commands/sample.js.map +1 -1
- package/dist/src/cli/commands/skill-extract.d.ts +20 -0
- package/dist/src/cli/commands/skill-extract.d.ts.map +1 -0
- package/dist/src/cli/commands/skill-extract.js +84 -0
- package/dist/src/cli/commands/skill-extract.js.map +1 -0
- package/dist/src/cli/commands/studio.d.ts +22 -1
- package/dist/src/cli/commands/studio.d.ts.map +1 -1
- package/dist/src/cli/commands/studio.js +111 -37
- package/dist/src/cli/commands/studio.js.map +1 -1
- package/dist/src/cli/index.js +15 -47
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/cli/lib/cli-exit.d.ts +16 -0
- package/dist/src/cli/lib/cli-exit.d.ts.map +1 -0
- package/dist/src/cli/lib/cli-exit.js +20 -0
- package/dist/src/cli/lib/cli-exit.js.map +1 -0
- package/dist/src/cli/lib/cmd-flags.d.ts +236 -0
- package/dist/src/cli/lib/cmd-flags.d.ts.map +1 -0
- package/dist/src/cli/lib/cmd-flags.js +46 -0
- package/dist/src/cli/lib/cmd-flags.js.map +1 -0
- package/dist/src/cli/{i18n-dict.d.ts → lib/i18n-dict.d.ts} +1 -1
- package/dist/src/cli/lib/i18n-dict.d.ts.map +1 -0
- package/dist/src/cli/{i18n-dict.js → lib/i18n-dict.js} +39 -264
- package/dist/src/cli/lib/i18n-dict.js.map +1 -0
- package/dist/src/cli/{i18n.d.ts → lib/i18n.d.ts} +11 -6
- package/dist/src/cli/lib/i18n.d.ts.map +1 -0
- package/dist/src/cli/{i18n.js → lib/i18n.js} +12 -7
- package/dist/src/cli/lib/i18n.js.map +1 -0
- package/dist/src/cli/{parse-run-config.d.ts → lib/parse-run-config.d.ts} +5 -6
- package/dist/src/cli/lib/parse-run-config.d.ts.map +1 -0
- package/dist/src/cli/{parse-run-config.js → lib/parse-run-config.js} +11 -61
- package/dist/src/cli/lib/parse-run-config.js.map +1 -0
- package/dist/src/cli/lib/progress.d.ts.map +1 -0
- package/dist/src/cli/lib/progress.js.map +1 -0
- package/dist/src/cli/{run-tally.d.ts → lib/run-tally.d.ts} +1 -1
- package/dist/src/cli/lib/run-tally.d.ts.map +1 -0
- package/dist/src/cli/lib/run-tally.js.map +1 -0
- package/dist/src/cli/{commands/_shared.d.ts → lib/shared.d.ts} +2 -2
- package/dist/src/cli/lib/shared.d.ts.map +1 -0
- package/dist/src/cli/{commands/_shared.js → lib/shared.js} +3 -3
- package/dist/src/cli/lib/shared.js.map +1 -0
- package/dist/src/cli/lib/update-check.d.ts.map +1 -0
- package/dist/src/cli/lib/update-check.js.map +1 -0
- package/dist/src/cli/oclif/base-command.d.ts +8 -0
- package/dist/src/cli/oclif/base-command.d.ts.map +1 -0
- package/dist/src/cli/oclif/base-command.js +27 -0
- package/dist/src/cli/oclif/base-command.js.map +1 -0
- package/dist/src/cli/oclif/help.d.ts +16 -0
- package/dist/src/cli/oclif/help.d.ts.map +1 -0
- package/dist/src/cli/oclif/help.js +33 -0
- package/dist/src/cli/oclif/help.js.map +1 -0
- package/dist/src/cli/oclif/i18n.d.ts +16 -0
- package/dist/src/cli/oclif/i18n.d.ts.map +1 -0
- package/dist/src/cli/oclif/i18n.js +62 -0
- package/dist/src/cli/oclif/i18n.js.map +1 -0
- package/dist/src/cli/oclif/parsers.d.ts +10 -0
- package/dist/src/cli/oclif/parsers.d.ts.map +1 -0
- package/dist/src/cli/oclif/parsers.js +102 -0
- package/dist/src/cli/oclif/parsers.js.map +1 -0
- package/dist/src/cli/oclif/projection.d.ts +23 -0
- package/dist/src/cli/oclif/projection.d.ts.map +1 -0
- package/dist/src/cli/oclif/projection.js +52 -0
- package/dist/src/cli/oclif/projection.js.map +1 -0
- package/dist/src/cli/oclif/run.d.ts +2 -0
- package/dist/src/cli/oclif/run.d.ts.map +1 -0
- package/dist/src/cli/oclif/run.js +11 -0
- package/dist/src/cli/oclif/run.js.map +1 -0
- package/dist/src/diagnosis/observe-mapper.d.ts +68 -0
- package/dist/src/diagnosis/observe-mapper.d.ts.map +1 -0
- package/dist/src/diagnosis/observe-mapper.js +276 -0
- package/dist/src/diagnosis/observe-mapper.js.map +1 -0
- package/dist/src/diagnosis/observe-producer.d.ts +9 -0
- package/dist/src/diagnosis/observe-producer.d.ts.map +1 -0
- package/dist/src/diagnosis/observe-producer.js +197 -0
- package/dist/src/diagnosis/observe-producer.js.map +1 -0
- package/dist/src/diagnosis/studio-projection.d.ts +14 -0
- package/dist/src/diagnosis/studio-projection.d.ts.map +1 -0
- package/dist/src/diagnosis/studio-projection.js +84 -0
- package/dist/src/diagnosis/studio-projection.js.map +1 -0
- package/dist/src/diagnosis/types.d.ts +79 -0
- package/dist/src/diagnosis/types.d.ts.map +1 -0
- package/dist/src/diagnosis/types.js +20 -0
- package/dist/src/diagnosis/types.js.map +1 -0
- package/dist/src/doctor/fixer.d.ts +12 -0
- package/dist/src/doctor/fixer.d.ts.map +1 -0
- package/dist/src/doctor/fixer.js +326 -0
- package/dist/src/doctor/fixer.js.map +1 -0
- package/dist/src/doctor/health/composer.js +7 -4
- package/dist/src/doctor/health/composer.js.map +1 -1
- package/dist/src/doctor/health/parser.d.ts +0 -1
- package/dist/src/doctor/health/parser.d.ts.map +1 -1
- package/dist/src/doctor/health/parser.js +37 -0
- package/dist/src/doctor/health/parser.js.map +1 -1
- package/dist/src/doctor/health/prompt-builder.d.ts +2 -21
- package/dist/src/doctor/health/prompt-builder.d.ts.map +1 -1
- package/dist/src/doctor/health/prompt-builder.js +1 -169
- package/dist/src/doctor/health/prompt-builder.js.map +1 -1
- package/dist/src/doctor/html-renderer.d.ts +1 -1
- package/dist/src/doctor/html-renderer.d.ts.map +1 -1
- package/dist/src/doctor/html-renderer.js +2 -2
- package/dist/src/doctor/html-renderer.js.map +1 -1
- package/dist/src/doctor/index.d.ts.map +1 -1
- package/dist/src/doctor/index.js +1 -0
- package/dist/src/doctor/index.js.map +1 -1
- package/dist/src/doctor/renderer.d.ts +1 -1
- package/dist/src/doctor/renderer.d.ts.map +1 -1
- package/dist/src/doctor/renderer.js +2 -2
- package/dist/src/doctor/renderer.js.map +1 -1
- package/dist/src/doctor/rules.js +1 -1
- package/dist/src/doctor/rules.js.map +1 -1
- package/dist/src/eval-core/mock-hook.cjs +13 -1
- package/dist/src/eval-core/mocks-runtime.d.ts +7 -3
- package/dist/src/eval-core/mocks-runtime.d.ts.map +1 -1
- package/dist/src/eval-core/mocks-runtime.js +240 -4
- package/dist/src/eval-core/mocks-runtime.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +1 -1
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/executors/claude-cli.d.ts.map +1 -1
- package/dist/src/executors/claude-cli.js +6 -4
- package/dist/src/executors/claude-cli.js.map +1 -1
- package/dist/src/observability/experience.d.ts +184 -3
- package/dist/src/observability/experience.d.ts.map +1 -1
- package/dist/src/observability/experience.js +1208 -41
- package/dist/src/observability/experience.js.map +1 -1
- package/dist/src/observability/inbox-view-model.d.ts +3 -0
- package/dist/src/observability/inbox-view-model.d.ts.map +1 -1
- package/dist/src/observability/inbox-view-model.js +9 -0
- package/dist/src/observability/inbox-view-model.js.map +1 -1
- package/dist/src/observability/inbox.d.ts +2 -0
- package/dist/src/observability/inbox.d.ts.map +1 -1
- package/dist/src/observability/inbox.js +16 -2
- package/dist/src/observability/inbox.js.map +1 -1
- package/dist/src/observability/problem-patterns.d.ts +1 -0
- package/dist/src/observability/problem-patterns.d.ts.map +1 -1
- package/dist/src/observability/problem-patterns.js +8 -8
- package/dist/src/observability/problem-patterns.js.map +1 -1
- package/dist/src/observability/resolved-review.d.ts +25 -0
- package/dist/src/observability/resolved-review.d.ts.map +1 -0
- package/dist/src/observability/resolved-review.js +231 -0
- package/dist/src/observability/resolved-review.js.map +1 -0
- package/dist/src/observability/review-state.d.ts +9 -2
- package/dist/src/observability/review-state.d.ts.map +1 -1
- package/dist/src/observability/review-state.js +40 -2
- package/dist/src/observability/review-state.js.map +1 -1
- package/dist/src/observability/skill-chain-advisories.js +4 -4
- package/dist/src/observability/skill-chain-advisories.js.map +1 -1
- package/dist/src/observability/skill-chain.d.ts +1 -0
- package/dist/src/observability/skill-chain.d.ts.map +1 -1
- package/dist/src/observability/skill-chain.js +72 -35
- package/dist/src/observability/skill-chain.js.map +1 -1
- package/dist/src/observability/soft-standards.d.ts +130 -0
- package/dist/src/observability/soft-standards.d.ts.map +1 -0
- package/dist/src/observability/soft-standards.js +426 -0
- package/dist/src/observability/soft-standards.js.map +1 -0
- package/dist/src/observability/text-signals.d.ts +7 -0
- package/dist/src/observability/text-signals.d.ts.map +1 -1
- package/dist/src/observability/text-signals.js +88 -4
- package/dist/src/observability/text-signals.js.map +1 -1
- package/dist/src/observability/trace-segmenter.d.ts.map +1 -1
- package/dist/src/observability/trace-segmenter.js +4 -2
- package/dist/src/observability/trace-segmenter.js.map +1 -1
- package/dist/src/observability/trace-source.d.ts.map +1 -1
- package/dist/src/observability/trace-source.js +13 -3
- package/dist/src/observability/trace-source.js.map +1 -1
- package/dist/src/renderer/observation-inbox-renderer.d.ts.map +1 -1
- package/dist/src/renderer/observation-inbox-renderer.js +5914 -924
- package/dist/src/renderer/observation-inbox-renderer.js.map +1 -1
- package/dist/src/renderer/skill-detail-renderer.d.ts.map +1 -1
- package/dist/src/renderer/skill-detail-renderer.js +15 -5
- package/dist/src/renderer/skill-detail-renderer.js.map +1 -1
- package/dist/src/renderer/skill-list-renderer.d.ts.map +1 -1
- package/dist/src/renderer/skill-list-renderer.js +10 -3
- package/dist/src/renderer/skill-list-renderer.js.map +1 -1
- package/dist/src/renderer/test-view.js +2 -2
- package/dist/src/renderer/test-view.js.map +1 -1
- package/dist/src/server/report-server.d.ts.map +1 -1
- package/dist/src/server/report-server.js +95 -2
- package/dist/src/server/report-server.js.map +1 -1
- package/dist/src/server/skill-index.d.ts +5 -1
- package/dist/src/server/skill-index.d.ts.map +1 -1
- package/dist/src/server/skill-index.js +72 -26
- package/dist/src/server/skill-index.js.map +1 -1
- package/dist/src/server/skill-insights.d.ts +10 -1
- package/dist/src/server/skill-insights.d.ts.map +1 -1
- package/dist/src/server/skill-insights.js +113 -1
- package/dist/src/server/skill-insights.js.map +1 -1
- package/dist/src/shared/hard-rules.d.ts +2 -0
- package/dist/src/shared/hard-rules.d.ts.map +1 -1
- package/dist/src/shared/hard-rules.js +42 -0
- package/dist/src/shared/hard-rules.js.map +1 -1
- package/dist/src/shared/llm-prompts/index.d.ts +14 -0
- package/dist/src/shared/llm-prompts/index.d.ts.map +1 -0
- package/dist/src/shared/llm-prompts/index.js +35 -0
- package/dist/src/shared/llm-prompts/index.js.map +1 -0
- package/dist/src/shared/llm-prompts/skill-health.d.ts +22 -0
- package/dist/src/shared/llm-prompts/skill-health.d.ts.map +1 -0
- package/dist/src/shared/llm-prompts/skill-health.js +170 -0
- package/dist/src/shared/llm-prompts/skill-health.js.map +1 -0
- package/dist/src/types/doctor.d.ts +2 -0
- package/dist/src/types/doctor.d.ts.map +1 -1
- package/dist/src/types/eval.d.ts +6 -1
- package/dist/src/types/eval.d.ts.map +1 -1
- package/dist/src/types/report.d.ts +6 -0
- package/dist/src/types/report.d.ts.map +1 -1
- package/docs/prompts/llm-enhanced-review.prompt.md +111 -0
- package/package.json +29 -4
- package/dist/src/cli/cli-exit.d.ts +0 -15
- package/dist/src/cli/cli-exit.d.ts.map +0 -1
- package/dist/src/cli/cli-exit.js +0 -19
- package/dist/src/cli/cli-exit.js.map +0 -1
- package/dist/src/cli/commands/_shared.d.ts.map +0 -1
- package/dist/src/cli/commands/_shared.js.map +0 -1
- package/dist/src/cli/commands/eval-gold.d.ts +0 -2
- package/dist/src/cli/commands/eval-gold.d.ts.map +0 -1
- package/dist/src/cli/commands/eval-gold.js +0 -137
- package/dist/src/cli/commands/eval-gold.js.map +0 -1
- package/dist/src/cli/commands/eval-runner.d.ts +0 -2
- package/dist/src/cli/commands/eval-runner.d.ts.map +0 -1
- package/dist/src/cli/commands/eval-runner.js.map +0 -1
- package/dist/src/cli/commands/eval.d.ts +0 -2
- package/dist/src/cli/commands/eval.d.ts.map +0 -1
- package/dist/src/cli/commands/eval.js +0 -11
- package/dist/src/cli/commands/eval.js.map +0 -1
- package/dist/src/cli/commands/observe.d.ts +0 -2
- package/dist/src/cli/commands/observe.d.ts.map +0 -1
- package/dist/src/cli/commands/observe.js +0 -247
- package/dist/src/cli/commands/observe.js.map +0 -1
- package/dist/src/cli/commands/registry.d.ts +0 -11
- package/dist/src/cli/commands/registry.d.ts.map +0 -1
- package/dist/src/cli/commands/registry.js +0 -32
- package/dist/src/cli/commands/registry.js.map +0 -1
- package/dist/src/cli/i18n-dict.d.ts.map +0 -1
- package/dist/src/cli/i18n-dict.js.map +0 -1
- package/dist/src/cli/i18n.d.ts.map +0 -1
- package/dist/src/cli/i18n.js.map +0 -1
- package/dist/src/cli/parse-run-config.d.ts.map +0 -1
- package/dist/src/cli/parse-run-config.js.map +0 -1
- package/dist/src/cli/parse-strict.d.ts +0 -7
- package/dist/src/cli/parse-strict.d.ts.map +0 -1
- package/dist/src/cli/parse-strict.js +0 -26
- package/dist/src/cli/parse-strict.js.map +0 -1
- package/dist/src/cli/progress.d.ts.map +0 -1
- package/dist/src/cli/progress.js.map +0 -1
- package/dist/src/cli/run-tally.d.ts.map +0 -1
- package/dist/src/cli/run-tally.js.map +0 -1
- package/dist/src/cli/update-check.d.ts.map +0 -1
- package/dist/src/cli/update-check.js.map +0 -1
- /package/dist/src/cli/{progress.d.ts → lib/progress.d.ts} +0 -0
- /package/dist/src/cli/{progress.js → lib/progress.js} +0 -0
- /package/dist/src/cli/{run-tally.js → lib/run-tally.js} +0 -0
- /package/dist/src/cli/{update-check.d.ts → lib/update-check.d.ts} +0 -0
- /package/dist/src/cli/{update-check.js → lib/update-check.js} +0 -0
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
2
5
|
import { buildExperienceProblemPatterns, mergeExperienceProblemPatterns, } from './problem-patterns.js';
|
|
3
|
-
import { observationMetricAnnotationVerdict } from './review-state.js';
|
|
6
|
+
import { observationMetricAnnotationVerdict, observationReviewStateKey } from './review-state.js';
|
|
4
7
|
import { extractCommandEnvelopeText, stripCommandEnvelopeText } from './trace-attribution.js';
|
|
5
|
-
import { hasAssistantDeliverySignalText, hasUserHardRuleText,
|
|
8
|
+
import { hasAssistantDeliverableArtifactText, hasAssistantDeliverySignalText, hasUserHardRuleText, isAssistantProgressUpdateText, isAssistantProtocolReplyText, isRuntimeProtocolPromptText, isSyntheticUserMessageText, isToolResultFailureText, isUserInteractionMetricText } from './text-signals.js';
|
|
6
9
|
import { durationMsBetween } from '../shared/time.js';
|
|
10
|
+
import { parseSkillFrontmatter, validateSkillHardRules, validateSkillWorkflows } from '../shared/hard-rules.js';
|
|
11
|
+
export function aggregateExperienceChecklistItemStatus(statuses) {
|
|
12
|
+
if (statuses.includes('degraded'))
|
|
13
|
+
return 'degraded';
|
|
14
|
+
if (statuses.includes('failed'))
|
|
15
|
+
return 'failed';
|
|
16
|
+
if (statuses.includes('unknown'))
|
|
17
|
+
return 'unknown';
|
|
18
|
+
if (statuses.includes('not_declared'))
|
|
19
|
+
return 'not_declared';
|
|
20
|
+
if (statuses.includes('passed'))
|
|
21
|
+
return 'passed';
|
|
22
|
+
return 'not_applicable';
|
|
23
|
+
}
|
|
7
24
|
const TEXT_BOUNDARY_CHARS = new Set([
|
|
8
25
|
' ', '\n', '\r', '\t',
|
|
9
26
|
',', ',', ';', ';', '、', '.', '。', '!', '!', '?', '?', ':', ':',
|
|
@@ -75,6 +92,22 @@ const NEGATIVE_FEEDBACK_TERMS = [
|
|
|
75
92
|
'菜啊',
|
|
76
93
|
];
|
|
77
94
|
const BOUNDED_NEGATIVE_FEEDBACK_TERMS = ['失败', '垃圾', '菜'];
|
|
95
|
+
const ADDITIONAL_NEGATIVE_FEEDBACK_TERMS = [
|
|
96
|
+
'不对',
|
|
97
|
+
'错了',
|
|
98
|
+
'有误',
|
|
99
|
+
'不可以',
|
|
100
|
+
'不是这个',
|
|
101
|
+
'不是我要的',
|
|
102
|
+
'跑偏',
|
|
103
|
+
'绕路',
|
|
104
|
+
'你没懂',
|
|
105
|
+
'没懂',
|
|
106
|
+
'重做',
|
|
107
|
+
'重新做',
|
|
108
|
+
'重来',
|
|
109
|
+
'不符合预期',
|
|
110
|
+
];
|
|
78
111
|
const POSITIVE_FEEDBACK_TERMS = [
|
|
79
112
|
'非常好',
|
|
80
113
|
'很好',
|
|
@@ -101,18 +134,22 @@ const POSITIVE_FEEDBACK_TERMS = [
|
|
|
101
134
|
'excellent',
|
|
102
135
|
'awesome',
|
|
103
136
|
];
|
|
104
|
-
const USER_INTERRUPTION_RE = /\[Request interrupted by user(?: for tool use)?\]|interrupted by user
|
|
137
|
+
const USER_INTERRUPTION_RE = /\[Request interrupted by user(?: for tool use)?\]|interrupted by user|用户中断|停止任务|停一下|先别|别动|等一下|等下|等等|取消(?:任务|执行)?|先暂停|暂停一下/i;
|
|
105
138
|
const TIMELINE_PREVIEW_EVENT_LIMIT = 240;
|
|
106
139
|
const ZERO_INDICATORS = {
|
|
107
140
|
userMessageCount: 0,
|
|
108
141
|
userFollowUpCount: 0,
|
|
109
142
|
userCorrectionCount: 0,
|
|
110
143
|
userInterruptionCount: 0,
|
|
144
|
+
sessionInterruptedCount: 0,
|
|
111
145
|
negativeFeedbackCount: 0,
|
|
112
146
|
positiveFeedbackCount: 0,
|
|
113
147
|
userGoalShiftCount: 0,
|
|
114
148
|
hardRuleTextHitCount: 0,
|
|
115
149
|
assistantDeliverySignalCount: 0,
|
|
150
|
+
deliverableArtifactSignalCount: 0,
|
|
151
|
+
selfCorrectionCount: 0,
|
|
152
|
+
repeatedExecutionCount: 0,
|
|
116
153
|
toolCallCount: 0,
|
|
117
154
|
toolFailureCount: 0,
|
|
118
155
|
highObservationCount: 0,
|
|
@@ -123,6 +160,7 @@ const ZERO_INDICATORS = {
|
|
|
123
160
|
export function buildObservationExperienceReport(input) {
|
|
124
161
|
const sessionsBySourceTrace = new Map(input.sessions.map((session) => [session.sourcePath, session]));
|
|
125
162
|
const sessionGroupsById = groupSessionsByLogicalId(input.sessions);
|
|
163
|
+
const sessionGroupsByKey = groupSessionsByExperienceKey(input.sessions);
|
|
126
164
|
const goalSlices = [];
|
|
127
165
|
const invocations = [];
|
|
128
166
|
for (const segment of input.segments) {
|
|
@@ -130,19 +168,22 @@ export function buildObservationExperienceReport(input) {
|
|
|
130
168
|
continue;
|
|
131
169
|
const session = sessionsBySourceTrace.get(segment.sourceTrace ?? '') ?? sessionGroupsById.get(segment.sessionId)?.[0];
|
|
132
170
|
const sourceTrace = segment.sourceTrace ?? session?.sourcePath ?? '';
|
|
171
|
+
const sessionGroupKey = session ? experienceSessionGroupKey(session) : `trace:${sourceTrace || segment.sessionId}`;
|
|
133
172
|
const relatedItems = relatedObservationItems(segment, input.items);
|
|
134
173
|
const bounds = session ? segmentRecordBounds(session, segment) : { start: 0, end: 0 };
|
|
135
174
|
const timeline = session ? buildTimeline(session, bounds.start, bounds.end) : [];
|
|
175
|
+
const metricScopeId = hashParts('session', segment.skillName, sessionGroupKey);
|
|
136
176
|
const userRefs = timeline.filter((event) => event.kind === 'user_message');
|
|
137
|
-
const indicators = indicatorsForSegment(segment, relatedItems, timeline, input.reviewState);
|
|
177
|
+
const indicators = indicatorsForSegment(segment, relatedItems, timeline, metricScopeId, input.reviewState);
|
|
138
178
|
const observationRefs = relatedItems.map(observationEvidenceRef);
|
|
139
179
|
const evidenceChain = evidenceChainForTimeline(timeline, observationRefs);
|
|
140
|
-
const ruleFindings = ruleFindingsForEvidence(indicators, timeline, observationRefs, evidenceChain, input.reviewState);
|
|
180
|
+
const ruleFindings = ruleFindingsForEvidence(indicators, timeline, observationRefs, evidenceChain, metricScopeId, input.reviewState);
|
|
141
181
|
const assistiveInference = assistiveInferenceForEvidence(indicators, evidenceChain, ruleFindings);
|
|
142
182
|
const problemPatterns = buildExperienceProblemPatterns({
|
|
143
183
|
skillName: segment.skillName,
|
|
144
184
|
sessionId: segment.sessionId,
|
|
145
185
|
timeline,
|
|
186
|
+
metricScopeId,
|
|
146
187
|
reviewState: input.reviewState,
|
|
147
188
|
});
|
|
148
189
|
const hasGoalShift = userRefs.some((ref) => hasUserGoalShiftSignal(ref.snippet ?? ''));
|
|
@@ -167,6 +208,7 @@ export function buildObservationExperienceReport(input) {
|
|
|
167
208
|
id: invocationId,
|
|
168
209
|
skillName: segment.skillName,
|
|
169
210
|
sessionId: segment.sessionId,
|
|
211
|
+
sessionGroupKey,
|
|
170
212
|
sourceTrace,
|
|
171
213
|
sourceKind: segment.sourceKind ?? sourceKindForPath(sourceTrace),
|
|
172
214
|
entrypoint: session ? session.entrypoint ?? inferEntrypointFromRecords(session) : undefined,
|
|
@@ -198,7 +240,7 @@ export function buildObservationExperienceReport(input) {
|
|
|
198
240
|
timeline,
|
|
199
241
|
});
|
|
200
242
|
}
|
|
201
|
-
const sessions = summarizeExperienceSessions(invocations,
|
|
243
|
+
const sessions = summarizeExperienceSessions(invocations, sessionGroupsByKey, input.generatedAt, input.reviewState);
|
|
202
244
|
const skills = summarizeExperienceSkills(sessions, invocations);
|
|
203
245
|
return {
|
|
204
246
|
kind: 'observe-experience',
|
|
@@ -226,6 +268,13 @@ function relatedObservationItems(segment, items) {
|
|
|
226
268
|
function logicalSessionId(session) {
|
|
227
269
|
return session.sessionGroupId ?? session.sessionId;
|
|
228
270
|
}
|
|
271
|
+
function experienceSessionGroupKey(session) {
|
|
272
|
+
const logicalId = logicalSessionId(session);
|
|
273
|
+
if (session.traceRole && session.traceRole !== 'standalone' && session.sessionGroupPath) {
|
|
274
|
+
return `group:${session.sessionGroupPath}\u0000${logicalId}`;
|
|
275
|
+
}
|
|
276
|
+
return `trace:${session.sourcePath}`;
|
|
277
|
+
}
|
|
229
278
|
function groupSessionsByLogicalId(sessions) {
|
|
230
279
|
const groups = new Map();
|
|
231
280
|
for (const session of sessions) {
|
|
@@ -239,6 +288,19 @@ function groupSessionsByLogicalId(sessions) {
|
|
|
239
288
|
}
|
|
240
289
|
return groups;
|
|
241
290
|
}
|
|
291
|
+
function groupSessionsByExperienceKey(sessions) {
|
|
292
|
+
const groups = new Map();
|
|
293
|
+
for (const session of sessions) {
|
|
294
|
+
const key = experienceSessionGroupKey(session);
|
|
295
|
+
const group = groups.get(key) ?? [];
|
|
296
|
+
group.push(session);
|
|
297
|
+
groups.set(key, group);
|
|
298
|
+
}
|
|
299
|
+
for (const [key, group] of groups.entries()) {
|
|
300
|
+
groups.set(key, group.sort(compareSessionsForTimeline));
|
|
301
|
+
}
|
|
302
|
+
return groups;
|
|
303
|
+
}
|
|
242
304
|
function compareSessionsForTimeline(a, b) {
|
|
243
305
|
const roleRank = (session) => session.traceRole === 'main' ? 0 : session.traceRole === 'standalone' ? 1 : 2;
|
|
244
306
|
const rank = roleRank(a) - roleRank(b);
|
|
@@ -331,6 +393,30 @@ function isAssistantDeliveryEvent(event) {
|
|
|
331
393
|
const text = event.fullText ?? event.snippet ?? '';
|
|
332
394
|
return hasAssistantDeliverySignalText(text);
|
|
333
395
|
}
|
|
396
|
+
function isAssistantDeliverableArtifactEvent(event) {
|
|
397
|
+
if (event.kind !== 'assistant_message')
|
|
398
|
+
return false;
|
|
399
|
+
const text = event.fullText ?? event.snippet ?? '';
|
|
400
|
+
return hasAssistantDeliverableArtifactText(text);
|
|
401
|
+
}
|
|
402
|
+
function isAssistantProgressUpdateEvent(event) {
|
|
403
|
+
if (event.kind !== 'assistant_message')
|
|
404
|
+
return false;
|
|
405
|
+
const text = event.fullText ?? event.snippet ?? '';
|
|
406
|
+
return isAssistantProgressUpdateText(text);
|
|
407
|
+
}
|
|
408
|
+
function hasSelfCorrectionSignal(event) {
|
|
409
|
+
if (event.kind !== 'assistant_message')
|
|
410
|
+
return false;
|
|
411
|
+
const text = event.fullText ?? event.snippet ?? '';
|
|
412
|
+
return /刚才.*(?:不对|错了|有误)|发现.*(?:不对|错了|问题|遗漏)|重新(?:检查|分析|执行|生成|整理|补(?:充)?(?:结论|答案|回答))|改用|换成|修正|我再(?:检查|重新|看)|跑偏|偏(?:题|了|向)|没(?:有)?(?:完整|完全)?(?:回答|覆盖)|漏(?:了|掉)|不完整|我刚追问了一版|按原问题(?:重新)?(?:补|回|答)|\b(?:recheck|retry|rerun|mistake|wrong)\b/i.test(text);
|
|
413
|
+
}
|
|
414
|
+
function hasRepeatedExecutionSignal(event) {
|
|
415
|
+
if (event.kind !== 'assistant_message' && event.kind !== 'tool_use')
|
|
416
|
+
return false;
|
|
417
|
+
const text = `${event.label ?? ''} ${event.toolName ?? ''} ${event.fullText ?? event.snippet ?? ''}`;
|
|
418
|
+
return /重复(?:执行|尝试|读取|搜索|调用)|再次(?:执行|读取|搜索|调用)|重新(?:执行|读取|搜索|调用|跑|运行)|再(?:执行|读取|搜索|调用|跑)一遍|重试|\b(?:retry|rerun)\b/i.test(text);
|
|
419
|
+
}
|
|
334
420
|
function buildTimeline(session, start, end) {
|
|
335
421
|
return buildTimelineWindow(session, start, end).slice(0, TIMELINE_PREVIEW_EVENT_LIMIT);
|
|
336
422
|
}
|
|
@@ -366,7 +452,36 @@ function timelineEventsFromRecord(session, record, messageIndex) {
|
|
|
366
452
|
const assistant = rec;
|
|
367
453
|
return assistantEvents(assistant, base, messageIndex);
|
|
368
454
|
}
|
|
369
|
-
return
|
|
455
|
+
return sessionEventFromRecord(rec, base, messageIndex);
|
|
456
|
+
}
|
|
457
|
+
function sessionEventFromRecord(record, base, messageIndex) {
|
|
458
|
+
const type = typeof record.type === 'string' ? record.type : '';
|
|
459
|
+
const rawText = safeRecordText(record);
|
|
460
|
+
if (!isSessionBoundaryType(type) && !hasAssistantTurnFailedText(rawText))
|
|
461
|
+
return [];
|
|
462
|
+
return [timelineEvent({
|
|
463
|
+
...base,
|
|
464
|
+
kind: 'runtime_context',
|
|
465
|
+
role: 'other',
|
|
466
|
+
order: messageIndex * 10,
|
|
467
|
+
snippet: snippet(rawText || type, 700),
|
|
468
|
+
fullText: fullText(rawText || type),
|
|
469
|
+
label: type || 'session event',
|
|
470
|
+
})];
|
|
471
|
+
}
|
|
472
|
+
function isSessionBoundaryType(type) {
|
|
473
|
+
return /^session[._-](?:started|ended)$/i.test(type);
|
|
474
|
+
}
|
|
475
|
+
function safeRecordText(record) {
|
|
476
|
+
try {
|
|
477
|
+
return JSON.stringify(record);
|
|
478
|
+
}
|
|
479
|
+
catch {
|
|
480
|
+
return String(record ?? '');
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
function hasAssistantTurnFailedText(value) {
|
|
484
|
+
return /\[assistant turn failed\]|assistant turn failed|assistant_turn_failed|turn failed/i.test(value);
|
|
370
485
|
}
|
|
371
486
|
function userEvents(record, base, messageIndex) {
|
|
372
487
|
const content = record.message.content;
|
|
@@ -387,16 +502,17 @@ function userEvents(record, base, messageIndex) {
|
|
|
387
502
|
else if (part.type === 'tool_result') {
|
|
388
503
|
const full = fullText(part.content);
|
|
389
504
|
const text = snippet(part.content, 900);
|
|
505
|
+
const failed = part.is_error === true || isToolResultFailureText(part.content);
|
|
390
506
|
events.push(timelineEvent({
|
|
391
507
|
...base,
|
|
392
508
|
kind: 'tool_result',
|
|
393
509
|
role: 'tool',
|
|
394
510
|
order: messageIndex * 10 + 5 + resultIndex,
|
|
395
511
|
toolUseId: part.tool_use_id,
|
|
396
|
-
isError:
|
|
512
|
+
isError: failed,
|
|
397
513
|
snippet: text,
|
|
398
514
|
fullText: full,
|
|
399
|
-
label:
|
|
515
|
+
label: failed ? 'tool result error' : 'tool result',
|
|
400
516
|
}));
|
|
401
517
|
resultIndex += 1;
|
|
402
518
|
}
|
|
@@ -425,7 +541,7 @@ function userTextEventsFromText(record, rawText, base, messageIndex, offset) {
|
|
|
425
541
|
events.push(timelineEvent({
|
|
426
542
|
...base,
|
|
427
543
|
kind,
|
|
428
|
-
role: kind === 'user_message' ? 'user' : 'tool',
|
|
544
|
+
role: kind === 'user_message' ? 'user' : kind === 'synthetic_user_event' ? 'other' : 'tool',
|
|
429
545
|
order: messageIndex * 10 + offset + (commandEnvelope ? 1 : 0),
|
|
430
546
|
snippet: text,
|
|
431
547
|
fullText: full,
|
|
@@ -460,14 +576,15 @@ function assistantEvents(record, base, messageIndex) {
|
|
|
460
576
|
const rawText = textParts.join('\n');
|
|
461
577
|
const text = snippet(rawText, 700);
|
|
462
578
|
if (text) {
|
|
579
|
+
const protocolReply = isAssistantProtocolReplyText(rawText);
|
|
463
580
|
events.unshift(timelineEvent({
|
|
464
581
|
...base,
|
|
465
|
-
kind: 'assistant_message',
|
|
582
|
+
kind: protocolReply ? 'runtime_context' : 'assistant_message',
|
|
466
583
|
role: 'assistant',
|
|
467
584
|
order: messageIndex * 10,
|
|
468
585
|
snippet: text,
|
|
469
586
|
fullText: fullText(rawText),
|
|
470
|
-
label: 'assistant message',
|
|
587
|
+
label: protocolReply ? 'assistant protocol reply' : 'assistant message',
|
|
471
588
|
}));
|
|
472
589
|
}
|
|
473
590
|
return events;
|
|
@@ -483,6 +600,8 @@ function userTextEventKind(record, text) {
|
|
|
483
600
|
return 'skill_context';
|
|
484
601
|
if (isRuntimeContextRecord(record, text))
|
|
485
602
|
return 'runtime_context';
|
|
603
|
+
if (isSyntheticUserMessageText(text))
|
|
604
|
+
return 'synthetic_user_event';
|
|
486
605
|
return 'user_message';
|
|
487
606
|
}
|
|
488
607
|
function userTextEventLabel(kind) {
|
|
@@ -490,6 +609,8 @@ function userTextEventLabel(kind) {
|
|
|
490
609
|
return 'skill context';
|
|
491
610
|
if (kind === 'runtime_context')
|
|
492
611
|
return 'runtime context';
|
|
612
|
+
if (kind === 'synthetic_user_event')
|
|
613
|
+
return 'synthetic user event';
|
|
493
614
|
return 'user message';
|
|
494
615
|
}
|
|
495
616
|
function observationEvidenceRef(item) {
|
|
@@ -526,13 +647,13 @@ function evidenceRefFromTimeline(event) {
|
|
|
526
647
|
}
|
|
527
648
|
function evidenceChainForTimeline(timeline, observationRefs) {
|
|
528
649
|
const events = uniqueTimelineEvents(timeline).sort(compareTimelineEvents);
|
|
529
|
-
const userEvents = events.filter((event) => event.kind === 'user_message');
|
|
650
|
+
const userEvents = events.filter((event) => event.kind === 'user_message' && !isSyntheticUserMessageText(event.snippet ?? ''));
|
|
530
651
|
const runtimeEvents = events.filter((event) => event.kind === 'runtime_context');
|
|
531
652
|
const skillEvents = events.filter((event) => event.kind === 'skill_context');
|
|
532
653
|
const assistantEvents = events.filter((event) => event.kind === 'assistant_message');
|
|
533
654
|
const toolUseEvents = events.filter((event) => event.kind === 'tool_use');
|
|
534
655
|
const toolResultEvents = events.filter((event) => event.kind === 'tool_result');
|
|
535
|
-
const toolFailureEvents = toolResultEvents.filter(
|
|
656
|
+
const toolFailureEvents = toolResultEvents.filter(isToolFailureEvent);
|
|
536
657
|
return {
|
|
537
658
|
userMessageCount: userEvents.length,
|
|
538
659
|
runtimeContextCount: runtimeEvents.length,
|
|
@@ -550,10 +671,10 @@ function evidenceChainForTimeline(timeline, observationRefs) {
|
|
|
550
671
|
lastAssistantMessage: assistantEvents.at(-1) ? evidenceRefFromTimeline(assistantEvents.at(-1)) : undefined,
|
|
551
672
|
};
|
|
552
673
|
}
|
|
553
|
-
function ruleFindingsForEvidence(indicators, timeline, observationRefs, evidenceChain, reviewState) {
|
|
674
|
+
function ruleFindingsForEvidence(indicators, timeline, observationRefs, evidenceChain, metricScopeId, reviewState) {
|
|
554
675
|
const events = uniqueTimelineEvents(timeline);
|
|
555
676
|
const userEvents = events.filter((event) => event.kind === 'user_message');
|
|
556
|
-
const metricUserEvents = userEvents.filter((event) =>
|
|
677
|
+
const metricUserEvents = userEvents.filter((event) => isUserInteractionMetricText(event.snippet ?? ''));
|
|
557
678
|
const refs = (matches) => matches.slice(0, 5).map(evidenceRefFromTimeline);
|
|
558
679
|
const findings = [];
|
|
559
680
|
const push = (code, level, count, evidenceRefs = []) => {
|
|
@@ -562,16 +683,17 @@ function ruleFindingsForEvidence(indicators, timeline, observationRefs, evidence
|
|
|
562
683
|
findings.push({ code, level, count, evidenceRefs: uniqueEvidenceRefs(evidenceRefs).slice(0, 5) });
|
|
563
684
|
};
|
|
564
685
|
push('high_observation_seen', 'attention', indicators.highObservationCount, observationRefs);
|
|
565
|
-
push('user_correction_seen', 'attention', indicators.userCorrectionCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_correction', hasUserCorrectionSignal(event.snippet ?? ''), reviewState))));
|
|
566
|
-
push('user_interruption_seen', 'attention', indicators.userInterruptionCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_interruption', USER_INTERRUPTION_RE.test(event.snippet ?? ''), reviewState))));
|
|
686
|
+
push('user_correction_seen', 'attention', indicators.userCorrectionCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_correction', hasUserCorrectionSignal(event.snippet ?? ''), reviewState, metricScopeId))));
|
|
687
|
+
push('user_interruption_seen', 'attention', indicators.userInterruptionCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_interruption', USER_INTERRUPTION_RE.test(event.snippet ?? ''), reviewState, metricScopeId))));
|
|
688
|
+
push('session_interrupted_seen', 'attention', indicators.sessionInterruptedCount, refs(sessionInterruptedEvents(events)));
|
|
567
689
|
push('negative_feedback_seen', 'attention', indicators.negativeFeedbackCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'negative_feedback', hasNegativeFeedbackSignal(event.snippet ?? ''), reviewState))));
|
|
568
|
-
push('tool_failure_seen', 'sample', indicators.toolFailureCount, refs(events.filter(
|
|
690
|
+
push('tool_failure_seen', 'sample', indicators.toolFailureCount, refs(events.filter(isToolFailureEvent)));
|
|
569
691
|
push('medium_observation_seen', 'sample', indicators.mediumObservationCount, observationRefs);
|
|
570
692
|
push('hedging_seen', 'sample', indicators.hedgingCount, observationRefs);
|
|
571
693
|
push('explicit_marker_seen', 'sample', indicators.explicitMarkerCount, observationRefs);
|
|
572
|
-
push('hard_rule_seen', 'sample', indicators.hardRuleTextHitCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'hard_rule', hasUserHardRuleText(event.snippet ?? ''), reviewState))));
|
|
694
|
+
push('hard_rule_seen', 'sample', indicators.hardRuleTextHitCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'hard_rule', hasUserHardRuleText(event.snippet ?? ''), reviewState, metricScopeId))));
|
|
573
695
|
push('positive_feedback_seen', 'normal', indicators.positiveFeedbackCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'positive_feedback', hasPositiveFeedbackSignal(event.snippet ?? ''), reviewState))));
|
|
574
|
-
push('user_goal_shift_seen', 'normal', indicators.userGoalShiftCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_goal_shift', hasUserGoalShiftSignal(event.snippet ?? ''), reviewState))));
|
|
696
|
+
push('user_goal_shift_seen', 'normal', indicators.userGoalShiftCount, refs(metricUserEvents.filter((event) => metricIsActive(event, 'user_goal_shift', hasUserGoalShiftSignal(event.snippet ?? ''), reviewState, metricScopeId))));
|
|
575
697
|
push('runtime_context_excluded', 'normal', evidenceChain.runtimeContextCount, evidenceChain.firstRuntimeContext ? [evidenceChain.firstRuntimeContext] : []);
|
|
576
698
|
push('skill_context_excluded', 'normal', evidenceChain.skillContextCount, evidenceChain.firstSkillContext ? [evidenceChain.firstSkillContext] : []);
|
|
577
699
|
if (findings.filter((finding) => finding.level !== 'normal').length === 0) {
|
|
@@ -642,48 +764,73 @@ function assistiveInferenceForEvidence(indicators, evidenceChain, ruleFindings)
|
|
|
642
764
|
evidenceRefs,
|
|
643
765
|
};
|
|
644
766
|
}
|
|
645
|
-
function indicatorsForSegment(segment, relatedItems, timeline, reviewState) {
|
|
767
|
+
function indicatorsForSegment(segment, relatedItems, timeline, metricScopeId, reviewState) {
|
|
646
768
|
const userRefs = timeline.filter((event) => event.kind === 'user_message');
|
|
647
|
-
const humanUserRefs = userRefs.filter((ref) => Boolean(ref.snippet));
|
|
648
|
-
const interactionUserRefs = humanUserRefs.filter((ref) =>
|
|
769
|
+
const humanUserRefs = userRefs.filter((ref) => Boolean(ref.snippet) && !isSyntheticUserMessageText(ref.snippet ?? ''));
|
|
770
|
+
const interactionUserRefs = humanUserRefs.filter((ref) => isUserInteractionMetricText(ref.snippet ?? ''));
|
|
649
771
|
return {
|
|
650
772
|
userMessageCount: humanUserRefs.length,
|
|
651
|
-
userFollowUpCount: interactionUserRefs.reduce((sum, ref, index) => sum + (metricIsActive(ref, 'user_follow_up', index > 0, reviewState) ? 1 : 0), 0),
|
|
652
|
-
userCorrectionCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'user_correction', findUserCorrectionMatches(ref.snippet ?? '').length, reviewState), 0),
|
|
653
|
-
userInterruptionCount: interactionUserRefs.reduce((sum, ref) => sum + (metricIsActive(ref, 'user_interruption', USER_INTERRUPTION_RE.test(ref.snippet ?? ''), reviewState) ? 1 : 0), 0),
|
|
773
|
+
userFollowUpCount: interactionUserRefs.reduce((sum, ref, index) => sum + (metricIsActive(ref, 'user_follow_up', index > 0 && !hasUserGoalShiftSignal(ref.snippet ?? ''), reviewState, metricScopeId) ? 1 : 0), 0),
|
|
774
|
+
userCorrectionCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'user_correction', findUserCorrectionMatches(ref.snippet ?? '').length, reviewState, metricScopeId), 0),
|
|
775
|
+
userInterruptionCount: interactionUserRefs.reduce((sum, ref) => sum + (metricIsActive(ref, 'user_interruption', USER_INTERRUPTION_RE.test(ref.snippet ?? ''), reviewState, metricScopeId) ? 1 : 0), 0),
|
|
776
|
+
sessionInterruptedCount: sessionInterruptedEvents(timeline).length,
|
|
654
777
|
negativeFeedbackCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'negative_feedback', findNegativeFeedbackMatches(ref.snippet ?? '').length, reviewState), 0),
|
|
655
778
|
positiveFeedbackCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'positive_feedback', findPositiveFeedbackMatches(ref.snippet ?? '').length, reviewState), 0),
|
|
656
|
-
userGoalShiftCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'user_goal_shift', findUserGoalShiftMatches(ref.snippet ?? '').length, reviewState), 0),
|
|
657
|
-
hardRuleTextHitCount: interactionUserRefs.reduce((sum, ref) => sum + (metricIsActive(ref, 'hard_rule', hasUserHardRuleText(ref.snippet ?? ''), reviewState) ? 1 : 0), 0),
|
|
779
|
+
userGoalShiftCount: interactionUserRefs.reduce((sum, ref) => sum + metricCount(ref, 'user_goal_shift', findUserGoalShiftMatches(ref.snippet ?? '').length, reviewState, metricScopeId), 0),
|
|
780
|
+
hardRuleTextHitCount: interactionUserRefs.reduce((sum, ref) => sum + (metricIsActive(ref, 'hard_rule', hasUserHardRuleText(ref.snippet ?? ''), reviewState, metricScopeId) ? 1 : 0), 0),
|
|
658
781
|
assistantDeliverySignalCount: timeline.filter(isAssistantDeliveryEvent).length,
|
|
782
|
+
deliverableArtifactSignalCount: timeline.reduce((sum, ref) => sum + (metricIsActive(ref, 'deliverable_artifact', isAssistantDeliverableArtifactEvent(ref), reviewState) ? 1 : 0), 0),
|
|
783
|
+
selfCorrectionCount: timeline.reduce((sum, ref) => sum + (metricIsActive(ref, 'self_correction', hasSelfCorrectionSignal(ref), reviewState) ? 1 : 0), 0),
|
|
784
|
+
repeatedExecutionCount: timeline.reduce((sum, ref) => sum + (metricIsActive(ref, 'repeated_execution', hasRepeatedExecutionSignal(ref), reviewState) ? 1 : 0), 0),
|
|
659
785
|
toolCallCount: segment.metrics.numToolCalls,
|
|
660
|
-
toolFailureCount: segment.metrics.numToolFailures,
|
|
786
|
+
toolFailureCount: Math.max(segment.metrics.numToolFailures, timeline.filter(isToolFailureEvent).length),
|
|
661
787
|
highObservationCount: relatedItems.filter((item) => item.severity === 'high').length,
|
|
662
788
|
mediumObservationCount: relatedItems.filter((item) => item.severity === 'medium').length,
|
|
663
789
|
hedgingCount: relatedItems.filter((item) => item.signalType === 'hedging').reduce((sum, item) => sum + item.occurrences, 0),
|
|
664
790
|
explicitMarkerCount: relatedItems.filter((item) => item.signalType === 'explicit_marker').reduce((sum, item) => sum + item.occurrences, 0),
|
|
665
791
|
};
|
|
666
792
|
}
|
|
667
|
-
function
|
|
668
|
-
|
|
793
|
+
function isToolFailureEvent(event) {
|
|
794
|
+
return event.kind === 'tool_result' && (event.isError === true || isToolResultFailureText(event.fullText ?? event.snippet ?? ''));
|
|
795
|
+
}
|
|
796
|
+
function sessionInterruptedEvents(timeline) {
|
|
797
|
+
const events = uniqueTimelineEvents(timeline).sort(compareTimelineEvents);
|
|
798
|
+
const interrupted = [];
|
|
799
|
+
for (let index = 0; index < events.length; index += 1) {
|
|
800
|
+
const event = events[index];
|
|
801
|
+
const text = `${event.label ?? ''}\n${event.snippet ?? ''}\n${event.fullText ?? ''}`;
|
|
802
|
+
if (hasAssistantTurnFailedText(text)) {
|
|
803
|
+
interrupted.push(event);
|
|
804
|
+
continue;
|
|
805
|
+
}
|
|
806
|
+
if (/^session[._-]ended$/i.test(event.label ?? '')) {
|
|
807
|
+
const next = events.slice(index + 1).find((candidate) => /^session[._-]started$/i.test(candidate.label ?? ''));
|
|
808
|
+
if (next)
|
|
809
|
+
interrupted.push(event);
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
return uniqueTimelineEvents(interrupted);
|
|
813
|
+
}
|
|
814
|
+
function metricIsActive(ref, metricKey, ruleDetected, reviewState, metricScopeId) {
|
|
815
|
+
const verdict = observationMetricAnnotationVerdict(reviewState, { ...ref, metricScopeId }, metricKey);
|
|
669
816
|
if (verdict === 'confirmed')
|
|
670
817
|
return true;
|
|
671
818
|
if (verdict === 'rejected')
|
|
672
819
|
return false;
|
|
673
820
|
return ruleDetected;
|
|
674
821
|
}
|
|
675
|
-
function metricCount(ref, metricKey, ruleCount, reviewState) {
|
|
676
|
-
const verdict = observationMetricAnnotationVerdict(reviewState, ref, metricKey);
|
|
822
|
+
function metricCount(ref, metricKey, ruleCount, reviewState, metricScopeId) {
|
|
823
|
+
const verdict = observationMetricAnnotationVerdict(reviewState, { ...ref, metricScopeId }, metricKey);
|
|
677
824
|
if (verdict === 'confirmed')
|
|
678
825
|
return Math.max(1, ruleCount);
|
|
679
826
|
if (verdict === 'rejected')
|
|
680
827
|
return 0;
|
|
681
828
|
return ruleCount;
|
|
682
829
|
}
|
|
683
|
-
function summarizeExperienceSessions(invocations,
|
|
830
|
+
function summarizeExperienceSessions(invocations, sessionGroupsByKey, generatedAt, reviewState) {
|
|
684
831
|
const byKey = new Map();
|
|
685
832
|
for (const invocation of invocations) {
|
|
686
|
-
const key = `${invocation.skillName}\u0000${invocation.
|
|
833
|
+
const key = `${invocation.skillName}\u0000${invocation.sessionGroupKey}`;
|
|
687
834
|
const group = byKey.get(key) ?? [];
|
|
688
835
|
group.push(invocation);
|
|
689
836
|
byKey.set(key, group);
|
|
@@ -695,7 +842,7 @@ function summarizeExperienceSessions(invocations, sessionGroupsById) {
|
|
|
695
842
|
const reviewBasisCodes = basisCodesForIndicators(indicators);
|
|
696
843
|
const reviewPriorityScore = scoreForIndicators(indicators);
|
|
697
844
|
const timeline = uniqueTimelineEvents(group.flatMap((invocation) => invocation.timeline)).sort(compareTimelineEvents);
|
|
698
|
-
const sessionGroup =
|
|
845
|
+
const sessionGroup = sessionGroupsByKey.get(first.sessionGroupKey) ?? [];
|
|
699
846
|
const sourceSessionStartTimestamp = minString(sessionGroup.map((session) => session.startTimestamp));
|
|
700
847
|
const sourceSessionEndTimestamp = maxString(sessionGroup.map((session) => session.endTimestamp));
|
|
701
848
|
const timelineTree = sessionGroup.length > 0 ? buildSessionTimelineTree(first.sessionId, sessionGroup) : undefined;
|
|
@@ -727,11 +874,13 @@ function summarizeExperienceSessions(invocations, sessionGroupsById) {
|
|
|
727
874
|
: fullSessionTimeline.filter((event) => typeof event.messageIndex === 'number' && event.messageIndex > previewEndRecordIndex).length;
|
|
728
875
|
const observationRefs = uniqueEvidenceRefs(group.flatMap((invocation) => invocation.evidenceRefs.filter((ref) => ref.kind === 'observation')));
|
|
729
876
|
const evidenceChain = evidenceChainForTimeline(timeline, observationRefs);
|
|
730
|
-
const
|
|
877
|
+
const metricScopeId = hashParts('session', first.skillName, first.sessionGroupKey);
|
|
878
|
+
const ruleFindings = ruleFindingsForEvidence(indicators, timeline, observationRefs, evidenceChain, metricScopeId);
|
|
731
879
|
const assistiveInference = assistiveInferenceForEvidence(indicators, evidenceChain, ruleFindings);
|
|
732
880
|
const problemPatterns = mergeExperienceProblemPatterns(group.flatMap((invocation) => invocation.problemPatterns));
|
|
733
|
-
|
|
734
|
-
|
|
881
|
+
const storyInvocations = invocations.filter((invocation) => invocation.sessionGroupKey === first.sessionGroupKey);
|
|
882
|
+
const baseSession = {
|
|
883
|
+
id: metricScopeId,
|
|
735
884
|
skillName: first.skillName,
|
|
736
885
|
sessionId: first.sessionId,
|
|
737
886
|
sourceTrace: first.sourceTrace,
|
|
@@ -777,12 +926,1007 @@ function summarizeExperienceSessions(invocations, sessionGroupsById) {
|
|
|
777
926
|
rawSkillRefs: unique(group.map((invocation) => invocation.attribution.rawSkillRef).filter((value) => Boolean(value))).sort(),
|
|
778
927
|
commandNames: unique(group.map((invocation) => invocation.attribution.commandName).filter((value) => Boolean(value))).sort(),
|
|
779
928
|
};
|
|
929
|
+
const sessionStory = buildSessionStory(baseSession, storyInvocations);
|
|
930
|
+
const sessionWithStory = {
|
|
931
|
+
...baseSession,
|
|
932
|
+
sessionStory,
|
|
933
|
+
};
|
|
934
|
+
const reviewerReport = buildReviewerReport(sessionWithStory, group, generatedAt, reviewState, storyInvocations, sessionStory);
|
|
935
|
+
return {
|
|
936
|
+
...sessionWithStory,
|
|
937
|
+
reviewPriority: priorityForReviewerFindings(sessionWithStory, reviewerReport.findings),
|
|
938
|
+
reviewerReport,
|
|
939
|
+
};
|
|
780
940
|
}).sort((a, b) => {
|
|
941
|
+
const priorityDiff = experiencePriorityRank(b.reviewPriority) - experiencePriorityRank(a.reviewPriority);
|
|
942
|
+
if (priorityDiff !== 0)
|
|
943
|
+
return priorityDiff;
|
|
781
944
|
if (b.reviewPriorityScore !== a.reviewPriorityScore)
|
|
782
945
|
return b.reviewPriorityScore - a.reviewPriorityScore;
|
|
783
946
|
return b.endTimestamp.localeCompare(a.endTimestamp);
|
|
784
947
|
});
|
|
785
948
|
}
|
|
949
|
+
function experiencePriorityRank(priority) {
|
|
950
|
+
if (priority === 'review_first')
|
|
951
|
+
return 2;
|
|
952
|
+
if (priority === 'sample_review')
|
|
953
|
+
return 1;
|
|
954
|
+
return 0;
|
|
955
|
+
}
|
|
956
|
+
const REVIEWER_REPORT_RULE_VERSION = 'reviewer-report.v1';
|
|
957
|
+
function buildReviewerReport(session, invocations, generatedAt, reviewState, storyInvocations = invocations, sessionStory = buildSessionStory(session, storyInvocations)) {
|
|
958
|
+
const indicators = session.indicators;
|
|
959
|
+
const scopeReasons = reviewerScopeReasonCodes(session);
|
|
960
|
+
const scopeKind = scopeReasons.length === 0 ? 'single_skill_single_goal' : 'degraded_complex';
|
|
961
|
+
const findings = reviewerFindingsForSession(session, reviewState);
|
|
962
|
+
const attentionCount = findings.filter((finding) => finding.level === 'attention').length;
|
|
963
|
+
const possibleFalsePositiveCount = findings.filter((finding) => finding.level === 'possible_false_positive').length;
|
|
964
|
+
const tokenUsage = sumTokenUsage(invocations);
|
|
965
|
+
const title = reviewerTitle(session, attentionCount, possibleFalsePositiveCount);
|
|
966
|
+
const expectedToolCheck = expectedToolCheckForSession(session);
|
|
967
|
+
const traceLinks = uniqueEvidenceRefs([
|
|
968
|
+
...(session.evidenceChain.firstUserMessage ? [session.evidenceChain.firstUserMessage] : []),
|
|
969
|
+
...(session.evidenceChain.firstToolUse ? [session.evidenceChain.firstToolUse] : []),
|
|
970
|
+
...(session.evidenceChain.firstToolFailure ? [session.evidenceChain.firstToolFailure] : []),
|
|
971
|
+
...(session.evidenceChain.lastAssistantMessage ? [session.evidenceChain.lastAssistantMessage] : []),
|
|
972
|
+
...findings.flatMap((finding) => finding.evidenceRefs),
|
|
973
|
+
]).slice(0, 10);
|
|
974
|
+
return {
|
|
975
|
+
schemaVersion: 1,
|
|
976
|
+
mode: 'deterministic_session_story',
|
|
977
|
+
generatedAt,
|
|
978
|
+
title,
|
|
979
|
+
summary: reviewerSummary(session, scopeKind, attentionCount, possibleFalsePositiveCount),
|
|
980
|
+
scope: {
|
|
981
|
+
kind: scopeKind,
|
|
982
|
+
reasonCodes: scopeReasons,
|
|
983
|
+
},
|
|
984
|
+
chainSteps: [
|
|
985
|
+
reviewerStep(1, '用户期待', userGoalStepText(session), session.evidenceChain.firstUserMessage ? 'ok' : 'unknown', session.evidenceChain.firstUserMessage ? [session.evidenceChain.firstUserMessage] : []),
|
|
986
|
+
reviewerStep(2, '选择能力', skillSelectionStepText(session), 'ok', [session.evidenceChain.firstSkillContext, session.evidenceChain.firstToolUse].filter((ref) => Boolean(ref))),
|
|
987
|
+
reviewerStep(3, '执行流程', executionStepText(session, expectedToolCheck), executionStepStatus(session, expectedToolCheck), session.evidenceChain.firstToolUse ? [session.evidenceChain.firstToolUse] : []),
|
|
988
|
+
reviewerStep(4, '结果 / 产物', deliveryStepText(session), indicators.assistantDeliverySignalCount > 0 ? 'ok' : 'unknown', session.evidenceChain.lastAssistantMessage ? [session.evidenceChain.lastAssistantMessage] : []),
|
|
989
|
+
reviewerStep(5, '用户反馈', userFeedbackStepText(session), userFeedbackStepStatus(session), userFeedbackEvidenceRefs(session)),
|
|
990
|
+
],
|
|
991
|
+
findings,
|
|
992
|
+
oneLookMetrics: {
|
|
993
|
+
toolCallCount: indicators.toolCallCount,
|
|
994
|
+
toolFailureCount: indicators.toolFailureCount,
|
|
995
|
+
userMessageCount: indicators.userMessageCount,
|
|
996
|
+
userFollowUpCount: indicators.userFollowUpCount,
|
|
997
|
+
assistantDeliverySignalCount: indicators.assistantDeliverySignalCount,
|
|
998
|
+
deliverableArtifactSignalCount: indicators.deliverableArtifactSignalCount,
|
|
999
|
+
assistantProgressUpdateCount: assistantProgressUpdateEvents(session).length,
|
|
1000
|
+
selfCorrectionCount: indicators.selfCorrectionCount,
|
|
1001
|
+
repeatedExecutionCount: indicators.repeatedExecutionCount,
|
|
1002
|
+
finalDeliverySignalCount: assistantFinalDeliveryEvents(session).length,
|
|
1003
|
+
traceEventCount: session.fullSessionTimeline.length || session.timelinePreview.length,
|
|
1004
|
+
tokenUsage: {
|
|
1005
|
+
...tokenUsage,
|
|
1006
|
+
attribution: 'skill_segment',
|
|
1007
|
+
},
|
|
1008
|
+
},
|
|
1009
|
+
sessionStory,
|
|
1010
|
+
authorSuggestions: reviewerAuthorSuggestions(session, findings),
|
|
1011
|
+
traceLinks,
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
function buildSessionStory(session, invocations) {
|
|
1015
|
+
const nodes = [];
|
|
1016
|
+
const push = (kind, label, status, text, evidenceRefs = []) => {
|
|
1017
|
+
nodes.push({
|
|
1018
|
+
id: hashParts('session-story-node', session.id, kind, String(nodes.length), text),
|
|
1019
|
+
order: nodes.length + 1,
|
|
1020
|
+
kind,
|
|
1021
|
+
label,
|
|
1022
|
+
status,
|
|
1023
|
+
text,
|
|
1024
|
+
evidenceRefs: uniqueEvidenceRefs(evidenceRefs).slice(0, 5),
|
|
1025
|
+
});
|
|
1026
|
+
return nodes[nodes.length - 1];
|
|
1027
|
+
};
|
|
1028
|
+
const goalSlices = sessionStoryGoalSlices(session, invocations);
|
|
1029
|
+
const subagentDispatches = sessionStorySubagentDispatches(session);
|
|
1030
|
+
const skillLinks = sessionStorySkillLinks(session, invocations);
|
|
1031
|
+
const progressUpdates = assistantProgressUpdateEvents(session);
|
|
1032
|
+
const finalDeliveries = assistantFinalDeliveryEvents(session);
|
|
1033
|
+
const userGoalNode = push('user_goal', '用户提出目标', session.evidenceChain.firstUserMessage ? 'ok' : 'unknown', goalSlices.length > 1
|
|
1034
|
+
? `识别到 ${goalSlices.length} 个目标段:${goalSlices.map((goal) => goal.inferredUserGoal ?? '未提取到明确目标').slice(0, 3).join(';')}${goalSlices.length > 3 ? ';...' : ''}`
|
|
1035
|
+
: session.evidenceChain.firstUserMessage?.snippet
|
|
1036
|
+
? `用户目标:${session.evidenceChain.firstUserMessage.snippet}`
|
|
1037
|
+
: '没有看到明确人工用户目标;当前只能按运行证据还原链路。', session.evidenceChain.firstUserMessage ? [session.evidenceChain.firstUserMessage] : []);
|
|
1038
|
+
const roleSummary = skillLinks.map((link) => `${link.skillName}:${skillRoleLabel(link.role)}`).join(';');
|
|
1039
|
+
const invocationText = skillLinks.length > 1
|
|
1040
|
+
? `本次链路识别到 ${skillLinks.length} 个能力:${roleSummary}。`
|
|
1041
|
+
: skillLinks[0]
|
|
1042
|
+
? `本次使用能力:${skillLinks[0].skillName},角色判断:${skillRoleLabel(skillLinks[0].role)}。`
|
|
1043
|
+
: `本次使用能力:${session.skillName}。`;
|
|
1044
|
+
const invocationNode = push('skill_invocation', '能力介入', 'ok', invocationText, uniqueEvidenceRefs([session.evidenceChain.firstSkillContext, session.evidenceChain.firstToolUse].filter((ref) => Boolean(ref))));
|
|
1045
|
+
let subagentNode;
|
|
1046
|
+
if (subagentDispatches.length > 0) {
|
|
1047
|
+
subagentNode = push('subagent_branch', '分支 / 子任务', 'unknown', `检测到 ${subagentDispatches.length} 条分支或子任务执行线;主线和分支已单独列出,仍需要结合原文确认真实委派关系。`, subagentDispatches.flatMap((dispatch) => dispatch.evidenceRefs.slice(0, 1)));
|
|
1048
|
+
}
|
|
1049
|
+
const executionNode = push('tool_execution', '执行过程', session.indicators.toolFailureCount > 0 ? 'attention' : session.indicators.toolCallCount > 0 ? 'ok' : 'unknown', session.indicators.toolCallCount > 0
|
|
1050
|
+
? `执行中看到 ${session.indicators.toolCallCount} 次工具调用${session.indicators.toolFailureCount > 0 ? `,其中失败 ${session.indicators.toolFailureCount} 次` : ''}。`
|
|
1051
|
+
: '没有看到明确工具调用;只能根据消息上下文复盘。', uniqueEvidenceRefs([
|
|
1052
|
+
session.evidenceChain.firstToolUse,
|
|
1053
|
+
session.evidenceChain.firstToolFailure,
|
|
1054
|
+
].filter((ref) => Boolean(ref))));
|
|
1055
|
+
const deliveryNode = push('delivery', '交付产物', finalDeliveries.length > 0 ? 'ok' : 'attention', deliveryStepText(session), uniqueEvidenceRefs([
|
|
1056
|
+
...finalDeliveries.slice(-2).map(evidenceRefFromTimeline),
|
|
1057
|
+
...progressUpdates.slice(-2).map(evidenceRefFromTimeline),
|
|
1058
|
+
session.evidenceChain.lastAssistantMessage,
|
|
1059
|
+
].filter((ref) => Boolean(ref))));
|
|
1060
|
+
const feedbackNode = push('user_feedback', '用户反馈', userFeedbackStepStatus(session), userFeedbackStepText(session), userFeedbackEvidenceRefs(session));
|
|
1061
|
+
if (session.indicators.userGoalShiftCount > 0) {
|
|
1062
|
+
push('goal_shift', '目标切换', 'unknown', `用户中途切换了 ${session.indicators.userGoalShiftCount} 次目标,后续诉求可能不属于当前 skill。`, userFeedbackEvidenceRefs(session));
|
|
1063
|
+
}
|
|
1064
|
+
const goalChecklistItems = checklistItemsForAnswer(session, 'goal_satisfaction');
|
|
1065
|
+
const declaredBehaviorChecklistItems = checklistItemsForAnswer(session, 'declared_behavior_fit');
|
|
1066
|
+
const userFeelingChecklistItems = checklistItemsForAnswer(session, 'user_feeling');
|
|
1067
|
+
const answers = [
|
|
1068
|
+
sessionStoryAnswer('goal_satisfaction', '用户目标有没有被满足', goalChecklistItems, [
|
|
1069
|
+
session.evidenceChain.firstUserMessage,
|
|
1070
|
+
session.evidenceChain.lastAssistantMessage,
|
|
1071
|
+
...userFeedbackEvidenceRefs(session),
|
|
1072
|
+
]),
|
|
1073
|
+
sessionStoryAnswer('declared_behavior_fit', '行为是否符合能力用途', declaredBehaviorChecklistItems, [
|
|
1074
|
+
session.evidenceChain.firstSkillContext,
|
|
1075
|
+
session.evidenceChain.firstToolUse,
|
|
1076
|
+
session.evidenceChain.firstToolFailure,
|
|
1077
|
+
]),
|
|
1078
|
+
sessionStoryAnswer('user_feeling', '用户是否觉得有用或绕路', userFeelingChecklistItems, userFeedbackEvidenceRefs(session)),
|
|
1079
|
+
];
|
|
1080
|
+
const summary = answers.some((answer) => answer.status === 'degraded')
|
|
1081
|
+
? '这次数据本身有可信度问题(比如 skill 没真的被加载),先确认数据再下判断。'
|
|
1082
|
+
: answers.some((answer) => answer.status === 'attention')
|
|
1083
|
+
? '这次有几条需要看一眼的事项,从红色标记的事实和原文开始看。'
|
|
1084
|
+
: answers.every((answer) => answer.status === 'ok')
|
|
1085
|
+
? '这次从目标、执行到反馈都没有明显异常,进入常规抽样。'
|
|
1086
|
+
: '这次链路已按语义节点展开,但部分结论仍需要人工结合原文判断。';
|
|
1087
|
+
return {
|
|
1088
|
+
schemaVersion: 1,
|
|
1089
|
+
summary,
|
|
1090
|
+
invocationCount: invocations.length,
|
|
1091
|
+
goalSliceCount: session.goalSliceIds.length,
|
|
1092
|
+
branchCount: subagentDispatches.length,
|
|
1093
|
+
progressUpdateCount: progressUpdates.length,
|
|
1094
|
+
finalDeliverySignalCount: finalDeliveries.length,
|
|
1095
|
+
mainlineNodeIds: [
|
|
1096
|
+
userGoalNode.id,
|
|
1097
|
+
invocationNode.id,
|
|
1098
|
+
...(subagentNode ? [subagentNode.id] : []),
|
|
1099
|
+
executionNode.id,
|
|
1100
|
+
deliveryNode.id,
|
|
1101
|
+
feedbackNode.id,
|
|
1102
|
+
],
|
|
1103
|
+
goalSlices,
|
|
1104
|
+
subagentDispatches,
|
|
1105
|
+
skillLinks,
|
|
1106
|
+
graph: sessionStoryGraph(nodes, skillLinks),
|
|
1107
|
+
nodes,
|
|
1108
|
+
answers,
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
function assistantFinalDeliveryEvents(session) {
|
|
1112
|
+
return (session.fullSessionTimeline.length > 0 ? session.fullSessionTimeline : session.timelinePreview)
|
|
1113
|
+
.filter(isAssistantDeliveryEvent);
|
|
1114
|
+
}
|
|
1115
|
+
function assistantProgressUpdateEvents(session) {
|
|
1116
|
+
return (session.fullSessionTimeline.length > 0 ? session.fullSessionTimeline : session.timelinePreview)
|
|
1117
|
+
.filter(isAssistantProgressUpdateEvent);
|
|
1118
|
+
}
|
|
1119
|
+
function sessionStoryGoalSlices(session, invocations) {
|
|
1120
|
+
const byId = new Map();
|
|
1121
|
+
for (const invocation of invocations) {
|
|
1122
|
+
const group = byId.get(invocation.goalSliceId) ?? [];
|
|
1123
|
+
group.push(invocation);
|
|
1124
|
+
byId.set(invocation.goalSliceId, group);
|
|
1125
|
+
}
|
|
1126
|
+
return Array.from(byId.entries()).map(([id, group], index) => {
|
|
1127
|
+
const timeline = uniqueTimelineEvents(group.flatMap((invocation) => invocation.timeline)).sort(compareTimelineEvents);
|
|
1128
|
+
const userEvents = timeline.filter((event) => event.kind === 'user_message');
|
|
1129
|
+
const startTimestamp = minString(group.map((invocation) => invocation.startTimestamp)) ?? session.startTimestamp;
|
|
1130
|
+
const endTimestamp = maxString(group.map((invocation) => invocation.endTimestamp)) ?? session.endTimestamp;
|
|
1131
|
+
const hasGoalShift = userEvents.some((event) => hasUserGoalShiftSignal(event.snippet ?? ''));
|
|
1132
|
+
const reasonCode = hasGoalShift
|
|
1133
|
+
? 'explicit_user_goal_shift'
|
|
1134
|
+
: group.length > 1 ? 'skill_segment_boundary' : 'default_session_slice';
|
|
1135
|
+
return {
|
|
1136
|
+
id,
|
|
1137
|
+
order: index + 1,
|
|
1138
|
+
skillNames: unique(group.map((invocation) => invocation.skillName)).sort(),
|
|
1139
|
+
startTimestamp,
|
|
1140
|
+
endTimestamp,
|
|
1141
|
+
reasonCode,
|
|
1142
|
+
inferredUserGoal: inferUserGoal(userEvents),
|
|
1143
|
+
evidenceRefs: userEvents.slice(0, 3).map(evidenceRefFromTimeline),
|
|
1144
|
+
};
|
|
1145
|
+
}).sort((a, b) => a.startTimestamp.localeCompare(b.startTimestamp));
|
|
1146
|
+
}
|
|
1147
|
+
function sessionStorySubagentDispatches(session) {
|
|
1148
|
+
const branches = session.timelineTree?.branches ?? [];
|
|
1149
|
+
return branches.map((branch, index) => ({
|
|
1150
|
+
id: hashParts('session-story-dispatch', session.id, branch.id),
|
|
1151
|
+
order: index + 1,
|
|
1152
|
+
branchId: branch.id,
|
|
1153
|
+
label: branch.label,
|
|
1154
|
+
sourceTrace: branch.sourceTrace,
|
|
1155
|
+
attachTo: branch.attachTo ? {
|
|
1156
|
+
messageIndex: branch.attachTo.messageIndex,
|
|
1157
|
+
toolUseId: branch.attachTo.toolUseId,
|
|
1158
|
+
label: branch.attachTo.label,
|
|
1159
|
+
} : undefined,
|
|
1160
|
+
eventCount: branch.events.length,
|
|
1161
|
+
evidenceRefs: branch.events.slice(0, 3).map(evidenceRefFromTimeline),
|
|
1162
|
+
}));
|
|
1163
|
+
}
|
|
1164
|
+
function sessionStorySkillLinks(session, invocations) {
|
|
1165
|
+
const bySkill = new Map();
|
|
1166
|
+
for (const invocation of invocations) {
|
|
1167
|
+
const group = bySkill.get(invocation.skillName) ?? [];
|
|
1168
|
+
group.push(invocation);
|
|
1169
|
+
bySkill.set(invocation.skillName, group);
|
|
1170
|
+
}
|
|
1171
|
+
return Array.from(bySkill.entries()).map(([skillName, group], index) => {
|
|
1172
|
+
const role = inferSkillRole(group, invocations, session);
|
|
1173
|
+
return {
|
|
1174
|
+
id: hashParts('session-story-skill-link', session.id, skillName),
|
|
1175
|
+
order: index + 1,
|
|
1176
|
+
skillName,
|
|
1177
|
+
role,
|
|
1178
|
+
invocationIds: group.map((invocation) => invocation.id),
|
|
1179
|
+
goalSliceIds: unique(group.map((invocation) => invocation.goalSliceId)),
|
|
1180
|
+
evidenceRefs: uniqueEvidenceRefs(group.flatMap((invocation) => [
|
|
1181
|
+
invocation.evidenceChain.firstSkillContext,
|
|
1182
|
+
invocation.evidenceChain.firstToolUse,
|
|
1183
|
+
...routingEvidenceEvents(invocation).map(evidenceRefFromTimeline),
|
|
1184
|
+
].filter((ref) => Boolean(ref)))).slice(0, 5),
|
|
1185
|
+
};
|
|
1186
|
+
}).sort((a, b) => a.order - b.order);
|
|
1187
|
+
}
|
|
1188
|
+
function inferSkillRole(group, allInvocations, session) {
|
|
1189
|
+
const hasRoutingSignal = group.some((invocation) => routingEvidenceEvents(invocation).length > 0);
|
|
1190
|
+
const hasBranchDispatch = (session.timelineTree?.branches.length ?? 0) > 0 && group.some((invocation) => invocation.timeline.some((event) => event.kind === 'tool_use' && /^(Task|Agent|Skill)$/i.test(event.toolName ?? '')));
|
|
1191
|
+
const hasExecutionSignal = group.some((invocation) => invocation.timeline.some((event) => event.kind === 'tool_use' && !/^(Task|Agent|Skill)$/i.test(event.toolName ?? '')));
|
|
1192
|
+
if ((hasRoutingSignal || hasBranchDispatch) && allInvocations.length > group.length)
|
|
1193
|
+
return hasExecutionSignal ? 'mixed' : 'router';
|
|
1194
|
+
if (hasRoutingSignal || hasBranchDispatch)
|
|
1195
|
+
return 'router';
|
|
1196
|
+
if (hasExecutionSignal)
|
|
1197
|
+
return 'executor';
|
|
1198
|
+
return 'unknown';
|
|
1199
|
+
}
|
|
1200
|
+
function routingEvidenceEvents(invocation) {
|
|
1201
|
+
return invocation.timeline.filter((event) => {
|
|
1202
|
+
if (event.kind !== 'assistant_message' && event.kind !== 'tool_use')
|
|
1203
|
+
return false;
|
|
1204
|
+
const text = event.fullText ?? event.snippet ?? '';
|
|
1205
|
+
return /子\s*Claude|subagent|子任务|分发|委派|调用.+skill|走\s*`?[\w-]+`?\s*skill|\/consult|task-runner/i.test(text);
|
|
1206
|
+
}).slice(0, 3);
|
|
1207
|
+
}
|
|
1208
|
+
function skillRoleLabel(role) {
|
|
1209
|
+
if (role === 'router')
|
|
1210
|
+
return '路由';
|
|
1211
|
+
if (role === 'executor')
|
|
1212
|
+
return '执行';
|
|
1213
|
+
if (role === 'mixed')
|
|
1214
|
+
return '路由 + 执行';
|
|
1215
|
+
return '未确认';
|
|
1216
|
+
}
|
|
1217
|
+
function sessionStoryGraph(nodes, skillLinks) {
|
|
1218
|
+
const graphNodes = nodes.map((node) => ({
|
|
1219
|
+
id: node.id,
|
|
1220
|
+
label: node.label,
|
|
1221
|
+
kind: node.kind,
|
|
1222
|
+
status: node.status,
|
|
1223
|
+
detailNodeId: node.id,
|
|
1224
|
+
}));
|
|
1225
|
+
for (const link of skillLinks) {
|
|
1226
|
+
graphNodes.push({
|
|
1227
|
+
id: link.id,
|
|
1228
|
+
label: `${link.skillName}(${skillRoleLabel(link.role)})`,
|
|
1229
|
+
kind: 'skill_invocation',
|
|
1230
|
+
status: link.role === 'unknown' ? 'unknown' : 'ok',
|
|
1231
|
+
role: link.role,
|
|
1232
|
+
});
|
|
1233
|
+
}
|
|
1234
|
+
const edges = [];
|
|
1235
|
+
for (let index = 1; index < nodes.length; index += 1) {
|
|
1236
|
+
edges.push({ fromId: nodes[index - 1].id, toId: nodes[index].id, label: '下一步' });
|
|
1237
|
+
}
|
|
1238
|
+
const goalNode = nodes.find((node) => node.kind === 'user_goal');
|
|
1239
|
+
const executionNode = nodes.find((node) => node.kind === 'tool_execution');
|
|
1240
|
+
if (goalNode && executionNode) {
|
|
1241
|
+
for (const link of skillLinks) {
|
|
1242
|
+
edges.push({ fromId: goalNode.id, toId: link.id, label: '触发' });
|
|
1243
|
+
edges.push({ fromId: link.id, toId: executionNode.id, label: skillRoleLabel(link.role) });
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
return { nodes: graphNodes, edges };
|
|
1247
|
+
}
|
|
1248
|
+
function sessionStoryAnswer(key, label, checklistItems, evidenceRefs) {
|
|
1249
|
+
const folded = foldExperienceChecklistItems(checklistItems);
|
|
1250
|
+
return {
|
|
1251
|
+
key,
|
|
1252
|
+
label,
|
|
1253
|
+
status: folded.status,
|
|
1254
|
+
reason: folded.reason,
|
|
1255
|
+
sourceItemKeys: folded.sourceItemKeys,
|
|
1256
|
+
text: sessionStoryAnswerText(key, folded.reason),
|
|
1257
|
+
evidenceRefs: uniqueEvidenceRefs(evidenceRefs.filter((ref) => Boolean(ref))).slice(0, 5),
|
|
1258
|
+
checklistItems,
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
function sessionStoryAnswerText(key, reason) {
|
|
1262
|
+
const texts = {
|
|
1263
|
+
goal_satisfaction: {
|
|
1264
|
+
data_degraded: '这次没看到 skill 真的被加载,先确认数据,再判断目标是否满足。',
|
|
1265
|
+
blocking_failed: '用户出现了不满或叫停,目标没真的满足。',
|
|
1266
|
+
attention_accumulated: '有几条要看一眼,打开原文确认目标是否真的达成。',
|
|
1267
|
+
unknown_dominant: '现有证据不够,判断不了目标是否满足。',
|
|
1268
|
+
all_passed: '关键信号都没问题,看起来目标已满足。',
|
|
1269
|
+
not_applicable: '当前场景不适合回答这个问题。',
|
|
1270
|
+
},
|
|
1271
|
+
declared_behavior_fit: {
|
|
1272
|
+
data_degraded: '没看到 skill 真的被加载,先确认这次任务是不是真的命中了你的 skill。',
|
|
1273
|
+
blocking_failed: '关键执行项没过,skill 行为需要优先看一眼。',
|
|
1274
|
+
attention_accumulated: 'SKILL.md 声明的标准流程、硬性规则或核心工具有几条要看一眼。',
|
|
1275
|
+
unknown_dominant: 'SKILL.md 声明不全或执行证据不够,判不出行为是否符合用途。',
|
|
1276
|
+
all_passed: '执行情况看起来符合 skill 声明的用途。',
|
|
1277
|
+
not_applicable: '当前场景不适合回答这个问题。',
|
|
1278
|
+
},
|
|
1279
|
+
user_feeling: {
|
|
1280
|
+
data_degraded: '这次数据本身有问题,用户感受判断先放一放。',
|
|
1281
|
+
blocking_failed: '看到用户不满或主动叫停,可能觉得没用或绕路了。',
|
|
1282
|
+
attention_accumulated: '看到用户纠正或追问,结合原文判断用户感受。',
|
|
1283
|
+
unknown_dominant: '用户没给明确反馈,判断不了是否觉得有用。',
|
|
1284
|
+
all_passed: '没看到明显不满,用户体感看起来正常。',
|
|
1285
|
+
not_applicable: '当前场景不适合回答这个问题。',
|
|
1286
|
+
},
|
|
1287
|
+
};
|
|
1288
|
+
return texts[key][reason];
|
|
1289
|
+
}
|
|
1290
|
+
function checklistItemsForAnswer(session, key) {
|
|
1291
|
+
if (key === 'goal_satisfaction')
|
|
1292
|
+
return goalSatisfactionChecklistItems(session);
|
|
1293
|
+
if (key === 'declared_behavior_fit')
|
|
1294
|
+
return declaredBehaviorChecklistItems(session);
|
|
1295
|
+
return userFeelingChecklistItems(session);
|
|
1296
|
+
}
|
|
1297
|
+
function attributionSourcesToLabel(sources) {
|
|
1298
|
+
if (sources.length === 0)
|
|
1299
|
+
return '未识别(旧数据可能没记录来源)';
|
|
1300
|
+
const map = {
|
|
1301
|
+
'skill-tool': 'assistant 调用 Skill 工具',
|
|
1302
|
+
'command-name': '用户用 slash command',
|
|
1303
|
+
'aima-cmd': '用户用 aima-cmd 块',
|
|
1304
|
+
'skill-script': '跑了 skills/<name>/scripts 脚本',
|
|
1305
|
+
'read-skill-md': 'LLM 主动 Read SKILL.md',
|
|
1306
|
+
unknown: '未知',
|
|
1307
|
+
};
|
|
1308
|
+
return sources.map((s) => map[s] ?? s).join(' + ');
|
|
1309
|
+
}
|
|
1310
|
+
export function isExperienceTraceInProgress(session) {
|
|
1311
|
+
if (session.indicators.assistantDeliverySignalCount > 0)
|
|
1312
|
+
return false;
|
|
1313
|
+
if (session.indicators.deliverableArtifactSignalCount > 0)
|
|
1314
|
+
return false;
|
|
1315
|
+
// 用户已经主动表达不满(纠正 / 负向 / 中断)属于「被打断」, 不是「在途中」, 仍需触发 final_delivery_absent。
|
|
1316
|
+
if (session.indicators.userCorrectionCount > 0)
|
|
1317
|
+
return false;
|
|
1318
|
+
if (session.indicators.negativeFeedbackCount > 0)
|
|
1319
|
+
return false;
|
|
1320
|
+
if (session.indicators.userInterruptionCount > 0)
|
|
1321
|
+
return false;
|
|
1322
|
+
const last = session.evidenceChain.lastAssistantMessage?.snippet ?? '';
|
|
1323
|
+
// 完全没有最后助手回复时不强判 in-progress, 走原 failed / unknown 路径。
|
|
1324
|
+
if (!last.trim())
|
|
1325
|
+
return false;
|
|
1326
|
+
if (isAssistantProgressUpdateText(last))
|
|
1327
|
+
return true;
|
|
1328
|
+
// 预备语(让我看 / 先看看 / 先读 / 接下来)也算 in-progress, 仍未给出最终回复。
|
|
1329
|
+
return /让我(?:先|来|看|读|检查|分析|确认|拉|获取|继续)|先(?:看(?:看|一?下)|读(?:取|一下)?|确认|检查|分析|拉取|获取)|接下来/i.test(last);
|
|
1330
|
+
}
|
|
1331
|
+
function checklistItem(input) {
|
|
1332
|
+
const { statusCandidates, ...item } = input;
|
|
1333
|
+
return {
|
|
1334
|
+
...item,
|
|
1335
|
+
status: aggregateExperienceChecklistItemStatus([input.status, ...(statusCandidates ?? [])]),
|
|
1336
|
+
source: input.source ?? 'deterministic_rule',
|
|
1337
|
+
evidenceRefs: uniqueEvidenceRefs((input.evidenceRefs ?? []).filter((ref) => Boolean(ref))).slice(0, 5),
|
|
1338
|
+
};
|
|
1339
|
+
}
|
|
1340
|
+
export function hasRecognizableUserGoalText(value) {
|
|
1341
|
+
const text = value?.trim() ?? '';
|
|
1342
|
+
if (!text)
|
|
1343
|
+
return false;
|
|
1344
|
+
if (/^(嗯+|啊+|好的|好|继续|可以|收到|ok|OK|yes|no|不用|不需要)[。.!!??\s]*$/.test(text))
|
|
1345
|
+
return false;
|
|
1346
|
+
if (/(帮我|请|需要|想要|我要|给我|看下|看一下|基于|根据|重新|继续|先|把|将)/.test(text)
|
|
1347
|
+
&& /(生成|创建|写|实现|修复|优化|调整|修改|新增|删除|分析|review|评价|检查|看|拉取|执行|运行|验证|整理|总结|回复|评论|设计|拆分|合并|标注|定位|排查|上传|导出|发布|查询|对齐|沉淀|补充|改|做)/i.test(text)) {
|
|
1348
|
+
return true;
|
|
1349
|
+
}
|
|
1350
|
+
if (/(生成|创建|写一个|实现|修复|优化|调整|修改|新增|分析|review|检查|排查|验证|总结|设计|标注|定位|查询|对齐|补充|改一下|做一个)/i.test(text))
|
|
1351
|
+
return true;
|
|
1352
|
+
return text.length >= 12 && /[??]/.test(text) && !/^(为什么|怎么|哪里)[??]?$/.test(text);
|
|
1353
|
+
}
|
|
1354
|
+
function hasRecognizableUserGoal(ref) {
|
|
1355
|
+
return hasRecognizableUserGoalText(ref?.snippet);
|
|
1356
|
+
}
|
|
1357
|
+
function goalSatisfactionChecklistItems(session) {
|
|
1358
|
+
const feedbackRefs = userFeedbackEvidenceRefs(session);
|
|
1359
|
+
const goalIdentified = hasRecognizableUserGoal(session.evidenceChain.firstUserMessage);
|
|
1360
|
+
const inProgress = isExperienceTraceInProgress(session);
|
|
1361
|
+
const hasDelivery = session.indicators.assistantDeliverySignalCount > 0;
|
|
1362
|
+
const hasArtifact = session.indicators.deliverableArtifactSignalCount > 0;
|
|
1363
|
+
return [
|
|
1364
|
+
checklistItem({
|
|
1365
|
+
key: 'goal_identified',
|
|
1366
|
+
label: goalIdentified ? '目标已识别' : '目标不明确',
|
|
1367
|
+
status: goalIdentified ? 'passed' : 'unknown',
|
|
1368
|
+
contribution: 'informational',
|
|
1369
|
+
reason: goalIdentified
|
|
1370
|
+
? '真实用户原文里能识别出目标动作或明确请求。'
|
|
1371
|
+
: session.evidenceChain.firstUserMessage ? '看到真实用户原文,但目标动作不够明确。' : '没有看到真实用户目标原文。',
|
|
1372
|
+
evidenceRefs: [session.evidenceChain.firstUserMessage],
|
|
1373
|
+
}),
|
|
1374
|
+
checklistItem({
|
|
1375
|
+
key: 'completion_result_present',
|
|
1376
|
+
label: hasDelivery ? '给了用户最终答复' : inProgress ? '会话进行中' : '没给用户最终答复',
|
|
1377
|
+
status: hasDelivery ? 'passed' : inProgress ? 'not_applicable' : 'failed',
|
|
1378
|
+
contribution: hasDelivery ? 'attention' : inProgress ? 'neutral' : 'attention',
|
|
1379
|
+
reason: hasDelivery
|
|
1380
|
+
? '看到 assistant 给出了明确的完成话术或结果反馈。'
|
|
1381
|
+
: inProgress
|
|
1382
|
+
? '最后一句还是过程态(「先看看」「让我」),任务还没收尾,先不判定。'
|
|
1383
|
+
: '没看到 assistant 给用户明确的完成话术或结果反馈。',
|
|
1384
|
+
evidenceRefs: [session.evidenceChain.lastAssistantMessage],
|
|
1385
|
+
suggestionKey: hasDelivery || inProgress ? undefined : 'final_delivery_absent',
|
|
1386
|
+
}),
|
|
1387
|
+
checklistItem({
|
|
1388
|
+
key: 'deliverable_artifact_present',
|
|
1389
|
+
label: hasArtifact ? '给了可点开的产物' : inProgress ? '会话进行中' : '没给可点开的产物',
|
|
1390
|
+
status: hasArtifact ? 'passed' : inProgress ? 'not_applicable' : 'unknown',
|
|
1391
|
+
contribution: hasArtifact ? 'informational' : inProgress ? 'neutral' : 'informational',
|
|
1392
|
+
reason: hasArtifact
|
|
1393
|
+
? '看到 assistant 回复里附了链接、路径、代码块或文件。'
|
|
1394
|
+
: inProgress
|
|
1395
|
+
? '任务还没收尾,先不判定产物。'
|
|
1396
|
+
: '没看到明确的链接、路径、代码块或文件;不一定失败,得按 skill 目标判断。',
|
|
1397
|
+
evidenceRefs: [session.evidenceChain.lastAssistantMessage],
|
|
1398
|
+
suggestionKey: hasArtifact || inProgress ? undefined : 'artifact_absent',
|
|
1399
|
+
}),
|
|
1400
|
+
checklistItem({
|
|
1401
|
+
key: 'negative_feedback_seen',
|
|
1402
|
+
label: session.indicators.negativeFeedbackCount > 0 ? '看到用户负向反馈' : '未见用户负向反馈',
|
|
1403
|
+
status: session.indicators.negativeFeedbackCount > 0 ? 'failed' : 'passed',
|
|
1404
|
+
contribution: 'blocking',
|
|
1405
|
+
reason: session.indicators.negativeFeedbackCount > 0 ? '看到用户负向表达,不能直接认为目标已满足。' : '没有看到用户负向表达。',
|
|
1406
|
+
evidenceRefs: feedbackRefs,
|
|
1407
|
+
suggestionKey: session.indicators.negativeFeedbackCount > 0 ? 'negative_feedback_review' : undefined,
|
|
1408
|
+
}),
|
|
1409
|
+
checklistItem({
|
|
1410
|
+
key: 'user_correction_seen',
|
|
1411
|
+
label: session.indicators.userCorrectionCount > 0 ? '看到用户纠正' : '未见用户纠正',
|
|
1412
|
+
status: session.indicators.userCorrectionCount > 0 ? 'failed' : 'passed',
|
|
1413
|
+
contribution: 'attention',
|
|
1414
|
+
reason: session.indicators.userCorrectionCount > 0 ? '用户中途纠正了方向,目标是否满足要打开原文看。' : '没有看到用户纠正。',
|
|
1415
|
+
evidenceRefs: feedbackRefs,
|
|
1416
|
+
suggestionKey: session.indicators.userCorrectionCount > 0 ? 'user_correction_review' : undefined,
|
|
1417
|
+
}),
|
|
1418
|
+
checklistItem({
|
|
1419
|
+
key: 'user_interruption_seen',
|
|
1420
|
+
label: session.indicators.userInterruptionCount > 0 ? '看到用户中断' : '未见用户中断',
|
|
1421
|
+
status: session.indicators.userInterruptionCount > 0 ? 'failed' : 'passed',
|
|
1422
|
+
contribution: 'blocking',
|
|
1423
|
+
reason: session.indicators.userInterruptionCount > 0 ? '看到用户中断或停止任务信号,不能认为执行链路自然完成。' : '没有看到用户中断信号。',
|
|
1424
|
+
evidenceRefs: feedbackRefs,
|
|
1425
|
+
suggestionKey: session.indicators.userInterruptionCount > 0 ? 'user_interruption_review' : undefined,
|
|
1426
|
+
}),
|
|
1427
|
+
checklistItem({
|
|
1428
|
+
key: 'goal_shift_seen',
|
|
1429
|
+
label: session.indicators.userGoalShiftCount > 0 ? '看到目标切换' : '未见目标切换',
|
|
1430
|
+
status: session.indicators.userGoalShiftCount > 0 ? 'failed' : 'passed',
|
|
1431
|
+
contribution: 'attention',
|
|
1432
|
+
reason: session.indicators.userGoalShiftCount > 0 ? '用户中途切换了目标,后续诉求可能不属于这个 skill。' : '没有看到目标切换。',
|
|
1433
|
+
evidenceRefs: feedbackRefs,
|
|
1434
|
+
suggestionKey: session.indicators.userGoalShiftCount > 0 ? 'goal_shift_review' : undefined,
|
|
1435
|
+
}),
|
|
1436
|
+
];
|
|
1437
|
+
}
|
|
1438
|
+
function declaredBehaviorChecklistItems(session) {
|
|
1439
|
+
const expectedToolCheck = expectedToolCheckForSession(session);
|
|
1440
|
+
const declarations = skillDeclarationCheckForSession(session);
|
|
1441
|
+
const hasSkillRead = session.evidenceChain.skillContextCount > 0;
|
|
1442
|
+
const attributionLabel = attributionSourcesToLabel(session.attributionSources ?? []);
|
|
1443
|
+
const items = [
|
|
1444
|
+
checklistItem({
|
|
1445
|
+
key: 'attribution_source',
|
|
1446
|
+
label: hasSkillRead
|
|
1447
|
+
? 'LLM 读了 SKILL.md'
|
|
1448
|
+
: `skill 判定来源:${attributionLabel}`,
|
|
1449
|
+
status: 'passed',
|
|
1450
|
+
contribution: 'informational',
|
|
1451
|
+
reason: hasSkillRead
|
|
1452
|
+
? `看到 ${session.evidenceChain.skillContextCount} 次 SKILL.md 加载事件,可作为能力归因证据。`
|
|
1453
|
+
: `日志里没看到 LLM 主动 Read SKILL.md。这次 skill 归因来自:${attributionLabel}。OpenClaw 脚本 / Skill 工具 / slash command 触发时这是常态,不代表 skill 没用上。`,
|
|
1454
|
+
evidenceRefs: [session.evidenceChain.firstSkillContext, session.evidenceChain.firstToolUse],
|
|
1455
|
+
}),
|
|
1456
|
+
checklistItem({
|
|
1457
|
+
key: 'workflow_declared',
|
|
1458
|
+
label: declarations.workflows.declared ? '标准流程已声明' : '标准流程未声明',
|
|
1459
|
+
status: declarations.workflows.declared ? 'passed' : 'not_declared',
|
|
1460
|
+
contribution: declarations.workflows.declared ? 'informational' : 'attention',
|
|
1461
|
+
reason: declarations.workflows.declared ? `SKILL.md 里声明了 ${declarations.workflows.count} 个标准流程节点。` : 'SKILL.md 没声明标准流程,运行时只能猜流程是否完整。',
|
|
1462
|
+
suggestionKey: declarations.workflows.declared ? undefined : 'workflow_not_declared',
|
|
1463
|
+
}),
|
|
1464
|
+
];
|
|
1465
|
+
if (declarations.workflows.declared) {
|
|
1466
|
+
const executed = session.indicators.toolCallCount > 0;
|
|
1467
|
+
items.push(checklistItem({
|
|
1468
|
+
key: 'workflow_executed',
|
|
1469
|
+
label: executed ? '标准流程已执行' : '标准流程未执行',
|
|
1470
|
+
status: executed ? 'unknown' : 'failed',
|
|
1471
|
+
contribution: 'attention',
|
|
1472
|
+
reason: executed ? '看到了工具调用,但是不是真的覆盖了完整流程,还要打开原文确认。' : '声明了标准流程,但没看到工具执行的证据。',
|
|
1473
|
+
evidenceRefs: [session.evidenceChain.firstToolUse],
|
|
1474
|
+
suggestionKey: 'workflow_execution_review',
|
|
1475
|
+
}));
|
|
1476
|
+
}
|
|
1477
|
+
items.push(checklistItem({
|
|
1478
|
+
key: 'hardrule_declared',
|
|
1479
|
+
label: declarations.hardRules.declared ? '硬性规则已声明' : '硬性规则未声明',
|
|
1480
|
+
status: declarations.hardRules.declared ? 'passed' : 'not_declared',
|
|
1481
|
+
contribution: declarations.hardRules.declared ? 'informational' : 'attention',
|
|
1482
|
+
reason: declarations.hardRules.declared ? `SKILL.md 里声明了 ${declarations.hardRules.count} 条硬性规则。` : 'SKILL.md 没声明硬性规则。',
|
|
1483
|
+
suggestionKey: declarations.hardRules.declared ? undefined : 'hardrule_not_declared',
|
|
1484
|
+
}));
|
|
1485
|
+
if (declarations.hardRules.declared) {
|
|
1486
|
+
items.push(checklistItem({
|
|
1487
|
+
key: 'hardrule_executed',
|
|
1488
|
+
label: '硬性规则执行情况需打开原文看',
|
|
1489
|
+
status: 'unknown',
|
|
1490
|
+
contribution: 'attention',
|
|
1491
|
+
reason: '声明了硬性规则,但当前规则没法完整证明每条都执行了,要打开原文看。',
|
|
1492
|
+
suggestionKey: 'hardrule_execution_review',
|
|
1493
|
+
}));
|
|
1494
|
+
}
|
|
1495
|
+
items.push(checklistItem({
|
|
1496
|
+
key: 'core_tools_declared',
|
|
1497
|
+
label: expectedToolCheck.declared ? '核心工具已声明' : '核心工具未声明',
|
|
1498
|
+
status: expectedToolCheck.declared ? 'passed' : 'not_declared',
|
|
1499
|
+
contribution: expectedToolCheck.declared ? 'informational' : 'attention',
|
|
1500
|
+
reason: expectedToolCheck.declared ? `SKILL.md 声明的核心工具:${expectedToolCheck.expectedTools.join('、')}。` : 'SKILL.md 没声明核心工具,分不出「真用上了 skill 工具」还是「只是随便调了个工具」。',
|
|
1501
|
+
suggestionKey: expectedToolCheck.declared ? undefined : 'expected_tools_not_declared',
|
|
1502
|
+
}));
|
|
1503
|
+
if (expectedToolCheck.declared) {
|
|
1504
|
+
const hit = expectedToolCheck.matchedTools.length > 0;
|
|
1505
|
+
items.push(checklistItem({
|
|
1506
|
+
key: 'core_tools_hit',
|
|
1507
|
+
label: hit ? '核心工具用上了' : '核心工具没用上',
|
|
1508
|
+
status: hit ? 'passed' : 'failed',
|
|
1509
|
+
contribution: 'blocking',
|
|
1510
|
+
reason: hit ? `用上了核心工具:${expectedToolCheck.matchedTools.join('、')}。` : '没用上 SKILL.md 声明的核心工具。',
|
|
1511
|
+
evidenceRefs: [session.evidenceChain.firstToolUse],
|
|
1512
|
+
suggestionKey: hit ? undefined : 'expected_tools_missed',
|
|
1513
|
+
}));
|
|
1514
|
+
}
|
|
1515
|
+
return items;
|
|
1516
|
+
}
|
|
1517
|
+
function userFeelingChecklistItems(session) {
|
|
1518
|
+
const feedbackRefs = userFeedbackEvidenceRefs(session);
|
|
1519
|
+
const hasAnyFeedback = session.indicators.positiveFeedbackCount > 0
|
|
1520
|
+
|| session.indicators.negativeFeedbackCount > 0
|
|
1521
|
+
|| session.indicators.userCorrectionCount > 0
|
|
1522
|
+
|| session.indicators.userInterruptionCount > 0
|
|
1523
|
+
|| session.indicators.userFollowUpCount > 0
|
|
1524
|
+
|| session.indicators.userGoalShiftCount > 0;
|
|
1525
|
+
return [
|
|
1526
|
+
checklistItem({
|
|
1527
|
+
key: 'user_feedback_signal_present',
|
|
1528
|
+
label: hasAnyFeedback ? '看到用户反馈信号' : '未见用户反馈信号',
|
|
1529
|
+
status: hasAnyFeedback ? 'passed' : 'unknown',
|
|
1530
|
+
contribution: 'informational',
|
|
1531
|
+
reason: hasAnyFeedback ? '看到至少一种用户反馈或后续行为信号。' : '没有看到明确用户反馈信号。',
|
|
1532
|
+
evidenceRefs: feedbackRefs,
|
|
1533
|
+
}),
|
|
1534
|
+
checklistItem({
|
|
1535
|
+
key: 'positive_feedback_seen',
|
|
1536
|
+
label: session.indicators.positiveFeedbackCount > 0 ? '看到用户正向反馈' : '未见用户正向反馈',
|
|
1537
|
+
status: session.indicators.positiveFeedbackCount > 0 ? 'passed' : 'unknown',
|
|
1538
|
+
contribution: session.indicators.positiveFeedbackCount > 0 ? 'positive' : 'neutral',
|
|
1539
|
+
reason: session.indicators.positiveFeedbackCount > 0 ? '看到用户认可或正向反馈。' : '没有看到明确正向反馈。',
|
|
1540
|
+
evidenceRefs: feedbackRefs,
|
|
1541
|
+
}),
|
|
1542
|
+
checklistItem({
|
|
1543
|
+
key: 'negative_feedback_seen',
|
|
1544
|
+
label: session.indicators.negativeFeedbackCount > 0 ? '看到用户负向反馈' : '未见用户负向反馈',
|
|
1545
|
+
status: session.indicators.negativeFeedbackCount > 0 ? 'failed' : 'passed',
|
|
1546
|
+
contribution: session.indicators.negativeFeedbackCount > 0 ? 'blocking' : 'neutral',
|
|
1547
|
+
reason: session.indicators.negativeFeedbackCount > 0 ? '看到用户负向表达。' : '没有看到用户负向表达。',
|
|
1548
|
+
evidenceRefs: feedbackRefs,
|
|
1549
|
+
suggestionKey: session.indicators.negativeFeedbackCount > 0 ? 'negative_feedback_review' : undefined,
|
|
1550
|
+
}),
|
|
1551
|
+
checklistItem({
|
|
1552
|
+
key: 'user_correction_seen',
|
|
1553
|
+
label: session.indicators.userCorrectionCount > 0 ? '看到用户纠正' : '未见用户纠正',
|
|
1554
|
+
status: session.indicators.userCorrectionCount > 0 ? 'failed' : 'passed',
|
|
1555
|
+
contribution: session.indicators.userCorrectionCount > 0 ? 'attention' : 'neutral',
|
|
1556
|
+
reason: session.indicators.userCorrectionCount > 0 ? '看到用户重新解释或要求修正。' : '没有看到用户纠正信号。',
|
|
1557
|
+
evidenceRefs: feedbackRefs,
|
|
1558
|
+
suggestionKey: session.indicators.userCorrectionCount > 0 ? 'user_correction_review' : undefined,
|
|
1559
|
+
}),
|
|
1560
|
+
checklistItem({
|
|
1561
|
+
key: 'user_follow_up_seen',
|
|
1562
|
+
label: session.indicators.userFollowUpCount > 0 ? '看到用户追问' : '未见用户追问',
|
|
1563
|
+
status: session.indicators.userFollowUpCount > 0 ? 'unknown' : 'passed',
|
|
1564
|
+
contribution: session.indicators.userFollowUpCount > 0 ? 'informational' : 'neutral',
|
|
1565
|
+
reason: session.indicators.userFollowUpCount > 0 ? '看到用户追问;需要结合上下文区分推进使用还是不满意。' : '没有看到用户追问。',
|
|
1566
|
+
evidenceRefs: feedbackRefs,
|
|
1567
|
+
suggestionKey: session.indicators.userFollowUpCount > 0 ? 'follow_up_review' : undefined,
|
|
1568
|
+
}),
|
|
1569
|
+
checklistItem({
|
|
1570
|
+
key: 'user_interruption_seen',
|
|
1571
|
+
label: session.indicators.userInterruptionCount > 0 ? '看到用户中断' : '未见用户中断',
|
|
1572
|
+
status: session.indicators.userInterruptionCount > 0 ? 'failed' : 'passed',
|
|
1573
|
+
contribution: session.indicators.userInterruptionCount > 0 ? 'blocking' : 'neutral',
|
|
1574
|
+
reason: session.indicators.userInterruptionCount > 0 ? '看到用户中断或停止任务信号。' : '没有看到用户中断信号。',
|
|
1575
|
+
evidenceRefs: feedbackRefs,
|
|
1576
|
+
suggestionKey: session.indicators.userInterruptionCount > 0 ? 'user_interruption_review' : undefined,
|
|
1577
|
+
}),
|
|
1578
|
+
];
|
|
1579
|
+
}
|
|
1580
|
+
export function foldExperienceChecklistItems(items) {
|
|
1581
|
+
const relevant = items.filter((item) => item.contribution !== 'neutral');
|
|
1582
|
+
const active = relevant.length > 0 ? relevant : items;
|
|
1583
|
+
const degraded = active.filter((item) => item.status === 'degraded');
|
|
1584
|
+
if (degraded.length > 0)
|
|
1585
|
+
return { status: 'degraded', reason: 'data_degraded', sourceItemKeys: degraded.map((item) => item.key) };
|
|
1586
|
+
const blockingFailed = active.filter((item) => item.contribution === 'blocking' && item.status === 'failed');
|
|
1587
|
+
if (blockingFailed.length > 0)
|
|
1588
|
+
return { status: 'attention', reason: 'blocking_failed', sourceItemKeys: blockingFailed.map((item) => item.key) };
|
|
1589
|
+
const attentionFailed = active.filter((item) => item.contribution === 'attention' && item.status === 'failed');
|
|
1590
|
+
if (attentionFailed.length > 0)
|
|
1591
|
+
return { status: 'attention', reason: 'attention_accumulated', sourceItemKeys: attentionFailed.map((item) => item.key) };
|
|
1592
|
+
const unknown = active.filter((item) => item.status === 'unknown' || item.status === 'not_declared');
|
|
1593
|
+
const positivePassed = active.filter((item) => item.status === 'passed' && item.contribution === 'positive');
|
|
1594
|
+
const decisivePassed = active.filter((item) => item.status === 'passed' && (item.contribution === 'blocking' || item.contribution === 'attention' || item.contribution === 'positive'));
|
|
1595
|
+
const informationalPassed = active.filter((item) => item.status === 'passed' && item.contribution === 'informational');
|
|
1596
|
+
const passed = [...positivePassed, ...decisivePassed.filter((item) => item.contribution !== 'positive'), ...informationalPassed];
|
|
1597
|
+
if (unknown.length > 0 && (decisivePassed.length === 0 || unknown.length >= passed.length)) {
|
|
1598
|
+
return { status: 'unknown', reason: 'unknown_dominant', sourceItemKeys: unknown.map((item) => item.key) };
|
|
1599
|
+
}
|
|
1600
|
+
if (passed.length > 0)
|
|
1601
|
+
return { status: 'ok', reason: 'all_passed', sourceItemKeys: passed.map((item) => item.key) };
|
|
1602
|
+
return { status: 'not_applicable', reason: 'not_applicable', sourceItemKeys: active.map((item) => item.key) };
|
|
1603
|
+
}
|
|
1604
|
+
function reviewerScopeReasonCodes(session) {
|
|
1605
|
+
const reasons = [];
|
|
1606
|
+
if (session.invocationIds.length !== 1)
|
|
1607
|
+
reasons.push('multiple_invocations');
|
|
1608
|
+
if (session.goalSliceIds.length !== 1)
|
|
1609
|
+
reasons.push('multiple_goal_slices');
|
|
1610
|
+
if ((session.timelineTree?.branches.length ?? 0) > 0)
|
|
1611
|
+
reasons.push('subagent_branches_present');
|
|
1612
|
+
if (session.pluginNames.length > 1 || session.commandNames.length > 1)
|
|
1613
|
+
reasons.push('multiple_skill_entrypoints');
|
|
1614
|
+
return reasons;
|
|
1615
|
+
}
|
|
1616
|
+
function reviewerStep(order, label, text, status, evidenceRefs = []) {
|
|
1617
|
+
return {
|
|
1618
|
+
order,
|
|
1619
|
+
label,
|
|
1620
|
+
status,
|
|
1621
|
+
text,
|
|
1622
|
+
evidenceRefs: uniqueEvidenceRefs(evidenceRefs).slice(0, 4),
|
|
1623
|
+
};
|
|
1624
|
+
}
|
|
1625
|
+
function userGoalStepText(session) {
|
|
1626
|
+
const goal = session.evidenceChain.firstUserMessage?.snippet;
|
|
1627
|
+
if (!goal)
|
|
1628
|
+
return '没有看到明确人工用户目标;当前只能按运行证据做常规复盘。';
|
|
1629
|
+
return `用户目标:${goal}`;
|
|
1630
|
+
}
|
|
1631
|
+
function skillSelectionStepText(session) {
|
|
1632
|
+
const entrypoint = session.commandNames.length > 0 ? `,入口 ${session.commandNames.join('、')}` : session.entrypoint ? `,入口 ${session.entrypoint}` : '';
|
|
1633
|
+
return `本次使用的能力:${session.skillName}${entrypoint}。`;
|
|
1634
|
+
}
|
|
1635
|
+
function executionStepStatus(session, expectedToolCheck) {
|
|
1636
|
+
if (session.indicators.toolFailureCount > 0 || expectedToolCheck.declared && expectedToolCheck.matchedTools.length === 0)
|
|
1637
|
+
return 'attention';
|
|
1638
|
+
if (session.indicators.toolCallCount > 0)
|
|
1639
|
+
return 'ok';
|
|
1640
|
+
return 'unknown';
|
|
1641
|
+
}
|
|
1642
|
+
function executionStepText(session, expectedToolCheck = expectedToolCheckForSession(session)) {
|
|
1643
|
+
const failures = session.indicators.toolFailureCount > 0 ? `,其中失败 ${session.indicators.toolFailureCount} 次` : '';
|
|
1644
|
+
const expected = expectedToolCheck.declared
|
|
1645
|
+
? expectedToolCheck.matchedTools.length > 0
|
|
1646
|
+
? `命中声明的核心工具:${expectedToolCheck.matchedTools.join('、')}。`
|
|
1647
|
+
: `但没有命中能力声明的核心工具:${expectedToolCheck.expectedTools.join('、')}。`
|
|
1648
|
+
: '';
|
|
1649
|
+
return `执行中看到 ${session.indicators.toolCallCount} 次工具调用${failures}。${expected}`;
|
|
1650
|
+
}
|
|
1651
|
+
function deliveryStepText(session) {
|
|
1652
|
+
if (session.indicators.assistantDeliverySignalCount > 0) {
|
|
1653
|
+
const artifact = session.indicators.deliverableArtifactSignalCount > 0
|
|
1654
|
+
? `,其中 ${session.indicators.deliverableArtifactSignalCount} 次包含具体产物线索`
|
|
1655
|
+
: ',但未看到明确产物线索';
|
|
1656
|
+
return `看到 ${session.indicators.assistantDeliverySignalCount} 次完成态或结果反馈${artifact}。`;
|
|
1657
|
+
}
|
|
1658
|
+
return '没有发现最后结果反馈;当前不能把过程进展当成完成。';
|
|
1659
|
+
}
|
|
1660
|
+
function userFeedbackStepText(session) {
|
|
1661
|
+
const parts = [
|
|
1662
|
+
session.indicators.userFollowUpCount > 0 ? `追问/补充 ${session.indicators.userFollowUpCount} 次` : '',
|
|
1663
|
+
session.indicators.userCorrectionCount > 0 ? `纠正 ${session.indicators.userCorrectionCount} 次` : '',
|
|
1664
|
+
session.indicators.negativeFeedbackCount > 0 ? `负向反馈 ${session.indicators.negativeFeedbackCount} 次` : '',
|
|
1665
|
+
session.indicators.positiveFeedbackCount > 0 ? `正向反馈 ${session.indicators.positiveFeedbackCount} 次` : '',
|
|
1666
|
+
session.indicators.userGoalShiftCount > 0 ? `目标切换 ${session.indicators.userGoalShiftCount} 次` : '',
|
|
1667
|
+
].filter(Boolean);
|
|
1668
|
+
return parts.length > 0 ? `用户反馈信号:${parts.join(',')}。` : '原始记录里没有看到人工追问、纠正、负向反馈或目标切换。';
|
|
1669
|
+
}
|
|
1670
|
+
function userFeedbackStepStatus(session) {
|
|
1671
|
+
if (session.indicators.userCorrectionCount > 0 || session.indicators.negativeFeedbackCount > 0 || session.indicators.userInterruptionCount > 0)
|
|
1672
|
+
return 'attention';
|
|
1673
|
+
if (session.indicators.positiveFeedbackCount > 0)
|
|
1674
|
+
return 'ok';
|
|
1675
|
+
return 'unknown';
|
|
1676
|
+
}
|
|
1677
|
+
function userFeedbackEvidenceRefs(session) {
|
|
1678
|
+
return uniqueEvidenceRefs(session.ruleFindings
|
|
1679
|
+
.filter((finding) => finding.code === 'user_correction_seen' || finding.code === 'negative_feedback_seen' || finding.code === 'positive_feedback_seen' || finding.code === 'user_goal_shift_seen' || finding.code === 'user_interruption_seen')
|
|
1680
|
+
.flatMap((finding) => finding.evidenceRefs)).slice(0, 5);
|
|
1681
|
+
}
|
|
1682
|
+
function expectedToolCheckForSession(session) {
|
|
1683
|
+
const expectedTools = loadExpectedToolsForSkill(session.skillName, session.cwd);
|
|
1684
|
+
if (expectedTools.length === 0)
|
|
1685
|
+
return { expectedTools: [], matchedTools: [], declared: false };
|
|
1686
|
+
const events = session.fullSessionTimeline.length > 0 ? session.fullSessionTimeline : session.timelinePreview;
|
|
1687
|
+
const matchedTools = expectedTools.filter((tool) => events.some((event) => eventMatchesExpectedTool(event, tool)));
|
|
1688
|
+
return {
|
|
1689
|
+
expectedTools,
|
|
1690
|
+
matchedTools,
|
|
1691
|
+
declared: true,
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
function skillDeclarationCheckForSession(session) {
|
|
1695
|
+
const path = findSkillMdPathForExperience(session.skillName, session.cwd ?? process.cwd());
|
|
1696
|
+
if (!path) {
|
|
1697
|
+
return {
|
|
1698
|
+
hardRules: { declared: false, count: 0 },
|
|
1699
|
+
workflows: { declared: false, count: 0 },
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
try {
|
|
1703
|
+
const content = readFileSync(path, 'utf-8');
|
|
1704
|
+
const hardRules = validateSkillHardRules(content);
|
|
1705
|
+
const workflows = validateSkillWorkflows(content);
|
|
1706
|
+
return {
|
|
1707
|
+
hardRules: { declared: hardRules.declared, count: hardRules.rules.length },
|
|
1708
|
+
workflows: { declared: workflows.declared, count: workflows.workflows.reduce((sum, workflow) => sum + workflow.nodes.length, 0) },
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
catch {
|
|
1712
|
+
return {
|
|
1713
|
+
hardRules: { declared: false, count: 0 },
|
|
1714
|
+
workflows: { declared: false, count: 0 },
|
|
1715
|
+
};
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
function loadExpectedToolsForSkill(skillName, cwd) {
|
|
1719
|
+
const path = findSkillMdPathForExperience(skillName, cwd ?? process.cwd());
|
|
1720
|
+
if (!path)
|
|
1721
|
+
return [];
|
|
1722
|
+
try {
|
|
1723
|
+
const parsed = parseSkillFrontmatter(readFileSync(path, 'utf-8'));
|
|
1724
|
+
const data = parsed.ok ? parsed.data : undefined;
|
|
1725
|
+
const value = data?.expected_tools ?? data?.expectedTools;
|
|
1726
|
+
return Array.isArray(value)
|
|
1727
|
+
? unique(value.filter((item) => typeof item === 'string' && Boolean(item.trim())).map((item) => item.trim()))
|
|
1728
|
+
: [];
|
|
1729
|
+
}
|
|
1730
|
+
catch {
|
|
1731
|
+
return [];
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
function eventMatchesExpectedTool(event, expectedTool) {
|
|
1735
|
+
if (event.kind !== 'tool_use')
|
|
1736
|
+
return false;
|
|
1737
|
+
const text = `${event.toolName ?? ''}\n${event.label ?? ''}\n${event.snippet ?? ''}\n${event.fullText ?? ''}`.toLowerCase();
|
|
1738
|
+
const normalized = expectedTool.toLowerCase().trim();
|
|
1739
|
+
const aliases = unique([
|
|
1740
|
+
normalized,
|
|
1741
|
+
normalized.replace(/[-_]?cli$/, ''),
|
|
1742
|
+
].filter(Boolean));
|
|
1743
|
+
return aliases.some((alias) => new RegExp(`(^|[^a-z0-9_-])${escapeRegExp(alias)}([^a-z0-9_-]|$)`, 'i').test(text));
|
|
1744
|
+
}
|
|
1745
|
+
function escapeRegExp(value) {
|
|
1746
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
1747
|
+
}
|
|
1748
|
+
function findSkillMdPathForExperience(skillName, cwd) {
|
|
1749
|
+
if (!/^[A-Za-z0-9_.-]+$/.test(skillName))
|
|
1750
|
+
return undefined;
|
|
1751
|
+
const candidates = [
|
|
1752
|
+
join(cwd, '.claude', 'skills', skillName, 'SKILL.md'),
|
|
1753
|
+
join(cwd, '.openclaw', 'workspace', 'skills', skillName, 'SKILL.md'),
|
|
1754
|
+
join(cwd, 'workspace', 'skills', skillName, 'SKILL.md'),
|
|
1755
|
+
join(cwd, 'skills', skillName, 'SKILL.md'),
|
|
1756
|
+
join(homedir(), '.openclaw', 'workspace', 'skills', skillName, 'SKILL.md'),
|
|
1757
|
+
join(homedir(), '.claude', 'skills', skillName, 'SKILL.md'),
|
|
1758
|
+
join(homedir(), '.codex', 'skills', skillName, 'SKILL.md'),
|
|
1759
|
+
join(homedir(), '.agents', 'skills', skillName, 'SKILL.md'),
|
|
1760
|
+
];
|
|
1761
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
1762
|
+
}
|
|
1763
|
+
function reviewerFindingsForSession(session, reviewState) {
|
|
1764
|
+
const findings = [];
|
|
1765
|
+
const push = (level, title, body, ruleSource, evidenceRefs = []) => {
|
|
1766
|
+
const id = hashParts('reviewer-finding', session.id, ruleSource, title);
|
|
1767
|
+
const judgmentId = hashParts('reviewer-judgment', session.id, ruleSource, title, evidenceRefs.map((ref) => ref.id).join('|'));
|
|
1768
|
+
const reviewEntry = reviewState?.entries[observationReviewStateKey('reviewer_judgment', judgmentId)];
|
|
1769
|
+
findings.push({
|
|
1770
|
+
id,
|
|
1771
|
+
judgmentId,
|
|
1772
|
+
source: 'deterministic_rule',
|
|
1773
|
+
level,
|
|
1774
|
+
title,
|
|
1775
|
+
body,
|
|
1776
|
+
ruleSource,
|
|
1777
|
+
ruleVersion: REVIEWER_REPORT_RULE_VERSION,
|
|
1778
|
+
evidenceRefs: uniqueEvidenceRefs(evidenceRefs).slice(0, 5),
|
|
1779
|
+
reviewStateRef: {
|
|
1780
|
+
targetType: 'reviewer_judgment',
|
|
1781
|
+
targetId: judgmentId,
|
|
1782
|
+
...(reviewEntry?.verdict ? { verdict: reviewEntry.verdict } : {}),
|
|
1783
|
+
...(reviewEntry?.reason ? { reason: reviewEntry.reason } : {}),
|
|
1784
|
+
...(reviewEntry?.note ? { note: reviewEntry.note } : {}),
|
|
1785
|
+
...(reviewEntry?.reviewedAt ? { reviewedAt: reviewEntry.reviewedAt } : {}),
|
|
1786
|
+
},
|
|
1787
|
+
});
|
|
1788
|
+
};
|
|
1789
|
+
const findingRefs = (code) => session.ruleFindings.filter((finding) => finding.code === code).flatMap((finding) => finding.evidenceRefs);
|
|
1790
|
+
const expectedToolCheck = expectedToolCheckForSession(session);
|
|
1791
|
+
if (session.indicators.toolFailureCount > 0) {
|
|
1792
|
+
push('attention', `工具调用失败 ${session.indicators.toolFailureCount} 次`, '执行中遇到工具报错。看下失败的步骤是否在 SKILL.md 里写明了重试或回退方式。', 'tool_error_recovery', findingRefs('tool_failure_seen'));
|
|
1793
|
+
}
|
|
1794
|
+
if (session.indicators.assistantDeliverySignalCount === 0) {
|
|
1795
|
+
push('attention', '没看到给用户的最终答复', 'assistant 没说「完成 / 结果如下」这种收尾,可能任务还没跑完,或收尾文案不够清楚让用户知道事情结束了。', 'final_delivery_absent', session.evidenceChain.lastAssistantMessage ? [session.evidenceChain.lastAssistantMessage] : []);
|
|
1796
|
+
}
|
|
1797
|
+
if (session.indicators.sessionInterruptedCount > 0) {
|
|
1798
|
+
push('attention', `会话异常断开 ${session.indicators.sessionInterruptedCount} 次`, '任务中途被异常中断或重启。如果是网络/超时,看是否要在 skill 里加重试;如果是程序原因,跟开发反馈。', 'session_interrupted', findingRefs('session_interrupted_seen'));
|
|
1799
|
+
}
|
|
1800
|
+
if (expectedToolCheck.declared && expectedToolCheck.matchedTools.length === 0) {
|
|
1801
|
+
push('attention', '没用上 SKILL.md 声明的核心工具', `SKILL.md 里声明 ${expectedToolCheck.expectedTools.join('、')} 是核心工具,但这次没看到调用。要么 description 指引不够清楚,要么用户的诉求不属于这个 skill 的场景。`, 'expected_tools_missed', session.evidenceChain.firstToolUse ? [session.evidenceChain.firstToolUse] : []);
|
|
1802
|
+
}
|
|
1803
|
+
if (session.indicators.userCorrectionCount > 0) {
|
|
1804
|
+
push('attention', `用户纠正 ${session.indicators.userCorrectionCount} 次`, '用户在过程中纠正了方向。看原文确认是 skill 理解偏差,还是 skill 不该处理这种诉求。', 'user_correction', findingRefs('user_correction_seen'));
|
|
1805
|
+
}
|
|
1806
|
+
if (session.indicators.userInterruptionCount > 0) {
|
|
1807
|
+
push('attention', `用户手动叫停 ${session.indicators.userInterruptionCount} 次`, '用户主动喊停了执行。常见原因:跑偏 / 太慢 / 用错工具。看原文定位是哪一步触发的。', 'user_interruption', findingRefs('user_interruption_seen'));
|
|
1808
|
+
}
|
|
1809
|
+
if (session.indicators.negativeFeedbackCount > 0) {
|
|
1810
|
+
push('attention', `用户说了 ${session.indicators.negativeFeedbackCount} 次不满意`, '用户出现了「不对 / 错了 / 不行」等负向表达。先看是 skill 给的结果不达预期,还是用户对方向本身有疑问。', 'negative_feedback', findingRefs('negative_feedback_seen'));
|
|
1811
|
+
}
|
|
1812
|
+
if (session.indicators.hardRuleTextHitCount > 0) {
|
|
1813
|
+
push('note', `用户提了 ${session.indicators.hardRuleTextHitCount} 次硬性要求`, '用户在对话里强调了某些必须做/不能做的规则。如果同类要求反复出现,可以沉淀到 SKILL.md 的 hardRules。', 'user_hard_rule', findingRefs('hard_rule_seen'));
|
|
1814
|
+
}
|
|
1815
|
+
if (reviewerScopeReasonCodes(session).length > 0) {
|
|
1816
|
+
push('note', '复杂链路降级展示', '本次不是严格的 1 次会话 × 1 个目标 × 1 个能力场景。当前先做通用展示,不强行拆分多能力、子任务或目标切换。', 'complex_scope_degraded', []);
|
|
1817
|
+
}
|
|
1818
|
+
if (findings.length === 0) {
|
|
1819
|
+
push('note', '未命中优先问题信号', '没有看到需要优先关注的纠正、中断、负向反馈、工具失败或没收尾的信号。', 'no_priority_signal', []);
|
|
1820
|
+
}
|
|
1821
|
+
return findings;
|
|
1822
|
+
}
|
|
1823
|
+
function reviewerTitle(session, attentionCount, possibleFalsePositiveCount) {
|
|
1824
|
+
const suffix = possibleFalsePositiveCount > 0 ? ` · ${possibleFalsePositiveCount} 项疑似误判` : '';
|
|
1825
|
+
if (attentionCount > 0)
|
|
1826
|
+
return `${session.skillName} · ${attentionCount} 项要看一眼${suffix}`;
|
|
1827
|
+
if (session.indicators.assistantDeliverySignalCount > 0)
|
|
1828
|
+
return `${session.skillName} · 看起来有结果 · 常规抽样${suffix}`;
|
|
1829
|
+
return `${session.skillName} · 常规抽样 · 未见高优先级信号${suffix}`;
|
|
1830
|
+
}
|
|
1831
|
+
function reviewerSummary(session, scopeKind, attentionCount, possibleFalsePositiveCount) {
|
|
1832
|
+
const scopeText = scopeKind === 'single_skill_single_goal'
|
|
1833
|
+
? '本次属于单个能力 / 单个目标报告范围。'
|
|
1834
|
+
: '本次是复杂链路,当前先做降级展示,不强行拆分语义分支。';
|
|
1835
|
+
const reviewText = attentionCount > 0
|
|
1836
|
+
? `发现 ${attentionCount} 条事实层复核点。`
|
|
1837
|
+
: '没有发现优先级较高的事实层复核点。';
|
|
1838
|
+
const falsePositiveText = possibleFalsePositiveCount > 0 ? `另有 ${possibleFalsePositiveCount} 条疑似误判需要人工确认。` : '';
|
|
1839
|
+
return [scopeText, reviewText, falsePositiveText].filter(Boolean).join(' ');
|
|
1840
|
+
}
|
|
1841
|
+
function reviewerAuthorSuggestions(session, findings) {
|
|
1842
|
+
const suggestions = new Map();
|
|
1843
|
+
const pushSuggestion = (key, text, severity) => {
|
|
1844
|
+
const existing = suggestions.get(key);
|
|
1845
|
+
if (!existing || severity > existing.severity)
|
|
1846
|
+
suggestions.set(key, { text, severity });
|
|
1847
|
+
};
|
|
1848
|
+
for (const answer of session.sessionStory?.answers ?? []) {
|
|
1849
|
+
for (const item of answer.checklistItems ?? []) {
|
|
1850
|
+
if (!item.suggestionKey)
|
|
1851
|
+
continue;
|
|
1852
|
+
const text = suggestionTextForChecklistItem(item.suggestionKey);
|
|
1853
|
+
if (!text)
|
|
1854
|
+
continue;
|
|
1855
|
+
pushSuggestion(item.suggestionKey, text, severityForChecklistStatus(item.status));
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
if (findings.some((finding) => finding.ruleSource === 'final_delivery_absent')) {
|
|
1859
|
+
pushSuggestion('final_delivery_absent', '补充明确的产物交付表达或交付标记,避免过程进展被当成完成。', 4);
|
|
1860
|
+
}
|
|
1861
|
+
if (findings.some((finding) => finding.ruleSource === 'tool_error_recovery')) {
|
|
1862
|
+
pushSuggestion('tool_error_recovery', '复查失败工具调用前后的执行流程,必要时把稳定路径写入能力说明文档。', 4);
|
|
1863
|
+
}
|
|
1864
|
+
if (findings.some((finding) => finding.ruleSource === 'session_interrupted')) {
|
|
1865
|
+
pushSuggestion('session_interrupted', '复查会话异常中断前后的上下文,确认是否需要补充中断恢复或重跑策略。', 4);
|
|
1866
|
+
}
|
|
1867
|
+
if (findings.some((finding) => finding.ruleSource === 'expected_tools_missed')) {
|
|
1868
|
+
pushSuggestion('expected_tools_missed', '如果能力依赖核心工具,请在能力定义里维护 expected_tools,并确认运行链路实际命中这些工具。', 4);
|
|
1869
|
+
}
|
|
1870
|
+
if (session.indicators.hardRuleTextHitCount > 0) {
|
|
1871
|
+
pushSuggestion('user_hard_rule', '把反复出现的用户硬性要求沉淀为能力规则,并在后续观测中追踪是否减少纠偏。', 2);
|
|
1872
|
+
}
|
|
1873
|
+
if (session.indicators.userCorrectionCount > 0 || session.indicators.negativeFeedbackCount > 0) {
|
|
1874
|
+
pushSuggestion('user_negative_review', '优先打开原始片段,确认用户纠正/负向反馈发生在交付前还是交付后。', 4);
|
|
1875
|
+
}
|
|
1876
|
+
if (reviewerScopeReasonCodes(session).length > 0) {
|
|
1877
|
+
pushSuggestion('complex_scope_review', '复杂链路暂按降级报告处理;后续再拆多能力、子任务或目标切换。', 1);
|
|
1878
|
+
}
|
|
1879
|
+
if (suggestions.size === 0)
|
|
1880
|
+
pushSuggestion('routine_sample', '进入常规抽样池,保留 evidenceRef 以便人工抽查。', 0);
|
|
1881
|
+
return Array.from(suggestions.values())
|
|
1882
|
+
.sort((a, b) => b.severity - a.severity || a.text.localeCompare(b.text))
|
|
1883
|
+
.map((entry) => entry.text);
|
|
1884
|
+
}
|
|
1885
|
+
function severityForChecklistStatus(status) {
|
|
1886
|
+
if (status === 'degraded')
|
|
1887
|
+
return 5;
|
|
1888
|
+
if (status === 'failed')
|
|
1889
|
+
return 4;
|
|
1890
|
+
if (status === 'unknown')
|
|
1891
|
+
return 3;
|
|
1892
|
+
if (status === 'not_declared')
|
|
1893
|
+
return 2;
|
|
1894
|
+
if (status === 'passed')
|
|
1895
|
+
return 1;
|
|
1896
|
+
return 0;
|
|
1897
|
+
}
|
|
1898
|
+
function suggestionTextForChecklistItem(key) {
|
|
1899
|
+
const suggestions = {
|
|
1900
|
+
final_delivery_absent: '在最后回复里加上「已完成 / 结果如下」之类的明确收尾,让用户知道任务跑完了。',
|
|
1901
|
+
artifact_absent: '如果 skill 应该产出文档、demo、代码或报告,最终回复里要附上文件路径、链接或代码块。',
|
|
1902
|
+
goal_shift_review: '用户中途切了目标,后续追问不属于这个 skill。看下是否要在 description 里说清楚 skill 的边界。',
|
|
1903
|
+
user_negative_or_interrupted: '用户出现了不满 / 纠正 / 叫停。先看原文是哪一步触发的,再决定改 description、补标准流程还是补硬性规则。',
|
|
1904
|
+
workflow_not_declared: '在 SKILL.md 里补一个标准流程声明,把这个 skill 的执行步骤写清楚。否则报告只能猜流程是否完整。',
|
|
1905
|
+
workflow_execution_review: '声明了标准流程但执行证据不够。补一下每个步骤的输出形态,让运行时能验证是否真的跑过。',
|
|
1906
|
+
hardrule_not_declared: '在 SKILL.md 里补硬性规则声明,把那些「必须做 / 不能做」的约束写明。',
|
|
1907
|
+
hardrule_execution_review: '声明了硬性规则但执行证据不够。补一下每条规则的触发场景,让运行时能验证。',
|
|
1908
|
+
expected_tools_not_declared: '在 SKILL.md frontmatter 里声明 expected_tools。否则报告分不出「真用上了 skill 工具」还是「只是随便调了个工具」。',
|
|
1909
|
+
expected_tools_missed: '声明了核心工具但没用上。先确认 description 是否清楚指引到这些工具,或者用户的诉求不属于这个 skill。',
|
|
1910
|
+
negative_feedback_review: '打开用户负向反馈的原文,看问题出在理解目标、执行过程还是最后没收尾。',
|
|
1911
|
+
user_correction_review: '用户纠正了多次。把纠正内容沉淀到 SKILL.md 的标准流程或硬性规则,避免下次同类返工。',
|
|
1912
|
+
follow_up_review: '用户追问比较多。看是围绕产物继续推进(好事),还是因为没拿到结果而反复问(要改)。',
|
|
1913
|
+
user_interruption_review: '用户叫停了执行。看下断的那一步是不是 skill 没声明标准流程导致跑偏。',
|
|
1914
|
+
};
|
|
1915
|
+
return suggestions[key];
|
|
1916
|
+
}
|
|
1917
|
+
function sumTokenUsage(invocations) {
|
|
1918
|
+
return invocations.reduce((sum, invocation) => ({
|
|
1919
|
+
inputTokens: sum.inputTokens + (invocation.metrics.inputTokens ?? 0),
|
|
1920
|
+
outputTokens: sum.outputTokens + (invocation.metrics.outputTokens ?? 0),
|
|
1921
|
+
cacheReadTokens: sum.cacheReadTokens + (invocation.metrics.cacheReadTokens ?? 0),
|
|
1922
|
+
cacheCreationTokens: sum.cacheCreationTokens + (invocation.metrics.cacheCreationTokens ?? 0),
|
|
1923
|
+
}), {
|
|
1924
|
+
inputTokens: 0,
|
|
1925
|
+
outputTokens: 0,
|
|
1926
|
+
cacheReadTokens: 0,
|
|
1927
|
+
cacheCreationTokens: 0,
|
|
1928
|
+
});
|
|
1929
|
+
}
|
|
786
1930
|
function summarizeExperienceSkills(sessions, invocations) {
|
|
787
1931
|
const bySkill = new Map();
|
|
788
1932
|
for (const session of sessions) {
|
|
@@ -885,6 +2029,7 @@ function scoreForIndicators(indicators) {
|
|
|
885
2029
|
+ indicators.mediumObservationCount
|
|
886
2030
|
+ indicators.userCorrectionCount * 2
|
|
887
2031
|
+ indicators.userInterruptionCount * 2
|
|
2032
|
+
+ indicators.sessionInterruptedCount * 2
|
|
888
2033
|
+ indicators.negativeFeedbackCount * 2
|
|
889
2034
|
+ indicators.hardRuleTextHitCount
|
|
890
2035
|
+ indicators.toolFailureCount
|
|
@@ -898,6 +2043,20 @@ function priorityForScore(score) {
|
|
|
898
2043
|
return 'sample_review';
|
|
899
2044
|
return 'routine_sample';
|
|
900
2045
|
}
|
|
2046
|
+
function priorityForReviewerFindings(session, findings) {
|
|
2047
|
+
const fallback = priorityForScore(session.reviewPriorityScore);
|
|
2048
|
+
const attentionFindings = findings.filter((finding) => finding.level === 'attention');
|
|
2049
|
+
if (attentionFindings.length === 0)
|
|
2050
|
+
return fallback;
|
|
2051
|
+
const criticalMissing = attentionFindings.some((finding) => finding.ruleSource === 'final_delivery_absent'
|
|
2052
|
+
|| finding.ruleSource === 'session_interrupted'
|
|
2053
|
+
|| finding.ruleSource === 'expected_tools_missed')
|
|
2054
|
+
|| session.indicators.userMessageCount === 0
|
|
2055
|
+
|| session.indicators.toolCallCount === 0;
|
|
2056
|
+
if (criticalMissing)
|
|
2057
|
+
return 'review_first';
|
|
2058
|
+
return fallback === 'routine_sample' ? 'sample_review' : fallback;
|
|
2059
|
+
}
|
|
901
2060
|
function basisCodesForIndicators(indicators) {
|
|
902
2061
|
const codes = [];
|
|
903
2062
|
if (indicators.highObservationCount > 0)
|
|
@@ -908,6 +2067,8 @@ function basisCodesForIndicators(indicators) {
|
|
|
908
2067
|
codes.push('user_correction');
|
|
909
2068
|
if (indicators.userInterruptionCount > 0)
|
|
910
2069
|
codes.push('user_interruption');
|
|
2070
|
+
if (indicators.sessionInterruptedCount > 0)
|
|
2071
|
+
codes.push('session_interrupted');
|
|
911
2072
|
if (indicators.negativeFeedbackCount > 0)
|
|
912
2073
|
codes.push('negative_feedback');
|
|
913
2074
|
if (indicators.hardRuleTextHitCount > 0)
|
|
@@ -949,11 +2110,15 @@ function sumIndicators(values) {
|
|
|
949
2110
|
userFollowUpCount: acc.userFollowUpCount + value.userFollowUpCount,
|
|
950
2111
|
userCorrectionCount: acc.userCorrectionCount + value.userCorrectionCount,
|
|
951
2112
|
userInterruptionCount: acc.userInterruptionCount + value.userInterruptionCount,
|
|
2113
|
+
sessionInterruptedCount: acc.sessionInterruptedCount + (value.sessionInterruptedCount ?? 0),
|
|
952
2114
|
negativeFeedbackCount: acc.negativeFeedbackCount + (value.negativeFeedbackCount ?? 0),
|
|
953
2115
|
positiveFeedbackCount: acc.positiveFeedbackCount + (value.positiveFeedbackCount ?? 0),
|
|
954
2116
|
userGoalShiftCount: acc.userGoalShiftCount + (value.userGoalShiftCount ?? 0),
|
|
955
2117
|
hardRuleTextHitCount: acc.hardRuleTextHitCount + value.hardRuleTextHitCount,
|
|
956
2118
|
assistantDeliverySignalCount: acc.assistantDeliverySignalCount + (value.assistantDeliverySignalCount ?? 0),
|
|
2119
|
+
deliverableArtifactSignalCount: acc.deliverableArtifactSignalCount + (value.deliverableArtifactSignalCount ?? 0),
|
|
2120
|
+
selfCorrectionCount: acc.selfCorrectionCount + (value.selfCorrectionCount ?? 0),
|
|
2121
|
+
repeatedExecutionCount: acc.repeatedExecutionCount + (value.repeatedExecutionCount ?? 0),
|
|
957
2122
|
toolCallCount: acc.toolCallCount + value.toolCallCount,
|
|
958
2123
|
toolFailureCount: acc.toolFailureCount + value.toolFailureCount,
|
|
959
2124
|
highObservationCount: acc.highObservationCount + value.highObservationCount,
|
|
@@ -1126,7 +2291,7 @@ export function hasUserGoalShiftSignal(value) {
|
|
|
1126
2291
|
}
|
|
1127
2292
|
export function findNegativeFeedbackMatches(value) {
|
|
1128
2293
|
const ranges = [];
|
|
1129
|
-
for (const term of [...NEGATIVE_FEEDBACK_TERMS].sort((a, b) => b.length - a.length)) {
|
|
2294
|
+
for (const term of [...NEGATIVE_FEEDBACK_TERMS, ...ADDITIONAL_NEGATIVE_FEEDBACK_TERMS].sort((a, b) => b.length - a.length)) {
|
|
1130
2295
|
pushTermMatches(value, term, ranges, false);
|
|
1131
2296
|
}
|
|
1132
2297
|
for (const term of [...BOUNDED_NEGATIVE_FEEDBACK_TERMS].sort((a, b) => b.length - a.length)) {
|
|
@@ -1173,6 +2338,8 @@ function isSkillContextRecord(record, text) {
|
|
|
1173
2338
|
}
|
|
1174
2339
|
function isRuntimeContextRecord(record, text) {
|
|
1175
2340
|
const meta = record;
|
|
2341
|
+
if (isRuntimeProtocolPromptText(text))
|
|
2342
|
+
return true;
|
|
1176
2343
|
if (/^Conversation info \(untrusted metadata\):\s*```json/i.test(text))
|
|
1177
2344
|
return true;
|
|
1178
2345
|
if (meta.entrypoint !== 'sdk-ts' || typeof meta.promptId !== 'string')
|