@principles/core 1.240.20 → 1.241.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime-v2/diagnostician/__tests__/diag-distiller-output.test.js +46 -0
- package/dist/runtime-v2/diagnostician/__tests__/diag-distiller-output.test.js.map +1 -1
- package/dist/runtime-v2/diagnostician/diag-distiller-output.d.ts +2 -0
- package/dist/runtime-v2/diagnostician/diag-distiller-output.d.ts.map +1 -1
- package/dist/runtime-v2/diagnostician/diag-distiller-output.js +29 -2
- package/dist/runtime-v2/diagnostician/diag-distiller-output.js.map +1 -1
- package/dist/runtime-v2/feature-flags/__tests__/artifact-summary-redundancy-flag.test.d.ts +15 -0
- package/dist/runtime-v2/feature-flags/__tests__/artifact-summary-redundancy-flag.test.d.ts.map +1 -0
- package/dist/runtime-v2/feature-flags/__tests__/artifact-summary-redundancy-flag.test.js +90 -0
- package/dist/runtime-v2/feature-flags/__tests__/artifact-summary-redundancy-flag.test.js.map +1 -0
- package/dist/runtime-v2/feature-flags/feature-flag-contract.d.ts.map +1 -1
- package/dist/runtime-v2/feature-flags/feature-flag-contract.js +6 -0
- package/dist/runtime-v2/feature-flags/feature-flag-contract.js.map +1 -1
- package/dist/runtime-v2/internalization/__tests__/artifact-content-hash.property.test.d.ts +11 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-content-hash.property.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-content-hash.property.test.js +124 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-content-hash.property.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-summary.property.test.d.ts +12 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-summary.property.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-summary.property.test.js +253 -0
- package/dist/runtime-v2/internalization/__tests__/artifact-summary.property.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/philosopher-toggle-characterization.test.d.ts +2 -0
- package/dist/runtime-v2/internalization/__tests__/philosopher-toggle-characterization.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/philosopher-toggle-characterization.test.js +279 -0
- package/dist/runtime-v2/internalization/__tests__/philosopher-toggle-characterization.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-scope-regression.test.d.ts +22 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-scope-regression.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-scope-regression.test.js +92 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-scope-regression.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.d.ts +165 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.js +657 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.test.d.ts +22 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.test.js +229 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike-fixtures.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike.test.d.ts +2 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike.test.js +1039 -0
- package/dist/runtime-v2/internalization/__tests__/progressive-disclosure-spike.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-chain-flow.property.test.d.ts +22 -0
- package/dist/runtime-v2/internalization/__tests__/summary-chain-flow.property.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-chain-flow.property.test.js +280 -0
- package/dist/runtime-v2/internalization/__tests__/summary-chain-flow.property.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-envelope-writer.property.test.d.ts +25 -0
- package/dist/runtime-v2/internalization/__tests__/summary-envelope-writer.property.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-envelope-writer.property.test.js +413 -0
- package/dist/runtime-v2/internalization/__tests__/summary-envelope-writer.property.test.js.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-freshness.property.test.d.ts +12 -0
- package/dist/runtime-v2/internalization/__tests__/summary-freshness.property.test.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/__tests__/summary-freshness.property.test.js +176 -0
- package/dist/runtime-v2/internalization/__tests__/summary-freshness.property.test.js.map +1 -0
- package/dist/runtime-v2/internalization/artifact-content-hash.d.ts +19 -0
- package/dist/runtime-v2/internalization/artifact-content-hash.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/artifact-content-hash.js +116 -0
- package/dist/runtime-v2/internalization/artifact-content-hash.js.map +1 -0
- package/dist/runtime-v2/internalization/artifact-summary.d.ts +106 -0
- package/dist/runtime-v2/internalization/artifact-summary.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/artifact-summary.js +331 -0
- package/dist/runtime-v2/internalization/artifact-summary.js.map +1 -0
- package/dist/runtime-v2/internalization/artificer-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/artificer-runner.js +29 -1
- package/dist/runtime-v2/internalization/artificer-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/attach-summary-envelope.d.ts +60 -0
- package/dist/runtime-v2/internalization/attach-summary-envelope.d.ts.map +1 -0
- package/dist/runtime-v2/internalization/attach-summary-envelope.js +126 -0
- package/dist/runtime-v2/internalization/attach-summary-envelope.js.map +1 -0
- package/dist/runtime-v2/internalization/diag-distiller-runner.d.ts +1 -1
- package/dist/runtime-v2/internalization/diag-distiller-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/diag-distiller-runner.js +20 -2
- package/dist/runtime-v2/internalization/diag-distiller-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/diag-rootcause-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/diag-rootcause-runner.js +6 -1
- package/dist/runtime-v2/internalization/diag-rootcause-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/diag-router-runner.d.ts +1 -1
- package/dist/runtime-v2/internalization/diag-router-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/diag-router-runner.js +25 -1
- package/dist/runtime-v2/internalization/diag-router-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/dreamer-runner.d.ts +9 -1
- package/dist/runtime-v2/internalization/dreamer-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/dreamer-runner.js +28 -6
- package/dist/runtime-v2/internalization/dreamer-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/evaluator-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/evaluator-runner.js +35 -2
- package/dist/runtime-v2/internalization/evaluator-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/philosopher-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/philosopher-runner.js +25 -1
- package/dist/runtime-v2/internalization/philosopher-runner.js.map +1 -1
- package/dist/runtime-v2/internalization/scribe-runner.d.ts.map +1 -1
- package/dist/runtime-v2/internalization/scribe-runner.js +23 -1
- package/dist/runtime-v2/internalization/scribe-runner.js.map +1 -1
- package/dist/runtime-v2/runner/base-peer-runner.d.ts +30 -0
- package/dist/runtime-v2/runner/base-peer-runner.d.ts.map +1 -1
- package/dist/runtime-v2/runner/base-peer-runner.js +106 -0
- package/dist/runtime-v2/runner/base-peer-runner.js.map +1 -1
- package/dist/runtime-v2/runner/peer-runner-types.d.ts +19 -0
- package/dist/runtime-v2/runner/peer-runner-types.d.ts.map +1 -1
- package/dist/runtime-v2/store/commit/diagnostician-committer.d.ts +15 -0
- package/dist/runtime-v2/store/commit/diagnostician-committer.d.ts.map +1 -1
- package/dist/runtime-v2/store/commit/diagnostician-committer.js +19 -1
- package/dist/runtime-v2/store/commit/diagnostician-committer.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,1039 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 0 Spike driver (tasks.md 1.2) — 价值验证阻塞门.
|
|
3
|
+
*
|
|
4
|
+
* 问题:只给**摘要级上下文**(每个 stage 一个 headline + 结构化 fields,
|
|
5
|
+
* 不含 `evidence` / 全量 `candidates` / `implementationCode` 等原始大字段),
|
|
6
|
+
* evaluator 能否指出「哪一段(pain_to_dreamer / dreamer_to_scribe /
|
|
7
|
+
* scribe_to_artificer)丢了哪个维度」,且对照链不被误报?
|
|
8
|
+
*
|
|
9
|
+
* 这个门只有在**真实 LLM** 从摘要级上下文独立判断时才有意义(ERR-088:
|
|
10
|
+
* 断言信号必须唯一指向被验证的路径)。因此:
|
|
11
|
+
* - 没有任何手写启发式 / 字符串匹配替代 LLM 判断;
|
|
12
|
+
* - LLM 不可达时测试**显式失败**并报 `LLM_UNAVAILABLE`(阻塞是合法结论,
|
|
13
|
+
* 模拟通过不是);
|
|
14
|
+
* - 摘要构造是确定性纯函数,且**不为了让答案对而调参**。
|
|
15
|
+
*
|
|
16
|
+
* 判断路由(Layer 2 尚不存在,如实记录保真度):
|
|
17
|
+
* Route A(主):spike-local prompt,要求 LLM 输出 design §6.5 的
|
|
18
|
+
* `segments[] / compressionFidelity / painCoverage` 形状。今天的
|
|
19
|
+
* `EvaluatorOutputV2` 没有 `painCoverage` / `compressionFidelity`
|
|
20
|
+
* (它们是 Layer 2 增量),所以这条路由用 spike-local prompt 承载判断,
|
|
21
|
+
* 字段构成照抄 design §6.5,注入字段照抄 `EVALUATOR_STAGE1_MANIFEST`。
|
|
22
|
+
* Route B(对照):生产 `EvaluatorPromptBuilder` + 生产
|
|
23
|
+
* `DefaultEvaluatorValidator`,上下文换成摘要级 stand-in。用来记录
|
|
24
|
+
* 「今天的 prompt 在摘要级上下文下会把判断落在 `evaluation.concerns` /
|
|
25
|
+
* `codeReview` 的哪里」。
|
|
26
|
+
*
|
|
27
|
+
* I/O 边界:本文件是 `*.test.ts`,按 eslint.config.js 的 PRI-450/462 豁免
|
|
28
|
+
* 允许 fs / child_process;core 生产代码仍是纯逻辑(AGENTS.md
|
|
29
|
+
* `antipattern-core-io`)。LLM 进程由生产同一个 `runCliProcess` 拉起
|
|
30
|
+
* (与 `OpenClawCliRuntimeAdapter` 相同的 spawn 语义)。
|
|
31
|
+
*
|
|
32
|
+
* ERR 清单:ERR-088(测试真实性:不用「没抛错」当依据,断言具体 segment /
|
|
33
|
+
* dimension),ERR-001 / ERR-005(LLM 输出保持 unknown 直到逐字段校验,
|
|
34
|
+
* 不用 `as` 绕过),ERR-013(读 unknown 键一律 `Object.hasOwn`),
|
|
35
|
+
* ERR-002(每条降级/不可达都带结构化原因,不静默跳过)。
|
|
36
|
+
*
|
|
37
|
+
* 2026-07-28 修订(design §12.1 的实测驱动修复,门禁重跑):
|
|
38
|
+
* - 维度不再是「五个等权」:`DIMENSION_COVERAGE_POLICY`(§6.5.1)把
|
|
39
|
+
* `betterDecision` / `rationale` / `riskLevel` 定为 required,`badDecision`
|
|
40
|
+
* 为 optional(出现在 antiPatterns 即覆盖,**缺失不算缺陷**),
|
|
41
|
+
* `strategicPerspective` 为 excluded(判据不得对它下结论)。
|
|
42
|
+
* - 覆盖判定口径(§6.5.2 六条)写进 prompt,不再让模型自己发明尺子。
|
|
43
|
+
* - 判据侧按政策过滤 LLM 给的 `missingDimensions`(§6.5.3):模型仍可能把
|
|
44
|
+
* optional / excluded / 不认识的字符串塞进去(rc-1 / rc-4 不可信输入),
|
|
45
|
+
* 一律忽略并记成 shape warning,不据此判失败。
|
|
46
|
+
*
|
|
47
|
+
* 2026-07-29 第四次门禁重跑(design §12.1「第三次门禁重跑」结论 4/5,Owner 已授权修补第 7 条规则冲突):
|
|
48
|
+
* - **第 7 条「看不见即不裁决」的字面表述改窄**:第三次重跑中模型把「scribe 没有一个
|
|
49
|
+
* 字面叫 riskLevel 的字段」误读成「riskLevel 在这一段不可判定」,导致缺陷链 A 的
|
|
50
|
+
* dreamer_to_scribe 段判 pass。改后明确「看不见」指的是「维度值本身在 dreamer 的
|
|
51
|
+
* injectedFields 里都没注入过」,不是「下游字段没有同名字段」——scribe 本来就没有
|
|
52
|
+
* riskLevel/betterDecision 这类字段,它只有 principleText/scope 自由文本字段,
|
|
53
|
+
* 这条规则不应被读成「因此不判定」。这是**判据内部规则冲突的修补**,不是对模型
|
|
54
|
+
* 行为的第三次拟合:冲突在设计阶段就存在,只是前两次重跑的链条形态没有精确命中它
|
|
55
|
+
* (对照链的 scribe 文本本身就含风险等级词,不会触发这条歧义)。
|
|
56
|
+
* - 按 §6.5.2 止损规则,本条本应是「不再调整判据」,但 Owner 已明确审阅根因分析并
|
|
57
|
+
* 授权本次修补(判据内部规则冲突 ≠ 拟合模型行为),故本次修正记为止损规则下的
|
|
58
|
+
* 显式例外,不视为规则被违反。
|
|
59
|
+
*
|
|
60
|
+
* 2026-07-28 第三次门禁重跑(design §6.5.2 clause 2 / §6.5.4 / §12.1 第二次重跑记录):
|
|
61
|
+
* - **具体性子句**:`betterDecision` 的覆盖判定同时要求存在性与具体性 ——
|
|
62
|
+
* 把可核验动作(审计文件树 / grep 全部 imports / 检查导出依赖图)换成不可核验
|
|
63
|
+
* 的抽象(理解架构 / 掌握整体结构)**不算覆盖**。第二次重跑正是因为口径只写了
|
|
64
|
+
* 「存在语义等价表述」,模型检出抽象化后仍裁定为覆盖,缺陷链 B 退化为 `pass`。
|
|
65
|
+
* 这是**判据的最后一次修正**(§6.5.2 止损规则)。
|
|
66
|
+
* - **输出契约路径**:回复经仓库既有的结构化输出修复通道
|
|
67
|
+
* (`attemptStructuredOutputRepair` + `output-repair-contract` 助手)处理,
|
|
68
|
+
* 本文件不再自带 JSON 提取器;`llm_unavailable`(模型没答)与
|
|
69
|
+
* `output_contract_violation`(答了但信封坏了)分两个桶、两条断言。
|
|
70
|
+
* - **结果文件带运行时间戳**:`.kiro/` 是 git-ignored,第二次重跑覆盖了第一次的原始记录。
|
|
71
|
+
* - Route B 本次不跑(第二次重跑中挂在传输层超时;纯记录性路由,不参与门禁判定)。
|
|
72
|
+
*
|
|
73
|
+
* @see .kiro/specs/internalization-progressive-disclosure/design.md §6.1 / §6.5.1 / §6.5.2 / §6.5.3 / §6.6.1 / §12.1
|
|
74
|
+
* @see Requirements 12.3、12.4
|
|
75
|
+
*/
|
|
76
|
+
import { describe, it, expect } from 'vitest';
|
|
77
|
+
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
|
78
|
+
import { join, dirname } from 'node:path';
|
|
79
|
+
import { runCliProcess } from '../../utils/cli-process-runner.js';
|
|
80
|
+
import { EvaluatorPromptBuilder } from '../evaluator-prompt-builder.js';
|
|
81
|
+
import { DefaultEvaluatorValidator } from '../evaluator-output.js';
|
|
82
|
+
// design §6.5.4:输出契约违规必须走**仓库既有**的结构化输出修复通道,
|
|
83
|
+
// 不得在本文件里再实现一份 JSON 提取器 / 修复循环。
|
|
84
|
+
import { attemptStructuredOutputRepair, extractJsonObject, } from '../../adapter/structured-output-repair.js';
|
|
85
|
+
import { repairMalformedJson } from '../../adapter/json-extractor.js';
|
|
86
|
+
import { MAX_REPAIR_ATTEMPTS, normalizeMaxRepairAttempts, safeStringifyPreview, } from '../../adapter/output-repair-contract.js';
|
|
87
|
+
import { SPIKE_CHAINS, assertSpikeChainLineageConsistent, } from './progressive-disclosure-spike-fixtures.js';
|
|
88
|
+
// ── 摘要级上下文的确定性构造(mirrors design §6.1 派生映射表)────────────────
|
|
89
|
+
//
|
|
90
|
+
// 这是 Spike 本地实现:Layer 0 的 `deriveArtifactSummary` 还不存在(它属于
|
|
91
|
+
// PR 1 / 任务 3.2,被 Phase 0 门禁阻塞)。字段选择与截断上限照抄 design §6.1,
|
|
92
|
+
// 目的是让 Spike 看到的信息量与 Layer 0 上线后一致 —— 既不多给(不含原始大
|
|
93
|
+
// 字段),也不少给。
|
|
94
|
+
const SUMMARY_HEADLINE_MAX_CHARS = 200;
|
|
95
|
+
const SUMMARY_FIELD_MAX_CHARS = 600;
|
|
96
|
+
function isRecord(value) {
|
|
97
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
98
|
+
}
|
|
99
|
+
/** rc-1 / rc-5:unknown 上按自有属性读字符串,空串视为缺失。 */
|
|
100
|
+
function readString(source, key) {
|
|
101
|
+
if (!isRecord(source) || !Object.hasOwn(source, key))
|
|
102
|
+
return null;
|
|
103
|
+
const value = source[key];
|
|
104
|
+
if (typeof value !== 'string' || value.trim() === '')
|
|
105
|
+
return null;
|
|
106
|
+
return value;
|
|
107
|
+
}
|
|
108
|
+
function readRecord(source, key) {
|
|
109
|
+
if (!isRecord(source) || !Object.hasOwn(source, key))
|
|
110
|
+
return null;
|
|
111
|
+
const value = source[key];
|
|
112
|
+
return isRecord(value) ? value : null;
|
|
113
|
+
}
|
|
114
|
+
function readArray(source, key) {
|
|
115
|
+
if (!isRecord(source) || !Object.hasOwn(source, key))
|
|
116
|
+
return null;
|
|
117
|
+
const value = source[key];
|
|
118
|
+
return Array.isArray(value) ? value : null;
|
|
119
|
+
}
|
|
120
|
+
/** rc-4:数组元素逐个校验为字符串后再拼接。 */
|
|
121
|
+
function readStringList(source, key) {
|
|
122
|
+
const list = readArray(source, key);
|
|
123
|
+
if (list === null)
|
|
124
|
+
return null;
|
|
125
|
+
const strings = list.filter((entry) => typeof entry === 'string' && entry.trim() !== '');
|
|
126
|
+
if (strings.length === 0)
|
|
127
|
+
return null;
|
|
128
|
+
return strings.join(' / ');
|
|
129
|
+
}
|
|
130
|
+
function clamp(text, max) {
|
|
131
|
+
return text.length <= max ? text : `${text.slice(0, max - 1)}…`;
|
|
132
|
+
}
|
|
133
|
+
/** 首句:以中文句号 / 分号 / 英文句点为界,确定性截断。 */
|
|
134
|
+
function firstSentence(text) {
|
|
135
|
+
const match = /^[\s\S]*?[。;;.]/.exec(text);
|
|
136
|
+
const sentence = match?.[0] ?? text;
|
|
137
|
+
return clamp(sentence.trim(), SUMMARY_HEADLINE_MAX_CHARS);
|
|
138
|
+
}
|
|
139
|
+
/** 派生一个 stage 的摘要:目标键缺失即进 omittedFields(design §6.1)。 */
|
|
140
|
+
function deriveSpikeSummary(runnerKind, targetKeys, headlineSource, resolved) {
|
|
141
|
+
const fields = {};
|
|
142
|
+
const omittedFields = [];
|
|
143
|
+
for (const key of targetKeys) {
|
|
144
|
+
const value = Object.hasOwn(resolved, key) ? resolved[key] : null;
|
|
145
|
+
if (typeof value === 'string' && value.trim() !== '') {
|
|
146
|
+
fields[key] = clamp(value, SUMMARY_FIELD_MAX_CHARS);
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
omittedFields.push(key);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
runnerKind,
|
|
154
|
+
headline: headlineSource === null ? '' : firstSentence(headlineSource),
|
|
155
|
+
fields,
|
|
156
|
+
omittedFields,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* 8 个 stage 的摘要。字段名是 design §6.1 的**目标键名**;实际 output schema
|
|
161
|
+
* 里不存在该语义的键一律进 `omittedFields`(不改 schema、不改 prompt)。
|
|
162
|
+
*/
|
|
163
|
+
function buildChainSummaries(chain) {
|
|
164
|
+
// rc-1:fixture 的 contentJson 一律按 unknown 读,不用 `as` 收窄(rc-2)
|
|
165
|
+
const rootCause = chain.diagRootCause.contentJson;
|
|
166
|
+
const distiller = chain.diagDistiller.contentJson;
|
|
167
|
+
const router = chain.diagRouter.contentJson;
|
|
168
|
+
const dreamer = chain.dreamer.contentJson;
|
|
169
|
+
const philosopher = chain.philosopher.contentJson;
|
|
170
|
+
const scribe = chain.scribe.contentJson;
|
|
171
|
+
const artificer = chain.artificer.contentJson;
|
|
172
|
+
const evaluator = chain.evaluator.contentJson;
|
|
173
|
+
// dreamer:只取第一个候选的五维(全量 candidates 属 tier2 原始大字段,排除)
|
|
174
|
+
const candidates = readArray(dreamer, 'candidates') ?? [];
|
|
175
|
+
const firstCandidate = candidates.length > 0 ? candidates[0] : null;
|
|
176
|
+
const principleDraft = readRecord(scribe, 'principleDraft');
|
|
177
|
+
const principleCandidate = readRecord(philosopher, 'principleCandidate');
|
|
178
|
+
const violated = readArray(router, 'violatedPrinciples') ?? [];
|
|
179
|
+
const violatedIds = violated
|
|
180
|
+
.map((entry) => readString(entry, 'principleId'))
|
|
181
|
+
.filter((id) => id !== null);
|
|
182
|
+
const recommendations = readArray(router, 'recommendations') ?? [];
|
|
183
|
+
const firstRecommendationKind = recommendations.length > 0 ? readString(recommendations[0], 'kind') : null;
|
|
184
|
+
const goldenTraceCases = readArray(artificer, 'goldenTraceCases') ?? [];
|
|
185
|
+
const evaluatorEvaluation = readRecord(evaluator, 'evaluation');
|
|
186
|
+
const evaluatorConcerns = evaluatorEvaluation === null ? null : readArray(evaluatorEvaluation, 'concerns');
|
|
187
|
+
const evaluatorCodeReview = readRecord(evaluator, 'codeReview');
|
|
188
|
+
const intentConsistency = evaluatorCodeReview === null ? null : readRecord(evaluatorCodeReview, 'intentConsistency');
|
|
189
|
+
return {
|
|
190
|
+
diag_rootcause: deriveSpikeSummary('diag_rootcause', ['rootSymptom', 'category', 'severity', 'rootCause'], readString(rootCause, 'rootCause'), {
|
|
191
|
+
rootSymptom: readString(rootCause, 'summary'),
|
|
192
|
+
category: readString(rootCause, 'rootCauseCategory'),
|
|
193
|
+
// DiagRootCauseOutputV1 没有 severity 语义字段 → omitted
|
|
194
|
+
severity: null,
|
|
195
|
+
rootCause: readString(rootCause, 'rootCause'),
|
|
196
|
+
}),
|
|
197
|
+
diag_distiller: deriveSpikeSummary('diag_distiller', ['rootCause', 'affectedComponents', 'category', 'severity'], readString(distiller, 'abstractedPrinciple'), {
|
|
198
|
+
rootCause: readString(distiller, 'rationale'),
|
|
199
|
+
// DiagDistillerOutputV1 没有 affectedComponents / severity → omitted
|
|
200
|
+
affectedComponents: null,
|
|
201
|
+
category: readString(distiller, 'scope'),
|
|
202
|
+
severity: null,
|
|
203
|
+
}),
|
|
204
|
+
diag_router: deriveSpikeSummary('diag_router', ['rootCause', 'affectedComponents', 'rootSymptom', 'category', 'severity'], readString(router, 'summary'), {
|
|
205
|
+
rootCause: readString(router, 'rootCause'),
|
|
206
|
+
affectedComponents: violatedIds.length > 0 ? violatedIds.join(' / ') : null,
|
|
207
|
+
rootSymptom: readString(router, 'summary'),
|
|
208
|
+
category: firstRecommendationKind,
|
|
209
|
+
severity: null,
|
|
210
|
+
}),
|
|
211
|
+
dreamer: deriveSpikeSummary('dreamer', ['badDecision', 'betterDecision', 'rationale', 'riskLevel', 'strategicPerspective'], readString(firstCandidate, 'betterDecision'), {
|
|
212
|
+
badDecision: readString(firstCandidate, 'badDecision'),
|
|
213
|
+
betterDecision: readString(firstCandidate, 'betterDecision'),
|
|
214
|
+
rationale: readString(firstCandidate, 'rationale'),
|
|
215
|
+
riskLevel: readString(firstCandidate, 'riskLevel'),
|
|
216
|
+
strategicPerspective: readString(firstCandidate, 'strategicPerspective'),
|
|
217
|
+
}),
|
|
218
|
+
philosopher: deriveSpikeSummary('philosopher', ['thesis', 'principleTitle', 'principleScope', 'principleConfidence'], principleCandidate === null ? null : readString(principleCandidate, 'title'), {
|
|
219
|
+
thesis: readString(philosopher, 'thesis'),
|
|
220
|
+
principleTitle: principleCandidate === null ? null : readString(principleCandidate, 'title'),
|
|
221
|
+
principleScope: principleCandidate === null ? null : readString(principleCandidate, 'scope'),
|
|
222
|
+
principleConfidence: principleCandidate !== null
|
|
223
|
+
&& Object.hasOwn(principleCandidate, 'confidence')
|
|
224
|
+
&& typeof principleCandidate.confidence === 'number'
|
|
225
|
+
? String(principleCandidate.confidence)
|
|
226
|
+
: null,
|
|
227
|
+
}),
|
|
228
|
+
scribe: deriveSpikeSummary('scribe', ['principleText', 'scope', 'exceptions'], principleDraft === null ? null : readString(principleDraft, 'statement'), {
|
|
229
|
+
principleText: principleDraft === null ? null : readString(principleDraft, 'statement'),
|
|
230
|
+
scope: principleDraft === null ? null : readStringList(principleDraft, 'applicability'),
|
|
231
|
+
exceptions: principleDraft === null ? null : readStringList(principleDraft, 'antiPatterns'),
|
|
232
|
+
}),
|
|
233
|
+
artificer: deriveSpikeSummary('artificer', ['changedFiles', 'apiSurface', 'risks'], readString(artificer, 'implementationSummary'), {
|
|
234
|
+
// 原始 implementationCode 属 tier2 大字段,摘要级只给受影响工具与摘要
|
|
235
|
+
changedFiles: readStringList(artificer, 'affectedTools'),
|
|
236
|
+
apiSurface: readString(artificer, 'implementationSummary'),
|
|
237
|
+
risks: readStringList(artificer, 'risks'),
|
|
238
|
+
}),
|
|
239
|
+
evaluator: deriveSpikeSummary('evaluator', ['verdict', 'concernCount', 'intentConsistency'], evaluatorEvaluation === null ? null : readString(evaluatorEvaluation, 'summary'), {
|
|
240
|
+
verdict: evaluatorEvaluation === null ? null : readString(evaluatorEvaluation, 'decision'),
|
|
241
|
+
concernCount: evaluatorConcerns === null ? null : String(evaluatorConcerns.length),
|
|
242
|
+
intentConsistency: intentConsistency !== null
|
|
243
|
+
&& Object.hasOwn(intentConsistency, 'aligned')
|
|
244
|
+
&& typeof intentConsistency.aligned === 'boolean'
|
|
245
|
+
? String(intentConsistency.aligned)
|
|
246
|
+
: null,
|
|
247
|
+
}),
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* 注入字段集合 = `EVALUATOR_STAGE1_MANIFEST` 的 tier0 ∪ tier1(design §6.6)。
|
|
252
|
+
* 逐字对齐,不额外加字段 —— Spike 看到的必须就是 Layer 1 会给的。
|
|
253
|
+
*/
|
|
254
|
+
const EVALUATOR_STAGE1_PATHS = [
|
|
255
|
+
// tier0
|
|
256
|
+
'artificer.summary.headline',
|
|
257
|
+
'artificer.predecessorSummary.headline',
|
|
258
|
+
// tier1
|
|
259
|
+
'scribe.summary.principleText',
|
|
260
|
+
'scribe.summary.scope',
|
|
261
|
+
'artificer.summary.changedFiles',
|
|
262
|
+
'artificer.summary.apiSurface',
|
|
263
|
+
'artificer.summary.risks',
|
|
264
|
+
'dreamer.summary.badDecision',
|
|
265
|
+
'dreamer.summary.betterDecision',
|
|
266
|
+
'dreamer.summary.rationale',
|
|
267
|
+
'dreamer.summary.riskLevel',
|
|
268
|
+
'pain.summary.rootSymptom',
|
|
269
|
+
'pain.summary.category',
|
|
270
|
+
];
|
|
271
|
+
function buildEvaluatorStage1Context(chain) {
|
|
272
|
+
const summaries = buildChainSummaries(chain);
|
|
273
|
+
const source = {
|
|
274
|
+
'artificer.summary.headline': summaries.artificer.headline,
|
|
275
|
+
// 一层冗余:artificer 的边上前驱是 scribe(design §6.1 前驱表)
|
|
276
|
+
'artificer.predecessorSummary.headline': summaries.scribe.headline,
|
|
277
|
+
'scribe.summary.principleText': summaries.scribe.fields.principleText,
|
|
278
|
+
'scribe.summary.scope': summaries.scribe.fields.scope,
|
|
279
|
+
'artificer.summary.changedFiles': summaries.artificer.fields.changedFiles,
|
|
280
|
+
'artificer.summary.apiSurface': summaries.artificer.fields.apiSurface,
|
|
281
|
+
'artificer.summary.risks': summaries.artificer.fields.risks,
|
|
282
|
+
'dreamer.summary.badDecision': summaries.dreamer.fields.badDecision,
|
|
283
|
+
'dreamer.summary.betterDecision': summaries.dreamer.fields.betterDecision,
|
|
284
|
+
'dreamer.summary.rationale': summaries.dreamer.fields.rationale,
|
|
285
|
+
'dreamer.summary.riskLevel': summaries.dreamer.fields.riskLevel,
|
|
286
|
+
'pain.summary.rootSymptom': summaries.diag_rootcause.fields.rootSymptom,
|
|
287
|
+
'pain.summary.category': summaries.diag_rootcause.fields.category,
|
|
288
|
+
};
|
|
289
|
+
const fields = {};
|
|
290
|
+
const absent = [];
|
|
291
|
+
for (const path of EVALUATOR_STAGE1_PATHS) {
|
|
292
|
+
const value = Object.hasOwn(source, path) ? source[path] : undefined;
|
|
293
|
+
if (typeof value === 'string' && value.trim() !== '') {
|
|
294
|
+
fields[path] = value;
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
absent.push(path);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return { fields, absent };
|
|
301
|
+
}
|
|
302
|
+
// ── LLM 传输(与生产 OpenClawCliRuntimeAdapter 同一个 runCliProcess)─────────
|
|
303
|
+
const LLM_TIMEOUT_MS = 420_000;
|
|
304
|
+
const REPO_ROOT = join(process.cwd(), '..', '..');
|
|
305
|
+
const SPIKE_TMP_DIR = join(REPO_ROOT, '.spike-tmp', 'spike-1.2');
|
|
306
|
+
/**
|
|
307
|
+
* 结果文件名带运行时间戳:`.kiro/` 是 git-ignored,第二次重跑覆盖掉了第一次的
|
|
308
|
+
* 原始记录(无法再回看首轮原文)。带时间戳后每次重跑各留一份。
|
|
309
|
+
*/
|
|
310
|
+
const RUN_TIMESTAMP = new Date().toISOString();
|
|
311
|
+
const RESULTS_PATH = join(REPO_ROOT, '.kiro', 'specs', 'internalization-progressive-disclosure', `phase0-spike-1.2-raw-results-${RUN_TIMESTAMP.replace(/[:.]/g, '-')}.json`);
|
|
312
|
+
/**
|
|
313
|
+
* 从 openclaw `--json` 信封里取出 agent 回复文本(local / gateway / 裸对象三种形态)。
|
|
314
|
+
*
|
|
315
|
+
* 兜底顺序:平衡括号扫描找 `payloads[].text` → 整体 `JSON.parse` → 直接抓
|
|
316
|
+
* `finalAssistantVisibleText`(信封里同一份回复的另一处出口)。三条都失败才
|
|
317
|
+
* 判定为拿不到回复,理由分别可区分(rc-9)。
|
|
318
|
+
*/
|
|
319
|
+
function extractAgentText(stdout, stderr) {
|
|
320
|
+
for (const source of [stdout, stderr]) {
|
|
321
|
+
if (!source)
|
|
322
|
+
continue;
|
|
323
|
+
const objects = [];
|
|
324
|
+
let depth = 0;
|
|
325
|
+
let start = -1;
|
|
326
|
+
for (let i = 0; i < source.length; i++) {
|
|
327
|
+
const ch = source[i];
|
|
328
|
+
if (ch === '{') {
|
|
329
|
+
if (start === -1)
|
|
330
|
+
start = i;
|
|
331
|
+
depth++;
|
|
332
|
+
}
|
|
333
|
+
else if (ch === '}') {
|
|
334
|
+
depth--;
|
|
335
|
+
if (depth === 0 && start !== -1) {
|
|
336
|
+
try {
|
|
337
|
+
objects.push(JSON.parse(source.slice(start, i + 1)));
|
|
338
|
+
}
|
|
339
|
+
catch {
|
|
340
|
+
// keep scanning
|
|
341
|
+
}
|
|
342
|
+
start = -1;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
for (let i = objects.length - 1; i >= 0; i--) {
|
|
347
|
+
const candidate = objects[i];
|
|
348
|
+
const payloads = readArray(candidate, 'payloads')
|
|
349
|
+
?? readArray(readRecord(candidate, 'result'), 'payloads');
|
|
350
|
+
if (payloads !== null && payloads.length > 0) {
|
|
351
|
+
const text = readString(payloads[payloads.length - 1], 'text');
|
|
352
|
+
if (text !== null)
|
|
353
|
+
return text;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
// 兜底 1:整体 JSON.parse(stdout 可能是被换行包裹的单个对象)
|
|
357
|
+
try {
|
|
358
|
+
const whole = JSON.parse(source);
|
|
359
|
+
const payloads = readArray(whole, 'payloads') ?? readArray(readRecord(whole, 'result'), 'payloads');
|
|
360
|
+
if (payloads !== null && payloads.length > 0) {
|
|
361
|
+
const text = readString(payloads[payloads.length - 1], 'text');
|
|
362
|
+
if (text !== null)
|
|
363
|
+
return text;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
// 继续兜底 2
|
|
368
|
+
}
|
|
369
|
+
// 兜底 2:信封的 meta.finalAssistantVisibleText / finalAssistantRawText
|
|
370
|
+
const visible = /"final(?:AssistantVisibleText|AssistantRawText)"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(source);
|
|
371
|
+
const captured = visible?.[1];
|
|
372
|
+
if (captured !== undefined && captured.trim() !== '') {
|
|
373
|
+
try {
|
|
374
|
+
const decoded = JSON.parse(`"${captured}"`);
|
|
375
|
+
if (typeof decoded === 'string' && decoded.trim() !== '')
|
|
376
|
+
return decoded;
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
// 放弃该兜底
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* `runCliProcess` 自己做 Windows 的 .cmd shim 解析(resolveCommandForWindows),
|
|
387
|
+
* 所以这里传裸命令名即可;ENOENT 会以 `spawnError` 形式回来 → LLM_UNAVAILABLE。
|
|
388
|
+
*/
|
|
389
|
+
const OPENCLAW_BINARY = 'openclaw';
|
|
390
|
+
/**
|
|
391
|
+
* 显式指定模型(2026-07-28 第三次门禁重跑)。
|
|
392
|
+
*
|
|
393
|
+
* 第三次重跑三条链全部 `exit_code:1`,`llm_unavailable` 桶正确捕获。stderr 证据
|
|
394
|
+
* 显示根因**不在判据也不在 prompt**:配置的默认模型返回
|
|
395
|
+
* `404 model route not found`,failover 无候选 →
|
|
396
|
+
* `FailoverError: The selected model was not found by the provider`。
|
|
397
|
+
*
|
|
398
|
+
* 因此 Spike 不再依赖 openclaw 的默认模型(它会随 Owner 的全局配置漂移),改为
|
|
399
|
+
* 显式传 `--model`:既让「模型不可用」这类环境问题一眼可辨,也避免为了跑 Spike
|
|
400
|
+
* 去改 Owner 的全局 `~/.openclaw/openclaw.json`(那超出本任务的改动边界)。
|
|
401
|
+
* 可用 `PD_SPIKE_MODEL` 覆盖。
|
|
402
|
+
*/
|
|
403
|
+
const SPIKE_MODEL = process.env.PD_SPIKE_MODEL ?? 'sensenova/deepseek-v4-flash';
|
|
404
|
+
async function askAgent(label, message) {
|
|
405
|
+
mkdirSync(SPIKE_TMP_DIR, { recursive: true });
|
|
406
|
+
const messagePath = join(SPIKE_TMP_DIR, `${label}-${Date.now()}.json`);
|
|
407
|
+
writeFileSync(messagePath, message, 'utf8');
|
|
408
|
+
try {
|
|
409
|
+
const result = await runCliProcess({
|
|
410
|
+
command: OPENCLAW_BINARY,
|
|
411
|
+
args: [
|
|
412
|
+
'agent',
|
|
413
|
+
'--agent', 'main',
|
|
414
|
+
// 显式模型:见 SPIKE_MODEL 注释(默认模型 404 曾让整轮门禁误判为不可达)
|
|
415
|
+
'--model', SPIKE_MODEL,
|
|
416
|
+
'--message-file', messagePath,
|
|
417
|
+
'--session-id', `pd-spike-1-2-${label}-${Date.now()}`,
|
|
418
|
+
'--json',
|
|
419
|
+
'--timeout', String(Math.floor(LLM_TIMEOUT_MS / 1000)),
|
|
420
|
+
],
|
|
421
|
+
// 传输层约束(由 .spike-tmp 的 probe4 / probe5 对照实验确定):在
|
|
422
|
+
// vitest 的环境标记(NODE_ENV=test / VITEST=*)与 package 级 cwd 下,
|
|
423
|
+
// openclaw 的 embedded agent 会以 exit 0 + **空 stdout** 结束(stderr 只留
|
|
424
|
+
// 一条 gateway fallback 提示)。同一 payload 在仓库根 cwd + 非 test 环境下
|
|
425
|
+
// 正常返回。因此这里显式还原成生产式环境,否则 LLM 判断根本不会发生。
|
|
426
|
+
cwd: REPO_ROOT,
|
|
427
|
+
env: {
|
|
428
|
+
NODE_ENV: 'production',
|
|
429
|
+
VITEST: '',
|
|
430
|
+
VITEST_WORKER_ID: '',
|
|
431
|
+
VITEST_POOL_ID: '',
|
|
432
|
+
VITEST_MODE: '',
|
|
433
|
+
},
|
|
434
|
+
timeoutMs: LLM_TIMEOUT_MS,
|
|
435
|
+
});
|
|
436
|
+
if (result.spawnError !== undefined) {
|
|
437
|
+
return { kind: 'unavailable', reason: `spawn_error:${result.spawnError}`, rawText: result.stderr };
|
|
438
|
+
}
|
|
439
|
+
if (result.timedOut) {
|
|
440
|
+
return { kind: 'unavailable', reason: 'timeout', rawText: result.stderr };
|
|
441
|
+
}
|
|
442
|
+
// 诊断留痕:整份 stdout / stderr 落盘,便于在 Spike 结论里逐字引用不可达原因
|
|
443
|
+
writeFileSync(join(SPIKE_TMP_DIR, `${label}.stdout.txt`), result.stdout, 'utf8');
|
|
444
|
+
writeFileSync(join(SPIKE_TMP_DIR, `${label}.stderr.txt`), result.stderr, 'utf8');
|
|
445
|
+
if (result.exitCode !== 0) {
|
|
446
|
+
return {
|
|
447
|
+
kind: 'unavailable',
|
|
448
|
+
reason: `exit_code:${String(result.exitCode)}`,
|
|
449
|
+
rawText: `${result.stdout}\n${result.stderr}`.slice(0, 4000),
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
if (result.stdout.trim() === '') {
|
|
453
|
+
return { kind: 'unavailable', reason: 'empty_stdout', rawText: result.stderr.slice(-4000) };
|
|
454
|
+
}
|
|
455
|
+
const text = extractAgentText(result.stdout, result.stderr);
|
|
456
|
+
if (text === null) {
|
|
457
|
+
return {
|
|
458
|
+
kind: 'unavailable',
|
|
459
|
+
reason: 'no_agent_text_in_envelope',
|
|
460
|
+
rawText: `${result.stdout}\n${result.stderr}`.slice(0, 4000),
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
// 拿到回复文本即为「模型答了」。信封是否合法由输出契约通道判定(design §6.5.4)。
|
|
464
|
+
return { kind: 'reply', rawText: text };
|
|
465
|
+
}
|
|
466
|
+
finally {
|
|
467
|
+
rmSync(messagePath, { force: true });
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
// ── Route A:spike-local 三段式 prompt(字段照抄 design §6.5)────────────────
|
|
471
|
+
const SEGMENTS = ['pain_to_dreamer', 'dreamer_to_scribe', 'scribe_to_artificer'];
|
|
472
|
+
const VERDICTS = ['pass', 'degraded', 'fail'];
|
|
473
|
+
const DIMENSION_COVERAGE_POLICY = {
|
|
474
|
+
betterDecision: 'required',
|
|
475
|
+
rationale: 'required',
|
|
476
|
+
riskLevel: 'required',
|
|
477
|
+
badDecision: 'optional',
|
|
478
|
+
strategicPerspective: 'excluded',
|
|
479
|
+
};
|
|
480
|
+
/** 由政策表派生,顺序稳定(确定性输出)。 */
|
|
481
|
+
const REQUIRED_FIDELITY_DIMENSIONS = [
|
|
482
|
+
'betterDecision',
|
|
483
|
+
'rationale',
|
|
484
|
+
'riskLevel',
|
|
485
|
+
];
|
|
486
|
+
const OPTIONAL_FIDELITY_DIMENSIONS = ['badDecision'];
|
|
487
|
+
const EXCLUDED_FIDELITY_DIMENSIONS = ['strategicPerspective'];
|
|
488
|
+
/**
|
|
489
|
+
* rc-5 / ERR-013:以 LLM 字符串为键查表用 `Map`,不用普通对象索引
|
|
490
|
+
* (`table['__proto__']` 会命中 `Object.prototype`)。
|
|
491
|
+
*/
|
|
492
|
+
const POLICY_BY_LOWERCASE_NAME = new Map(Object.entries(DIMENSION_COVERAGE_POLICY).map(([name, coverageClass]) => [
|
|
493
|
+
name.toLowerCase(),
|
|
494
|
+
coverageClass,
|
|
495
|
+
]));
|
|
496
|
+
/**
|
|
497
|
+
* rc-1 / rc-4:LLM 给的维度名是不可信输入。按政策表分类,认不出来的一律 unknown,
|
|
498
|
+
* 不做任何猜测性归并(大小写与首尾空白容错,因为那是书写形式而非语义差异)。
|
|
499
|
+
*/
|
|
500
|
+
function classifyDimension(raw) {
|
|
501
|
+
return POLICY_BY_LOWERCASE_NAME.get(raw.trim().toLowerCase()) ?? 'unknown';
|
|
502
|
+
}
|
|
503
|
+
function partitionByPolicy(values) {
|
|
504
|
+
const required = [];
|
|
505
|
+
const optional = [];
|
|
506
|
+
const ignored = [];
|
|
507
|
+
for (const value of values) {
|
|
508
|
+
switch (classifyDimension(value)) {
|
|
509
|
+
case 'required':
|
|
510
|
+
required.push(value);
|
|
511
|
+
break;
|
|
512
|
+
case 'optional':
|
|
513
|
+
optional.push(value);
|
|
514
|
+
break;
|
|
515
|
+
default:
|
|
516
|
+
ignored.push(value);
|
|
517
|
+
break;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
return { required, optional, ignored };
|
|
521
|
+
}
|
|
522
|
+
/** rc-2:用类型守卫收窄 LLM 给的字符串,不用 `as`。 */
|
|
523
|
+
function isSegmentId(value) {
|
|
524
|
+
return SEGMENTS.some((segment) => segment === value);
|
|
525
|
+
}
|
|
526
|
+
function isSegmentVerdict(value) {
|
|
527
|
+
return VERDICTS.some((verdict) => verdict === value);
|
|
528
|
+
}
|
|
529
|
+
const ROUTE_A_INSTRUCTION = `你是内化管道的 Evaluator。你只能看到每一跳的**摘要级上下文**(headline + 结构化字段),看不到任何原始大字段(没有 pain 证据全文、没有 dreamer 全量候选、没有 artificer 实现代码)。
|
|
530
|
+
|
|
531
|
+
管道是一条逐级压缩的信息链:pain/诊断 → dreamer(维度:badDecision / betterDecision / rationale / riskLevel / strategicPerspective)→ scribe(原则文本)→ artificer(实现)。每一跳都可能丢字段或把具体动作模糊成抽象表述。
|
|
532
|
+
|
|
533
|
+
你的任务:判断这条链上是否发生了信息丢失或语义模糊。若有,指出**发生在哪一段**以及**丢了哪个维度**。若没有,如实判 pass —— 不要为了显得严谨而编造缺失。
|
|
534
|
+
|
|
535
|
+
三段的定义:
|
|
536
|
+
- pain_to_dreamer:pain/诊断的根因与症状,是否在 dreamer 的决策里被覆盖。
|
|
537
|
+
- dreamer_to_scribe:dreamer 的维度,是否在 scribe 的原则文本里被保留(维度值本身丢失,或具体动作被抽象化,都算缺陷)。
|
|
538
|
+
- scribe_to_artificer:scribe 原则文本的约束意图,是否在 artificer 的实现摘要里被忠实实现。
|
|
539
|
+
|
|
540
|
+
【维度分级政策】(**不是五个等权维度**,必须严格按下面分级判定,不得自行升级或降级)
|
|
541
|
+
- required(必需,缺失即缺陷):${REQUIRED_FIDELITY_DIMENSIONS.join(' / ')}。
|
|
542
|
+
- optional(可选,缺失**不是**缺陷):${OPTIONAL_FIDELITY_DIMENSIONS.join(' / ')}。它出现在原则文本的 antiPatterns(反模式/禁止行为清单)里即算覆盖;**不出现不算缺陷**,只写进 optionalUncovered,禁止写进 missingDimensions,也禁止因此把 verdict 判成 degraded 或 fail。
|
|
543
|
+
- excluded(不参与判定):${EXCLUDED_FIDELITY_DIMENSIONS.join(' / ')}。**不得对它下任何结论**:不得写进 missingDimensions,不得写进 optionalUncovered,不得因它影响任何 verdict,也不要在 detail 里把它当成缺失。
|
|
544
|
+
|
|
545
|
+
【覆盖判定口径】(口径由本 prompt 规定,不要自行发明;同一条链上各维度必须用同一把尺)
|
|
546
|
+
1. 覆盖 = **语义等价的对应表述存在**。允许改写、允许换词、允许合并进其他句子;不要求逐字出现,也不要求出现字段名。**但本条不是一张空白许可**:这里的「改写」指用不同措辞表达**同一份可核验内容**;**抽象化不是改写**(见第 2 条)——把可核验的具体动作换成无法核验的概括属于内容丢失,不在本条允许范围内。第 1 条与第 2 条**必须连读**,只按第 1 条判定即误用。
|
|
547
|
+
2. **betterDecision 的覆盖必须同时满足存在性与具体性**。仅当**可核验的具体动作被保留**时才算覆盖。把具体、可核验的动作(例如「审计文件树」「grep 全部 imports」「检查导出依赖图」)替换为**无法核验的抽象表述**(例如「理解架构」「掌握整体结构」「充分评估」)**不算覆盖**——即使抽象表述在语义上指向同一意图、即使核心意图被保留。此时必须把 betterDecision 写进该段的 missingDimensions,并且 compressionFidelity.betterDecisionCovered = false。判定依据只有一条:该表述能否让 Owner 或一段规则代码**判断它有没有被执行**;不能,即为具体性丢失,即不算覆盖。
|
|
548
|
+
3. riskLevel:以风险等级词(high / medium / low 或中文等价)**或**等价的风险描述(例如「跨包改动一旦漏掉调用方会直接编译失败」)体现,两者任一即算覆盖。
|
|
549
|
+
4. badDecision:出现在 antiPatterns(反模式/禁止行为清单)即算覆盖;**不出现不算缺陷**。
|
|
550
|
+
5. strategicPerspective:不参与保真度判定,不要对它下结论,也不得让它出现在 missingDimensions 里。
|
|
551
|
+
6. 未覆盖必须**指名 required 维度名**(${REQUIRED_FIDELITY_DIMENSIONS.join(' / ')}),不得只给定性描述,也不得填别的名字。
|
|
552
|
+
7. **看不见即不裁决,但这条只管「维度的值本身从未被注入过」,不管「下游字段没有同名字段」**:某个维度(riskLevel / betterDecision / rationale / badDecision)由 dreamer 产生,只要 \`dreamer.summary.<维度名>\` 出现在下面的 injectedFields 里,这个维度就**已经可判定**,即使 scribe / artificer 那一跳没有一个字面叫同样名字的字段——scribe 本来就没有 riskLevel / betterDecision 这类字段,它只有 principleText / scope 这类自由文本字段,dreamer_to_scribe 段要判的正是「dreamer 已注入的维度值,有没有被 scribe 的自由文本吸收」,**不是「scribe 有没有一个同名字段」**。只有当某个维度在 dreamer 的 injectedFields 里也完全不存在时,才适用「看不见即不裁决」,判定为该维度在此链路上不可判定,不得据此报缺失。
|
|
553
|
+
|
|
554
|
+
【verdict 规则】
|
|
555
|
+
- 某段的 required 维度全部覆盖 → verdict = "pass",missingDimensions = []。optional 维度未覆盖**不影响** pass。
|
|
556
|
+
- 该段有 required 维度缺失 → "degraded"(部分丢失)或 "fail"(约束意图整体丢失),并在 missingDimensions 里指名。
|
|
557
|
+
- 某段判定 required 维度所需的字段在 injectedFields 里根本不存在 → "degraded",并在 detail 里说明缺哪个字段,不要猜。
|
|
558
|
+
|
|
559
|
+
只输出下面这个 JSON 对象,不要 markdown 代码围栏,不要任何解释性文字:
|
|
560
|
+
{"segments":[{"segment":"pain_to_dreamer","verdict":"pass|degraded|fail","missingDimensions":[],"optionalUncovered":[],"detail":"简述依据"},{"segment":"dreamer_to_scribe","verdict":"pass|degraded|fail","missingDimensions":["riskLevel"],"optionalUncovered":["badDecision"],"detail":"简述依据"},{"segment":"scribe_to_artificer","verdict":"pass|degraded|fail","missingDimensions":[],"optionalUncovered":[],"detail":"简述依据"}],"compressionFidelity":{"betterDecisionCovered":true,"rationaleCovered":true,"riskLevelCovered":false,"badDecisionCovered":true,"missingDimensions":["riskLevel"],"optionalUncovered":[],"explanation":"简述依据"},"painCoverage":{"fullyCovered":true,"uncoveredAspects":[],"explanation":"简述依据"}}
|
|
561
|
+
|
|
562
|
+
约束:
|
|
563
|
+
- 不要调用任何工具,不要读写任何文件,直接基于下面给出的字段作答(这是传输层约束,不影响判断内容)。
|
|
564
|
+
- segments 必须恰好三条,segment 取值必须是 pain_to_dreamer / dreamer_to_scribe / scribe_to_artificer。
|
|
565
|
+
- verdict 必须是 pass / degraded / fail 之一。
|
|
566
|
+
- missingDimensions **只能**含 required 维度名(${REQUIRED_FIDELITY_DIMENSIONS.join(' / ')});没有缺失就给空数组。
|
|
567
|
+
- optionalUncovered **只能**含 optional 维度名(${OPTIONAL_FIDELITY_DIMENSIONS.join(' / ')});没有就给空数组。它只是诊断参考,不是缺陷。
|
|
568
|
+
- compressionFidelity 里**没有** strategicPerspectiveCovered 字段,不要补上。`;
|
|
569
|
+
/**
|
|
570
|
+
* flagged = reasons 非空(design §6.5.3 的后置条件)。
|
|
571
|
+
*
|
|
572
|
+
* 与生产判据的两个差异,如实记录而非悄悄折衷:
|
|
573
|
+
* - 第一条判据只看 **required 维度缺失**(政策过滤后),不是
|
|
574
|
+
* `missingDimensions.length > 0`;optionalUncovered 永不进 reasons。
|
|
575
|
+
* - 第三条判据 `implementationFidelity.score < 0.7` 不在本 Spike 的 prompt 形状里
|
|
576
|
+
* (`implementationFidelity` 属 Layer 2 增量,Route A 不索取),因此它既不进
|
|
577
|
+
* reasons 也不进 undetermined —— 它不是「依据缺失」,而是本次判定范围之外。
|
|
578
|
+
*/
|
|
579
|
+
function evaluateSpikeFlagged(judgment) {
|
|
580
|
+
const reasons = [];
|
|
581
|
+
const undetermined = [];
|
|
582
|
+
if (judgment.compressionFidelityMissing.length > 0) {
|
|
583
|
+
reasons.push('missing_dimensions');
|
|
584
|
+
}
|
|
585
|
+
if (judgment.painFullyCovered === false) {
|
|
586
|
+
reasons.push('pain_not_fully_covered');
|
|
587
|
+
}
|
|
588
|
+
if (judgment.painFullyCovered === null) {
|
|
589
|
+
undetermined.push('painCoverage.fullyCovered missing from reply');
|
|
590
|
+
}
|
|
591
|
+
return { flagged: reasons.length > 0, reasons, undetermined };
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* 容错读取 `compressionFidelity` / `painCoverage`:既接受顶层,也接受被模型
|
|
595
|
+
* 误塞进 `segments` 数组元素里的形态(实测 sensenova-flash-lite 会这样写)。
|
|
596
|
+
*/
|
|
597
|
+
function findNestedRecord(parsed, key) {
|
|
598
|
+
const top = readRecord(parsed, key);
|
|
599
|
+
if (top !== null)
|
|
600
|
+
return top;
|
|
601
|
+
const rawSegments = readArray(parsed, 'segments');
|
|
602
|
+
if (rawSegments === null)
|
|
603
|
+
return null;
|
|
604
|
+
for (const entry of rawSegments) {
|
|
605
|
+
const nested = readRecord(entry, key);
|
|
606
|
+
if (nested !== null)
|
|
607
|
+
return nested;
|
|
608
|
+
}
|
|
609
|
+
return null;
|
|
610
|
+
}
|
|
611
|
+
/** rc-2 / rc-3 / rc-4:逐字段校验 LLM 输出,不用 `as` 收窄,缺失即失败。 */
|
|
612
|
+
function validateRouteA(parsed) {
|
|
613
|
+
const errors = [];
|
|
614
|
+
if (!isRecord(parsed))
|
|
615
|
+
return { ok: false, errors: ['reply is not an object'] };
|
|
616
|
+
const shapeWarnings = [];
|
|
617
|
+
const rawSegments = readArray(parsed, 'segments');
|
|
618
|
+
const segments = [];
|
|
619
|
+
if (rawSegments === null) {
|
|
620
|
+
errors.push('segments must be an array');
|
|
621
|
+
}
|
|
622
|
+
else {
|
|
623
|
+
for (const entry of rawSegments) {
|
|
624
|
+
const segment = readString(entry, 'segment');
|
|
625
|
+
const verdict = readString(entry, 'verdict');
|
|
626
|
+
if (segment === null || !isSegmentId(segment)) {
|
|
627
|
+
shapeWarnings.push(`segments[] entry without a valid segment id: ${String(segment)}`);
|
|
628
|
+
continue;
|
|
629
|
+
}
|
|
630
|
+
if (verdict === null || !isSegmentVerdict(verdict)) {
|
|
631
|
+
errors.push(`segments[].verdict invalid for ${segment}: ${String(verdict)}`);
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
// rc-4:数组元素逐个校验为字符串,再按政策表过滤(design §6.5.3)
|
|
635
|
+
const missingRaw = (readArray(entry, 'missingDimensions') ?? []).filter((v) => typeof v === 'string');
|
|
636
|
+
const declaredOptional = (readArray(entry, 'optionalUncovered') ?? []).filter((v) => typeof v === 'string');
|
|
637
|
+
const partition = partitionByPolicy(missingRaw);
|
|
638
|
+
if (partition.optional.length > 0) {
|
|
639
|
+
shapeWarnings.push(`${segment}.missingDimensions contained optional dimensions (ignored per policy): ${partition.optional.join(', ')}`);
|
|
640
|
+
}
|
|
641
|
+
if (partition.ignored.length > 0) {
|
|
642
|
+
shapeWarnings.push(`${segment}.missingDimensions contained excluded/unknown entries (ignored per policy): ${partition.ignored.join(', ')}`);
|
|
643
|
+
}
|
|
644
|
+
segments.push({
|
|
645
|
+
segment,
|
|
646
|
+
verdict,
|
|
647
|
+
rawMissingDimensions: missingRaw,
|
|
648
|
+
missingDimensions: partition.required,
|
|
649
|
+
optionalUncovered: [...declaredOptional, ...partition.optional],
|
|
650
|
+
detail: readString(entry, 'detail') ?? '',
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
// rc-3:三段必须齐全,缺一即判定不可用(不能静默当作「没问题」)
|
|
655
|
+
for (const required of SEGMENTS) {
|
|
656
|
+
if (!segments.some((s) => s.segment === required)) {
|
|
657
|
+
errors.push(`segment missing from reply: ${required}`);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
const fidelity = findNestedRecord(parsed, 'compressionFidelity');
|
|
661
|
+
if (fidelity !== null && readRecord(parsed, 'compressionFidelity') === null) {
|
|
662
|
+
shapeWarnings.push('compressionFidelity was nested inside segments[] instead of top level');
|
|
663
|
+
}
|
|
664
|
+
const fidelityMissingRaw = (fidelity === null ? [] : readArray(fidelity, 'missingDimensions') ?? []).filter((v) => typeof v === 'string');
|
|
665
|
+
const fidelityDeclaredOptional = (fidelity === null ? [] : readArray(fidelity, 'optionalUncovered') ?? []).filter((v) => typeof v === 'string');
|
|
666
|
+
const fidelityPartition = partitionByPolicy(fidelityMissingRaw);
|
|
667
|
+
if (fidelityPartition.optional.length > 0) {
|
|
668
|
+
shapeWarnings.push(`compressionFidelity.missingDimensions contained optional dimensions (ignored per policy): ${fidelityPartition.optional.join(', ')}`);
|
|
669
|
+
}
|
|
670
|
+
if (fidelityPartition.ignored.length > 0) {
|
|
671
|
+
shapeWarnings.push(`compressionFidelity.missingDimensions contained excluded/unknown entries (ignored per policy): ${fidelityPartition.ignored.join(', ')}`);
|
|
672
|
+
}
|
|
673
|
+
if (fidelity !== null && Object.hasOwn(fidelity, 'strategicPerspectiveCovered')) {
|
|
674
|
+
shapeWarnings.push('compressionFidelity carried strategicPerspectiveCovered (excluded dimension; ignored per policy)');
|
|
675
|
+
}
|
|
676
|
+
const painCoverage = findNestedRecord(parsed, 'painCoverage');
|
|
677
|
+
if (painCoverage !== null && readRecord(parsed, 'painCoverage') === null) {
|
|
678
|
+
shapeWarnings.push('painCoverage was nested inside segments[] instead of top level');
|
|
679
|
+
}
|
|
680
|
+
let painFullyCovered = null;
|
|
681
|
+
if (painCoverage !== null && Object.hasOwn(painCoverage, 'fullyCovered') && typeof painCoverage.fullyCovered === 'boolean') {
|
|
682
|
+
painFullyCovered = painCoverage.fullyCovered;
|
|
683
|
+
}
|
|
684
|
+
if (errors.length > 0)
|
|
685
|
+
return { ok: false, errors };
|
|
686
|
+
return {
|
|
687
|
+
ok: true,
|
|
688
|
+
value: {
|
|
689
|
+
segments,
|
|
690
|
+
compressionFidelityMissing: fidelityPartition.required,
|
|
691
|
+
compressionFidelityMissingRaw: fidelityMissingRaw,
|
|
692
|
+
compressionFidelityOptionalUncovered: [
|
|
693
|
+
...fidelityDeclaredOptional,
|
|
694
|
+
...fidelityPartition.optional,
|
|
695
|
+
],
|
|
696
|
+
painFullyCovered,
|
|
697
|
+
shapeWarnings,
|
|
698
|
+
},
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
function buildRouteAMessage(chain, context) {
|
|
702
|
+
return JSON.stringify({
|
|
703
|
+
instruction: ROUTE_A_INSTRUCTION,
|
|
704
|
+
contextLevel: 'summary_only',
|
|
705
|
+
manifestId: 'evaluator.stage1.v1',
|
|
706
|
+
excludedRawFields: ['pain.raw.evidence', 'dreamer.raw.candidates', 'artificer.raw.implementationCode'],
|
|
707
|
+
injectedFields: context.fields,
|
|
708
|
+
absentFields: context.absent,
|
|
709
|
+
chainId: chain.chainId,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
// ── 输出契约通道(design §6.5.4)───────────────────────────────────────────────
|
|
713
|
+
//
|
|
714
|
+
// 第二次门禁重跑的首要失败形态是「判断内容正确但信封损坏」:`painCoverage` 被写成
|
|
715
|
+
// 兄弟对象、外层对象从未闭合,3/3 不可解析。这里**复用仓库既有通道**处理该路径:
|
|
716
|
+
// 1. `extractJsonObject`(adapter/json-extractor,经 structured-output-repair 再导出)
|
|
717
|
+
// —— 处理 markdown 围栏与前后散文;
|
|
718
|
+
// 2. `repairMalformedJson`(同一模块)—— 本地语法修复,不花 LLM 往返;
|
|
719
|
+
// 3. `attemptStructuredOutputRepair` —— 有界重试(`normalizeMaxRepairAttempts` /
|
|
720
|
+
// `MAX_REPAIR_ATTEMPTS`)、逐次刷新错误(rc-7 / ERR-015 / ERR-018 / ERR-019)、
|
|
721
|
+
// 产出 `repairAttempts` 证据。
|
|
722
|
+
// 本文件不再自带 brace 扫描器(旧 `extractJsonObject` 已删)。
|
|
723
|
+
const ROUTE_A_SCHEMA_REF = 'spike.routeA.stage1.v1';
|
|
724
|
+
/** 供修复 prompt 用的目标形状描述(`RepairConfig.schemaSummary`,PRI-271 A2)。 */
|
|
725
|
+
const ROUTE_A_SCHEMA_SUMMARY = [
|
|
726
|
+
' segments: object[] (required) — 恰好 3 条,segment ∈ {pain_to_dreamer, dreamer_to_scribe, scribe_to_artificer}',
|
|
727
|
+
' segments[].verdict: enum(pass | degraded | fail) (required)',
|
|
728
|
+
' segments[].missingDimensions: string[] (required) — 只含 required 维度名',
|
|
729
|
+
' segments[].optionalUncovered: string[] (required)',
|
|
730
|
+
' segments[].detail: string (required)',
|
|
731
|
+
' compressionFidelity: object (required) — **顶层键**,不得放进 segments 数组',
|
|
732
|
+
' compressionFidelity.{betterDecisionCovered,rationaleCovered,riskLevelCovered,badDecisionCovered}: boolean (required)',
|
|
733
|
+
' compressionFidelity.{missingDimensions,optionalUncovered}: string[] (required)',
|
|
734
|
+
' compressionFidelity.explanation: string (required)',
|
|
735
|
+
' painCoverage: object (required) — **顶层键**,与 compressionFidelity 平级,不得写成 segments 的兄弟对象',
|
|
736
|
+
' painCoverage.fullyCovered: boolean (required)',
|
|
737
|
+
' painCoverage.uncoveredAspects: string[] (required)',
|
|
738
|
+
' painCoverage.explanation: string (required)',
|
|
739
|
+
].join('\n');
|
|
740
|
+
/** 本 Spike 允许的 LLM 修复往返次数(上限由 MAX_REPAIR_ATTEMPTS 收口)。 */
|
|
741
|
+
const ROUTE_A_REPAIR_ATTEMPTS = normalizeMaxRepairAttempts(2, MAX_REPAIR_ATTEMPTS);
|
|
742
|
+
/** 把本地形状校验错误转成修复通道的错误形状(不引入新的错误类型)。 */
|
|
743
|
+
function toSchemaValidationErrors(errors) {
|
|
744
|
+
return errors.map((message) => ({ path: ROUTE_A_SCHEMA_REF, message, value: undefined }));
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* 从回复文本解析 + 校验 Route A 判定;失败则经既有修复通道有界重试。
|
|
748
|
+
*
|
|
749
|
+
* rc-1 / rc-2:解析结果保持 `unknown`,一律经 `validateRouteA` 逐字段校验后才使用;
|
|
750
|
+
* 修复通道的 `output` 也不用 `as` 收窄(泛参显式给 `unknown`)。
|
|
751
|
+
* rc-9 / ERR-002:修复失败带 `repairAttempts` 与经 `safeStringifyPreview` 截断的原文预览。
|
|
752
|
+
*/
|
|
753
|
+
async function resolveRouteAContract(label, rawText) {
|
|
754
|
+
const firstPass = extractJsonObject(rawText) ?? repairMalformedJson(rawText);
|
|
755
|
+
if (firstPass !== null) {
|
|
756
|
+
const validated = validateRouteA(firstPass);
|
|
757
|
+
if (validated.ok) {
|
|
758
|
+
return { kind: 'ok', judgment: validated.value, repairAttempts: 0, repaired: false };
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
const initialErrors = firstPass === null
|
|
762
|
+
? ['reply text contains no parseable JSON object (unbalanced or mis-nested envelope)']
|
|
763
|
+
: (() => {
|
|
764
|
+
const validated = validateRouteA(firstPass);
|
|
765
|
+
return validated.ok ? [] : validated.errors;
|
|
766
|
+
})();
|
|
767
|
+
let repairRound = 0;
|
|
768
|
+
const repair = await attemptStructuredOutputRepair(firstPass ?? rawText, toSchemaValidationErrors(initialErrors), {
|
|
769
|
+
llmCaller: async (prompt) => {
|
|
770
|
+
repairRound += 1;
|
|
771
|
+
const call = await askAgent(`${label}-repair${String(repairRound)}`, prompt);
|
|
772
|
+
return call.kind === 'reply' ? call.rawText : null;
|
|
773
|
+
},
|
|
774
|
+
schemaCheck: (value) => validateRouteA(value).ok,
|
|
775
|
+
// rc-7:每一轮读当轮的新错误,不复用上一轮的陈旧错误
|
|
776
|
+
schemaErrors: (value) => {
|
|
777
|
+
const validated = validateRouteA(value);
|
|
778
|
+
return validated.ok ? [] : toSchemaValidationErrors(validated.errors);
|
|
779
|
+
},
|
|
780
|
+
}, {
|
|
781
|
+
maxRepairAttempts: ROUTE_A_REPAIR_ATTEMPTS,
|
|
782
|
+
schemaRef: ROUTE_A_SCHEMA_REF,
|
|
783
|
+
schemaSummary: ROUTE_A_SCHEMA_SUMMARY,
|
|
784
|
+
// 回复本体 ~2-3 KB:预览必须覆盖全文,否则模型看不到自己写坏的那一段
|
|
785
|
+
maxRawOutputChars: 8000,
|
|
786
|
+
_testJitterMs: 0,
|
|
787
|
+
});
|
|
788
|
+
if (repair.repaired) {
|
|
789
|
+
const validated = validateRouteA(repair.output);
|
|
790
|
+
if (validated.ok) {
|
|
791
|
+
return {
|
|
792
|
+
kind: 'ok',
|
|
793
|
+
judgment: validated.value,
|
|
794
|
+
repairAttempts: repair.attemptsUsed,
|
|
795
|
+
repaired: true,
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
return {
|
|
799
|
+
kind: 'output_contract_violation',
|
|
800
|
+
errors: validated.errors,
|
|
801
|
+
repairAttempts: repair.attemptsUsed,
|
|
802
|
+
rawPreview: safeStringifyPreview(repair.output, 1200),
|
|
803
|
+
};
|
|
804
|
+
}
|
|
805
|
+
return {
|
|
806
|
+
kind: 'output_contract_violation',
|
|
807
|
+
errors: initialErrors.length > 0 ? initialErrors : [repair.repairSummary],
|
|
808
|
+
repairAttempts: repair.attemptsUsed,
|
|
809
|
+
rawPreview: safeStringifyPreview(firstPass ?? rawText, 1200),
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
// ── Route B:生产 evaluator prompt + 生产 validator,上下文换成摘要级 ────────
|
|
813
|
+
function buildRouteBMessage(chain, context) {
|
|
814
|
+
const builder = new EvaluatorPromptBuilder();
|
|
815
|
+
const { message } = builder.buildPrompt({
|
|
816
|
+
taskId: chain.evaluator.taskId,
|
|
817
|
+
contextHash: `spike-summary-${chain.chainId}`,
|
|
818
|
+
sourceArtificerArtifactId: chain.artificer.artifactId,
|
|
819
|
+
// 摘要级 stand-in:只含 headline + 结构化字段,无 implementationCode
|
|
820
|
+
artificerArtifact: {
|
|
821
|
+
summary: {
|
|
822
|
+
headline: context.fields['artificer.summary.headline'],
|
|
823
|
+
changedFiles: context.fields['artificer.summary.changedFiles'],
|
|
824
|
+
apiSurface: context.fields['artificer.summary.apiSurface'],
|
|
825
|
+
risks: context.fields['artificer.summary.risks'],
|
|
826
|
+
},
|
|
827
|
+
predecessorSummary: {
|
|
828
|
+
runnerKind: 'scribe',
|
|
829
|
+
headline: context.fields['artificer.predecessorSummary.headline'],
|
|
830
|
+
},
|
|
831
|
+
upstreamSummaries: {
|
|
832
|
+
dreamer: {
|
|
833
|
+
badDecision: context.fields['dreamer.summary.badDecision'],
|
|
834
|
+
betterDecision: context.fields['dreamer.summary.betterDecision'],
|
|
835
|
+
rationale: context.fields['dreamer.summary.rationale'],
|
|
836
|
+
riskLevel: context.fields['dreamer.summary.riskLevel'],
|
|
837
|
+
},
|
|
838
|
+
pain: {
|
|
839
|
+
rootSymptom: context.fields['pain.summary.rootSymptom'],
|
|
840
|
+
category: context.fields['pain.summary.category'],
|
|
841
|
+
},
|
|
842
|
+
},
|
|
843
|
+
absentFields: context.absent,
|
|
844
|
+
},
|
|
845
|
+
scribeArtifact: {
|
|
846
|
+
summary: {
|
|
847
|
+
principleText: context.fields['scribe.summary.principleText'],
|
|
848
|
+
scope: context.fields['scribe.summary.scope'],
|
|
849
|
+
},
|
|
850
|
+
},
|
|
851
|
+
});
|
|
852
|
+
return message;
|
|
853
|
+
}
|
|
854
|
+
const results = {
|
|
855
|
+
generatedAt: RUN_TIMESTAMP,
|
|
856
|
+
llmRoute: 'openclaw agent --agent main --json (runCliProcess, same spawn path as OpenClawCliRuntimeAdapter)',
|
|
857
|
+
routeA: [],
|
|
858
|
+
routeB: [],
|
|
859
|
+
llmUnavailable: [],
|
|
860
|
+
outputContractViolations: [],
|
|
861
|
+
};
|
|
862
|
+
function persistResults() {
|
|
863
|
+
// CodeRabbit PR #1273 #7: RESULTS_PATH targets .kiro/specs/... which is
|
|
864
|
+
// git-ignored and may not exist on a fresh clone / CI. mkdirSync recursive
|
|
865
|
+
// mirrors askAgent's handling of SPIKE_TMP_DIR (line 513).
|
|
866
|
+
mkdirSync(dirname(RESULTS_PATH), { recursive: true });
|
|
867
|
+
writeFileSync(RESULTS_PATH, `${JSON.stringify(results, null, 2)}\n`, 'utf8');
|
|
868
|
+
}
|
|
869
|
+
// ── Tests ────────────────────────────────────────────────────────────────────
|
|
870
|
+
describe('Phase 0 Spike 1.2 — 摘要级上下文的定位能力(阻塞门)', () => {
|
|
871
|
+
it('摘要级上下文只含 headline + 结构化字段,不含任何原始大字段', () => {
|
|
872
|
+
for (const chain of SPIKE_CHAINS) {
|
|
873
|
+
assertSpikeChainLineageConsistent(chain);
|
|
874
|
+
const context = buildEvaluatorStage1Context(chain);
|
|
875
|
+
const serialized = JSON.stringify(context);
|
|
876
|
+
// 原始大字段的哨兵串:evidence note / 全量候选的 confidence / 实现代码
|
|
877
|
+
expect(serialized).not.toContain('run://spike-rename');
|
|
878
|
+
expect(serialized).not.toContain('function evaluate(input)');
|
|
879
|
+
expect(serialized).not.toContain('goldenTraceCases');
|
|
880
|
+
expect(serialized).not.toContain('causalChain');
|
|
881
|
+
// 注入路径恰好是 EVALUATOR_STAGE1_MANIFEST 的 tier0 ∪ tier1
|
|
882
|
+
expect([...Object.keys(context.fields), ...context.absent].sort()).toEqual([...EVALUATOR_STAGE1_PATHS].sort());
|
|
883
|
+
// 缺陷判定所需的两个关键字段必须在摘要级上下文里可用(否则门无意义)
|
|
884
|
+
expect(context.fields['dreamer.summary.riskLevel']).toBeTruthy();
|
|
885
|
+
expect(context.fields['scribe.summary.principleText']).toBeTruthy();
|
|
886
|
+
}
|
|
887
|
+
});
|
|
888
|
+
// Route A is the real-LLM Phase 0 value-validation gate (design §12.1).
|
|
889
|
+
// The gate has PASSED (2026-07-30, 4th rerun) — its blocking purpose is
|
|
890
|
+
// served. It must NOT run in regular CI: it spawns a real LLM agent
|
|
891
|
+
// (`openclaw agent --agent main`) requiring credentials/keys unavailable
|
|
892
|
+
// in CI, and it is non-deterministic by nature (LLM judgement).
|
|
893
|
+
//
|
|
894
|
+
// To re-run the gate manually (e.g. after a model swap or a criteria
|
|
895
|
+
// change), set PD_SPIKE_RUN_LLM=1 locally with the LLM key configured.
|
|
896
|
+
// Otherwise the test is skipped — never silently passes, never fails CI.
|
|
897
|
+
const runRouteA = process.env.PD_SPIKE_RUN_LLM === '1';
|
|
898
|
+
(runRouteA ? it : it.skip)('Route A:真实 LLM 仅凭摘要级上下文判断三段与维度;对照链不得被误报' + (runRouteA ? '' : '(skipped: PD_SPIKE_RUN_LLM 未设;门禁已于 2026-07-30 通过,CI 不重跑真实 LLM)'), async () => {
|
|
899
|
+
for (const chain of SPIKE_CHAINS) {
|
|
900
|
+
const context = buildEvaluatorStage1Context(chain);
|
|
901
|
+
const call = await askAgent(`routeA-${chain.chainId}`, buildRouteAMessage(chain, context));
|
|
902
|
+
// 桶 1:模型没答(design §6.5.4 的 llm_unavailable)
|
|
903
|
+
if (call.kind === 'unavailable') {
|
|
904
|
+
results.llmUnavailable.push(`${chain.chainId}: ${call.reason}`);
|
|
905
|
+
results.routeA.push({
|
|
906
|
+
chainId: chain.chainId,
|
|
907
|
+
label: chain.label,
|
|
908
|
+
expectedDefect: chain.expectedDefect,
|
|
909
|
+
injectedFieldPaths: Object.keys(context.fields),
|
|
910
|
+
absentFieldPaths: context.absent,
|
|
911
|
+
llmRawReply: call.rawText,
|
|
912
|
+
validation: 'llm_unavailable',
|
|
913
|
+
validationErrors: [call.reason],
|
|
914
|
+
});
|
|
915
|
+
continue;
|
|
916
|
+
}
|
|
917
|
+
const resolution = await resolveRouteAContract(`routeA-${chain.chainId}`, call.rawText);
|
|
918
|
+
// 桶 2:模型答了但信封坏了,且修复通道用尽仍非法
|
|
919
|
+
if (resolution.kind === 'output_contract_violation') {
|
|
920
|
+
results.outputContractViolations.push(`${chain.chainId}: ${resolution.errors.join(' | ')} (repairAttempts=${String(resolution.repairAttempts)})`);
|
|
921
|
+
results.routeA.push({
|
|
922
|
+
chainId: chain.chainId,
|
|
923
|
+
label: chain.label,
|
|
924
|
+
expectedDefect: chain.expectedDefect,
|
|
925
|
+
injectedFieldPaths: Object.keys(context.fields),
|
|
926
|
+
absentFieldPaths: context.absent,
|
|
927
|
+
llmRawReply: call.rawText,
|
|
928
|
+
validation: 'output_contract_violation',
|
|
929
|
+
validationErrors: resolution.errors,
|
|
930
|
+
repairAttempts: resolution.repairAttempts,
|
|
931
|
+
repaired: false,
|
|
932
|
+
rawPreview: resolution.rawPreview,
|
|
933
|
+
});
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
const validated = { ok: true, value: resolution.judgment };
|
|
937
|
+
const flaggedDecision = evaluateSpikeFlagged(validated.value);
|
|
938
|
+
results.routeA.push({
|
|
939
|
+
chainId: chain.chainId,
|
|
940
|
+
label: chain.label,
|
|
941
|
+
expectedDefect: chain.expectedDefect,
|
|
942
|
+
injectedFieldPaths: Object.keys(context.fields),
|
|
943
|
+
absentFieldPaths: context.absent,
|
|
944
|
+
llmRawReply: call.rawText,
|
|
945
|
+
validation: 'ok',
|
|
946
|
+
repairAttempts: resolution.repairAttempts,
|
|
947
|
+
repaired: resolution.repaired,
|
|
948
|
+
namedSegments: validated.value.segments.map((s) => ({
|
|
949
|
+
segment: s.segment,
|
|
950
|
+
verdict: s.verdict,
|
|
951
|
+
missingDimensions: s.missingDimensions,
|
|
952
|
+
rawMissingDimensions: s.rawMissingDimensions,
|
|
953
|
+
optionalUncovered: s.optionalUncovered,
|
|
954
|
+
detail: s.detail,
|
|
955
|
+
})),
|
|
956
|
+
compressionFidelityMissing: validated.value.compressionFidelityMissing,
|
|
957
|
+
compressionFidelityMissingRaw: validated.value.compressionFidelityMissingRaw,
|
|
958
|
+
compressionFidelityOptionalUncovered: validated.value.compressionFidelityOptionalUncovered,
|
|
959
|
+
painFullyCovered: validated.value.painFullyCovered,
|
|
960
|
+
flagged: flaggedDecision.flagged,
|
|
961
|
+
flaggedReasons: flaggedDecision.reasons,
|
|
962
|
+
flaggedUndetermined: flaggedDecision.undetermined,
|
|
963
|
+
shapeWarnings: validated.value.shapeWarnings,
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
persistResults();
|
|
967
|
+
// rc-9 / ERR-002 / ERR-088:两种失败**分两条断言**,诊断信号唯一。
|
|
968
|
+
// 桶 1「模型没答」→ 运行时/环境问题(换 runtime 配置)。
|
|
969
|
+
expect(results.llmUnavailable, `LLM_UNAVAILABLE (model did not answer — spawn error / timeout / empty stdout / no reply in envelope): ${results.llmUnavailable.join('; ')}`).toEqual([]);
|
|
970
|
+
// 桶 2「模型答了但信封坏了」→ 能力/prompt 问题(换模型或改 prompt)。
|
|
971
|
+
expect(results.outputContractViolations, `OUTPUT_CONTRACT_VIOLATION (model answered but shape stayed invalid after up to ${String(ROUTE_A_REPAIR_ATTEMPTS)} repair round(s) via attemptStructuredOutputRepair): ${results.outputContractViolations.join('; ')}`).toEqual([]);
|
|
972
|
+
for (const record of results.routeA) {
|
|
973
|
+
expect(record.validation, `chain ${record.chainId} (${record.validation}) raw reply: ${record.llmRawReply}`).toBe('ok');
|
|
974
|
+
}
|
|
975
|
+
const byChain = new Map(results.routeA.map((r) => [r.chainId, r]));
|
|
976
|
+
// 缺陷链 A:必须点名 dreamer_to_scribe 段 + riskLevel 维度
|
|
977
|
+
const defectA = byChain.get('defect_a_risk_level_dropped');
|
|
978
|
+
expect(defectA).toBeDefined();
|
|
979
|
+
const defectASegment = defectA?.namedSegments?.find((s) => s.segment === 'dreamer_to_scribe');
|
|
980
|
+
expect(defectASegment?.verdict, `defect A raw: ${defectA?.llmRawReply ?? ''}`).not.toBe('pass');
|
|
981
|
+
expect([...(defectASegment?.missingDimensions ?? []), ...(defectA?.compressionFidelityMissing ?? [])]
|
|
982
|
+
.join(' ')
|
|
983
|
+
.toLowerCase(), `defect A raw: ${defectA?.llmRawReply ?? ''}`).toContain('risklevel');
|
|
984
|
+
// 缺陷链 B:必须点名 dreamer_to_scribe 段 + betterDecision(具体动作被抽象化)
|
|
985
|
+
const defectB = byChain.get('defect_b_action_abstracted');
|
|
986
|
+
expect(defectB).toBeDefined();
|
|
987
|
+
const defectBSegment = defectB?.namedSegments?.find((s) => s.segment === 'dreamer_to_scribe');
|
|
988
|
+
expect(defectBSegment?.verdict, `defect B raw: ${defectB?.llmRawReply ?? ''}`).not.toBe('pass');
|
|
989
|
+
expect([...(defectBSegment?.missingDimensions ?? []), ...(defectB?.compressionFidelityMissing ?? [])]
|
|
990
|
+
.join(' ')
|
|
991
|
+
.toLowerCase(), `defect B raw: ${defectB?.llmRawReply ?? ''}`).toContain('betterdecision');
|
|
992
|
+
// 对照链:假阳性守卫(design §12.1「门禁再跑」+ CP-36)——
|
|
993
|
+
// dreamer_to_scribe 判 pass、政策过滤后的 required 缺失为空、flagged 为 false。
|
|
994
|
+
// `badDecision`(optional)/ `strategicPerspective`(excluded)出现在
|
|
995
|
+
// optionalUncovered 里**不构成失败**,只作为诊断留痕。
|
|
996
|
+
const control = byChain.get('control_no_defect');
|
|
997
|
+
expect(control).toBeDefined();
|
|
998
|
+
const controlSegment = control?.namedSegments?.find((s) => s.segment === 'dreamer_to_scribe');
|
|
999
|
+
expect(controlSegment?.verdict, `control raw: ${control?.llmRawReply ?? ''}`).toBe('pass');
|
|
1000
|
+
expect([...(controlSegment?.missingDimensions ?? []), ...(control?.compressionFidelityMissing ?? [])], `control raw: ${control?.llmRawReply ?? ''}`).toEqual([]);
|
|
1001
|
+
expect(control?.flagged, `control flagged reasons: ${(control?.flaggedReasons ?? []).join(', ')}; raw: ${control?.llmRawReply ?? ''}`).toBe(false);
|
|
1002
|
+
}, LLM_TIMEOUT_MS * 4);
|
|
1003
|
+
// Route B 在第三次门禁重跑中**不执行**:第二次重跑里它每条链都跑过传输层超时
|
|
1004
|
+
// (生产 evaluator prompt 体积远大于 Route A 的 spike-local prompt),整个测试挂到
|
|
1005
|
+
// 超时上限之后才结束,且它是纯记录性路由(不参与门禁判定)。保留代码与形状,
|
|
1006
|
+
// 显式 skip 并写明原因(不删测试、不假装通过)。
|
|
1007
|
+
it.skip('Route B(第三次门禁重跑中跳过:第二次重跑挂在传输层超时;纯记录性路由,不参与门禁判定)', async () => {
|
|
1008
|
+
const validator = new DefaultEvaluatorValidator();
|
|
1009
|
+
for (const chain of SPIKE_CHAINS) {
|
|
1010
|
+
const context = buildEvaluatorStage1Context(chain);
|
|
1011
|
+
const call = await askAgent(`routeB-${chain.chainId}`, buildRouteBMessage(chain, context));
|
|
1012
|
+
const rawText = call.rawText;
|
|
1013
|
+
const parsed = call.kind === 'reply'
|
|
1014
|
+
? extractJsonObject(rawText) ?? repairMalformedJson(rawText)
|
|
1015
|
+
: null;
|
|
1016
|
+
const validation = await validator.validate(parsed, chain.evaluator.taskId, chain.artificer.artifactId);
|
|
1017
|
+
const evaluation = readRecord(parsed, 'evaluation');
|
|
1018
|
+
const concerns = evaluation === null ? null : readArray(evaluation, 'concerns');
|
|
1019
|
+
const requiredChanges = evaluation === null ? null : readArray(evaluation, 'requiredChanges');
|
|
1020
|
+
results.routeB.push({
|
|
1021
|
+
chainId: chain.chainId,
|
|
1022
|
+
llmRawReply: rawText,
|
|
1023
|
+
productionValidatorValid: validation.valid,
|
|
1024
|
+
productionValidatorErrors: validation.errors,
|
|
1025
|
+
concerns: (concerns ?? []).filter((c) => typeof c === 'string'),
|
|
1026
|
+
requiredChanges: (requiredChanges ?? []).filter((c) => typeof c === 'string'),
|
|
1027
|
+
decision: evaluation === null ? null : readString(evaluation, 'decision'),
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
persistResults();
|
|
1031
|
+
// ERR-088:不能用「拿到了某段文本」当依据 —— 传输层不可达时 stderr 也是文本。
|
|
1032
|
+
// 唯一指向「LLM 真的判过」的信号是生产 validator 至少看到一个对象化输出,
|
|
1033
|
+
// 因此断言输出被解析成对象(校验错误可以有,但不能是 'Output is not an object')。
|
|
1034
|
+
for (const record of results.routeB) {
|
|
1035
|
+
expect(record.productionValidatorErrors, `chain ${record.chainId} produced no parseable LLM output; raw: ${record.llmRawReply.slice(0, 500)}`).not.toContain('Output is not an object');
|
|
1036
|
+
}
|
|
1037
|
+
}, LLM_TIMEOUT_MS * 4);
|
|
1038
|
+
});
|
|
1039
|
+
//# sourceMappingURL=progressive-disclosure-spike.test.js.map
|