dsh-plan-completion 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,392 @@
1
+ /**
2
+ * host/evidence.js — 从 SessionEvent 数组采集「证据」(纯函数,零依赖)。
3
+ *
4
+ * 事件契约已按 probe-verify 的实测结果(`research/p0-contract.md` §9)对齐:
5
+ * SessionEvent = { type: string, seq: number, time: number, data: <按 type 映射>, ignorable? }
6
+ * turn/start data = { turn }
7
+ * turn/end data = { turn, reason: { kind, error? } }
8
+ * user/message data = UserMessage { role:'user', content: ContentBlock[] }
9
+ * assistant/message data = { turn, step, message: AssistantMessage { content: ContentBlock[] }, stream, usage? }
10
+ * tool/call data = { turn, step, callId, name, arguments: JSON string }
11
+ * tool/result data = { turn, step, message: ToolResultMessage { toolCallId, isError?, content }, error?: {name,code,reason?} }
12
+ * 文本只取 `type === 'text'` 的 block(reasoning / tool-call 等块不算正文)。
13
+ *
14
+ * 输出字段(与 index.js 接线约定一致):
15
+ * turn 本回合编号(number)
16
+ * lastAssistantText 最后一条 assistant 正文(截断 1500 字)
17
+ * toolCalls 本回合工具调用 [{name, ok, callId?, error?, path?}](失败结果按 callId 回填)
18
+ * failedTools 其中失败的项
19
+ * lastUserMessages 最近 2 条用户消息(string[],各截断 500 字)
20
+ * todoSteps 当前计划里未完成步骤 [{id,title,status}]
21
+ * 另附:sessionId、seq、watermark、intent、firstUserMessage、recentMessages、files、
22
+ * hasQuestion、turnEnded、turnEndReason、turnEndError、interrupted、plan、counts。
23
+ *
24
+ * 纯函数:不修改 events 数组、不修改数组元素、不修改传入的 plan。
25
+ */
26
+
27
+ import { DEFAULTS } from './config.js';
28
+ import { clampChars, looksLikeQuestion, oneLine, toText } from './text.js';
29
+
30
+ const TYPE_SETS = {
31
+ turnStart: new Set(['turn/start', 'turn.start', 'turn_start', 'turn/started']),
32
+ turnEnd: new Set(['turn/end', 'turn.end', 'turn_end', 'turn/ended', 'turn/complete', 'turn/completed']),
33
+ user: new Set(['user/message', 'user.message', 'user_message', 'message/user']),
34
+ assistant: new Set(['assistant/message', 'assistant.message', 'assistant_message', 'message/assistant']),
35
+ toolCall: new Set(['tool/call', 'tool.call', 'tool_call', 'tool/start', 'tool/begin']),
36
+ toolResult: new Set(['tool/result', 'tool.result', 'tool_result', 'tool/end', 'tool/done', 'tool/finish']),
37
+ };
38
+
39
+ const BLOCK_IGNORED_TYPES = new Set(['reasoning', 'image', 'file', 'tool-call', 'tool-addition', 'tool-removal']);
40
+
41
+ const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
42
+
43
+ function typeOf(event) {
44
+ if (!isObj(event)) return '';
45
+ const raw = event.type ?? event.kind ?? event.event ?? event.name;
46
+ return toText(raw).toLowerCase();
47
+ }
48
+
49
+ function payloadOf(event) {
50
+ if (!isObj(event)) return {};
51
+ const candidates = [event.data, event.payload, event.detail, event.value];
52
+ for (const candidate of candidates) {
53
+ if (isObj(candidate)) return candidate;
54
+ }
55
+ return event;
56
+ }
57
+
58
+ function seqOf(event, index) {
59
+ const direct = event?.seq ?? event?.sequence ?? payloadOf(event).seq;
60
+ if (Number.isInteger(direct) && direct >= 0) return direct;
61
+ return index + 1;
62
+ }
63
+
64
+ function intOrNull(value) {
65
+ return Number.isInteger(value) && value >= 0 ? value : null;
66
+ }
67
+
68
+ /**
69
+ * 从消息节点里提取正文:只取 `type === 'text'` 的 ContentBlock。
70
+ * @param {unknown} node
71
+ * @param {number} [depth]
72
+ * @returns {string}
73
+ */
74
+ function extractText(node, depth = 0) {
75
+ if (depth > 6 || node == null) return '';
76
+ if (typeof node === 'string') return node;
77
+ if (typeof node === 'number') return String(node);
78
+ if (Array.isArray(node)) {
79
+ return node
80
+ .map((item) => extractText(item, depth + 1))
81
+ .filter(Boolean)
82
+ .join('');
83
+ }
84
+ if (!isObj(node)) return '';
85
+ const blockType = typeof node.type === 'string' ? node.type : '';
86
+ if (blockType) {
87
+ if (blockType === 'text') return typeof node.text === 'string' ? node.text : '';
88
+ if (BLOCK_IGNORED_TYPES.has(blockType)) return '';
89
+ }
90
+ if (typeof node.text === 'string') return node.text;
91
+ if (typeof node.value === 'string') return node.value;
92
+ if (typeof node.delta === 'string') return node.delta;
93
+ for (const key of ['content', 'message', 'parts', 'blocks', 'body', 'output']) {
94
+ if (node[key] != null) {
95
+ const text = extractText(node[key], depth + 1);
96
+ if (text) return text;
97
+ }
98
+ }
99
+ return '';
100
+ }
101
+
102
+ function messageOf(payload) {
103
+ const message = payload?.message ?? payload?.msg ?? payload?.content ?? payload;
104
+ const text = oneLine(extractText(message));
105
+ const roleRaw = payload?.role ?? message?.role ?? payload?.message?.role;
106
+ return { text, role: toText(roleRaw).toLowerCase(), message };
107
+ }
108
+
109
+ function errorTextOf(raw) {
110
+ if (raw == null) return '';
111
+ if (typeof raw === 'string') return oneLine(raw);
112
+ if (Array.isArray(raw)) {
113
+ return raw
114
+ .map((item) => errorTextOf(item))
115
+ .filter(Boolean)
116
+ .join('; ');
117
+ }
118
+ if (!isObj(raw)) return oneLine(String(raw));
119
+ const parts = [raw.message, raw.reason, raw.error?.message, raw.name, raw.code, raw.status]
120
+ .map((part) => toText(part))
121
+ .filter(Boolean);
122
+ if (parts.length) return oneLine(parts.join(' '));
123
+ return oneLine(extractText(raw));
124
+ }
125
+
126
+ function parseMaybeJson(value) {
127
+ if (typeof value !== 'string') return value;
128
+ const text = value.trim();
129
+ if (!text) return value;
130
+ if (!text.startsWith('{') && !text.startsWith('[')) return value;
131
+ try {
132
+ return JSON.parse(text);
133
+ } catch {
134
+ return value;
135
+ }
136
+ }
137
+
138
+ function inputPathOf(input) {
139
+ const parsed = parseMaybeJson(input);
140
+ if (!isObj(parsed)) return '';
141
+ for (const key of ['file_path', 'filePath', 'path', 'file', 'target', 'filename']) {
142
+ const value = parsed[key];
143
+ if (typeof value === 'string' && value.trim()) return value.trim();
144
+ }
145
+ return '';
146
+ }
147
+
148
+ function toolCallOf(payload) {
149
+ const call = isObj(payload?.call)
150
+ ? payload.call
151
+ : isObj(payload?.toolCall)
152
+ ? payload.toolCall
153
+ : isObj(payload?.tool)
154
+ ? payload.tool
155
+ : payload;
156
+ const name =
157
+ toText(call?.name ?? call?.toolName ?? call?.tool_name ?? payload?.name ?? payload?.toolName ?? '') || '(unknown)';
158
+ const callId =
159
+ toText(
160
+ call?.callId ??
161
+ call?.call_id ??
162
+ call?.id ??
163
+ call?.toolCallId ??
164
+ payload?.callId ??
165
+ payload?.call_id ??
166
+ payload?.toolCallId ??
167
+ payload?.id ??
168
+ '',
169
+ ) || '';
170
+ const input = parseMaybeJson(
171
+ call?.input ?? call?.args ?? call?.arguments ?? payload?.input ?? payload?.args ?? payload?.arguments ?? null,
172
+ );
173
+ return { name, callId, input };
174
+ }
175
+
176
+ function isErrorResult(payload) {
177
+ if (!isObj(payload)) return false;
178
+ const message = payload.message;
179
+ if (isObj(message) && message.isError === true) return true;
180
+ if (payload.isError === true || payload.is_error === true) return true;
181
+ if (payload.ok === false || payload.success === false || payload.failed === true) return true;
182
+ const status = toText(payload.status).toLowerCase();
183
+ if (status === 'error' || status === 'failed' || status === 'failure') return true;
184
+ return payload.error != null;
185
+ }
186
+
187
+ function resultErrorText(payload) {
188
+ const message = isObj(payload?.message) ? payload.message : null;
189
+ if (message && message.isError === true) {
190
+ const fromBlock = oneLine(extractText(message.content));
191
+ if (fromBlock) return fromBlock;
192
+ }
193
+ const raw = payload?.error ?? payload?.result?.error ?? payload?.output?.error ?? message?.error ?? null;
194
+ const fromError = errorTextOf(raw);
195
+ if (fromError) return fromError;
196
+ return oneLine(extractText(message?.content));
197
+ }
198
+
199
+ function todoStepsOf(plan) {
200
+ if (!isObj(plan) || !Array.isArray(plan.steps)) return [];
201
+ return plan.steps
202
+ .filter((step) => isObj(step) && step.status !== 'done' && step.status !== 'dropped')
203
+ .map((step) => ({ id: toText(step.id), title: toText(step.title), status: toText(step.status) || 'todo' }));
204
+ }
205
+
206
+ function emptyEvidence(error) {
207
+ return {
208
+ ok: false,
209
+ error,
210
+ sessionId: '',
211
+ turn: 0,
212
+ seq: 0,
213
+ watermark: { turn: 0, seq: 0 },
214
+ intent: '',
215
+ plan: null,
216
+ lastAssistantText: '',
217
+ toolCalls: [],
218
+ failedTools: [],
219
+ lastUserMessages: [],
220
+ firstUserMessage: '',
221
+ recentMessages: [],
222
+ todoSteps: [],
223
+ files: [],
224
+ hasQuestion: false,
225
+ turnEnded: false,
226
+ turnEndReason: '',
227
+ turnEndError: '',
228
+ interrupted: false,
229
+ counts: { events: 0, toolCalls: 0, failedTools: 0, messages: 0, userMessages: 0 },
230
+ };
231
+ }
232
+
233
+ function messageRecord(payload) {
234
+ return messageOf(payload);
235
+ }
236
+
237
+ /**
238
+ * 从 SessionEvent 数组采集证据。
239
+ *
240
+ * @param {{events?: unknown, plan?: object|null, turn?: number, sessionId?: string}} params
241
+ * @returns {object} 证据对象(见文件头注释;`ok:false` 表示入参不是事件数组)
242
+ */
243
+ export function collectFromEvents(params = {}) {
244
+ try {
245
+ const events = params?.events;
246
+ if (!Array.isArray(events)) return emptyEvidence('events 必须是数组');
247
+ const basePlan = isObj(params?.plan) ? params.plan : null;
248
+
249
+ // ── 1. 水位:全量事件的最大 seq;回合:最后一个 turn/start 之后 ──────
250
+ let seq = 0;
251
+ let lastTurnStartIndex = -1;
252
+ let turnFromEvents = null;
253
+ let turnEnded = false;
254
+ let turnEndReason = '';
255
+ let turnEndError = '';
256
+ for (let i = 0; i < events.length; i += 1) {
257
+ const event = events[i];
258
+ const type = typeOf(event);
259
+ const eventSeq = seqOf(event, i);
260
+ if (eventSeq > seq) seq = eventSeq;
261
+ if (TYPE_SETS.turnStart.has(type)) {
262
+ lastTurnStartIndex = i;
263
+ const data = payloadOf(event);
264
+ turnFromEvents = intOrNull(data.turn) ?? turnFromEvents;
265
+ turnEnded = false;
266
+ turnEndReason = '';
267
+ turnEndError = '';
268
+ } else if (TYPE_SETS.turnEnd.has(type)) {
269
+ turnEnded = true;
270
+ const data = payloadOf(event);
271
+ turnFromEvents = intOrNull(data.turn) ?? turnFromEvents;
272
+ const reason = isObj(data.reason) ? data.reason : null;
273
+ turnEndReason = toText(reason?.kind) || toText(data.reason);
274
+ turnEndError = errorTextOf(reason?.error ?? data.error);
275
+ }
276
+ }
277
+ const scopeStart = lastTurnStartIndex >= 0 ? lastTurnStartIndex : 0;
278
+ const explicitTurn = intOrNull(params?.turn);
279
+ const turn = explicitTurn ?? turnFromEvents ?? 0;
280
+
281
+ // ── 2. 消息:全量扫描(最近 2 条用户消息可能跨回合)──────────────
282
+ let lastAssistantText = '';
283
+ let firstUserMessage = '';
284
+ const userTexts = [];
285
+ const recentMessages = [];
286
+
287
+ for (let i = 0; i < events.length; i += 1) {
288
+ const event = events[i];
289
+ const type = typeOf(event);
290
+ const payload = payloadOf(event);
291
+ const isUser = TYPE_SETS.user.has(type) || (!TYPE_SETS.assistant.has(type) && messageRecord(payload).role === 'user');
292
+ const isAssistant = TYPE_SETS.assistant.has(type) || messageRecord(payload).role === 'assistant';
293
+ if (!isUser && !isAssistant) continue;
294
+ const record = messageRecord(payload);
295
+ if (!record.text) continue;
296
+ if (isUser) {
297
+ const text = clampChars(record.text, DEFAULTS.userMessageMaxChars);
298
+ userTexts.push(text);
299
+ if (!firstUserMessage) firstUserMessage = text;
300
+ recentMessages.push({ role: 'user', text: clampChars(record.text, DEFAULTS.recentMessageChars) });
301
+ } else {
302
+ lastAssistantText = clampChars(record.text, DEFAULTS.assistantTextMaxChars);
303
+ recentMessages.push({ role: 'assistant', text: clampChars(record.text, DEFAULTS.recentMessageChars) });
304
+ }
305
+ }
306
+
307
+ // ── 3. 工具调用:只取本回合(last turn/start 之后)──────────────
308
+ const toolCalls = [];
309
+ const callIndex = new Map();
310
+ for (let i = scopeStart; i < events.length; i += 1) {
311
+ const event = events[i];
312
+ const type = typeOf(event);
313
+ const payload = payloadOf(event);
314
+ if (TYPE_SETS.toolCall.has(type)) {
315
+ const call = toolCallOf(payload);
316
+ const record = { name: call.name, ok: true };
317
+ if (call.callId) record.callId = call.callId;
318
+ const path = inputPathOf(call.input);
319
+ if (path) record.path = path;
320
+ toolCalls.push(record);
321
+ if (call.callId) callIndex.set(call.callId, toolCalls.length - 1);
322
+ continue;
323
+ }
324
+ if (TYPE_SETS.toolResult.has(type)) {
325
+ const message = isObj(payload?.message) ? payload.message : null;
326
+ const callId = toText(
327
+ payload?.callId ??
328
+ payload?.call_id ??
329
+ payload?.toolCallId ??
330
+ payload?.id ??
331
+ message?.toolCallId ??
332
+ message?.source?.callId ??
333
+ '',
334
+ );
335
+ const ok = !isErrorResult(payload);
336
+ let record = callId ? toolCalls[callIndex.get(callId)] : undefined;
337
+ if (!record) {
338
+ const fallbackName = toText(payload?.name ?? payload?.toolName) || toolCallOf(payload).name;
339
+ record = { name: fallbackName && fallbackName !== '(unknown)' ? fallbackName : 'tool', ok: true };
340
+ toolCalls.push(record);
341
+ if (callId) record.callId = callId;
342
+ }
343
+ record.ok = ok;
344
+ if (!ok) record.error = resultErrorText(payload) || '工具执行失败';
345
+ }
346
+ }
347
+
348
+ // ── 4. 汇总 ────────────────────────────────────────────────
349
+ const failedTools = toolCalls.filter((call) => call.ok === false);
350
+ const dedupedUserTexts = [];
351
+ for (const text of userTexts) {
352
+ if (dedupedUserTexts[dedupedUserTexts.length - 1] !== text) dedupedUserTexts.push(text);
353
+ }
354
+ const lastUserMessages = dedupedUserTexts.slice(-2);
355
+ const plan = basePlan;
356
+ const files = [...new Set(toolCalls.filter((call) => call.path).map((call) => call.path))].slice(-DEFAULTS.maxToolRecords);
357
+ const intent = isObj(plan) && typeof plan.intent === 'string' && plan.intent ? plan.intent : firstUserMessage;
358
+
359
+ return {
360
+ ok: true,
361
+ error: '',
362
+ sessionId: toText(params?.sessionId) || (isObj(plan) ? toText(plan.sessionId) : ''),
363
+ turn,
364
+ seq,
365
+ watermark: { turn, seq },
366
+ intent: clampChars(intent, DEFAULTS.maxIntentChars),
367
+ plan,
368
+ lastAssistantText,
369
+ toolCalls: toolCalls.slice(0, DEFAULTS.maxToolRecords),
370
+ failedTools,
371
+ lastUserMessages,
372
+ firstUserMessage,
373
+ recentMessages: recentMessages.slice(-DEFAULTS.recentMessagesMax),
374
+ todoSteps: todoStepsOf(plan),
375
+ files,
376
+ hasQuestion: looksLikeQuestion(lastAssistantText),
377
+ turnEnded,
378
+ turnEndReason,
379
+ turnEndError,
380
+ interrupted: turnEndReason === 'interrupted' || turnEndReason === 'aborted' || turnEndReason === 'error',
381
+ counts: {
382
+ events: events.length,
383
+ toolCalls: toolCalls.length,
384
+ failedTools: failedTools.length,
385
+ messages: recentMessages.length,
386
+ userMessages: dedupedUserTexts.length,
387
+ },
388
+ };
389
+ } catch (err) {
390
+ return emptyEvidence(`证据采集异常:${err && err.message ? err.message : String(err)}`);
391
+ }
392
+ }
@@ -0,0 +1,261 @@
1
+ /**
2
+ * host/experiment.js — isolated prompt-budget experiment (round 2).
3
+ *
4
+ * Fixed session snapshot + fixed plan; variables:
5
+ * - evidenceChars: explicit character budget for the serialized evidence object
6
+ * (truncated from the least important fields first), and
7
+ * - maxTokens: the output budget for the plan/suggestion JSON.
8
+ *
9
+ * Each variant runs at temperature 0 and is scored by a second (judge) model call.
10
+ * Exposed only through the temporary `/plan-completion/experiment` route.
11
+ */
12
+
13
+ function parseLooseJson(text) {
14
+ try {
15
+ const cleaned = String(text ?? '').replace(/```(?:json)?/gi, '').trim();
16
+ const match = cleaned.match(/\{[\s\S]*\}/);
17
+ if (!match) return null;
18
+ return JSON.parse(match[0]);
19
+ } catch (error) {
20
+ return null;
21
+ }
22
+ }
23
+
24
+ const JUDGE_SYSTEM = [
25
+ '你是严格但务实的评分员。只输出一个 JSON 对象,不要输出任何解释、markdown 或代码块。',
26
+ '评分维度(1-5 分,5 最好):',
27
+ ' coherence:计划步骤的状态与实际证据是否一致(无凭空完成、无遗漏明显进度)。',
28
+ ' specificity:下一步建议是否具体可执行(有明确动作和对象),而不是空话。',
29
+ ' consistency:建议是否与计划不矛盾、是否指向计划的下一步。',
30
+ ' usable:该建议是否是一句人可以直接发送的话(<=60 字,单行,无解释)。',
31
+ '重要:若候选输出声称某个步骤完成、而基准计划尚未体现,但候选原始输出给出了具体证据',
32
+ '(工具名/文件/命令结果),视为合理进度更新,不算 coherence 扣分;只有“无证据的完成声明”才扣分。',
33
+ '输出格式:{"coherence":4,"specificity":5,"consistency":4,"usable":true,"issue":"<=20字"}',
34
+ ].join('\n');
35
+
36
+ function judgeMessage(basePlan, rawOutput, suggestion) {
37
+ return [
38
+ '【评估对象】一次「维护会话计划 + 生成下一条用户建议」的模型输出。',
39
+ '【基准计划(当前真实状态)】',
40
+ JSON.stringify(
41
+ basePlan
42
+ ? {
43
+ version: basePlan.version,
44
+ intent: basePlan.intent,
45
+ steps: (basePlan.steps || []).map(step => ({ id: step.id, title: step.title, status: step.status })),
46
+ }
47
+ : null,
48
+ null,
49
+ 2,
50
+ ),
51
+ '【候选建议】',
52
+ suggestion || '',
53
+ '【候选原始输出前 900 字】',
54
+ String(rawOutput || '').slice(0, 900),
55
+ '【评分要求】对照基准计划评估候选输出,按系统提示的 JSON 格式输出。',
56
+ ].join('\n');
57
+ }
58
+
59
+ /** Trim the serialized evidence down to `targetChars`, dropping the least important parts first. */
60
+ export function sliceEvidence(evidence, targetChars) {
61
+ const copy = (() => {
62
+ try {
63
+ return JSON.parse(JSON.stringify(evidence ?? {}));
64
+ } catch (error) {
65
+ return { ok: false };
66
+ }
67
+ })();
68
+ const size = () => JSON.stringify(copy).length;
69
+ if (size() <= targetChars) return copy;
70
+
71
+ // 1. Trim the long assistant tail.
72
+ if (typeof copy.lastAssistantText === 'string') {
73
+ const overflow = size() - targetChars;
74
+ copy.lastAssistantText = copy.lastAssistantText.slice(0, Math.max(0, copy.lastAssistantText.length - overflow - 40));
75
+ if (!copy.lastAssistantText) delete copy.lastAssistantText;
76
+ }
77
+ // 2. Drop recent messages from the oldest end.
78
+ while (size() > targetChars && Array.isArray(copy.recentMessages) && copy.recentMessages.length > 1) {
79
+ copy.recentMessages.shift();
80
+ }
81
+ // 3. Drop tool calls from the oldest end.
82
+ while (size() > targetChars && Array.isArray(copy.toolCalls) && copy.toolCalls.length > 1) {
83
+ copy.toolCalls.shift();
84
+ }
85
+ // 4. Drop secondary fields entirely.
86
+ for (const field of ['failedTools', 'files', 'todoSteps', 'counts', 'recentMessages', 'toolCalls', 'evidence', 'lastUserMessages']) {
87
+ if (size() <= targetChars) break;
88
+ if (field in copy) delete copy[field];
89
+ }
90
+ // 5. Hard cap as a last resort.
91
+ if (size() > targetChars && typeof copy.lastAssistantText === 'string') {
92
+ copy.lastAssistantText = copy.lastAssistantText.slice(0, Math.max(0, targetChars - 160));
93
+ }
94
+ return copy;
95
+ }
96
+
97
+ async function runVariant({ deps, evidenceChars, maxTokens, rep, index, total }) {
98
+ const {
99
+ session,
100
+ sessionId,
101
+ basePlan,
102
+ collectFromEvents,
103
+ buildPatchPrompt,
104
+ parseModelOutput,
105
+ callModel,
106
+ route,
107
+ judgeRoute,
108
+ debugLog,
109
+ } = deps;
110
+ const startedAt = Date.now();
111
+ let seqNow = 0;
112
+ try {
113
+ seqNow = Number(session.seq ?? 0);
114
+ } catch (error) {
115
+ seqNow = 0;
116
+ }
117
+ let events = [];
118
+ try {
119
+ events = Array.from(session.snapshotEvents(0) ?? []);
120
+ } catch (error) {
121
+ events = [];
122
+ }
123
+ const fullEvidence = (() => {
124
+ try {
125
+ return collectFromEvents({ events, plan: basePlan, turn: Number(basePlan?.watermark?.turn ?? 0), sessionId });
126
+ } catch (error) {
127
+ return { ok: false };
128
+ }
129
+ })();
130
+ const evidence = deps.evidenceChars
131
+ ? sliceEvidence(fullEvidence, evidenceChars)
132
+ : fullEvidence;
133
+ const evidenceSize = JSON.stringify(evidence).length;
134
+ const prompt = (() => {
135
+ try {
136
+ return buildPatchPrompt(evidence);
137
+ } catch (error) {
138
+ return null;
139
+ }
140
+ })();
141
+
142
+ const result = {
143
+ index,
144
+ total,
145
+ evidenceChars,
146
+ fullEvidenceChars: JSON.stringify(fullEvidence).length,
147
+ slicedEvidenceChars: evidenceSize,
148
+ maxTokens,
149
+ rep,
150
+ parseOk: false,
151
+ suggestion: '',
152
+ suggestionChars: 0,
153
+ changeCount: 0,
154
+ inputTokens: 0,
155
+ outputTokens: 0,
156
+ latencyMs: 0,
157
+ judge: null,
158
+ judgeRaw: '',
159
+ error: '',
160
+ };
161
+ if (!prompt) {
162
+ result.error = 'no-prompt';
163
+ return result;
164
+ }
165
+ prompt.maxTokens = maxTokens;
166
+ prompt.temperature = 0;
167
+
168
+ try {
169
+ const response = await callModel(deps.services, route, prompt, undefined);
170
+ result.inputTokens = response.usage?.inputTokens ?? 0;
171
+ result.outputTokens = response.usage?.outputTokens ?? 0;
172
+ const parsed = parseModelOutput(response.text);
173
+ result.parseOk = parsed?.ok === true;
174
+ if (parsed?.ok) {
175
+ result.suggestion = parsed.output.suggestion || '';
176
+ result.suggestionChars = result.suggestion.length;
177
+ result.changeCount = (parsed.output.changes || []).length + (parsed.output.steps || []).length;
178
+ }
179
+ result.latencyMs = Date.now() - startedAt;
180
+
181
+ try {
182
+ const judgePrompt = {
183
+ system: JUDGE_SYSTEM,
184
+ messages: [{ role: 'user', content: judgeMessage(basePlan, response.text, result.suggestion) }],
185
+ maxTokens: 500,
186
+ temperature: 0,
187
+ };
188
+ const judged = await callModel(deps.services, judgeRoute || route, judgePrompt, undefined);
189
+ result.judgeRaw = String(judged.text || '').slice(0, 400);
190
+ result.judgeModel = judgeRoute ? judgeRoute.model : route.model;
191
+ const score = parseLooseJson(judged.text);
192
+ if (score && typeof score === 'object') {
193
+ const clamp = value => (typeof value === 'number' ? Math.max(1, Math.min(5, Math.round(value))) : null);
194
+ result.judge = {
195
+ coherence: clamp(score.coherence),
196
+ specificity: clamp(score.specificity),
197
+ consistency: clamp(score.consistency),
198
+ usable: score.usable === true,
199
+ issue: typeof score.issue === 'string' ? score.issue.slice(0, 40) : '',
200
+ };
201
+ }
202
+ } catch (error) {
203
+ result.error = `judge:${String(error)}`;
204
+ }
205
+ } catch (error) {
206
+ result.error = String(error);
207
+ result.latencyMs = Date.now() - startedAt;
208
+ }
209
+ debugLog({
210
+ phase: 'experiment-variant',
211
+ sessionId,
212
+ evidenceChars,
213
+ maxTokens,
214
+ rep,
215
+ parseOk: result.parseOk,
216
+ suggestionChars: result.suggestionChars,
217
+ inputTokens: result.inputTokens,
218
+ outputTokens: result.outputTokens,
219
+ judge: result.judge,
220
+ });
221
+ return result;
222
+ }
223
+
224
+ export async function runExperiment(deps) {
225
+ const budgets = Array.isArray(deps.maxTokens) && deps.maxTokens.length
226
+ ? deps.maxTokens.map(Number).filter(value => Number.isFinite(value) && value > 0)
227
+ : [200, 300, 500];
228
+ const slices = Array.isArray(deps.evidenceChars) && deps.evidenceChars.length
229
+ ? deps.evidenceChars.map(Number).filter(value => Number.isFinite(value) && value > 0)
230
+ : [800, 2000, 4000];
231
+ const reps = Math.max(1, Math.min(3, Number(deps.reps) || 3));
232
+ const total = slices.length * budgets.length * reps;
233
+ const results = [];
234
+ let index = 0;
235
+ for (const evidenceChars of slices) {
236
+ for (const maxTokens of budgets) {
237
+ for (let rep = 0; rep < reps; rep += 1) {
238
+ index += 1;
239
+ try {
240
+ const result = await runVariant({ deps, evidenceChars, maxTokens, rep, index, total });
241
+ results.push(result);
242
+ if (deps.onProgress) deps.onProgress(index, total, result);
243
+ } catch (error) {
244
+ results.push({ index, evidenceChars, maxTokens, rep, error: String(error) });
245
+ }
246
+ }
247
+ }
248
+ }
249
+ return {
250
+ finishedAt: new Date().toISOString(),
251
+ mode: 'explicit-evidence-slice',
252
+ temperature: 0,
253
+ model: deps.route ? { provider: deps.route.provider, model: deps.route.model } : null,
254
+ judgeModel: deps.judgeRoute ? { provider: deps.judgeRoute.provider, model: deps.judgeRoute.model } : null,
255
+ slices,
256
+ budgets,
257
+ reps,
258
+ total,
259
+ results,
260
+ };
261
+ }