dsh-plan-completion 0.0.0-stage → 0.1.1

package/host/experiment.js

@@ -0,0 +1,261 @@
+/**
+ * host/experiment.js — isolated prompt-budget experiment (round 2).
+ *
+ * Fixed session snapshot + fixed plan; variables:
+ *   - evidenceChars: explicit character budget for the serialized evidence object
+ *     (truncated from the least important fields first), and
+ *   - maxTokens: the output budget for the plan/suggestion JSON.
+ *
+ * Each variant runs at temperature 0 and is scored by a second (judge) model call.
+ * Exposed only through the temporary `/plan-completion/experiment` route.
+ */
+
+function parseLooseJson(text) {
+  try {
+    const cleaned = String(text ?? '').replace(/```(?:json)?/gi, '').trim();
+    const match = cleaned.match(/\{[\s\S]*\}/);
+    if (!match) return null;
+    return JSON.parse(match[0]);
+  } catch (error) {
+    return null;
+  }
+}
+
+const JUDGE_SYSTEM = [
+  '你是严格但务实的评分员。只输出一个 JSON 对象，不要输出任何解释、markdown 或代码块。',
+  '评分维度（1-5 分，5 最好）：',
+  '  coherence：计划步骤的状态与实际证据是否一致（无凭空完成、无遗漏明显进度）。',
+  '  specificity：下一步建议是否具体可执行（有明确动作和对象），而不是空话。',
+  '  consistency：建议是否与计划不矛盾、是否指向计划的下一步。',
+  '  usable：该建议是否是一句人可以直接发送的话（<=60 字，单行，无解释）。',
+  '重要：若候选输出声称某个步骤完成、而基准计划尚未体现，但候选原始输出给出了具体证据',
+  '（工具名/文件/命令结果），视为合理进度更新，不算 coherence 扣分；只有“无证据的完成声明”才扣分。',
+  '输出格式：{"coherence":4,"specificity":5,"consistency":4,"usable":true,"issue":"<=20字"}',
+].join('\n');
+
+function judgeMessage(basePlan, rawOutput, suggestion) {
+  return [
+    '【评估对象】一次「维护会话计划 + 生成下一条用户建议」的模型输出。',
+    '【基准计划（当前真实状态）】',
+    JSON.stringify(
+      basePlan
+        ? {
+            version: basePlan.version,
+            intent: basePlan.intent,
+            steps: (basePlan.steps || []).map(step => ({ id: step.id, title: step.title, status: step.status })),
+          }
+        : null,
+      null,
+      2,
+    ),
+    '【候选建议】',
+    suggestion || '',
+    '【候选原始输出前 900 字】',
+    String(rawOutput || '').slice(0, 900),
+    '【评分要求】对照基准计划评估候选输出，按系统提示的 JSON 格式输出。',
+  ].join('\n');
+}
+
+/** Trim the serialized evidence down to `targetChars`, dropping the least important parts first. */
+export function sliceEvidence(evidence, targetChars) {
+  const copy = (() => {
+    try {
+      return JSON.parse(JSON.stringify(evidence ?? {}));
+    } catch (error) {
+      return { ok: false };
+    }
+  })();
+  const size = () => JSON.stringify(copy).length;
+  if (size() <= targetChars) return copy;
+
+  // 1. Trim the long assistant tail.
+  if (typeof copy.lastAssistantText === 'string') {
+    const overflow = size() - targetChars;
+    copy.lastAssistantText = copy.lastAssistantText.slice(0, Math.max(0, copy.lastAssistantText.length - overflow - 40));
+    if (!copy.lastAssistantText) delete copy.lastAssistantText;
+  }
+  // 2. Drop recent messages from the oldest end.
+  while (size() > targetChars && Array.isArray(copy.recentMessages) && copy.recentMessages.length > 1) {
+    copy.recentMessages.shift();
+  }
+  // 3. Drop tool calls from the oldest end.
+  while (size() > targetChars && Array.isArray(copy.toolCalls) && copy.toolCalls.length > 1) {
+    copy.toolCalls.shift();
+  }
+  // 4. Drop secondary fields entirely.
+  for (const field of ['failedTools', 'files', 'todoSteps', 'counts', 'recentMessages', 'toolCalls', 'evidence', 'lastUserMessages']) {
+    if (size() <= targetChars) break;
+    if (field in copy) delete copy[field];
+  }
+  // 5. Hard cap as a last resort.
+  if (size() > targetChars && typeof copy.lastAssistantText === 'string') {
+    copy.lastAssistantText = copy.lastAssistantText.slice(0, Math.max(0, targetChars - 160));
+  }
+  return copy;
+}
+
+async function runVariant({ deps, evidenceChars, maxTokens, rep, index, total }) {
+  const {
+    session,
+    sessionId,
+    basePlan,
+    collectFromEvents,
+    buildPatchPrompt,
+    parseModelOutput,
+    callModel,
+    route,
+    judgeRoute,
+    debugLog,
+  } = deps;
+  const startedAt = Date.now();
+  let seqNow = 0;
+  try {
+    seqNow = Number(session.seq ?? 0);
+  } catch (error) {
+    seqNow = 0;
+  }
+  let events = [];
+  try {
+    events = Array.from(session.snapshotEvents(0) ?? []);
+  } catch (error) {
+    events = [];
+  }
+  const fullEvidence = (() => {
+    try {
+      return collectFromEvents({ events, plan: basePlan, turn: Number(basePlan?.watermark?.turn ?? 0), sessionId });
+    } catch (error) {
+      return { ok: false };
+    }
+  })();
+  const evidence = deps.evidenceChars
+    ? sliceEvidence(fullEvidence, evidenceChars)
+    : fullEvidence;
+  const evidenceSize = JSON.stringify(evidence).length;
+  const prompt = (() => {
+    try {
+      return buildPatchPrompt(evidence);
+    } catch (error) {
+      return null;
+    }
+  })();
+
+  const result = {
+    index,
+    total,
+    evidenceChars,
+    fullEvidenceChars: JSON.stringify(fullEvidence).length,
+    slicedEvidenceChars: evidenceSize,
+    maxTokens,
+    rep,
+    parseOk: false,
+    suggestion: '',
+    suggestionChars: 0,
+    changeCount: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    latencyMs: 0,
+    judge: null,
+    judgeRaw: '',
+    error: '',
+  };
+  if (!prompt) {
+    result.error = 'no-prompt';
+    return result;
+  }
+  prompt.maxTokens = maxTokens;
+  prompt.temperature = 0;
+
+  try {
+    const response = await callModel(deps.services, route, prompt, undefined);
+    result.inputTokens = response.usage?.inputTokens ?? 0;
+    result.outputTokens = response.usage?.outputTokens ?? 0;
+    const parsed = parseModelOutput(response.text);
+    result.parseOk = parsed?.ok === true;
+    if (parsed?.ok) {
+      result.suggestion = parsed.output.suggestion || '';
+      result.suggestionChars = result.suggestion.length;
+      result.changeCount = (parsed.output.changes || []).length + (parsed.output.steps || []).length;
+    }
+    result.latencyMs = Date.now() - startedAt;
+
+    try {
+      const judgePrompt = {
+        system: JUDGE_SYSTEM,
+        messages: [{ role: 'user', content: judgeMessage(basePlan, response.text, result.suggestion) }],
+        maxTokens: 500,
+        temperature: 0,
+      };
+      const judged = await callModel(deps.services, judgeRoute || route, judgePrompt, undefined);
+      result.judgeRaw = String(judged.text || '').slice(0, 400);
+      result.judgeModel = judgeRoute ? judgeRoute.model : route.model;
+      const score = parseLooseJson(judged.text);
+      if (score && typeof score === 'object') {
+        const clamp = value => (typeof value === 'number' ? Math.max(1, Math.min(5, Math.round(value))) : null);
+        result.judge = {
+          coherence: clamp(score.coherence),
+          specificity: clamp(score.specificity),
+          consistency: clamp(score.consistency),
+          usable: score.usable === true,
+          issue: typeof score.issue === 'string' ? score.issue.slice(0, 40) : '',
+        };
+      }
+    } catch (error) {
+      result.error = `judge:${String(error)}`;
+    }
+  } catch (error) {
+    result.error = String(error);
+    result.latencyMs = Date.now() - startedAt;
+  }
+  debugLog({
+    phase: 'experiment-variant',
+    sessionId,
+    evidenceChars,
+    maxTokens,
+    rep,
+    parseOk: result.parseOk,
+    suggestionChars: result.suggestionChars,
+    inputTokens: result.inputTokens,
+    outputTokens: result.outputTokens,
+    judge: result.judge,
+  });
+  return result;
+}
+
+export async function runExperiment(deps) {
+  const budgets = Array.isArray(deps.maxTokens) && deps.maxTokens.length
+    ? deps.maxTokens.map(Number).filter(value => Number.isFinite(value) && value > 0)
+    : [200, 300, 500];
+  const slices = Array.isArray(deps.evidenceChars) && deps.evidenceChars.length
+    ? deps.evidenceChars.map(Number).filter(value => Number.isFinite(value) && value > 0)
+    : [800, 2000, 4000];
+  const reps = Math.max(1, Math.min(3, Number(deps.reps) || 3));
+  const total = slices.length * budgets.length * reps;
+  const results = [];
+  let index = 0;
+  for (const evidenceChars of slices) {
+    for (const maxTokens of budgets) {
+      for (let rep = 0; rep < reps; rep += 1) {
+        index += 1;
+        try {
+          const result = await runVariant({ deps, evidenceChars, maxTokens, rep, index, total });
+          results.push(result);
+          if (deps.onProgress) deps.onProgress(index, total, result);
+        } catch (error) {
+          results.push({ index, evidenceChars, maxTokens, rep, error: String(error) });
+        }
+      }
+    }
+  }
+  return {
+    finishedAt: new Date().toISOString(),
+    mode: 'explicit-evidence-slice',
+    temperature: 0,
+    model: deps.route ? { provider: deps.route.provider, model: deps.route.model } : null,
+    judgeModel: deps.judgeRoute ? { provider: deps.judgeRoute.provider, model: deps.judgeRoute.model } : null,
+    slices,
+    budgets,
+    reps,
+    total,
+    results,
+  };
+}

package/host/heuristics.js

@@ -0,0 +1,160 @@
+/**
+ * host/heuristics.js — 启发式建议与降级计划骨架（纯函数，零依赖）。
+ *
+ * 设计依据：v1.2 §3.5.4（模型不可用 → 启发式）、§3.6（降级链：
+ * 「计划下一步 / 失败工具 / 未完成 todo」；无模型 → 首条消息骨架 + todo）。
+ *
+ * 优先级（与 task 验收一致）：
+ *   1. 失败工具 → 修报错
+ *   2. 有未完成 todo/doing → 下一个
+ *   3. 最后一条 assistant 是提问 → 给倾向性决策
+ *   4. 全部 done/dropped → 验收
+ *   5. 否则 → 计划下一步 title / 按 intent 起步
+ */
+
+import { DEFAULTS } from './config.js';
+import { statusFor } from './plan.js';
+import { clampChars, looksLikeQuestion, oneLine, sanitizeSuggestion } from './text.js';
+
+const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+
+function make(text, kind) {
+  const clean = sanitizeSuggestion(text, DEFAULTS.suggestionMaxChars);
+  if (!clean.ok) return { ok: false, text: '', kind: 'none' };
+  return { ok: true, text: clean.text, kind };
+}
+
+function firstFailedTool(evidence) {
+  const failed = Array.isArray(evidence?.failedTools) ? evidence.failedTools.filter((tool) => isObj(tool)) : [];
+  if (failed.length) return failed[failed.length - 1];
+  const calls = Array.isArray(evidence?.toolCalls) ? evidence.toolCalls : [];
+  const fromCalls = calls.filter((tool) => isObj(tool) && tool.ok === false);
+  return fromCalls.length ? fromCalls[fromCalls.length - 1] : null;
+}
+
+function nextOpenStep(plan) {
+  if (!isObj(plan) || !Array.isArray(plan.steps)) return null;
+  const doing = plan.steps.find((step) => isObj(step) && step.status === 'doing');
+  if (doing) return doing;
+  const todo = plan.steps.find((step) => isObj(step) && step.status === 'todo');
+  return todo ?? null;
+}
+
+/**
+ * 生成启发式建议（模型不可用/超时/非法输出时的兜底）。
+ *
+ * @param {object} evidence collectFromEvents 的输出（也可传任意同名字段对象）
+ * @returns {{ok:true, text:string, kind:string}|{ok:false, text:'', kind:'none'}}
+ */
+export function heuristicSuggestion(evidence = {}) {
+  try {
+    const source = isObj(evidence) ? evidence : {};
+    const plan = isObj(source.plan) ? source.plan : null;
+
+    // 1. 失败工具 → 修报错
+    const failed = firstFailedTool(source);
+    if (failed) {
+      const name = clampChars(oneLine(failed.name) || '工具', 14);
+      const error = clampChars(oneLine(failed.error), 20);
+      if (error) return make(`先修 ${name} 的报错：${error}`, 'fix-error');
+      return make(`先修 ${name} 的报错再继续下一步`, 'fix-error');
+    }
+
+    // 2. 有未完成步骤 → 下一个
+    const next = nextOpenStep(plan);
+    if (next) {
+      const title = clampChars(oneLine(next.title), 24) || '下一步';
+      return make(`接着做「${title}」，先跑通再补细节`, 'next-step');
+    }
+
+    // 3. 提问回合 → 给倾向性决策
+    if (source.hasQuestion === true || looksLikeQuestion(source.lastAssistantText)) {
+      return make('按方案 A 继续，先把最小可用版本跑通', 'decide');
+    }
+
+    // 4. 全部完成 → 验收
+    if (plan && statusFor(plan.steps) === 'completed') {
+      return make('功能已完成，先自测验收再收尾', 'accept');
+    }
+
+    // 5. 否则 → 计划下一步 title / intent 起步
+    if (plan && Array.isArray(plan.steps) && plan.steps.length) {
+      const last = plan.steps[plan.steps.length - 1];
+      const title = clampChars(oneLine(last?.title), 22) || clampChars(oneLine(plan.intent), 22);
+      if (title) return make(`先收尾「${title}」并确认结果`, 'next-title');
+    }
+    const intent = clampChars(
+      oneLine(plan?.intent) || oneLine(source.intent) || oneLine(source.firstUserMessage) || '',
+      22,
+    );
+    if (intent) return make(`先摸清现状，把「${intent}」拆成 3 步开始`, 'intent');
+    return { ok: false, text: '', kind: 'none' };
+  } catch {
+    return { ok: false, text: '', kind: 'none' };
+  }
+}
+
+/** 骨架模板：按 intent 关键词选择 3~5 个里程碑。 */
+const SKELETONS = [
+  {
+    match: /(bug|报错|错误|修复|崩溃|异常|失败|fix|error|crash)/i,
+    titles: ['复现并定位报错根因', '做最小修复并验证', '补回归测试防止复发'],
+  },
+  {
+    match: /(测试|用例|回归|覆盖率|test|spec)/i,
+    titles: ['梳理测试范围与现状', '补关键路径测试用例', '跑通回归并修复问题'],
+  },
+  {
+    match: /(设计|方案|架构|重构|优化|选型)/i,
+    titles: ['确认目标、约束与验收标准', '出 2 个方案并对比取舍', '落地最小可行版本', '验证效果并收尾'],
+  },
+  {
+    match: /(文档|readme|说明|注释|doc)/i,
+    titles: ['梳理要写的文档范围', '补写核心文档与示例', '校对并验证示例可运行'],
+  },
+  {
+    match: /(接口|api|联调|集成|部署|发布)/i,
+    titles: ['确认接口契约与环境', '实现并打通主流程', '联调异常路径', '验收并发布'],
+  },
+];
+
+/**
+ * 降级计划骨架：无模型时按首条消息生成 3~5 步（交给 createSeedPlan 落版本 0.0）。
+ *
+ * @param {string} [intent] 用户目标（首条消息 / 现计划 intent）
+ * @returns {{ok:true, intent:string, steps:Array<{id:string,title:string,status:'todo',source:'model'}>}|{ok:false, code:string, error:string}}
+ */
+export function fallbackPlan(intent) {
+  try {
+    const goal = clampChars(oneLine(intent), 24);
+    let template = SKELETONS[0];
+    if (goal) {
+      for (const item of SKELETONS) {
+        if (item.match.test(goal)) {
+          template = item;
+          break;
+        }
+      }
+      if (template === SKELETONS[0] && !SKELETONS[0].match.test(goal)) template = null;
+    } else {
+      template = null;
+    }
+    let titles = template ? [...template.titles] : ['摸清现状与约束', '实现核心改动', '自测并修复问题', '补测试与文档'];
+    if (goal) {
+      // 把目标自然嵌进第一/第二步，让骨架与用户意图相关。
+      titles = titles.map((title, index) => (index === 0 && !title.includes(goal) ? `${title}：${goal}` : title));
+    }
+    const steps = titles.map((title, index) => ({
+      id: `s${index + 1}`,
+      title: clampChars(title, DEFAULTS.maxTitleChars),
+      status: 'todo',
+      source: 'model',
+    }));
+    if (steps.length < DEFAULTS.minSteps || steps.length > DEFAULTS.maxSteps) {
+      return { ok: false, code: 'bad-skeleton', error: '骨架步骤数不在 3~5 之间' };
+    }
+    return { ok: true, intent: clampChars(oneLine(intent), DEFAULTS.maxIntentChars), steps };
+  } catch (err) {
+    return { ok: false, code: 'internal', error: `骨架生成异常：${err && err.message ? err.message : String(err)}` };
+  }
+}