@boyingliu01/xp-gate 0.19.2 → 0.20.0

package/plugins/dsh/src/delphi-run.test.ts

@@ -0,0 +1,181 @@
+import { describe, it, expect } from "vitest"
+import { formatOutcome, runReview } from "./delphi-run.js"
+import type { ExpertSeat } from "./delphi.js"
+
+/**
+ * @test REQ-DSH-016
+ * @intent 验证多轮编排的收敛语义：首轮匿名、达阈值即停、未达阈值继续、
+ *         触及轮次上限则如实返回 NO CONSENSUS；并验证一份畸形报告
+ *         无法靠其余全票凑出共识（分母是全体席位）。
+ * @covers AC-DSH-016-03
+ */
+
+const seats: ExpertSeat[] = [
+  { id: "A", role: "architecture", model: "m-a" },
+  { id: "B", role: "technical", model: "m-b" },
+  { id: "C", role: "feasibility", model: "m-c" },
+]
+
+/**
+ * Fetch stub that answers per (model, call index). `plan[n]` is the verdict map
+ * used on the n-th round, so a sequence of rounds can be scripted.
+ */
+function scriptedFetch(plan: Array<Record<string, string>>) {
+  let round = 0
+  let seen = 0
+  return (async (_url: string | URL | Request, init?: RequestInit) => {
+    const model = JSON.parse(String(init?.body)).model as string
+    // Three seats per round, collected in seat order.
+    const idx = round
+    const table = plan[Math.min(idx, plan.length - 1)]
+    seen += 1
+    if (seen % 3 === 0) round += 1
+    const body = table[model]
+    if (body === undefined) return new Response("nope", { status: 404 })
+    return new Response(
+      JSON.stringify({
+        model: `resolved-${model}`,
+        choices: [{ finish_reason: "stop", message: { content: body } }],
+      }),
+      { status: 200, headers: { "Content-Type": "application/json" } },
+    )
+  }) as unknown as typeof fetch
+}
+
+const APPROVE = "### VERDICT\nAPPROVED\n### CONFIDENCE\n8"
+const REJECT = "### VERDICT\nREQUEST_CHANGES\n### CONFIDENCE\n7"
+
+describe("runReview", () => {
+  it("stops at round 1 when the whole panel approves", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "https://x.test/gw",
+        apiKey: "k",
+        seats,
+        fetchImpl: scriptedFetch([{ "m-a": APPROVE, "m-b": APPROVE, "m-c": APPROVE }]),
+      },
+      "subject",
+    )
+    expect(out.consensus).toBe(true)
+    expect(out.totalRounds).toBe(1)
+    expect(out.finalRatio).toBe(1)
+  })
+
+  it("iterates when the first round disagrees, then converges", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "https://x.test/gw",
+        apiKey: "k",
+        seats,
+        fetchImpl: scriptedFetch([
+          { "m-a": REJECT, "m-b": APPROVE, "m-c": REJECT },
+          { "m-a": APPROVE, "m-b": APPROVE, "m-c": APPROVE },
+        ]),
+      },
+      "subject",
+    )
+    expect(out.totalRounds).toBe(2)
+    expect(out.consensus).toBe(true)
+    // Round 1 must be recorded even though it failed.
+    expect(out.rounds[0].approvals).toBe(1)
+    expect(out.rounds[1].approvals).toBe(3)
+  })
+
+  it("reports NO CONSENSUS after hitting the round cap", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "https://x.test/gw",
+        apiKey: "k",
+        seats,
+        maxRounds: 3,
+        fetchImpl: scriptedFetch([{ "m-a": APPROVE, "m-b": REJECT, "m-c": REJECT }]),
+      },
+      "subject",
+    )
+    expect(out.consensus).toBe(false)
+    expect(out.totalRounds).toBe(3)
+    expect(out.finalRatio).toBeCloseTo(1 / 3, 5)
+  })
+
+  it("never reaches consensus when a report lacks a verdict, even if the rest approve", async () => {
+    // 2/3 approvals with one malformed report. A naive implementation that
+    // divides by valid responses would call this 100%.
+    const out = await runReview(
+      {
+        baseUrl: "https://x.test/gw",
+        apiKey: "k",
+        seats,
+        maxRounds: 2,
+        fetchImpl: scriptedFetch([
+          { "m-a": APPROVE, "m-b": APPROVE, "m-c": "I reviewed it but will not say." },
+        ]),
+      },
+      "subject",
+    )
+    expect(out.consensus).toBe(false)
+    expect(out.finalRatio).toBeCloseTo(2 / 3, 5)
+  })
+
+  it("uses the configured threshold rather than the default", async () => {
+    // 2/3 = 67%. Reachable at a 60% threshold, not at 90%.
+    const mk = () =>
+      scriptedFetch([{ "m-a": APPROVE, "m-b": APPROVE, "m-c": REJECT }])
+    const lax = await runReview(
+      { baseUrl: "u", apiKey: "k", seats, thresholdPercent: 60, fetchImpl: mk() },
+      "s",
+    )
+    expect(lax.consensus).toBe(true)
+
+    const strict = await runReview(
+      { baseUrl: "u", apiKey: "k", seats, thresholdPercent: 90, fetchImpl: mk() },
+      "s",
+    )
+    expect(strict.consensus).toBe(false)
+  })
+
+  it("defaults to at most 5 rounds", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "u",
+        apiKey: "k",
+        seats,
+        fetchImpl: scriptedFetch([{ "m-a": REJECT, "m-b": REJECT, "m-c": REJECT }]),
+      },
+      "s",
+    )
+    expect(out.totalRounds).toBe(5)
+    expect(out.consensus).toBe(false)
+  })
+})
+
+describe("formatOutcome", () => {
+  it("names each seat, its resolved model, and the final ratio", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "u",
+        apiKey: "k",
+        seats,
+        fetchImpl: scriptedFetch([{ "m-a": APPROVE, "m-b": APPROVE, "m-c": APPROVE }]),
+      },
+      "s",
+    )
+    const text = formatOutcome(out)
+    expect(text).toContain("resolved-m-a")
+    expect(text).toContain("architecture")
+    expect(text).toMatch(/CONSENSUS REACHED/)
+  })
+
+  it("says NO CONSENSUS explicitly when the cap is reached", async () => {
+    const out = await runReview(
+      {
+        baseUrl: "u",
+        apiKey: "k",
+        seats,
+        maxRounds: 1,
+        fetchImpl: scriptedFetch([{ "m-a": REJECT, "m-b": REJECT, "m-c": REJECT }]),
+      },
+      "s",
+    )
+    expect(formatOutcome(out)).toMatch(/NO CONSENSUS/)
+  })
+})

package/plugins/dsh/src/delphi-run.ts

@@ -0,0 +1,129 @@
+/**
+ * Delphi review orchestration: iterate rounds until consensus or the cap.
+ *
+ * @test REQ-DSH-016
+ * @intent 编排多轮交叉评审：首轮匿名独立评审，后续轮次反馈上一轮汇总，
+ *         直到达到共识阈值（默认 90%）或触及轮次上限（默认 5），
+ *         并把结果整理成可写成 .code-walkthrough-result.json 的证据。
+ * @covers AC-DSH-016-03
+ */
+
+import type { DelphiOptions, ExpertSeat, RoundResult } from "./delphi.js"
+import { runRound } from "./delphi.js"
+
+/** Mirrors delphi.ts; declared here so orchestrators need not import internals. */
+const DEFAULT_THRESHOLD_PERCENT = 90
+const DEFAULT_MAX_ROUNDS = 5
+
+export interface ReviewOutcome {
+  readonly rounds: readonly RoundResult[]
+  readonly consensus: boolean
+  readonly finalRatio: number
+  readonly totalRounds: number
+}
+
+/** Anonymous first round: each seat answers the same question with no peers. */
+function roundOnePrompt(seat: ExpertSeat, subject: string): string {
+  return (
+    `You are the ${seat.role.toUpperCase()} expert. Review the material below INDEPENDENTLY. ` +
+    `Do not assume other reviewers exist or agree with you.\n\n${subject}\n\n` +
+    `Respond with <=300 words under the headings: "### Findings" (a markdown table of ` +
+    `ID | Severity | Description | Location | Fix, or a single NONE row), then "### VERDICT" ` +
+    `followed by exactly APPROVED or REQUEST_CHANGES, then "### CONFIDENCE" followed by an ` +
+    `integer 1-10. Approve only if you would personally ship this.`
+  )
+}
+
+/**
+ * Later rounds: feed back the aggregate so seats can revise their position.
+ * Individual reports stay anonymous -- only the tally and the addressed items
+ * are shared, which is what keeps the method honest.
+ */
+function laterRoundPrompt(
+  seat: ExpertSeat,
+  subject: string,
+  previous: RoundResult,
+  addressed: string,
+): string {
+  const tally = previous.verdicts
+    .map((v) => `- ${v.role}: ${v.verdict}`)
+    .join("\n")
+
+  return (
+    `You are the ${seat.role.toUpperCase()} expert. This is ROUND ${previous.round + 1} of a ` +
+    `Delphi consensus review. Individual peer reports are anonymous; only this aggregate is shared:\n\n` +
+    `${tally}\nApprovals: ${previous.approvals}/${previous.validCount} ` +
+    `(threshold ${DEFAULT_THRESHOLD_PERCENT}%).\n\n` +
+    `${addressed ? `What changed since the last round:\n${addressed}\n\n` : ""}` +
+    `Re-examine the material and revise or hold your position.\n\n${subject}\n\n` +
+    `Respond with <=300 words: "### Round assessment" (2-3 sentences), "### Remaining blocking ` +
+    `issues" (table, or a single NONE row), "### VERDICT" followed by exactly APPROVED or ` +
+    `REQUEST_CHANGES, then "### CONFIDENCE" and an integer 1-10.`
+  )
+}
+
+/**
+ * Run a full review. Round 1 is anonymous; each later round sees the aggregate
+ * plus whatever the caller says was addressed, then re-votes.
+ */
+export async function runReview(
+  opts: DelphiOptions,
+  subject: string,
+  addressed = "",
+): Promise<ReviewOutcome> {
+  const threshold = opts.thresholdPercent ?? DEFAULT_THRESHOLD_PERCENT
+  const maxRounds = opts.maxRounds ?? DEFAULT_MAX_ROUNDS
+  const rounds: RoundResult[] = []
+
+  for (let n = 1; n <= maxRounds; n++) {
+    const previous = rounds[rounds.length - 1]
+    const result = previous
+      ? await runRound(opts, n, "You are a rigorous senior code reviewer.", (seat) =>
+          laterRoundPrompt(seat, subject, previous, addressed),
+        )
+      : await runRound(opts, 1, "You are a rigorous senior code reviewer.", (seat) =>
+          roundOnePrompt(seat, subject),
+        )
+
+    rounds.push(result)
+
+    // Threshold is a percentage of the whole panel, so a round with a malformed
+    // report cannot reach consensus even if every valid seat approved.
+    if (result.approvals / opts.seats.length * 100 >= threshold) {
+      return {
+        rounds,
+        consensus: true,
+        finalRatio: result.approvals / opts.seats.length,
+        totalRounds: n,
+      }
+    }
+  }
+
+  const last = rounds[rounds.length - 1]
+  return {
+    rounds,
+    consensus: false,
+    finalRatio: last ? last.approvals / opts.seats.length : 0,
+    totalRounds: rounds.length,
+  }
+}
+
+/** Human-readable transcript for the tool result. */
+export function formatOutcome(outcome: ReviewOutcome): string {
+  const lines: string[] = []
+  for (const r of outcome.rounds) {
+    lines.push(`## Round ${r.round} — approvals ${r.approvals}/${r.validCount} valid`)
+    for (const v of r.verdicts) {
+      const model = v.resolvedModel ? `${v.requestedModel} -> ${v.resolvedModel}` : v.requestedModel
+      const note = v.error ? ` (${v.error})` : ""
+      lines.push(`- [${v.role}] ${v.id} (${model}): ${v.verdict} confidence=${v.confidence}${note}`)
+    }
+    lines.push("")
+  }
+  lines.push(
+    outcome.consensus
+      ? `CONSENSUS REACHED: ${(outcome.finalRatio * 100).toFixed(0)}% after ${outcome.totalRounds} round(s).`
+      : `NO CONSENSUS: ${(outcome.finalRatio * 100).toFixed(0)}% after ${outcome.totalRounds} round(s) (cap reached).`,
+  )
+  return lines.join("\n")
+}

package/plugins/dsh/src/delphi.test.ts

@@ -0,0 +1,222 @@
+import { describe, it, expect } from "vitest"
+import { DelphiConfigError, parseVerdict, runRound, validateSeats } from "./delphi.js"
+import type { ExpertSeat } from "./delphi.js"
+
+/**
+ * @test REQ-DSH-016
+ * @intent 验证多模型交叉评审的核心不变量：恰好 3 位专家、requested_model 两两不同、
+ *         缺少模型或未展开的 ${ENV} 占位符必须在校验期而非网络期报错；
+ *         并验证裁决解析与共识计算不会被畸形报告污染。
+ * @covers AC-DSH-016-01
+ */
+
+const seats: ExpertSeat[] = [
+  { id: "A", role: "architecture", model: "g-glm-5.3-flash", maxTokens: 32000 },
+  { id: "B", role: "technical", model: "g-qwen3.8-flash" },
+  { id: "C", role: "feasibility", model: "g-deepseek-flash" },
+]
+
+describe("validateSeats", () => {
+  it("accepts exactly three seats with distinct models", () => {
+    expect(() => validateSeats(seats)).not.toThrow()
+  })
+
+  it("rejects a panel that is not exactly three", () => {
+    expect(() => validateSeats(seats.slice(0, 2))).toThrow(DelphiConfigError)
+    expect(() => validateSeats([...seats, { id: "D", role: "x", model: "m4" }])).toThrow(
+      /exactly 3/,
+    )
+  })
+
+  it("rejects duplicate models, since Delphi rests on reviewers differing", () => {
+    const dup = [seats[0], seats[1], { ...seats[2], model: "g-qwen3.8-flash" }]
+    expect(() => validateSeats(dup)).toThrow(/distinct/)
+  })
+
+  it("treats models differing only by whitespace as duplicates", () => {
+    const padded = [seats[0], seats[1], { ...seats[2], model: " g-qwen3.8-flash " }]
+    expect(() => validateSeats(padded)).toThrow(/distinct/)
+  })
+
+  it("rejects an empty model (a local fallback cannot count)", () => {
+    expect(() => validateSeats([seats[0], seats[1], { ...seats[2], model: "  " }])).toThrow(
+      /fallback|no model/i,
+    )
+  })
+
+  it("rejects an unexpanded env placeholder before any network call", () => {
+    expect(() => validateSeats([seats[0], seats[1], { ...seats[2], model: "${MODEL_ID}" }])).toThrow(
+      /unexpanded/,
+    )
+  })
+})
+
+describe("parseVerdict", () => {
+  it("reads the verdict on the line after the heading", () => {
+    // The real reports put the token on the next line; a same-line
+    // `VERDICT: X` pattern misses every one of them.
+    expect(parseVerdict("### VERDICT\nAPPROVED\n\n### CONFIDENCE\n8").verdict).toBe("APPROVED")
+    expect(parseVerdict("### VERDICT\nREQUEST_CHANGES\n### CONFIDENCE\n7").verdict).toBe(
+      "REQUEST_CHANGES",
+    )
+  })
+
+  it("also accepts a same-line form", () => {
+    expect(parseVerdict("VERDICT: APPROVED\nCONFIDENCE: 9").verdict).toBe("APPROVED")
+  })
+
+  it("maps PASS_WITH_CAVEATS and BLOCKED to non-approval", () => {
+    // Neither reaches the 90% bar as an approval.
+    expect(parseVerdict("### VERDICT\nPASS_WITH_CAVEATS\nCONFIDENCE 6").verdict).toBe(
+      "REQUEST_CHANGES",
+    )
+    expect(parseVerdict("### VERDICT\nBLOCKED\nCONFIDENCE 2").verdict).toBe("REQUEST_CHANGES")
+  })
+
+  it("marks a report with no verdict as INVALID rather than guessing", () => {
+    // Observed in practice: a reviewer analysed everything but omitted the
+    // verdict. Inferring approval from prose would pass a gate on a guess.
+    const proseOnly = "The fixes look correct. Non-blocking concerns only. No critical defects."
+    expect(parseVerdict(proseOnly).verdict).toBe("INVALID_NO_VERDICT")
+    expect(parseVerdict(proseOnly).confidence).toBe(0)
+  })
+
+  it("clamps confidence into 1-10 and defaults to 0 when absent", () => {
+    expect(parseVerdict("VERDICT APPROVED CONFIDENCE 99").confidence).toBe(10)
+    expect(parseVerdict("VERDICT APPROVED").confidence).toBe(0)
+  })
+})
+
+describe("runRound", () => {
+  function fakeFetch(bodies: Record<string, string>, status = 200) {
+    return (async (_url: string | URL | Request, init?: RequestInit) => {
+      const model = JSON.parse(String(init?.body)).model as string
+      if (!(model in bodies)) {
+        return new Response("no such model", { status: 404 })
+      }
+      return new Response(
+        JSON.stringify({
+          model: `resolved-${model}`,
+          choices: [{ finish_reason: "stop", message: { content: bodies[model] } }],
+        }),
+        { status, headers: { "Content-Type": "application/json" } },
+      )
+    }) as unknown as typeof fetch
+  }
+
+  const opts = {
+    baseUrl: "https://example.test/gw",
+    apiKey: "k",
+    seats,
+    fetchImpl: fakeFetch({
+      "g-glm-5.3-flash": "### VERDICT\nAPPROVED\n### CONFIDENCE\n8",
+      "g-qwen3.8-flash": "### VERDICT\nAPPROVED\n### CONFIDENCE\n9",
+      "g-deepseek-flash": "### VERDICT\nREQUEST_CHANGES\n### CONFIDENCE\n7",
+    }),
+  }
+
+  it("runs every seat and records the resolved model per seat", async () => {
+    const r = await runRound(opts, 1, "sys", () => "task")
+    expect(r.verdicts).toHaveLength(3)
+    expect(r.verdicts.map((v) => v.requestedModel)).toEqual([
+      "g-glm-5.3-flash",
+      "g-qwen3.8-flash",
+      "g-deepseek-flash",
+    ])
+    expect(r.verdicts[0].resolvedModel).toBe("resolved-g-glm-5.3-flash")
+  })
+
+  it("computes the ratio over valid responses and does not call 2/3 consensus", async () => {
+    const r = await runRound(opts, 1, "sys", () => "task")
+    expect(r.approvals).toBe(2)
+    expect(r.validCount).toBe(3)
+    expect(r.consensusRatio).toBeCloseTo(2 / 3, 5)
+    // 67% < 90%
+    expect(r.reachedConsensus).toBe(false)
+  })
+
+  it("cannot reach consensus when a report is malformed, even if all valid seats approve", async () => {
+    const r = await runRound(
+      {
+        ...opts,
+        fetchImpl: fakeFetch({
+          "g-glm-5.3-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          "g-qwen3.8-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          // No verdict section at all -- must not count as agreement.
+          "g-deepseek-flash": "Everything looks fine to me.",
+        }),
+      },
+      2,
+      "sys",
+      () => "task",
+    )
+    expect(r.verdicts[2].verdict).toBe("INVALID_NO_VERDICT")
+    expect(r.verdicts[2].counted).toBe(false)
+    expect(r.validCount).toBe(2)
+    expect(r.reachedConsensus).toBe(false)
+  })
+
+  it("records a transport failure as ERROR without aborting the round", async () => {
+    const r = await runRound(
+      {
+        ...opts,
+        fetchImpl: fakeFetch({
+          "g-glm-5.3-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          "g-qwen3.8-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          // g-deepseek-flash is absent from the map, so it 404s.
+        }),
+      },
+      1,
+      "sys",
+      () => "task",
+    )
+    const errored = r.verdicts.filter((v) => v.verdict === "ERROR")
+    expect(errored).toHaveLength(1)
+    expect(errored[0].counted).toBe(false)
+    expect(errored[0].error).toMatch(/404/)
+    // The two healthy seats still produced counts; the failure is isolated.
+    expect(r.validCount).toBe(2)
+    // A dead seat means the panel is incomplete, so no consensus.
+    expect(r.reachedConsensus).toBe(false)
+  })
+
+  it("treats empty content as an error rather than an empty review", async () => {
+    // Reasoning models can spend their entire budget on hidden tokens and
+    // return nothing; that must surface as a failure, not a blank approval.
+    const r = await runRound(
+      {
+        ...opts,
+        fetchImpl: fakeFetch({
+          "g-glm-5.3-flash": "",
+          "g-qwen3.8-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          "g-deepseek-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+        }),
+      },
+      1,
+      "sys",
+      () => "task",
+    )
+    const blank = r.verdicts.find((v) => v.requestedModel === "g-glm-5.3-flash")
+    expect(blank?.verdict).toBe("ERROR")
+    expect(blank?.error).toMatch(/empty content/)
+  })
+
+  it("reaches consensus when all three approve", async () => {
+    const r = await runRound(
+      {
+        ...opts,
+        fetchImpl: fakeFetch({
+          "g-glm-5.3-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+          "g-qwen3.8-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 9",
+          "g-deepseek-flash": "### VERDICT\nAPPROVED\nCONFIDENCE 8",
+        }),
+      },
+      3,
+      "sys",
+      () => "task",
+    )
+    expect(r.approvals).toBe(3)
+    expect(r.consensusRatio).toBe(1)
+    expect(r.reachedConsensus).toBe(true)
+  })
+})