@runweave/server-linux-x64 0.1.8 → 0.1.9

package/payload/plugins/toolkit/skills/review-with-runtime-evidence/scripts/run-static-review.mjs

@@ -0,0 +1,389 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import process from "node:process";
+import {
+  aggregateCase,
+  buildPrompt,
+  OUTPUT_SCHEMA,
+  REVIEWERS,
+  validateManifest,
+  validateReview,
+} from "./static-review-contract.mjs";
+
+function usage() {
+  return [
+    "Usage:",
+    "  node run-static-review.mjs --manifest=<file> --output=<dir> [--prepare-only] [--timeout-ms=<ms>]",
+    "",
+    "The manifest contract is documented in references/contracts.md.",
+  ].join("\n");
+}
+
+function parseArgs(argv) {
+  const options = {
+    manifest: null,
+    output: null,
+    prepareOnly: false,
+    timeoutMs: 12 * 60 * 1000,
+  };
+  for (const arg of argv) {
+    if (arg === "--help" || arg === "-h") {
+      process.stdout.write(`${usage()}\n`);
+      process.exit(0);
+    }
+    if (arg === "--prepare-only") {
+      options.prepareOnly = true;
+    } else if (arg.startsWith("--manifest=")) {
+      options.manifest = path.resolve(arg.slice("--manifest=".length));
+    } else if (arg.startsWith("--output=")) {
+      options.output = path.resolve(arg.slice("--output=".length));
+    } else if (arg.startsWith("--timeout-ms=")) {
+      options.timeoutMs = Number.parseInt(
+        arg.slice("--timeout-ms=".length),
+        10,
+      );
+    } else {
+      throw new Error(`unknown argument: ${arg}`);
+    }
+  }
+  if (!options.manifest || !options.output) {
+    throw new Error("--manifest and --output are required");
+  }
+  if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) {
+    throw new Error("--timeout-ms must be a positive integer");
+  }
+  return options;
+}
+
+async function writePrivate(filePath, content) {
+  await mkdir(path.dirname(filePath), {
+    recursive: true,
+    mode: 0o700,
+  });
+  await writeFile(filePath, content, { mode: 0o600 });
+  await chmod(filePath, 0o600);
+}
+
+function reviewerArgs(reviewer, schemaPath, outputPath, sandboxDir) {
+  return [
+    "exec",
+    "--ephemeral",
+    "--skip-git-repo-check",
+    "--ignore-rules",
+    "--ignore-user-config",
+    "--disable",
+    "plugins",
+    "--disable",
+    "remote_plugin",
+    "--sandbox",
+    "read-only",
+    "-C",
+    sandboxDir,
+    "-m",
+    reviewer.model,
+    "-c",
+    'model_reasoning_effort="high"',
+    "--output-schema",
+    schemaPath,
+    "--json",
+    "--output-last-message",
+    outputPath,
+    "-",
+  ];
+}
+
+async function runCommand(command, args, input, timeoutMs) {
+  const startedAt = Date.now();
+  const child = spawn(command, args, {
+    env: {
+      ...process.env,
+      NO_COLOR: "1",
+      TERM: "dumb",
+    },
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  let stdout = "";
+  let stderr = "";
+  child.stdout.setEncoding("utf8");
+  child.stderr.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    stdout += chunk;
+  });
+  child.stderr.on("data", (chunk) => {
+    stderr += chunk;
+  });
+  child.stdin.end(input);
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    child.kill("SIGTERM");
+    setTimeout(() => child.kill("SIGKILL"), 5_000).unref();
+  }, timeoutMs);
+  const status = await new Promise((resolve) => {
+    child.on("error", (error) => {
+      resolve({ exitCode: null, signal: null, spawnError: error.message });
+    });
+    child.on("close", (exitCode, signal) => {
+      resolve({ exitCode, signal, spawnError: null });
+    });
+  });
+  clearTimeout(timer);
+  return {
+    ...status,
+    timedOut,
+    durationMs: Date.now() - startedAt,
+    stdout,
+    stderr,
+  };
+}
+
+function parseJsonLines(stdout) {
+  return stdout
+    .split("\n")
+    .filter(Boolean)
+    .flatMap((line) => {
+      try {
+        return [JSON.parse(line)];
+      } catch {
+        return [];
+      }
+    });
+}
+
+function parseStructuredValue(value) {
+  const trimmed = value.trim();
+  const fenced = /```(?:json)?\s*([\s\S]*?)\s*```/i.exec(trimmed);
+  return JSON.parse(fenced ? fenced[1] : trimmed);
+}
+
+async function readReviewerResult(outputPath, stdout) {
+  try {
+    return parseStructuredValue(await readFile(outputPath, "utf8"));
+  } catch {
+    const agentMessage = parseJsonLines(stdout)
+      .slice()
+      .reverse()
+      .find(
+        (item) =>
+          item.type === "item.completed" &&
+          item.item?.type === "agent_message" &&
+          typeof item.item.text === "string",
+      );
+    if (!agentMessage) {
+      throw new Error("reviewer produced no parseable result");
+    }
+    return parseStructuredValue(agentMessage.item.text);
+  }
+}
+
+async function invokeReviewer({
+  reviewer,
+  testCase,
+  prompt,
+  outputDir,
+  schemaPath,
+  sandboxDir,
+  timeoutMs,
+}) {
+  const prefix = `${testCase.id}-${reviewer.id}`;
+  await writePrivate(path.join(outputDir, "prompts", `${prefix}.txt`), prompt);
+  let lastReceipt = null;
+  for (let attempt = 1; attempt <= 2; attempt += 1) {
+    const attemptPrefix = `${prefix}-attempt-${attempt}`;
+    const resultPath = path.join(
+      outputDir,
+      "raw",
+      `${attemptPrefix}.result.json`,
+    );
+    await mkdir(path.dirname(resultPath), {
+      recursive: true,
+      mode: 0o700,
+    });
+    const commandResult = await runCommand(
+      reviewer.command,
+      reviewerArgs(reviewer, schemaPath, resultPath, sandboxDir),
+      prompt,
+      timeoutMs,
+    );
+    await writePrivate(
+      path.join(outputDir, "raw", `${attemptPrefix}.stdout`),
+      commandResult.stdout,
+    );
+    await writePrivate(
+      path.join(outputDir, "raw", `${attemptPrefix}.stderr`),
+      commandResult.stderr,
+    );
+    let review = null;
+    let parseError = null;
+    try {
+      review = await readReviewerResult(resultPath, commandResult.stdout);
+      validateReview(review, testCase);
+      await writePrivate(resultPath, `${JSON.stringify(review, null, 2)}\n`);
+    } catch (error) {
+      parseError = error instanceof Error ? error.message : String(error);
+    }
+    lastReceipt = {
+      caseId: testCase.id,
+      reviewerId: reviewer.id,
+      model: reviewer.model,
+      attempt,
+      ok:
+        commandResult.exitCode === 0 && !commandResult.timedOut && !parseError,
+      review,
+      parseError,
+      ...commandResult,
+    };
+    if (lastReceipt.ok) {
+      return lastReceipt;
+    }
+  }
+  return lastReceipt;
+}
+
+async function runQueue(tasks, concurrency, worker) {
+  const results = new Array(tasks.length);
+  let nextIndex = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
+      while (nextIndex < tasks.length) {
+        const index = nextIndex;
+        nextIndex += 1;
+        results[index] = await worker(tasks[index]);
+      }
+    }),
+  );
+  return results;
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  const manifestDir = path.dirname(options.manifest);
+  const manifest = validateManifest(
+    JSON.parse(await readFile(options.manifest, "utf8")),
+    manifestDir,
+  );
+  await mkdir(options.output, { recursive: true, mode: 0o700 });
+  const sandboxDir = path.join(options.output, "empty-sandbox");
+  await mkdir(sandboxDir, { recursive: true, mode: 0o700 });
+  const schemaPath = path.join(options.output, "review.schema.json");
+  await writePrivate(schemaPath, `${JSON.stringify(OUTPUT_SCHEMA, null, 2)}\n`);
+
+  const preparedCases = [];
+  for (const testCase of manifest.cases) {
+    const patchText = await readFile(testCase.patchPath, "utf8");
+    const prompt = buildPrompt(testCase, patchText);
+    preparedCases.push({
+      ...testCase,
+      patchSha256: createHash("sha256").update(patchText).digest("hex"),
+      prompt,
+    });
+    for (const reviewer of REVIEWERS) {
+      await writePrivate(
+        path.join(
+          options.output,
+          "prompts",
+          `${testCase.id}-${reviewer.id}.txt`,
+        ),
+        prompt,
+      );
+    }
+  }
+  await writePrivate(
+    path.join(options.output, "prepared.json"),
+    `${JSON.stringify(
+      {
+        reviewers: REVIEWERS,
+        cases: preparedCases.map((testCase) => ({
+          id: testCase.id,
+          requirements: testCase.requirements,
+          patchPath: testCase.patchPath,
+          patchSha256: testCase.patchSha256,
+        })),
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  if (options.prepareOnly) {
+    process.stdout.write(
+      `${JSON.stringify(
+        {
+          prepared: true,
+          output: options.output,
+          caseCount: preparedCases.length,
+        },
+        null,
+        2,
+      )}\n`,
+    );
+    return;
+  }
+
+  const tasks = preparedCases.flatMap((testCase) =>
+    REVIEWERS.map((reviewer) => ({
+      reviewer,
+      testCase,
+      prompt: testCase.prompt,
+    })),
+  );
+  const receipts = await runQueue(tasks, REVIEWERS.length, (task) =>
+    invokeReviewer({
+      ...task,
+      outputDir: options.output,
+      schemaPath,
+      sandboxDir,
+      timeoutMs: options.timeoutMs,
+    }),
+  );
+  const cases = preparedCases.map((testCase) =>
+    aggregateCase(
+      testCase,
+      receipts.filter((receipt) => receipt.caseId === testCase.id),
+    ),
+  );
+  const protocolFailures = receipts
+    .filter((receipt) => !receipt.ok)
+    .map((receipt) => ({
+      caseId: receipt.caseId,
+      reviewerId: receipt.reviewerId,
+      model: receipt.model,
+      attempts: receipt.attempt,
+      exitCode: receipt.exitCode,
+      timedOut: receipt.timedOut,
+      spawnError: receipt.spawnError,
+      parseError: receipt.parseError,
+    }));
+  const result = {
+    version: 1,
+    createdAt: new Date().toISOString(),
+    reviewers: REVIEWERS,
+    cases,
+    protocolFailures,
+    receipts,
+  };
+  await writePrivate(
+    path.join(options.output, "result.json"),
+    `${JSON.stringify(result, null, 2)}\n`,
+  );
+  process.stdout.write(
+    `${JSON.stringify(
+      {
+        ok: protocolFailures.length === 0,
+        output: options.output,
+        cases,
+        protocolFailures,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+  if (protocolFailures.length > 0) {
+    process.exitCode = 1;
+  }
+}
+
+await main();

package/payload/plugins/toolkit/skills/review-with-runtime-evidence/scripts/static-review-contract.mjs

@@ -0,0 +1,243 @@
+import path from "node:path";
+
+export const REVIEWERS = [
+  {
+    id: "codex",
+    command: "codex",
+    model: "gpt-5.6-sol",
+  },
+  {
+    id: "gpt54",
+    command: "traex",
+    model: "gpt-5.4",
+  },
+  {
+    id: "openrouter",
+    command: "traex",
+    model: "openrouter-3o",
+  },
+];
+
+export const OUTPUT_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  properties: {
+    caseId: { type: "string" },
+    summary: { type: "string" },
+    findings: {
+      type: "array",
+      items: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+          id: { type: "string" },
+          requirementIds: {
+            type: "array",
+            items: { type: "string" },
+          },
+          severity: {
+            type: "string",
+            enum: ["P0", "P1", "P2", "P3"],
+          },
+          file: { type: "string" },
+          line: {
+            anyOf: [{ type: "integer" }, { type: "null" }],
+          },
+          problem: { type: "string" },
+          impact: { type: "string" },
+          evidence: { type: "string" },
+          confidence: {
+            type: "integer",
+            minimum: 0,
+            maximum: 100,
+          },
+        },
+        required: [
+          "id",
+          "requirementIds",
+          "severity",
+          "file",
+          "line",
+          "problem",
+          "impact",
+          "evidence",
+          "confidence",
+        ],
+      },
+    },
+  },
+  required: ["caseId", "summary", "findings"],
+};
+
+export function validateManifest(value, manifestDir) {
+  if (!value || typeof value !== "object" || !Array.isArray(value.cases)) {
+    throw new Error("manifest must contain a cases array");
+  }
+  if (value.cases.length === 0) {
+    throw new Error("manifest cases must not be empty");
+  }
+  const caseIds = new Set();
+  return {
+    cases: value.cases.map((testCase, index) => {
+      if (!testCase || typeof testCase !== "object") {
+        throw new Error(`cases[${index}] must be an object`);
+      }
+      if (
+        typeof testCase.id !== "string" ||
+        !testCase.id.trim() ||
+        caseIds.has(testCase.id)
+      ) {
+        throw new Error(`cases[${index}].id must be unique and non-empty`);
+      }
+      caseIds.add(testCase.id);
+      if (
+        !Array.isArray(testCase.requirements) ||
+        testCase.requirements.length === 0
+      ) {
+        throw new Error(
+          `case ${testCase.id} must contain at least one requirement`,
+        );
+      }
+      const requirementIds = new Set();
+      const requirements = testCase.requirements.map(
+        (requirement, requirementIndex) => {
+          if (
+            !requirement ||
+            typeof requirement.id !== "string" ||
+            !requirement.id.trim() ||
+            requirementIds.has(requirement.id) ||
+            typeof requirement.text !== "string" ||
+            !requirement.text.trim()
+          ) {
+            throw new Error(
+              `case ${testCase.id} requirement ${requirementIndex} is invalid`,
+            );
+          }
+          requirementIds.add(requirement.id);
+          return {
+            id: requirement.id,
+            text: requirement.text,
+          };
+        },
+      );
+      if (
+        typeof testCase.patchPath !== "string" ||
+        !testCase.patchPath.trim()
+      ) {
+        throw new Error(`case ${testCase.id}.patchPath is required`);
+      }
+      return {
+        id: testCase.id,
+        requirements,
+        requirementIds,
+        patchPath: path.resolve(manifestDir, testCase.patchPath),
+      };
+    }),
+  };
+}
+
+export function buildPrompt(testCase, patchText) {
+  return [
+    "You are one independent code-review cell.",
+    "Review only the supplied patch against the supplied requirements.",
+    "Do not assume a requirement is violated merely because it is listed.",
+    "Report P0/P1 only for a concrete executable path that violates a requirement or introduces an equally concrete blocking defect.",
+    "Use P2/P3 for non-blocking risks. Return findings: [] when the patch is correct.",
+    "Use exact supplied requirement IDs. Use [] only for a concrete defect outside the supplied requirements.",
+    "Do not use tools, repository access, prior reviewer output, or external context.",
+    "Return exactly one JSON object matching the output schema.",
+    "",
+    `CASE_ID: ${testCase.id}`,
+    `REQUIREMENTS: ${JSON.stringify(testCase.requirements)}`,
+    "",
+    "PATCH:",
+    patchText,
+  ].join("\n");
+}
+
+export function validateReview(review, testCase) {
+  if (
+    !review ||
+    typeof review !== "object" ||
+    Array.isArray(review) ||
+    review.caseId !== testCase.id ||
+    typeof review.summary !== "string" ||
+    !Array.isArray(review.findings)
+  ) {
+    throw new Error("review output has an invalid top-level shape");
+  }
+  for (const finding of review.findings) {
+    if (
+      !finding ||
+      typeof finding !== "object" ||
+      typeof finding.id !== "string" ||
+      !Array.isArray(finding.requirementIds) ||
+      !["P0", "P1", "P2", "P3"].includes(finding.severity) ||
+      typeof finding.file !== "string" ||
+      !(Number.isInteger(finding.line) || finding.line === null) ||
+      typeof finding.problem !== "string" ||
+      typeof finding.impact !== "string" ||
+      typeof finding.evidence !== "string" ||
+      !Number.isInteger(finding.confidence) ||
+      finding.confidence < 0 ||
+      finding.confidence > 100
+    ) {
+      throw new Error(`finding ${finding?.id ?? "unknown"} is invalid`);
+    }
+    for (const requirementId of finding.requirementIds) {
+      if (!testCase.requirementIds.has(requirementId)) {
+        throw new Error(
+          `finding ${finding.id} used unknown requirement ${requirementId}`,
+        );
+      }
+    }
+  }
+}
+
+export function aggregateCase(testCase, receipts) {
+  const highSeverities = new Set(["P0", "P1"]);
+  const byRequirement = new Map();
+  const unmappedSevere = [];
+  for (const receipt of receipts.filter((item) => item.ok)) {
+    for (const finding of receipt.review.findings.filter((item) =>
+      highSeverities.has(item.severity),
+    )) {
+      if (finding.requirementIds.length === 0) {
+        unmappedSevere.push({
+          reviewerId: receipt.reviewerId,
+          finding,
+        });
+        continue;
+      }
+      for (const requirementId of finding.requirementIds) {
+        const reviewers = byRequirement.get(requirementId) ?? new Map();
+        if (!reviewers.has(receipt.reviewerId)) {
+          reviewers.set(receipt.reviewerId, finding);
+        }
+        byRequirement.set(requirementId, reviewers);
+      }
+    }
+  }
+  const candidates = [...byRequirement.entries()].map(
+    ([requirementId, reviewers]) => ({
+      requirementId,
+      independentVotes: reviewers.size,
+      reviewers: [...reviewers.keys()],
+      proposals: [...reviewers.entries()].map(([reviewerId, finding]) => ({
+        reviewerId,
+        finding,
+      })),
+    }),
+  );
+  return {
+    caseId: testCase.id,
+    validReviewerCount: receipts.filter((item) => item.ok).length,
+    majorityCandidates: candidates.filter(
+      (candidate) => candidate.independentVotes >= 2,
+    ),
+    singletonCandidates: candidates.filter(
+      (candidate) => candidate.independentVotes === 1,
+    ),
+    unmappedSevere,
+  };
+}

package/payload/plugins/toolkit/skills/run-test-cases/SKILL.md

@@ -0,0 +1,63 @@
+---
+name: run-test-cases
+description: 当需要执行或验收 docs/testing/**/*.testplan.yaml 测试计划或用户给出的新格式 YAML 用例时使用。先校验最小 schema，再在真实浏览器、桌面或后端环境逐条取证；默认遇到首个失败即停，不负责编写用例，不兼容 Markdown 测试案例。
+---
+
+# 执行测试用例
+
+把测试计划视为不可偷改的验收合同。只执行 `.testplan.yaml`；收到 Markdown 测试案例时停止，
+要求先用 `write-test-cases` 迁移，不自行兼容解析。
+
+## 独立执行校验
+
+1. 读取完整 YAML，确认：
+   - 顶层只有 `version`、`name`、`description`、`cases`，且 `version: 1`。
+   - `cases` 包含 1-20 条；至少一条 `required: true`。
+   - 每条 case 只有 `id`、`name`、`required`、`description`、`preconditions`、`steps`。
+   - `id` 符合 `<统一大写前缀>-<三位连续编号>`；字符串非空，`preconditions` 和 `steps`
+     都是非空字符串数组。
+2. 仅当当前仓库的 `AGENTS.md`、项目文档或 package script **显式声明**测试计划 validator
+   时运行它；不得根据 pnpm、Rush、npm 等仓库类型猜测命令。Runweave 仓库显式入口为：
+
+   ```bash
+   pnpm testplan:validate <测试计划路径>
+   ```
+
+3. 仓库没有额外 validator 时，第一步的最小 schema 校验结果即为格式校验结果，不能据此
+   写 environment skip。
+4. 校验失败立即停止，不猜测或修补格式。
+5. 默认只执行 `required: true` 的 case，并按文件顺序逐条执行。只有用户明确要求全量回归时，
+   才额外执行 `required: false`。
+
+## 执行铁律
+
+- 默认严格逐条执行，任一 case 失败立即停止。
+- 不为了通过而修改测试计划。发现用例与已确认需求或当前前提矛盾时，停止并说明。
+- 逐项执行 `preconditions` 检查，再按 `steps` 的自然语言完成动作和结果核对。
+- 浏览器行为必须使用 `$toolkit:playwright-cli` 操作真实页面并获取 DOM、截图、控制台或网络
+  证据；桌面端联动使用 `$computer-use`；后端、协议和 CLI 运行真实命令并保留关键输出。
+- `typecheck` 和 `lint` 只是门禁，不能替代 UI 或行为取证。
+- 承诺过真实验证但无法执行时，写明“未执行 + 阻塞原因”，不得用代码阅读或推断判为通过。
+
+用户显式要求“失败后自己修”时，可以修实现并重跑当前 case；仍然不得擅自修改测试计划。
+
+## 失败归因
+
+以下情况视为用例问题并停止确认：
+
+- 描述与已确认需求或当前代码事实直接矛盾。
+- 前提条件已经不存在。
+- 步骤没有写出可观察结果，无法判断通过或失败。
+- 让它通过需要明显超出当前任务范围的行为修改。
+
+其余可稳定复现的偏差按实现问题报告；只有用户授权自愈时才修改实现。
+
+## 汇报
+
+使用中文报告：
+
+- 测试计划路径、格式校验依据与结果。
+- 每条 case ID、通过/失败/未执行、操作或命令、关键证据。
+- 未执行的 `required: false` case。
+- 首个失败点、实际结果、期望结果和建议下一步。
+- 自愈模式下列出修改文件与重验结果。