karajan-code 4.57.3 → 4.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/register-pipeline.js +11 -0
- package/src/harden/sandbox.js +7 -2
- package/src/harden/sentinel/sentinel-bash-write.mjs +27 -1
- package/src/harden/sentinel-hooks.js +10 -3
- package/src/privacy/scan.js +18 -0
- package/src/review/pr-security.js +5 -2
- package/src/review/publish-pr.js +1 -1
- package/src/review/repo-privacy.js +36 -0
- package/src/review/review-lens.js +19 -0
- package/src/review/third-party-review.js +5 -2
- package/src/review/verify-pr.js +59 -0
- package/templates/roles/reviewer-qa.md +18 -0
package/package.json
CHANGED
|
@@ -15,6 +15,7 @@ import { statusCommand } from "../commands/status.js";
|
|
|
15
15
|
import { loadPlan } from "../plan/plan-store.js";
|
|
16
16
|
import { withConfig } from "./_shared.js";
|
|
17
17
|
import { publishPrReview } from "../review/publish-pr.js";
|
|
18
|
+
import { verifyPrReview } from "../review/verify-pr.js";
|
|
18
19
|
import { thirdPartyReviewCommand } from "../review/third-party-review.js";
|
|
19
20
|
|
|
20
21
|
/**
|
|
@@ -258,8 +259,10 @@ export function registerPipeline(program, { pkgVersion }) {
|
|
|
258
259
|
.option("--walked <route>", "Recorrido comprobado como lo haría una persona (repetible) — queda en el veredicto", (v, prev) => [...(prev || []), v], [])
|
|
259
260
|
.option("--walked-with <tool>", "Con qué se comprobó el recorrido (por ejemplo chrome-devtools)")
|
|
260
261
|
.option("--publish-pr <n>", "Publish the cross-AI verdicts of a PR's commits on it as one GitHub review")
|
|
262
|
+
.option("--verify-pr <n>", "CI check: every commit of the PR is listed, with its diff hash, in the kj review a member published")
|
|
261
263
|
.option("--third-party", "Review someone else's PR (--range base...head): reviewer only, no own-work gates, no verdict stored")
|
|
262
264
|
.option("--json", "With --third-party: print the review as one JSON document on stdout")
|
|
265
|
+
.option("--lens <lens>", "With --third-party: what the reviewer looks at; qa = test coverage, weak assertions, flakiness, test data")
|
|
263
266
|
.action(async (task, flags) => {
|
|
264
267
|
await withConfig(pkgVersion, "review", flags, async ({ config, logger }) => {
|
|
265
268
|
// KJC-TSK-1075 (#2227): a third-party PR, for reviewer agents; own work keeps every gate.
|
|
@@ -273,6 +276,14 @@ export function registerPipeline(program, { pkgVersion }) {
|
|
|
273
276
|
logger.info(r.published ? `✓ PR #${flags.publishPr}: review published (${r.commits} commit(s))` : `PR #${flags.publishPr}: ${r.reason}`);
|
|
274
277
|
return;
|
|
275
278
|
}
|
|
279
|
+
// KJC-TSK-1088: the CI side of the same review; a commit without its verdict fails the check.
|
|
280
|
+
if (flags.verifyPr) {
|
|
281
|
+
const r = await verifyPrReview({ projectDir: config.projectDir || process.cwd(), pr: flags.verifyPr });
|
|
282
|
+
if (r.ok) logger.info(`✓ PR #${flags.verifyPr}: ${r.commits} commit(s), each with its cross-AI verdict`);
|
|
283
|
+
else logger.error(`✗ PR #${flags.verifyPr}:\n - ${r.problems.join("\n - ")}`);
|
|
284
|
+
process.exitCode = r.ok ? 0 : 1;
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
276
287
|
// ENV-B1 (KJC-TSK-0637): the gate mode records a verdict tied to
|
|
277
288
|
// the exact diff so the pre-commit hook can enforce cross-AI review.
|
|
278
289
|
if (flags.staged || flags.check || flags.range || flags.installGate || flags.prune) {
|
package/src/harden/sandbox.js
CHANGED
|
@@ -60,8 +60,13 @@ export function probeSandbox({ platform = process.platform, which = whichOnPath,
|
|
|
60
60
|
// Outside the project only kj's state and package caches are writable, and the
|
|
61
61
|
// supervisor and the phone key stay out of reach.
|
|
62
62
|
const METHOD_SANDBOX = {
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
// KJC-TSK-1114: with these denied, the Sentinel's Bash-text guards step aside (plan KJC-TSK-1081);
|
|
64
|
+
// kj worktree runs outside, it is the sanctioned way to create and remove lanes.
|
|
65
|
+
excludedCommands: ["gh *", "kj review *", "kj worktree *"],
|
|
66
|
+
denyWrite: [
|
|
67
|
+
".karajan/harness", ".karajan/hooks", ".claude/settings.json", ".karajan/supervisor-provenance.json", "~/.karajan/supervisor-phone.json",
|
|
68
|
+
"kj.config.yml", ".ragignore", ".kj/worktrees",
|
|
69
|
+
],
|
|
65
70
|
allowWrite: ["~/.karajan", "~/.npm/_cacache", "~/.npm/_logs", "~/.npm/_npx", "~/.cache", "~/.config/configstore"],
|
|
66
71
|
allowedDomains: [
|
|
67
72
|
"github.com", "api.github.com", "*.githubusercontent.com", "uploads.github.com", "registry.npmjs.org",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// ADR 0014). Inside the repo, files are written through Edit/Write only, the path
|
|
3
3
|
// every gate guards. Copied byte for byte into .karajan/harness; `root` injected.
|
|
4
4
|
// The stated limit: a program you run can write files; only a sandbox stops that.
|
|
5
|
-
import { existsSync, realpathSync } from "node:fs";
|
|
5
|
+
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
6
6
|
import { homedir } from "node:os";
|
|
7
7
|
import { dirname, join, relative, resolve } from "node:path";
|
|
8
8
|
import { headIndex, shellSegments, withoutRedirections } from "./sentinel-shell.mjs";
|
|
@@ -100,3 +100,29 @@ export const writesRepo = (t, root) => {
|
|
|
100
100
|
}
|
|
101
101
|
return !rel.startsWith("..") && !rel.startsWith("/");
|
|
102
102
|
};
|
|
103
|
+
|
|
104
|
+
/** The project's sandbox settings: the local file wins over the shared one; unreadable is none. */
|
|
105
|
+
const sandboxOf = (root) => {
|
|
106
|
+
for (const file of ["settings.local.json", "settings.json"]) {
|
|
107
|
+
const p = join(root, ".claude", file);
|
|
108
|
+
if (!existsSync(p)) continue;
|
|
109
|
+
try {
|
|
110
|
+
return JSON.parse(readFileSync(p, "utf8")).sandbox ?? null;
|
|
111
|
+
} catch {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* KJC-TSK-1114 (plan KJC-TSK-1081): does the host sandbox already deny Bash writes to
|
|
120
|
+
* every one of these repo-relative paths? Only then does a guard that reads Bash text
|
|
121
|
+
* step aside. A sandbox that is off, can be bypassed or misses a path covers nothing.
|
|
122
|
+
*/
|
|
123
|
+
export const sandboxCovers = (root, paths) => {
|
|
124
|
+
const sb = sandboxOf(root);
|
|
125
|
+
if (sb?.enabled !== true || sb.allowUnsandboxedCommands !== false) return false;
|
|
126
|
+
const denied = (sb.filesystem?.denyWrite ?? []).map((d) => String(d).replace(/^\.\//, "").replace(/\/+$/, ""));
|
|
127
|
+
return paths.every((p) => denied.some((d) => p === d || p.startsWith(d + "/")));
|
|
128
|
+
};
|
|
@@ -479,7 +479,7 @@ import { doc, CODE, TESTS, ROOT, BASE_BRANCHES, CARD, STATE, branchOf, foreignLa
|
|
|
479
479
|
// KJC-TSK-0915 (ADR 0014): the shell reader is a real, unit-tested module copied here as is.
|
|
480
480
|
import { isGhIssueCreate, isPublishCommand, optionValues, shellSegments, stripInertQuotes, stripTextOptionValues, stripTextOnlyCommandValues } from "./sentinel-shell.mjs";
|
|
481
481
|
import { DISCARD_VERBS, discardOf, foreignLost, removedFiles } from "./sentinel-discard.mjs";
|
|
482
|
-
import { shellWrites, writesRepo } from "./sentinel-bash-write.mjs";
|
|
482
|
+
import { sandboxCovers, shellWrites, writesRepo } from "./sentinel-bash-write.mjs";
|
|
483
483
|
const EDIT_TOOLS = ["Write", "Edit", "MultiEdit", "NotebookEdit"];
|
|
484
484
|
const inOrder = (want, got) => { let i = 0; for (const w of got) if (w === want[i] && ++i === want.length) return true; return false; };
|
|
485
485
|
// KJC-BUG-0333: a publish is read per simple command by the shell reader, so
|
|
@@ -547,7 +547,9 @@ process.stdin.on("end", () => {
|
|
|
547
547
|
// the Read/Grep tools are for. Shell indirection (variables, globs,
|
|
548
548
|
// substitution) cannot be resolved from command text: that vector is
|
|
549
549
|
// caught by the Stop gate's checksum verification, which blocks the turn.
|
|
550
|
-
|
|
550
|
+
// KJC-TSK-1114: where the host sandbox denies writing them, the sandbox guards them instead.
|
|
551
|
+
const SUPERVISOR_PATHS = [".claude/settings.json", ".karajan/hooks", ".karajan/harness"];
|
|
552
|
+
if (tool === "Bash" && PROTECTED.test(String(input.command || "").replaceAll("\\\\", "/")) && !sandboxCovers(ROOT, SUPERVISOR_PATHS)) {
|
|
551
553
|
console.error("karajan sentinel: los ficheros del supervisor no se tocan desde Bash — consultalos con la tool Read/Grep; solo el humano los modifica, fuera de la sesion." + doc("supervisor"));
|
|
552
554
|
process.exit(2);
|
|
553
555
|
}
|
|
@@ -745,6 +747,8 @@ process.stdin.on("end", () => {
|
|
|
745
747
|
console.error("karajan sentinel: git worktree add/remove/prune no se ejecuta desde una sesion — estructura del repo, sin escape; usa kj worktree start|done." + doc("cross-lane"));
|
|
746
748
|
process.exit(2);
|
|
747
749
|
}
|
|
750
|
+
// KJC-TSK-1114: where the host sandbox denies writing the lanes, it confines Bash itself.
|
|
751
|
+
if (!sandboxCovers(ROOT, [".kj/worktrees"])) {
|
|
748
752
|
// Deny-unless-known-read-only: un allowlist de verbos mutadores era
|
|
749
753
|
// esquivable (dd, interpretes). Solo un comando SIMPLE de lectura (sin
|
|
750
754
|
// encadenar ni redirigir) puede nombrar otro carril; leer carriles es
|
|
@@ -876,10 +880,13 @@ process.stdin.on("end", () => {
|
|
|
876
880
|
if (lane && laneDeny(m[1], lane)) process.exit(2);
|
|
877
881
|
}
|
|
878
882
|
}
|
|
883
|
+
}
|
|
879
884
|
}
|
|
880
885
|
// KJC-BUG-0237 (#1886): the repo is written through Edit/Write only, the path every gate
|
|
881
886
|
// guards. After the lane guard, whose message is the precise one for another lane.
|
|
882
|
-
|
|
887
|
+
// KJC-TSK-1114: gates run at commit now; what only this guard kept, the governance
|
|
888
|
+
// files, the host sandbox denies where it covers them.
|
|
889
|
+
if (tool === "Bash" && !sandboxCovers(ROOT, ["kj.config.yml", ".ragignore"])) {
|
|
883
890
|
let moved = false; // after cd/pushd/popd, a relative target cannot be placed: fail-closed
|
|
884
891
|
// $( ) and backticks run even inside double quotes: their bodies, read from the raw text, are commands too.
|
|
885
892
|
const raw = String(input.command || "");
|
package/src/privacy/scan.js
CHANGED
|
@@ -82,6 +82,16 @@ const SEMVER_JOIN = String.raw`\s*(?:(?:\|\||-)\s*${SEMVER_OP}?|${SEMVER_OP})\s*
|
|
|
82
82
|
const SEMVER_VALUE = new RegExp(String.raw`^\s*${SEMVER_OP}?\s*${SEMVER_TOKEN}(?:${SEMVER_JOIN}${SEMVER_TOKEN})*\s*$`);
|
|
83
83
|
|
|
84
84
|
/** KJC-BUG-0380: a pair is discarded only when it is a real latitude, longitude. */
|
|
85
|
+
// KJC-TSK-1118: the test card numbers payment providers publish (Stripe, Braintree,
|
|
86
|
+
// Adyen, PayPal docs). Only these exact numbers; a card one digit away still warns.
|
|
87
|
+
const TEST_CARDS = new Set([
|
|
88
|
+
"4242424242424242", "4000056655665556", "4111111111111111", "4012888888881881", "4000000000000002",
|
|
89
|
+
"5555555555554444", "5105105105105100", "2223003122003222", "5200828282828210",
|
|
90
|
+
"378282246310005", "371449635398431", "6011111111111117", "6011000990139424",
|
|
91
|
+
"3056930009020004", "36227206271667", "3530111333300000", "3566002020360505",
|
|
92
|
+
]);
|
|
93
|
+
const isTestCard = (match) => TEST_CARDS.has(match.replaceAll(/[\s-]/g, ""));
|
|
94
|
+
|
|
85
95
|
function inGpsRange(_match, lat, lng) {
|
|
86
96
|
return Math.abs(Number(lat)) <= 90 && Math.abs(Number(lng)) <= 180;
|
|
87
97
|
}
|
|
@@ -124,6 +134,14 @@ const CONTEXT_DISCARDS = [
|
|
|
124
134
|
// range (latitude up to 90, longitude up to 180), joined by a comma. A lone
|
|
125
135
|
// number or a pair out of range still warns.
|
|
126
136
|
{ type: "gps-pair", re: /(?<![\d.])(-?\d{1,3}\.\d{4,})\s*,\s*(-?\d{1,3}\.\d{4,})(?![\d.])/g, keep: inGpsRange },
|
|
137
|
+
// KJC-TSK-1118: numbers reserved for fiction and tests, the good practice for test data.
|
|
138
|
+
// NANP 555-0100..0199 (with an optional +1 and area code); Ofcom's drama ranges:
|
|
139
|
+
// 07700 900xxx, 020 7946 0xxx and 01632 960xxx (0 or +44). Nothing around them.
|
|
140
|
+
{ type: "fictional-phone", re: /(?<![\d+])(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]?555[\s.-]?01\d{2}(?!\d)/g },
|
|
141
|
+
{ type: "fictional-phone-uk", re: /(?<![\d+])(?:\+44\s?|0)7700\s?900\d{3}(?!\d)/g },
|
|
142
|
+
{ type: "fictional-phone-uk", re: /(?<![\d+])(?:\+44\s?|0)20\s?7946\s?0\d{3}(?!\d)/g },
|
|
143
|
+
{ type: "fictional-phone-uk", re: /(?<![\d+])(?:\+44\s?|0)1632\s?960\d{3}(?!\d)/g },
|
|
144
|
+
{ type: "test-card", re: /(?<!\d)\d(?:[\s-]?\d){13,18}(?!\d)/g, keep: isTestCard },
|
|
127
145
|
{ type: "plan-id", re:/(?<![A-Za-z0-9])plan-20\d\d(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\d|3[01])(?:[01]\d|2[0-3])[0-5]\d[0-5]\d-[a-z0-9]{4}(?![A-Za-z0-9])/g },
|
|
128
146
|
];
|
|
129
147
|
|
|
@@ -11,6 +11,7 @@ import { loadPrivacyList, scanText } from "../privacy/scan.js";
|
|
|
11
11
|
import { isGeneratedPath, splitAddedByFile } from "../privacy/diff-scope.js";
|
|
12
12
|
import { collectSemgrepFindings } from "../audit/semgrep-findings.js";
|
|
13
13
|
import { cooldownAdvisories } from "./dep-cooldown.js";
|
|
14
|
+
import { repoPrivacyAllow, withRepoAllow } from "./repo-privacy.js";
|
|
14
15
|
|
|
15
16
|
const CREDENTIAL_IDS = new Set(CREDENTIAL_PATTERNS.map((p) => p.id));
|
|
16
17
|
const BLOCKING_SEMGREP = new Set(["ERROR", "WARNING"]);
|
|
@@ -58,11 +59,13 @@ export async function semgrepCheck({ projectDir, files, collect = collectSemgrep
|
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
/** Every security and privacy check of the PR's diff. */
|
|
61
|
-
export async function prSecurityChecks({ diff, projectDir, config }) {
|
|
62
|
+
export async function prSecurityChecks({ diff, projectDir, config, range }) {
|
|
62
63
|
const files = splitAddedByFile(diff).map((f) => f.file);
|
|
64
|
+
// KJC-TSK-1112: the reviewed repo's allow list, as it stands at the base of the range.
|
|
65
|
+
const list = withRepoAllow(loadPrivacyList(), await repoPrivacyAllow({ projectDir, range }));
|
|
63
66
|
return [
|
|
64
67
|
secretsCheck(diff, config),
|
|
65
|
-
privacyCheck(diff),
|
|
68
|
+
privacyCheck(diff, { list }),
|
|
66
69
|
injectionCheck(diff),
|
|
67
70
|
await cooldownCheck(diff, { thresholdHours: config?.supply_chain?.cooldown_hours }),
|
|
68
71
|
await semgrepCheck({ projectDir, files }),
|
package/src/review/publish-pr.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import { runCommand } from "../utils/process.js";
|
|
8
8
|
import { diffHash, loadVerdict } from "./verdict-store.js";
|
|
9
9
|
|
|
10
|
-
const MARKER = (head) => `<!-- kj-review head=${head} -->`;
|
|
10
|
+
export const MARKER = (head) => `<!-- kj-review head=${head} -->`;
|
|
11
11
|
|
|
12
12
|
// gh runs in the project's own directory, so it reads and writes the same repo the verdicts belong to.
|
|
13
13
|
async function json(run, cmd, args, cwd) {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KJC-TSK-1112: the reviewed repo's own allow list for the privacy check of a
|
|
3
|
+
* third-party PR. Only `allow` is read: a personal list never lives in a repo
|
|
4
|
+
* (it is sensitive, see privacy/scan.js). The file is read at the BASE of the
|
|
5
|
+
* range, never at the head, so a PR cannot add the exemption it needs.
|
|
6
|
+
*/
|
|
7
|
+
import { parse as parseYaml } from "yaml";
|
|
8
|
+
import { runCommand } from "../utils/process.js";
|
|
9
|
+
|
|
10
|
+
const REPO_PRIVACY = ".karajan/privacy.yml";
|
|
11
|
+
|
|
12
|
+
const baseOf = (range) => {
|
|
13
|
+
const base = String(range).split(/\.{2,3}/)[0];
|
|
14
|
+
if (!base || base === range) throw new Error(`cannot read the base of the range ${range}: pass <base>...<head>`);
|
|
15
|
+
return base;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** The repo's allow list at the base of the range; empty when the base has no such file. */
|
|
19
|
+
export async function repoPrivacyAllow({ projectDir, range, run = runCommand }) {
|
|
20
|
+
const base = baseOf(range);
|
|
21
|
+
const res = await run("git", ["-C", projectDir, "show", `${base}:${REPO_PRIVACY}`]);
|
|
22
|
+
if (res.exitCode !== 0) {
|
|
23
|
+
if (/does not exist|exists on disk, but not in/.test(res.stderr ?? "")) return [];
|
|
24
|
+
throw new Error(`git show ${base}:${REPO_PRIVACY} failed: ${(res.stderr || "").trim()}`);
|
|
25
|
+
}
|
|
26
|
+
let raw;
|
|
27
|
+
try {
|
|
28
|
+
raw = parseYaml(res.stdout) || {};
|
|
29
|
+
} catch (err) {
|
|
30
|
+
throw new Error(`${REPO_PRIVACY} at ${base} is not valid YAML: ${err.message}`, { cause: err });
|
|
31
|
+
}
|
|
32
|
+
return Array.isArray(raw.allow) ? raw.allow.map(String).map((s) => s.trim()).filter(Boolean) : [];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The user's privacy list with the repo's allow added; the personal list stays the user's own. */
|
|
36
|
+
export const withRepoAllow = (list, repoAllow) => ({ ...list, allow: [...list.allow, ...repoAllow] });
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KJC-TSK-1117: a review lens changes what the reviewer looks at, not how strict
|
|
3
|
+
* it is (that is review_mode). The qa lens puts a QA rubric before the task, so a
|
|
4
|
+
* PR of test automation is judged on whether its tests would catch bugs.
|
|
5
|
+
*/
|
|
6
|
+
import { readFile } from "node:fs/promises";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { getTemplatesRoot } from "../utils/templates-root.js";
|
|
9
|
+
|
|
10
|
+
const RUBRICS = { qa: "reviewer-qa.md" };
|
|
11
|
+
export const LENSES = Object.keys(RUBRICS);
|
|
12
|
+
|
|
13
|
+
/** The reviewer's task under a lens; no lens leaves it as it is. */
|
|
14
|
+
export async function lensTask(lens, task) {
|
|
15
|
+
if (!lens) return task;
|
|
16
|
+
if (!Object.hasOwn(RUBRICS, lens)) throw new Error(`unknown review lens "${lens}": use ${LENSES.join(", ")}`);
|
|
17
|
+
const rubric = (await readFile(join(getTemplatesRoot(), "roles", RUBRICS[lens]), "utf8")).trim();
|
|
18
|
+
return task ? `${rubric}\n\n## Task\n\n${task}` : rubric;
|
|
19
|
+
}
|
|
@@ -14,6 +14,7 @@ import { isGenerated } from "./loc-budget.js";
|
|
|
14
14
|
import { prSecurityChecks } from "./pr-security.js";
|
|
15
15
|
import { sonarCheck } from "./pr-sonar.js";
|
|
16
16
|
import { batchDiff, splitByFile } from "./diff-batches.js";
|
|
17
|
+
import { lensTask } from "./review-lens.js";
|
|
17
18
|
import { DIFF_CLIP_CHARS } from "../prompts/diff-clip.js";
|
|
18
19
|
|
|
19
20
|
// KJC-BUG-0378: a reviewer call per batch; past this many, files are left out and named.
|
|
@@ -37,7 +38,7 @@ export async function thirdPartyReview({ range, config, projectDir, logger, task
|
|
|
37
38
|
// KJC-BUG-0379 (#2263): the house requirements (size, commits, attribution, tests) travel apart:
|
|
38
39
|
// on another repo with other conventions they must not read as findings next to security.
|
|
39
40
|
const policyChecks = await requirements({ range, projectDir, config });
|
|
40
|
-
const checks = [...await security({ diff, projectDir, config }), await sonar({ diff, config, projectDir })];
|
|
41
|
+
const checks = [...await security({ diff, projectDir, config, range }), await sonar({ diff, config, projectDir })];
|
|
41
42
|
// KJC-BUG-0378 (#2263): the whole diff, in batches of whole files that fit the reviewer's
|
|
42
43
|
// prompt; what does not fit is named in the coverage and the verdict cannot be approved.
|
|
43
44
|
const provider = resolveRole(config, "reviewer").provider || null;
|
|
@@ -91,7 +92,9 @@ export async function thirdPartyReviewCommand({ flags, config, out = process.std
|
|
|
91
92
|
const log = (msg) => err.write(`${msg}\n`);
|
|
92
93
|
const logger = { info: log, warn: log, error: log, debug: () => {} };
|
|
93
94
|
try {
|
|
94
|
-
|
|
95
|
+
// KJC-TSK-1117: a lens (qa) puts its rubric before the task.
|
|
96
|
+
const task = await lensTask(flags.lens ?? null, flags.task ?? null);
|
|
97
|
+
print(await review({ range: flags.range, config, projectDir: config.projectDir || process.cwd(), logger, task }));
|
|
95
98
|
return 0;
|
|
96
99
|
} catch (e) {
|
|
97
100
|
print({ error: e.message });
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KJC-TSK-1088: CI proof that every commit of a PR carries its cross-AI
|
|
3
|
+
* verdict, so skipping the pre-commit hook (--no-verify) cannot leave a commit
|
|
4
|
+
* unreviewed. It reads the review kj review --publish-pr posted for the PR's
|
|
5
|
+
* head, written by a member of the repo, and recomputes the hash of the diff
|
|
6
|
+
* each commit brought: a listed hash that no longer matches is no proof.
|
|
7
|
+
*/
|
|
8
|
+
import { runCommand } from "../utils/process.js";
|
|
9
|
+
import { diffHash } from "./verdict-store.js";
|
|
10
|
+
import { MARKER } from "./publish-pr.js";
|
|
11
|
+
|
|
12
|
+
const TRUSTED = new Set(["OWNER", "MEMBER", "COLLABORATOR"]);
|
|
13
|
+
// One line per commit, as publish-pr writes it: - `<sha7>` headline: approved by X (diff <hash12>)...
|
|
14
|
+
const LISTED = /^- `([0-9a-f]{7})` .*?\(diff ([0-9a-f]{12})\)/gm;
|
|
15
|
+
|
|
16
|
+
async function json(run, args, cwd) {
|
|
17
|
+
const res = await run("gh", args, { cwd });
|
|
18
|
+
if (res.exitCode !== 0) throw new Error(`gh ${args.slice(0, 3).join(" ")} failed: ${(res.stderr || res.stdout || "").trim()}`);
|
|
19
|
+
return JSON.parse(res.stdout || "null");
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// KJC-BUG-0386: git abbreviates the object ids of the index lines by the clone's object
|
|
23
|
+
// count (or core.abbrev, 4 up to the full 40), so the publisher and CI may print the
|
|
24
|
+
// same diff with ids of other lengths.
|
|
25
|
+
const ABBREVS = Array.from({ length: 37 }, (_, i) => i + 4);
|
|
26
|
+
|
|
27
|
+
/** The diff hashes of a commit under every abbreviation git may have used where it was reviewed. */
|
|
28
|
+
async function hashesOf(projectDir, sha, run) {
|
|
29
|
+
const hashes = new Set();
|
|
30
|
+
for (const n of ABBREVS) {
|
|
31
|
+
const res = await run("git", ["-C", projectDir, "diff", `--abbrev=${n}`, `${sha}^`, sha]);
|
|
32
|
+
if (res.exitCode !== 0) throw new Error(`git diff ${sha.slice(0, 7)} failed: ${(res.stderr || "").trim()}`);
|
|
33
|
+
hashes.add(diffHash(res.stdout).slice(0, 12));
|
|
34
|
+
}
|
|
35
|
+
return hashes;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @returns {Promise<{ok: boolean, commits: number, problems: string[]}>}
|
|
40
|
+
*/
|
|
41
|
+
export async function verifyPrReview({ projectDir, pr, run = runCommand }) {
|
|
42
|
+
const info = await json(run, ["pr", "view", String(pr), "--json", "headRefOid,commits"], projectDir);
|
|
43
|
+
const commits = info.commits ?? [];
|
|
44
|
+
// One review per line, every page: --slurp is missing from the gh that distros ship.
|
|
45
|
+
const res = await run("gh", ["api", "--paginate", `repos/{owner}/{repo}/pulls/${pr}/reviews`, "--jq", ".[] | {body, author_association}"], { cwd: projectDir });
|
|
46
|
+
if (res.exitCode !== 0) throw new Error(`gh api reviews failed: ${(res.stderr || res.stdout || "").trim()}`);
|
|
47
|
+
const reviews = (res.stdout || "").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
48
|
+
const review = reviews.findLast((r) => TRUSTED.has(r.author_association) && String(r.body ?? "").includes(MARKER(info.headRefOid)));
|
|
49
|
+
if (!review) {
|
|
50
|
+
return { ok: false, commits: commits.length, problems: [`no kj review published by a member for head ${info.headRefOid.slice(0, 7)}: run kj review --publish-pr ${pr}`] };
|
|
51
|
+
}
|
|
52
|
+
const listed = new Map([...review.body.matchAll(LISTED)].map((m) => [m[1], m[2]]));
|
|
53
|
+
const problems = [];
|
|
54
|
+
for (const c of commits) {
|
|
55
|
+
const hash = listed.get(c.oid.slice(0, 7));
|
|
56
|
+
if (!hash || !(await hashesOf(projectDir, c.oid, run)).has(hash)) problems.push(`${c.oid.slice(0, 7)} (${c.messageHeadline}): no approved verdict for the diff it brings`);
|
|
57
|
+
}
|
|
58
|
+
return { ok: problems.length === 0, commits: commits.length, problems };
|
|
59
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Review lens: QA
|
|
2
|
+
|
|
3
|
+
Review this PR as a QA engineer, not only as a developer. Most of it may be test code: judge whether the tests would catch the bugs they exist for.
|
|
4
|
+
|
|
5
|
+
## Blocking (in the diff only)
|
|
6
|
+
|
|
7
|
+
- **No test for new behavior**: the change adds or changes behavior and no test exercises it.
|
|
8
|
+
- **Tests that cannot fail**: no assertion, assertions only on mocks, `toBeTruthy`/`assert True` on values that are always truthy, a caught exception that swallows the failure.
|
|
9
|
+
- **Flaky by construction**: fixed sleeps or timeouts instead of waiting on a condition, order dependence between tests, shared mutable state, real clock, randomness or network without control, retries that hide a failure.
|
|
10
|
+
- **Real personal data or production credentials in test data.**
|
|
11
|
+
|
|
12
|
+
## Suggestions (non-blocking)
|
|
13
|
+
|
|
14
|
+
- **Missing scenarios**: name the edge and error cases the new tests skip (empty, boundary, invalid input, permissions, concurrency).
|
|
15
|
+
- **Weak assertions**: only a status code, only "element exists", snapshots standing in for a precise check.
|
|
16
|
+
- **Fragile selectors**: CSS chains, XPath by index or text that changes; prefer roles, labels or `data-testid`.
|
|
17
|
+
- **Test data**: prefer values reserved for testing (emails on `example.com`/`example.org`/`.test`, fictional phone ranges such as 555-01xx or 07700 900xxx, the test card numbers each payment provider publishes) or generated data (factories, faker), created and cleaned up by each test.
|
|
18
|
+
- **Structure**: one behavior per test, names that say the scenario and the expected result, page objects and flows that keep selectors out of the test body, no duplicated setup.
|