@dev-loops/core 1.0.0-rc.5 → 1.0.0-rc.7

package/src/projects/list-queue-items.mjs

@@ -1,6 +1,6 @@
 import { runChild as _runChild } from "../cli/primitives.mjs";
-import { parseJsonText } from "../github/review-threads.mjs";
 import { resolveProjectSelector, findProject } from "./resolve-project.mjs";
+import { ghGraphql } from "../github/gh.mjs";
 
 // ── Validation ───────────────────────────────────────────────────────────
 
@@ -30,32 +30,6 @@ function validateRepo(repo) {
   return repo;
 }
 
-// ── API helpers ──────────────────────────────────────────────────────────
-
-async function ghGraphql(query, vars, env, runChild = _runChild) {
-  const fieldArgs = [];
-  for (const [key, value] of Object.entries(vars)) {
-    fieldArgs.push("--field", `${key}=${value}`);
-  }
-  const result = await runChild(
-    "gh",
-    ["api", "graphql", "--field", `query=${query}`, ...fieldArgs],
-    env,
-  );
-  if (result.code !== 0) {
-    const detail = result.stderr.trim() || `exit code ${result.code}`;
-    throw Object.assign(new Error(`gh api graphql failed: ${detail}`), { code: "GH_API_ERROR" });
-  }
-  const payload = parseJsonText(result.stdout);
-  if (payload.errors && payload.errors.length > 0) {
-    throw Object.assign(
-      new Error(`GraphQL errors: ${payload.errors.map((e) => e.message).join("; ")}`),
-      { code: "GRAPHQL_ERROR" },
-    );
-  }
-  return payload;
-}
-
 // ── GraphQL fragments ────────────────────────────────────────────────────
 
 const GET_USER_ID = [

package/src/projects/move-queue-item.mjs

@@ -1,6 +1,8 @@
 import { runChild as _runChild } from "../cli/primitives.mjs";
-import { parseJsonText } from "../github/review-threads.mjs";
+import { runPickupRefinementGate } from "../loop/issue-refinement-artifact.mjs";
+import { loadStateColumnMap, LOGICAL_COLUMN } from "../loop/queue-board-sync.mjs";
 import { resolveProjectSelector, findProject, parseItemRef } from "./resolve-project.mjs";
+import { ghGraphql } from "../github/gh.mjs";
 
 // ── Validation ───────────────────────────────────────────────────────────
 
@@ -27,32 +29,6 @@ function validateRepo(repo) {
   return repo;
 }
 
-// ── API helpers ──────────────────────────────────────────────────────────
-
-async function ghGraphql(query, vars, env, runChild = _runChild) {
-  const fieldArgs = [];
-  for (const [key, value] of Object.entries(vars)) {
-    fieldArgs.push("--field", `${key}=${value}`);
-  }
-  const result = await runChild(
-    "gh",
-    ["api", "graphql", "--field", `query=${query}`, ...fieldArgs],
-    env,
-  );
-  if (result.code !== 0) {
-    const detail = result.stderr.trim() || `exit code ${result.code}`;
-    throw Object.assign(new Error(`gh api graphql failed: ${detail}`), { code: "GH_API_ERROR" });
-  }
-  const payload = parseJsonText(result.stdout);
-  if (payload.errors && payload.errors.length > 0) {
-    throw Object.assign(
-      new Error(`GraphQL errors: ${payload.errors.map((e) => e.message).join("; ")}`),
-      { code: "GRAPHQL_ERROR" },
-    );
-  }
-  return payload;
-}
-
 // ── GraphQL fragments ────────────────────────────────────────────────────
 
 const GET_USER_ID = [
@@ -252,6 +228,7 @@ function statusOf(node) {
 function classifyExitCode(err) {
   if (err.code === "INVALID_REPO" || err.code === "INVALID_PROJECT" || err.code === "INVALID_ITEM" ||
       err.code === "INVALID_COLUMN" || err.code === "INVALID_ARGS") return 1;
+  if (err.code === "MISSING_REFINEMENT_ARTIFACT") return 4;
   if (err.code === "PROJECT_NOT_FOUND" || err.code === "FIELD_NOT_FOUND" || err.code === "COLUMN_NOT_FOUND" ||
       err.code === "ITEM_NOT_FOUND") return 3;
   return 2;
@@ -259,7 +236,7 @@ function classifyExitCode(err) {
 
 // ── Main logic ──────────────────────────────────────────────────────────
 
-async function main(args, { env = process.env, runChild } = {}) {
+async function main(args, { env = process.env, runChild, cwd = null } = {}) {
   const child = runChild ?? _runChild;
   const repo = validateRepo(args.repo);
   const [owner, repoName] = repo.split("/");
@@ -365,6 +342,38 @@ async function main(args, { env = process.env, runChild } = {}) {
     };
   }
 
+  // 5b. QUEUE-ENQUEUE-REFINEMENT-GATE: moving an ISSUE into the pickup column
+  // must pass the same refinement gate `queue add` applies — a guard at one
+  // entry point but not its sibling is exactly the asymmetry this closes. Put
+  // here (in core, not the script wrapper) so every caller routing through core
+  // is covered — reconcile-queue.mjs included. reconcile is unaffected because
+  // it passes no cwd and never derives the pickup column. The column name is
+  // only derivable when cwd/config is supplied: an interactive `queue move` from
+  // a worktree passes cwd (gate fires), while headless callers (e.g.
+  // reconcile-queue.mjs) omit it and are unaffected — we simply never derive
+  // the column.
+  let refinement = null;
+  if (issueNumber !== null && typeof cwd === "string" && cwd.length > 0) {
+    const { columnNames, error: columnError } = loadStateColumnMap(cwd);
+    const pickupColumn = columnNames[LOGICAL_COLUMN.NEXT_UP];
+    if (pickupColumn && toColumn === pickupColumn) {
+      // Mirror queue add's fail-closed posture on a malformed `.devloops` when the
+      // interactive path supplied cwd AND the target is actually the pickup
+      // column: a config parse failure must never silently bypass the just-moved
+      // pickup refinement gate (the exact sibling asymmetry this issue closes).
+      // Scoped here (after the pickup-column guard) so an unrelated column move
+      // is not over-broadened into a hard CONFIG_ERROR failure.
+      if (columnError) {
+        throw Object.assign(
+          new Error(`could not resolve the pickup column (config read/parse error: ${columnError})`),
+          { code: "CONFIG_ERROR" },
+        );
+      }
+      const decision = await runPickupRefinementGate({ issueNumber, repo, env, runChild: child, auto: false });
+      refinement = { refined: decision.action === "enqueue" };
+    }
+  }
+
   // 6. Update Status via mutation
   const updatePayload = await ghGraphql(UPDATE_ITEM_FIELD, {
     projectId: project.id,
@@ -388,6 +397,7 @@ async function main(args, { env = process.env, runChild } = {}) {
       newColumn: toColumn,
       unchanged: false,
     },
+    ...(refinement ? { refinement } : {}),
   };
 }
 

package/src/security/secret-scan.mjs

@@ -0,0 +1,330 @@
+/**
+ * Fail-closed secret scan over the ADDED lines of a unified git diff.
+ *
+ * No external dependency (no gitleaks/trufflehog): this module is a small,
+ * self-contained detector set, deliberately kept auditable rather than
+ * outsourced. It is deterministic and side-effect free — it never reads or
+ * writes anything itself; callers (the CLI, the git hook) supply the diff
+ * text and act on the result.
+ *
+ * Three detector classes (see DETECTOR_CLASSES):
+ *  - literal-credential: a known provider token PREFIX (ghp_, xoxb-, AKIA...,
+ *    a PEM private-key header, ...), literal OR base64-encoded.
+ *  - high-entropy: a long single-token run whose Shannon entropy is above a
+ *    tuned threshold — catches a credential with no recognized prefix.
+ *  - sink-pattern: a secret-NAMED variable (*TOKEN*, *SECRET*, ...) and an
+ *    output sink (echo/printf/tee/base64/redirect/workflow `::` directive)
+ *    on the SAME line — no literal secret value has to be present in the diff
+ *    for this to fire; it is what catches a secret-named var piped into a
+ *    logged/echoed stream.
+ *
+ * A finding NEVER carries the matched value — only file/line/detector-class
+ * and a canned, value-free reason string. A secret, once flagged, is treated
+ * as unrecoverable: there is no "show me the match" affordance anywhere in
+ * this module.
+ *
+ * Allowlisting: an inline `secret-scan:allow` marker plus a trailing reason
+ * (comment syntax is irrelevant — this is a plain substring test) on the SAME
+ * line exempts that one line from every detector. There is no global disable
+ * and no central baseline file — every exemption is visible in the diff it
+ * exempts.
+ */
+
+export const DETECTOR_CLASSES = Object.freeze({
+  LITERAL_CREDENTIAL: "literal-credential",
+  HIGH_ENTROPY: "high-entropy",
+  SINK_PATTERN: "sink-pattern",
+});
+
+export const ALLOW_MARKER = "secret-scan:allow";
+const ALLOW_RE = /secret-scan:allow\s+(\S.*?)\s*$/u;
+
+/**
+ * Known provider token/credential PREFIX formats. Each pattern requires the
+ * full token shape (prefix + a plausible body length), not just the prefix
+ * alone — a bare mention of a prefix constant (as in this module's own
+ * source) is short of the required body length and never self-matches.
+ */
+const LITERAL_PATTERNS = [
+  { name: "github-pat-classic", re: /\bghp_[A-Za-z0-9]{20,}\b/ },
+  { name: "github-pat-fine-grained", re: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/ },
+  { name: "github-oauth-token", re: /\bgho_[A-Za-z0-9]{20,}\b/ },
+  { name: "github-user-to-server-token", re: /\bghu_[A-Za-z0-9]{20,}\b/ },
+  { name: "github-server-to-server-token", re: /\bghs_[A-Za-z0-9]{20,}\b/ },
+  { name: "github-refresh-token", re: /\bghr_[A-Za-z0-9]{20,}\b/ },
+  { name: "slack-token", re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ },
+  { name: "aws-access-key-id", re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
+  { name: "pem-private-key", re: /-----BEGIN [A-Z0-9]+(?: [A-Z0-9]+)* PRIVATE KEY-----/ },
+];
+
+/** True when `text` matches any known literal credential prefix format. */
+function matchLiteralPattern(text) {
+  for (const pattern of LITERAL_PATTERNS) {
+    if (pattern.re.test(text)) return pattern.name;
+  }
+  return null;
+}
+
+// A base64-shaped run long enough to plausibly carry an encoded credential.
+// `g` (global) so every candidate run on the line is checked, not just the
+// first.
+const BASE64_CANDIDATE_RE = /[A-Za-z0-9+/]{20,}={0,2}/gu;
+
+/**
+ * Whether `text` contains a base64-shaped substring that DECODES to a known
+ * literal credential format — the "value stored encoded" case a plain
+ * literal-prefix scan over the RAW diff text would miss.
+ */
+function matchBase64Credential(text) {
+  const candidates = text.match(BASE64_CANDIDATE_RE) ?? [];
+  for (const candidate of candidates) {
+    let decoded;
+    try {
+      decoded = Buffer.from(candidate, "base64").toString("utf8");
+    } catch {
+      continue;
+    }
+    const hit = matchLiteralPattern(decoded);
+    if (hit) return hit;
+  }
+  return null;
+}
+
+/**
+ * Shannon entropy in bits/character. Pure math, no I/O — exported so the
+ * threshold this module tunes is independently testable.
+ * @param {string} str
+ * @returns {number}
+ */
+export function shannonEntropy(str) {
+  if (str.length === 0) return 0;
+  const counts = new Map();
+  for (const ch of str) counts.set(ch, (counts.get(ch) ?? 0) + 1);
+  let entropy = 0;
+  for (const count of counts.values()) {
+    const p = count / str.length;
+    entropy -= p * Math.log2(p);
+  }
+  return entropy;
+}
+
+// A candidate is a single unbroken run (no whitespace, no `=`) of the
+// characters a token/secret literal is typically made of. `=` is
+// deliberately EXCLUDED from the run (unlike the base64-candidate pattern
+// above, which keeps it for trailing padding): an ordinary `KEY=value`
+// shell/env assignment is otherwise read as one inflated, higher-entropy
+// candidate spanning both sides of the `=` — splitting there scores the key
+// and the value separately, each far more likely to fall under the length or
+// digit/letter floor below on its own. Tuned length/threshold: 20 chars is
+// short enough to catch a real secret, long enough that ordinary identifiers
+// rarely reach it; 4.3 bits/char sits between plain lowercase English (~4.0
+// max, usually much lower) and true base64/hex randomness (~4.5-6). Both
+// digit AND letter are required so a long plain word (no digits) never
+// qualifies — false positives here cost only an allowlist line; a missed
+// real secret costs a leak, so the length/threshold pair leans toward
+// catching more, not fewer.
+const ENTROPY_CANDIDATE_RE = /[A-Za-z0-9+/_.-]{20,}/gu;
+const ENTROPY_MIN_LENGTH = 20;
+const ENTROPY_THRESHOLD = 4.3;
+
+function hasHighEntropyToken(text) {
+  const candidates = text.match(ENTROPY_CANDIDATE_RE) ?? [];
+  for (const candidate of candidates) {
+    if (candidate.length < ENTROPY_MIN_LENGTH) continue;
+    if (!/[0-9]/.test(candidate) || !/[A-Za-z]/.test(candidate)) continue;
+    if (shannonEntropy(candidate) >= ENTROPY_THRESHOLD) return true;
+  }
+  return false;
+}
+
+// A variable/env-key NAME that reads as a secret. Substring match for
+// TOKEN/SECRET/PASSWORD(/PASSWD) (`API_TOKEN`, `authToken`, `DB_PASSWORD`,
+// ...), suffix match for `_PAT`/`_KEY` (deliberately a suffix, not a
+// substring — "_KEY" alone would otherwise fire on ordinary words like
+// "keyboard"), plus the literal incident-motivating name. Tested per
+// IDENTIFIER, not per whole line — see hasSecretNamedIdentifier.
+const SECRET_KEYWORD_RE = /(?:TOKEN|SECRET|PASSWORD|PASSWD)|_PAT$|_KEY$/iu;
+const IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_]*/gu;
+
+/**
+ * True when `identifier` (an already-extracted `[A-Za-z_][A-Za-z0-9_]*` run)
+ * is SHAPED like a real variable/env-key name rather than a plain English
+ * word that merely contains a secret keyword as a substring: it has an
+ * underscore (snake_case / SCREAMING_SNAKE env-var shape, e.g.
+ * `GITHUB_TOKEN`, `foo_api_key`), mixes upper and lower case
+ * (camelCase/PascalCase, e.g. `authToken`), or is entirely uppercase letters
+ * (a bare ALL-CAPS identifier, e.g. `SECRET` used standalone). A plain
+ * single-case word with no underscore — "token", "secretary", "tokenize" —
+ * is not identifier-shaped, even though it may contain a secret keyword as a
+ * substring: "token-economical" and a bare "token" in prose don't qualify.
+ */
+function isIdentifierShaped(identifier) {
+  if (identifier.includes("_")) return true;
+  const hasUpper = /[A-Z]/u.test(identifier);
+  const hasLower = /[a-z]/u.test(identifier);
+  if (hasUpper && hasLower) return true; // camelCase / PascalCase
+  return hasUpper && !hasLower; // bare ALL-CAPS identifier (e.g. `SECRET`)
+}
+
+function hasSecretNamedIdentifier(text) {
+  for (const identifier of text.match(IDENTIFIER_RE) ?? []) {
+    if (identifier.toUpperCase() === "BUNDLE_GITHUB__COM") return true;
+    if (SECRET_KEYWORD_RE.test(identifier) && isIdentifierShaped(identifier)) return true;
+  }
+  return false;
+}
+
+// An angle-bracket PLACEHOLDER or generic type — `<owner/name>`, `<x>`, a
+// `Map` of two type parameters where the second is a credential-named type —
+// whose trailing angle bracket is not a shell redirect. The content charset
+// is deliberately narrow (identifier/path-ish chars only: letters, digits,
+// `_ , . / : -` and interior spaces) and must both START and END on an
+// identifier/path char (never on a space): this is what keeps a REAL
+// redirect pair like `<input.txt >output.txt` (space right before the
+// trailing bracket) from being misread as one placeholder spanning both —
+// the bracket-strip below would otherwise swallow a genuine redirect target.
+// Applied in a loop so nested placeholders (two levels of generic type
+// parameters) are fully stripped, innermost first.
+const ANGLE_PLACEHOLDER_RE = /<[A-Za-z0-9_](?:[A-Za-z0-9_ ,./:-]*[A-Za-z0-9_.])?>/gu;
+
+function stripAnglePlaceholders(text) {
+  let stripped = text;
+  let previous;
+  do {
+    previous = stripped;
+    stripped = stripped.replace(ANGLE_PLACEHOLDER_RE, "");
+  } while (stripped !== previous);
+  return stripped;
+}
+
+// Output sinks named in the motivating incident: echo/printf/tee, base64
+// (encode-then-emit), a `>`/`>>` redirect, and a GitHub Actions workflow
+// `::directive::`. This is deliberately a same-line, no-literal-value-
+// required class, so a variable name reaching a sink is enough on its own.
+// A workflow mask directive next to the sink call does not change the
+// verdict: masking only redacts what CI's log renderer shows LATER, so a
+// value still reaches this stream unmasked, and the flow is still a hit.
+// The redirect branch excludes `=>`/`->`/`>=` (lookbehind/lookahead around the
+// bare `>`/`>>`) — those are an arrow function or a comparison in ordinary
+// source, not a shell redirect, and would otherwise fire on nearly any JS/TS
+// line that also happens to name a *TOKEN*/*SECRET*/... variable — and
+// requires a plausible target to follow (`> file`, `>>out.log`,
+// `>/dev/null`), never a bare `>` with nothing after it. Run against the
+// angle-placeholder-stripped text (see stripAnglePlaceholders) so a `>` that
+// merely closes a `<owner/name>` placeholder or a `Map<string, AuthToken>`
+// generic is never read as a redirect.
+const SINK_RE = /\b(?:echo|printf|tee|base64)\b|(?<![=-])>{1,2}(?!=)(?=\s*\S)|::[A-Za-z][\w-]*::/u;
+
+function hasSecretNameToSinkFlow(text) {
+  return hasSecretNamedIdentifier(text) && SINK_RE.test(stripAnglePlaceholders(text));
+}
+
+/**
+ * The reason a matched line is allowlisted, or `null` when it is not. Any
+ * comment syntax works (`#`, `//`, `<!--`, ...): this is a plain substring
+ * test, not a language-aware parse — the `secret-scan:allow` marker plus a
+ * trailing reason exempts the ONE line it appears on, nothing else.
+ * @param {string} text
+ * @returns {string|null}
+ */
+export function allowlistReason(text) {
+  const match = ALLOW_RE.exec(text);
+  return match ? match[1] : null;
+}
+
+/**
+ * Scan one added line's text against all three detector classes.
+ * @param {string} text
+ * @returns {{ detectorClass: string, reason: string }[]}
+ */
+export function scanLineText(text) {
+  if (allowlistReason(text) !== null) return [];
+  const findings = [];
+  const literal = matchLiteralPattern(text) ?? matchBase64Credential(text);
+  if (literal) {
+    findings.push({
+      detectorClass: DETECTOR_CLASSES.LITERAL_CREDENTIAL,
+      reason: `matches known credential format (${literal})`,
+    });
+  }
+  if (hasHighEntropyToken(text)) {
+    findings.push({
+      detectorClass: DETECTOR_CLASSES.HIGH_ENTROPY,
+      reason: "high-entropy token-shaped literal",
+    });
+  }
+  if (hasSecretNameToSinkFlow(text)) {
+    findings.push({
+      detectorClass: DETECTOR_CLASSES.SINK_PATTERN,
+      reason: "secret-named variable flows into an output sink on this line",
+    });
+  }
+  return findings;
+}
+
+/**
+ * Parse a unified diff (`git diff --cached` output) into `{ file, line, text
+ * }` entries — one per ADDED line, `line` the new-file 1-based line number.
+ * Removed/context lines are skipped (but context lines still advance the
+ * new-file line counter). A binary-file diff (no `@@` hunk) yields no
+ * entries for that file.
+ * @param {string} diffText
+ * @returns {{ file: string, line: number, text: string }[]}
+ */
+export function parseAddedLines(diffText) {
+  const entries = [];
+  let file = null;
+  let newLine = 0;
+  // `+++ `/`--- ` are STRUCTURAL file headers only before the first `@@`
+  // hunk of a file — inside a hunk, a line starting with `+` (even `++ ` or
+  // `--- `-shaped content) is added/removed CONTENT, never a header. Gate the
+  // header checks on hunk position (`inHunk`), not a naive prefix test alone,
+  // so a planted credential on a line whose content happens to start with
+  // `+ ` can never be misread as a header and skipped from every detector.
+  let inHunk = false;
+  for (const rawLine of diffText.split("\n")) {
+    if (rawLine.startsWith("diff --git ")) {
+      inHunk = false;
+      continue;
+    }
+    if (!inHunk && rawLine.startsWith("+++ ")) {
+      const target = rawLine.slice(4).replace(/\t.*$/u, "");
+      file = target === "/dev/null" ? null : target.replace(/^b\//u, "");
+      continue;
+    }
+    if (!inHunk && rawLine.startsWith("--- ")) continue;
+    if (rawLine.startsWith("@@")) {
+      const match = rawLine.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/u);
+      newLine = match ? Number.parseInt(match[1], 10) : 0;
+      inHunk = true;
+      continue;
+    }
+    if (rawLine.startsWith("+")) {
+      entries.push({ file, line: newLine, text: rawLine.slice(1) });
+      newLine += 1;
+      continue;
+    }
+    if (rawLine.startsWith("-")) continue; // removed line — never advances the new-file counter
+    if (rawLine.startsWith("\\")) continue; // ""
+    // A context line (leading space) or a blank line inside a hunk both
+    // still exist in the new file, so both advance the counter.
+    if (newLine > 0) newLine += 1;
+  }
+  return entries;
+}
+
+/**
+ * Scan a full unified diff. Findings never carry the matched substring —
+ * only file/line/detectorClass/reason.
+ * @param {string} diffText
+ * @returns {{ ok: boolean, findings: { file: string, line: number, detectorClass: string, reason: string }[] }}
+ */
+export function scanDiffText(diffText) {
+  const findings = [];
+  for (const entry of parseAddedLines(diffText)) {
+    for (const hit of scanLineText(entry.text)) {
+      findings.push({ file: entry.file, line: entry.line, ...hit });
+    }
+  }
+  return { ok: findings.length === 0, findings };
+}