@ryuenn3123/agentic-senior-core 3.0.49 → 4.0.0

package/scripts/llm-judge/constants.mjs

@@ -0,0 +1,66 @@
+// @ts-check
+
+/**
+ * Static configuration for the LLM judge CI gate. Centralizes paths, env-driven
+ * settings, and severity normalization so the rest of the pipeline reads from a
+ * single config surface.
+ */
+
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+export const REPOSITORY_ROOT = resolve(__dirname, '..', '..');
+export const PR_CHECKLIST_PATH = resolve(REPOSITORY_ROOT, '.agent-context/review-checklists/pr-checklist.md');
+export const THRESHOLDS_PATH = resolve(REPOSITORY_ROOT, '.agent-context/policies/llm-judge-threshold.json');
+export const DEFAULT_MACHINE_REPORT_PATH = resolve(REPOSITORY_ROOT, '.agent-context/state/llm-judge-report.json');
+
+export const MAX_DIFF_CHARS = parseInt(process.env.LLM_MAX_DIFF_CHARS ?? '12000', 10);
+export const IS_DRY_RUN = process.argv.includes('--dry-run');
+export const SHOULD_EMIT_MACHINE_REPORT = process.env.LLM_JUDGE_EMIT_JSON !== 'false';
+export const MACHINE_REPORT_PATH = process.env.LLM_JUDGE_OUTPUT_PATH || DEFAULT_MACHINE_REPORT_PATH;
+
+/** @type {Record<string, string>} */
+export const SEVERITY_NORMALIZATION_TABLE = {
+  critical: 'critical',
+  blocker: 'critical',
+  severe: 'critical',
+  high: 'high',
+  major: 'high',
+  medium: 'medium',
+  moderate: 'medium',
+  low: 'low',
+  minor: 'low',
+  info: 'low',
+  informational: 'low',
+};
+
+/**
+ * @typedef {{
+ *   rule: string,
+ *   problem: string,
+ *   severity: string,
+ * }} Violation
+ */
+
+/**
+ * @typedef {{
+ *   generatedAt: string,
+ *   schemaVersion: string,
+ *   profile: string,
+ *   provider: string,
+ *   ciProvider: string,
+ *   blockingSeverities: string[],
+ *   failDecision: boolean,
+ *   malformedVerdict: boolean,
+ *   providerError: boolean,
+ *   dryRun: boolean,
+ *   summary: {
+ *     totalViolations: number,
+ *     blockingViolations: number,
+ *   },
+ *   violations: Violation[],
+ * }} MachineReportPayload
+ */

package/scripts/llm-judge/diff-collection.mjs

@@ -0,0 +1,74 @@
+// @ts-check
+
+/**
+ * Git diff collection for the LLM judge. Selects the best available diff source
+ * based on environment signals (PR_DIFF override, GitHub Actions, GitLab CI),
+ * with a local HEAD~1..HEAD fallback that handles the initial-commit edge case
+ * by diffing against Git's empty-tree object.
+ */
+
+import { execSync } from 'node:child_process';
+
+import { REPOSITORY_ROOT } from './constants.mjs';
+
+export function detectCiProvider() {
+  if (process.env.GITHUB_ACTIONS === 'true') {
+    return 'github';
+  }
+
+  if (process.env.GITLAB_CI === 'true') {
+    return 'gitlab';
+  }
+
+  return 'local';
+}
+
+/**
+ * Collects the pull request diff from the best available source:
+ * 1. PR_DIFF env var (direct injection, highest priority)
+ * 2. GitHub Actions env vars (GITHUB_BASE_SHA / GITHUB_HEAD_SHA)
+ * 3. GitLab CI env vars (CI_MERGE_REQUEST_DIFF_BASE_SHA / CI_COMMIT_SHA)
+ * 4. Local fallback: HEAD~1..HEAD, then empty-tree if no parent exists
+ *
+ * @returns {string} The raw git diff output
+ */
+export function collectPullRequestDiff() {
+  if (process.env.PR_DIFF) {
+    console.log('  Source: PR_DIFF env variable');
+    return process.env.PR_DIFF;
+  }
+
+  const execOptions = {
+    cwd: REPOSITORY_ROOT,
+    encoding: /** @type {'utf-8'} */ ('utf-8'),
+    maxBuffer: 1024 * 1024 * 8,
+    stdio: /** @type {['ignore', 'pipe', 'ignore']} */ (['ignore', 'pipe', 'ignore']),
+  };
+
+  const githubBaseSha = process.env.GITHUB_BASE_SHA;
+  const githubHeadSha = process.env.GITHUB_HEAD_SHA ?? 'HEAD';
+  if (githubBaseSha) {
+    console.log(`  Source: GitHub Actions diff (${githubBaseSha.slice(0, 8)}...${githubHeadSha.slice(0, 8)})`);
+    return execSync(`git diff "${githubBaseSha}...${githubHeadSha}"`, execOptions);
+  }
+
+  const gitlabBaseSha = process.env.CI_MERGE_REQUEST_DIFF_BASE_SHA;
+  const gitlabHeadSha = process.env.CI_COMMIT_SHA ?? 'HEAD';
+  if (gitlabBaseSha) {
+    console.log(`  Source: GitLab CI diff (${gitlabBaseSha.slice(0, 8)}...${gitlabHeadSha.slice(0, 8)})`);
+    return execSync(`git diff "${gitlabBaseSha}...${gitlabHeadSha}"`, execOptions);
+  }
+
+  console.log('  Source: local HEAD~1..HEAD fallback');
+  try {
+    return execSync('git diff HEAD~1 HEAD', execOptions);
+  } catch {
+    try {
+      const emptyTreeSha = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';
+      return execSync(`git diff "${emptyTreeSha}" HEAD`, execOptions);
+    } catch {
+      console.warn('  ⚠️   Unable to execute git diff. Defaulting to empty diff.');
+      return '';
+    }
+  }
+}

package/scripts/llm-judge/prompting.mjs

@@ -0,0 +1,78 @@
+// @ts-check
+
+/**
+ * Prompt construction for the LLM judge. Keeps the system role and the
+ * user-message diff packaging in one place so the contract surface (severity
+ * scheme + JSON_VERDICT format) stays inspectable.
+ */
+
+import { MAX_DIFF_CHARS } from './constants.mjs';
+
+/**
+ * Returns the system-level instruction for the LLM judge role.
+ *
+ * @returns {string}
+ */
+export function buildSystemPrompt() {
+  return `You are a Senior Software Architect performing an automated code review for a CI/CD pipeline.
+
+Your job: evaluate a git diff against the provided PR checklist and identify violations.
+You must categorize each violation with a severity level: critical, high, medium, or low.
+
+## Severity classification:
+- critical: Security vulnerabilities (hardcoded secrets, SQL/command injection, missing auth checks, CORS), unvalidated external inputs.
+- high: N+1 database queries, swallowed errors (empty catch blocks without re-throw/recovery), layer boundary violations.
+- medium: TypeScript \`any\` type used without justification, missing test coverage, bad architectural patterns.
+- low: Style preferences, minor naming nitpicks, documentation nitpicks, performance micro-optimizations.
+
+## Mandatory output format:
+You MUST output your findings in EXACTLY this structure:
+
+\`\`\`
+## PR REVIEW RESULTS
+━━━━━━━━━━━━━━━━━━━
+
+✅ [Section Name] — Passes
+❌ [Section Name] — FAILS
+   📌 Rule: [rule file and section]
+   ❌ Problem: [exact description of the issue found in the diff]
+   ⚠️ Severity: [critical | high | medium | low]
+   ✅ Fix: [specific actionable fix]
+
+\`\`\`
+
+Rules:
+- Then at the absolute LAST line of your response, output a JSON array of the failed checks. Each object should have 'rule', 'problem', 'severity'. If there are no failures, output an empty array [].
+- Make sure the JSON array is perfectly valid JSON on a single line starting with \`JSON_VERDICT: \`. For example:
+JSON_VERDICT: [{"rule": "Security", "problem": "Hardcoded secret", "severity": "critical"}]
+- If the diff is empty, contains only documentation changes, or has no source code changes, output JSON_VERDICT: [] immediately.`;
+}
+
+/**
+ * Builds the user message combining the checklist and the (possibly truncated)
+ * diff. Truncation is annotated so the model knows the diff is partial.
+ *
+ * @param {string} prChecklistContent
+ * @param {string} diffContent
+ * @returns {string}
+ */
+export function buildUserMessage(prChecklistContent, diffContent) {
+  const truncatedDiff =
+    diffContent.length > MAX_DIFF_CHARS
+      ? `${diffContent.slice(0, MAX_DIFF_CHARS)}\n\n[DIFF TRUNCATED — ${(diffContent.length - MAX_DIFF_CHARS).toLocaleString()} additional characters omitted to stay within token limits]`
+      : diffContent;
+
+  return `## PR Checklist Reference
+
+${prChecklistContent}
+
+---
+
+## Git Diff to Review
+
+\`\`\`diff
+${truncatedDiff.trim() || '(empty diff — no source code changes detected)'}
+\`\`\`
+
+Review the diff against the checklist. Report your findings in the required format, ending with VERDICT: PASS ✅ or VERDICT: FAIL ❌.`;
+}

package/scripts/llm-judge/providers.mjs

@@ -0,0 +1,111 @@
+// @ts-check
+
+/**
+ * LLM provider invocations and selection logic. Each provider sticks to its
+ * native API contract; the selection helper picks the first one whose env key
+ * is set so callers do not need provider-specific glue code.
+ */
+
+/**
+ * @typedef {{ providerName: string, invokeProvider: (sys: string, usr: string) => Promise<string> }} SelectedProvider
+ */
+
+async function callOpenAiProvider(systemPrompt, userMessage) {
+  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'gpt-4o-mini';
+  const apiResponse = await fetch('https://api.openai.com/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+    },
+    body: JSON.stringify({
+      model: selectedModel,
+      max_tokens: 2048,
+      temperature: 0,
+      messages: [
+        { role: 'system', content: systemPrompt },
+        { role: 'user', content: userMessage },
+      ],
+    }),
+  });
+
+  if (!apiResponse.ok) {
+    const errorBody = await apiResponse.text();
+    throw new Error(`OpenAI API returned ${apiResponse.status}: ${errorBody}`);
+  }
+
+  /** @type {{ choices: Array<{ message: { content: string } }> }} */
+  const responsePayload = await apiResponse.json();
+  return responsePayload.choices[0].message.content;
+}
+
+async function callAnthropicProvider(systemPrompt, userMessage) {
+  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'claude-3-5-haiku-latest';
+  const apiResponse = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': process.env.ANTHROPIC_API_KEY ?? '',
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify({
+      model: selectedModel,
+      max_tokens: 2048,
+      system: systemPrompt,
+      messages: [{ role: 'user', content: userMessage }],
+    }),
+  });
+
+  if (!apiResponse.ok) {
+    const errorBody = await apiResponse.text();
+    throw new Error(`Anthropic API returned ${apiResponse.status}: ${errorBody}`);
+  }
+
+  /** @type {{ content: Array<{ text: string }> }} */
+  const responsePayload = await apiResponse.json();
+  return responsePayload.content[0].text;
+}
+
+async function callGeminiProvider(systemPrompt, userMessage) {
+  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'gemini-2.0-flash';
+  const apiKey = process.env.GEMINI_API_KEY ?? '';
+  const endpointUrl = `https://generativelanguage.googleapis.com/v1beta/models/${selectedModel}:generateContent?key=${apiKey}`;
+
+  const apiResponse = await fetch(endpointUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      system_instruction: { parts: [{ text: systemPrompt }] },
+      contents: [{ role: 'user', parts: [{ text: userMessage }] }],
+      generationConfig: { temperature: 0, maxOutputTokens: 2048 },
+    }),
+  });
+
+  if (!apiResponse.ok) {
+    const errorBody = await apiResponse.text();
+    throw new Error(`Gemini API returned ${apiResponse.status}: ${errorBody}`);
+  }
+
+  /** @type {{ candidates: Array<{ content: { parts: Array<{ text: string }> } }> }} */
+  const responsePayload = await apiResponse.json();
+  return responsePayload.candidates[0].content.parts[0].text;
+}
+
+/**
+ * Returns the first available LLM provider based on environment keys.
+ * Priority: OpenAI, then Anthropic, then Gemini.
+ *
+ * @returns {SelectedProvider | null}
+ */
+export function selectAvailableProvider() {
+  if (process.env.OPENAI_API_KEY) {
+    return { providerName: 'OpenAI (gpt-4o-mini)', invokeProvider: callOpenAiProvider };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { providerName: 'Anthropic (claude-3-5-haiku-latest)', invokeProvider: callAnthropicProvider };
+  }
+  if (process.env.GEMINI_API_KEY) {
+    return { providerName: 'Google Gemini (gemini-2.0-flash)', invokeProvider: callGeminiProvider };
+  }
+  return null;
+}

package/scripts/llm-judge/verdict.mjs

@@ -0,0 +1,134 @@
+// @ts-check
+
+/**
+ * Verdict parsing and machine-readable report assembly for the LLM judge. Keeps
+ * the pipeline output deterministic so downstream gates and CI dashboards can
+ * consume a stable schema.
+ */
+
+import { writeFileSync } from 'node:fs';
+
+import {
+  MACHINE_REPORT_PATH,
+  SEVERITY_NORMALIZATION_TABLE,
+  SHOULD_EMIT_MACHINE_REPORT,
+} from './constants.mjs';
+import { detectCiProvider } from './diff-collection.mjs';
+
+/**
+ * @typedef {import('./constants.mjs').MachineReportPayload} MachineReportPayload
+ * @typedef {import('./constants.mjs').Violation} Violation
+ */
+
+/**
+ * @param {string | undefined} rawSeverityValue
+ * @returns {string}
+ */
+export function normalizeSeverity(rawSeverityValue) {
+  const normalizedSeverityKey = String(rawSeverityValue || '').trim().toLowerCase();
+  return SEVERITY_NORMALIZATION_TABLE[normalizedSeverityKey] || 'low';
+}
+
+/**
+ * @param {MachineReportPayload} machineReportPayload
+ * @returns {string}
+ */
+export function formatMachineReadableLine(machineReportPayload) {
+  return `JSON_REPORT: ${JSON.stringify(machineReportPayload)}`;
+}
+
+/**
+ * @param {MachineReportPayload} machineReportPayload
+ */
+export function emitMachineReadableReport(machineReportPayload) {
+  if (!SHOULD_EMIT_MACHINE_REPORT) {
+    return;
+  }
+
+  writeFileSync(MACHINE_REPORT_PATH, `${JSON.stringify(machineReportPayload, null, 2)}\n`, 'utf-8');
+  console.log(formatMachineReadableLine(machineReportPayload));
+  console.log(`📎  Machine report saved: ${MACHINE_REPORT_PATH}`);
+}
+
+/**
+ * Extracts and parses the JSON verdict from the LLM response.
+ *
+ * @param {string} llmResponseText
+ * @param {boolean} failOnMalformedResponse
+ * @returns {Array<{ rule: string, problem: string, severity: string }>}
+ */
+export function extractVerdict(llmResponseText, failOnMalformedResponse) {
+  const match = llmResponseText.match(/JSON_VERDICT:\s*(\[.*\])/i);
+  if (!match) {
+    console.warn('⚠️  LLM response did not include a valid JSON_VERDICT line.');
+    if (failOnMalformedResponse) {
+      console.error('❌  Failing pipeline because malformed responses are not allowed by the profile.');
+      process.exit(1);
+    }
+    return [];
+  }
+  try {
+    return JSON.parse(match[1]);
+  } catch (err) {
+    const parseError = /** @type {Error} */ (err);
+    console.error('⚠️  Failed to parse JSON_VERDICT:', parseError.message);
+    if (failOnMalformedResponse) {
+      process.exit(1);
+    }
+    return [];
+  }
+}
+
+/**
+ * @param {Array<{ rule?: string, problem?: string, severity?: string }>} violations
+ * @returns {Violation[]}
+ */
+export function normalizeViolations(violations) {
+  return violations.map((violationItem) => ({
+    rule: String(violationItem.rule || 'Unknown Rule'),
+    problem: String(violationItem.problem || 'No problem description provided.'),
+    severity: normalizeSeverity(violationItem.severity),
+  }));
+}
+
+/**
+ * @param {{
+ *   provider: string,
+ *   selectedProfile: string,
+ *   blockingSeverities: string[],
+ *   finalViolations: Violation[],
+ *   blockingFound: Violation[],
+ *   isDryRun: boolean,
+ *   malformedVerdict: boolean,
+ *   providerError: boolean,
+ * }} payloadInput
+ * @returns {MachineReportPayload}
+ */
+export function buildMachineReportPayload({
+  provider,
+  selectedProfile,
+  blockingSeverities,
+  finalViolations,
+  blockingFound,
+  isDryRun,
+  malformedVerdict,
+  providerError,
+}) {
+  return {
+    generatedAt: new Date().toISOString(),
+    schemaVersion: '1.0',
+    profile: selectedProfile,
+    provider,
+    ciProvider: detectCiProvider(),
+    blockingSeverities,
+    failDecision: blockingFound.length > 0 || malformedVerdict || providerError,
+    malformedVerdict,
+    providerError,
+    dryRun: isDryRun,
+    summary: {
+      totalViolations: finalViolations.length,
+      blockingViolations: blockingFound.length,
+    },
+    violations: finalViolations,
+  };
+}