@ryuenn3123/agentic-senior-core 4.4.0 → 5.1.0

package/scripts/llm-judge/prompting.mjs

@@ -1,78 +0,0 @@
-// @ts-check
-
-/**
- * Prompt construction for the LLM judge. Keeps the system role and the
- * user-message diff packaging in one place so the contract surface (severity
- * scheme + JSON_VERDICT format) stays inspectable.
- */
-
-import { MAX_DIFF_CHARS } from './constants.mjs';
-
-/**
- * Returns the system-level instruction for the LLM judge role.
- *
- * @returns {string}
- */
-export function buildSystemPrompt() {
-  return `You are a Senior Software Architect performing an automated code review for a CI/CD pipeline.
-
-Your job: evaluate a git diff against the provided PR checklist and identify violations.
-You must categorize each violation with a severity level: critical, high, medium, or low.
-
-## Severity classification:
-- critical: Security vulnerabilities (hardcoded secrets, SQL/command injection, missing auth checks, CORS), unvalidated external inputs.
-- high: N+1 database queries, swallowed errors (empty catch blocks without re-throw/recovery), layer boundary violations.
-- medium: TypeScript \`any\` type used without justification, missing test coverage, bad architectural patterns.
-- low: Style preferences, minor naming nitpicks, documentation nitpicks, performance micro-optimizations.
-
-## Mandatory output format:
-You MUST output your findings in EXACTLY this structure:
-
-\`\`\`
-## PR REVIEW RESULTS
-━━━━━━━━━━━━━━━━━━━
-
-✅ [Section Name] — Passes
-❌ [Section Name] — FAILS
-   📌 Rule: [rule file and section]
-   ❌ Problem: [exact description of the issue found in the diff]
-   ⚠️ Severity: [critical | high | medium | low]
-   ✅ Fix: [specific actionable fix]
-
-\`\`\`
-
-Rules:
-- Then at the absolute LAST line of your response, output a JSON array of the failed checks. Each object should have 'rule', 'problem', 'severity'. If there are no failures, output an empty array [].
-- Make sure the JSON array is perfectly valid JSON on a single line starting with \`JSON_VERDICT: \`. For example:
-JSON_VERDICT: [{"rule": "Security", "problem": "Hardcoded secret", "severity": "critical"}]
-- If the diff is empty, contains only documentation changes, or has no source code changes, output JSON_VERDICT: [] immediately.`;
-}
-
-/**
- * Builds the user message combining the checklist and the (possibly truncated)
- * diff. Truncation is annotated so the model knows the diff is partial.
- *
- * @param {string} prChecklistContent
- * @param {string} diffContent
- * @returns {string}
- */
-export function buildUserMessage(prChecklistContent, diffContent) {
-  const truncatedDiff =
-    diffContent.length > MAX_DIFF_CHARS
-      ? `${diffContent.slice(0, MAX_DIFF_CHARS)}\n\n[DIFF TRUNCATED — ${(diffContent.length - MAX_DIFF_CHARS).toLocaleString()} additional characters omitted to stay within token limits]`
-      : diffContent;
-
-  return `## PR Checklist Reference
-
-${prChecklistContent}
-
----
-
-## Git Diff to Review
-
-\`\`\`diff
-${truncatedDiff.trim() || '(empty diff — no source code changes detected)'}
-\`\`\`
-
-Review the diff against the checklist. Report your findings in the required format, ending with VERDICT: PASS ✅ or VERDICT: FAIL ❌.`;
-}

package/scripts/llm-judge/providers.mjs

@@ -1,111 +0,0 @@
-// @ts-check
-
-/**
- * LLM provider invocations and selection logic. Each provider sticks to its
- * native API contract; the selection helper picks the first one whose env key
- * is set so callers do not need provider-specific glue code.
- */
-
-/**
- * @typedef {{ providerName: string, invokeProvider: (sys: string, usr: string) => Promise<string> }} SelectedProvider
- */
-
-async function callOpenAiProvider(systemPrompt, userMessage) {
-  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'gpt-4o-mini';
-  const apiResponse = await fetch('https://api.openai.com/v1/chat/completions', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
-    },
-    body: JSON.stringify({
-      model: selectedModel,
-      max_tokens: 2048,
-      temperature: 0,
-      messages: [
-        { role: 'system', content: systemPrompt },
-        { role: 'user', content: userMessage },
-      ],
-    }),
-  });
-
-  if (!apiResponse.ok) {
-    const errorBody = await apiResponse.text();
-    throw new Error(`OpenAI API returned ${apiResponse.status}: ${errorBody}`);
-  }
-
-  /** @type {{ choices: Array<{ message: { content: string } }> }} */
-  const responsePayload = await apiResponse.json();
-  return responsePayload.choices[0].message.content;
-}
-
-async function callAnthropicProvider(systemPrompt, userMessage) {
-  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'claude-3-5-haiku-latest';
-  const apiResponse = await fetch('https://api.anthropic.com/v1/messages', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'x-api-key': process.env.ANTHROPIC_API_KEY ?? '',
-      'anthropic-version': '2023-06-01',
-    },
-    body: JSON.stringify({
-      model: selectedModel,
-      max_tokens: 2048,
-      system: systemPrompt,
-      messages: [{ role: 'user', content: userMessage }],
-    }),
-  });
-
-  if (!apiResponse.ok) {
-    const errorBody = await apiResponse.text();
-    throw new Error(`Anthropic API returned ${apiResponse.status}: ${errorBody}`);
-  }
-
-  /** @type {{ content: Array<{ text: string }> }} */
-  const responsePayload = await apiResponse.json();
-  return responsePayload.content[0].text;
-}
-
-async function callGeminiProvider(systemPrompt, userMessage) {
-  const selectedModel = process.env.LLM_JUDGE_MODEL ?? 'gemini-2.0-flash';
-  const apiKey = process.env.GEMINI_API_KEY ?? '';
-  const endpointUrl = `https://generativelanguage.googleapis.com/v1beta/models/${selectedModel}:generateContent?key=${apiKey}`;
-
-  const apiResponse = await fetch(endpointUrl, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      system_instruction: { parts: [{ text: systemPrompt }] },
-      contents: [{ role: 'user', parts: [{ text: userMessage }] }],
-      generationConfig: { temperature: 0, maxOutputTokens: 2048 },
-    }),
-  });
-
-  if (!apiResponse.ok) {
-    const errorBody = await apiResponse.text();
-    throw new Error(`Gemini API returned ${apiResponse.status}: ${errorBody}`);
-  }
-
-  /** @type {{ candidates: Array<{ content: { parts: Array<{ text: string }> } }> }} */
-  const responsePayload = await apiResponse.json();
-  return responsePayload.candidates[0].content.parts[0].text;
-}
-
-/**
- * Returns the first available LLM provider based on environment keys.
- * Priority: OpenAI, then Anthropic, then Gemini.
- *
- * @returns {SelectedProvider | null}
- */
-export function selectAvailableProvider() {
-  if (process.env.OPENAI_API_KEY) {
-    return { providerName: 'OpenAI (gpt-4o-mini)', invokeProvider: callOpenAiProvider };
-  }
-  if (process.env.ANTHROPIC_API_KEY) {
-    return { providerName: 'Anthropic (claude-3-5-haiku-latest)', invokeProvider: callAnthropicProvider };
-  }
-  if (process.env.GEMINI_API_KEY) {
-    return { providerName: 'Google Gemini (gemini-2.0-flash)', invokeProvider: callGeminiProvider };
-  }
-  return null;
-}

package/scripts/llm-judge/verdict.mjs

@@ -1,134 +0,0 @@
-// @ts-check
-
-/**
- * Verdict parsing and machine-readable report assembly for the LLM judge. Keeps
- * the pipeline output deterministic so downstream gates and CI dashboards can
- * consume a stable schema.
- */
-
-import { writeFileSync } from 'node:fs';
-
-import {
-  MACHINE_REPORT_PATH,
-  SEVERITY_NORMALIZATION_TABLE,
-  SHOULD_EMIT_MACHINE_REPORT,
-} from './constants.mjs';
-import { detectCiProvider } from './diff-collection.mjs';
-
-/**
- * @typedef {import('./constants.mjs').MachineReportPayload} MachineReportPayload
- * @typedef {import('./constants.mjs').Violation} Violation
- */
-
-/**
- * @param {string | undefined} rawSeverityValue
- * @returns {string}
- */
-export function normalizeSeverity(rawSeverityValue) {
-  const normalizedSeverityKey = String(rawSeverityValue || '').trim().toLowerCase();
-  return SEVERITY_NORMALIZATION_TABLE[normalizedSeverityKey] || 'low';
-}
-
-/**
- * @param {MachineReportPayload} machineReportPayload
- * @returns {string}
- */
-export function formatMachineReadableLine(machineReportPayload) {
-  return `JSON_REPORT: ${JSON.stringify(machineReportPayload)}`;
-}
-
-/**
- * @param {MachineReportPayload} machineReportPayload
- */
-export function emitMachineReadableReport(machineReportPayload) {
-  if (!SHOULD_EMIT_MACHINE_REPORT) {
-    return;
-  }
-
-  writeFileSync(MACHINE_REPORT_PATH, `${JSON.stringify(machineReportPayload, null, 2)}\n`, 'utf-8');
-  console.log(formatMachineReadableLine(machineReportPayload));
-  console.log(`📎  Machine report saved: ${MACHINE_REPORT_PATH}`);
-}
-
-/**
- * Extracts and parses the JSON verdict from the LLM response.
- *
- * @param {string} llmResponseText
- * @param {boolean} failOnMalformedResponse
- * @returns {Array<{ rule: string, problem: string, severity: string }>}
- */
-export function extractVerdict(llmResponseText, failOnMalformedResponse) {
-  const match = llmResponseText.match(/JSON_VERDICT:\s*(\[.*\])/i);
-  if (!match) {
-    console.warn('⚠️  LLM response did not include a valid JSON_VERDICT line.');
-    if (failOnMalformedResponse) {
-      console.error('❌  Failing pipeline because malformed responses are not allowed by the profile.');
-      process.exit(1);
-    }
-    return [];
-  }
-  try {
-    return JSON.parse(match[1]);
-  } catch (err) {
-    const parseError = /** @type {Error} */ (err);
-    console.error('⚠️  Failed to parse JSON_VERDICT:', parseError.message);
-    if (failOnMalformedResponse) {
-      process.exit(1);
-    }
-    return [];
-  }
-}
-
-/**
- * @param {Array<{ rule?: string, problem?: string, severity?: string }>} violations
- * @returns {Violation[]}
- */
-export function normalizeViolations(violations) {
-  return violations.map((violationItem) => ({
-    rule: String(violationItem.rule || 'Unknown Rule'),
-    problem: String(violationItem.problem || 'No problem description provided.'),
-    severity: normalizeSeverity(violationItem.severity),
-  }));
-}
-
-/**
- * @param {{
- *   provider: string,
- *   selectedProfile: string,
- *   blockingSeverities: string[],
- *   finalViolations: Violation[],
- *   blockingFound: Violation[],
- *   isDryRun: boolean,
- *   malformedVerdict: boolean,
- *   providerError: boolean,
- * }} payloadInput
- * @returns {MachineReportPayload}
- */
-export function buildMachineReportPayload({
-  provider,
-  selectedProfile,
-  blockingSeverities,
-  finalViolations,
-  blockingFound,
-  isDryRun,
-  malformedVerdict,
-  providerError,
-}) {
-  return {
-    generatedAt: new Date().toISOString(),
-    schemaVersion: '1.0',
-    profile: selectedProfile,
-    provider,
-    ciProvider: detectCiProvider(),
-    blockingSeverities,
-    failDecision: blockingFound.length > 0 || malformedVerdict || providerError,
-    malformedVerdict,
-    providerError,
-    dryRun: isDryRun,
-    summary: {
-      totalViolations: finalViolations.length,
-      blockingViolations: blockingFound.length,
-    },
-    violations: finalViolations,
-  };
-}

package/scripts/llm-judge.mjs

@@ -1,200 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-
-/**
- * scripts/llm-judge.mjs
- *
- * LLM-as-a-Judge CI gate. Enforces pr-checklist.md on every pull request.
- *
- * Reads the git diff of the current PR, loads the PR checklist, sends both
- * to the first available LLM provider, and exits 1 when CRITICAL findings
- * exist (security gaps, N+1 queries, swallowed errors, hardcoded secrets,
- * layer boundary violations, SQL injection risks).
- *
- * Implementation is split per concern under scripts/llm-judge/. This entry
- * file owns the CLI orchestration only.
- *
- * Supported providers (auto-selected by first available env key):
- *   OPENAI_API_KEY    -> gpt-4o-mini  (override with LLM_JUDGE_MODEL)
- *   ANTHROPIC_API_KEY -> claude-3-5-haiku-latest
- *   GEMINI_API_KEY    -> gemini-2.0-flash
- *
- * Usage:
- *   node scripts/llm-judge.mjs            (auto-detect diff, call LLM)
- *   node scripts/llm-judge.mjs --dry-run  (print prompt, skip LLM call)
- *
- * Environment variables:
- *   OPENAI_API_KEY         OpenAI secret key
- *   ANTHROPIC_API_KEY      Anthropic secret key
- *   GEMINI_API_KEY         Google Gemini API key
- *   LLM_JUDGE_MODEL        Override model name for the selected provider
- *   LLM_MAX_DIFF_CHARS     Max characters of diff to send (default: 12000)
- *   PR_DIFF                Inject diff directly (bypasses git commands)
- *   GITHUB_BASE_SHA        Base commit SHA for GitHub Actions PR diffs
- *   GITHUB_HEAD_SHA        Head commit SHA for GitHub Actions PR diffs
- *   CI_MERGE_REQUEST_DIFF_BASE_SHA  Base SHA for GitLab MR diffs
- *   CI_COMMIT_SHA          Head SHA for GitLab MR diffs
- *
- * Zero external dependencies — uses Node.js built-ins only (Node 18+).
- */
-
-import { IS_DRY_RUN, MAX_DIFF_CHARS } from './llm-judge/constants.mjs';
-import { collectPullRequestDiff } from './llm-judge/diff-collection.mjs';
-import { loadPrChecklist, loadThresholds } from './llm-judge/checklist-loader.mjs';
-import { buildSystemPrompt, buildUserMessage } from './llm-judge/prompting.mjs';
-import { selectAvailableProvider } from './llm-judge/providers.mjs';
-import {
-  buildMachineReportPayload,
-  emitMachineReadableReport,
-  extractVerdict,
-  normalizeViolations,
-} from './llm-judge/verdict.mjs';
-
-async function main() {
-  console.log('');
-  console.log('🔍  LLM Judge — Automated Code Review Gate');
-  console.log('════════════════════════════════════════════');
-  console.log('');
-
-  // ── Step 1: Load checklist and thresholds ──────────────
-  const prChecklistContent = loadPrChecklist();
-  const thresholdsObj = loadThresholds();
-  const selectedProfile = thresholdsObj.selectedProfile || 'balanced';
-  const profileConfig = thresholdsObj.profileThresholds[selectedProfile] || {};
-  const blockingSeverities = profileConfig.blockingSeverities || ['critical', 'high'];
-  const failOnMalformedResponse = profileConfig.failOnMalformedResponse !== false;
-  const failOnProviderError = profileConfig.failOnProviderError || false;
-
-  console.log(`✅  PR checklist loaded (${prChecklistContent.length} chars)`);
-  console.log(`✅  Threshold profile loaded: ${selectedProfile} (blocking: ${blockingSeverities.join(', ')})`);
-
-  // ── Step 2: Collect diff ────────────────────────────────
-  const rawDiff = collectPullRequestDiff();
-  console.log(`✅  Git diff collected (${rawDiff.length} chars${rawDiff.length > MAX_DIFF_CHARS ? ` — will truncate to ${MAX_DIFF_CHARS}` : ''})`);
-
-  // ── Step 3: Build prompt ────────────────────────────────
-  const systemPrompt = buildSystemPrompt();
-  const userMessage = buildUserMessage(prChecklistContent, rawDiff);
-
-  // ── Step 4: Dry run mode ────────────────────────────────
-  if (IS_DRY_RUN) {
-    console.log('');
-    console.log('── DRY RUN MODE ──────────────────────────────────────────');
-    console.log('[SYSTEM PROMPT PREVIEW]');
-    console.log(systemPrompt.slice(0, 400) + '...');
-    console.log('');
-    console.log('[USER MESSAGE PREVIEW]');
-    console.log(userMessage.slice(0, 400) + '...');
-    console.log('─────────────────────────────────────────────────────────');
-    console.log('');
-    const dryRunReportPayload = buildMachineReportPayload({
-      provider: 'dry-run',
-      selectedProfile,
-      blockingSeverities,
-      finalViolations: [],
-      blockingFound: [],
-      isDryRun: true,
-      malformedVerdict: false,
-      providerError: false,
-    });
-    emitMachineReadableReport(dryRunReportPayload);
-    console.log('VERDICT: JSON_VERDICT: []  (dry run — no LLM call made)');
-    process.exit(0);
-  }
-
-  // ── Step 5: Select provider ─────────────────────────────
-  const selectedProvider = selectAvailableProvider();
-  if (!selectedProvider) {
-    console.warn('');
-    console.warn('⚠️   No LLM API key detected.');
-    console.warn('    Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GEMINI_API_KEY');
-    console.warn('    to enable automated code review.');
-    console.warn('');
-    console.warn('⏭️   Skipping LLM review — pipeline continues (PASS).');
-    const skippedReportPayload = buildMachineReportPayload({
-      provider: 'none',
-      selectedProfile,
-      blockingSeverities,
-      finalViolations: [],
-      blockingFound: [],
-      isDryRun: false,
-      malformedVerdict: false,
-      providerError: false,
-    });
-    emitMachineReadableReport(skippedReportPayload);
-    process.exit(0);
-  }
-
-  console.log(`✅  Provider selected: ${selectedProvider.providerName}`);
-  if (process.env.LLM_JUDGE_MODEL) {
-    console.log(`    Model override: ${process.env.LLM_JUDGE_MODEL}`);
-  }
-  console.log('');
-  console.log('⏳  Sending diff to LLM for review...');
-  console.log('');
-
-  // ── Step 6: Call LLM ────────────────────────────────────
-  let llmReviewText;
-  try {
-    llmReviewText = await selectedProvider.invokeProvider(systemPrompt, userMessage);
-  } catch (providerCallError) {
-    console.warn(`⚠️   LLM call failed: ${/** @type {Error} */ (providerCallError).message}`);
-    const providerErrorReportPayload = buildMachineReportPayload({
-      provider: selectedProvider.providerName,
-      selectedProfile,
-      blockingSeverities,
-      finalViolations: [],
-      blockingFound: [],
-      isDryRun: false,
-      malformedVerdict: false,
-      providerError: Boolean(failOnProviderError),
-    });
-    emitMachineReadableReport(providerErrorReportPayload);
-    if (failOnProviderError) {
-      console.error('❌  Failing pipeline because provider errors are not allowed by the profile.');
-      process.exit(1);
-    }
-    console.warn('    Skipping LLM review — pipeline continues (PASS).');
-    process.exit(0);
-  }
-
-  // ── Step 7: Print report ────────────────────────────────
-  console.log('── LLM Review Report ─────────────────────────────────────');
-  console.log('');
-  console.log(llmReviewText);
-  console.log('');
-  console.log('──────────────────────────────────────────────────────────');
-  console.log('');
-
-  // ── Step 8: Enforce verdict ─────────────────────────────
-  const rawVerdictViolations = extractVerdict(llmReviewText, failOnMalformedResponse);
-  const finalViolations = normalizeViolations(rawVerdictViolations);
-  const hasMalformedVerdict = !/JSON_VERDICT:\s*\[/i.test(llmReviewText);
-
-  const blockingFound = finalViolations.filter((v) => blockingSeverities.includes(v.severity.toLowerCase()));
-  const machineReportPayload = buildMachineReportPayload({
-    provider: selectedProvider.providerName,
-    selectedProfile,
-    blockingSeverities,
-    finalViolations,
-    blockingFound,
-    isDryRun: false,
-    malformedVerdict: hasMalformedVerdict,
-    providerError: false,
-  });
-  emitMachineReadableReport(machineReportPayload);
-
-  if (blockingFound.length > 0) {
-    console.error(`❌  LLM Judge: ${blockingFound.length} blocking violations found (severities: ${blockingSeverities.join(', ')}). Pipeline FAILED.`);
-    console.error('    Fix the issues listed above before merging.');
-    process.exit(1);
-  }
-
-  console.log('✅  LLM Judge: No blocking violations. Pipeline PASSED.');
-  process.exit(0);
-}
-
-main().catch((unexpectedError) => {
-  console.error('❌  Unexpected error in llm-judge:', unexpectedError);
-  process.exit(1);
-});