create-qa-architect 5.13.5 → 5.14.0

package/lib/commands/ci-doctor.js

@@ -0,0 +1,341 @@
+/**
+ * CI Doctor — workflow waste + flaky-test detection.
+ *
+ * Extends --analyze-ci with deeper diagnostic checks:
+ *   - Duplicated jobs (same runs-on + steps signature)
+ *   - Workflows triggered on every push without `paths:` filters
+ *   - Oversized matrix (>10 cells)
+ *   - Unnecessary scheduled runs (more frequent than weekly)
+ *   - Flaky tests (parsed from `gh run list` if gh CLI is authenticated)
+ *
+ * Gated behind hasFeature('ciDoctor'). Invoked via `--analyze-ci --doctor`.
+ *
+ * All process invocations use spawnSync with argv arrays (no shell).
+ */
+
+const crypto = require('crypto')
+const { spawnSync } = require('child_process')
+
+const SEVERITY = {
+  HIGH: 'high',
+  MEDIUM: 'medium',
+  LOW: 'low',
+}
+
+const SEVERITY_ICON = {
+  high: '🔴',
+  medium: '🟡',
+  low: '🟢',
+}
+
+const MATRIX_CELL_THRESHOLD = 10
+const FLAKY_THRESHOLD_PCT = 90
+const FLAKY_MIN_RUNS = 5
+
+function fingerprintSteps(steps) {
+  if (!Array.isArray(steps)) return ''
+  const sig = steps
+    .map(step => {
+      if (step.uses) return `uses:${step.uses}`
+      if (step.run) return `run:${step.run.trim().slice(0, 200)}`
+      return 'unknown'
+    })
+    .join('|')
+  return crypto.createHash('sha256').update(sig).digest('hex').slice(0, 12)
+}
+
+function detectDuplicatedJobs(workflows) {
+  const findings = []
+  const seen = new Map()
+
+  for (const wf of workflows) {
+    const jobs = (wf.parsed && wf.parsed.jobs) || {}
+    for (const [jobName, job] of Object.entries(jobs)) {
+      if (!job || !Array.isArray(job.steps)) continue
+      const fp = `${job['runs-on'] || ''}|${fingerprintSteps(job.steps)}`
+      if (seen.has(fp)) {
+        const prev = seen.get(fp)
+        findings.push({
+          id: 'duplicated-job',
+          severity: SEVERITY.MEDIUM,
+          title: 'Duplicated job',
+          location: `${wf.name}#${jobName}`,
+          description: `Job "${jobName}" has the same runs-on + steps signature as "${prev.job}" in ${prev.workflow}.`,
+          fix: 'Extract into a reusable workflow (`workflow_call`) or composite action.',
+        })
+      } else {
+        seen.set(fp, { workflow: wf.name, job: jobName })
+      }
+    }
+  }
+  return findings
+}
+
+function workflowTriggersOnPush(parsed) {
+  if (!parsed || !parsed.on) return false
+  if (parsed.on === 'push') return true
+  if (typeof parsed.on === 'object' && parsed.on !== null) {
+    return Object.prototype.hasOwnProperty.call(parsed.on, 'push')
+  }
+  if (Array.isArray(parsed.on)) return parsed.on.includes('push')
+  return false
+}
+
+function pushHasPathFilter(parsed) {
+  if (!parsed || !parsed.on || typeof parsed.on !== 'object') return false
+  const push = parsed.on.push
+  if (!push || typeof push !== 'object') return false
+  return Boolean(push.paths || push['paths-ignore'])
+}
+
+function detectMissingPathFilters(workflows) {
+  const findings = []
+  for (const wf of workflows) {
+    const parsed = wf.parsed
+    if (!workflowTriggersOnPush(parsed)) continue
+    if (pushHasPathFilter(parsed)) continue
+    findings.push({
+      id: 'missing-path-filter',
+      severity: SEVERITY.MEDIUM,
+      title: 'Missing path filter',
+      location: wf.name,
+      description:
+        'Workflow runs on every push without `paths:` or `paths-ignore:` filters.',
+      fix: 'Add `paths:` filter so docs/test-only changes do not trigger this workflow.',
+    })
+  }
+  return findings
+}
+
+function matrixCellCount(matrix) {
+  if (!matrix || typeof matrix !== 'object') return 0
+  // Use the existing logic from analyze-ci if available; fall back to local.
+  let product = 1
+  let hasAxis = false
+  for (const [key, value] of Object.entries(matrix)) {
+    if (key === 'include' || key === 'exclude') continue
+    if (Array.isArray(value)) {
+      hasAxis = true
+      product *= value.length || 1
+    }
+  }
+  if (!hasAxis) return 0
+  if (Array.isArray(matrix.include)) product += matrix.include.length
+  if (Array.isArray(matrix.exclude)) product -= matrix.exclude.length
+  return Math.max(product, 0)
+}
+
+function detectExpensiveMatrix(workflows) {
+  const findings = []
+  for (const wf of workflows) {
+    const jobs = (wf.parsed && wf.parsed.jobs) || {}
+    for (const [jobName, job] of Object.entries(jobs)) {
+      if (!job || !job.strategy || !job.strategy.matrix) continue
+      const cells = matrixCellCount(job.strategy.matrix)
+      if (cells > MATRIX_CELL_THRESHOLD) {
+        findings.push({
+          id: 'expensive-matrix',
+          severity: SEVERITY.HIGH,
+          title: 'Oversized matrix',
+          location: `${wf.name}#${jobName}`,
+          description: `Matrix has ${cells} combinations (threshold: ${MATRIX_CELL_THRESHOLD}).`,
+          fix: 'Prune axes or use `include`/`exclude` to keep only the combinations that matter.',
+        })
+      }
+    }
+  }
+  return findings
+}
+
+function parseCronFrequency(cron) {
+  // Returns approximate runs per week for a cron expression.
+  // We only need to flag "more frequent than weekly".
+  // Cron: minute hour dom month dow
+  if (typeof cron !== 'string') return 0
+  const parts = cron.trim().split(/\s+/)
+  if (parts.length < 5) return 0
+  const [minute, hour, dom, , dow] = parts
+
+  // If every minute or every hour, very frequent.
+  if (minute === '*' || /\*\//.test(minute)) return 7 * 24 * 60
+  if (hour === '*' || /\*\//.test(hour)) return 7 * 24
+  // Daily if dom and dow are unrestricted.
+  if (dom === '*' && dow === '*') return 7
+  // Weekly if dow is a specific day.
+  if (dow !== '*') return 1
+  return 1
+}
+
+function detectUnnecessarySchedules(workflows) {
+  const findings = []
+  for (const wf of workflows) {
+    const on = wf.parsed && wf.parsed.on
+    if (!on || typeof on !== 'object') continue
+    const schedule = on.schedule
+    if (!Array.isArray(schedule)) continue
+    for (const entry of schedule) {
+      if (!entry || !entry.cron) continue
+      const runsPerWeek = parseCronFrequency(entry.cron)
+      if (runsPerWeek > 1) {
+        findings.push({
+          id: 'frequent-schedule',
+          severity: SEVERITY.LOW,
+          title: 'Frequent scheduled run',
+          location: `${wf.name} (cron: ${entry.cron})`,
+          description: `Cron runs ~${runsPerWeek}× per week. Most maintenance jobs only need weekly.`,
+          fix: 'Reduce frequency unless this is load-bearing (e.g. dependency updates can be weekly).',
+        })
+      }
+    }
+  }
+  return findings
+}
+
+function ghCliAvailable() {
+  const r = spawnSync('gh', ['--version'], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    shell: false,
+    timeout: 5_000,
+  })
+  return r.status === 0
+}
+
+function ghAuthenticated() {
+  const r = spawnSync('gh', ['auth', 'status'], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    shell: false,
+    timeout: 5_000,
+  })
+  return r.status === 0
+}
+
+function fetchRecentRuns(projectPath, limit) {
+  const r = spawnSync(
+    'gh',
+    [
+      'run',
+      'list',
+      '--limit',
+      String(limit),
+      '--json',
+      'workflowName,conclusion,name,status',
+    ],
+    {
+      cwd: projectPath,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      shell: false,
+      timeout: 15_000,
+    }
+  )
+  if (r.status !== 0) return null
+  try {
+    return JSON.parse(r.stdout || '[]')
+  } catch {
+    return null
+  }
+}
+
+function tallyRunsByWorkflow(runs) {
+  const byWorkflow = new Map()
+  for (const run of runs) {
+    const key = run.workflowName || run.name
+    if (!key || run.status !== 'completed') continue
+    if (!byWorkflow.has(key)) {
+      byWorkflow.set(key, { total: 0, success: 0 })
+    }
+    const bucket = byWorkflow.get(key)
+    bucket.total++
+    if (run.conclusion === 'success') bucket.success++
+  }
+  return byWorkflow
+}
+
+function flakyFinding(workflowName, total, success) {
+  const successPct = Math.round((success / total) * 100)
+  if (successPct >= FLAKY_THRESHOLD_PCT) return null
+  return {
+    id: 'flaky-workflow',
+    severity: SEVERITY.HIGH,
+    title: 'Flaky workflow',
+    location: workflowName,
+    description: `${success}/${total} recent runs succeeded (${successPct}%, threshold ${FLAKY_THRESHOLD_PCT}%).`,
+    fix: 'Identify the flaky job/test — retry-with-backoff is a smell. Fix root cause (timing, external service, shared state).',
+  }
+}
+
+function detectFlakyWorkflows(projectPath, options) {
+  if (options && options.skipFlakyCheck) return []
+  if (!ghCliAvailable() || !ghAuthenticated()) return []
+
+  const runs = fetchRecentRuns(projectPath, 50)
+  if (!runs || runs.length === 0) return []
+
+  const byWorkflow = tallyRunsByWorkflow(runs)
+  const findings = []
+  for (const [workflowName, { total, success }] of byWorkflow.entries()) {
+    if (total < FLAKY_MIN_RUNS) continue
+    const finding = flakyFinding(workflowName, total, success)
+    if (finding) findings.push(finding)
+  }
+  return findings
+}
+
+function runDoctorChecks(workflows, projectPath, options = {}) {
+  const findings = []
+  findings.push(...detectDuplicatedJobs(workflows))
+  findings.push(...detectMissingPathFilters(workflows))
+  findings.push(...detectExpensiveMatrix(workflows))
+  findings.push(...detectUnnecessarySchedules(workflows))
+  if (projectPath) {
+    findings.push(...detectFlakyWorkflows(projectPath, options))
+  }
+  // Order: HIGH first, then MEDIUM, then LOW.
+  const order = { high: 0, medium: 1, low: 2 }
+  findings.sort((a, b) => order[a.severity] - order[b.severity])
+  return findings
+}
+
+function buildDoctorReport(findings) {
+  const lines = []
+  lines.push('')
+  lines.push('🩺 CI Doctor')
+  lines.push('─'.repeat(60))
+  if (findings.length === 0) {
+    lines.push('✅ No CI health issues detected.')
+    lines.push('')
+    return lines.join('\n')
+  }
+
+  const counts = { high: 0, medium: 0, low: 0 }
+  for (const f of findings) counts[f.severity]++
+  lines.push(
+    `Findings: ${counts.high} high · ${counts.medium} medium · ${counts.low} low`
+  )
+  lines.push('─'.repeat(60))
+
+  for (const f of findings) {
+    const icon = SEVERITY_ICON[f.severity] || ' '
+    lines.push(`${icon} ${f.title} — ${f.location}`)
+    lines.push(`   ${f.description}`)
+    lines.push(`   Fix: ${f.fix}`)
+    lines.push('')
+  }
+  return lines.join('\n')
+}
+
+module.exports = {
+  runDoctorChecks,
+  buildDoctorReport,
+  detectDuplicatedJobs,
+  detectMissingPathFilters,
+  detectExpensiveMatrix,
+  detectUnnecessarySchedules,
+  detectFlakyWorkflows,
+  matrixCellCount,
+  parseCronFrequency,
+  fingerprintSteps,
+  SEVERITY,
+}

package/lib/commands/history-scan.js

@@ -0,0 +1,342 @@
+/**
+ * Historical Secrets Scan — full git-history audit via gitleaks.
+ *
+ * Runs gitleaks against the project's git history (default: all commits,
+ * with a configurable --depth limit), parses JSON output, deduplicates,
+ * and reports leaked secrets with commit SHAs and file paths.
+ *
+ * Gated behind Pro tier (hasFeature('historicalSecretsScan'), with
+ * fallback to hasFeature('securityScanning') for backwards compatibility).
+ *
+ * All process invocations use spawnSync with argv arrays (no shell).
+ */
+
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const { spawnSync } = require('child_process')
+const {
+  hasFeature,
+  showUpgradeMessage,
+  ensureLicenseFresh,
+} = require('../licensing')
+const { ConfigSecurityScanner } = require('../validation/config-security')
+
+const SAFE_DEPTH_MAX = 10_000
+
+function parseDepth(input) {
+  if (!input) return null
+  const n = parseInt(input, 10)
+  if (!Number.isFinite(n) || n <= 0) return null
+  return Math.min(n, SAFE_DEPTH_MAX)
+}
+
+function inGitRepo(projectPath) {
+  const r = spawnSync(
+    'git',
+    ['-C', projectPath, 'rev-parse', '--is-inside-work-tree'],
+    {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      shell: false,
+      timeout: 10_000,
+    }
+  )
+  return r.status === 0 && (r.stdout || '').trim() === 'true'
+}
+
+function countCommits(projectPath) {
+  const r = spawnSync(
+    'git',
+    ['-C', projectPath, 'rev-list', '--count', 'HEAD'],
+    {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      shell: false,
+      timeout: 10_000,
+    }
+  )
+  if (r.status !== 0) return null
+  const n = parseInt((r.stdout || '').trim(), 10)
+  return Number.isFinite(n) ? n : null
+}
+
+function buildGitleaksArgs(reportPath, depth) {
+  // Note: argv array, no shell. `--log-opts` value is a single argv element.
+  // We always pass --log-opts so gitleaks scans history (not just the
+  // working tree). Use `--max-count=N` for depth limits — `HEAD~N..HEAD`
+  // produces a git fatal error on shallow repos with fewer than N commits.
+  const args = [
+    'detect',
+    '--no-banner',
+    '--redact',
+    '--report-format',
+    'json',
+    '--report-path',
+    reportPath,
+  ]
+
+  if (depth) {
+    args.push(`--log-opts=--max-count=${depth}`)
+  } else {
+    args.push('--log-opts=--all')
+  }
+  return args
+}
+
+function dedupeFindings(findings) {
+  const seen = new Set()
+  const out = []
+  for (const f of findings) {
+    const key = [f.Commit || f.commit, f.File || f.file, f.Secret || f.secret]
+      .map(v => String(v || ''))
+      .join('|')
+    if (seen.has(key)) continue
+    seen.add(key)
+    out.push(f)
+  }
+  return out
+}
+
+function normalizeFinding(f) {
+  return {
+    commit: f.Commit || f.commit || '',
+    file: f.File || f.file || '',
+    line: f.StartLine || f.startLine || null,
+    secretType: f.RuleID || f.ruleId || f.Description || f.description || '',
+    author: f.Author || f.author || '',
+    date: f.Date || f.date || '',
+  }
+}
+
+function countBySecretType(normalized) {
+  const counts = {}
+  for (const f of normalized) {
+    const key = f.secretType || 'unknown'
+    counts[key] = (counts[key] || 0) + 1
+  }
+  return counts
+}
+
+function oldestExposures(normalized, limit) {
+  // Sort by date ascending (oldest first); fall back to commit string compare.
+  const sorted = [...normalized].sort((a, b) => {
+    if (a.date && b.date) return a.date.localeCompare(b.date)
+    return String(a.commit).localeCompare(String(b.commit))
+  })
+  return sorted.slice(0, limit)
+}
+
+function buildHumanReport(report) {
+  const lines = []
+  lines.push('')
+  lines.push('🔐 Historical Secrets Scan')
+  lines.push('─'.repeat(60))
+  lines.push(`Scope: ${report.scope}`)
+  lines.push(`Commits scanned: ${report.commitsScanned ?? 'unknown'}`)
+  lines.push(`Findings: ${report.findings.length}`)
+  lines.push('─'.repeat(60))
+
+  if (report.findings.length === 0) {
+    lines.push('✅ No secrets detected in git history.')
+    lines.push('')
+    return lines.join('\n')
+  }
+
+  const counts = countBySecretType(report.findings)
+  lines.push('By secret type:')
+  for (const [type, count] of Object.entries(counts)) {
+    lines.push(`  • ${type}: ${count}`)
+  }
+  lines.push('')
+
+  const oldest = oldestExposures(report.findings, 10)
+  lines.push('Top 10 oldest exposures:')
+  for (const f of oldest) {
+    const date = f.date ? f.date.slice(0, 10) : 'unknown'
+    const sha = (f.commit || '').slice(0, 8)
+    lines.push(`  • ${date}  ${sha}  ${f.secretType}  ${f.file}`)
+  }
+  lines.push('')
+  lines.push('❌ Secrets found in history. Rotate exposed credentials and')
+  lines.push('   consider rewriting history with git-filter-repo or BFG.')
+  lines.push('')
+  return lines.join('\n')
+}
+
+function buildMarkdown(report) {
+  const lines = []
+  lines.push('# Historical Secrets Scan')
+  lines.push('')
+  lines.push(`- **Scope:** ${report.scope}`)
+  lines.push(`- **Commits scanned:** ${report.commitsScanned ?? 'unknown'}`)
+  lines.push(`- **Findings:** ${report.findings.length}`)
+  lines.push('')
+
+  if (report.findings.length === 0) {
+    lines.push('✅ No secrets detected in git history.')
+    lines.push('')
+    return `${lines.join('\n')}\n`
+  }
+
+  const counts = countBySecretType(report.findings)
+  lines.push('## By secret type')
+  lines.push('')
+  for (const [type, count] of Object.entries(counts)) {
+    lines.push(`- \`${type}\`: ${count}`)
+  }
+  lines.push('')
+
+  const oldest = oldestExposures(report.findings, 10)
+  lines.push('## Top 10 oldest exposures')
+  lines.push('')
+  lines.push('| Date | Commit | Secret type | File |')
+  lines.push('| --- | --- | --- | --- |')
+  for (const f of oldest) {
+    const date = f.date ? f.date.slice(0, 10) : 'unknown'
+    const sha = (f.commit || '').slice(0, 8)
+    lines.push(`| ${date} | \`${sha}\` | ${f.secretType} | \`${f.file}\` |`)
+  }
+  lines.push('')
+  lines.push('### Next steps')
+  lines.push('1. **Rotate every exposed credential immediately.**')
+  lines.push('2. Confirm logs/services for unauthorized use of leaked keys.')
+  lines.push(
+    '3. Rewrite history with `git-filter-repo` or BFG if you need to purge.'
+  )
+  lines.push('4. Add a pre-commit `gitleaks` hook to prevent recurrence.')
+  lines.push('')
+  return `${lines.join('\n')}\n`
+}
+
+/**
+ * Validate a completed gitleaks invocation. Returns an error string when
+ * the run is untrustworthy (so we never silently report "clean"), or null
+ * when the result can be parsed.
+ */
+function validateGitleaksRun(result, tmpReport) {
+  const stderr = (result.stderr || '').trim()
+  if (result.status !== 0 && result.status !== 1) {
+    return `gitleaks failed (exit ${result.status}): ${stderr.slice(0, 500)}`
+  }
+  if (/^fatal:/m.test(stderr)) {
+    return `git rev-walk failed during scan: ${stderr.slice(0, 500)}`
+  }
+  if (!fs.existsSync(tmpReport) && result.status === 1) {
+    return `gitleaks reported leaks but produced no report file (stderr: ${stderr.slice(0, 300)})`
+  }
+  return null
+}
+
+/**
+ * Read and parse the gitleaks JSON report, cleaning up the temp file.
+ * Returns { findings } on success or { error } on parse failure.
+ */
+function readGitleaksReport(tmpReport) {
+  if (!fs.existsSync(tmpReport)) return { findings: [] }
+  try {
+    const content = fs.readFileSync(tmpReport, 'utf8').trim()
+    if (!content) return { findings: [] }
+    const parsed = JSON.parse(content)
+    return { findings: Array.isArray(parsed) ? parsed : [] }
+  } catch (err) {
+    return { error: `Failed to parse gitleaks report: ${err.message}` }
+  } finally {
+    try {
+      fs.unlinkSync(tmpReport)
+    } catch {
+      // cleanup is best-effort
+    }
+  }
+}
+
+async function runHistoryScan(projectPath, options = {}) {
+  if (!inGitRepo(projectPath)) {
+    return { error: 'Not a git repository' }
+  }
+
+  const depth = parseDepth(options.depth)
+  const scope = depth
+    ? `last ${depth} commit(s) of HEAD`
+    : 'full git history (--all)'
+
+  const scanner = new ConfigSecurityScanner({ quiet: true })
+  let binary
+  try {
+    binary = await scanner.resolveGitleaksBinary()
+  } catch (err) {
+    return { error: `Failed to resolve gitleaks binary: ${err.message}` }
+  }
+
+  const tmpReport = path.join(
+    os.tmpdir(),
+    `qaa-history-scan-${Date.now()}.json`
+  )
+  const args = buildGitleaksArgs(tmpReport, depth)
+
+  const result = spawnSync(binary, args, {
+    cwd: projectPath,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    shell: false,
+    timeout: options.timeoutMs || 10 * 60 * 1000,
+  })
+
+  const runError = validateGitleaksRun(result, tmpReport)
+  if (runError) return { error: runError }
+
+  const parseResult = readGitleaksReport(tmpReport)
+  if (parseResult.error) return { error: parseResult.error }
+
+  const normalized = dedupeFindings(parseResult.findings).map(normalizeFinding)
+  return {
+    scope,
+    commitsScanned: countCommits(projectPath),
+    findings: normalized,
+    generatedAt: new Date().toISOString(),
+  }
+}
+
+async function handleHistoryScan(options = {}) {
+  await ensureLicenseFresh()
+  if (!hasFeature('historicalSecretsScan') && !hasFeature('securityScanning')) {
+    showUpgradeMessage('Historical secrets scan')
+    process.exit(1)
+  }
+
+  const projectPath = options.projectPath || process.cwd()
+  const report = await runHistoryScan(projectPath, options)
+
+  if (report.error) {
+    process.stderr.write(`❌ ${report.error}\n`)
+    process.exit(1)
+  }
+
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`)
+  } else {
+    process.stdout.write(buildHumanReport(report))
+  }
+
+  if (options.outPath) {
+    fs.writeFileSync(options.outPath, buildMarkdown(report), 'utf8')
+    if (!options.json) {
+      process.stdout.write(
+        `\n📄 Markdown report written to ${options.outPath}\n`
+      )
+    }
+  }
+
+  process.exit(report.findings.length > 0 ? 1 : 0)
+}
+
+module.exports = {
+  runHistoryScan,
+  handleHistoryScan,
+  buildHumanReport,
+  buildMarkdown,
+  buildGitleaksArgs,
+  dedupeFindings,
+  normalizeFinding,
+  parseDepth,
+}

package/lib/commands/index.js

@@ -13,6 +13,9 @@ const {
   detectRubyProject,
 } = require('./deps')
 const { handleAnalyzeCi } = require('./analyze-ci')
+const { handleShipCheck } = require('./ship-check')
+const { handlePrCheck } = require('./pr-check')
+const { handleHistoryScan } = require('./history-scan')
 
 module.exports = {
   // Validation commands
@@ -26,4 +29,9 @@ module.exports = {
 
   // CI/CD optimization commands
   handleAnalyzeCi,
+
+  // Pro release-confidence commands
+  handleShipCheck,
+  handlePrCheck,
+  handleHistoryScan,
 }