@1aboveio/skills 0.10.0

package/skills/engineering/resolve-release/scripts/candidate-hygiene.mjs

@@ -0,0 +1,407 @@
+// candidate-hygiene — pre-locate cleanup of stale candidate-* traffic tags.
+//
+// Previous successful releases can leave `candidate-<sha>` tags on the now-100% revision
+// (the incumbent) or on an orphan 0% revision. Those tags are NOT a second release in flight,
+// but they make `candidate-identity.mjs locate` report `e:second-candidate-in-flight`. This
+// script reads the same traffic capture `locate` uses, classifies the foreign tags, and emits the
+// `gcloud run services update-traffic --remove-tags` command that removes only the stale ones.
+//
+// A foreign tag is stale only when the shared traffic parser proves it has 0%, matching
+// terminal release evidence identifies the same tag and revision, and the product-scoped lease
+// report is valid and empty. Traffic, ambiguous traffic, missing/nonterminal evidence, or a
+// held/unknown lease remains a competing release and refuses. A missing lease report after the
+// other proofs reports `decision: "needs-lease-report"` and exits 3.
+//
+// The script is PURE: it does not shell out to gcloud or change production. The release lane runs
+// the emitted command, then re-runs `candidate-identity.mjs locate` on the cleaned traffic.
+//
+// Usage:
+//   candidate-hygiene locate --sha <shipped> --traffic <file>
+//                            [--candidate-tag-prefix <prefix-ending-in-hyphen>]
+//                            [--lease-report <file>] [--terminal-evidence <file>] [--service <svc>] [--region <r>]
+//
+// Input `--traffic` is the same flattened / JSON form accepted by `candidate-identity.mjs`.
+// `--lease-report` is the JSON output of `run-state.mjs check-release-lease-scoped` for the
+// same product. `--terminal-evidence` is a JSON array of canonical resolve-release manifests
+// or immutable release evidence bundles. Each stale candidate must match one verified terminal
+// attempt by exact tag and revision. Without either proof, cleanup is not authorized.
+//
+// Exit codes:
+//   0  clean / no-action — only stale tags (or none), and no true rival
+//   1  refuse — a true second release is present; the failing ids are in `refuse`
+//   2  usage / unreadable input
+//   3  needs-lease-report — a 0% foreign candidate requires a lease report to classify; supply one and re-run
+
+import fs from 'node:fs'
+import { isMainModule } from '../../engineering-runtime/scripts/main-module.mjs'
+import { parseTraffic, normalizeTrafficPercents } from './candidate-traffic.mjs'
+import { MIN_SHORT_SHA } from './candidate-identity.mjs'
+import { RELEASE_STATE_CONTRACT } from '../../resolve-issues/scripts/release-state-contract.mjs'
+import { validateBundle } from './evidence-bundle.mjs'
+
+const SHA_RE = new RegExp(`^[0-9a-f]{${MIN_SHORT_SHA},40}$`, 'i')
+const isSha = (s) => typeof s === 'string' && SHA_RE.test(s)
+
+const shaMatches = (a, b) => {
+  if (!isSha(a) || !isSha(b)) return false
+  const [x, y] = [a.toLowerCase(), b.toLowerCase()]
+  const short = x.length <= y.length ? x : y
+  const long = short === x ? y : x
+  return long.startsWith(short)
+}
+
+const DEFAULT_CANDIDATE_TAG_PREFIX = 'candidate-'
+
+function validCandidateTagPrefix(prefix) {
+  return typeof prefix === 'string' && /^[a-z][a-z0-9-]{0,30}-$/.test(prefix)
+}
+
+function classifyTag(tag, sha, candidateTagPrefix = DEFAULT_CANDIDATE_TAG_PREFIX) {
+  if (tag === 'candidate') return 'shared'
+  if (!validCandidateTagPrefix(candidateTagPrefix) || !tag.startsWith(candidateTagPrefix)) {
+    return 'other'
+  }
+  const suffix = tag.slice(candidateTagPrefix.length)
+  if (!/^[0-9a-f]{4,40}$/i.test(suffix) || !isSha(suffix)) return 'other'
+  return shaMatches(suffix, sha) ? 'ours' : 'foreign'
+}
+
+function readJson(file, what) {
+  if (!file) return { error: `usage: --${what} <file> is required` }
+  let raw
+  try {
+    raw = fs.readFileSync(file, 'utf8')
+  } catch {
+    return { error: `${what} not readable: ${file}` }
+  }
+  try {
+    return { value: JSON.parse(raw) }
+  } catch {
+    return { error: `${what} is not valid JSON: ${file}` }
+  }
+}
+
+const isObject = (value) => value != null && typeof value === 'object' && !Array.isArray(value)
+const hasText = (value) => typeof value === 'string' && value.trim().length > 0
+
+function matchingTerminalEvidence(terminalEvidence, candidate) {
+  if (!Array.isArray(terminalEvidence)) return null
+  const { leaseClosingStates, closeCauses } = RELEASE_STATE_CONTRACT.attempt
+  for (const evidence of terminalEvidence) {
+    if (!isObject(evidence)) continue
+
+    if (evidence.skill === 'resolve-release') {
+      const release = isObject(evidence.release) ? evidence.release : null
+      const attempt = release && isObject(release.attempt) ? release.attempt : null
+      const identity = release && isObject(release.candidateIdentity) ? release.candidateIdentity : null
+      if (
+        attempt && identity && hasText(attempt.id) && leaseClosingStates.includes(attempt.state) &&
+        attempt.closedBy === closeCauses[attempt.state] && hasText(attempt.evidence) &&
+        identity.tag === candidate.tag && identity.revision === candidate.revision
+      ) {
+        return { attemptId: attempt.id, source: 'manifest' }
+      }
+      continue
+    }
+
+    const bundle = validateBundle(evidence)
+    if (!bundle.valid) continue
+    const receipt = isObject(evidence.candidateReceipt) ? evidence.candidateReceipt : null
+    if (
+      hasText(evidence.attemptId) && receipt && leaseClosingStates.includes(evidence.terminalState) &&
+      evidence.closedBy === closeCauses[evidence.terminalState] &&
+      receipt.tag === candidate.tag && receipt.revision === candidate.revision
+    ) {
+      return { attemptId: evidence.attemptId, source: 'evidence-bundle' }
+    }
+  }
+  return null
+}
+
+function classifyLeaseReport(leaseReport) {
+  if (leaseReport === null || leaseReport === undefined) return { state: 'missing' }
+  if (!Array.isArray(leaseReport)) return { state: 'unknown', reason: 'product lease report is not an array' }
+  if (leaseReport.length === 0) return { state: 'clear' }
+
+  let held = false
+  for (const entry of leaseReport) {
+    if (!isObject(entry) || !isObject(entry.lease) || !isObject(entry.productMatch)) {
+      return { state: 'unknown', reason: 'product lease report contains an unreadable matching entry' }
+    }
+    if (entry.productMatch.repo === 'unknown' || entry.productMatch.service === 'unknown') {
+      return { state: 'unknown', reason: 'product lease identity is unknown' }
+    }
+    if (entry.productMatch.repo === false || entry.productMatch.service === false) {
+      return { state: 'unknown', reason: 'product lease report contains an entry outside its declared scope' }
+    }
+    if (entry.lease.code === 2 || entry.lease.state === 'unknown' || entry.lease.state === 'ambiguous') {
+      return { state: 'unknown', reason: entry.lease.reason || 'matching product lease is ambiguous' }
+    }
+    if (entry.lease.code === 1 || entry.lease.state === 'held') held = true
+    else return { state: 'unknown', reason: 'product lease report contains an unrecognized lease state' }
+  }
+  return held
+    ? { state: 'held', reason: 'a release attempt holds the matching product lease' }
+    : { state: 'unknown', reason: 'product lease report did not prove the matching product free' }
+}
+
+export function evaluateHygiene({
+  sha,
+  traffic,
+  leaseReport = null,
+  terminalEvidence = null,
+  incumbentRevision = null,
+  candidateTagPrefix = DEFAULT_CANDIDATE_TAG_PREFIX,
+}) {
+  if (!isSha(sha)) {
+    return { decision: 'usage', refuse: [], sha, evidence: { error: 'e:unusable-shipped-sha' } }
+  }
+  if (!validCandidateTagPrefix(candidateTagPrefix)) {
+    return {
+      decision: 'usage',
+      refuse: [],
+      sha,
+      evidence: { error: 'e:invalid-candidate-tag-prefix', candidateTagPrefix },
+    }
+  }
+  const rows = normalizeTrafficPercents(traffic.rows)
+  const classified = rows.map((row) => ({
+    ...row,
+    kind: classifyTag(row.tag, sha, candidateTagPrefix),
+  }))
+  const ours = classified.filter((row) => row.kind === 'ours')
+  const foreign = classified.filter((row) => row.kind === 'foreign')
+
+  let incumbent = incumbentRevision
+  if (!incumbent) {
+    const hundredRows = classified.filter((row) => row.percent === 100 && row.revision)
+    if (hundredRows.length === 1) incumbent = hundredRows[0].revision
+  }
+
+  const lease = classifyLeaseReport(leaseReport)
+  const staleTags = []
+  const rivals = []
+  let needsLeaseReport = false
+
+  for (const row of foreign) {
+    if (
+      row.percent === null || !Number.isFinite(row.percent) ||
+      row.percent < 0 || row.percent > 100
+    ) {
+      rivals.push({ ...candidateConflict(row, 'foreign candidate has ambiguous traffic') })
+      continue
+    }
+    if (row.percent !== 0) {
+      rivals.push({ ...candidateConflict(row, 'foreign candidate has non-zero traffic') })
+      continue
+    }
+
+    const terminal = matchingTerminalEvidence(terminalEvidence, row)
+    if (!terminal) {
+      rivals.push({ ...candidateConflict(row, 'foreign zero-traffic candidate has no matching terminal release evidence') })
+      continue
+    }
+    if (lease.state === 'missing') {
+      needsLeaseReport = true
+      continue
+    }
+    if (lease.state === 'held' || lease.state === 'unknown') {
+      rivals.push({ ...candidateConflict(row, `foreign zero-traffic candidate is competing because the matching product lease is ${lease.state}: ${lease.reason}`) })
+      continue
+    }
+
+    staleTags.push({
+      tag: row.tag,
+      revision: row.revision,
+      percent: row.percent,
+      kind: row.revision && row.revision === incumbent ? 'stale-on-incumbent' : 'stale-foreign-zero',
+      terminalAttempt: terminal.attemptId,
+    })
+  }
+
+  const shared = classified.filter((row) => row.kind === 'shared')
+  if (ours.length > 0) {
+    for (const row of shared) {
+      if (row.revision && row.revision !== ours[0].revision) {
+        rivals.push(candidateConflict(row, 'shared candidate tag resolves to a different revision'))
+      }
+    }
+  }
+  if (ours.length > 1) {
+    rivals.push({
+      tag: ours.map((row) => row.tag).join(', '),
+      revision: null,
+      percent: null,
+      reason: 'multiple entries claim this SHA-keyed candidate tag',
+    })
+  }
+
+  const cleanupCommand = staleTags.length > 0
+    ? `gcloud run services update-traffic <service> --region <region> --remove-tags ${staleTags.map((tag) => tag.tag).join(',')}`
+    : null
+  const evidence = {
+    trafficEntries: classified.length,
+    ourTagCount: ours.length,
+    staleTagCount: staleTags.length,
+    ...(rivals.length > 0 ? { rivalCount: rivals.length } : {}),
+  }
+
+  if (rivals.length > 0) {
+    return {
+      decision: 'refuse', sha, incumbentRevision: incumbent || null, staleTags, rivals,
+      cleanupCommand: null, refuse: ['e:second-candidate-in-flight'], evidence,
+    }
+  }
+  if (needsLeaseReport) {
+    return {
+      decision: 'needs-lease-report', sha, incumbentRevision: incumbent || null, staleTags, rivals,
+      cleanupCommand: null, refuse: [], evidence: { ...evidence, foreignZeroWithoutLeaseReport: true },
+    }
+  }
+  return {
+    decision: staleTags.length > 0 ? 'clean' : 'no-action',
+    sha,
+    incumbentRevision: incumbent || null,
+    staleTags,
+    rivals,
+    cleanupCommand,
+    refuse: [],
+    evidence,
+  }
+}
+
+function candidateConflict(row, reason) {
+  return { tag: row.tag, revision: row.revision, percent: row.percent, reason }
+}
+
+const USAGE = `candidate-hygiene — pre-locate cleanup of stale candidate-* traffic tags.
+
+Reads the same normalized traffic capture as candidate identity. A foreign \`candidate-*\`
+tag is stale only with proven zero traffic, matching terminal release evidence, and a valid
+empty product-scoped lease report; every ambiguous or competing shape refuses.
+
+Usage:
+  candidate-hygiene locate --sha <shipped> --traffic <file>
+                             [--candidate-tag-prefix <prefix-ending-in-hyphen>]
+                             [--lease-report <file>] [--terminal-evidence <file>]
+                             [--service <svc>] [--region <r>]
+
+Options:
+  --sha <shipped>          the promoted SHA
+  --traffic <file>         the flattened or JSON traffic capture (same form as locate)
+  --candidate-tag-prefix   repository SHA-keyed namespace; defaults to candidate-
+  --lease-report <file>    JSON output of run-state.mjs check-release-lease-scoped for the
+                           same product; without it, a 0% foreign candidate cannot be
+                           classified as stale and the script reports "needs-lease-report"
+  --terminal-evidence <file>  JSON array of canonical resolve-release manifests or immutable
+                           evidence bundles; exact candidate tag/revision must match a verified
+                           terminal close
+  --service <svc>          Cloud Run service name (used to render a concrete cleanup command)
+  --region <r>             Cloud Run region (used to render a concrete cleanup command)
+
+Exit codes:
+  0  clean or no-action — stale tags were found (or none) and no true rival
+  1  refuse — a true second release is present; \`refuse\` contains e:second-candidate-in-flight
+  2  usage / unreadable input
+  3  needs-lease-report — a 0% foreign candidate requires a lease report to classify; supply one and re-run
+`
+
+function parseArgs(argv) {
+  const opts = {}
+  const positional = []
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]
+    if (a === '--help' || a === '-h') opts.help = true
+    else if (a.startsWith('--')) opts[a.slice(2)] = argv[++i]
+    else positional.push(a)
+  }
+  return { opts, positional }
+}
+
+export function main(argv) {
+  const { opts, positional } = parseArgs(argv)
+  const cmd = positional[0]
+  if (opts.help || cmd === 'help') {
+    console.log(USAGE)
+    return 0
+  }
+  if (cmd !== 'locate') {
+    const what = cmd ? `unknown command "${cmd}"` : 'no subcommand given'
+    console.error(`candidate-hygiene: ${what}. Use locate (see --help).`)
+    return 2
+  }
+  if (!opts.sha) {
+    console.error('candidate-hygiene: usage: locate requires --sha <shipped>')
+    return 2
+  }
+  if (
+    'candidate-tag-prefix' in opts &&
+    (typeof opts['candidate-tag-prefix'] !== 'string' ||
+      !opts['candidate-tag-prefix'] ||
+      opts['candidate-tag-prefix'].startsWith('--'))
+  ) {
+    console.error('candidate-hygiene: --candidate-tag-prefix requires a value')
+    return 2
+  }
+  if (!opts.traffic) {
+    console.error('candidate-hygiene: usage: locate requires --traffic <file>')
+    return 2
+  }
+
+  let text
+  try {
+    text = fs.readFileSync(opts.traffic, 'utf8')
+  } catch {
+    console.error(`candidate-hygiene: traffic read not readable: ${opts.traffic}`)
+    return 2
+  }
+  const traffic = parseTraffic(text)
+  if (traffic.error) {
+    console.error(`candidate-hygiene: ${traffic.error}`)
+    return 2
+  }
+
+  let leaseReport = null
+  let terminalEvidence = null
+  if (opts['lease-report']) {
+    const read = readJson(opts['lease-report'], 'lease-report')
+    if (read.error) {
+      console.error(`candidate-hygiene: ${read.error}`)
+      return 2
+    }
+    leaseReport = read.value
+  }
+
+  if (opts['terminal-evidence']) {
+    const read = readJson(opts['terminal-evidence'], 'terminal-evidence')
+    if (read.error) {
+      console.error(`candidate-hygiene: ${read.error}`)
+      return 2
+    }
+    terminalEvidence = read.value
+  }
+
+  const result = evaluateHygiene({
+    sha: opts.sha,
+    traffic,
+    leaseReport,
+    terminalEvidence,
+    incumbentRevision: opts['incumbent-revision'] || null,
+    candidateTagPrefix: opts['candidate-tag-prefix'] || DEFAULT_CANDIDATE_TAG_PREFIX,
+  })
+
+  // Replace the <service>/<region> placeholders with concrete values if the caller supplied them.
+  if (result.cleanupCommand && opts.service && opts.region) {
+    result.cleanupCommand = result.cleanupCommand.replace('<service>', opts.service).replace('<region>', opts.region)
+  }
+
+  console.log(JSON.stringify(result, null, 2))
+  if (result.decision === 'refuse') return 1
+  if (result.decision === 'usage') return 2
+  if (result.decision === 'needs-lease-report') return 3
+  return 0
+}
+
+if (isMainModule(import.meta.url)) {
+  process.exit(main(process.argv.slice(2)))
+}