@1aboveio/skills 0.10.0

package/skills/engineering/resolve-release/scripts/postmortem.mjs

@@ -0,0 +1,381 @@
+// The release lane's post-mortem: what this release left behind, and what to change.
+//
+// MANDATORY and AUTOMATIC, on the same terms as the delivery lane's — `report.mjs` saves it
+// when a release reaches a terminal step, and the value is the CROSS-RUN series. It shares
+// that series, its renderer and its persistence with `resolve-issues`; only the questions
+// differ, and they differ a lot.
+//
+// WHAT THIS PASS IS FOR, AND WHY IT IS NOT A SPEED PASS. The delivery lane's post-mortem
+// hunts wasted wall-clock, because that is what an epic loses. A release does not — the
+// measured median is well under an hour, since the lane is mostly waiting on watchers. What
+// a release loses is not time but ACCOUNT: the record of what happened to production. So
+// every finder below asks a completeness question, and the severities are about how much of
+// a release's own history is unreachable from state, not about how long it took.
+//
+// **The corpus measurements live in one place: `references/postmortem.md`.** Each finder was
+// added with the count of real releases it would have caught, and that table is dated and
+// re-measurable. They are deliberately NOT restated here or in the strings this module
+// renders: a number quoted in four files is a number that goes stale in three of them, which
+// is the exact failure the delivery lane's `inFlightCap` finding exists because of (a default
+// documented as 2 for nine days after it became 4). Read the table there before editing a
+// severity or a threshold — the thresholds below are derived from it.
+//
+// Rollbacks and burned versions are reported as facts rather than faulted: a rollback that
+// happened and was recorded is the machine WORKING, and a finder that scolds for it would
+// train the reader to ignore the pass.
+
+import {
+  profile, renderProfile, renderProfileIssue, saveProfile,
+} from '../../resolve-issues/scripts/postmortem.mjs'
+import {
+  CANONICAL_RELEASE_MANIFEST_FIELDS,
+  RELEASE_STEPS,
+  classifyReleaseManifest,
+  RELEASE_TERMINAL_STEPS,
+  validateReleaseHistory,
+} from '../../resolve-issues/scripts/release-state-contract.mjs'
+import { metadataPrStatus } from './metadata-pr-status.mjs'
+
+const isObject = (v) => v != null && typeof v === 'object' && !Array.isArray(v)
+const nonEmpty = (v) => typeof v === 'string' && v.trim() !== ''
+
+function versionString(v) {
+  if (nonEmpty(v)) return v
+  if (isObject(v)) {
+    return nonEmpty(v.tag) ? v.tag : (nonEmpty(v.next) ? v.next : null)
+  }
+  return null
+}
+
+function resolveShippedSha(m) {
+  const rel = isObject(m.release) ? m.release : {}
+  return rel.tagSha || rel.candidateSha || (rel.candidateIdentity && rel.candidateIdentity.sourceSha) || m.shippedSha || null
+}
+
+// A release is over when it reached one of these. `discard` and `rollback` are terminal and
+// SUCCESSFUL outcomes of the machine — the candidate was caught before or after exposure and
+// the lane did its job — so a release that ends at either is complete, not failed.
+const TERMINAL_STEPS = new Set(RELEASE_TERMINAL_STEPS)
+
+// The keys the release schema defines (run-state.md § Release variant), plus the general
+// run-level fields every manifest carries. Anything else is state written where nothing
+// reads it. Deliberately generous: the point is to catch `z4Fix`, not to police a field
+// somebody added for a good reason and documented.
+const KNOWN_KEYS = new Set(CANONICAL_RELEASE_MANIFEST_FIELDS)
+
+const hrs = (a, b) => {
+  const x = Date.parse(a); const y = Date.parse(b)
+  return Number.isFinite(x) && Number.isFinite(y) ? (y - x) / 3600000 : null
+}
+const fmtH = (h) => (h == null ? '—' : h < 1 ? `${Math.round(h * 60)}m` : `${h.toFixed(1)}h`)
+
+export function releaseTimeline(manifest = {}) {
+  const errors = validateReleaseHistory(manifest)
+  if (errors.length) {
+    return { status: 'malformed', errors, events: [], intervals: [], metrics: null, truncated: null }
+  }
+  const history = isObject(manifest?.releaseHistory) ? manifest.releaseHistory : { events: [] }
+  const events = Array.isArray(history.events) ? structuredClone(history.events) : []
+  const intents = new Map((history.truncated?.openIntents || []).map((intent) => [
+    `${intent.subject}:${intent.correlationId}`, intent,
+  ]))
+  const intervals = []
+  for (const event of events) {
+    if (event.kind === 'intent') intents.set(`${event.subject}:${event.correlationId}`, event)
+    if (event.kind !== 'result') continue
+    const key = `${event.subject}:${event.correlationId}`
+    const intent = intents.get(key)
+    if (!intent) continue
+    intervals.push({
+      correlationId: event.correlationId,
+      subject: event.subject,
+      status: event.outcome,
+      startedAt: intent.at,
+      endedAt: event.at,
+      durationMs: Date.parse(event.at) - Date.parse(intent.at),
+    })
+    intents.delete(key)
+  }
+  for (const intent of intents.values()) {
+    intervals.push({
+      correlationId: intent.correlationId,
+      subject: intent.subject,
+      status: 'waiting',
+      startedAt: intent.at,
+      endedAt: null,
+      durationMs: null,
+    })
+  }
+  intervals.sort((left, right) => Date.parse(left.startedAt) - Date.parse(right.startedAt))
+  return {
+    status: history.truncated ? 'truncated' : 'complete',
+    events,
+    intervals,
+    metrics: {
+      eventCount: events.length + (history.truncated?.count || 0),
+      completedWaitMs: (history.truncated?.completedWaitMs || 0) + intervals.reduce((total, interval) => total + (interval.durationMs || 0), 0),
+      failedWaitMs: (history.truncated?.failedWaitMs || 0) + intervals
+        .filter((interval) => interval.status === 'failed')
+        .reduce((total, interval) => total + (interval.durationMs || 0), 0),
+      unresolvedWaits: intervals.filter((interval) => interval.status === 'waiting').length,
+    },
+    truncated: history.truncated ? structuredClone(history.truncated) : null,
+  }
+}
+
+function formatDuration(durationMs) {
+  if (durationMs == null) return 'waiting'
+  const minutes = Math.round(durationMs / 60000)
+  return minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, '0')}m`
+}
+
+function renderReleaseTimeline(manifest) {
+  const timeline = releaseTimeline(manifest)
+  if (timeline.status === 'malformed') {
+    return `RELEASE TIMELINE: MALFORMED\n  ${timeline.errors.join('\n  ')}\n`
+  }
+  const rows = ['RELEASE TIMELINE']
+  if (timeline.truncated) {
+    rows.push(`  truncated: ${timeline.truncated.count} event(s), release-event-${timeline.truncated.firstSequence} through release-event-${timeline.truncated.throughSequence}; boundary ${timeline.truncated.boundary.step ?? 'null'} / ${timeline.truncated.boundary.attemptState ?? 'null'}`)
+  }
+  if (!timeline.events.length) rows.push('  No machine-generated release events recorded.')
+  for (const event of timeline.events) {
+    if (event.kind === 'state') {
+      rows.push(`  ${event.id} ${event.at} state: ${event.prior.step ?? 'null'} -> ${event.next.step ?? 'null'}; ${event.prior.attemptState ?? 'null'} -> ${event.next.attemptState ?? 'null'}`)
+    } else {
+      rows.push(`  ${event.id} ${event.at} ${event.kind}: ${event.subject} ${event.correlationId}${event.outcome ? ` ${event.outcome}` : ''}`)
+    }
+  }
+  if (timeline.intervals.length) rows.push('  intervals:')
+  for (const interval of timeline.intervals) {
+    rows.push(`    ${interval.subject} ${interval.correlationId}: ${interval.status} (${formatDuration(interval.durationMs)})`)
+  }
+  return `${rows.join('\n')}\n`
+}
+
+// ---- the finders ----
+
+function findStepOffEnum(m) {
+  const step = m.step
+  if (!nonEmpty(step) || RELEASE_STEPS.includes(step)) return null
+  return {
+    id: 'step-off-enum',
+    severity: 'high',
+    costMinutes: null,
+    headline: `step is \`${step}\`, which is not a release step`,
+    detail: `\`step\` is the resume key — the one field a dead session re-enters at — and it is an enum: ${RELEASE_STEPS.join(' | ')}. A freeform value cannot be routed on, so a resumed session cannot tell whether this release shipped, is mid-deploy, or needs a rollback.`,
+    evidence: [
+      `recorded: ${JSON.stringify(step)}`,
+      '`run-state.mjs validate` REFUSES this value — so validate was never run on this release',
+    ],
+    recommendation: 'Record the enum step and put the detail elsewhere (`terminalState`, `note`). Then run `run-state.mjs validate <slug>` at the handoff — it already catches this, so every occurrence is a release where validate was not run.',
+  }
+}
+
+function findStoppedMidMachine(m) {
+  const step = m.step
+  // An off-enum step is already the louder finding, and terminality is undecidable there —
+  // saying so twice in different words reads as two problems.
+  if (!nonEmpty(step) || !RELEASE_STEPS.includes(step) || TERMINAL_STEPS.has(step)) return null
+  const age = hrs(m.updatedAt, new Date().toISOString())
+  return {
+    id: 'stopped-mid-machine',
+    severity: 'high',
+    costMinutes: null,
+    headline: `the release stopped at \`${step}\` and never recorded a terminal step`,
+    detail: 'A release that stops mid-machine leaves production in a state only the tree can answer for. At `validate` a candidate may be serving nobody and costing money; at `expose` a traffic shift may be half-applied. Neither is visible from state, and the manifest is what a later session reads first.',
+    evidence: [
+      `last write: ${m.updatedAt ?? '—'}${age != null && age > 24 ? ` (${Math.round(age / 24)} days ago)` : ''}`,
+      `terminal steps are: ${[...TERMINAL_STEPS].join(', ')}`,
+      ...(isObject(m.release) && nonEmpty(m.release.candidateRevision)
+        ? [`a candidate revision is recorded and unresolved: ${m.release.candidateRevision} — check whether it still exists at 0% traffic`] : []),
+    ],
+    recommendation: 'Finish the machine or record the outcome that actually happened — `done`, `discard` (the candidate was deleted pre-exposure), or `rollback`. If the release was abandoned, `discard` with a note is the honest record; silence is not.',
+  }
+}
+
+function findNoVerification(m) {
+  if (isObject(m.verification) || !nonEmpty(m.step)) return null
+  // Only owed once the machine has got far enough to have verified something. A release
+  // that stopped at intake owes no verification record and faulting it is noise.
+  const reached = ['verify', 'expose', 'done'].includes(m.step) || m.step === 'validate'
+  if (!reached) return null
+  return {
+    id: 'no-verification-record',
+    severity: 'medium',
+    costMinutes: null,
+    headline: 'no `verification` record, on a release that reached ' + `\`${m.step}\``,
+    detail: 'The verification block is the evidence that the thing which shipped was checked — the candidate\'s ALIVE verdict, the re-check fence readings, the serving confirmation. Without it the handoff is asserting a verified release from memory, which is exactly the claim the record exists to make checkable.',
+    evidence: [`step: ${m.step}`, 'verification: absent'],
+    recommendation: 'Record the verification as it happens rather than at the handoff. Across the corpus this is the lane\'s most-skipped record — see references/postmortem.md for the count.',
+  }
+}
+
+function findAdHocKeys(m) {
+  const extra = Object.keys(m).filter((k) => !KNOWN_KEYS.has(k))
+  if (!extra.length) return null
+  return {
+    id: 'adhoc-manifest-keys',
+    severity: extra.length >= 5 ? 'medium' : 'low',
+    costMinutes: null,
+    headline: `${extra.length} manifest key(s) outside the release schema`,
+    detail: 'State written to a key nothing reads is state that did not survive: no gate checks it, no resume routes on it, and `report.mjs` will not render it. It looks like a record and behaves like a comment.',
+    evidence: [extra.slice(0, 12).join(', ') + (extra.length > 12 ? `, … (+${extra.length - 12})` : '')],
+    recommendation: 'Put it in a schema field, or in `note`/`terminalState` where a reader will find it. Roughly half of all releases wrote at least one such key — see references/postmortem.md.',
+  }
+}
+
+function findLongRelease(m) {
+  const h = hrs(m.createdAt, m.updatedAt)
+  // Well past "a queue was slow" — an order of magnitude over the measured median
+  // (references/postmortem.md). Kept as a bare threshold so the median can be re-measured
+  // there without this constant silently disagreeing with it.
+  if (h == null || h < 8) return null
+  return {
+    id: 'long-release',
+    severity: 'low',
+    costMinutes: Math.round(h * 60),
+    headline: `the release ran ${fmtH(h)} — an order of magnitude over the usual`,
+    detail: 'Releases are normally fast — the lane is mostly waiting on watchers. A long one is usually a release paused for a human and resumed later, which is fine, or one stalled on a gate nobody was watching, which is not. The manifest cannot tell those apart, so this is a prompt to check rather than a defect.',
+    evidence: [`createdAt ${m.createdAt ?? '—'}`, `updatedAt ${m.updatedAt ?? '—'}`],
+    recommendation: 'If it was waiting on a human, that is the answer and no change is owed. If it stalled on a watcher, that is a continuation-trigger miss (SKILL.md: a watcher firing is a continuation, not a checkpoint).',
+  }
+}
+
+function findMetadataPrIncomplete(m) {
+  const rel = isObject(m.release) ? m.release : {}
+  const attempt = rel.attempt || {}
+  // Only verified-sha releases that actually reached `done` owe a metadata PR. Rollbacks
+  // and discards do not; a release that stopped mid-machine already has that finding.
+  if (m.step !== 'done' || attempt.state !== 'released') return null
+  const status = metadataPrStatus(rel)
+  if (!status || status.complete) return null
+  return {
+    id: 'metadata-pr-incomplete',
+    severity: 'high',
+    costMinutes: null,
+    headline: 'released attempt has no valid metadata PR',
+    detail: 'The verified-sha-github-flow contract requires `metadata-sync.mjs open` or `metadata-sync.mjs na` to record `release.metadataPr` after the final claim. A released attempt without a valid record is incomplete; report.mjs flags it owed, and the handoff must not treat it as done.',
+    evidence: [
+      `step: ${m.step}`,
+      `attempt.state: ${attempt.state}`,
+      `release.metadataPr: ${JSON.stringify(rel.metadataPr)}`,
+      ...(isObject(rel.metadataSync)
+        ? [`legacy metadataSync present: ${JSON.stringify(rel.metadataSync)}`] : []),
+    ],
+    recommendation: 'Run `metadata-sync.mjs open` if a CHANGELOG/package version PR is owed, or `metadata-sync.mjs na --reason <text>` if not applicable. If a legacy `metadataSync` record exists, run `metadata-sync.mjs migrate`.',
+  }
+}
+
+const RELEASE_FINDERS = [
+  findStepOffEnum,
+  findStoppedMidMachine,
+  findNoVerification,
+  findAdHocKeys,
+  findLongRelease,
+  findMetadataPrIncomplete,
+]
+
+// The version the attempts of this release were cut for — what the version-level post-mortem
+// (#961, version-postmortem.mjs) discovers durable bundles by. Same resolution the summary uses.
+export function releaseVersion(m) {
+  const rel = isObject(m?.release) ? m.release : {}
+  return versionString(rel.version) ?? versionString(isObject(m?.envelope) ? m.envelope.version : null)
+}
+
+// The rc circuit breaker (#962), reported as a fact — like a rollback, a hold that happened
+// and was recorded is the machine WORKING, so this is never a faulting finding: what the
+// post-mortem owes is whether the breaker fired and how it was resolved (the issue's AC).
+// The record lives at `release.circuitBreaker` — the decision JSON rc-circuit-breaker.mjs
+// printed and the lane merged onto the manifest before the rc mint.
+export function circuitBreakerStatus(m) {
+  const rel = isObject(m.release) ? m.release : {}
+  const cb = isObject(rel.circuitBreaker) ? rel.circuitBreaker : null
+  if (!cb || !nonEmpty(cb.decision)) return null
+  if (cb.decision === 'proceed') {
+    return { fired: false, resolution: null, line: 'rc circuit breaker: did not fire (evaluated below threshold)' }
+  }
+  const count = Number.isInteger(cb.qualifyingCount) ? cb.qualifyingCount : '?'
+  if (cb.decision === 'authorized-continue' && isObject(cb.authorization) && nonEmpty(cb.authorization.authorizedBy)) {
+    return {
+      fired: true,
+      resolution: `authorized-continue by ${cb.authorization.authorizedBy}`,
+      line: `rc circuit breaker fired — authorized-continue by ${cb.authorization.authorizedBy} (${count} qualifying failure(s))`,
+    }
+  }
+  return {
+    fired: true,
+    resolution: 'unresolved at this record',
+    line: `rc circuit breaker FIRED (${count} qualifying) — unresolved`,
+  }
+}
+
+function releaseSummary(m) {
+  const rel = isObject(m.release) ? m.release : {}
+  const version = releaseVersion(m)
+  const shippedSha = resolveShippedSha(m)
+  const metrics = {
+    releaseState: classifyReleaseManifest(m),
+    step: m.step ?? null,
+    version: version ?? null,
+    shippedSha: nonEmpty(shippedSha) ? shippedSha.slice(0, 7) : null,
+    runHours: (() => { const h = hrs(m.createdAt, m.updatedAt); return h == null ? null : Number(h.toFixed(2)) })(),
+    rolledBack: Boolean(m.rollback),
+    burned: Boolean(rel.burned),
+    // #962 — present only when the lane recorded a breaker decision: an absent record and a
+    // breaker that did not fire are different statements, and a null here would read as a
+    // measurement (the same trap the inFlightCap column guard names).
+    ...(circuitBreakerStatus(m)
+      ? { circuitBreaker: { fired: circuitBreakerStatus(m).fired, resolution: circuitBreakerStatus(m).resolution } }
+      : {}),
+  }
+  const cbLine = circuitBreakerStatus(m)?.line
+  const bits = [
+    `step ${metrics.step ?? '—'}`,
+    version ? `version ${version}` : 'unversioned (dev/staging)',
+    `${fmtH(metrics.runHours)} wall-clock`,
+    ...(metrics.rolledBack ? ['ROLLED BACK'] : []),
+    ...(metrics.burned ? ['version burned'] : []),
+    ...(cbLine ? [cbLine] : []),
+  ]
+  return {
+    kind: 'release',
+    metrics,
+    summaryLine: bits.join(' · '),
+    blindSpots: 'what the tree and the live service actually hold — a candidate revision this pass reports as recorded may already be gone, and one it cannot see may still be serving',
+  }
+}
+
+export function releaseProfile(manifest = {}) {
+  return {
+    ...profile(manifest, { finders: RELEASE_FINDERS, summary: releaseSummary }),
+    timeline: releaseTimeline(manifest),
+  }
+}
+
+export function renderReleasePostmortem(manifest = {}) {
+  return `${renderProfile(releaseProfile(manifest))}\n${renderReleaseTimeline(manifest)}`
+}
+
+export function renderReleasePostmortemIssue(manifest = {}) {
+  const issue = renderProfileIssue(releaseProfile(manifest), 'resolve-release')
+  return {
+    ...issue,
+    body: `${issue.body}\n\n## Release timeline\n\n\`\`\`text\n${renderReleaseTimeline(manifest)}\`\`\``,
+  }
+}
+
+export function saveReleasePostmortem(manifest = {}, env = process.env, now = new Date()) {
+  return saveProfile(releaseProfile(manifest), env, now)
+}
+
+// A release is profiled once it reaches a terminal step. Unlike the delivery lane — where an
+// unfinished run is simply in flight — a release that never reaches one is itself a finding
+// (`stopped-mid-machine`), so this is deliberately NOT the gate on whether the pass has
+// anything to say. It is only the gate on the AUTOMATIC save, and the standalone command
+// profiles a stalled release on demand, which is how that finding gets seen at all.
+export function releaseIsTerminal(manifest = {}) {
+  return TERMINAL_STEPS.has(manifest?.step)
+}
+
+export { RELEASE_FINDERS, TERMINAL_STEPS, KNOWN_KEYS }