@defrex/autobuild 0.8.0

package/packages/core/src/processes/infrastructure-failure-budget.ts

@@ -0,0 +1,146 @@
+import type { AbEvent } from '../events/catalog'
+import { DISPATCHER } from '../events/envelope'
+import type { EventPayload } from '../events/payloads'
+import type { IdSource } from '../ids'
+import type { BuildStore } from '../store/types'
+
+export type InfrastructureOperation = EventPayload<'infrastructure.failed'>['operation']
+export type InfrastructureCause = EventPayload<'infrastructure.failed'>['cause']
+
+export interface InfrastructureFailureInput {
+  slug: string
+  events: readonly AbEvent[]
+  maxAttempts: number
+  provider: string
+  workspaceRef: string
+  instance: string
+  environmentId?: string
+  sessionId?: string
+  operation: InfrastructureOperation
+  error: unknown
+  cleanupPending: boolean
+}
+
+export interface InfrastructureFailureDependencies {
+  store: Pick<BuildStore, 'append' | 'getEvents'>
+  ids: IdSource
+}
+
+/** Infrastructure retries are re-armed only by a confirmed execution boundary
+ * or a retry answer to this policy's own previously-raised exhaustion escalation. */
+export function infrastructureFailureResetSeq(events: readonly AbEvent[]): number {
+  const infrastructureEscalations = new Set<string>()
+  let resetSeq = 0
+  for (const event of events) {
+    if (
+      event.type === 'escalation.raised' &&
+      event.payload.policyCause === 'infrastructure-failure-limit'
+    ) {
+      infrastructureEscalations.add(event.payload.id)
+    } else if (
+      event.type === 'execution.ended' ||
+      (event.type === 'escalation.answered' &&
+        event.payload.resolution === 'retry' &&
+        infrastructureEscalations.has(event.payload.id))
+    ) {
+      resetSeq = event.seq
+    }
+  }
+  return resetSeq
+}
+
+export function normalizeInfrastructureError(error: unknown): string {
+  const seen = new Set<unknown>()
+  const collect = (value: unknown): string[] => {
+    if ((typeof value === 'object' && value !== null) || typeof value === 'function') {
+      if (seen.has(value)) return []
+      seen.add(value)
+    }
+    if (value instanceof AggregateError) {
+      return [value.message, ...value.errors.flatMap(collect), ...collect(value.cause)].filter(
+        (part) => part.trim().length > 0,
+      )
+    }
+    if (value instanceof Error) {
+      return [value.message, ...collect(value.cause)].filter((part) => part.trim().length > 0)
+    }
+    if (value === undefined) return []
+    const text = String(value).trim()
+    return text === '' ? [] : [text]
+  }
+  const messages = [...new Set(collect(error).map((part) => part.trim()))]
+  return messages.join('; ') || 'provider operation failed without an error message'
+}
+
+/** One deterministic provider-failure taxonomy shared by every dispatch path. */
+export function classifyInfrastructureFailure(
+  message: string,
+  cleanupPending: boolean,
+): InfrastructureCause {
+  const lower = message.toLowerCase()
+  if (/limit|quota|cpu|duration/.test(lower)) return 'provider-limit'
+  if (/timeout|abort/.test(lower)) return 'timeout'
+  if (/no longer exists|not found|missing/.test(lower)) return 'missing'
+  return cleanupPending ? 'unknown-outcome' : 'provider-error'
+}
+
+/** Append one durable infrastructure failure and, when its epoch is exhausted,
+ * the epoch's single policy escalation. Returned events let snapshot-based
+ * callers immediately include these writes in the rest of their tick. */
+export async function recordInfrastructureFailure(
+  deps: InfrastructureFailureDependencies,
+  input: InfrastructureFailureInput,
+): Promise<AbEvent[]> {
+  const lastReset = infrastructureFailureResetSeq(input.events)
+  const attempt =
+    input.events.filter((event) => event.type === 'infrastructure.failed' && event.seq > lastReset)
+      .length + 1
+  const message = normalizeInfrastructureError(input.error)
+  const failure = await deps.store.append(input.slug, {
+    actor: DISPATCHER,
+    type: 'infrastructure.failed',
+    payload: {
+      provider: input.provider,
+      workspaceRef: input.workspaceRef,
+      instance: input.instance,
+      ...(input.environmentId !== undefined ? { environmentId: input.environmentId } : {}),
+      ...(input.sessionId !== undefined ? { sessionId: input.sessionId } : {}),
+      operation: input.operation,
+      cause: classifyInfrastructureFailure(message, input.cleanupPending),
+      attempt,
+      retryable: true,
+      cleanupPending: input.cleanupPending,
+      error: message,
+    },
+  })
+  const appended: AbEvent[] = [failure]
+
+  if (attempt < input.maxAttempts) return appended
+
+  const currentEvents = await deps.store.getEvents(input.slug)
+  if (
+    !currentEvents.some(
+      (event) =>
+        event.seq > lastReset &&
+        event.type === 'escalation.raised' &&
+        event.payload.policyCause === 'infrastructure-failure-limit',
+    )
+  ) {
+    appended.push(
+      await deps.store.append(input.slug, {
+        actor: DISPATCHER,
+        type: 'escalation.raised',
+        payload: {
+          id: deps.ids('esc'),
+          phase: 'setup',
+          source: 'policy',
+          policyCause: 'infrastructure-failure-limit',
+          question: `maxInfrastructureAttempts (${input.maxAttempts}) exhausted during ${input.operation}; provider cleanup/recovery must succeed before retry: ${message}`,
+          refs: [input.workspaceRef],
+        },
+      }),
+    )
+  }
+
+  return appended
+}

package/packages/core/src/processes/phase-failure-budget.ts

@@ -0,0 +1,52 @@
+import type { EscalationSource, EscalationTarget, Phase, PolicyEscalationCause } from '../ontology'
+
+export type PhaseFailureBudgetResetScope = 'none' | 'matching-round' | 'all-phase-rounds'
+
+/**
+ * Closed policy semantics for answering a phase-failure budget escalation.
+ * Adding a policy cause must choose its effect here rather than inheriting one
+ * from the incidental presence or absence of `round`.
+ */
+export const POLICY_PHASE_FAILURE_RESET_SCOPES = {
+  'review-round-limit': 'matching-round',
+  'verify-failure-limit': 'all-phase-rounds',
+  'reconcile-no-progress': 'none',
+  'setup-failure-limit': 'none',
+  'infrastructure-failure-limit': 'none',
+  'phase-attempt-limit': 'matching-round',
+  'non-retryable-phase-failure': 'matching-round',
+} as const satisfies Record<PolicyEscalationCause, PhaseFailureBudgetResetScope>
+
+export interface PhaseFailureResetEscalation {
+  phase: EscalationTarget
+  round?: number
+  source: EscalationSource
+  policyCause?: PolicyEscalationCause
+}
+
+/**
+ * Whether answering one raise re-arms the queried phase+round failure budget.
+ * Cause-less historical policy raises and non-policy raises retain the legacy
+ * shape rule so old logs replay without migration.
+ */
+export function resetsPhaseFailureBudget(
+  raise: PhaseFailureResetEscalation,
+  phase: Phase,
+  round: number,
+): boolean {
+  if (raise.phase !== phase) return false
+
+  if (raise.source !== 'policy' || raise.policyCause === undefined) {
+    return raise.round === undefined || raise.round === round
+  }
+
+  const scope = POLICY_PHASE_FAILURE_RESET_SCOPES[raise.policyCause]
+  switch (scope) {
+    case 'none':
+      return false
+    case 'matching-round':
+      return raise.round === round
+    case 'all-phase-rounds':
+      return true
+  }
+}

package/packages/core/src/processes/process-group-reaper.ts

@@ -0,0 +1,35 @@
+import { distributionPath } from '../distribution'
+import { DEFAULT_PROCESS_GROUP_STOP_TIMEOUT_MS } from './process-group'
+
+export const PROCESS_GROUP_REAPER_OPTIONS_ENV = 'AB_PROCESS_GROUP_REAPER_OPTIONS'
+
+export interface ProcessGroupReaperOptions {
+  groupId: number
+  stopTimeoutMs: number
+}
+
+export function processGroupReaperOptions(groupId: number): ProcessGroupReaperOptions {
+  return { groupId, stopTimeoutMs: DEFAULT_PROCESS_GROUP_STOP_TIMEOUT_MS }
+}
+
+/** Start an owner outside the dying build's session so group-wide SIGKILL
+ * cannot kill the process responsible for escalating to it. The synchronous
+ * return means the target leader can retain its process-group identity until
+ * the helper has been created. */
+export function launchProcessGroupReaper(
+  options: ProcessGroupReaperOptions,
+  spawn: typeof Bun.spawn = Bun.spawn,
+): number {
+  const child = spawn([process.execPath, distributionPath('bin', 'ab-process-group-reaper.ts')], {
+    env: {
+      ...process.env,
+      [PROCESS_GROUP_REAPER_OPTIONS_ENV]: JSON.stringify(options),
+    },
+    stdin: 'ignore',
+    stdout: 'ignore',
+    stderr: 'ignore',
+    detached: true,
+  })
+  child.unref()
+  return child.pid
+}

package/packages/core/src/processes/process-group.ts

@@ -0,0 +1,65 @@
+export const DEFAULT_PROCESS_GROUP_STOP_TIMEOUT_MS = 5_000
+
+function validateGroupId(groupId: number): void {
+  if (!Number.isSafeInteger(groupId) || groupId <= 0) {
+    throw new Error(`invalid process group id ${JSON.stringify(groupId)}`)
+  }
+}
+
+function isGone(error: unknown): boolean {
+  return error instanceof Error && 'code' in error && error.code === 'ESRCH'
+}
+
+/** Probe a POSIX process group without ever falling back to its leader PID. */
+export function processGroupAlive(groupId: number): boolean {
+  validateGroupId(groupId)
+  try {
+    process.kill(-groupId, 0)
+    return true
+  } catch (error) {
+    if (isGone(error)) return false
+    throw error
+  }
+}
+
+/** Signal a complete POSIX process group. An already-gone group is success. */
+export function signalProcessGroup(groupId: number, signal: NodeJS.Signals): boolean {
+  validateGroupId(groupId)
+  try {
+    process.kill(-groupId, signal)
+    return true
+  } catch (error) {
+    if (isGone(error)) return false
+    throw error
+  }
+}
+
+async function waitForGroupExit(groupId: number, deadline: number): Promise<boolean> {
+  while (processGroupAlive(groupId)) {
+    const remaining = deadline - Date.now()
+    if (remaining <= 0) return false
+    await Bun.sleep(Math.min(10, remaining))
+  }
+  return true
+}
+
+/**
+ * Gracefully stop a complete local execution group, then force the same group
+ * after the bounded stop delay. The force path waits until no member remains;
+ * callers may therefore use resolution as the workspace-teardown boundary.
+ */
+export async function terminateProcessGroup(
+  groupId: number,
+  stopTimeoutMs = DEFAULT_PROCESS_GROUP_STOP_TIMEOUT_MS,
+): Promise<void> {
+  validateGroupId(groupId)
+  if (!Number.isFinite(stopTimeoutMs) || stopTimeoutMs < 0) {
+    throw new Error(`invalid process-group stop timeout ${JSON.stringify(stopTimeoutMs)}`)
+  }
+  if (!signalProcessGroup(groupId, 'SIGTERM')) return
+
+  if (await waitForGroupExit(groupId, Date.now() + stopTimeoutMs)) return
+  if (!signalProcessGroup(groupId, 'SIGKILL')) return
+
+  while (processGroupAlive(groupId)) await Bun.sleep(10)
+}

package/packages/core/src/processes/publication-loss.ts

@@ -0,0 +1,62 @@
+/**
+ * Last-chance publication settlement at the workspace-release boundary
+ * (AUT-328). A pending `publication.requested` whose workspace is destroyed
+ * takes its only copy of un-pushed commits with it, so every destructive
+ * release path must first attempt trusted settlement and then, if the request
+ * still cannot be completed, durably record the loss BEFORE the release fact.
+ * A release must never proceed past an unrecorded pending request: if the
+ * loss record cannot be written, the append failure propagates and the caller
+ * aborts the release (the next tick retries it).
+ */
+import { DISPATCHER } from '../events/envelope'
+import type { AbEvent } from '../events/catalog'
+import {
+  latestUncompletedPublicationRequest,
+  publicationLostRecorded,
+  publicationRequestCompleted,
+} from './publication-state'
+import type { BuildStore } from '../store/types'
+
+export interface PublicationLossDeps {
+  store: BuildStore
+  /** Trusted publication settlement (dispatcher-injected). A throw means the
+   * request is uncompleted — the guard continues to the loss record. */
+  settlePublication?: (slug: string) => Promise<void>
+}
+
+/**
+ * Settle or record the loss for the latest uncompleted publication request
+ * ahead of a workspace release. Returns refreshed events for the caller's
+ * subsequent reads. Idempotent: a completed or already-recorded request
+ * changes nothing, and repeated runs append at most one `publication.lost`.
+ */
+export async function settlePublicationBeforeRelease(
+  deps: PublicationLossDeps,
+  slug: string,
+  events: AbEvent[],
+  reason: string,
+): Promise<AbEvent[]> {
+  const request = latestUncompletedPublicationRequest(events)
+  if (request === undefined) return events
+  try {
+    await deps.settlePublication?.(slug)
+  } catch {
+    // A throw means the request is still uncompleted (or settlement failed);
+    // either way the loss record below is what makes the release safe.
+  }
+  const refreshed = await deps.store.getEvents(slug)
+  if (publicationRequestCompleted(refreshed, request)) return refreshed
+  if (publicationLostRecorded(refreshed, request)) return refreshed
+  const appended = await deps.store.append(slug, {
+    actor: DISPATCHER,
+    type: 'publication.lost',
+    payload: {
+      request: request.seq,
+      operation: request.payload.operation,
+      branch: request.payload.branch,
+      sha: request.payload.sha,
+      reason,
+    },
+  })
+  return [...refreshed, appended]
+}

package/packages/core/src/processes/publication-state.ts

@@ -0,0 +1,93 @@
+import type { AbEvent } from '../events/catalog'
+
+export type PublicationRequest = Extract<AbEvent, { type: 'publication.requested' }>
+
+export function publicationRequestCompleted(
+  events: readonly AbEvent[],
+  request: PublicationRequest,
+): boolean {
+  return events.some((event) => {
+    if (event.seq <= request.seq) return false
+    if (request.payload.operation === 'implement')
+      return (
+        event.type === 'implement.completed' &&
+        event.payload.round === request.payload.round &&
+        event.payload.commits.base === request.payload.base &&
+        event.payload.commits.head === request.payload.sha
+      )
+    if (request.payload.operation === 'reconcile')
+      return (
+        event.type === 'reconcile.completed' && event.payload.mergeCommit === request.payload.sha
+      )
+    if (request.payload.operation === 'finalize')
+      return event.type === 'finalize.completed' && event.payload.pr.headSha === request.payload.sha
+    return (
+      event.type === 'finalize.step-completed' &&
+      event.payload.step === request.payload.step &&
+      (!event.payload.ok || event.payload.headSha === request.payload.sha)
+    )
+  })
+}
+
+/** A request is no longer publishable after its workspace is released. The
+ * commit may have existed only in that disposable environment, so replacement
+ * runners must rerun the phase rather than trying to settle the stale SHA. */
+export function publicationRequestSettled(
+  events: readonly AbEvent[],
+  request: PublicationRequest,
+): boolean {
+  let workspaceRef: string | undefined
+  for (const event of events) {
+    if (event.seq > request.seq) break
+    if (event.type === 'workspace.provisioned') workspaceRef = event.payload.ref
+    else if (event.type === 'workspace.released') workspaceRef = undefined
+  }
+  return (
+    publicationRequestCompleted(events, request) ||
+    events.some(
+      (event) =>
+        event.seq > request.seq &&
+        event.type === 'workspace.released' &&
+        (!('ref' in event.payload) ||
+          workspaceRef === undefined ||
+          event.payload.ref === workspaceRef),
+    )
+  )
+}
+
+/** The latest `publication.requested` with no settling completion fact — the
+ * single settlement target for every caller that would previously re-derive it
+ * with a private `findLast`. */
+export function latestUncompletedPublicationRequest(
+  events: readonly AbEvent[],
+): PublicationRequest | undefined {
+  const request = events.findLast(
+    (event) =>
+      event.type === 'publication.requested' && !publicationRequestCompleted(events, event),
+  )
+  return request?.type === 'publication.requested' ? request : undefined
+}
+
+/** Whether the durable loss record (AUT-328) has already been appended for
+ * `request`. The release guard and the settlement backstop both consult this
+ * so repeated ticks append at most one `publication.lost` per request. */
+export function publicationLostRecorded(
+  events: readonly AbEvent[],
+  request: PublicationRequest,
+): boolean {
+  return events.some(
+    (event) => event.type === 'publication.lost' && event.payload.request === request.seq,
+  )
+}
+
+export function abandonedPublicationPending(events: readonly AbEvent[]): boolean {
+  const request = latestUncompletedPublicationRequest(events)
+  return request !== undefined && publicationRequestSettled(events, request)
+}
+
+export function publicationPending(events: readonly AbEvent[]): boolean {
+  return events.some(
+    (request) =>
+      request.type === 'publication.requested' && !publicationRequestSettled(events, request),
+  )
+}

package/packages/core/src/processes/sandbox-state.ts

@@ -0,0 +1,101 @@
+/**
+ * Durable operator-sandbox state (AUT-340): the repository-journal projection
+ * the sandbox service, the dispatcher's idle-settlement stage, and the
+ * repository status projection read instead of process memory. The lifecycle
+ * facts (`orchestrator.sandbox.provisioned`/`resumed`/`activity`/`stopped`/
+ * `released`/`reset`) bracket one environment per operator × repository.
+ * Every helper here is a pure function over the journal, mirroring the
+ * harvest execution settlement, so any dispatcher invocation can settle an
+ * orphaned environment.
+ */
+import type { RepositoryEvent } from '../events/repository'
+
+export type SandboxEnvironmentState = 'live' | 'stopped' | 'released'
+
+export interface SandboxEnvironmentSnapshot {
+  operator: string
+  environmentId: string
+  provider: string
+  state: SandboxEnvironmentState
+  /** Journal timestamp of the latest lifecycle/activity fact for this
+   * environment — the evidence the idle settlement compares against
+   * `[orchestrator].sandbox.idleMinutes`. */
+  lastEvidenceTs: string
+  sessionId?: string
+}
+
+type MutableSnapshot = {
+  operator: string
+  environmentId: string
+  provider: string
+  state: SandboxEnvironmentState
+  lastEvidenceTs: string
+  sessionId?: string
+}
+
+/** Per-environment sandbox state, keyed deterministically by
+ * `environmentId` (one environment per operator × repository, and the id is
+ * derived from both). Latest fact wins; `released` environments stay in the
+ * result with their closing evidence so callers can distinguish them. */
+export function sandboxStates(events: readonly RepositoryEvent[]): SandboxEnvironmentSnapshot[] {
+  const byEnvironment = new Map<string, MutableSnapshot>()
+  for (const event of events) {
+    const ts = event.ts
+    if (event.type === 'orchestrator.sandbox.provisioned') {
+      byEnvironment.set(event.payload.environmentId, {
+        operator: event.payload.operator,
+        environmentId: event.payload.environmentId,
+        provider: event.payload.provider,
+        state: 'live',
+        lastEvidenceTs: ts,
+        ...(event.payload.sessionId !== undefined ? { sessionId: event.payload.sessionId } : {}),
+      })
+    } else if (event.type === 'orchestrator.sandbox.resumed') {
+      const previous = byEnvironment.get(event.payload.environmentId)
+      byEnvironment.set(event.payload.environmentId, {
+        operator: event.payload.operator,
+        environmentId: event.payload.environmentId,
+        provider: event.payload.provider,
+        state: 'live',
+        lastEvidenceTs: ts,
+        ...(event.payload.sessionId !== undefined
+          ? { sessionId: event.payload.sessionId }
+          : previous?.sessionId !== undefined
+            ? { sessionId: previous.sessionId }
+            : {}),
+      })
+    } else if (event.type === 'orchestrator.sandbox.activity') {
+      const current = byEnvironment.get(event.payload.environmentId)
+      if (current !== undefined && current.lastEvidenceTs <= ts) {
+        current.lastEvidenceTs = ts
+      }
+    } else if (event.type === 'orchestrator.sandbox.stopped') {
+      const current = byEnvironment.get(event.payload.environmentId)
+      if (current !== undefined && current.lastEvidenceTs <= ts) {
+        current.state = 'stopped'
+        current.lastEvidenceTs = ts
+      }
+    } else if (event.type === 'orchestrator.sandbox.reset') {
+      // Destructive marker only: the following released/provisioned facts
+      // carry the evidence. Refresh the evidence timestamp so a reset alone
+      // counts as recent activity for the environment it names.
+      const current = byEnvironment.get(event.payload.environmentId)
+      if (current !== undefined && current.lastEvidenceTs <= ts) {
+        current.lastEvidenceTs = ts
+      }
+    } else if (event.type === 'orchestrator.sandbox.released') {
+      // The released payload carries no provider; keep the one the
+      // environment's earlier facts recorded, when one exists.
+      const existing = byEnvironment.get(event.payload.environmentId)
+      byEnvironment.set(event.payload.environmentId, {
+        operator: event.payload.operator,
+        environmentId: event.payload.environmentId,
+        provider: existing?.provider ?? '',
+        state: 'released',
+        lastEvidenceTs: ts,
+        ...(existing?.sessionId !== undefined ? { sessionId: existing.sessionId } : {}),
+      })
+    }
+  }
+  return [...byEnvironment.values()]
+}

package/packages/core/src/registry.ts

@@ -0,0 +1,24 @@
+/**
+ * npm registry addressing shared by self-update (version lookup) and the
+ * guest distribution source (tarball download). Pure URL arithmetic; the
+ * callers own their transport seams.
+ */
+
+export const DEFAULT_NPM_REGISTRY = 'https://registry.npmjs.org'
+
+/** The registry to read: the operator's configured `NPM_CONFIG_REGISTRY`
+ * when set, else the public registry, without a trailing slash. */
+export function registryBaseUrl(
+  env: Readonly<Record<string, string | undefined>> | undefined,
+): string {
+  const configured = env?.NPM_CONFIG_REGISTRY?.trim()
+  return (
+    configured !== undefined && configured !== '' ? configured : DEFAULT_NPM_REGISTRY
+  ).replace(/\/+$/, '')
+}
+
+/** `<registry>/<encoded name>/<version|latest>` — the registry's version
+ * document endpoint, with the scope separator encoded as npm clients do. */
+export function registryVersionUrl(base: string, packageName: string, version?: string): string {
+  return `${base}/${packageName.replace('/', '%2f')}/${version ?? 'latest'}`
+}