@defrex/autobuild 0.10.1 → 0.11.0

package/packages/core/src/cli/dispatch.ts

@@ -1,3 +1,5 @@
+import { readDispatcherLiveness, StandbyIdentityResolver } from './dispatcher-liveness'
+import { formatDispatcherIdentity } from '../kernel/dispatcher-liveness'
 /**
  * `ab dispatch` — the operator's entry into the outer loop (SPEC §3.3, §12).
  * Runs OUTSIDE build sessions like init/upgrade/ticket (§16.3): it takes a
@@ -691,6 +693,9 @@ class DispatchLoop {
    * discarded on Escape. All read semantics live in the shared feed module. */
   private sessionFeed: SessionStreamFeed | undefined
   private warningLine: string | undefined
+  /** Invocation-local ownership; independent of every notice slot. */
+  private standby: string | undefined
+  private standbyResolver: StandbyIdentityResolver | undefined
   /** Process-local, persistent release notice. It never shares the replaceable
    * warning slot and is re-applied to every store projection. */
   private availableUpgrade: string | undefined
@@ -1190,6 +1195,7 @@ class DispatchLoop {
       selection: _oldSelection,
       warningLines: _oldWarningLines,
       availableUpgrade: _oldAvailableUpgrade,
+      standby: _oldStandby,
       resumeInput: _oldResumeInput,
       abortConfirmation: _oldAbortConfirmation,
       view: _oldView,
@@ -1204,6 +1210,7 @@ class DispatchLoop {
     ]
     let effective: DashboardModel = {
       ...base,
+      ...(this.standby !== undefined ? { standby: this.standby } : {}),
       ...(this.availableUpgrade !== undefined ? { availableUpgrade: this.availableUpgrade } : {}),
       ...(warningLines.length > 0 ? { warningLines } : {}),
       ...(this.selection !== undefined ? { selection: this.selection } : {}),
@@ -1258,7 +1265,13 @@ class DispatchLoop {
     const limit =
       terminal === undefined
         ? 0
-        : sessionScrollLimit(view, terminal.columns, paintableRows(terminal.rows), upgrade)
+        : sessionScrollLimit(
+            view,
+            terminal.columns,
+            paintableRows(terminal.rows),
+            upgrade,
+            this.standby !== undefined,
+          )
     const current = view.follow ? limit : view.scroll
     const scroll =
       terminal === undefined
@@ -1270,6 +1283,7 @@ class DispatchLoop {
             current,
             delta,
             upgrade,
+            this.standby !== undefined,
           )
     let follow = view.follow
     if (delta < 0 && follow && scroll < limit) follow = false
@@ -1291,6 +1305,7 @@ class DispatchLoop {
       terminal.columns,
       paintableRows(terminal.rows),
       this.availableUpgrade !== undefined,
+      this.standby !== undefined,
     )
     return { ...view, scroll: view.follow ? limit : Math.min(view.scroll, limit) }
   }
@@ -1350,6 +1365,7 @@ class DispatchLoop {
                 this.view.scroll,
                 delta,
                 this.availableUpgrade !== undefined,
+                this.standby !== undefined,
               ),
       }
       this.syncModelControls()
@@ -3389,6 +3405,18 @@ class DispatchLoop {
     const claimed = await this.wiring.store
       .claimRepoLease(this.repoIdentity, holder, REPO_LEASE_TTL_MS)
       .catch(() => false)
+    if (this.dashboard && !this.opts.once) {
+      this.standbyResolver ??= new StandbyIdentityResolver(this.wiring.store, this.repoIdentity)
+      this.standby = claimed
+        ? undefined
+        : await this.standbyResolver.refresh(
+            this.runId,
+            this.yieldedTo ?? 'another invocation',
+            this.wiring.clock(),
+          )
+      this.syncModelControls()
+      this.paint()
+    }
     if (!claimed) {
       if (this.repoLeaseHolder !== undefined) {
         // We held the lease and the store says it is gone: a peer took over.
@@ -3474,10 +3502,17 @@ class DispatchLoop {
     const holder = this.runId
     if (!(await this.ensureRepoLease(holder))) {
       await this.recordTickYielded()
-      const peer = (await this.wiring.store.getRepo(this.repoIdentity).catch(() => null))?.lease
-        ?.holder
-      const message = `tick yielded: repository held by ${peer ?? 'another invocation'}`
-      this.warn(message)
+      if ((!this.dashboard && this.opts.kernelRunId === undefined) || this.opts.once) {
+        const peer = (await this.wiring.store.getRepo(this.repoIdentity).catch(() => null))?.lease
+          ?.holder
+        const liveness = await readDispatcherLiveness(
+          this.wiring.store,
+          this.repoIdentity,
+          this.wiring.clock(),
+        ).catch(() => null)
+        const message = `tick yielded: repository held by ${liveness?.dispatcher.holder ? formatDispatcherIdentity(liveness.dispatcher.holder) : (peer ?? 'another invocation')}`
+        this.warn(message)
+      }
       if (this.opts.once === true) return
       // Watch mode stays alive and retries the claim at every tick.
     }
@@ -3498,7 +3533,17 @@ class DispatchLoop {
         // lease and any provisioning state are released for the next
         // invocation.
         if (this.pastDeadline()) return
-        const initial = await this.dispatcherTick(true)
+        const initial = await this.dispatcherTick(true).catch(async (error: unknown) => {
+          await this.appendStatus({
+            actor: DISPATCHER,
+            type: 'dispatcher.tick-failed',
+            payload: {
+              run: this.runId,
+              error: error instanceof Error ? error.message : String(error),
+            },
+          })
+          throw error
+        })
         const initialPrinted = this.printReport(initial, false)
         await this.drainInFlight()
         const settledReports = await this.publishSettlementReports()
@@ -3963,6 +4008,7 @@ export async function abDispatch(opts: DispatchOpts): Promise<void> {
         payload: {
           run: runId,
           pid: process.pid,
+          host: hostname(),
           effectiveConfig: { kind: artifact.kind, rev: artifact.revision },
           roleWarnings: roleKeyWarnings(config),
         },
@@ -3973,6 +4019,17 @@ export async function abDispatch(opts: DispatchOpts): Promise<void> {
   const loop = new DispatchLoop(liveConfig, wiring, resolvedOpts, view)
   try {
     await loop.run()
+    const stopRequest = (
+      resolvedOpts.kernelRunId && resolvedOpts.signal?.aborted
+        ? await wiring.store.getRepoEvents(state.repo)
+        : []
+    ).find((event) => event.type === 'dispatcher.stop-requested' && event.payload.run === runId)
+    const stoppedSignal =
+      stopRequest?.type === 'dispatcher.stop-requested'
+        ? stopRequest.payload.signal
+        : typeof resolvedOpts.signal?.reason === 'string'
+          ? resolvedOpts.signal.reason
+          : undefined
     await view.appendRepo(state.repo, {
       actor: DISPATCHER,
       type: 'dispatcher.run-stopped',
@@ -3980,6 +4037,7 @@ export async function abDispatch(opts: DispatchOpts): Promise<void> {
         run: runId,
         outcome: 'normal',
         exitCode: 0,
+        ...(stoppedSignal === undefined ? {} : { signal: stoppedSignal }),
         ...(loop.supersededByPeer ? { reason: 'superseded' as const } : {}),
       },
     })

package/packages/core/src/cli/dispatcher-liveness.ts

@@ -0,0 +1,91 @@
+import type { RepositoryEvent } from '../events/repository'
+import type { BuildStore, RepositoryRecord } from '../store/types'
+import { formatDispatcherIdentity, projectDispatcherLiveness } from '../kernel/dispatcher-liveness'
+
+/** Bounded first; a later yielding invocation can hide the owner's identity or
+ * an earlier stop. Only this correlation query falls back to full history. */
+export async function readDispatcherLiveness(
+  store: BuildStore,
+  repo: string,
+  now = new Date(),
+  subset?: RepositoryEvent[],
+) {
+  const record = await store.getRepo(repo)
+  let events = record === null ? [] : (subset ?? (await store.getRepoStateEvents(repo)))
+  const bounded = projectDispatcherLiveness(record, events, now)
+  if (
+    record &&
+    ((record.lease &&
+      !events.some(
+        (e) => e.type === 'dispatcher.run-started' && e.payload.run === record.lease!.holder,
+      )) ||
+      (bounded.state !== 'held' &&
+        events.some((e) => e.type === 'dispatcher.run-started') &&
+        !events.some((e) => e.type === 'dispatcher.run-stopped')))
+  ) {
+    events = await store.getRepoEvents(repo)
+  }
+  return { record, events, dispatcher: projectDispatcherLiveness(record, events, now) }
+}
+
+/** Invocation-local identity cache. Only start facts survive refreshes; ages
+ * always come from the current lease record and clock. Failed correlations are
+ * negatively cached so a standby cannot repeatedly replay a large journal. */
+export class StandbyIdentityResolver {
+  private readonly starts = new Map<string, RepositoryEvent>()
+  private readonly attempted = new Set<string>()
+  private readonly pending = new Map<string, Promise<void>>()
+
+  constructor(
+    private readonly store: BuildStore,
+    private readonly repo: string,
+  ) {}
+
+  ingest(events: readonly RepositoryEvent[]): void {
+    for (const event of events) {
+      if (event.type === 'dispatcher.run-started') this.starts.set(event.payload.run, event)
+    }
+  }
+
+  private async correlate(holder: string): Promise<void> {
+    const pending = this.pending.get(holder)
+    if (pending !== undefined) return pending
+    if (this.starts.has(holder) || this.attempted.has(holder)) return
+    this.attempted.add(holder)
+    const lookup = (async () => {
+      try {
+        this.ingest(await this.store.getRepoStateEvents(this.repo))
+        if (!this.starts.has(holder)) this.ingest(await this.store.getRepoEvents(this.repo, 0))
+      } catch {
+        // Identity is best effort; the sampled holder id remains sufficient.
+      }
+    })()
+    this.pending.set(holder, lookup)
+    try {
+      await lookup
+    } finally {
+      this.pending.delete(holder)
+    }
+  }
+
+  async refresh(
+    run: string,
+    yieldedHolder: string | undefined,
+    now: Date,
+  ): Promise<string | undefined> {
+    let record: RepositoryRecord | null
+    try {
+      record = await this.store.getRepo(this.repo)
+    } catch {
+      return yieldedHolder
+    }
+    const holder = record?.lease?.holder
+    if (holder === run) return undefined
+    if (holder === undefined) return yieldedHolder
+    await this.correlate(holder)
+    if (!this.starts.has(holder)) return holder
+    const projection = projectDispatcherLiveness(record, [...this.starts.values()], now)
+    const identity = projection.holder ?? projection.expiredHolder
+    return identity?.run === holder ? formatDispatcherIdentity(identity) : holder
+  }
+}

package/packages/core/src/cli/help.ts

@@ -282,7 +282,7 @@ export const HELP_CATALOG: readonly HelpEntry[] = [
       '  ab watch [<slug>...] [--repository] [--event <glob>]... [--since <cursor>] [--timeout <duration>] [--interval <duration>] [--count <n>] [--json] [--store <ref>]',
       '',
       'Follow the event logs of one, several, or all active builds in this repository —',
-      'optionally the repository journal — and emit one record per matching event as it',
+      'the repository journal by default with no slugs — and emit one record per matching event as it',
       'lands. With no slug, every nonterminal build (queued, running, paused, blocked) is',
       'watched, and a build created while the watch runs joins it without a restart.',
       'Naming one or more slugs watches exactly those builds; an unknown slug is an error',
@@ -291,16 +291,18 @@ export const HELP_CATALOG: readonly HelpEntry[] = [
       '',
       'Options:',
       '  --repository       Also follow the repository journal (dispatcher and harvest',
-      '                     events). Without it, repository events are never emitted.',
+      '                     events) for named watches. No-slug watches include it by default.',
       '  --event <glob>     Repeatable glob over event type names: * matches any run of',
       '                     characters, ? one character (escalation.*, pr.merged, *). Every',
       '                     glob must match at least one type in the build catalog — or,',
-      '                     with --repository, the repository catalog — or the command fails',
+      '                     with no slugs or --repository, the repository catalog — or the command fails',
       '                     before any store read. Any --event replaces the default filter.',
       '  --since <cursor>   Resume from a cursor emitted by an earlier record; every matching',
       '                     event after that cursor is delivered exactly once, including',
       '                     events that landed while no watch was running. A cursor from a',
-      '                     different repository or store is rejected.',
+      '                     different repository or store is rejected. A legacy cursor without',
+      '                     a repository position baselines that stream now; reconcile earlier',
+      '                     source conditions with ab repository status.',
       '  --timeout <dur>    End the watch after this duration (30, 45s, 5m, 1h). Default',
       '                     30 minutes; 0 means unbounded.',
       '  --interval <dur>   Poll cadence. Default: 5s for an http(s) store, 1s for a local store.',
@@ -313,7 +315,8 @@ export const HELP_CATALOG: readonly HelpEntry[] = [
       'infrastructure.failed, runner.setup-failed, dispatch.failed, publication.lost,',
       'finalize.completed, pr.conflicted, pr.merged, pr.closed, build.completed,',
       'build.aborted. For the repository journal: harvest.escalated, harvest.failed,',
-      'dispatcher.tick-failed, dispatcher.config-rejected, dispatcher.harvest-runner-failed.',
+      'dispatcher.tick-failed, dispatcher.config-rejected, dispatcher.harvest-runner-failed,',
+      'dispatcher.ticket-source-degraded, dispatcher.ticket-source-recovered.',
       '',
       'Record shape (--json): build (slug, or null for a repository event), repo (repository',
       'id, present only for repository events), event (the complete durable envelope: build,',

package/packages/core/src/cli/repository-status.ts

@@ -1,3 +1,13 @@
+import {
+  formatDispatcherIdentity,
+  projectDispatcherLiveness,
+  type DispatcherLiveness,
+} from '../kernel/dispatcher-liveness'
+import { readDispatcherLiveness } from './dispatcher-liveness'
+import {
+  reduceTicketSourceHealth,
+  type TicketSourceDegradation,
+} from '../kernel/ticket-source-health'
 import type { RepositoryEvent } from '../events/repository'
 import { reduceDispatchSettings, type DispatchSettings } from '../kernel/dispatch-settings'
 import { sandboxStates } from '../processes/sandbox-state'
@@ -7,6 +17,8 @@ import { withSessionlessStore, type StoreOpener } from './store-opening'
 /** Script-facing projection returned by `ab repository status`. */
 export interface RepositoryStatus extends DispatchSettings {
   repo: string
+  dispatcher: DispatcherLiveness
+  ticketSourceDegradation?: TicketSourceDegradation
   /** Live operator sandboxes (AUT-340); released environments are omitted. */
   sandboxes: Array<{
     operator: string
@@ -38,10 +50,15 @@ export interface RepositoryStatusOpts {
   json?: boolean
   /** Explicit --store reference; precedence is owned by repo-state.ts. */
   storeRef?: string
+  now?: () => Date
   openStore?: StoreOpener
 }
 
-export function projectRepositoryStatus(repo: string, events: RepositoryEvent[]): RepositoryStatus {
+export function projectRepositoryStatus(
+  repo: string,
+  events: RepositoryEvent[],
+  dispatcher = projectDispatcherLiveness(null, [], new Date()),
+): RepositoryStatus {
   const publications: RepositoryStatus['publications'] = []
   for (const event of events) {
     if (event.type !== 'orchestrator.sandbox.published') continue
@@ -63,8 +80,11 @@ export function projectRepositoryStatus(repo: string, events: RepositoryEvent[])
     })
   }
   publications.reverse()
+  const active = reduceTicketSourceHealth(events)
   return {
+    ...(active === undefined ? {} : { ticketSourceDegradation: active }),
     repo,
+    dispatcher,
     ...reduceDispatchSettings(events),
     sandboxes: sandboxStates(events)
       .filter((state) => state.state !== 'released')
@@ -81,8 +101,30 @@ export function projectRepositoryStatus(repo: string, events: RepositoryEvent[])
 
 export function renderRepositoryStatus(status: RepositoryStatus): string[] {
   const setting = (enabled: boolean): string => (enabled ? 'ON' : 'OFF')
+  const active = status.ticketSourceDegradation
   return [
+    ...(active === undefined
+      ? []
+      : [
+          `ticket source ${active.source}: ${active.cause} (${active.operation}) — ${active.message}${'resetAt' in active ? `; reset ${active.resetAt}` : ''}`,
+        ]),
     `repository: ${status.repo}`,
+    `dispatcher: ${status.dispatcher.state}${status.dispatcher.holder ? ` — ${formatDispatcherIdentity(status.dispatcher.holder)}` : ' — no dispatcher holds the repository'}`,
+    ...(status.dispatcher.expiredHolder
+      ? [`expired dispatcher: ${formatDispatcherIdentity(status.dispatcher.expiredHolder)}`]
+      : []),
+    ...[status.dispatcher.holder ?? status.dispatcher.expiredHolder].flatMap((run) =>
+      run
+        ? [
+            `last tick: ${run.lastTickAt ?? 'unknown'} (${run.lastTickAgeMs ?? 'unknown'}ms ago); ${JSON.stringify(run.lastTick)}`,
+          ]
+        : [],
+    ),
+    ...(status.dispatcher.latestStoppedRun
+      ? [
+          `last dispatcher stop: ${status.dispatcher.latestStoppedRun.stoppedAt}; ${status.dispatcher.latestStoppedRun.stop?.outcome}; signal ${status.dispatcher.latestStoppedRun.stop?.signal ?? 'none'}`,
+        ]
+      : []),
     `intake: ${setting(status.intake)}`,
     `repository pause: ${setting(status.paused)}`,
     `default auto-merge: ${setting(status.defaultAutoMerge)}`,
@@ -105,10 +147,8 @@ export async function abRepositoryStatus(opts: RepositoryStatusOpts): Promise<vo
   await withSessionlessStore(opts, async ({ store, repo }) => {
     // A fresh store has no repository row. Do not call ensureRepo: this query's
     // empty-stream defaults must remain a genuinely read-only operation.
-    const record = await store.getRepo(repo)
-    // Bounded read (AUT-489): projectRepositoryStatus reduces durable types only.
-    const events = record === null ? [] : await store.getRepoStateEvents(repo)
-    const status = projectRepositoryStatus(repo, events)
+    const { events, dispatcher } = await readDispatcherLiveness(store, repo, opts.now?.())
+    const status = projectRepositoryStatus(repo, events, dispatcher)
     if (opts.json === true) {
       opts.stdout(JSON.stringify(status, null, 2))
       return