@defrex/autobuild 0.8.0 → 0.9.0

package/packages/core/src/processes/harvest-runner.ts

@@ -46,6 +46,7 @@ import { defaultProposalState } from './dispatcher'
 import {
   artifactRef,
   evaluateHarvestPressure,
+  evaluateHarvestPressureFromDigests,
   HARVEST_REPORT_ARTIFACT,
   HARVEST_SCAN_ARTIFACT,
   HARVEST_TRANSCRIPT_ARTIFACT,
@@ -230,7 +231,9 @@ export class HarvestRunner {
     try {
       await this.startHeartbeat()
       await this.ensureLease()
-      let events = await store.getRepoEvents(repo)
+      // Bounded read (AUT-489): reduceHarvest and the verdict/failure filters
+      // below consume durable types only, so the subset is replay-equivalent.
+      let events = await store.getRepoStateEvents(repo)
       let state = reduceHarvest(events)
       run = actionableHarvestRun(state)
 
@@ -238,19 +241,48 @@ export class HarvestRunner {
       // selected run comes from the full journal, so a shadowed failure or
       // exhaustion barrier is recovered/reported before any later open run.
       await this.controlBoundary(run?.run)
-      events = await store.getRepoEvents(repo)
+      events = await store.getRepoStateEvents(repo)
       state = reduceHarvest(events)
       run = openHarvestRun(state)
 
       if (!run) {
         await this.controlBoundary()
-        const scan = await scanUnclaimedObservations(store, repo)
-        // Scanning may span many build streams. Treat its completion as a
-        // boundary even when the threshold is not met, so a request arriving
-        // during the read is acknowledged before this runner returns idle.
+        // Flat-cost gate (AUT-521): the pressure trigger is evaluated from
+        // the journal and one repo-scoped digest batch read — no per-build
+        // history reads — so the idle evaluation no longer grows with
+        // accumulated finished builds. Only a tripped gate pays for the full
+        // unclaimed-observation scan, because the packet artifact needs the
+        // observation payloads no bounded source carries.
+        // Bounded read (AUT-489): the gate's journal input is consumed only
+        // through reduceHarvest, so the subset is replay-equivalent here too.
+        const [gateEvents, digests] = await Promise.all([
+          store.getRepoStateEvents(repo),
+          store.getRepoBuildDigests(repo),
+        ])
+        const pressure = evaluateHarvestPressureFromDigests({
+          digests,
+          harvestEvents: gateEvents,
+          policy: this.currentConfig().policy,
+        })
+        if (pressure.trigger === undefined) {
+          // Treat the read's completion as a boundary even when the gate did
+          // not trip, so a request arriving during the read is acknowledged
+          // before this runner returns idle.
+          await this.controlBoundary()
+          return { outcome: 'idle' }
+        }
         await this.controlBoundary()
-        const pressure = evaluateHarvestPressure(scan, this.currentConfig().policy)
-        if (pressure.trigger === undefined) return { outcome: 'idle' }
+        const scan = await scanUnclaimedObservations(store, repo)
+        // The trigger came from an earlier read than this scan, and a claim
+        // may have landed in the window. Re-confirm the gate against the
+        // fresh scan (pure, free): an emptied window returns idle exactly as
+        // if the gate had not tripped, so a packet is only ever built from a
+        // scan that itself triggered.
+        const confirmed = evaluateHarvestPressure(scan, this.currentConfig().policy)
+        if (confirmed.trigger === undefined) {
+          await this.controlBoundary()
+          return { outcome: 'idle' }
+        }
         const runId = this.deps.ids('harvest')
         const packet = await makeHarvestScanPacket({
           store,
@@ -280,14 +312,15 @@ export class HarvestRunner {
               run: runId,
               observations: packet.observations.map((item) => item.occurrence),
               scan: artifactRef(deposited[0]!),
-              trigger: pressure.trigger,
+              trigger: confirmed.trigger,
               ...(this.deps.environment !== undefined
                 ? { environment: this.deps.environment }
                 : {}),
             },
           }),
         )
-        events = await store.getRepoEvents(repo)
+        // Bounded read (AUT-489): reduceHarvest consumes durable types only.
+        events = await store.getRepoStateEvents(repo)
         run = reduceHarvest(events).runs.find((candidate) => candidate.run === runId)
         if (!run) throw new Error(`harvest.started did not reduce run "${runId}"`)
         // A request racing the claim parks this exact run; its snapshot stays
@@ -448,7 +481,9 @@ export class HarvestRunner {
             existing.verdict,
             existing.artifact,
           )
-          const event = (await this.deps.store.getRepoEvents(this.deps.repo)).find(
+          // Bounded read (AUT-489): only harvest.review.verdict facts are
+          // filtered here, and those are durable.
+          const event = (await this.deps.store.getRepoStateEvents(this.deps.repo)).find(
             (
               candidate,
             ): candidate is Extract<RepositoryEvent, { type: 'harvest.review.verdict' }> =>
@@ -515,7 +550,9 @@ export class HarvestRunner {
         event.actor.kind === 'agent' &&
         event.actor.session === session,
     })
-    const events = await this.deps.store.getRepoEvents(this.deps.repo)
+    // Bounded read (AUT-489): only harvest.review.verdict facts are filtered
+    // here, and those are durable.
+    const events = await this.deps.store.getRepoStateEvents(this.deps.repo)
     const verdict = [...events]
       .reverse()
       .find(
@@ -565,7 +602,9 @@ export class HarvestRunner {
   ): Promise<void> {
     const { store, repo, ids, workspacePath } = this.deps
     await this.ensureLease()
-    const events = await store.getRepoEvents(repo)
+    // Bounded read (AUT-489): reduceHarvest and the harvest.failed filter
+    // consume durable types only.
+    const events = await store.getRepoStateEvents(repo)
     const matchingFailures = events.filter(
       (event): event is Extract<RepositoryEvent, { type: 'harvest.failed' }> =>
         event.type === 'harvest.failed' &&
@@ -823,7 +862,9 @@ export class HarvestRunner {
   }): Promise<void> {
     const { store, repo, ids, workspacePath } = this.deps
     await this.ensureLease()
-    const events = await store.getRepoEvents(repo)
+    // Bounded read (AUT-489): reduceHarvest and the harvest.failed filter
+    // consume durable types only.
+    const events = await store.getRepoStateEvents(repo)
     const matchingFailures = events.filter(
       (event): event is Extract<RepositoryEvent, { type: 'harvest.failed' }> =>
         event.type === 'harvest.failed' &&
@@ -1296,7 +1337,9 @@ export class HarvestRunner {
 
   private async recordWorkflowFailure(run: string, error: unknown): Promise<void> {
     await this.ensureLease()
-    const events = await this.deps.store.getRepoEvents(this.deps.repo)
+    // Bounded read (AUT-489): reduceHarvest and the harvest.failed filter
+    // consume durable types only.
+    const events = await this.deps.store.getRepoStateEvents(this.deps.repo)
     const state = reduceHarvest(events)
     const current = state.runs.find((candidate) => candidate.run === run)
     if (current?.status !== 'running') return
@@ -1349,7 +1392,8 @@ export class HarvestRunner {
    * outer budget and therefore remain recoverable. */
   private async settleRecoveryExhaustion(run: string): Promise<void> {
     await this.ensureLease()
-    const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
+    // Bounded read (AUT-489): reduceHarvest consumes durable types only.
+    const state = reduceHarvest(await this.deps.store.getRepoStateEvents(this.deps.repo))
     const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
     if (decision.kind !== 'exhaust-recovery' || decision.run !== run) return
     await this.finalizeRecoveryExhaustion(decision.run, decision.attempts, decision.limit)
@@ -1374,7 +1418,8 @@ export class HarvestRunner {
     })
 
     await this.ensureLease()
-    const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
+    // Bounded read (AUT-489): reduceHarvest consumes durable types only.
+    const state = reduceHarvest(await this.deps.store.getRepoStateEvents(this.deps.repo))
     const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
     if (
       decision.kind !== 'exhaust-recovery' ||
@@ -1410,7 +1455,8 @@ export class HarvestRunner {
   private async controlBoundary(run?: string): Promise<void> {
     while (true) {
       await this.ensureLease()
-      const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
+      // Bounded read (AUT-489): reduceHarvest consumes durable types only.
+      const state = reduceHarvest(await this.deps.store.getRepoStateEvents(this.deps.repo))
       const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
       if (decision.kind === 'proceed') return
       if (decision.kind === 'park') {
@@ -1453,7 +1499,8 @@ export class HarvestRunner {
   }
 
   private async refreshRun(run: string): Promise<HarvestRunState> {
-    const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
+    // Bounded read (AUT-489): reduceHarvest consumes durable types only.
+    const state = reduceHarvest(await this.deps.store.getRepoStateEvents(this.deps.repo))
     const found = state.runs.find((candidate) => candidate.run === run)
     if (!found) throw new Error(`unknown harvest run "${run}"`)
     return found

package/packages/core/src/processes/harvest.ts

@@ -1,5 +1,6 @@
 /** Deterministic halves of observation harvest: scan, dedup, validation, and
  * proposal rendering. Agent judgment is deliberately absent from this file. */
+import type { AbEvent } from '../events/catalog'
 import type { ArtifactRef } from '../ontology'
 import {
   harvestProposalSetSchema,
@@ -26,7 +27,14 @@ import {
 } from '../kernel/harvest'
 import type { TicketSource } from '../ports/types'
 import { specConformance } from '../spec-standard'
-import { contentHash, toBytes, type BuildStore } from '../store/types'
+import type { RepositoryEvent } from '../events/repository'
+import {
+  contentHash,
+  toBytes,
+  type BuildDigest,
+  type BuildRecord,
+  type BuildStore,
+} from '../store/types'
 
 export const HARVEST_SCAN_ARTIFACT = 'harvest-scan'
 export const HARVEST_PROPOSALS_ARTIFACT = 'harvest-proposals'
@@ -52,21 +60,33 @@ export interface HarvestPressure {
   trigger?: HarvestTrigger
 }
 
-/** Pure two-dimensional Harvest gate. Drift is measured from the oldest
- * unclaimed observation and excludes that observation's own build. */
-export function evaluateHarvestPressure(
-  scan: Pick<HarvestScanResult, 'observations' | 'merges'>,
+/** The build- and occurrence-shaped facts both pressure evaluations project
+ * onto: the unclaimed occurrences with their event timestamps, and the merge
+ * facts. Keeping the oldest-observation selection and drift arithmetic in one
+ * place makes the scan-based and digest-based evaluations equal by
+ * construction rather than by parallel maintenance (AUT-521). */
+interface HarvestPressureFacts {
+  unclaimed: { build: string; seq: number; ts: string }[]
+  merges: { build: string; ts: string }[]
+}
+
+function evaluateHarvestPressureFacts(
+  facts: HarvestPressureFacts,
   policy: { harvestThreshold: number; harvestMaxDrift: number },
 ): HarvestPressure {
-  const observationCount = scan.observations.length
+  const observationCount = facts.unclaimed.length
   if (observationCount === 0) return { observationCount, drift: 0 }
 
-  const oldest = scan.observations.reduce((candidate, observation) =>
-    observation.ts < candidate.ts ? observation : candidate,
-  )
+  // The oldest unclaimed occurrence: min ts, then build ascending, then seq
+  // ascending — exactly what the scan projection's sort-by-(build,seq) then
+  // strict-`<`-reduce produced, now stated as one comparator.
+  const oldest = [...facts.unclaimed].sort(
+    (a, b) =>
+      (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0) || a.build.localeCompare(b.build) || a.seq - b.seq,
+  )[0]!
   const mergedBuilds = new Set<string>()
-  for (const merge of scan.merges) {
-    if (merge.build === oldest.occurrence.build || merge.ts <= oldest.ts) continue
+  for (const merge of facts.merges) {
+    if (merge.build === oldest.build || merge.ts <= oldest.ts) continue
     mergedBuilds.add(merge.build)
   }
   const drift = mergedBuilds.size
@@ -87,23 +107,94 @@ export function evaluateHarvestPressure(
   }
 }
 
-/** Raw structured `observation.recorded` envelopes across this repository.
- * The pair `{build, seq}` is the occurrence key; payload ids are not assumed
- * globally unique. */
-export async function scanUnclaimedObservations(
-  store: BuildStore,
-  repo: string,
-): Promise<HarvestScanResult> {
-  await store.ensureRepo(repo)
-  const harvestEvents = await store.getRepoEvents(repo)
-  const state = reduceHarvest(harvestEvents)
+/** Pure two-dimensional Harvest gate. Drift is measured from the oldest
+ * unclaimed observation and excludes that observation's own build. */
+export function evaluateHarvestPressure(
+  scan: Pick<HarvestScanResult, 'observations' | 'merges'>,
+  policy: { harvestThreshold: number; harvestMaxDrift: number },
+): HarvestPressure {
+  return evaluateHarvestPressureFacts(
+    {
+      unclaimed: scan.observations.map((observation) => ({
+        build: observation.occurrence.build,
+        seq: observation.occurrence.seq,
+        ts: observation.ts,
+      })),
+      merges: scan.merges,
+    },
+    policy,
+  )
+}
+
+/** The same gate from build digests and the repository journal alone
+ * (AUT-521): claimedness comes from the journal, unclaimed occurrences and
+ * merge facts from the digests — no per-build history reads. The merge facts
+ * use each digest's latest `pr.merged` ts; drift counts a build when any of
+ * its merges is strictly after the oldest unclaimed observation, and
+ * `max(merge.ts) > oldest.ts` is equivalent to `any(merge.ts) > oldest.ts`,
+ * so latest-wins carries exactly the drift fact the scan collects. */
+export function evaluateHarvestPressureFromDigests(input: {
+  digests: Map<string, BuildDigest>
+  harvestEvents: RepositoryEvent[]
+  policy: { harvestThreshold: number; harvestMaxDrift: number }
+}): HarvestPressure {
+  const claimed = claimedOccurrenceKeys(reduceHarvest(input.harvestEvents))
+  const unclaimed: HarvestPressureFacts['unclaimed'] = []
+  const merges: HarvestPressureFacts['merges'] = []
+  for (const digest of input.digests.values()) {
+    for (const observation of digest.observations) {
+      if (claimed.has(occurrenceKey({ build: digest.slug, seq: observation.seq }))) continue
+      unclaimed.push({ build: digest.slug, seq: observation.seq, ts: observation.ts })
+    }
+    if (digest.merged !== undefined) merges.push({ build: digest.slug, ts: digest.merged })
+  }
+  return evaluateHarvestPressureFacts({ unclaimed, merges }, input.policy)
+}
+
+/** The dispatcher harvest-launch gate's read sequence as one named seam
+ * (AUT-521): `ensureRepo`, one repo-scoped digest batch read, and the pure
+ * digest evaluation with the caller's policy. Per-evaluation cost is one
+ * journal read (passed in by the caller, which already holds it) plus one
+ * bounded digest read — flat in the number of accumulated finished builds,
+ * where the previous scan read every build's full history per evaluation. */
+export async function evaluateHarvestPressureFromStore(input: {
+  store: BuildStore
+  repo: string
+  harvestEvents: RepositoryEvent[]
+  policy: { harvestThreshold: number; harvestMaxDrift: number }
+}): Promise<HarvestPressure> {
+  await input.store.ensureRepo(input.repo)
+  const digests = await input.store.getRepoBuildDigests(input.repo)
+  return evaluateHarvestPressureFromDigests({
+    digests,
+    harvestEvents: input.harvestEvents,
+    policy: input.policy,
+  })
+}
+
+/** The deterministic core of the unclaimed-observation scan, over
+ * already-loaded streams: exactly the reduce/claim/collect/sort the
+ * store-reading scan performs, with no store calls. The dashboard consumes
+ * this directly with the journal and per-build event arrays it has already
+ * fetched (AUT-486), while `scanUnclaimedObservations` keeps the
+ * ensure/read/delegate shape its other callers depend on. */
+export function collectUnclaimedObservations(input: {
+  repo: string
+  records: BuildRecord[]
+  eventsByBuild: Map<string, AbEvent[]>
+  harvestEvents: RepositoryEvent[]
+}): HarvestScanResult {
+  const state = reduceHarvest(input.harvestEvents)
   const claimed = claimedOccurrenceKeys(state)
   const observations: HarvestObservation[] = []
   const merges: HarvestMerge[] = []
 
-  for (const record of await store.listBuilds()) {
-    if (record.repo !== repo) continue
-    const events = await store.getEvents(record.slug)
+  for (const record of input.records) {
+    if (record.repo !== input.repo) continue
+    const events = input.eventsByBuild.get(record.slug)
+    if (events === undefined) {
+      throw new Error(`eventsByBuild is missing an entry for build "${record.slug}"`)
+    }
     for (const event of events) {
       if (event.type === 'pr.merged') {
         merges.push({ build: record.slug, ts: event.ts })
@@ -132,6 +223,87 @@ export async function scanUnclaimedObservations(
   return { observations, merges, state }
 }
 
+/** The unclaimed-observation count from build digests and the repository
+ * journal alone (AUT-487): the same reduce/claim/count the store-reading scan
+ * performs, with no per-build history reads. The operator query consumes this
+ * directly, and the terminal dashboards consume it through
+ * `sampleUnclaimedObservationCount` (AUT-524), while `scanUnclaimedObservations`
+ * keeps the full-scan shape its remaining callers depend on (the harvest
+ * runner's packet scan — the only source of observation payloads — and its
+ * gate re-confirmation; the dispatcher's harvest-launch gate itself evaluates
+ * from digests, AUT-521). Occurrences are keyed `{build, seq}`; payload ids
+ * are not assumed globally unique. */
+export function unclaimedObservationCount(input: {
+  digests: Map<string, BuildDigest>
+  harvestEvents: RepositoryEvent[]
+}): number {
+  const claimed = claimedOccurrenceKeys(reduceHarvest(input.harvestEvents))
+  let count = 0
+  for (const digest of input.digests.values()) {
+    for (const observation of digest.observations) {
+      if (!claimed.has(occurrenceKey({ build: digest.slug, seq: observation.seq }))) count += 1
+    }
+  }
+  return count
+}
+
+/** The shared missing-record journal read (AUT-524): a repository whose
+ * journal record does not yet exist has an empty journal by definition, so
+ * the probe answers `[]` instead of letting `getRepoStateEvents` reject. The
+ * read writes nothing — a display path must not `ensureRepo` — while
+ * `scanUnclaimedObservations` keeps its materializing `ensureRepo` for its
+ * remaining callers.
+ *
+ * The journal read is bounded (AUT-489): every consumer — the terminal
+ * dashboards' observation-pressure sample, the operator query
+ * (operator/query.ts), the dispatcher's repository-wide controls and ready
+ * scans, and the operator sandbox service's journal reads — reduces durable
+ * event types plus the latest-run tail only, so the subset is
+ * replay-equivalent to a full journal replay. */
+export async function readRepoEventsIfRecorded(
+  store: BuildStore,
+  repo: string,
+): Promise<RepositoryEvent[]> {
+  return (await store.getRepo(repo)) === null ? [] : store.getRepoStateEvents(repo)
+}
+
+/** The terminal dashboards' observation-pressure sample (AUT-487): build
+ * digests plus the repository journal, reduced to the unclaimed-observation
+ * count. Store traffic is flat in the finished-build count — one journal read
+ * plus one repo-scoped digest read, plus the journal-record probe inside
+ * `readRepoEventsIfRecorded`. Its missing-record treatment (AUT-524) is the
+ * shared helper's, above. */
+export async function sampleUnclaimedObservationCount(
+  store: BuildStore,
+  repo: string,
+): Promise<number> {
+  const [digests, harvestEvents] = await Promise.all([
+    store.getRepoBuildDigests(repo),
+    readRepoEventsIfRecorded(store, repo),
+  ])
+  return unclaimedObservationCount({ digests, harvestEvents })
+}
+
+/** Raw structured `observation.recorded` envelopes across this repository.
+ * The pair `{build, seq}` is the occurrence key; payload ids are not assumed
+ * globally unique. */
+export async function scanUnclaimedObservations(
+  store: BuildStore,
+  repo: string,
+): Promise<HarvestScanResult> {
+  await store.ensureRepo(repo)
+  // Bounded read (AUT-489): only harvest facts are consumed here; the
+  // per-build history reads stay as AUT-487 left them.
+  const harvestEvents = await store.getRepoStateEvents(repo)
+  const records = await store.listBuilds()
+  const eventsByBuild = new Map<string, AbEvent[]>()
+  for (const record of records) {
+    if (record.repo !== repo) continue
+    eventsByBuild.set(record.slug, await store.getEvents(record.slug))
+  }
+  return collectUnclaimedObservations({ repo, records, eventsByBuild, harvestEvents })
+}
+
 /** Distinct previously filed/joined proposal tickets, reconciled through the
  * TicketSource's native lifecycle semantics. Resolved and missing entries stay
  * in the packet as tombstones rather than disappearing and being re-filed. */