@defrex/autobuild 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +192 -0
- package/SPEC.md +2269 -0
- package/bin/ab-build-runner.ts +47 -0
- package/bin/ab-dev.ts +27 -0
- package/bin/ab-dispatch-kernel.ts +92 -0
- package/bin/ab-harvest-runner.ts +29 -0
- package/bin/ab-init-probe.ts +19 -0
- package/bin/ab-process-group-reaper.ts +32 -0
- package/bin/ab.ts +6 -0
- package/bin/agent/ab +3 -0
- package/docs/answering-blocked-builds.md +89 -0
- package/docs/architecture.md +726 -0
- package/docs/assets/headline-wide.png +0 -0
- package/docs/configuration.md +1623 -0
- package/docs/hosted-dispatcher-cutover-validation.md +178 -0
- package/docs/hosted-dispatcher.md +204 -0
- package/docs/mcp.md +119 -0
- package/docs/operator-api.md +186 -0
- package/docs/remote-store-protocol.md +972 -0
- package/docs/remote-ticket-protocol.md +79 -0
- package/docs/setup.md +289 -0
- package/docs/spec-standard.md +63 -0
- package/docs/ticket-dependencies.md +59 -0
- package/docs/vercel-sandbox-rollout-probe.md +7 -0
- package/package.json +135 -0
- package/packages/core/src/cli/args.ts +74 -0
- package/packages/core/src/cli/artifact.ts +182 -0
- package/packages/core/src/cli/binary.ts +198 -0
- package/packages/core/src/cli/build-control.ts +789 -0
- package/packages/core/src/cli/build-progress.ts +55 -0
- package/packages/core/src/cli/bulk-control.ts +385 -0
- package/packages/core/src/cli/context.ts +470 -0
- package/packages/core/src/cli/dashboard/actions.ts +59 -0
- package/packages/core/src/cli/dashboard/cells.ts +92 -0
- package/packages/core/src/cli/dashboard/composer.ts +230 -0
- package/packages/core/src/cli/dashboard/detail.ts +100 -0
- package/packages/core/src/cli/dashboard/frame-image.ts +373 -0
- package/packages/core/src/cli/dashboard/live.ts +152 -0
- package/packages/core/src/cli/dashboard/model.ts +1173 -0
- package/packages/core/src/cli/dashboard/poll.ts +196 -0
- package/packages/core/src/cli/dashboard/render.ts +1568 -0
- package/packages/core/src/cli/dashboard/selection.ts +66 -0
- package/packages/core/src/cli/dashboard/session-feed.ts +244 -0
- package/packages/core/src/cli/dashboard/session-view.ts +506 -0
- package/packages/core/src/cli/dashboard/transcript.ts +108 -0
- package/packages/core/src/cli/dev.ts +46 -0
- package/packages/core/src/cli/dispatch-frontend.ts +1045 -0
- package/packages/core/src/cli/dispatch-process.ts +259 -0
- package/packages/core/src/cli/dispatch.ts +3692 -0
- package/packages/core/src/cli/dotenv.ts +48 -0
- package/packages/core/src/cli/env.ts +206 -0
- package/packages/core/src/cli/harvest.ts +698 -0
- package/packages/core/src/cli/help.ts +713 -0
- package/packages/core/src/cli/init-agent.ts +106 -0
- package/packages/core/src/cli/init-validation.ts +583 -0
- package/packages/core/src/cli/init.ts +756 -0
- package/packages/core/src/cli/installation.ts +333 -0
- package/packages/core/src/cli/keyboard.ts +261 -0
- package/packages/core/src/cli/main.ts +1399 -0
- package/packages/core/src/cli/mcp.ts +279 -0
- package/packages/core/src/cli/models.ts +65 -0
- package/packages/core/src/cli/observe.ts +55 -0
- package/packages/core/src/cli/plugin.ts +227 -0
- package/packages/core/src/cli/pr-attachments.ts +138 -0
- package/packages/core/src/cli/pr-summary.ts +124 -0
- package/packages/core/src/cli/publication-settlement.ts +188 -0
- package/packages/core/src/cli/repo-state.ts +213 -0
- package/packages/core/src/cli/repository-status.ts +80 -0
- package/packages/core/src/cli/self-update.ts +443 -0
- package/packages/core/src/cli/status.ts +895 -0
- package/packages/core/src/cli/store-opening.contract.ts +289 -0
- package/packages/core/src/cli/store-opening.ts +167 -0
- package/packages/core/src/cli/store-ref.ts +33 -0
- package/packages/core/src/cli/terminal-restore.ts +174 -0
- package/packages/core/src/cli/terminal.ts +361 -0
- package/packages/core/src/cli/terminals.ts +995 -0
- package/packages/core/src/cli/testkit.ts +194 -0
- package/packages/core/src/cli/ticket.ts +484 -0
- package/packages/core/src/cli/upgrade-agent.ts +180 -0
- package/packages/core/src/cli/upgrade-commit.ts +468 -0
- package/packages/core/src/cli/upgrade-notice.ts +91 -0
- package/packages/core/src/cli/upgrade-progress.ts +110 -0
- package/packages/core/src/cli/upgrade.ts +888 -0
- package/packages/core/src/cli/wait.ts +587 -0
- package/packages/core/src/cli/watch.ts +932 -0
- package/packages/core/src/config/live.ts +255 -0
- package/packages/core/src/config/load.ts +98 -0
- package/packages/core/src/config/roles.ts +438 -0
- package/packages/core/src/config/schema.ts +774 -0
- package/packages/core/src/distribution.ts +24 -0
- package/packages/core/src/events/catalog.ts +201 -0
- package/packages/core/src/events/envelope.ts +59 -0
- package/packages/core/src/events/payloads.ts +618 -0
- package/packages/core/src/events/repository.ts +559 -0
- package/packages/core/src/events/sessions.ts +180 -0
- package/packages/core/src/harvest/schema.ts +147 -0
- package/packages/core/src/ids.ts +45 -0
- package/packages/core/src/integration/harness.ts +681 -0
- package/packages/core/src/kernel/auto-merge.ts +259 -0
- package/packages/core/src/kernel/converge.ts +117 -0
- package/packages/core/src/kernel/dispatch-settings.ts +60 -0
- package/packages/core/src/kernel/dispatch-status.ts +141 -0
- package/packages/core/src/kernel/engine.ts +1047 -0
- package/packages/core/src/kernel/harvest.ts +823 -0
- package/packages/core/src/kernel/origin.ts +69 -0
- package/packages/core/src/kernel/phases.ts +196 -0
- package/packages/core/src/kernel/plan-verify-selection.ts +105 -0
- package/packages/core/src/kernel/pr-attachments.ts +82 -0
- package/packages/core/src/kernel/reducer.ts +752 -0
- package/packages/core/src/kernel/stall.ts +155 -0
- package/packages/core/src/kernel/verify-applicability.ts +52 -0
- package/packages/core/src/markdown.ts +67 -0
- package/packages/core/src/ontology.ts +288 -0
- package/packages/core/src/open-map.ts +203 -0
- package/packages/core/src/operator/annotations.ts +187 -0
- package/packages/core/src/operator/client.ts +311 -0
- package/packages/core/src/operator/control.ts +117 -0
- package/packages/core/src/operator/index.ts +62 -0
- package/packages/core/src/operator/presentation.ts +22 -0
- package/packages/core/src/operator/protocol.ts +98 -0
- package/packages/core/src/operator/query.ts +201 -0
- package/packages/core/src/operator/registry.ts +1043 -0
- package/packages/core/src/operator/requests.ts +138 -0
- package/packages/core/src/operator/sandbox.ts +458 -0
- package/packages/core/src/operator/server.ts +616 -0
- package/packages/core/src/operator/sessions.ts +223 -0
- package/packages/core/src/operator/tickets.ts +177 -0
- package/packages/core/src/plugin-sdk/index.ts +174 -0
- package/packages/core/src/plugins/contract-entry.ts +74 -0
- package/packages/core/src/plugins/contract-env.ts +3 -0
- package/packages/core/src/plugins/load.ts +259 -0
- package/packages/core/src/plugins/manifest.ts +236 -0
- package/packages/core/src/plugins/registry.ts +274 -0
- package/packages/core/src/plugins/runtimes.ts +75 -0
- package/packages/core/src/ports/forge/contract.ts +258 -0
- package/packages/core/src/ports/forge/create.ts +74 -0
- package/packages/core/src/ports/forge/fake.ts +488 -0
- package/packages/core/src/ports/forge/github-pr-attachments.ts +351 -0
- package/packages/core/src/ports/forge/github-transport.ts +352 -0
- package/packages/core/src/ports/forge/github.ts +1162 -0
- package/packages/core/src/ports/forge/local-git.ts +568 -0
- package/packages/core/src/ports/runner/claude.ts +873 -0
- package/packages/core/src/ports/runner/cli-stream.ts +90 -0
- package/packages/core/src/ports/runner/codex.ts +873 -0
- package/packages/core/src/ports/runner/contract.ts +492 -0
- package/packages/core/src/ports/runner/fake.ts +213 -0
- package/packages/core/src/ports/runner/json-record.ts +2 -0
- package/packages/core/src/ports/runner/live-contract-fixture.ts +176 -0
- package/packages/core/src/ports/runner/one-shot.ts +29 -0
- package/packages/core/src/ports/runner/pi-bridge-core.ts +136 -0
- package/packages/core/src/ports/runner/pi-bridge.js +6 -0
- package/packages/core/src/ports/runner/pi-rpc.ts +454 -0
- package/packages/core/src/ports/runner/pi.ts +617 -0
- package/packages/core/src/ports/runner/production.ts +74 -0
- package/packages/core/src/ports/runner/provider-error.ts +121 -0
- package/packages/core/src/ports/runner/routing.ts +234 -0
- package/packages/core/src/ports/runner/runtime.ts +214 -0
- package/packages/core/src/ports/runner/session-env.ts +34 -0
- package/packages/core/src/ports/runner/stream-parts.ts +103 -0
- package/packages/core/src/ports/tickets/contract.ts +570 -0
- package/packages/core/src/ports/tickets/create.ts +135 -0
- package/packages/core/src/ports/tickets/fake.ts +215 -0
- package/packages/core/src/ports/tickets/file.ts +559 -0
- package/packages/core/src/ports/tickets/linear.ts +811 -0
- package/packages/core/src/ports/tickets/operations.ts +161 -0
- package/packages/core/src/ports/tickets/remote-index.ts +13 -0
- package/packages/core/src/ports/tickets/remote-protocol.ts +80 -0
- package/packages/core/src/ports/tickets/remote-server.ts +185 -0
- package/packages/core/src/ports/tickets/remote.ts +130 -0
- package/packages/core/src/ports/tickets/update.ts +51 -0
- package/packages/core/src/ports/types.ts +528 -0
- package/packages/core/src/ports/workspace/build-execution.ts +123 -0
- package/packages/core/src/ports/workspace/contract.ts +136 -0
- package/packages/core/src/ports/workspace/create.ts +132 -0
- package/packages/core/src/ports/workspace/distribution-archive.ts +317 -0
- package/packages/core/src/ports/workspace/fake.ts +432 -0
- package/packages/core/src/ports/workspace/git-worktree.ts +848 -0
- package/packages/core/src/ports/workspace/harvest-execution.ts +111 -0
- package/packages/core/src/ports/workspace/in-process-build-execution.ts +30 -0
- package/packages/core/src/ports/workspace/local-build-execution.ts +88 -0
- package/packages/core/src/ports/workspace/operator-sandbox.ts +142 -0
- package/packages/core/src/ports/workspace/phase-scratch.ts +156 -0
- package/packages/core/src/ports/workspace/vercel-sandbox.ts +2541 -0
- package/packages/core/src/processes/build-child-exit.ts +78 -0
- package/packages/core/src/processes/build-child.ts +119 -0
- package/packages/core/src/processes/build-execution-state.ts +95 -0
- package/packages/core/src/processes/build-parent-watch.ts +35 -0
- package/packages/core/src/processes/build-runner.ts +2614 -0
- package/packages/core/src/processes/dispatcher.ts +2825 -0
- package/packages/core/src/processes/execution-settlement.ts +171 -0
- package/packages/core/src/processes/harvest-child.ts +115 -0
- package/packages/core/src/processes/harvest-execution-state.ts +121 -0
- package/packages/core/src/processes/harvest-runner.ts +1656 -0
- package/packages/core/src/processes/harvest.ts +595 -0
- package/packages/core/src/processes/infrastructure-failure-budget.ts +146 -0
- package/packages/core/src/processes/phase-failure-budget.ts +52 -0
- package/packages/core/src/processes/process-group-reaper.ts +35 -0
- package/packages/core/src/processes/process-group.ts +65 -0
- package/packages/core/src/processes/publication-loss.ts +62 -0
- package/packages/core/src/processes/publication-state.ts +93 -0
- package/packages/core/src/processes/sandbox-state.ts +101 -0
- package/packages/core/src/registry.ts +24 -0
- package/packages/core/src/sessions/reducer.ts +185 -0
- package/packages/core/src/skills.ts +7 -0
- package/packages/core/src/spec-standard.ts +47 -0
- package/packages/core/src/store/adapter.ts +80 -0
- package/packages/core/src/store/build-scope.ts +272 -0
- package/packages/core/src/store/contract.ts +2581 -0
- package/packages/core/src/store/local/blobs.ts +38 -0
- package/packages/core/src/store/local/schema.ts +180 -0
- package/packages/core/src/store/local/store.ts +1713 -0
- package/packages/core/src/store/memory.ts +1138 -0
- package/packages/core/src/store/phase-session.ts +425 -0
- package/packages/core/src/store/remote/client.ts +780 -0
- package/packages/core/src/store/remote/index.ts +12 -0
- package/packages/core/src/store/remote/protocol.ts +348 -0
- package/packages/core/src/store/remote/server.ts +923 -0
- package/packages/core/src/store/remote/token.ts +137 -0
- package/packages/core/src/store/remote/version.ts +10 -0
- package/packages/core/src/store/retention.ts +91 -0
- package/packages/core/src/store/session-handle.ts +265 -0
- package/packages/core/src/store/streams/assemble.ts +79 -0
- package/packages/core/src/store/streams/lock.ts +42 -0
- package/packages/core/src/store/streams/session-writer.ts +306 -0
- package/packages/core/src/store/streams/types.ts +171 -0
- package/packages/core/src/store/streams/wait.ts +127 -0
- package/packages/core/src/store/subscribe.ts +105 -0
- package/packages/core/src/store/types.ts +466 -0
- package/packages/core/src/testing/fixed.ts +44 -0
- package/packages/core/src/testing/store-failures.ts +32 -0
- package/packages/core/src/zod-issues.ts +91 -0
- package/skills/code-review/SKILL.md +100 -0
- package/skills/finalize/SKILL.md +44 -0
- package/skills/guide/SKILL.md +1781 -0
- package/skills/guide/references/operator-api.md +164 -0
- package/skills/guide/references/plugin-authoring.md +354 -0
- package/skills/guide/references/remote-store-protocol.md +972 -0
- package/skills/guide/references/remote-ticket-protocol.md +79 -0
- package/skills/guide/references/setup.md +289 -0
- package/skills/guide/references/spec-standard.md +63 -0
- package/skills/guide/references/ticket-dependencies.md +59 -0
- package/skills/harvest/SKILL.md +66 -0
- package/skills/harvest-review/SKILL.md +63 -0
- package/skills/implement/SKILL.md +82 -0
- package/skills/plan/SKILL.md +123 -0
- package/skills/plan-review/SKILL.md +121 -0
- package/skills/reconcile/SKILL.md +65 -0
- package/skills/spec/SKILL.md +132 -0
- package/skills/tickets/SKILL.md +139 -0
- package/templates/autobuild.toml +36 -0
|
@@ -0,0 +1,1656 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crash-safe repository-scoped harvest workflow:
|
|
3
|
+
* scan -> synthesize <-> review -> file. It reuses `converge` for adversarial
|
|
4
|
+
* review semantics while every durable boundary lives in the repository
|
|
5
|
+
* journal, so a replacement process resumes the claimed snapshot rather than
|
|
6
|
+
* starting a duplicate run.
|
|
7
|
+
*/
|
|
8
|
+
import { HARVEST_REVIEW_ROLE, HARVEST_ROLE } from '../config/roles'
|
|
9
|
+
import type { Config } from '../config/schema'
|
|
10
|
+
import type { HarvestEventPayload, RepositoryEvent } from '../events/repository'
|
|
11
|
+
import { KERNEL } from '../events/envelope'
|
|
12
|
+
import type { IdSource, UuidSource } from '../ids'
|
|
13
|
+
import { occurrenceKey, type HarvestDisposition } from '../harvest/schema'
|
|
14
|
+
import { converge } from '../kernel/converge'
|
|
15
|
+
import { stalledChains } from '../kernel/stall'
|
|
16
|
+
import {
|
|
17
|
+
DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS,
|
|
18
|
+
actionableHarvestRun,
|
|
19
|
+
decideHarvestControl,
|
|
20
|
+
openHarvestRun,
|
|
21
|
+
proposalArtifactForRound,
|
|
22
|
+
reduceHarvest,
|
|
23
|
+
type HarvestRunState,
|
|
24
|
+
} from '../kernel/harvest'
|
|
25
|
+
import type { ArtifactRef, Feedback, Verdict } from '../ontology'
|
|
26
|
+
import {
|
|
27
|
+
createRuntimeResolver,
|
|
28
|
+
type ResolvedRuntime,
|
|
29
|
+
type RuntimeResolver,
|
|
30
|
+
} from '../ports/runner/routing'
|
|
31
|
+
import { isAlternateEligible, mayRetryPhase } from '../ports/runner/provider-error'
|
|
32
|
+
import { sessionPart } from '../ports/runner/stream-parts'
|
|
33
|
+
import type { SessionStreamInfo, RuntimeRegistry } from '../ports/runner/runtime'
|
|
34
|
+
import { createSessionStreamSink } from '../store/streams/session-writer'
|
|
35
|
+
import type { StreamOutcome } from '../store/streams/types'
|
|
36
|
+
import type {
|
|
37
|
+
AgentRunner,
|
|
38
|
+
AgentSessionHandle,
|
|
39
|
+
AgentTurnFailure,
|
|
40
|
+
AgentTurnResult,
|
|
41
|
+
SessionStreamSink,
|
|
42
|
+
TicketSource,
|
|
43
|
+
} from '../ports/types'
|
|
44
|
+
import { installedSkillName } from '../skills'
|
|
45
|
+
import { defaultProposalState } from './dispatcher'
|
|
46
|
+
import {
|
|
47
|
+
artifactRef,
|
|
48
|
+
evaluateHarvestPressure,
|
|
49
|
+
HARVEST_REPORT_ARTIFACT,
|
|
50
|
+
HARVEST_SCAN_ARTIFACT,
|
|
51
|
+
HARVEST_TRANSCRIPT_ARTIFACT,
|
|
52
|
+
harvestProposalKey,
|
|
53
|
+
loadScanPacket,
|
|
54
|
+
makeHarvestScanPacket,
|
|
55
|
+
parseApprovedProposalSet,
|
|
56
|
+
partitionHarvestExhaustion,
|
|
57
|
+
renderHarvestProposal,
|
|
58
|
+
resolveHarvestCreateBlockers,
|
|
59
|
+
scanUnclaimedObservations,
|
|
60
|
+
} from './harvest'
|
|
61
|
+
import type { BuildStore, Clock } from '../store/types'
|
|
62
|
+
|
|
63
|
+
/** Reserved provenance marker for deterministic harvest-created proposals. */
|
|
64
|
+
export const HARVEST_PROPOSAL_LABEL = 'autobuild:proposal'
|
|
65
|
+
|
|
66
|
+
export interface HarvestRunnerOpts {
|
|
67
|
+
leaseTtlMs?: number
|
|
68
|
+
heartbeatMs?: number
|
|
69
|
+
/** Retry budget inside one synthesize/review/file occurrence. */
|
|
70
|
+
maxSessionAttempts?: number
|
|
71
|
+
/** Durable outer reopen budget for a stopped run. */
|
|
72
|
+
maxRecoveryAttempts?: number
|
|
73
|
+
/** Stream each agent session bracket as AI SDK UI message parts (SPEC §9,
|
|
74
|
+
* the same wiring the build runners use). Default true. */
|
|
75
|
+
streamSessions?: boolean
|
|
76
|
+
/** Diagnostics hook for contained stream failures. Default console.error. */
|
|
77
|
+
onDiagnostic?: (message: string) => void
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface HarvestRunnerDeps {
|
|
81
|
+
store: BuildStore
|
|
82
|
+
tickets: TicketSource
|
|
83
|
+
config: Config
|
|
84
|
+
/** Current dispatcher snapshot, sampled only at workflow/session boundaries. */
|
|
85
|
+
getConfig?: () => Config
|
|
86
|
+
runtimes: RuntimeRegistry
|
|
87
|
+
repo: string
|
|
88
|
+
workspacePath: string
|
|
89
|
+
ids: IdSource
|
|
90
|
+
uuids: UuidSource
|
|
91
|
+
clock: Clock
|
|
92
|
+
instance: string
|
|
93
|
+
/** Repository-lease holder adopted from the owning DispatchLoop: when
|
|
94
|
+
* present, the runner skips its own claim, heartbeats with that holder, and
|
|
95
|
+
* never releases at the end — the owning loop holds the lease for its
|
|
96
|
+
* lifetime. Absent keeps the standalone self-claiming behavior. */
|
|
97
|
+
leaseHolder?: string
|
|
98
|
+
/** Provider-native identity of the disposable environment this run
|
|
99
|
+
* executes in (hosted harvest). Stamped into the run's harvest.started
|
|
100
|
+
* payload so the journal records where it ran; absent on local runs. */
|
|
101
|
+
environment?: { provider: string; environmentId: string; sessionId?: string }
|
|
102
|
+
sessionEnv?: Record<string, string>
|
|
103
|
+
opts?: HarvestRunnerOpts
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export type HarvestRunnerResult =
|
|
107
|
+
| { outcome: 'idle' }
|
|
108
|
+
| { outcome: 'held' }
|
|
109
|
+
| { outcome: 'parked'; run?: string }
|
|
110
|
+
| {
|
|
111
|
+
outcome: 'completed' | 'escalated' | 'failed'
|
|
112
|
+
launch: 'started' | 'resumed'
|
|
113
|
+
run: string
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
interface ProducerSession {
|
|
117
|
+
handle: AgentSessionHandle
|
|
118
|
+
runner: AgentRunner
|
|
119
|
+
route: string
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
type HarvestProviderAttempt = NonNullable<
|
|
123
|
+
HarvestEventPayload<'harvest.failed'>['providerAttempts']
|
|
124
|
+
>[number]
|
|
125
|
+
|
|
126
|
+
/** One session bracket's live stream (SPEC §9): the sink the harvest runner
|
|
127
|
+
* opened through the resolved target's streaming capability and the store id
|
|
128
|
+
* it returned. Undefined ⇒ the runtime declared no capability or the opener
|
|
129
|
+
* declined — the bracket then behaves exactly as before streams existed. */
|
|
130
|
+
interface SessionBracketStream {
|
|
131
|
+
sink: SessionStreamSink
|
|
132
|
+
streamId: string
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
class SessionFailure extends Error {
|
|
136
|
+
constructor(message: string) {
|
|
137
|
+
super(message)
|
|
138
|
+
this.name = 'HarvestSessionFailure'
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Internal, non-error unwind from a durable control boundary. It must never
|
|
143
|
+
* consume failure budget or emit harvest.failed. */
|
|
144
|
+
class HarvestParkedSignal extends Error {
|
|
145
|
+
constructor(readonly run?: string) {
|
|
146
|
+
super('harvest parked by operator')
|
|
147
|
+
this.name = 'HarvestParkedSignal'
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** A heartbeat proved this process no longer owns the repository lease and a
|
|
152
|
+
* replacement claimed it before we could re-acquire. This is not workflow
|
|
153
|
+
* failure: the replacement resumes the same journal at the next boundary. */
|
|
154
|
+
class HarvestLeaseLostError extends Error {
|
|
155
|
+
constructor(repo: string, instance: string) {
|
|
156
|
+
super(`harvest lease for ${repo} is held by another runner ` + `(former holder ${instance})`)
|
|
157
|
+
this.name = 'HarvestLeaseLostError'
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function errorMessage(error: unknown): string {
|
|
162
|
+
return error instanceof Error ? error.message : String(error)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function verdictFromEvent(
|
|
166
|
+
event: Extract<RepositoryEvent, { type: 'harvest.review.verdict' }>,
|
|
167
|
+
): Verdict {
|
|
168
|
+
if (event.payload.verdict === 'approve') return { verdict: 'approve' }
|
|
169
|
+
if (event.payload.verdict === 'revise') {
|
|
170
|
+
return { verdict: 'revise', findings: event.payload.findings }
|
|
171
|
+
}
|
|
172
|
+
return {
|
|
173
|
+
verdict: 'escalate',
|
|
174
|
+
reason: event.payload.reason ?? 'harvest reviewer escalated',
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export class HarvestRunner {
|
|
179
|
+
private readonly leaseTtlMs: number
|
|
180
|
+
private readonly heartbeatMs: number
|
|
181
|
+
private readonly maxSessionAttempts: number
|
|
182
|
+
private readonly maxRecoveryAttempts: number
|
|
183
|
+
private readonly streamSessions: boolean
|
|
184
|
+
private readonly onDiagnostic: (message: string) => void
|
|
185
|
+
private heartbeatTimer: ReturnType<typeof setInterval> | undefined
|
|
186
|
+
/** Set only when the store positively reports a lapsed/stolen lease. A
|
|
187
|
+
* rejected heartbeat is an outage; later beats retry until one can decide. */
|
|
188
|
+
private leaseLost = false
|
|
189
|
+
private producer: ProducerSession | undefined
|
|
190
|
+
|
|
191
|
+
constructor(private readonly deps: HarvestRunnerDeps) {
|
|
192
|
+
this.leaseTtlMs = deps.opts?.leaseTtlMs ?? 60_000
|
|
193
|
+
this.heartbeatMs = deps.opts?.heartbeatMs ?? 15_000
|
|
194
|
+
this.maxSessionAttempts = deps.opts?.maxSessionAttempts ?? 2
|
|
195
|
+
this.maxRecoveryAttempts =
|
|
196
|
+
deps.opts?.maxRecoveryAttempts ?? DEFAULT_MAX_HARVEST_RECOVERY_ATTEMPTS
|
|
197
|
+
this.streamSessions = deps.opts?.streamSessions ?? true
|
|
198
|
+
this.onDiagnostic = deps.opts?.onDiagnostic ?? ((message) => console.error(message))
|
|
199
|
+
if (!Number.isInteger(this.maxRecoveryAttempts) || this.maxRecoveryAttempts <= 0) {
|
|
200
|
+
throw new Error('maxRecoveryAttempts must be a positive integer')
|
|
201
|
+
}
|
|
202
|
+
// Preserve eager validation for static callers and startup failures.
|
|
203
|
+
createRuntimeResolver(deps.runtimes, deps.config.roles, deps.config.policy.sessionBudgetSeconds)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
private currentConfig(): Config {
|
|
207
|
+
return this.deps.getConfig?.() ?? this.deps.config
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
private currentResolver(): RuntimeResolver {
|
|
211
|
+
const config = this.currentConfig()
|
|
212
|
+
return createRuntimeResolver(
|
|
213
|
+
this.deps.runtimes,
|
|
214
|
+
config.roles,
|
|
215
|
+
config.policy.sessionBudgetSeconds,
|
|
216
|
+
)
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async run(): Promise<HarvestRunnerResult> {
|
|
220
|
+
const { store, repo, instance } = this.deps
|
|
221
|
+
await store.ensureRepo(repo)
|
|
222
|
+
if (
|
|
223
|
+
this.deps.leaseHolder === undefined &&
|
|
224
|
+
!(await store.claimRepoLease(repo, instance, this.leaseTtlMs))
|
|
225
|
+
) {
|
|
226
|
+
return { outcome: 'held' }
|
|
227
|
+
}
|
|
228
|
+
let initial: 'started' | 'resumed' = 'resumed'
|
|
229
|
+
let run: HarvestRunState | undefined
|
|
230
|
+
try {
|
|
231
|
+
await this.startHeartbeat()
|
|
232
|
+
await this.ensureLease()
|
|
233
|
+
let events = await store.getRepoEvents(repo)
|
|
234
|
+
let state = reduceHarvest(events)
|
|
235
|
+
run = actionableHarvestRun(state)
|
|
236
|
+
|
|
237
|
+
// Settle durable operator control before scanning or resuming work. The
|
|
238
|
+
// selected run comes from the full journal, so a shadowed failure or
|
|
239
|
+
// exhaustion barrier is recovered/reported before any later open run.
|
|
240
|
+
await this.controlBoundary(run?.run)
|
|
241
|
+
events = await store.getRepoEvents(repo)
|
|
242
|
+
state = reduceHarvest(events)
|
|
243
|
+
run = openHarvestRun(state)
|
|
244
|
+
|
|
245
|
+
if (!run) {
|
|
246
|
+
await this.controlBoundary()
|
|
247
|
+
const scan = await scanUnclaimedObservations(store, repo)
|
|
248
|
+
// Scanning may span many build streams. Treat its completion as a
|
|
249
|
+
// boundary even when the threshold is not met, so a request arriving
|
|
250
|
+
// during the read is acknowledged before this runner returns idle.
|
|
251
|
+
await this.controlBoundary()
|
|
252
|
+
const pressure = evaluateHarvestPressure(scan, this.currentConfig().policy)
|
|
253
|
+
if (pressure.trigger === undefined) return { outcome: 'idle' }
|
|
254
|
+
const runId = this.deps.ids('harvest')
|
|
255
|
+
const packet = await makeHarvestScanPacket({
|
|
256
|
+
store,
|
|
257
|
+
tickets: this.deps.tickets,
|
|
258
|
+
repo,
|
|
259
|
+
run: runId,
|
|
260
|
+
observations: scan.observations,
|
|
261
|
+
state: scan.state,
|
|
262
|
+
})
|
|
263
|
+
// Scan preparation is not a claim. Re-read operator control immediately
|
|
264
|
+
// before atomically committing the immutable snapshot.
|
|
265
|
+
await this.controlBoundary()
|
|
266
|
+
initial = 'started'
|
|
267
|
+
await store.appendRepoWithArtifacts(
|
|
268
|
+
repo,
|
|
269
|
+
[
|
|
270
|
+
{
|
|
271
|
+
kind: HARVEST_SCAN_ARTIFACT,
|
|
272
|
+
content: `${JSON.stringify(packet, null, 2)}\n`,
|
|
273
|
+
metadata: { run: runId, observations: packet.observations.length },
|
|
274
|
+
},
|
|
275
|
+
],
|
|
276
|
+
(deposited) => ({
|
|
277
|
+
actor: KERNEL,
|
|
278
|
+
type: 'harvest.started',
|
|
279
|
+
payload: {
|
|
280
|
+
run: runId,
|
|
281
|
+
observations: packet.observations.map((item) => item.occurrence),
|
|
282
|
+
scan: artifactRef(deposited[0]!),
|
|
283
|
+
trigger: pressure.trigger,
|
|
284
|
+
...(this.deps.environment !== undefined
|
|
285
|
+
? { environment: this.deps.environment }
|
|
286
|
+
: {}),
|
|
287
|
+
},
|
|
288
|
+
}),
|
|
289
|
+
)
|
|
290
|
+
events = await store.getRepoEvents(repo)
|
|
291
|
+
run = reduceHarvest(events).runs.find((candidate) => candidate.run === runId)
|
|
292
|
+
if (!run) throw new Error(`harvest.started did not reduce run "${runId}"`)
|
|
293
|
+
// A request racing the claim parks this exact run; its snapshot stays
|
|
294
|
+
// open and is resumed rather than scanned again.
|
|
295
|
+
await this.controlBoundary(run.run)
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
try {
|
|
299
|
+
await this.ensureScanStep(run)
|
|
300
|
+
await this.controlBoundary(run.run)
|
|
301
|
+
const outcome = await this.executeWorkflow(run)
|
|
302
|
+
return { outcome, launch: initial, run: run.run }
|
|
303
|
+
} catch (error) {
|
|
304
|
+
if (error instanceof HarvestLeaseLostError || error instanceof HarvestParkedSignal) {
|
|
305
|
+
throw error
|
|
306
|
+
}
|
|
307
|
+
if (error instanceof SessionFailure) {
|
|
308
|
+
await this.settleRecoveryExhaustion(run.run)
|
|
309
|
+
return { outcome: 'failed', launch: initial, run: run.run }
|
|
310
|
+
}
|
|
311
|
+
await this.recordWorkflowFailure(run.run, error)
|
|
312
|
+
await this.settleRecoveryExhaustion(run.run)
|
|
313
|
+
return { outcome: 'failed', launch: initial, run: run.run }
|
|
314
|
+
}
|
|
315
|
+
} catch (error) {
|
|
316
|
+
if (error instanceof HarvestParkedSignal) {
|
|
317
|
+
const parkedRun = error.run ?? run?.run
|
|
318
|
+
return parkedRun === undefined
|
|
319
|
+
? { outcome: 'parked' }
|
|
320
|
+
: { outcome: 'parked', run: parkedRun }
|
|
321
|
+
}
|
|
322
|
+
if (error instanceof HarvestLeaseLostError) return { outcome: 'held' }
|
|
323
|
+
throw error
|
|
324
|
+
} finally {
|
|
325
|
+
this.stopHeartbeat()
|
|
326
|
+
if (this.producer !== undefined) {
|
|
327
|
+
try {
|
|
328
|
+
await this.producer.runner.end(this.producer.handle)
|
|
329
|
+
} catch {
|
|
330
|
+
// Every successful turn already has its own transcript artifact.
|
|
331
|
+
}
|
|
332
|
+
this.producer = undefined
|
|
333
|
+
}
|
|
334
|
+
// An adopted holder is released by the owning DispatchLoop, never here.
|
|
335
|
+
if (this.deps.leaseHolder === undefined) await store.releaseRepoLease(repo, instance)
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
private async executeWorkflow(
|
|
340
|
+
initialRun: HarvestRunState,
|
|
341
|
+
): Promise<'completed' | 'escalated' | 'failed'> {
|
|
342
|
+
let run = await this.refreshRun(initialRun.run)
|
|
343
|
+
// Repair any process-death gap between a typed agent terminal and the
|
|
344
|
+
// kernel-authored step result before deciding where to resume.
|
|
345
|
+
for (const proposal of run.proposals) {
|
|
346
|
+
await this.ensureStepCompleted(
|
|
347
|
+
run.run,
|
|
348
|
+
'synthesize',
|
|
349
|
+
proposal.round,
|
|
350
|
+
'completed',
|
|
351
|
+
proposal.artifact,
|
|
352
|
+
)
|
|
353
|
+
}
|
|
354
|
+
run = await this.refreshRun(run.run)
|
|
355
|
+
for (const review of run.reviews) {
|
|
356
|
+
await this.ensureStepCompleted(
|
|
357
|
+
run.run,
|
|
358
|
+
'review',
|
|
359
|
+
review.round,
|
|
360
|
+
review.verdict,
|
|
361
|
+
review.artifact,
|
|
362
|
+
)
|
|
363
|
+
}
|
|
364
|
+
run = await this.refreshRun(run.run)
|
|
365
|
+
await this.controlBoundary(run.run)
|
|
366
|
+
const existingApproval = [...run.reviews]
|
|
367
|
+
.reverse()
|
|
368
|
+
.find((review) => review.verdict === 'approve')
|
|
369
|
+
if (existingApproval !== undefined) {
|
|
370
|
+
const approved = proposalArtifactForRound(run, existingApproval.round)
|
|
371
|
+
if (!approved) {
|
|
372
|
+
throw new Error(
|
|
373
|
+
`harvest ${run.run} approval r${existingApproval.round} has no proposal artifact`,
|
|
374
|
+
)
|
|
375
|
+
}
|
|
376
|
+
await this.file(run, approved)
|
|
377
|
+
return 'completed'
|
|
378
|
+
}
|
|
379
|
+
const existingEscalation = [...run.reviews]
|
|
380
|
+
.reverse()
|
|
381
|
+
.find((review) => review.verdict === 'escalate')
|
|
382
|
+
if (existingEscalation !== undefined) {
|
|
383
|
+
await this.escalate(
|
|
384
|
+
run,
|
|
385
|
+
'agent',
|
|
386
|
+
existingEscalation.reason ?? 'harvest reviewer escalated',
|
|
387
|
+
existingEscalation.round,
|
|
388
|
+
)
|
|
389
|
+
return 'escalated'
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const reviseRounds = run.reviews
|
|
393
|
+
.filter((review) => review.verdict === 'revise')
|
|
394
|
+
.sort((a, b) => a.round - b.round)
|
|
395
|
+
const stalled = stalledChains(
|
|
396
|
+
reviseRounds.map((review) => review.findings),
|
|
397
|
+
this.currentConfig().policy.stallRounds,
|
|
398
|
+
)
|
|
399
|
+
if (stalled.length > 0) {
|
|
400
|
+
const chain = stalled.reduce((deepest, candidate) =>
|
|
401
|
+
candidate.rounds > deepest.rounds ? candidate : deepest,
|
|
402
|
+
)
|
|
403
|
+
await this.escalate(
|
|
404
|
+
run,
|
|
405
|
+
'stall',
|
|
406
|
+
`finding chain persisted ${chain.rounds} rounds: ${chain.ids.join(' -> ')}`,
|
|
407
|
+
reviseRounds.at(-1)?.round,
|
|
408
|
+
)
|
|
409
|
+
return 'escalated'
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const startRound = (reviseRounds.at(-1)?.round ?? 0) + 1
|
|
413
|
+
const initialFeedback: Feedback | null =
|
|
414
|
+
reviseRounds.length === 0
|
|
415
|
+
? null
|
|
416
|
+
: {
|
|
417
|
+
findings: reviseRounds.at(-1)!.findings.map((finding) => finding.id),
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const outcome = await converge<ArtifactRef>({
|
|
421
|
+
startRound,
|
|
422
|
+
priorRounds: reviseRounds.map((review) => review.findings),
|
|
423
|
+
initialFeedback,
|
|
424
|
+
policy: () => {
|
|
425
|
+
const policy = this.currentConfig().policy
|
|
426
|
+
return { maxRounds: policy.maxReviewRounds, stallRounds: policy.stallRounds }
|
|
427
|
+
},
|
|
428
|
+
produce: async (_feedback, round) => {
|
|
429
|
+
run = await this.refreshRun(run.run)
|
|
430
|
+
await this.controlBoundary(run.run)
|
|
431
|
+
const existing = proposalArtifactForRound(run, round)
|
|
432
|
+
if (existing !== undefined) {
|
|
433
|
+
await this.ensureStepCompleted(run.run, 'synthesize', round, 'completed', existing)
|
|
434
|
+
await this.controlBoundary(run.run)
|
|
435
|
+
return existing
|
|
436
|
+
}
|
|
437
|
+
return this.synthesize(run, round)
|
|
438
|
+
},
|
|
439
|
+
review: async (_artifact, round) => {
|
|
440
|
+
run = await this.refreshRun(run.run)
|
|
441
|
+
await this.controlBoundary(run.run)
|
|
442
|
+
const existing = [...run.reviews].reverse().find((review) => review.round === round)
|
|
443
|
+
if (existing !== undefined) {
|
|
444
|
+
await this.ensureStepCompleted(
|
|
445
|
+
run.run,
|
|
446
|
+
'review',
|
|
447
|
+
round,
|
|
448
|
+
existing.verdict,
|
|
449
|
+
existing.artifact,
|
|
450
|
+
)
|
|
451
|
+
const event = (await this.deps.store.getRepoEvents(this.deps.repo)).find(
|
|
452
|
+
(
|
|
453
|
+
candidate,
|
|
454
|
+
): candidate is Extract<RepositoryEvent, { type: 'harvest.review.verdict' }> =>
|
|
455
|
+
candidate.type === 'harvest.review.verdict' &&
|
|
456
|
+
candidate.payload.run === run.run &&
|
|
457
|
+
candidate.payload.round === round,
|
|
458
|
+
)
|
|
459
|
+
if (!event) throw new Error('review reducer/event mismatch')
|
|
460
|
+
await this.controlBoundary(run.run)
|
|
461
|
+
return verdictFromEvent(event)
|
|
462
|
+
}
|
|
463
|
+
return this.review(run, round)
|
|
464
|
+
},
|
|
465
|
+
})
|
|
466
|
+
|
|
467
|
+
run = await this.refreshRun(run.run)
|
|
468
|
+
if (outcome.outcome === 'approved') {
|
|
469
|
+
await this.file(run, outcome.artifact)
|
|
470
|
+
return 'completed'
|
|
471
|
+
}
|
|
472
|
+
await this.escalate(run, outcome.source, outcome.reason, outcome.rounds)
|
|
473
|
+
return 'escalated'
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
private async synthesize(run: HarvestRunState, round: number): Promise<ArtifactRef> {
|
|
477
|
+
await this.controlBoundary(run.run)
|
|
478
|
+
await this.startStep(run.run, 'synthesize', round)
|
|
479
|
+
await this.executeSession({
|
|
480
|
+
run: run.run,
|
|
481
|
+
role: HARVEST_ROLE,
|
|
482
|
+
skill: installedSkillName('harvest'),
|
|
483
|
+
step: 'synthesize',
|
|
484
|
+
round,
|
|
485
|
+
producer: true,
|
|
486
|
+
terminal: (event, session) =>
|
|
487
|
+
event.type === 'harvest.proposals.submitted' &&
|
|
488
|
+
event.payload.run === run.run &&
|
|
489
|
+
event.payload.round === round &&
|
|
490
|
+
event.actor.kind === 'agent' &&
|
|
491
|
+
event.actor.session === session,
|
|
492
|
+
})
|
|
493
|
+
const refreshed = await this.refreshRun(run.run)
|
|
494
|
+
const artifact = proposalArtifactForRound(refreshed, round)
|
|
495
|
+
if (!artifact) throw new Error(`synthesize@${round} terminal has no artifact`)
|
|
496
|
+
await this.completeStep(run.run, 'synthesize', round, 'completed', artifact)
|
|
497
|
+
await this.controlBoundary(run.run)
|
|
498
|
+
return artifact
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
private async review(run: HarvestRunState, round: number): Promise<Verdict> {
|
|
502
|
+
await this.controlBoundary(run.run)
|
|
503
|
+
await this.startStep(run.run, 'review', round)
|
|
504
|
+
await this.executeSession({
|
|
505
|
+
run: run.run,
|
|
506
|
+
role: HARVEST_REVIEW_ROLE,
|
|
507
|
+
skill: installedSkillName('harvest-review'),
|
|
508
|
+
step: 'review',
|
|
509
|
+
round,
|
|
510
|
+
producer: false,
|
|
511
|
+
terminal: (event, session) =>
|
|
512
|
+
event.type === 'harvest.review.verdict' &&
|
|
513
|
+
event.payload.run === run.run &&
|
|
514
|
+
event.payload.round === round &&
|
|
515
|
+
event.actor.kind === 'agent' &&
|
|
516
|
+
event.actor.session === session,
|
|
517
|
+
})
|
|
518
|
+
const events = await this.deps.store.getRepoEvents(this.deps.repo)
|
|
519
|
+
const verdict = [...events]
|
|
520
|
+
.reverse()
|
|
521
|
+
.find(
|
|
522
|
+
(event): event is Extract<RepositoryEvent, { type: 'harvest.review.verdict' }> =>
|
|
523
|
+
event.type === 'harvest.review.verdict' &&
|
|
524
|
+
event.payload.run === run.run &&
|
|
525
|
+
event.payload.round === round,
|
|
526
|
+
)
|
|
527
|
+
if (!verdict) throw new Error(`review@${round} terminal is missing`)
|
|
528
|
+
await this.completeStep(
|
|
529
|
+
run.run,
|
|
530
|
+
'review',
|
|
531
|
+
round,
|
|
532
|
+
verdict.payload.verdict,
|
|
533
|
+
verdict.payload.artifact,
|
|
534
|
+
)
|
|
535
|
+
await this.controlBoundary(run.run)
|
|
536
|
+
return verdictFromEvent(verdict)
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
private async executeSession(spec: {
|
|
540
|
+
run: string
|
|
541
|
+
role: 'harvest' | 'harvest-review'
|
|
542
|
+
skill: string
|
|
543
|
+
step: 'synthesize' | 'review'
|
|
544
|
+
round: number
|
|
545
|
+
producer: boolean
|
|
546
|
+
terminal: (event: RepositoryEvent, session: string) => boolean
|
|
547
|
+
}): Promise<void> {
|
|
548
|
+
const resolved = this.currentResolver().resolve(spec.role)
|
|
549
|
+
if (resolved.alternates.length === 0) return this.executeSingleSession(spec)
|
|
550
|
+
return this.executeSessionChain(spec, resolved, resolved.alternates)
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
private async executeSessionChain(
|
|
554
|
+
spec: {
|
|
555
|
+
run: string
|
|
556
|
+
role: 'harvest' | 'harvest-review'
|
|
557
|
+
skill: string
|
|
558
|
+
step: 'synthesize' | 'review'
|
|
559
|
+
round: number
|
|
560
|
+
producer: boolean
|
|
561
|
+
terminal: (event: RepositoryEvent, session: string) => boolean
|
|
562
|
+
},
|
|
563
|
+
primary: ResolvedRuntime,
|
|
564
|
+
alternates: readonly ResolvedRuntime[],
|
|
565
|
+
): Promise<void> {
|
|
566
|
+
const { store, repo, ids, workspacePath } = this.deps
|
|
567
|
+
await this.ensureLease()
|
|
568
|
+
const events = await store.getRepoEvents(repo)
|
|
569
|
+
const matchingFailures = events.filter(
|
|
570
|
+
(event): event is Extract<RepositoryEvent, { type: 'harvest.failed' }> =>
|
|
571
|
+
event.type === 'harvest.failed' &&
|
|
572
|
+
event.payload.run === spec.run &&
|
|
573
|
+
event.payload.step === spec.step &&
|
|
574
|
+
event.payload.round === spec.round,
|
|
575
|
+
)
|
|
576
|
+
const failures = matchingFailures.length
|
|
577
|
+
const latestFailure = matchingFailures.at(-1)
|
|
578
|
+
const current = reduceHarvest(events).runs.find((candidate) => candidate.run === spec.run)
|
|
579
|
+
const resumedAfterTerminalFailure =
|
|
580
|
+
latestFailure?.payload.willRetry === false &&
|
|
581
|
+
current?.status === 'running' &&
|
|
582
|
+
current.failure === undefined &&
|
|
583
|
+
events.some((event) => event.type === 'harvest.resumed' && event.seq > latestFailure.seq)
|
|
584
|
+
if (failures >= this.maxSessionAttempts && !resumedAfterTerminalFailure) {
|
|
585
|
+
throw new SessionFailure(`${spec.step}@${spec.round} exhausted retries`)
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
const targets = [primary, ...alternates]
|
|
589
|
+
const attempts: HarvestProviderAttempt[] = []
|
|
590
|
+
let substitution: HarvestProviderAttempt | undefined
|
|
591
|
+
const primaryRoute = JSON.stringify({
|
|
592
|
+
runtime: primary.runtime,
|
|
593
|
+
model: primary.model,
|
|
594
|
+
args: primary.args,
|
|
595
|
+
})
|
|
596
|
+
let primaryLive = spec.producer ? this.producer : undefined
|
|
597
|
+
if (
|
|
598
|
+
primaryLive !== undefined &&
|
|
599
|
+
(primaryLive.runner !== primary.runner || primaryLive.route !== primaryRoute)
|
|
600
|
+
) {
|
|
601
|
+
try {
|
|
602
|
+
await primaryLive.runner.end(primaryLive.handle)
|
|
603
|
+
} catch {
|
|
604
|
+
// Best-effort cleanup after the previous round reached its boundary.
|
|
605
|
+
}
|
|
606
|
+
this.producer = undefined
|
|
607
|
+
primaryLive = undefined
|
|
608
|
+
}
|
|
609
|
+
for (let index = 0; index < targets.length; index += 1) {
|
|
610
|
+
const target = targets[index]!
|
|
611
|
+
const session = ids('hs')
|
|
612
|
+
// A pause requested while a provider failed is honored before another
|
|
613
|
+
// provider starts; the incomplete step safely restarts at the primary.
|
|
614
|
+
if (index > 0) await this.controlBoundary(spec.run)
|
|
615
|
+
else await this.ensureLease()
|
|
616
|
+
const streamBracket = await this.openSessionStream(target, {
|
|
617
|
+
session,
|
|
618
|
+
role: spec.role,
|
|
619
|
+
runner: target.runtime,
|
|
620
|
+
...(target.model !== undefined ? { model: target.model } : {}),
|
|
621
|
+
phase: `harvest:${spec.step}`,
|
|
622
|
+
round: spec.round,
|
|
623
|
+
})
|
|
624
|
+
const stream = streamBracket?.sink
|
|
625
|
+
const started = await store.appendRepo(repo, {
|
|
626
|
+
actor: KERNEL,
|
|
627
|
+
type: 'harvest.session.started',
|
|
628
|
+
payload: {
|
|
629
|
+
run: spec.run,
|
|
630
|
+
session,
|
|
631
|
+
role: spec.role,
|
|
632
|
+
runner: target.runtime,
|
|
633
|
+
...(target.model !== undefined ? { model: target.model } : {}),
|
|
634
|
+
args: [...target.args],
|
|
635
|
+
step: spec.step,
|
|
636
|
+
round: spec.round,
|
|
637
|
+
...(streamBracket !== undefined ? { stream: streamBracket.streamId } : {}),
|
|
638
|
+
...(substitution !== undefined
|
|
639
|
+
? { substitution: { failed: substitution, selectedIndex: index } }
|
|
640
|
+
: {}),
|
|
641
|
+
},
|
|
642
|
+
})
|
|
643
|
+
|
|
644
|
+
let handle: AgentSessionHandle | undefined
|
|
645
|
+
let result: AgentTurnResult | undefined
|
|
646
|
+
let turnError: unknown
|
|
647
|
+
const live = index === 0 ? primaryLive : undefined
|
|
648
|
+
try {
|
|
649
|
+
if (live !== undefined) {
|
|
650
|
+
handle = live.handle
|
|
651
|
+
result = await live.runner.continue(
|
|
652
|
+
live.handle,
|
|
653
|
+
`Revise harvest proposals for round ${spec.round}: run ab harvest context, address .ab/findings.json, then submit.`,
|
|
654
|
+
{
|
|
655
|
+
env: this.sessionEnv(spec.run, spec.step, spec.round, session),
|
|
656
|
+
...(stream !== undefined ? { stream } : {}),
|
|
657
|
+
},
|
|
658
|
+
)
|
|
659
|
+
} else {
|
|
660
|
+
const turn = await target.runner.start({
|
|
661
|
+
skill: spec.skill,
|
|
662
|
+
invocation: spec.run,
|
|
663
|
+
workspacePath,
|
|
664
|
+
...(target.model !== undefined ? { model: target.model } : {}),
|
|
665
|
+
...(target.args !== undefined ? { args: target.args } : {}),
|
|
666
|
+
env: this.sessionEnv(spec.run, spec.step, spec.round, session),
|
|
667
|
+
...(stream !== undefined ? { stream } : {}),
|
|
668
|
+
})
|
|
669
|
+
handle = turn.session
|
|
670
|
+
result = turn.result
|
|
671
|
+
}
|
|
672
|
+
} catch (error) {
|
|
673
|
+
turnError = error
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
try {
|
|
677
|
+
await this.ensureLease()
|
|
678
|
+
} catch (error) {
|
|
679
|
+
if (handle !== undefined && live === undefined) {
|
|
680
|
+
try {
|
|
681
|
+
await target.runner.end(handle)
|
|
682
|
+
} catch {
|
|
683
|
+
// A dead session may have nothing left to close.
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
687
|
+
throw error
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
const since = await store.getRepoEvents(repo, started.seq)
|
|
691
|
+
const terminal =
|
|
692
|
+
turnError === undefined && since.some((event) => spec.terminal(event, session))
|
|
693
|
+
const owner = live?.runner ?? target.runner
|
|
694
|
+
const close =
|
|
695
|
+
streamBracket !== undefined
|
|
696
|
+
? { stream: streamBracket, outcome: 'completed' as const }
|
|
697
|
+
: undefined
|
|
698
|
+
if (terminal && handle !== undefined && result !== undefined) {
|
|
699
|
+
if (spec.producer && index === 0) {
|
|
700
|
+
this.producer = { handle, runner: owner, route: primaryRoute }
|
|
701
|
+
await this.depositTranscript(
|
|
702
|
+
spec,
|
|
703
|
+
session,
|
|
704
|
+
JSON.stringify(
|
|
705
|
+
{
|
|
706
|
+
session,
|
|
707
|
+
run: spec.run,
|
|
708
|
+
step: spec.step,
|
|
709
|
+
round: spec.round,
|
|
710
|
+
turn: result,
|
|
711
|
+
note: 'producer session kept live for convergence continuation',
|
|
712
|
+
},
|
|
713
|
+
null,
|
|
714
|
+
2,
|
|
715
|
+
),
|
|
716
|
+
result.usage,
|
|
717
|
+
target.model,
|
|
718
|
+
close,
|
|
719
|
+
)
|
|
720
|
+
} else {
|
|
721
|
+
const transcript = await owner.end(handle)
|
|
722
|
+
await this.depositTranscript(
|
|
723
|
+
spec,
|
|
724
|
+
session,
|
|
725
|
+
transcript.content,
|
|
726
|
+
transcript.metadata.usage,
|
|
727
|
+
transcript.metadata.model ?? target.model,
|
|
728
|
+
close,
|
|
729
|
+
)
|
|
730
|
+
if (spec.producer) this.producer = undefined
|
|
731
|
+
}
|
|
732
|
+
return
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
if (handle !== undefined) {
|
|
736
|
+
try {
|
|
737
|
+
const transcript = await owner.end(handle)
|
|
738
|
+
await this.depositTranscript(
|
|
739
|
+
spec,
|
|
740
|
+
session,
|
|
741
|
+
transcript.content,
|
|
742
|
+
transcript.metadata.usage,
|
|
743
|
+
transcript.metadata.model ?? target.model,
|
|
744
|
+
streamBracket !== undefined
|
|
745
|
+
? { stream: streamBracket, outcome: 'aborted' as const }
|
|
746
|
+
: undefined,
|
|
747
|
+
)
|
|
748
|
+
} catch (error) {
|
|
749
|
+
// The deposit was skipped, so the stream must not stay open behind
|
|
750
|
+
// the rethrow (lease lost) or the failure appends below.
|
|
751
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
752
|
+
if (error instanceof HarvestLeaseLostError) throw error
|
|
753
|
+
}
|
|
754
|
+
} else {
|
|
755
|
+
// No handle: the turn failed before the adapter owned a session, so
|
|
756
|
+
// no deposit will run — close the bracket's stream here.
|
|
757
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
758
|
+
}
|
|
759
|
+
if (spec.producer) this.producer = undefined
|
|
760
|
+
|
|
761
|
+
const failure: AgentTurnFailure =
|
|
762
|
+
turnError !== undefined
|
|
763
|
+
? { message: errorMessage(turnError), permanent: false, cause: 'availability' }
|
|
764
|
+
: result?.kind === 'failed'
|
|
765
|
+
? result.failure
|
|
766
|
+
: { message: 'no-terminal', permanent: false }
|
|
767
|
+
const attempt: HarvestProviderAttempt = {
|
|
768
|
+
index,
|
|
769
|
+
session,
|
|
770
|
+
runner: target.runtime,
|
|
771
|
+
...(target.model !== undefined ? { model: target.model } : {}),
|
|
772
|
+
error: failure.message,
|
|
773
|
+
...(failure.cause !== undefined ? { cause: failure.cause } : {}),
|
|
774
|
+
}
|
|
775
|
+
attempts.push(attempt)
|
|
776
|
+
const eligibleForAlternate =
|
|
777
|
+
turnError !== undefined ||
|
|
778
|
+
(result?.kind === 'failed' && isAlternateEligible(result.failure))
|
|
779
|
+
if (index + 1 < targets.length && eligibleForAlternate) {
|
|
780
|
+
substitution = attempt
|
|
781
|
+
continue
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
const attemptNumber = failures + 1
|
|
785
|
+
await this.ensureLease()
|
|
786
|
+
await store.appendRepo(repo, {
|
|
787
|
+
actor: KERNEL,
|
|
788
|
+
type: 'harvest.failed',
|
|
789
|
+
payload: {
|
|
790
|
+
run: spec.run,
|
|
791
|
+
step: spec.step,
|
|
792
|
+
round: spec.round,
|
|
793
|
+
attempt: attemptNumber,
|
|
794
|
+
error: failure.message,
|
|
795
|
+
willRetry: mayRetryPhase(failure) && attemptNumber < this.maxSessionAttempts,
|
|
796
|
+
providerAttempts: attempts,
|
|
797
|
+
},
|
|
798
|
+
})
|
|
799
|
+
await this.ensureLease()
|
|
800
|
+
await store.appendRepo(repo, {
|
|
801
|
+
actor: KERNEL,
|
|
802
|
+
type: 'harvest.step.completed',
|
|
803
|
+
payload: {
|
|
804
|
+
run: spec.run,
|
|
805
|
+
step: spec.step,
|
|
806
|
+
round: spec.round,
|
|
807
|
+
outcome: 'failed',
|
|
808
|
+
detail: failure.message,
|
|
809
|
+
},
|
|
810
|
+
})
|
|
811
|
+
throw new SessionFailure(`${spec.step}@${spec.round} failed: ${failure.message}`)
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
private async executeSingleSession(spec: {
|
|
816
|
+
run: string
|
|
817
|
+
role: 'harvest' | 'harvest-review'
|
|
818
|
+
skill: string
|
|
819
|
+
step: 'synthesize' | 'review'
|
|
820
|
+
round: number
|
|
821
|
+
producer: boolean
|
|
822
|
+
terminal: (event: RepositoryEvent, session: string) => boolean
|
|
823
|
+
}): Promise<void> {
|
|
824
|
+
const { store, repo, ids, workspacePath } = this.deps
|
|
825
|
+
await this.ensureLease()
|
|
826
|
+
const events = await store.getRepoEvents(repo)
|
|
827
|
+
const matchingFailures = events.filter(
|
|
828
|
+
(event): event is Extract<RepositoryEvent, { type: 'harvest.failed' }> =>
|
|
829
|
+
event.type === 'harvest.failed' &&
|
|
830
|
+
event.payload.run === spec.run &&
|
|
831
|
+
event.payload.step === spec.step &&
|
|
832
|
+
event.payload.round === spec.round,
|
|
833
|
+
)
|
|
834
|
+
const failures = matchingFailures.length
|
|
835
|
+
const latestFailure = matchingFailures.at(-1)
|
|
836
|
+
const current = reduceHarvest(events).runs.find((candidate) => candidate.run === spec.run)
|
|
837
|
+
// A stopping failure may already have consumed the ordinary retry budget.
|
|
838
|
+
// harvest.resumed deliberately clears that reduced stop and grants one real
|
|
839
|
+
// re-entry without deleting history or resetting attempt numbers. The next
|
|
840
|
+
// failure fact restores the guard (and parks the run again).
|
|
841
|
+
const resumedAfterTerminalFailure =
|
|
842
|
+
latestFailure?.payload.willRetry === false &&
|
|
843
|
+
current?.status === 'running' &&
|
|
844
|
+
current.failure === undefined &&
|
|
845
|
+
events.some((event) => event.type === 'harvest.resumed' && event.seq > latestFailure.seq)
|
|
846
|
+
if (failures >= this.maxSessionAttempts && !resumedAfterTerminalFailure) {
|
|
847
|
+
throw new SessionFailure(`${spec.step}@${spec.round} exhausted retries`)
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
const session = ids('hs')
|
|
851
|
+
const resolved = this.currentResolver().resolve(spec.role)
|
|
852
|
+
const route = JSON.stringify({
|
|
853
|
+
runtime: resolved.runtime,
|
|
854
|
+
model: resolved.model,
|
|
855
|
+
args: resolved.args,
|
|
856
|
+
})
|
|
857
|
+
let live = spec.producer ? this.producer : undefined
|
|
858
|
+
if (live !== undefined && (live.runner !== resolved.runner || live.route !== route)) {
|
|
859
|
+
try {
|
|
860
|
+
await live.runner.end(live.handle)
|
|
861
|
+
} catch {
|
|
862
|
+
// Best-effort cleanup after the old round reached a durable boundary.
|
|
863
|
+
}
|
|
864
|
+
this.producer = undefined
|
|
865
|
+
live = undefined
|
|
866
|
+
}
|
|
867
|
+
await this.ensureLease()
|
|
868
|
+
const streamBracket = await this.openSessionStream(resolved, {
|
|
869
|
+
session,
|
|
870
|
+
role: spec.role,
|
|
871
|
+
runner: resolved.runtime,
|
|
872
|
+
...(resolved.model !== undefined ? { model: resolved.model } : {}),
|
|
873
|
+
phase: `harvest:${spec.step}`,
|
|
874
|
+
round: spec.round,
|
|
875
|
+
})
|
|
876
|
+
const stream = streamBracket?.sink
|
|
877
|
+
const started = await store.appendRepo(repo, {
|
|
878
|
+
actor: KERNEL,
|
|
879
|
+
type: 'harvest.session.started',
|
|
880
|
+
payload: {
|
|
881
|
+
run: spec.run,
|
|
882
|
+
session,
|
|
883
|
+
role: spec.role,
|
|
884
|
+
runner: resolved.runtime,
|
|
885
|
+
...(resolved.model !== undefined ? { model: resolved.model } : {}),
|
|
886
|
+
args: [...resolved.args],
|
|
887
|
+
step: spec.step,
|
|
888
|
+
round: spec.round,
|
|
889
|
+
...(streamBracket !== undefined ? { stream: streamBracket.streamId } : {}),
|
|
890
|
+
},
|
|
891
|
+
})
|
|
892
|
+
|
|
893
|
+
let handle: AgentSessionHandle | undefined
|
|
894
|
+
let result: AgentTurnResult | undefined
|
|
895
|
+
let turnError: unknown
|
|
896
|
+
try {
|
|
897
|
+
if (live !== undefined) {
|
|
898
|
+
handle = live.handle
|
|
899
|
+
result = await live.runner.continue(
|
|
900
|
+
live.handle,
|
|
901
|
+
`Revise harvest proposals for round ${spec.round}: run ab harvest context, address .ab/findings.json, then submit.`,
|
|
902
|
+
{
|
|
903
|
+
env: this.sessionEnv(spec.run, spec.step, spec.round, session),
|
|
904
|
+
...(stream !== undefined ? { stream } : {}),
|
|
905
|
+
},
|
|
906
|
+
)
|
|
907
|
+
} else {
|
|
908
|
+
const turn = await resolved.runner.start({
|
|
909
|
+
skill: spec.skill,
|
|
910
|
+
invocation: spec.run,
|
|
911
|
+
workspacePath,
|
|
912
|
+
...(resolved.model !== undefined ? { model: resolved.model } : {}),
|
|
913
|
+
...(resolved.args !== undefined ? { args: resolved.args } : {}),
|
|
914
|
+
env: this.sessionEnv(spec.run, spec.step, spec.round, session),
|
|
915
|
+
...(stream !== undefined ? { stream } : {}),
|
|
916
|
+
})
|
|
917
|
+
handle = turn.session
|
|
918
|
+
result = turn.result
|
|
919
|
+
}
|
|
920
|
+
} catch (error) {
|
|
921
|
+
turnError = error
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
try {
|
|
925
|
+
await this.ensureLease()
|
|
926
|
+
} catch (error) {
|
|
927
|
+
// The turn may have crossed the lease-expiry boundary. Close adapter
|
|
928
|
+
// resources, but deposit no transcript/result after a replacement has
|
|
929
|
+
// taken ownership. Continued producers are closed by run()'s finally.
|
|
930
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
931
|
+
if (handle !== undefined && live === undefined) {
|
|
932
|
+
try {
|
|
933
|
+
await resolved.runner.end(handle)
|
|
934
|
+
} catch {
|
|
935
|
+
// A dead session may have nothing left to close.
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
throw error
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
const since = await store.getRepoEvents(repo, started.seq)
|
|
942
|
+
const terminal = turnError === undefined && since.some((event) => spec.terminal(event, session))
|
|
943
|
+
|
|
944
|
+
if (terminal && handle !== undefined && result !== undefined) {
|
|
945
|
+
const close =
|
|
946
|
+
streamBracket !== undefined
|
|
947
|
+
? { stream: streamBracket, outcome: 'completed' as const }
|
|
948
|
+
: undefined
|
|
949
|
+
if (spec.producer) {
|
|
950
|
+
this.producer = {
|
|
951
|
+
handle,
|
|
952
|
+
runner: live?.runner ?? resolved.runner,
|
|
953
|
+
route,
|
|
954
|
+
}
|
|
955
|
+
await this.depositTranscript(
|
|
956
|
+
spec,
|
|
957
|
+
session,
|
|
958
|
+
JSON.stringify(
|
|
959
|
+
{
|
|
960
|
+
session,
|
|
961
|
+
run: spec.run,
|
|
962
|
+
step: spec.step,
|
|
963
|
+
round: spec.round,
|
|
964
|
+
turn: result,
|
|
965
|
+
note: 'producer session kept live for convergence continuation',
|
|
966
|
+
},
|
|
967
|
+
null,
|
|
968
|
+
2,
|
|
969
|
+
),
|
|
970
|
+
result.usage,
|
|
971
|
+
resolved.model,
|
|
972
|
+
close,
|
|
973
|
+
)
|
|
974
|
+
} else {
|
|
975
|
+
const transcript = await resolved.runner.end(handle)
|
|
976
|
+
await this.depositTranscript(
|
|
977
|
+
spec,
|
|
978
|
+
session,
|
|
979
|
+
transcript.content,
|
|
980
|
+
transcript.metadata.usage,
|
|
981
|
+
transcript.metadata.model ?? resolved.model,
|
|
982
|
+
close,
|
|
983
|
+
)
|
|
984
|
+
}
|
|
985
|
+
return
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
if (handle !== undefined) {
|
|
989
|
+
try {
|
|
990
|
+
const owner = live?.runner ?? resolved.runner
|
|
991
|
+
const transcript = await owner.end(handle)
|
|
992
|
+
await this.depositTranscript(
|
|
993
|
+
spec,
|
|
994
|
+
session,
|
|
995
|
+
transcript.content,
|
|
996
|
+
transcript.metadata.usage,
|
|
997
|
+
transcript.metadata.model ?? resolved.model,
|
|
998
|
+
streamBracket !== undefined
|
|
999
|
+
? { stream: streamBracket, outcome: 'aborted' as const }
|
|
1000
|
+
: undefined,
|
|
1001
|
+
)
|
|
1002
|
+
} catch (error) {
|
|
1003
|
+
// The deposit was skipped, so the stream must not stay open behind
|
|
1004
|
+
// the rethrow (lease lost) or the failure appends below.
|
|
1005
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
1006
|
+
if (error instanceof HarvestLeaseLostError) throw error
|
|
1007
|
+
// A dead agent may have no recoverable transcript.
|
|
1008
|
+
}
|
|
1009
|
+
} else {
|
|
1010
|
+
// No handle: the turn failed before the adapter owned a session, so
|
|
1011
|
+
// no deposit will run — close the bracket's stream here.
|
|
1012
|
+
this.closeBracketStream(streamBracket, 'aborted')
|
|
1013
|
+
}
|
|
1014
|
+
if (spec.producer) this.producer = undefined
|
|
1015
|
+
const attempt = failures + 1
|
|
1016
|
+
const structuredFailure =
|
|
1017
|
+
turnError === undefined && result?.kind === 'failed' ? result.failure : undefined
|
|
1018
|
+
const failureMessage =
|
|
1019
|
+
turnError !== undefined
|
|
1020
|
+
? errorMessage(turnError)
|
|
1021
|
+
: (structuredFailure?.message ?? 'no-terminal')
|
|
1022
|
+
const willRetry = structuredFailure?.permanent !== true && attempt < this.maxSessionAttempts
|
|
1023
|
+
await this.ensureLease()
|
|
1024
|
+
await store.appendRepo(repo, {
|
|
1025
|
+
actor: KERNEL,
|
|
1026
|
+
type: 'harvest.failed',
|
|
1027
|
+
payload: {
|
|
1028
|
+
run: spec.run,
|
|
1029
|
+
step: spec.step,
|
|
1030
|
+
round: spec.round,
|
|
1031
|
+
attempt,
|
|
1032
|
+
error: failureMessage,
|
|
1033
|
+
willRetry,
|
|
1034
|
+
},
|
|
1035
|
+
})
|
|
1036
|
+
await this.ensureLease()
|
|
1037
|
+
await store.appendRepo(repo, {
|
|
1038
|
+
actor: KERNEL,
|
|
1039
|
+
type: 'harvest.step.completed',
|
|
1040
|
+
payload: {
|
|
1041
|
+
run: spec.run,
|
|
1042
|
+
step: spec.step,
|
|
1043
|
+
round: spec.round,
|
|
1044
|
+
outcome: 'failed',
|
|
1045
|
+
detail: failureMessage,
|
|
1046
|
+
},
|
|
1047
|
+
})
|
|
1048
|
+
throw new SessionFailure(`${spec.step}@${spec.round} failed: ${failureMessage}`)
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
private async depositTranscript(
|
|
1052
|
+
spec: {
|
|
1053
|
+
run: string
|
|
1054
|
+
role: 'harvest' | 'harvest-review'
|
|
1055
|
+
step: 'synthesize' | 'review'
|
|
1056
|
+
round: number
|
|
1057
|
+
},
|
|
1058
|
+
session: string,
|
|
1059
|
+
content: string,
|
|
1060
|
+
usage: { inputTokens: number; outputTokens: number; turns: number },
|
|
1061
|
+
model?: string,
|
|
1062
|
+
close?: { stream: SessionBracketStream; outcome: StreamOutcome },
|
|
1063
|
+
): Promise<void> {
|
|
1064
|
+
if (close !== undefined) {
|
|
1065
|
+
// Close must land no later than `harvest.session.ended`; this makes
|
|
1066
|
+
// the ordering structural rather than call-site discipline.
|
|
1067
|
+
await close.stream.sink.close(close.outcome)
|
|
1068
|
+
}
|
|
1069
|
+
await this.ensureLease()
|
|
1070
|
+
await this.deps.store.appendRepoWithArtifacts(
|
|
1071
|
+
this.deps.repo,
|
|
1072
|
+
[
|
|
1073
|
+
{
|
|
1074
|
+
kind: HARVEST_TRANSCRIPT_ARTIFACT,
|
|
1075
|
+
content,
|
|
1076
|
+
metadata: {
|
|
1077
|
+
run: spec.run,
|
|
1078
|
+
step: spec.step,
|
|
1079
|
+
round: spec.round,
|
|
1080
|
+
role: spec.role,
|
|
1081
|
+
session,
|
|
1082
|
+
...(model !== undefined ? { model } : {}),
|
|
1083
|
+
usage,
|
|
1084
|
+
},
|
|
1085
|
+
},
|
|
1086
|
+
],
|
|
1087
|
+
(deposited) => ({
|
|
1088
|
+
actor: KERNEL,
|
|
1089
|
+
type: 'harvest.session.ended',
|
|
1090
|
+
payload: {
|
|
1091
|
+
run: spec.run,
|
|
1092
|
+
session,
|
|
1093
|
+
transcript: artifactRef(deposited[0]!),
|
|
1094
|
+
usage,
|
|
1095
|
+
},
|
|
1096
|
+
}),
|
|
1097
|
+
)
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
private async file(run: HarvestRunState, approved: ArtifactRef): Promise<void> {
|
|
1101
|
+
const { store, repo, tickets, uuids } = this.deps
|
|
1102
|
+
const config = this.currentConfig()
|
|
1103
|
+
// Filing is one deterministic side-effecting unit: settle pause before it,
|
|
1104
|
+
// then leave reservation/create/adoption bookkeeping uninterrupted.
|
|
1105
|
+
await this.controlBoundary(run.run)
|
|
1106
|
+
await this.ensureLease()
|
|
1107
|
+
const current = await this.refreshRun(run.run)
|
|
1108
|
+
if (current.status === 'completed') return
|
|
1109
|
+
await this.startStep(run.run, 'file')
|
|
1110
|
+
const artifact = await store.getRepoArtifact(repo, approved.kind, approved.rev)
|
|
1111
|
+
if (!artifact) {
|
|
1112
|
+
throw new Error(`approved proposal artifact ${approved.kind}@${approved.rev} is missing`)
|
|
1113
|
+
}
|
|
1114
|
+
const set = parseApprovedProposalSet(
|
|
1115
|
+
new TextDecoder().decode(artifact.content),
|
|
1116
|
+
current.observations,
|
|
1117
|
+
)
|
|
1118
|
+
const packet = await loadScanPacket(store, repo, current.scan)
|
|
1119
|
+
const observations = packet.observations
|
|
1120
|
+
const knownLedger = new Map(
|
|
1121
|
+
packet.ledger.map((entry) => [`${entry.ticket.source}:${entry.ticket.id}`, entry]),
|
|
1122
|
+
)
|
|
1123
|
+
const alreadyFiled = new Map(current.filed.map((entry) => [entry.proposalKey, entry]))
|
|
1124
|
+
const reservations = new Map(current.reservations.map((entry) => [entry.proposalKey, entry.id]))
|
|
1125
|
+
const dispositions: HarvestDisposition[] = []
|
|
1126
|
+
const report: Array<Record<string, unknown>> = []
|
|
1127
|
+
|
|
1128
|
+
for (const proposal of set.proposals) {
|
|
1129
|
+
await this.ensureLease()
|
|
1130
|
+
if (proposal.action === 'create') {
|
|
1131
|
+
const proposalKey = harvestProposalKey(proposal)
|
|
1132
|
+
let filed = alreadyFiled.get(proposalKey)
|
|
1133
|
+
if (filed === undefined) {
|
|
1134
|
+
// Resolution must precede even the durable reservation: an invalid
|
|
1135
|
+
// declared prerequisite cannot leave behind the start of a ready-
|
|
1136
|
+
// ticket create path, while origins are refreshed at this boundary.
|
|
1137
|
+
const blockers = await resolveHarvestCreateBlockers(proposal, observations, tickets)
|
|
1138
|
+
let reservedId = reservations.get(proposalKey)
|
|
1139
|
+
if (reservedId === undefined) {
|
|
1140
|
+
reservedId = uuids()
|
|
1141
|
+
await this.ensureLease()
|
|
1142
|
+
await store.appendRepo(repo, {
|
|
1143
|
+
actor: KERNEL,
|
|
1144
|
+
type: 'harvest.proposal.id-reserved',
|
|
1145
|
+
payload: { run: run.run, proposalKey, id: reservedId },
|
|
1146
|
+
})
|
|
1147
|
+
await this.ensureLease()
|
|
1148
|
+
reservations.set(proposalKey, reservedId)
|
|
1149
|
+
}
|
|
1150
|
+
const body = renderHarvestProposal(proposal, observations)
|
|
1151
|
+
const ticket = (
|
|
1152
|
+
await tickets.create(
|
|
1153
|
+
{
|
|
1154
|
+
title: proposal.title,
|
|
1155
|
+
body,
|
|
1156
|
+
labels: [HARVEST_PROPOSAL_LABEL],
|
|
1157
|
+
...(blockers.blockedBy.length > 0 ? { blockedBy: blockers.blockedBy } : {}),
|
|
1158
|
+
},
|
|
1159
|
+
{
|
|
1160
|
+
state: defaultProposalState(config),
|
|
1161
|
+
idempotencyKey: reservedId,
|
|
1162
|
+
},
|
|
1163
|
+
)
|
|
1164
|
+
).ref
|
|
1165
|
+
// If the process stops before create, the replacement reads the
|
|
1166
|
+
// reservation. If it stops after create but before this filing fact,
|
|
1167
|
+
// it sends the same reserved id and adopts the external ticket.
|
|
1168
|
+
await this.ensureLease()
|
|
1169
|
+
const filedEvent = await store.appendRepo(repo, {
|
|
1170
|
+
actor: KERNEL,
|
|
1171
|
+
type: 'harvest.proposal.filed',
|
|
1172
|
+
payload: {
|
|
1173
|
+
run: run.run,
|
|
1174
|
+
proposalKey,
|
|
1175
|
+
ticket,
|
|
1176
|
+
blockers: blockers.provenance,
|
|
1177
|
+
},
|
|
1178
|
+
})
|
|
1179
|
+
filed = {
|
|
1180
|
+
proposalKey,
|
|
1181
|
+
ticket,
|
|
1182
|
+
blockers: blockers.provenance,
|
|
1183
|
+
seq: filedEvent.seq,
|
|
1184
|
+
}
|
|
1185
|
+
alreadyFiled.set(proposalKey, filed)
|
|
1186
|
+
}
|
|
1187
|
+
for (const occurrence of proposal.observations) {
|
|
1188
|
+
dispositions.push({
|
|
1189
|
+
occurrence,
|
|
1190
|
+
action: 'filed',
|
|
1191
|
+
proposalKey,
|
|
1192
|
+
ticket: filed.ticket,
|
|
1193
|
+
})
|
|
1194
|
+
}
|
|
1195
|
+
report.push({
|
|
1196
|
+
action: 'filed',
|
|
1197
|
+
proposalKey,
|
|
1198
|
+
ticket: filed.ticket,
|
|
1199
|
+
...(filed.blockers !== undefined ? { blockers: filed.blockers } : {}),
|
|
1200
|
+
})
|
|
1201
|
+
} else if (proposal.action === 'join') {
|
|
1202
|
+
const known = knownLedger.get(`${proposal.ticket.source}:${proposal.ticket.id}`)
|
|
1203
|
+
if (!known) {
|
|
1204
|
+
throw new Error(
|
|
1205
|
+
`approved join target ${proposal.ticket.source}:${proposal.ticket.id} is not in the ledger context`,
|
|
1206
|
+
)
|
|
1207
|
+
}
|
|
1208
|
+
if (!known.exists || known.resolved) {
|
|
1209
|
+
throw new Error(
|
|
1210
|
+
`approved join target ${proposal.ticket.id} is ${
|
|
1211
|
+
!known.exists ? 'missing' : 'resolved'
|
|
1212
|
+
}; it is a tombstone and must be suppressed, not joined`,
|
|
1213
|
+
)
|
|
1214
|
+
}
|
|
1215
|
+
for (const occurrence of proposal.observations) {
|
|
1216
|
+
dispositions.push({
|
|
1217
|
+
occurrence,
|
|
1218
|
+
action: 'joined',
|
|
1219
|
+
proposalKey: known.proposalKey,
|
|
1220
|
+
ticket: known.ticket,
|
|
1221
|
+
reason: proposal.reason,
|
|
1222
|
+
})
|
|
1223
|
+
}
|
|
1224
|
+
report.push({
|
|
1225
|
+
action: 'joined',
|
|
1226
|
+
proposalKey: known.proposalKey,
|
|
1227
|
+
ticket: known.ticket,
|
|
1228
|
+
})
|
|
1229
|
+
} else {
|
|
1230
|
+
const proposalKey = harvestProposalKey(proposal)
|
|
1231
|
+
for (const occurrence of proposal.observations) {
|
|
1232
|
+
dispositions.push({
|
|
1233
|
+
occurrence,
|
|
1234
|
+
action: 'suppressed',
|
|
1235
|
+
proposalKey,
|
|
1236
|
+
reason: proposal.reason,
|
|
1237
|
+
})
|
|
1238
|
+
}
|
|
1239
|
+
report.push({ action: 'suppressed', proposalKey, reason: proposal.reason })
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
const expected = new Set(current.observations.map(occurrenceKey))
|
|
1244
|
+
if (
|
|
1245
|
+
dispositions.length !== expected.size ||
|
|
1246
|
+
dispositions.some((entry) => !expected.has(occurrenceKey(entry.occurrence)))
|
|
1247
|
+
) {
|
|
1248
|
+
throw new Error('filing dispositions do not partition the claimed snapshot')
|
|
1249
|
+
}
|
|
1250
|
+
await this.completeStep(run.run, 'file', undefined, 'completed')
|
|
1251
|
+
await this.ensureLease()
|
|
1252
|
+
await store.appendRepoWithArtifacts(
|
|
1253
|
+
repo,
|
|
1254
|
+
[
|
|
1255
|
+
{
|
|
1256
|
+
kind: HARVEST_REPORT_ARTIFACT,
|
|
1257
|
+
content: `${JSON.stringify({ run: run.run, dispositions, proposals: report }, null, 2)}\n`,
|
|
1258
|
+
metadata: { run: run.run, observations: dispositions.length },
|
|
1259
|
+
},
|
|
1260
|
+
],
|
|
1261
|
+
(deposited) => ({
|
|
1262
|
+
actor: KERNEL,
|
|
1263
|
+
type: 'harvest.completed',
|
|
1264
|
+
payload: {
|
|
1265
|
+
run: run.run,
|
|
1266
|
+
dispositions,
|
|
1267
|
+
report: artifactRef(deposited[0]!),
|
|
1268
|
+
},
|
|
1269
|
+
}),
|
|
1270
|
+
)
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
private async escalate(
|
|
1274
|
+
run: HarvestRunState,
|
|
1275
|
+
source: 'agent' | 'stall' | 'policy',
|
|
1276
|
+
reason: string,
|
|
1277
|
+
round?: number,
|
|
1278
|
+
): Promise<void> {
|
|
1279
|
+
await this.controlBoundary(run.run)
|
|
1280
|
+
await this.ensureLease()
|
|
1281
|
+
const refreshed = await this.refreshRun(run.run)
|
|
1282
|
+
if (refreshed.status !== 'running') return
|
|
1283
|
+
await this.ensureLease()
|
|
1284
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1285
|
+
actor: KERNEL,
|
|
1286
|
+
type: 'harvest.escalated',
|
|
1287
|
+
payload: {
|
|
1288
|
+
run: run.run,
|
|
1289
|
+
source,
|
|
1290
|
+
reason,
|
|
1291
|
+
...(round !== undefined && round > 0 ? { round } : {}),
|
|
1292
|
+
observations: refreshed.observations,
|
|
1293
|
+
},
|
|
1294
|
+
})
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
private async recordWorkflowFailure(run: string, error: unknown): Promise<void> {
|
|
1298
|
+
await this.ensureLease()
|
|
1299
|
+
const events = await this.deps.store.getRepoEvents(this.deps.repo)
|
|
1300
|
+
const state = reduceHarvest(events)
|
|
1301
|
+
const current = state.runs.find((candidate) => candidate.run === run)
|
|
1302
|
+
if (current?.status !== 'running') return
|
|
1303
|
+
const open = [...current.steps]
|
|
1304
|
+
.reverse()
|
|
1305
|
+
.find((occurrence) => occurrence.completedSeq === undefined)
|
|
1306
|
+
const step = open?.step ?? 'file'
|
|
1307
|
+
const round = open?.round
|
|
1308
|
+
const prior = events.filter(
|
|
1309
|
+
(event) =>
|
|
1310
|
+
event.type === 'harvest.failed' &&
|
|
1311
|
+
event.payload.run === run &&
|
|
1312
|
+
event.payload.step === step &&
|
|
1313
|
+
event.payload.round === round,
|
|
1314
|
+
).length
|
|
1315
|
+
const attempt = prior + 1
|
|
1316
|
+
const willRetry = attempt < this.maxSessionAttempts
|
|
1317
|
+
const message = errorMessage(error)
|
|
1318
|
+
await this.ensureLease()
|
|
1319
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1320
|
+
actor: KERNEL,
|
|
1321
|
+
type: 'harvest.failed',
|
|
1322
|
+
payload: {
|
|
1323
|
+
run,
|
|
1324
|
+
step,
|
|
1325
|
+
...(round !== undefined ? { round } : {}),
|
|
1326
|
+
attempt,
|
|
1327
|
+
error: message,
|
|
1328
|
+
willRetry,
|
|
1329
|
+
},
|
|
1330
|
+
})
|
|
1331
|
+
if (open !== undefined) {
|
|
1332
|
+
await this.ensureLease()
|
|
1333
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1334
|
+
actor: KERNEL,
|
|
1335
|
+
type: 'harvest.step.completed',
|
|
1336
|
+
payload: {
|
|
1337
|
+
run,
|
|
1338
|
+
step,
|
|
1339
|
+
...(round !== undefined ? { round } : {}),
|
|
1340
|
+
outcome: 'failed',
|
|
1341
|
+
detail: message,
|
|
1342
|
+
},
|
|
1343
|
+
})
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/** If the just-failed execution consumed the final durable reopen, settle
|
|
1348
|
+
* give-up before returning. Initial/within-step failures do not spend this
|
|
1349
|
+
* outer budget and therefore remain recoverable. */
|
|
1350
|
+
private async settleRecoveryExhaustion(run: string): Promise<void> {
|
|
1351
|
+
await this.ensureLease()
|
|
1352
|
+
const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
|
|
1353
|
+
const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
|
|
1354
|
+
if (decision.kind !== 'exhaust-recovery' || decision.run !== run) return
|
|
1355
|
+
await this.finalizeRecoveryExhaustion(decision.run, decision.attempts, decision.limit)
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
/** Compute and append the selective-release boundary. The second decision
|
|
1359
|
+
* check closes the race with a human pause/resume request that can be written
|
|
1360
|
+
* without this process's repository lease. */
|
|
1361
|
+
private async finalizeRecoveryExhaustion(
|
|
1362
|
+
runId: string,
|
|
1363
|
+
attempts: number,
|
|
1364
|
+
limit: number,
|
|
1365
|
+
): Promise<boolean> {
|
|
1366
|
+
await this.ensureLease()
|
|
1367
|
+
let run = await this.refreshRun(runId)
|
|
1368
|
+
if (run.recoveryExhaustion !== undefined) return true
|
|
1369
|
+
if (run.status !== 'failed' || run.failure === undefined) return false
|
|
1370
|
+
const partition = await partitionHarvestExhaustion({
|
|
1371
|
+
store: this.deps.store,
|
|
1372
|
+
repo: this.deps.repo,
|
|
1373
|
+
run,
|
|
1374
|
+
})
|
|
1375
|
+
|
|
1376
|
+
await this.ensureLease()
|
|
1377
|
+
const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
|
|
1378
|
+
const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
|
|
1379
|
+
if (
|
|
1380
|
+
decision.kind !== 'exhaust-recovery' ||
|
|
1381
|
+
decision.run !== runId ||
|
|
1382
|
+
decision.attempts !== attempts ||
|
|
1383
|
+
decision.limit !== limit
|
|
1384
|
+
) {
|
|
1385
|
+
return false
|
|
1386
|
+
}
|
|
1387
|
+
run = state.runs.find((candidate) => candidate.run === runId)!
|
|
1388
|
+
if (run.failure === undefined) return false
|
|
1389
|
+
await this.ensureLease()
|
|
1390
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1391
|
+
actor: KERNEL,
|
|
1392
|
+
type: 'harvest.recovery-exhausted',
|
|
1393
|
+
payload: {
|
|
1394
|
+
run: runId,
|
|
1395
|
+
step: run.failure.step,
|
|
1396
|
+
...(run.failure.round !== undefined ? { round: run.failure.round } : {}),
|
|
1397
|
+
error: run.failure.error,
|
|
1398
|
+
attempts,
|
|
1399
|
+
limit,
|
|
1400
|
+
...partition,
|
|
1401
|
+
},
|
|
1402
|
+
})
|
|
1403
|
+
return true
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
/** Settle repository-wide commands and automatic recovery only at a durable
|
|
1407
|
+
* workflow boundary. Requests and acknowledgements are separate facts, so a
|
|
1408
|
+
* replacement process can finish the same transition without spending a
|
|
1409
|
+
* second recovery attempt. */
|
|
1410
|
+
private async controlBoundary(run?: string): Promise<void> {
|
|
1411
|
+
while (true) {
|
|
1412
|
+
await this.ensureLease()
|
|
1413
|
+
const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
|
|
1414
|
+
const decision = decideHarvestControl(state, this.maxRecoveryAttempts)
|
|
1415
|
+
if (decision.kind === 'proceed') return
|
|
1416
|
+
if (decision.kind === 'park') {
|
|
1417
|
+
throw new HarvestParkedSignal(actionableHarvestRun(state)?.run ?? run)
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
if (decision.kind === 'request-recovery') {
|
|
1421
|
+
await this.ensureLease()
|
|
1422
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1423
|
+
actor: KERNEL,
|
|
1424
|
+
type: 'harvest.recovery-requested',
|
|
1425
|
+
payload: {
|
|
1426
|
+
run: decision.run,
|
|
1427
|
+
attempt: decision.attempt,
|
|
1428
|
+
limit: decision.limit,
|
|
1429
|
+
},
|
|
1430
|
+
})
|
|
1431
|
+
continue
|
|
1432
|
+
}
|
|
1433
|
+
if (decision.kind === 'exhaust-recovery') {
|
|
1434
|
+
const exhausted = await this.finalizeRecoveryExhaustion(
|
|
1435
|
+
decision.run,
|
|
1436
|
+
decision.attempts,
|
|
1437
|
+
decision.limit,
|
|
1438
|
+
)
|
|
1439
|
+
if (exhausted) throw new HarvestParkedSignal(decision.run)
|
|
1440
|
+
continue
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
await this.ensureLease()
|
|
1444
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1445
|
+
actor: KERNEL,
|
|
1446
|
+
type: decision.command === 'pause' ? 'harvest.paused' : 'harvest.resumed',
|
|
1447
|
+
payload: {},
|
|
1448
|
+
})
|
|
1449
|
+
// Re-reduce after every acknowledgement. An opposing request may have
|
|
1450
|
+
// raced the append; the newest durable intent must win before this
|
|
1451
|
+
// runner either parks or starts another unit.
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
private async refreshRun(run: string): Promise<HarvestRunState> {
|
|
1456
|
+
const state = reduceHarvest(await this.deps.store.getRepoEvents(this.deps.repo))
|
|
1457
|
+
const found = state.runs.find((candidate) => candidate.run === run)
|
|
1458
|
+
if (!found) throw new Error(`unknown harvest run "${run}"`)
|
|
1459
|
+
return found
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
private async ensureScanStep(run: HarvestRunState): Promise<void> {
|
|
1463
|
+
await this.ensureLease()
|
|
1464
|
+
const completed = run.steps.some(
|
|
1465
|
+
(occurrence) =>
|
|
1466
|
+
occurrence.step === 'scan' &&
|
|
1467
|
+
occurrence.completedSeq !== undefined &&
|
|
1468
|
+
occurrence.outcome === 'completed',
|
|
1469
|
+
)
|
|
1470
|
+
if (completed) return
|
|
1471
|
+
const open = run.steps.some(
|
|
1472
|
+
(occurrence) => occurrence.step === 'scan' && occurrence.completedSeq === undefined,
|
|
1473
|
+
)
|
|
1474
|
+
if (!open) {
|
|
1475
|
+
await this.ensureLease()
|
|
1476
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1477
|
+
actor: KERNEL,
|
|
1478
|
+
type: 'harvest.step.started',
|
|
1479
|
+
payload: { run: run.run, step: 'scan' },
|
|
1480
|
+
})
|
|
1481
|
+
}
|
|
1482
|
+
await this.ensureLease()
|
|
1483
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1484
|
+
actor: KERNEL,
|
|
1485
|
+
type: 'harvest.step.completed',
|
|
1486
|
+
payload: {
|
|
1487
|
+
run: run.run,
|
|
1488
|
+
step: 'scan',
|
|
1489
|
+
outcome: 'completed',
|
|
1490
|
+
artifact: run.scan,
|
|
1491
|
+
},
|
|
1492
|
+
})
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
private async startStep(
|
|
1496
|
+
run: string,
|
|
1497
|
+
step: 'synthesize' | 'review' | 'file',
|
|
1498
|
+
round?: number,
|
|
1499
|
+
): Promise<void> {
|
|
1500
|
+
await this.ensureLease()
|
|
1501
|
+
const refreshed = await this.refreshRun(run)
|
|
1502
|
+
const open = refreshed.steps.some(
|
|
1503
|
+
(occurrence) =>
|
|
1504
|
+
occurrence.step === step &&
|
|
1505
|
+
occurrence.round === round &&
|
|
1506
|
+
occurrence.completedSeq === undefined,
|
|
1507
|
+
)
|
|
1508
|
+
if (open) return
|
|
1509
|
+
await this.ensureLease()
|
|
1510
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1511
|
+
actor: KERNEL,
|
|
1512
|
+
type: 'harvest.step.started',
|
|
1513
|
+
payload: { run, step, ...(round !== undefined ? { round } : {}) },
|
|
1514
|
+
})
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
private async completeStep(
|
|
1518
|
+
run: string,
|
|
1519
|
+
step: 'synthesize' | 'review' | 'file',
|
|
1520
|
+
round: number | undefined,
|
|
1521
|
+
outcome: 'completed' | 'approve' | 'revise' | 'escalate' | 'failed',
|
|
1522
|
+
artifact?: ArtifactRef,
|
|
1523
|
+
): Promise<void> {
|
|
1524
|
+
await this.ensureLease()
|
|
1525
|
+
await this.deps.store.appendRepo(this.deps.repo, {
|
|
1526
|
+
actor: KERNEL,
|
|
1527
|
+
type: 'harvest.step.completed',
|
|
1528
|
+
payload: {
|
|
1529
|
+
run,
|
|
1530
|
+
step,
|
|
1531
|
+
outcome,
|
|
1532
|
+
...(round !== undefined ? { round } : {}),
|
|
1533
|
+
...(artifact !== undefined ? { artifact } : {}),
|
|
1534
|
+
},
|
|
1535
|
+
})
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
private async ensureStepCompleted(
|
|
1539
|
+
run: string,
|
|
1540
|
+
step: 'synthesize' | 'review',
|
|
1541
|
+
round: number,
|
|
1542
|
+
outcome: 'completed' | 'approve' | 'revise' | 'escalate',
|
|
1543
|
+
artifact: ArtifactRef,
|
|
1544
|
+
): Promise<void> {
|
|
1545
|
+
const refreshed = await this.refreshRun(run)
|
|
1546
|
+
const done = refreshed.steps.some(
|
|
1547
|
+
(occurrence) =>
|
|
1548
|
+
occurrence.step === step &&
|
|
1549
|
+
occurrence.round === round &&
|
|
1550
|
+
occurrence.completedSeq !== undefined &&
|
|
1551
|
+
occurrence.outcome === outcome,
|
|
1552
|
+
)
|
|
1553
|
+
if (!done) {
|
|
1554
|
+
await this.completeStep(run, step, round, outcome, artifact)
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
private sessionEnv(
|
|
1559
|
+
run: string,
|
|
1560
|
+
step: 'synthesize' | 'review',
|
|
1561
|
+
round: number,
|
|
1562
|
+
session: string,
|
|
1563
|
+
): Record<string, string> {
|
|
1564
|
+
return {
|
|
1565
|
+
AB_STORE: 'local',
|
|
1566
|
+
...this.deps.sessionEnv,
|
|
1567
|
+
AB_REPO: this.deps.repo,
|
|
1568
|
+
AB_HARVEST: run,
|
|
1569
|
+
AB_PHASE: `${step}@${round}`,
|
|
1570
|
+
AB_SESSION: session,
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
// ── Session streams (SPEC §9) ────────────────────────────────────────────
|
|
1575
|
+
|
|
1576
|
+
/**
|
|
1577
|
+
* Open one stream for this harvest session bracket through the resolved
|
|
1578
|
+
* target's streaming capability: create the store-backed sink, call the
|
|
1579
|
+
* capability, and — once it returns an id — emit the bracket's
|
|
1580
|
+
* `data-ab-session` part. Returns undefined (and reports once) when
|
|
1581
|
+
* streaming is disabled, the runtime declared no capability, or the opener
|
|
1582
|
+
* declined/failed; the bracket then proceeds exactly as without streams.
|
|
1583
|
+
* Harvest streams are repository-scoped, the harvest analog of the
|
|
1584
|
+
* build-runner's build-scoped bracket streams.
|
|
1585
|
+
*/
|
|
1586
|
+
private async openSessionStream(
|
|
1587
|
+
target: ResolvedRuntime,
|
|
1588
|
+
info: SessionStreamInfo,
|
|
1589
|
+
): Promise<SessionBracketStream | undefined> {
|
|
1590
|
+
if (!this.streamSessions) return undefined
|
|
1591
|
+
const open = target.openSessionStream
|
|
1592
|
+
if (open === undefined) return undefined
|
|
1593
|
+
const sink = createSessionStreamSink({
|
|
1594
|
+
store: this.deps.store,
|
|
1595
|
+
scope: { kind: 'repo', repo: this.deps.repo },
|
|
1596
|
+
onDiagnostic: this.onDiagnostic,
|
|
1597
|
+
})
|
|
1598
|
+
try {
|
|
1599
|
+
const streamId = await open(sink, info)
|
|
1600
|
+
if (streamId === undefined) return undefined
|
|
1601
|
+
sink.append([sessionPart(info)])
|
|
1602
|
+
return { sink, streamId }
|
|
1603
|
+
} catch (error) {
|
|
1604
|
+
this.onDiagnostic(
|
|
1605
|
+
`session stream for "${info.session}" failed to open: ${errorMessage(error)}`,
|
|
1606
|
+
)
|
|
1607
|
+
return undefined
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/** Best-effort belt-and-braces close; idempotency makes repeated calls
|
|
1612
|
+
* harmless and lets the deposit path own the close-then-append ordering. */
|
|
1613
|
+
private closeBracketStream(
|
|
1614
|
+
stream: SessionBracketStream | undefined,
|
|
1615
|
+
outcome: StreamOutcome,
|
|
1616
|
+
): void {
|
|
1617
|
+
if (stream === undefined) return
|
|
1618
|
+
void stream.sink.close(outcome)
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
/** Re-acquire a lapsed lease only when no replacement won it. Checked at
|
|
1622
|
+
* every durable boundary; work already in an agent turn cannot be revoked,
|
|
1623
|
+
* but this process performs no subsequent kernel/file action without
|
|
1624
|
+
* ownership. */
|
|
1625
|
+
private async ensureLease(): Promise<void> {
|
|
1626
|
+
if (!this.leaseLost) return
|
|
1627
|
+
const { store, repo } = this.deps
|
|
1628
|
+
const holder = this.deps.leaseHolder ?? this.deps.instance
|
|
1629
|
+
const reclaimed = await store.claimRepoLease(repo, holder, this.leaseTtlMs)
|
|
1630
|
+
if (!reclaimed) throw new HarvestLeaseLostError(repo, holder)
|
|
1631
|
+
this.leaseLost = false
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
private async startHeartbeat(): Promise<void> {
|
|
1635
|
+
const { store, repo } = this.deps
|
|
1636
|
+
const holder = this.deps.leaseHolder ?? this.deps.instance
|
|
1637
|
+
if (!(await store.heartbeatRepo(repo, holder))) this.leaseLost = true
|
|
1638
|
+
this.heartbeatTimer = setInterval(() => {
|
|
1639
|
+
store.heartbeatRepo(repo, holder).then(
|
|
1640
|
+
(alive) => {
|
|
1641
|
+
if (!alive) this.leaseLost = true
|
|
1642
|
+
},
|
|
1643
|
+
() => {
|
|
1644
|
+
// Store unreachable: retry on the next beat. A later false result
|
|
1645
|
+
// proves expiry/takeover; rejecting timer promises are contained.
|
|
1646
|
+
},
|
|
1647
|
+
)
|
|
1648
|
+
}, this.heartbeatMs)
|
|
1649
|
+
this.heartbeatTimer.unref?.()
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
private stopHeartbeat(): void {
|
|
1653
|
+
if (this.heartbeatTimer !== undefined) clearInterval(this.heartbeatTimer)
|
|
1654
|
+
this.heartbeatTimer = undefined
|
|
1655
|
+
}
|
|
1656
|
+
}
|