@contextrail/assess 1.0.0-alpha.3 → 1.0.0-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -71,6 +71,121 @@ Source excerpts and selected guidance are sent to your chosen model provider. De
71
71
  quotes and conversations are not included in ContextRail's assessment-feedback payload.
72
72
  See [where data goes](docs/security.md) before connecting a repository.
73
73
 
74
+ ## Give large pull requests more capacity
75
+
76
+ A standard run keeps cost and time predictable. A large pull request can stop before it finishes:
77
+ the PR comment reports an incomplete assessment, and the run's stop reason is `max_time_ms` (time
78
+ limit reached) or `model_call_limit` (a verification used all its model calls). You do not have to
79
+ change how your team sizes pull requests. Instead, let a maintainer re-run that PR with more capacity
80
+ by adding a second label.
81
+
82
+ Three Action inputs set a run's capacity:
83
+
84
+ | Input | Standard run | Suggested extended run | What it changes |
85
+ | ------------------ | ------------------ | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
86
+ | `speed` | `normal` (default) | `fast` | Requests the provider's priority tier: quicker calls at a higher per-token price. `ultrafast` is also accepted. If a model doesn't serve a tier, Assess logs `provider_speed_unavailable` and continues. |
87
+ | `max-time-ms` | `600000` (10 min) | `1500000` (25 min) | Model time limit for the run. Keep the job's `timeout-minutes` about 10 minutes above it for setup and publishing. |
88
+ | `max-verify-calls` | empty (12) | `30` | Model calls each finding's independent verification may use. |
89
+
90
+ ### 1. Create the labels
91
+
92
+ ```sh
93
+ gh label create contextrail-assess --color 1D76DB \
94
+ --description "Run ContextRail Assess on this pull request"
95
+ gh label create contextrail-assess-extended --color 5319E7 \
96
+ --description "Re-run ContextRail Assess with extended capacity"
97
+ ```
98
+
99
+ In the GitHub UI: **Pull requests → Labels → New label**. Any names work if the workflow uses the same ones.
100
+
101
+ ### 2. Optionally tune the extended run
102
+
103
+ The workflow below has built-in defaults. To change them without editing it, set repository variables:
104
+
105
+ ```sh
106
+ gh variable set ASSESS_EXTENDED_SPEED --body fast
107
+ gh variable set ASSESS_EXTENDED_MAX_TIME_MS --body 1500000
108
+ gh variable set ASSESS_EXTENDED_MAX_VERIFY_CALLS --body 30
109
+ ```
110
+
111
+ In the GitHub UI: **Settings → Secrets and variables → Actions → Variables → New repository variable**.
112
+ These names belong to your workflow; the Action itself reads only its inputs. Set up the connection
113
+ variables and secrets first, as described in [GitHub Actions](docs/github-actions.md#prepare-the-repository-settings).
114
+
115
+ ### 3. Add the workflow
116
+
117
+ Save as `.github/workflows/contextrail-assess.yml` on the default branch and replace the reviewer skill ID:
118
+
119
+ ```yaml
120
+ name: ContextRail Assess
121
+ on:
122
+ pull_request_target:
123
+ types: [labeled]
124
+ permissions: {}
125
+ concurrency:
126
+ group: contextrail-assess-${{ github.repository }}-${{ github.event.pull_request.number }}
127
+ cancel-in-progress: false
128
+ jobs:
129
+ assess:
130
+ # Only maintainers with triage or write access can add labels. Same-repository human PRs only.
131
+ if: >-
132
+ (github.event.label.name == 'contextrail-assess' ||
133
+ github.event.label.name == 'contextrail-assess-extended') &&
134
+ github.event.pull_request.head.repo.full_name == github.repository &&
135
+ github.event.pull_request.user.type == 'User'
136
+ runs-on: ubuntu-latest
137
+ timeout-minutes: 35
138
+ permissions:
139
+ contents: read
140
+ pull-requests: write
141
+ env:
142
+ EXTENDED: ${{ github.event.label.name == 'contextrail-assess-extended' }}
143
+ steps:
144
+ - name: Assess proposed changes
145
+ uses: contextrail/assess/action@v1-alpha
146
+ with:
147
+ pr-number: ${{ github.event.pull_request.number }}
148
+ reviewer: skill://engineering/your-reviewer
149
+ mcp-url: ${{ vars.CONTEXTRAIL_MCP_URL }}
150
+ mcp-org-id: ${{ vars.CONTEXTRAIL_MCP_ORG_ID }}
151
+ mcp-auth-token: ${{ secrets.CONTEXTRAIL_MCP_AUTH_TOKEN }}
152
+ provider: openai
153
+ model: ${{ vars.ASSESS_MODEL }}
154
+ openai-api-key: ${{ secrets.OPENAI_API_KEY }}
155
+ enforcement: advisory
156
+ post-comment: 'true'
157
+ speed: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_SPEED || 'fast') || 'normal' }}
158
+ max-time-ms: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_MAX_TIME_MS || '1500000') || '600000' }}
159
+ max-verify-calls: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_MAX_VERIFY_CALLS || '30') || '' }}
160
+ ```
161
+
162
+ The job never checks out or runs pull request code, which is what makes `pull_request_target`
163
+ safe to use with secrets here. Do not add steps that do.
164
+
165
+ To also review every pull request when it is opened, set `types: [opened, ready_for_review, labeled]`
166
+ and make the first part of `if` accept those events too. Each opened pull request then starts one paid run:
167
+
168
+ ```yaml
169
+ if: >-
170
+ (((github.event.action == 'opened' || github.event.action == 'ready_for_review') &&
171
+ !github.event.pull_request.draft) ||
172
+ github.event.label.name == 'contextrail-assess' ||
173
+ github.event.label.name == 'contextrail-assess-extended') &&
174
+ github.event.pull_request.head.repo.full_name == github.repository &&
175
+ github.event.pull_request.user.type == 'User'
176
+ ```
177
+
178
+ ### 4. Run it
179
+
180
+ ```sh
181
+ gh pr edit 123 --add-label contextrail-assess # standard run
182
+ gh pr edit 123 --add-label contextrail-assess-extended # after an incomplete run
183
+ ```
184
+
185
+ In the GitHub UI: open the pull request and choose the label under **Labels** in the sidebar.
186
+ Each label addition starts a new paid run. To run again with the same label, remove it and add it
187
+ back. **Re-run jobs** replays the original event and is rejected if the pull request has changed since.
188
+
74
189
  ## Package and platform
75
190
 
76
191
  Assess is an open-source execution tool for the ContextRail platform. Your organization maintains
package/dist/assess.d.ts CHANGED
@@ -5,6 +5,7 @@ import { type ReportingStatus, type AssessmentFeedbackSender } from './feedback.
5
5
  import { type PreparedState } from './prepare.js';
6
6
  import { decideExit, type AssessmentEntry, type AssessmentPolicy } from './results.js';
7
7
  import { type ProviderConfiguration, type AssessmentProvider, type ProviderFailureCode } from './providers.js';
8
+ import { type ProviderSpeed } from './speed.js';
8
9
  import { type SourceLimits } from './source.js';
9
10
  import { type PhaseRecord } from './phases.js';
10
11
  import { DEFAULT_EXECUTION_LIMITS, type ExecutionLimits, type ModelPhase } from './budget.js';
@@ -96,6 +97,7 @@ type AssessmentOptions = {
96
97
  prContext?: PrContext;
97
98
  signal?: AbortSignal;
98
99
  fetch?: typeof fetch;
100
+ speed?: ProviderSpeed;
99
101
  feedback?: 'send' | 'off';
100
102
  feedbackSender?: AssessmentFeedbackSender;
101
103
  repositoryAttribution?: boolean;
package/dist/assess.js CHANGED
@@ -14,10 +14,11 @@ import { atomicJson, digest } from './prepare.js';
14
14
  import { CitationError, EvidenceLedger, ModelIdentifiers, modelEvidence, resolveCitationSelections, } from './evidence.js';
15
15
  import { DEFAULT_POLICY, assessmentPolicySchema, decideExit, } from './results.js';
16
16
  import { assessmentModel, ProviderAccountingError, recoverProviderCalls, reportedUsage, providerFailureDiagnostics, providerResponseDiagnostics, providerRequestDiagnostics, providerTransportDiagnostics, } from './providers.js';
17
+ import { speedFetch } from './speed.js';
17
18
  import { FixedSnapshotSource, SourceAccessError } from './source.js';
18
19
  import { publicationSchema } from './contracts.js';
19
20
  import { createPhaseRecord, validatePhaseRecord } from './phases.js';
20
- import { AssessmentBudget, BudgetStop, DEFAULT_EXECUTION_LIMITS, } from './budget.js';
21
+ import { AssessmentBudget, BudgetStop, DEFAULT_EXECUTION_LIMITS, DEFAULT_VERIFY_CALLS, } from './budget.js';
21
22
  import { Diagnostics, sanitizeRecord, systemFailureDiagnostics, submissionDiagnostics, submissionToolDiagnostics, } from './diagnostics.js';
22
23
  import { exactIds, PlanCitationError } from './planning.js';
23
24
  import { loadAssessmentCheckpoint } from './resume.js';
@@ -131,35 +132,39 @@ function initialAssessmentState(options, source, budget, prContext, settings) {
131
132
  }
132
133
  function restoreCheckpoint(options, directory, state, ledger) {
133
134
  const { prepared } = options;
134
- if (options.resume) {
135
- const loaded = loadAssessmentCheckpoint(directory, {
136
- artifact: prepared.state.artifact,
137
- reviewerId: prepared.state.requestedReviewerId,
138
- });
139
- if (digest(loaded.checkpoint) !== digest(options.resume) ||
140
- digest(loaded.prepared.state) !== digest(prepared.state))
141
- throw new Error('resume_checkpoint_changed');
142
- const old = options.resume.state;
143
- if (old.phases.some((p) => p.phase === 'assess' && p.configuration.promptVersion !== 'candidate-observation-v3'))
144
- throw Error('resume_method_changed_start_new_assessment');
145
- if (old.runId !== state.runId ||
146
- digest(old.artifact) !== digest(state.artifact) ||
147
- old.scopeIdentity !== state.scopeIdentity ||
148
- digest(old.execution.phaseModels) !== digest(state.execution.phaseModels) ||
149
- digest(old.execution.sourceLimits) !== digest(state.execution.sourceLimits) ||
150
- digest(old.execution.policy) !== digest(state.execution.policy) ||
151
- old.execution.purpose !== state.execution.purpose ||
152
- old.execution.prContext?.hash !== state.execution.prContext?.hash)
153
- throw new Error('resume_configuration_changed');
154
- state.phases = structuredClone(old.phases);
155
- state.execution.usage = structuredClone(old.execution.usage);
156
- state.execution.providerUsage = structuredClone(old.execution.providerUsage);
157
- state.execution.returnedIdentities = structuredClone(old.execution.returnedIdentities);
158
- state.execution.calls = old.execution.calls;
159
- state.execution.steps = old.execution.steps;
160
- for (const record of Object.values(options.resume.evidence))
161
- ledger.add(record.text, record.provenance, record.fragments);
162
- }
135
+ if (!options.resume)
136
+ return;
137
+ const loaded = loadAssessmentCheckpoint(directory, {
138
+ artifact: prepared.state.artifact,
139
+ reviewerId: prepared.state.requestedReviewerId,
140
+ });
141
+ if (digest(loaded.checkpoint) !== digest(options.resume) ||
142
+ digest(loaded.prepared.state) !== digest(prepared.state))
143
+ throw new Error('resume_checkpoint_changed');
144
+ const old = options.resume.state;
145
+ if (old.phases.some((p) => p.phase === 'assess' && p.configuration.promptVersion !== 'candidate-observation-v3'))
146
+ throw Error('resume_method_changed_start_new_assessment');
147
+ if (!sameResumeConfiguration(old, state))
148
+ throw new Error('resume_configuration_changed');
149
+ state.phases = structuredClone(old.phases);
150
+ state.execution.usage = structuredClone(old.execution.usage);
151
+ state.execution.providerUsage = structuredClone(old.execution.providerUsage);
152
+ state.execution.returnedIdentities = structuredClone(old.execution.returnedIdentities);
153
+ state.execution.calls = old.execution.calls;
154
+ state.execution.steps = old.execution.steps;
155
+ for (const record of Object.values(options.resume.evidence))
156
+ ledger.add(record.text, record.provenance, record.fragments);
157
+ }
158
+ /** A resumed run must be the same run: artifact, scope, models, limits, policy, purpose and PR context. */
159
+ function sameResumeConfiguration(old, state) {
160
+ return (old.runId === state.runId &&
161
+ digest(old.artifact) === digest(state.artifact) &&
162
+ old.scopeIdentity === state.scopeIdentity &&
163
+ digest(old.execution.phaseModels) === digest(state.execution.phaseModels) &&
164
+ digest(old.execution.sourceLimits) === digest(state.execution.sourceLimits) &&
165
+ digest(old.execution.policy) === digest(state.execution.policy) &&
166
+ old.execution.purpose === state.execution.purpose &&
167
+ old.execution.prContext?.hash === state.execution.prContext?.hash);
163
168
  }
164
169
  async function restorePreparationManifest(options, state, directory, _material, ledger, preparedManifest) {
165
170
  if (options.resume) {
@@ -187,6 +192,7 @@ async function createAssessmentRuntime(options) {
187
192
  : null;
188
193
  const state = initialAssessmentState(options, source, budget, prContext, { models, logLevel, limits });
189
194
  const diagnostics = new Diagnostics(directory, state.runId, logLevel), ledger = new EvidenceLedger();
195
+ const providerFetch = speedFetch(options.fetch ?? fetch, options.speed ?? 'normal', (event) => diagnostics.event('provider_speed_unavailable', event).catch(() => undefined));
190
196
  restoreCheckpoint(options, directory, state, ledger);
191
197
  const invocationPhaseIndex = state.phases.length;
192
198
  const previousCalls = state.execution.calls, previousElapsed = options.resume?.state.execution.elapsedMs ?? 0;
@@ -216,6 +222,7 @@ async function createAssessmentRuntime(options) {
216
222
  state,
217
223
  ledger,
218
224
  diagnostics,
225
+ providerFetch,
219
226
  persist,
220
227
  material,
221
228
  preparedManifest,
@@ -233,8 +240,20 @@ async function createAssessmentRuntime(options) {
233
240
  const correctionRounds = (session) => remainingContexts(session).length < session.remainingAtLastStep
234
241
  ? 0
235
242
  : session.rejected + Number(session.stepRejected);
243
+ /** The manifest, publications and every completed phase on disk still match what this run recorded. */
244
+ async function assertUpstreamUnchanged(run) {
245
+ const { state, directory, manifestHash, materialHash, prepared } = run;
246
+ if (digest(JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8'))) !== manifestHash ||
247
+ digest(JSON.parse(await readFile(prepared.materialPath, 'utf8'))) !== materialHash)
248
+ throw new Error('upstream_artifact_changed');
249
+ for (const p of state.phases.filter((p) => p.status === 'complete')) {
250
+ const name = phaseFilename(p.phase, p.workUnitId, p.attempt);
251
+ if (digest(JSON.parse(await readFile(join(directory, 'phases', name), 'utf8'))) !== digest(p))
252
+ throw new Error('upstream_phase_changed');
253
+ }
254
+ }
236
255
  async function createPhaseRuntime(run, phase, workUnitId, assigned, payload, schema, validate, attempt = 1) {
237
- const { state, directory, budget, ledger, options, manifestHash, materialHash, retainedProposals, prepared } = run;
256
+ const { state, budget, ledger, options, manifestHash, materialHash, retainedProposals } = run;
238
257
  const startedAt = new Date().toISOString(), inputRefs = [
239
258
  { ref: 'manifest.json', hash: manifestHash },
240
259
  { ref: 'publications.json', hash: materialHash },
@@ -243,17 +262,10 @@ async function createPhaseRuntime(run, phase, workUnitId, assigned, payload, sch
243
262
  .map((p) => ({ ref: `phase:${p.phase}:${p.workUnitId}:${p.attempt}`, hash: digest(p) })),
244
263
  { ref: `input:${phase}:${workUnitId}:${attempt}`, hash: digest(sanitizeRecord(payload)) },
245
264
  ], phaseCalls = [];
246
- if (digest(JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8'))) !== manifestHash ||
247
- digest(JSON.parse(await readFile(prepared.materialPath, 'utf8'))) !== materialHash)
248
- throw new Error('upstream_artifact_changed');
249
- for (const p of state.phases.filter((p) => p.status === 'complete')) {
250
- const name = phaseFilename(p.phase, p.workUnitId, p.attempt);
251
- if (digest(JSON.parse(await readFile(join(directory, 'phases', name), 'utf8'))) !== digest(p))
252
- throw new Error('upstream_phase_changed');
253
- }
265
+ await assertUpstreamUnchanged(run);
254
266
  const remaining = budget.remainingMs(phase), controller = new AbortController();
255
267
  if (remaining !== null && remaining <= 0)
256
- throw new BudgetStop(phase === 'verify' ? 'max_time_ms' : 'max_time_ms');
268
+ throw new BudgetStop('max_time_ms');
257
269
  const timer = remaining === null ? null : setTimeout(() => controller.abort(), Math.max(1, remaining));
258
270
  const signal = options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
259
271
  let output;
@@ -470,59 +482,62 @@ function revalidateSummary(value, ledger) {
470
482
  }, ledger);
471
483
  }
472
484
  async function captureChangeSummary(session, value) {
473
- if (value === undefined || value === null) {
474
- // An omitted summary contributes no update; after feedback it explicitly abandons correction.
475
- session.summaryCorrection = null;
476
- if (value === undefined || session.phase === 'assess' || session.run.state.changeSummary?.status === 'verified')
477
- return;
478
- }
479
- else
480
- session.summaryCorrection = null;
481
- const { state, ledger, diagnostics } = session.run;
485
+ // An omitted summary contributes no update; after feedback it explicitly abandons correction.
486
+ session.summaryCorrection = null;
487
+ const { state, diagnostics } = session.run;
488
+ // Correction batches need not repeat an already checked summary.
489
+ if (value === undefined ||
490
+ (value === null && (session.phase === 'assess' || state.changeSummary?.status === 'verified')))
491
+ return;
482
492
  if (session.phase === 'verify' && value === false) {
483
493
  state.changeSummary = { candidate: state.changeSummary?.candidate ?? null, accepted: null, status: 'rejected' };
484
494
  await diagnostics.event('change_summary_withdrawn', { phase: session.phase });
485
495
  return;
486
496
  }
487
- // Correction batches need not repeat an already checked summary.
488
- if (value === null && state.changeSummary?.status === 'verified')
489
- return;
490
497
  try {
491
- const summary = value === null ? null : validateChangeSummary(resolveCitationSelections(value, ledger), ledger);
492
- if (session.phase === 'assess') {
493
- if (summary)
494
- state.changeSummary = { candidate: summary, accepted: null, status: 'pending' };
495
- }
496
- else {
497
- const accepted = state.changeSummary?.candidate ? summary : null;
498
- state.changeSummary = {
499
- candidate: state.changeSummary?.candidate ?? null,
500
- accepted,
501
- status: accepted ? 'verified' : 'rejected',
502
- };
503
- }
498
+ applySummary(session, value);
504
499
  }
505
500
  catch (error) {
506
- session.summaryFailures++;
507
- const { code, details } = submissionDiagnostics(error);
508
- session.summaryCorrection = {
509
- code: error instanceof Error && error.message === 'change_summary_source_required' ? error.message : code,
510
- correction: submissionCorrection(error, code),
511
- };
512
- if (session.phase === 'verify' && state.changeSummary?.status !== 'verified')
513
- state.changeSummary = {
514
- candidate: state.changeSummary?.candidate ?? null,
515
- accepted: null,
516
- status: 'rejected',
517
- };
518
- await diagnostics.event('change_summary_rejected', {
519
- phase: session.phase,
520
- ...session.summaryCorrection,
521
- details,
522
- attempt: session.summaryFailures,
523
- retryable: session.summaryFailures < 2,
524
- });
501
+ await rejectSummary(session, error);
502
+ }
503
+ }
504
+ /** Assess proposes a pending candidate; verify accepts it only when there was a candidate to check. */
505
+ function applySummary(session, value) {
506
+ const { state, ledger } = session.run;
507
+ const summary = value === null ? null : validateChangeSummary(resolveCitationSelections(value, ledger), ledger);
508
+ if (session.phase === 'assess') {
509
+ if (summary)
510
+ state.changeSummary = { candidate: summary, accepted: null, status: 'pending' };
511
+ return;
525
512
  }
513
+ const accepted = state.changeSummary?.candidate ? summary : null;
514
+ state.changeSummary = {
515
+ candidate: state.changeSummary?.candidate ?? null,
516
+ accepted,
517
+ status: accepted ? 'verified' : 'rejected',
518
+ };
519
+ }
520
+ async function rejectSummary(session, error) {
521
+ const { state, diagnostics } = session.run;
522
+ session.summaryFailures++;
523
+ const { code, details } = submissionDiagnostics(error);
524
+ session.summaryCorrection = {
525
+ code: error instanceof Error && error.message === 'change_summary_source_required' ? error.message : code,
526
+ correction: submissionCorrection(error, code),
527
+ };
528
+ if (session.phase === 'verify' && state.changeSummary?.status !== 'verified')
529
+ state.changeSummary = {
530
+ candidate: state.changeSummary?.candidate ?? null,
531
+ accepted: null,
532
+ status: 'rejected',
533
+ };
534
+ await diagnostics.event('change_summary_rejected', {
535
+ phase: session.phase,
536
+ ...session.summaryCorrection,
537
+ details,
538
+ attempt: session.summaryFailures,
539
+ retryable: session.summaryFailures < 2,
540
+ });
526
541
  }
527
542
  function summarySubmissionFeedback(session) {
528
543
  const rejected = session.summaryCorrection;
@@ -737,7 +752,7 @@ function createPhaseTools(session) {
737
752
  }
738
753
  function createProviderFetch(session) {
739
754
  const { run, phase, workUnitId, attempt, phaseCalls, signal } = session;
740
- const { state, diagnostics, models, options, budget, persist, previousCalls } = run;
755
+ const { state, diagnostics, models, providerFetch, budget, persist, previousCalls } = run;
741
756
  return async (input, init) => {
742
757
  signal.throwIfAborted();
743
758
  const call = previousCalls + budget.reserve(phase);
@@ -766,7 +781,7 @@ function createProviderFetch(session) {
766
781
  });
767
782
  let recorded = false;
768
783
  try {
769
- const response = await (options.fetch ?? fetch)(input, init);
784
+ const response = await providerFetch(input, init);
770
785
  if (!response.ok) {
771
786
  budget.record(null);
772
787
  recorded = true;
@@ -872,7 +887,7 @@ async function runPhaseModel(session) {
872
887
  stopWhen: () => session.output !== undefined ||
873
888
  session.rejected >= 3 ||
874
889
  session.sdkStop !== null ||
875
- session.phaseCalls.length >= (phase === 'assess' ? 60 : 12),
890
+ session.phaseCalls.length >= phaseCallLimit(session),
876
891
  prepareStep: () => {
877
892
  session.toolsThisStep = 0;
878
893
  return phase === 'assess' && session.phaseCalls.length >= 58
@@ -917,7 +932,7 @@ async function runPhaseModel(session) {
917
932
  if (session.output !== undefined ||
918
933
  session.rejected >= 3 ||
919
934
  session.sdkStop !== null ||
920
- session.phaseCalls.length >= (phase === 'assess' ? 60 : 12) ||
935
+ session.phaseCalls.length >= phaseCallLimit(session) ||
921
936
  continuation >= 2)
922
937
  break;
923
938
  continuation++;
@@ -944,7 +959,7 @@ async function runPhaseModel(session) {
944
959
  if (session.sdkStop !== null)
945
960
  throw new BudgetStop(session.sdkStop);
946
961
  if (session.output === undefined)
947
- throw new BudgetStop(session.phaseCalls.length >= (phase === 'assess' ? 60 : 12)
962
+ throw new BudgetStop(session.phaseCalls.length >= phaseCallLimit(session)
948
963
  ? 'model_call_limit'
949
964
  : session.rejected >= 3
950
965
  ? 'phase_corrections_exhausted'
@@ -970,24 +985,61 @@ function phaseStopReason(session, failure) {
970
985
  if (session.signal.aborted) {
971
986
  if (session.run.options.signal?.aborted)
972
987
  return 'cancelled';
973
- return session.phase === 'verify' ? 'max_time_ms' : 'max_time_ms';
988
+ return 'max_time_ms';
974
989
  }
975
990
  return knownExecutionFailure(failure);
976
991
  }
977
- async function persistPhaseResult(session, failure) {
978
- const { run, phase, workUnitId, assigned, attempt, startedAt, inputRefs, phaseCalls, signal, instructions } = session;
979
- const { state, directory, ledger, diagnostics, models, persist } = run;
980
- const partial = failure && phase === 'assess' && session.assessmentAccumulator?.snapshot().length
981
- ? {
992
+ /** What a failed phase already accepted, so a resumed run continues instead of repeating it. */
993
+ function partialProgress(session) {
994
+ if (session.phase === 'assess' && session.assessmentAccumulator?.snapshot().length)
995
+ return {
982
996
  assessments: session.assessmentAccumulator.snapshot(),
983
997
  remaining: session.assessmentAccumulator.remaining(),
984
- }
985
- : failure && session.verifications?.size
986
- ? {
987
- verifications: [...session.verifications.values()].map((entry) => entry.result),
988
- remaining: remainingContexts(session),
989
- }
990
- : null;
998
+ };
999
+ if (session.verifications?.size)
1000
+ return {
1001
+ verifications: [...session.verifications.values()].map((entry) => entry.result),
1002
+ remaining: remainingContexts(session),
1003
+ };
1004
+ return null;
1005
+ }
1006
+ function phaseCallLimit({ phase, run }) {
1007
+ return phase === 'assess' ? 60 : (run.limits.maxVerifyCalls ?? DEFAULT_VERIFY_CALLS);
1008
+ }
1009
+ function phaseRecordConfiguration(session, model, partial) {
1010
+ const { phase, instructions } = session;
1011
+ return {
1012
+ provider: model.provider,
1013
+ model: model.model,
1014
+ effort: model.effort ?? null,
1015
+ ...(partial
1016
+ ? { kind: phase === 'assess' ? 'partial_assessment_progress' : 'partial_verification_progress' }
1017
+ : {}),
1018
+ modelIdentifiers: session.identifiers.snapshot(),
1019
+ promptVersion: 'candidate-observation-v3',
1020
+ maxPhaseCalls: phaseCallLimit(session),
1021
+ maxSourceToolsPerResponse: 50,
1022
+ validationConcurrency: 2,
1023
+ promptHash: digest(instructions),
1024
+ sdkVersion: runtimeVersion('ai'),
1025
+ adapter: '@ai-sdk/openai',
1026
+ adapterVersion: runtimeVersion('@ai-sdk/openai'),
1027
+ };
1028
+ }
1029
+ /** The phase output, carrying the change summary as assess proposed it or verify accepted it. */
1030
+ function phaseOutput(session, partial) {
1031
+ const summary = session.run.state.changeSummary;
1032
+ if (!summary)
1033
+ return partial ?? session.output ?? null;
1034
+ return {
1035
+ ...(partial ?? { [session.phase === 'assess' ? 'assessments' : 'entries']: session.output ?? [] }),
1036
+ changeSummary: session.phase === 'assess' ? summary.candidate : summary.accepted,
1037
+ };
1038
+ }
1039
+ async function persistPhaseResult(session, failure) {
1040
+ const { run, phase, workUnitId, assigned, attempt, startedAt, inputRefs, phaseCalls, signal } = session;
1041
+ const { state, directory, ledger, diagnostics, models, persist } = run;
1042
+ const partial = failure ? partialProgress(session) : null;
991
1043
  if (failure && phase === 'verify' && state.changeSummary) {
992
1044
  state.changeSummary.accepted = null;
993
1045
  state.changeSummary.status = 'pending';
@@ -1007,30 +1059,9 @@ async function persistPhaseResult(session, failure) {
1007
1059
  evidenceRefs: Object.keys(ledger.records),
1008
1060
  limitations: [],
1009
1061
  stopReason: phaseStopReason(session, failure),
1010
- configuration: {
1011
- provider: models[phase].provider,
1012
- model: models[phase].model,
1013
- effort: models[phase].effort ?? null,
1014
- ...(partial
1015
- ? { kind: phase === 'assess' ? 'partial_assessment_progress' : 'partial_verification_progress' }
1016
- : {}),
1017
- modelIdentifiers: session.identifiers.snapshot(),
1018
- promptVersion: 'candidate-observation-v3',
1019
- maxPhaseCalls: phase === 'assess' ? 60 : 12,
1020
- maxSourceToolsPerResponse: 50,
1021
- validationConcurrency: 2,
1022
- promptHash: digest(instructions),
1023
- sdkVersion: runtimeVersion('ai'),
1024
- adapter: '@ai-sdk/openai',
1025
- adapterVersion: runtimeVersion('@ai-sdk/openai'),
1026
- },
1062
+ configuration: phaseRecordConfiguration(session, models[phase], partial !== null),
1027
1063
  usageRefs: phaseCalls.map((c) => `call:${c}`),
1028
- output: run.state.changeSummary
1029
- ? {
1030
- ...(partial ?? { [phase === 'assess' ? 'assessments' : 'entries']: session.output ?? [] }),
1031
- changeSummary: phase === 'assess' ? run.state.changeSummary.candidate : run.state.changeSummary.accepted,
1032
- }
1033
- : (partial ?? session.output ?? null),
1064
+ output: phaseOutput(session, partial),
1034
1065
  });
1035
1066
  const filename = phaseFilename(phase, workUnitId, attempt);
1036
1067
  await atomicJson(join(directory, 'phases', filename), sanitizeRecord(record));
@@ -1080,6 +1111,12 @@ async function recoverSummaryOnlyFailure(session, error) {
1080
1111
  });
1081
1112
  return true;
1082
1113
  }
1114
+ /** A budget stop keeps its own code; an aborted phase hit its deadline or was cancelled; anything else failed in the provider or execution. */
1115
+ function phaseFailureCode(session, error) {
1116
+ if (error instanceof BudgetStop)
1117
+ return error.code;
1118
+ return session.signal.aborted ? 'phase_deadline_or_cancelled' : 'provider_or_execution_failed';
1119
+ }
1083
1120
  async function invokePhase(run, phase, workUnitId, assigned, payload, schema, validate, attempt = 1) {
1084
1121
  const session = await createPhaseRuntime(run, phase, workUnitId, assigned, payload, schema, validate, attempt);
1085
1122
  await run.diagnostics.event('phase_started', { phase, workUnitId, attempt, inputHash: session.inputRefs[0].hash });
@@ -1095,11 +1132,7 @@ async function invokePhase(run, phase, workUnitId, assigned, payload, schema, va
1095
1132
  phase,
1096
1133
  workUnitId,
1097
1134
  attempt,
1098
- code: error instanceof BudgetStop
1099
- ? error.code
1100
- : session.signal.aborted
1101
- ? 'phase_deadline_or_cancelled'
1102
- : 'provider_or_execution_failed',
1135
+ code: phaseFailureCode(session, error),
1103
1136
  details: { ...providerFailureDiagnostics(error), ...systemFailureDiagnostics(error) },
1104
1137
  });
1105
1138
  }
@@ -1168,6 +1201,16 @@ async function prepareRunEvidence(run) {
1168
1201
  await atomicJson(join(directory, 'prepared.json'), sanitizeRecord(prepared.state));
1169
1202
  if (prContext)
1170
1203
  await atomicJson(join(directory, 'pr-context.json'), sanitizeRecord(prContext));
1204
+ assertPinnedMaterial(state, material);
1205
+ for (const publication of Object.values(material))
1206
+ ledger.publication(publication);
1207
+ run.evidencePack = prepareEvidencePack({ state: prepared.state, publications: material, ledger });
1208
+ await writeEvidencePack(directory, run.evidencePack);
1209
+ run.repositoryInstructions = await repositoryInstructionsWithinBudget(run);
1210
+ await run.persist();
1211
+ }
1212
+ /** Each assessed entry's pinned publication is still the exact version and content it was prepared with. */
1213
+ function assertPinnedMaterial(state, material) {
1171
1214
  for (const entry of state.entries) {
1172
1215
  if (!entry.material)
1173
1216
  continue;
@@ -1178,24 +1221,22 @@ async function prepareRunEvidence(run) {
1178
1221
  digest(pinned.content) !== entry.contentHash)
1179
1222
  throw new Error('pinned_material_changed');
1180
1223
  }
1181
- for (const publication of Object.values(material))
1182
- ledger.publication(publication);
1183
- run.evidencePack = prepareEvidencePack({ state: prepared.state, publications: material, ledger });
1184
- await writeEvidencePack(directory, run.evidencePack);
1224
+ }
1225
+ /** Repository instruction files count against the assess time budget; running out is a budget stop. */
1226
+ async function repositoryInstructionsWithinBudget(run) {
1185
1227
  const remaining = run.budget.remainingMs('assess');
1186
1228
  if (remaining !== null && remaining <= 0)
1187
1229
  throw new BudgetStop('max_time_ms');
1188
1230
  const deadline = remaining === null ? undefined : AbortSignal.timeout(Math.ceil(remaining));
1189
1231
  const signals = [run.options.signal, deadline].filter((value) => value !== undefined);
1190
1232
  try {
1191
- run.repositoryInstructions = await repositoryInstructionsEvidence(run.source, ledger, signals.length ? AbortSignal.any(signals) : undefined);
1233
+ return await repositoryInstructionsEvidence(run.source, run.ledger, signals.length ? AbortSignal.any(signals) : undefined);
1192
1234
  }
1193
1235
  catch (error) {
1194
1236
  if (deadline?.aborted)
1195
1237
  throw new BudgetStop('max_time_ms');
1196
1238
  throw error;
1197
1239
  }
1198
- await run.persist();
1199
1240
  }
1200
1241
  async function restorePartialProposals(run) {
1201
1242
  if (!run.options.resume)
@@ -500,19 +500,6 @@ export declare function deriveAssessmentCitations(input: unknown, entries: Asses
500
500
  checkIndices: number[];
501
501
  }[];
502
502
  };
503
- export declare class AssessmentAccumulator {
504
- private readonly entries;
505
- private readonly ledger;
506
- private readonly structured;
507
- private readonly accepted;
508
- constructor(entries: AssessmentEntry[], ledger: EvidenceLedger, structured?: Record<string, Array<{
509
- path: string;
510
- value: unknown;
511
- }>>, seed?: AssessmentProposal[]);
512
- submit(input: unknown): AssessmentProgress;
513
- remaining(): string[];
514
- snapshot(): AssessmentProposal[];
515
- }
516
503
  export type AssessmentProgress = {
517
504
  complete: boolean;
518
505
  newlyAccepted: string[];
@@ -547,7 +534,7 @@ export type AssessmentFinding = {
547
534
  };
548
535
  verified: boolean;
549
536
  };
550
- export type AssessmentFollowUp = Omit<z.infer<typeof followUpSchema>, 'kind' | 'passage' | 'sourceRefs'> & {
537
+ type AssessmentFollowUp = Omit<z.infer<typeof followUpSchema>, 'kind' | 'passage' | 'sourceRefs'> & {
551
538
  kind: 'human_check' | 'operational_check' | 'external_evidence';
552
539
  externalEvidence?: {
553
540
  kind: 'external_dependency' | 'operational_state' | 'business_record';