@contextrail/assess 1.0.0-alpha.2 → 1.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/assess.d.ts +2 -0
- package/dist/assess.js +177 -136
- package/dist/assessment.d.ts +1 -14
- package/dist/assessment.js +0 -46
- package/dist/budget.d.ts +3 -0
- package/dist/budget.js +2 -0
- package/dist/candidate.js +135 -104
- package/dist/change-summary.d.ts +2 -1
- package/dist/change-summary.js +1 -1
- package/dist/cli.js +13 -1
- package/dist/contracts.d.ts +2 -8
- package/dist/contracts.js +2 -2
- package/dist/diagnostics.js +1 -0
- package/dist/evidence.d.ts +4 -0
- package/dist/evidence.js +21 -12
- package/dist/feedback.js +27 -23
- package/dist/git.d.ts +6 -0
- package/dist/git.js +30 -3
- package/dist/github.js +80 -53
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp.d.ts +2 -0
- package/dist/mcp.js +67 -56
- package/dist/pack.js +8 -1
- package/dist/pr-context.d.ts +2 -0
- package/dist/pr-context.js +27 -17
- package/dist/presentation.js +113 -71
- package/dist/provider-diagnostics.js +17 -12
- package/dist/providers.js +24 -20
- package/dist/report-contract.js +63 -57
- package/dist/results.js +5 -1
- package/dist/source.js +20 -11
- package/dist/speed.d.ts +13 -0
- package/dist/speed.js +72 -0
- package/docs/action-reference.md +6 -3
- package/docs/configuration.md +12 -1
- package/docs/contributing.md +10 -3
- package/docs/github-actions.md +22 -10
- package/docs/operations.md +4 -2
- package/examples/assess-pull-request.yml +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -71,6 +71,121 @@ Source excerpts and selected guidance are sent to your chosen model provider. De
|
|
|
71
71
|
quotes and conversations are not included in ContextRail's assessment-feedback payload.
|
|
72
72
|
See [where data goes](docs/security.md) before connecting a repository.
|
|
73
73
|
|
|
74
|
+
## Give large pull requests more capacity
|
|
75
|
+
|
|
76
|
+
A standard run keeps cost and time predictable. A large pull request can stop before it finishes:
|
|
77
|
+
the PR comment reports an incomplete assessment, and the run's stop reason is `max_time_ms` (time
|
|
78
|
+
limit reached) or `model_call_limit` (a verification used all its model calls). You do not have to
|
|
79
|
+
change how your team sizes pull requests. Instead, let a maintainer re-run that PR with more capacity
|
|
80
|
+
by adding a second label.
|
|
81
|
+
|
|
82
|
+
Three Action inputs set a run's capacity:
|
|
83
|
+
|
|
84
|
+
| Input | Standard run | Suggested extended run | What it changes |
|
|
85
|
+
| ------------------ | ------------------ | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
86
|
+
| `speed` | `normal` (default) | `fast` | Requests the provider's priority tier: quicker calls at a higher per-token price. `ultrafast` is also accepted. If a model doesn't serve a tier, Assess logs `provider_speed_unavailable` and continues. |
|
|
87
|
+
| `max-time-ms` | `600000` (10 min) | `1500000` (25 min) | Model time limit for the run. Keep the job's `timeout-minutes` about 10 minutes above it for setup and publishing. |
|
|
88
|
+
| `max-verify-calls` | empty (12) | `30` | Model calls each finding's independent verification may use. |
|
|
89
|
+
|
|
90
|
+
### 1. Create the labels
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
gh label create contextrail-assess --color 1D76DB \
|
|
94
|
+
--description "Run ContextRail Assess on this pull request"
|
|
95
|
+
gh label create contextrail-assess-extended --color 5319E7 \
|
|
96
|
+
--description "Re-run ContextRail Assess with extended capacity"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
In the GitHub UI: **Pull requests → Labels → New label**. Any names work if the workflow uses the same ones.
|
|
100
|
+
|
|
101
|
+
### 2. Optionally tune the extended run
|
|
102
|
+
|
|
103
|
+
The workflow below has built-in defaults. To change them without editing it, set repository variables:
|
|
104
|
+
|
|
105
|
+
```sh
|
|
106
|
+
gh variable set ASSESS_EXTENDED_SPEED --body fast
|
|
107
|
+
gh variable set ASSESS_EXTENDED_MAX_TIME_MS --body 1500000
|
|
108
|
+
gh variable set ASSESS_EXTENDED_MAX_VERIFY_CALLS --body 30
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
In the GitHub UI: **Settings → Secrets and variables → Actions → Variables → New repository variable**.
|
|
112
|
+
These names belong to your workflow; the Action itself reads only its inputs. Set up the connection
|
|
113
|
+
variables and secrets first, as described in [GitHub Actions](docs/github-actions.md#prepare-the-repository-settings).
|
|
114
|
+
|
|
115
|
+
### 3. Add the workflow
|
|
116
|
+
|
|
117
|
+
Save as `.github/workflows/contextrail-assess.yml` on the default branch and replace the reviewer skill ID:
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
name: ContextRail Assess
|
|
121
|
+
on:
|
|
122
|
+
pull_request_target:
|
|
123
|
+
types: [labeled]
|
|
124
|
+
permissions: {}
|
|
125
|
+
concurrency:
|
|
126
|
+
group: contextrail-assess-${{ github.repository }}-${{ github.event.pull_request.number }}
|
|
127
|
+
cancel-in-progress: false
|
|
128
|
+
jobs:
|
|
129
|
+
assess:
|
|
130
|
+
# Only maintainers with triage or write access can add labels. Same-repository human PRs only.
|
|
131
|
+
if: >-
|
|
132
|
+
(github.event.label.name == 'contextrail-assess' ||
|
|
133
|
+
github.event.label.name == 'contextrail-assess-extended') &&
|
|
134
|
+
github.event.pull_request.head.repo.full_name == github.repository &&
|
|
135
|
+
github.event.pull_request.user.type == 'User'
|
|
136
|
+
runs-on: ubuntu-latest
|
|
137
|
+
timeout-minutes: 35
|
|
138
|
+
permissions:
|
|
139
|
+
contents: read
|
|
140
|
+
pull-requests: write
|
|
141
|
+
env:
|
|
142
|
+
EXTENDED: ${{ github.event.label.name == 'contextrail-assess-extended' }}
|
|
143
|
+
steps:
|
|
144
|
+
- name: Assess proposed changes
|
|
145
|
+
uses: contextrail/assess/action@v1-alpha
|
|
146
|
+
with:
|
|
147
|
+
pr-number: ${{ github.event.pull_request.number }}
|
|
148
|
+
reviewer: skill://engineering/your-reviewer
|
|
149
|
+
mcp-url: ${{ vars.CONTEXTRAIL_MCP_URL }}
|
|
150
|
+
mcp-org-id: ${{ vars.CONTEXTRAIL_MCP_ORG_ID }}
|
|
151
|
+
mcp-auth-token: ${{ secrets.CONTEXTRAIL_MCP_AUTH_TOKEN }}
|
|
152
|
+
provider: openai
|
|
153
|
+
model: ${{ vars.ASSESS_MODEL }}
|
|
154
|
+
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
155
|
+
enforcement: advisory
|
|
156
|
+
post-comment: 'true'
|
|
157
|
+
speed: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_SPEED || 'fast') || 'normal' }}
|
|
158
|
+
max-time-ms: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_MAX_TIME_MS || '1500000') || '600000' }}
|
|
159
|
+
max-verify-calls: ${{ env.EXTENDED == 'true' && (vars.ASSESS_EXTENDED_MAX_VERIFY_CALLS || '30') || '' }}
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The job never checks out or runs pull request code, which is what makes `pull_request_target`
|
|
163
|
+
safe to use with secrets here. Do not add steps that do.
|
|
164
|
+
|
|
165
|
+
To also review every pull request when it is opened, set `types: [opened, ready_for_review, labeled]`
|
|
166
|
+
and make the first part of `if` accept those events too. Each opened pull request then starts one paid run:
|
|
167
|
+
|
|
168
|
+
```yaml
|
|
169
|
+
if: >-
|
|
170
|
+
(((github.event.action == 'opened' || github.event.action == 'ready_for_review') &&
|
|
171
|
+
!github.event.pull_request.draft) ||
|
|
172
|
+
github.event.label.name == 'contextrail-assess' ||
|
|
173
|
+
github.event.label.name == 'contextrail-assess-extended') &&
|
|
174
|
+
github.event.pull_request.head.repo.full_name == github.repository &&
|
|
175
|
+
github.event.pull_request.user.type == 'User'
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
### 4. Run it
|
|
179
|
+
|
|
180
|
+
```sh
|
|
181
|
+
gh pr edit 123 --add-label contextrail-assess # standard run
|
|
182
|
+
gh pr edit 123 --add-label contextrail-assess-extended # after an incomplete run
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
In the GitHub UI: open the pull request and choose the label under **Labels** in the sidebar.
|
|
186
|
+
Each label addition starts a new paid run. To run again with the same label, remove it and add it
|
|
187
|
+
back. **Re-run jobs** replays the original event and is rejected if the pull request has changed since.
|
|
188
|
+
|
|
74
189
|
## Package and platform
|
|
75
190
|
|
|
76
191
|
Assess is an open-source execution tool for the ContextRail platform. Your organization maintains
|
package/dist/assess.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { type ReportingStatus, type AssessmentFeedbackSender } from './feedback.
|
|
|
5
5
|
import { type PreparedState } from './prepare.js';
|
|
6
6
|
import { decideExit, type AssessmentEntry, type AssessmentPolicy } from './results.js';
|
|
7
7
|
import { type ProviderConfiguration, type AssessmentProvider, type ProviderFailureCode } from './providers.js';
|
|
8
|
+
import { type ProviderSpeed } from './speed.js';
|
|
8
9
|
import { type SourceLimits } from './source.js';
|
|
9
10
|
import { type PhaseRecord } from './phases.js';
|
|
10
11
|
import { DEFAULT_EXECUTION_LIMITS, type ExecutionLimits, type ModelPhase } from './budget.js';
|
|
@@ -96,6 +97,7 @@ type AssessmentOptions = {
|
|
|
96
97
|
prContext?: PrContext;
|
|
97
98
|
signal?: AbortSignal;
|
|
98
99
|
fetch?: typeof fetch;
|
|
100
|
+
speed?: ProviderSpeed;
|
|
99
101
|
feedback?: 'send' | 'off';
|
|
100
102
|
feedbackSender?: AssessmentFeedbackSender;
|
|
101
103
|
repositoryAttribution?: boolean;
|
package/dist/assess.js
CHANGED
|
@@ -14,10 +14,11 @@ import { atomicJson, digest } from './prepare.js';
|
|
|
14
14
|
import { CitationError, EvidenceLedger, ModelIdentifiers, modelEvidence, resolveCitationSelections, } from './evidence.js';
|
|
15
15
|
import { DEFAULT_POLICY, assessmentPolicySchema, decideExit, } from './results.js';
|
|
16
16
|
import { assessmentModel, ProviderAccountingError, recoverProviderCalls, reportedUsage, providerFailureDiagnostics, providerResponseDiagnostics, providerRequestDiagnostics, providerTransportDiagnostics, } from './providers.js';
|
|
17
|
+
import { speedFetch } from './speed.js';
|
|
17
18
|
import { FixedSnapshotSource, SourceAccessError } from './source.js';
|
|
18
19
|
import { publicationSchema } from './contracts.js';
|
|
19
20
|
import { createPhaseRecord, validatePhaseRecord } from './phases.js';
|
|
20
|
-
import { AssessmentBudget, BudgetStop, DEFAULT_EXECUTION_LIMITS, } from './budget.js';
|
|
21
|
+
import { AssessmentBudget, BudgetStop, DEFAULT_EXECUTION_LIMITS, DEFAULT_VERIFY_CALLS, } from './budget.js';
|
|
21
22
|
import { Diagnostics, sanitizeRecord, systemFailureDiagnostics, submissionDiagnostics, submissionToolDiagnostics, } from './diagnostics.js';
|
|
22
23
|
import { exactIds, PlanCitationError } from './planning.js';
|
|
23
24
|
import { loadAssessmentCheckpoint } from './resume.js';
|
|
@@ -131,35 +132,39 @@ function initialAssessmentState(options, source, budget, prContext, settings) {
|
|
|
131
132
|
}
|
|
132
133
|
function restoreCheckpoint(options, directory, state, ledger) {
|
|
133
134
|
const { prepared } = options;
|
|
134
|
-
if (options.resume)
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
135
|
+
if (!options.resume)
|
|
136
|
+
return;
|
|
137
|
+
const loaded = loadAssessmentCheckpoint(directory, {
|
|
138
|
+
artifact: prepared.state.artifact,
|
|
139
|
+
reviewerId: prepared.state.requestedReviewerId,
|
|
140
|
+
});
|
|
141
|
+
if (digest(loaded.checkpoint) !== digest(options.resume) ||
|
|
142
|
+
digest(loaded.prepared.state) !== digest(prepared.state))
|
|
143
|
+
throw new Error('resume_checkpoint_changed');
|
|
144
|
+
const old = options.resume.state;
|
|
145
|
+
if (old.phases.some((p) => p.phase === 'assess' && p.configuration.promptVersion !== 'candidate-observation-v3'))
|
|
146
|
+
throw Error('resume_method_changed_start_new_assessment');
|
|
147
|
+
if (!sameResumeConfiguration(old, state))
|
|
148
|
+
throw new Error('resume_configuration_changed');
|
|
149
|
+
state.phases = structuredClone(old.phases);
|
|
150
|
+
state.execution.usage = structuredClone(old.execution.usage);
|
|
151
|
+
state.execution.providerUsage = structuredClone(old.execution.providerUsage);
|
|
152
|
+
state.execution.returnedIdentities = structuredClone(old.execution.returnedIdentities);
|
|
153
|
+
state.execution.calls = old.execution.calls;
|
|
154
|
+
state.execution.steps = old.execution.steps;
|
|
155
|
+
for (const record of Object.values(options.resume.evidence))
|
|
156
|
+
ledger.add(record.text, record.provenance, record.fragments);
|
|
157
|
+
}
|
|
158
|
+
/** A resumed run must be the same run: artifact, scope, models, limits, policy, purpose and PR context. */
|
|
159
|
+
function sameResumeConfiguration(old, state) {
|
|
160
|
+
return (old.runId === state.runId &&
|
|
161
|
+
digest(old.artifact) === digest(state.artifact) &&
|
|
162
|
+
old.scopeIdentity === state.scopeIdentity &&
|
|
163
|
+
digest(old.execution.phaseModels) === digest(state.execution.phaseModels) &&
|
|
164
|
+
digest(old.execution.sourceLimits) === digest(state.execution.sourceLimits) &&
|
|
165
|
+
digest(old.execution.policy) === digest(state.execution.policy) &&
|
|
166
|
+
old.execution.purpose === state.execution.purpose &&
|
|
167
|
+
old.execution.prContext?.hash === state.execution.prContext?.hash);
|
|
163
168
|
}
|
|
164
169
|
async function restorePreparationManifest(options, state, directory, _material, ledger, preparedManifest) {
|
|
165
170
|
if (options.resume) {
|
|
@@ -187,6 +192,7 @@ async function createAssessmentRuntime(options) {
|
|
|
187
192
|
: null;
|
|
188
193
|
const state = initialAssessmentState(options, source, budget, prContext, { models, logLevel, limits });
|
|
189
194
|
const diagnostics = new Diagnostics(directory, state.runId, logLevel), ledger = new EvidenceLedger();
|
|
195
|
+
const providerFetch = speedFetch(options.fetch ?? fetch, options.speed ?? 'normal', (event) => diagnostics.event('provider_speed_unavailable', event).catch(() => undefined));
|
|
190
196
|
restoreCheckpoint(options, directory, state, ledger);
|
|
191
197
|
const invocationPhaseIndex = state.phases.length;
|
|
192
198
|
const previousCalls = state.execution.calls, previousElapsed = options.resume?.state.execution.elapsedMs ?? 0;
|
|
@@ -216,6 +222,7 @@ async function createAssessmentRuntime(options) {
|
|
|
216
222
|
state,
|
|
217
223
|
ledger,
|
|
218
224
|
diagnostics,
|
|
225
|
+
providerFetch,
|
|
219
226
|
persist,
|
|
220
227
|
material,
|
|
221
228
|
preparedManifest,
|
|
@@ -233,8 +240,20 @@ async function createAssessmentRuntime(options) {
|
|
|
233
240
|
const correctionRounds = (session) => remainingContexts(session).length < session.remainingAtLastStep
|
|
234
241
|
? 0
|
|
235
242
|
: session.rejected + Number(session.stepRejected);
|
|
243
|
+
/** The manifest, publications and every completed phase on disk still match what this run recorded. */
|
|
244
|
+
async function assertUpstreamUnchanged(run) {
|
|
245
|
+
const { state, directory, manifestHash, materialHash, prepared } = run;
|
|
246
|
+
if (digest(JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8'))) !== manifestHash ||
|
|
247
|
+
digest(JSON.parse(await readFile(prepared.materialPath, 'utf8'))) !== materialHash)
|
|
248
|
+
throw new Error('upstream_artifact_changed');
|
|
249
|
+
for (const p of state.phases.filter((p) => p.status === 'complete')) {
|
|
250
|
+
const name = phaseFilename(p.phase, p.workUnitId, p.attempt);
|
|
251
|
+
if (digest(JSON.parse(await readFile(join(directory, 'phases', name), 'utf8'))) !== digest(p))
|
|
252
|
+
throw new Error('upstream_phase_changed');
|
|
253
|
+
}
|
|
254
|
+
}
|
|
236
255
|
async function createPhaseRuntime(run, phase, workUnitId, assigned, payload, schema, validate, attempt = 1) {
|
|
237
|
-
const { state,
|
|
256
|
+
const { state, budget, ledger, options, manifestHash, materialHash, retainedProposals } = run;
|
|
238
257
|
const startedAt = new Date().toISOString(), inputRefs = [
|
|
239
258
|
{ ref: 'manifest.json', hash: manifestHash },
|
|
240
259
|
{ ref: 'publications.json', hash: materialHash },
|
|
@@ -243,17 +262,10 @@ async function createPhaseRuntime(run, phase, workUnitId, assigned, payload, sch
|
|
|
243
262
|
.map((p) => ({ ref: `phase:${p.phase}:${p.workUnitId}:${p.attempt}`, hash: digest(p) })),
|
|
244
263
|
{ ref: `input:${phase}:${workUnitId}:${attempt}`, hash: digest(sanitizeRecord(payload)) },
|
|
245
264
|
], phaseCalls = [];
|
|
246
|
-
|
|
247
|
-
digest(JSON.parse(await readFile(prepared.materialPath, 'utf8'))) !== materialHash)
|
|
248
|
-
throw new Error('upstream_artifact_changed');
|
|
249
|
-
for (const p of state.phases.filter((p) => p.status === 'complete')) {
|
|
250
|
-
const name = phaseFilename(p.phase, p.workUnitId, p.attempt);
|
|
251
|
-
if (digest(JSON.parse(await readFile(join(directory, 'phases', name), 'utf8'))) !== digest(p))
|
|
252
|
-
throw new Error('upstream_phase_changed');
|
|
253
|
-
}
|
|
265
|
+
await assertUpstreamUnchanged(run);
|
|
254
266
|
const remaining = budget.remainingMs(phase), controller = new AbortController();
|
|
255
267
|
if (remaining !== null && remaining <= 0)
|
|
256
|
-
throw new BudgetStop(
|
|
268
|
+
throw new BudgetStop('max_time_ms');
|
|
257
269
|
const timer = remaining === null ? null : setTimeout(() => controller.abort(), Math.max(1, remaining));
|
|
258
270
|
const signal = options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
|
|
259
271
|
let output;
|
|
@@ -470,59 +482,62 @@ function revalidateSummary(value, ledger) {
|
|
|
470
482
|
}, ledger);
|
|
471
483
|
}
|
|
472
484
|
async function captureChangeSummary(session, value) {
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
session.summaryCorrection = null;
|
|
481
|
-
const { state, ledger, diagnostics } = session.run;
|
|
485
|
+
// An omitted summary contributes no update; after feedback it explicitly abandons correction.
|
|
486
|
+
session.summaryCorrection = null;
|
|
487
|
+
const { state, diagnostics } = session.run;
|
|
488
|
+
// Correction batches need not repeat an already checked summary.
|
|
489
|
+
if (value === undefined ||
|
|
490
|
+
(value === null && (session.phase === 'assess' || state.changeSummary?.status === 'verified')))
|
|
491
|
+
return;
|
|
482
492
|
if (session.phase === 'verify' && value === false) {
|
|
483
493
|
state.changeSummary = { candidate: state.changeSummary?.candidate ?? null, accepted: null, status: 'rejected' };
|
|
484
494
|
await diagnostics.event('change_summary_withdrawn', { phase: session.phase });
|
|
485
495
|
return;
|
|
486
496
|
}
|
|
487
|
-
// Correction batches need not repeat an already checked summary.
|
|
488
|
-
if (value === null && state.changeSummary?.status === 'verified')
|
|
489
|
-
return;
|
|
490
497
|
try {
|
|
491
|
-
|
|
492
|
-
if (session.phase === 'assess') {
|
|
493
|
-
if (summary)
|
|
494
|
-
state.changeSummary = { candidate: summary, accepted: null, status: 'pending' };
|
|
495
|
-
}
|
|
496
|
-
else {
|
|
497
|
-
const accepted = state.changeSummary?.candidate ? summary : null;
|
|
498
|
-
state.changeSummary = {
|
|
499
|
-
candidate: state.changeSummary?.candidate ?? null,
|
|
500
|
-
accepted,
|
|
501
|
-
status: accepted ? 'verified' : 'rejected',
|
|
502
|
-
};
|
|
503
|
-
}
|
|
498
|
+
applySummary(session, value);
|
|
504
499
|
}
|
|
505
500
|
catch (error) {
|
|
506
|
-
session
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
};
|
|
518
|
-
await diagnostics.event('change_summary_rejected', {
|
|
519
|
-
phase: session.phase,
|
|
520
|
-
...session.summaryCorrection,
|
|
521
|
-
details,
|
|
522
|
-
attempt: session.summaryFailures,
|
|
523
|
-
retryable: session.summaryFailures < 2,
|
|
524
|
-
});
|
|
501
|
+
await rejectSummary(session, error);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
/** Assess proposes a pending candidate; verify accepts it only when there was a candidate to check. */
|
|
505
|
+
function applySummary(session, value) {
|
|
506
|
+
const { state, ledger } = session.run;
|
|
507
|
+
const summary = value === null ? null : validateChangeSummary(resolveCitationSelections(value, ledger), ledger);
|
|
508
|
+
if (session.phase === 'assess') {
|
|
509
|
+
if (summary)
|
|
510
|
+
state.changeSummary = { candidate: summary, accepted: null, status: 'pending' };
|
|
511
|
+
return;
|
|
525
512
|
}
|
|
513
|
+
const accepted = state.changeSummary?.candidate ? summary : null;
|
|
514
|
+
state.changeSummary = {
|
|
515
|
+
candidate: state.changeSummary?.candidate ?? null,
|
|
516
|
+
accepted,
|
|
517
|
+
status: accepted ? 'verified' : 'rejected',
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
async function rejectSummary(session, error) {
|
|
521
|
+
const { state, diagnostics } = session.run;
|
|
522
|
+
session.summaryFailures++;
|
|
523
|
+
const { code, details } = submissionDiagnostics(error);
|
|
524
|
+
session.summaryCorrection = {
|
|
525
|
+
code: error instanceof Error && error.message === 'change_summary_source_required' ? error.message : code,
|
|
526
|
+
correction: submissionCorrection(error, code),
|
|
527
|
+
};
|
|
528
|
+
if (session.phase === 'verify' && state.changeSummary?.status !== 'verified')
|
|
529
|
+
state.changeSummary = {
|
|
530
|
+
candidate: state.changeSummary?.candidate ?? null,
|
|
531
|
+
accepted: null,
|
|
532
|
+
status: 'rejected',
|
|
533
|
+
};
|
|
534
|
+
await diagnostics.event('change_summary_rejected', {
|
|
535
|
+
phase: session.phase,
|
|
536
|
+
...session.summaryCorrection,
|
|
537
|
+
details,
|
|
538
|
+
attempt: session.summaryFailures,
|
|
539
|
+
retryable: session.summaryFailures < 2,
|
|
540
|
+
});
|
|
526
541
|
}
|
|
527
542
|
function summarySubmissionFeedback(session) {
|
|
528
543
|
const rejected = session.summaryCorrection;
|
|
@@ -737,7 +752,7 @@ function createPhaseTools(session) {
|
|
|
737
752
|
}
|
|
738
753
|
function createProviderFetch(session) {
|
|
739
754
|
const { run, phase, workUnitId, attempt, phaseCalls, signal } = session;
|
|
740
|
-
const { state, diagnostics, models,
|
|
755
|
+
const { state, diagnostics, models, providerFetch, budget, persist, previousCalls } = run;
|
|
741
756
|
return async (input, init) => {
|
|
742
757
|
signal.throwIfAborted();
|
|
743
758
|
const call = previousCalls + budget.reserve(phase);
|
|
@@ -766,7 +781,7 @@ function createProviderFetch(session) {
|
|
|
766
781
|
});
|
|
767
782
|
let recorded = false;
|
|
768
783
|
try {
|
|
769
|
-
const response = await (
|
|
784
|
+
const response = await providerFetch(input, init);
|
|
770
785
|
if (!response.ok) {
|
|
771
786
|
budget.record(null);
|
|
772
787
|
recorded = true;
|
|
@@ -872,7 +887,7 @@ async function runPhaseModel(session) {
|
|
|
872
887
|
stopWhen: () => session.output !== undefined ||
|
|
873
888
|
session.rejected >= 3 ||
|
|
874
889
|
session.sdkStop !== null ||
|
|
875
|
-
session.phaseCalls.length >= (
|
|
890
|
+
session.phaseCalls.length >= phaseCallLimit(session),
|
|
876
891
|
prepareStep: () => {
|
|
877
892
|
session.toolsThisStep = 0;
|
|
878
893
|
return phase === 'assess' && session.phaseCalls.length >= 58
|
|
@@ -917,7 +932,7 @@ async function runPhaseModel(session) {
|
|
|
917
932
|
if (session.output !== undefined ||
|
|
918
933
|
session.rejected >= 3 ||
|
|
919
934
|
session.sdkStop !== null ||
|
|
920
|
-
session.phaseCalls.length >= (
|
|
935
|
+
session.phaseCalls.length >= phaseCallLimit(session) ||
|
|
921
936
|
continuation >= 2)
|
|
922
937
|
break;
|
|
923
938
|
continuation++;
|
|
@@ -944,7 +959,7 @@ async function runPhaseModel(session) {
|
|
|
944
959
|
if (session.sdkStop !== null)
|
|
945
960
|
throw new BudgetStop(session.sdkStop);
|
|
946
961
|
if (session.output === undefined)
|
|
947
|
-
throw new BudgetStop(session.phaseCalls.length >= (
|
|
962
|
+
throw new BudgetStop(session.phaseCalls.length >= phaseCallLimit(session)
|
|
948
963
|
? 'model_call_limit'
|
|
949
964
|
: session.rejected >= 3
|
|
950
965
|
? 'phase_corrections_exhausted'
|
|
@@ -970,24 +985,61 @@ function phaseStopReason(session, failure) {
|
|
|
970
985
|
if (session.signal.aborted) {
|
|
971
986
|
if (session.run.options.signal?.aborted)
|
|
972
987
|
return 'cancelled';
|
|
973
|
-
return
|
|
988
|
+
return 'max_time_ms';
|
|
974
989
|
}
|
|
975
990
|
return knownExecutionFailure(failure);
|
|
976
991
|
}
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
? {
|
|
992
|
+
/** What a failed phase already accepted, so a resumed run continues instead of repeating it. */
|
|
993
|
+
function partialProgress(session) {
|
|
994
|
+
if (session.phase === 'assess' && session.assessmentAccumulator?.snapshot().length)
|
|
995
|
+
return {
|
|
982
996
|
assessments: session.assessmentAccumulator.snapshot(),
|
|
983
997
|
remaining: session.assessmentAccumulator.remaining(),
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
998
|
+
};
|
|
999
|
+
if (session.verifications?.size)
|
|
1000
|
+
return {
|
|
1001
|
+
verifications: [...session.verifications.values()].map((entry) => entry.result),
|
|
1002
|
+
remaining: remainingContexts(session),
|
|
1003
|
+
};
|
|
1004
|
+
return null;
|
|
1005
|
+
}
|
|
1006
|
+
function phaseCallLimit({ phase, run }) {
|
|
1007
|
+
return phase === 'assess' ? 60 : (run.limits.maxVerifyCalls ?? DEFAULT_VERIFY_CALLS);
|
|
1008
|
+
}
|
|
1009
|
+
function phaseRecordConfiguration(session, model, partial) {
|
|
1010
|
+
const { phase, instructions } = session;
|
|
1011
|
+
return {
|
|
1012
|
+
provider: model.provider,
|
|
1013
|
+
model: model.model,
|
|
1014
|
+
effort: model.effort ?? null,
|
|
1015
|
+
...(partial
|
|
1016
|
+
? { kind: phase === 'assess' ? 'partial_assessment_progress' : 'partial_verification_progress' }
|
|
1017
|
+
: {}),
|
|
1018
|
+
modelIdentifiers: session.identifiers.snapshot(),
|
|
1019
|
+
promptVersion: 'candidate-observation-v3',
|
|
1020
|
+
maxPhaseCalls: phaseCallLimit(session),
|
|
1021
|
+
maxSourceToolsPerResponse: 50,
|
|
1022
|
+
validationConcurrency: 2,
|
|
1023
|
+
promptHash: digest(instructions),
|
|
1024
|
+
sdkVersion: runtimeVersion('ai'),
|
|
1025
|
+
adapter: '@ai-sdk/openai',
|
|
1026
|
+
adapterVersion: runtimeVersion('@ai-sdk/openai'),
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
/** The phase output, carrying the change summary as assess proposed it or verify accepted it. */
|
|
1030
|
+
function phaseOutput(session, partial) {
|
|
1031
|
+
const summary = session.run.state.changeSummary;
|
|
1032
|
+
if (!summary)
|
|
1033
|
+
return partial ?? session.output ?? null;
|
|
1034
|
+
return {
|
|
1035
|
+
...(partial ?? { [session.phase === 'assess' ? 'assessments' : 'entries']: session.output ?? [] }),
|
|
1036
|
+
changeSummary: session.phase === 'assess' ? summary.candidate : summary.accepted,
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
async function persistPhaseResult(session, failure) {
|
|
1040
|
+
const { run, phase, workUnitId, assigned, attempt, startedAt, inputRefs, phaseCalls, signal } = session;
|
|
1041
|
+
const { state, directory, ledger, diagnostics, models, persist } = run;
|
|
1042
|
+
const partial = failure ? partialProgress(session) : null;
|
|
991
1043
|
if (failure && phase === 'verify' && state.changeSummary) {
|
|
992
1044
|
state.changeSummary.accepted = null;
|
|
993
1045
|
state.changeSummary.status = 'pending';
|
|
@@ -1007,30 +1059,9 @@ async function persistPhaseResult(session, failure) {
|
|
|
1007
1059
|
evidenceRefs: Object.keys(ledger.records),
|
|
1008
1060
|
limitations: [],
|
|
1009
1061
|
stopReason: phaseStopReason(session, failure),
|
|
1010
|
-
configuration:
|
|
1011
|
-
provider: models[phase].provider,
|
|
1012
|
-
model: models[phase].model,
|
|
1013
|
-
effort: models[phase].effort ?? null,
|
|
1014
|
-
...(partial
|
|
1015
|
-
? { kind: phase === 'assess' ? 'partial_assessment_progress' : 'partial_verification_progress' }
|
|
1016
|
-
: {}),
|
|
1017
|
-
modelIdentifiers: session.identifiers.snapshot(),
|
|
1018
|
-
promptVersion: 'candidate-observation-v3',
|
|
1019
|
-
maxPhaseCalls: phase === 'assess' ? 60 : 12,
|
|
1020
|
-
maxSourceToolsPerResponse: 50,
|
|
1021
|
-
validationConcurrency: 2,
|
|
1022
|
-
promptHash: digest(instructions),
|
|
1023
|
-
sdkVersion: runtimeVersion('ai'),
|
|
1024
|
-
adapter: '@ai-sdk/openai',
|
|
1025
|
-
adapterVersion: runtimeVersion('@ai-sdk/openai'),
|
|
1026
|
-
},
|
|
1062
|
+
configuration: phaseRecordConfiguration(session, models[phase], partial !== null),
|
|
1027
1063
|
usageRefs: phaseCalls.map((c) => `call:${c}`),
|
|
1028
|
-
output:
|
|
1029
|
-
? {
|
|
1030
|
-
...(partial ?? { [phase === 'assess' ? 'assessments' : 'entries']: session.output ?? [] }),
|
|
1031
|
-
changeSummary: phase === 'assess' ? run.state.changeSummary.candidate : run.state.changeSummary.accepted,
|
|
1032
|
-
}
|
|
1033
|
-
: (partial ?? session.output ?? null),
|
|
1064
|
+
output: phaseOutput(session, partial),
|
|
1034
1065
|
});
|
|
1035
1066
|
const filename = phaseFilename(phase, workUnitId, attempt);
|
|
1036
1067
|
await atomicJson(join(directory, 'phases', filename), sanitizeRecord(record));
|
|
@@ -1080,6 +1111,12 @@ async function recoverSummaryOnlyFailure(session, error) {
|
|
|
1080
1111
|
});
|
|
1081
1112
|
return true;
|
|
1082
1113
|
}
|
|
1114
|
+
/** A budget stop keeps its own code; an aborted phase hit its deadline or was cancelled; anything else failed in the provider or execution. */
|
|
1115
|
+
function phaseFailureCode(session, error) {
|
|
1116
|
+
if (error instanceof BudgetStop)
|
|
1117
|
+
return error.code;
|
|
1118
|
+
return session.signal.aborted ? 'phase_deadline_or_cancelled' : 'provider_or_execution_failed';
|
|
1119
|
+
}
|
|
1083
1120
|
async function invokePhase(run, phase, workUnitId, assigned, payload, schema, validate, attempt = 1) {
|
|
1084
1121
|
const session = await createPhaseRuntime(run, phase, workUnitId, assigned, payload, schema, validate, attempt);
|
|
1085
1122
|
await run.diagnostics.event('phase_started', { phase, workUnitId, attempt, inputHash: session.inputRefs[0].hash });
|
|
@@ -1095,11 +1132,7 @@ async function invokePhase(run, phase, workUnitId, assigned, payload, schema, va
|
|
|
1095
1132
|
phase,
|
|
1096
1133
|
workUnitId,
|
|
1097
1134
|
attempt,
|
|
1098
|
-
code: error
|
|
1099
|
-
? error.code
|
|
1100
|
-
: session.signal.aborted
|
|
1101
|
-
? 'phase_deadline_or_cancelled'
|
|
1102
|
-
: 'provider_or_execution_failed',
|
|
1135
|
+
code: phaseFailureCode(session, error),
|
|
1103
1136
|
details: { ...providerFailureDiagnostics(error), ...systemFailureDiagnostics(error) },
|
|
1104
1137
|
});
|
|
1105
1138
|
}
|
|
@@ -1168,6 +1201,16 @@ async function prepareRunEvidence(run) {
|
|
|
1168
1201
|
await atomicJson(join(directory, 'prepared.json'), sanitizeRecord(prepared.state));
|
|
1169
1202
|
if (prContext)
|
|
1170
1203
|
await atomicJson(join(directory, 'pr-context.json'), sanitizeRecord(prContext));
|
|
1204
|
+
assertPinnedMaterial(state, material);
|
|
1205
|
+
for (const publication of Object.values(material))
|
|
1206
|
+
ledger.publication(publication);
|
|
1207
|
+
run.evidencePack = prepareEvidencePack({ state: prepared.state, publications: material, ledger });
|
|
1208
|
+
await writeEvidencePack(directory, run.evidencePack);
|
|
1209
|
+
run.repositoryInstructions = await repositoryInstructionsWithinBudget(run);
|
|
1210
|
+
await run.persist();
|
|
1211
|
+
}
|
|
1212
|
+
/** Each assessed entry's pinned publication is still the exact version and content it was prepared with. */
|
|
1213
|
+
function assertPinnedMaterial(state, material) {
|
|
1171
1214
|
for (const entry of state.entries) {
|
|
1172
1215
|
if (!entry.material)
|
|
1173
1216
|
continue;
|
|
@@ -1178,24 +1221,22 @@ async function prepareRunEvidence(run) {
|
|
|
1178
1221
|
digest(pinned.content) !== entry.contentHash)
|
|
1179
1222
|
throw new Error('pinned_material_changed');
|
|
1180
1223
|
}
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
await writeEvidencePack(directory, run.evidencePack);
|
|
1224
|
+
}
|
|
1225
|
+
/** Repository instruction files count against the assess time budget; running out is a budget stop. */
|
|
1226
|
+
async function repositoryInstructionsWithinBudget(run) {
|
|
1185
1227
|
const remaining = run.budget.remainingMs('assess');
|
|
1186
1228
|
if (remaining !== null && remaining <= 0)
|
|
1187
1229
|
throw new BudgetStop('max_time_ms');
|
|
1188
1230
|
const deadline = remaining === null ? undefined : AbortSignal.timeout(Math.ceil(remaining));
|
|
1189
1231
|
const signals = [run.options.signal, deadline].filter((value) => value !== undefined);
|
|
1190
1232
|
try {
|
|
1191
|
-
|
|
1233
|
+
return await repositoryInstructionsEvidence(run.source, run.ledger, signals.length ? AbortSignal.any(signals) : undefined);
|
|
1192
1234
|
}
|
|
1193
1235
|
catch (error) {
|
|
1194
1236
|
if (deadline?.aborted)
|
|
1195
1237
|
throw new BudgetStop('max_time_ms');
|
|
1196
1238
|
throw error;
|
|
1197
1239
|
}
|
|
1198
|
-
await run.persist();
|
|
1199
1240
|
}
|
|
1200
1241
|
async function restorePartialProposals(run) {
|
|
1201
1242
|
if (!run.options.resume)
|
package/dist/assessment.d.ts
CHANGED
|
@@ -500,19 +500,6 @@ export declare function deriveAssessmentCitations(input: unknown, entries: Asses
|
|
|
500
500
|
checkIndices: number[];
|
|
501
501
|
}[];
|
|
502
502
|
};
|
|
503
|
-
export declare class AssessmentAccumulator {
|
|
504
|
-
private readonly entries;
|
|
505
|
-
private readonly ledger;
|
|
506
|
-
private readonly structured;
|
|
507
|
-
private readonly accepted;
|
|
508
|
-
constructor(entries: AssessmentEntry[], ledger: EvidenceLedger, structured?: Record<string, Array<{
|
|
509
|
-
path: string;
|
|
510
|
-
value: unknown;
|
|
511
|
-
}>>, seed?: AssessmentProposal[]);
|
|
512
|
-
submit(input: unknown): AssessmentProgress;
|
|
513
|
-
remaining(): string[];
|
|
514
|
-
snapshot(): AssessmentProposal[];
|
|
515
|
-
}
|
|
516
503
|
export type AssessmentProgress = {
|
|
517
504
|
complete: boolean;
|
|
518
505
|
newlyAccepted: string[];
|
|
@@ -547,7 +534,7 @@ export type AssessmentFinding = {
|
|
|
547
534
|
};
|
|
548
535
|
verified: boolean;
|
|
549
536
|
};
|
|
550
|
-
|
|
537
|
+
type AssessmentFollowUp = Omit<z.infer<typeof followUpSchema>, 'kind' | 'passage' | 'sourceRefs'> & {
|
|
551
538
|
kind: 'human_check' | 'operational_check' | 'external_evidence';
|
|
552
539
|
externalEvidence?: {
|
|
553
540
|
kind: 'external_dependency' | 'operational_state' | 'business_record';
|