@contextrail/assess 1.0.0-alpha.3 → 1.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/assess.d.ts +2 -0
- package/dist/assess.js +177 -136
- package/dist/assessment.d.ts +1 -14
- package/dist/assessment.js +0 -46
- package/dist/budget.d.ts +3 -0
- package/dist/budget.js +2 -0
- package/dist/candidate.js +135 -104
- package/dist/change-summary.d.ts +2 -1
- package/dist/change-summary.js +1 -1
- package/dist/cli.js +13 -1
- package/dist/contracts.d.ts +2 -8
- package/dist/contracts.js +2 -2
- package/dist/diagnostics.js +1 -0
- package/dist/evidence.d.ts +4 -0
- package/dist/evidence.js +21 -12
- package/dist/feedback.js +27 -23
- package/dist/git.d.ts +6 -0
- package/dist/git.js +30 -3
- package/dist/github.js +78 -52
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp.d.ts +2 -0
- package/dist/mcp.js +67 -56
- package/dist/pack.js +8 -1
- package/dist/pr-context.js +26 -16
- package/dist/presentation.js +113 -71
- package/dist/provider-diagnostics.js +17 -12
- package/dist/providers.js +24 -20
- package/dist/report-contract.js +63 -57
- package/dist/results.js +5 -1
- package/dist/source.js +20 -11
- package/dist/speed.d.ts +13 -0
- package/dist/speed.js +72 -0
- package/docs/action-reference.md +2 -0
- package/docs/configuration.md +12 -1
- package/docs/github-actions.md +1 -0
- package/examples/assess-pull-request.yml +1 -1
- package/package.json +1 -1
package/dist/assessment.js
CHANGED
|
@@ -228,52 +228,6 @@ export function deriveAssessmentCitations(input, entries) {
|
|
|
228
228
|
};
|
|
229
229
|
return candidate;
|
|
230
230
|
}
|
|
231
|
-
export class AssessmentAccumulator {
|
|
232
|
-
entries;
|
|
233
|
-
ledger;
|
|
234
|
-
structured;
|
|
235
|
-
accepted = new Map();
|
|
236
|
-
constructor(entries, ledger, structured = {}, seed = []) {
|
|
237
|
-
this.entries = entries;
|
|
238
|
-
this.ledger = ledger;
|
|
239
|
-
this.structured = structured;
|
|
240
|
-
for (const proposal of seed) {
|
|
241
|
-
const validated = revalidateProposal(proposal, entries, ledger, structured);
|
|
242
|
-
if (this.accepted.has(validated.result.assessmentId))
|
|
243
|
-
throw new Error('assessment_coverage_mismatch');
|
|
244
|
-
this.accepted.set(validated.result.assessmentId, validated);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
submit(input) {
|
|
248
|
-
const candidate = deriveAssessmentCitations(input, this.entries);
|
|
249
|
-
const id = candidate.result.assessmentId;
|
|
250
|
-
if (!this.entries.some((e) => e.assessmentId === id))
|
|
251
|
-
throw new Error('foreign_assessment_id');
|
|
252
|
-
if (this.accepted.has(id))
|
|
253
|
-
return {
|
|
254
|
-
complete: this.remaining().length === 0,
|
|
255
|
-
newlyAccepted: [],
|
|
256
|
-
alreadyAccepted: [id],
|
|
257
|
-
remaining: this.remaining(),
|
|
258
|
-
};
|
|
259
|
-
this.accepted.set(id, validateAssessmentSubmission({ assessments: [candidate] }, this.entries, this.ledger, this.structured));
|
|
260
|
-
return {
|
|
261
|
-
complete: this.remaining().length === 0,
|
|
262
|
-
newlyAccepted: [id],
|
|
263
|
-
alreadyAccepted: [],
|
|
264
|
-
remaining: this.remaining(),
|
|
265
|
-
};
|
|
266
|
-
}
|
|
267
|
-
remaining() {
|
|
268
|
-
return this.entries.filter((e) => !this.accepted.has(e.assessmentId)).map((e) => e.assessmentId);
|
|
269
|
-
}
|
|
270
|
-
snapshot() {
|
|
271
|
-
return this.entries.flatMap((e) => {
|
|
272
|
-
const p = this.accepted.get(e.assessmentId);
|
|
273
|
-
return p ? [structuredClone(p)] : [];
|
|
274
|
-
});
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
231
|
function validateFollowUps(input, entry, ledger, prefix) {
|
|
278
232
|
return input.map((followUp, index) => {
|
|
279
233
|
if (followUp.required && followUp.kind !== 'operational_check')
|
package/dist/budget.d.ts
CHANGED
|
@@ -4,7 +4,10 @@ export type ExecutionLimits = {
|
|
|
4
4
|
maxOutputTokens: number | null;
|
|
5
5
|
maxTotalTokens: number | null;
|
|
6
6
|
maxTimeMs: number | null;
|
|
7
|
+
/** Model calls per verification work unit; null keeps the package default. */
|
|
8
|
+
maxVerifyCalls: number | null;
|
|
7
9
|
};
|
|
10
|
+
export declare const DEFAULT_VERIFY_CALLS = 12;
|
|
8
11
|
export declare const DEFAULT_EXECUTION_LIMITS: ExecutionLimits;
|
|
9
12
|
export declare class BudgetStop extends Error {
|
|
10
13
|
readonly code: string;
|
package/dist/budget.js
CHANGED
package/dist/candidate.js
CHANGED
|
@@ -216,25 +216,41 @@ export function validateConcern(input, observation, ledger) {
|
|
|
216
216
|
})
|
|
217
217
|
.parse(input);
|
|
218
218
|
const sourceRefs = sourceCitations(parsed.sourceRefs.map((c) => ledger.cite(c)), observation.passage);
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
throw Error(
|
|
222
|
-
if (parsed.decision === 'supported_violation' && parsed.applicability !== 'applicable')
|
|
223
|
-
throw Error('violation_applicability_required');
|
|
224
|
-
if (parsed.decision === 'not_applicable' && parsed.applicability !== 'not_applicable')
|
|
225
|
-
throw Error('inapplicable_decision_conflict');
|
|
226
|
-
if (parsed.decision === 'external_follow_up') {
|
|
227
|
-
if (!parsed.externalEvidence || !sourceRefs.length || !parsed.correction.trim())
|
|
228
|
-
throw Error('external_follow_up_requires_boundary_source_and_action');
|
|
229
|
-
if (parsed.applicability === 'not_applicable')
|
|
230
|
-
throw Error('irrelevant_external_follow_up');
|
|
231
|
-
}
|
|
232
|
-
else if (parsed.externalEvidence)
|
|
233
|
-
throw Error('external_evidence_decision_mismatch');
|
|
234
|
-
if (parsed.suggestionAccepted && (parsed.decision !== 'supported_violation' || !observation.proposedReplacement))
|
|
235
|
-
throw Error('unsupported_replacement_acceptance');
|
|
219
|
+
const conflict = decisionConflict(parsed, sourceRefs.length > 0, observation);
|
|
220
|
+
if (conflict)
|
|
221
|
+
throw Error(conflict);
|
|
236
222
|
return { ...parsed, sourceRefs, observationId: observation.id };
|
|
237
223
|
}
|
|
224
|
+
/** The first rule a concern decision breaks, as its error code. */
|
|
225
|
+
function decisionConflict(parsed, hasSourceRefs, observation) {
|
|
226
|
+
const violation = parsed.decision === 'supported_violation';
|
|
227
|
+
return ((violation ? violationConflict(parsed, hasSourceRefs) : undefined) ??
|
|
228
|
+
(parsed.decision === 'not_applicable' && parsed.applicability !== 'not_applicable'
|
|
229
|
+
? 'inapplicable_decision_conflict'
|
|
230
|
+
: undefined) ??
|
|
231
|
+
externalEvidenceConflict(parsed, hasSourceRefs) ??
|
|
232
|
+
(parsed.suggestionAccepted && (!violation || !observation.proposedReplacement)
|
|
233
|
+
? 'unsupported_replacement_acceptance'
|
|
234
|
+
: undefined));
|
|
235
|
+
}
|
|
236
|
+
/** A supported violation needs source evidence, a consequence and a correction, and must apply. */
|
|
237
|
+
function violationConflict(parsed, hasSourceRefs) {
|
|
238
|
+
if (!hasSourceRefs || !parsed.consequence.trim() || !parsed.correction.trim())
|
|
239
|
+
return 'supported_violation_requires_evidence_and_correction';
|
|
240
|
+
if (parsed.applicability !== 'applicable')
|
|
241
|
+
return 'violation_applicability_required';
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
/** External evidence belongs only to an applicable external follow-up with a boundary, source and action. */
|
|
245
|
+
function externalEvidenceConflict(parsed, hasSourceRefs) {
|
|
246
|
+
if (parsed.decision !== 'external_follow_up')
|
|
247
|
+
return parsed.externalEvidence ? 'external_evidence_decision_mismatch' : undefined;
|
|
248
|
+
if (!parsed.externalEvidence || !hasSourceRefs || !parsed.correction.trim())
|
|
249
|
+
return 'external_follow_up_requires_boundary_source_and_action';
|
|
250
|
+
if (parsed.applicability === 'not_applicable')
|
|
251
|
+
return 'irrelevant_external_follow_up';
|
|
252
|
+
return undefined;
|
|
253
|
+
}
|
|
238
254
|
function projectCheck(check, decision) {
|
|
239
255
|
if (!decision)
|
|
240
256
|
return { ...check, verified: false };
|
|
@@ -258,104 +274,119 @@ export function projectCandidates(state, proposals, decisions) {
|
|
|
258
274
|
state.followUps = [];
|
|
259
275
|
for (const proposal of proposals) {
|
|
260
276
|
const entry = state.entries.find((e) => e.assessmentId === proposal.result.assessmentId);
|
|
261
|
-
const
|
|
262
|
-
const judgments = observations.map((o) => ({
|
|
277
|
+
const judgments = (proposal.observations ?? []).map((o) => ({
|
|
263
278
|
observation: o,
|
|
264
279
|
decision: decisions.find((d) => d.observationId === o.id),
|
|
265
280
|
}));
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
281
|
+
state.followUps.push(...externalFollowUps(entry.assessmentId, judgments));
|
|
282
|
+
entry.result = projectedResult(proposal.result, judgments);
|
|
283
|
+
state.checks.push(...proposal.checks.map((check) => projectCheck(check, decisions.find((d) => d.observationId === check.checkId))));
|
|
284
|
+
for (const { observation, decision } of judgments.filter(isViolation))
|
|
285
|
+
mergeFinding(state.findings, violationFinding(entry.assessmentId, observation, decision));
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
function isViolation(judgment) {
|
|
289
|
+
return judgment.decision?.decision === 'supported_violation';
|
|
290
|
+
}
|
|
291
|
+
function externalFollowUps(assessmentId, judgments) {
|
|
292
|
+
return judgments.flatMap(({ observation: o, decision: d }) => d?.decision !== 'external_follow_up' || !d.externalEvidence
|
|
293
|
+
? []
|
|
294
|
+
: [
|
|
295
|
+
{
|
|
274
296
|
kind: 'external_evidence',
|
|
275
297
|
required: false,
|
|
276
298
|
verified: true,
|
|
277
|
-
assessmentId
|
|
299
|
+
assessmentId,
|
|
278
300
|
passage: o.passage,
|
|
279
301
|
sourceRefs: d.sourceRefs,
|
|
280
302
|
explanation: d.reason,
|
|
281
303
|
action: d.correction,
|
|
282
304
|
externalEvidence: d.externalEvidence,
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
classification: d.duty === 'binding' ? 'binding_violation' : 'guidance_departure',
|
|
331
|
-
locationRefs: d.sourceRefs,
|
|
332
|
-
explanation: d.reason,
|
|
333
|
-
consequence: d.consequence,
|
|
334
|
-
correction: d.correction,
|
|
335
|
-
limitations: o.assumptions,
|
|
336
|
-
verified: true,
|
|
337
|
-
};
|
|
338
|
-
const duplicate = state.findings.find((f) => f.classification === finding.classification &&
|
|
339
|
-
f.explanation.trim() === finding.explanation.trim() &&
|
|
340
|
-
f.consequence.trim() === finding.consequence.trim() &&
|
|
341
|
-
f.correction.trim() === finding.correction.trim() &&
|
|
342
|
-
f.locationRefs.some((c) => finding.locationRefs.some((r) => r.ref === c.ref && r.start === c.start && r.end === c.end)));
|
|
343
|
-
if (duplicate) {
|
|
344
|
-
if (finding.classification === 'binding_violation')
|
|
345
|
-
duplicate.classification = 'binding_violation';
|
|
346
|
-
duplicate.locationRefs = [
|
|
347
|
-
...duplicate.locationRefs,
|
|
348
|
-
...finding.locationRefs.filter((c) => !duplicate.locationRefs.some((r) => r.ref === c.ref && r.start === c.start && r.end === c.end)),
|
|
349
|
-
];
|
|
350
|
-
duplicate.assessmentIds = [...new Set([...duplicate.assessmentIds, ...finding.assessmentIds])];
|
|
351
|
-
duplicate.checkIds = [...new Set([...duplicate.checkIds, ...finding.checkIds])];
|
|
352
|
-
duplicate.sourceFindingIds = [
|
|
353
|
-
...(duplicate.sourceFindingIds ?? [duplicate.findingId]),
|
|
354
|
-
finding.findingId,
|
|
355
|
-
];
|
|
356
|
-
}
|
|
357
|
-
else
|
|
358
|
-
state.findings.push(finding);
|
|
305
|
+
},
|
|
306
|
+
]);
|
|
307
|
+
}
|
|
308
|
+
/** The assessment result after its nominated concerns were independently decided. */
|
|
309
|
+
function projectedResult(result, judgments) {
|
|
310
|
+
const violations = judgments.filter(isViolation);
|
|
311
|
+
const unresolved = judgments.filter((x) => x.observation.concern
|
|
312
|
+
? !x.decision || ['missing_evidence', 'ambiguous_rule'].includes(x.decision.decision)
|
|
313
|
+
: x.observation.unknowns.length > 0);
|
|
314
|
+
const explanation = judgments
|
|
315
|
+
.map((x) => x.decision?.reason || x.observation.observation || x.observation.unknowns.join(' '))
|
|
316
|
+
.join('\n');
|
|
317
|
+
return {
|
|
318
|
+
...result,
|
|
319
|
+
explanation,
|
|
320
|
+
limitations: [
|
|
321
|
+
...result.limitations,
|
|
322
|
+
...judgments
|
|
323
|
+
.filter((x) => x.decision?.decision === 'external_follow_up')
|
|
324
|
+
.map((x) => `Outside artifact evidence boundary: ${x.decision.externalEvidence.subject}. Not assessed; no compliance conclusion.`),
|
|
325
|
+
],
|
|
326
|
+
citations: [...result.citations, ...judgments.flatMap((x) => x.decision?.sourceRefs ?? [])],
|
|
327
|
+
disposition: unresolved.length ? 'unresolved' : violations.length ? 'evaluated' : 'observed',
|
|
328
|
+
adherence: violations.length && !unresolved.length ? 0 : null,
|
|
329
|
+
confidence: null,
|
|
330
|
+
finding: violations.length
|
|
331
|
+
? violations.some((x) => x.decision.duty === 'binding')
|
|
332
|
+
? 'binding_violation'
|
|
333
|
+
: 'guidance_departure'
|
|
334
|
+
: 'none',
|
|
335
|
+
consequence: violations.map((x) => x.decision.consequence).join('\n'),
|
|
336
|
+
correction: violations.map((x) => x.decision.correction).join('\n'),
|
|
337
|
+
confidenceReason: 'No calibrated confidence estimate is supplied; nominated concerns have independent evidence-based decisions.',
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
function violationFinding(assessmentId, o, d) {
|
|
341
|
+
const replacement = o.proposedReplacement;
|
|
342
|
+
const before = replacement?.source.quote.replace(/\n$/u, '');
|
|
343
|
+
const suggestion = replacement && d.suggestionAccepted
|
|
344
|
+
? {
|
|
345
|
+
path: String(replacement.source.provenance.path),
|
|
346
|
+
blob: String(replacement.source.provenance.object),
|
|
347
|
+
startLine: replacement.source.location.line,
|
|
348
|
+
endLine: replacement.source.location.line + before.split('\n').length - 1,
|
|
349
|
+
before: before,
|
|
350
|
+
replacement: replacement.replacement,
|
|
351
|
+
verified: true,
|
|
359
352
|
}
|
|
353
|
+
: undefined;
|
|
354
|
+
return {
|
|
355
|
+
proposedReplacement: suggestion,
|
|
356
|
+
findingId: `finding:${digest({ assessmentId, id: o.id })}`,
|
|
357
|
+
assessmentIds: [assessmentId],
|
|
358
|
+
checkIds: [o.id],
|
|
359
|
+
classification: d.duty === 'binding' ? 'binding_violation' : 'guidance_departure',
|
|
360
|
+
locationRefs: d.sourceRefs,
|
|
361
|
+
explanation: d.reason,
|
|
362
|
+
consequence: d.consequence,
|
|
363
|
+
correction: d.correction,
|
|
364
|
+
limitations: o.assumptions,
|
|
365
|
+
verified: true,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
function sameRange(a, b) {
|
|
369
|
+
return a.ref === b.ref && a.start === b.start && a.end === b.end;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* A finding with the same classification and text at an overlapping location joins the earlier
|
|
373
|
+
* one; locations, assessments, checks and source finding IDs accumulate.
|
|
374
|
+
*/
|
|
375
|
+
function mergeFinding(findings, finding) {
|
|
376
|
+
const duplicate = findings.find((f) => f.classification === finding.classification &&
|
|
377
|
+
f.explanation.trim() === finding.explanation.trim() &&
|
|
378
|
+
f.consequence.trim() === finding.consequence.trim() &&
|
|
379
|
+
f.correction.trim() === finding.correction.trim() &&
|
|
380
|
+
f.locationRefs.some((c) => finding.locationRefs.some((r) => sameRange(r, c))));
|
|
381
|
+
if (!duplicate) {
|
|
382
|
+
findings.push(finding);
|
|
383
|
+
return;
|
|
360
384
|
}
|
|
385
|
+
duplicate.locationRefs = [
|
|
386
|
+
...duplicate.locationRefs,
|
|
387
|
+
...finding.locationRefs.filter((c) => !duplicate.locationRefs.some((r) => sameRange(r, c))),
|
|
388
|
+
];
|
|
389
|
+
duplicate.assessmentIds = [...new Set([...duplicate.assessmentIds, ...finding.assessmentIds])];
|
|
390
|
+
duplicate.checkIds = [...new Set([...duplicate.checkIds, ...finding.checkIds])];
|
|
391
|
+
duplicate.sourceFindingIds = [...(duplicate.sourceFindingIds ?? [duplicate.findingId]), finding.findingId];
|
|
361
392
|
}
|
package/dist/change-summary.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { type Citation, type EvidenceLedger } from './evidence.js';
|
|
3
3
|
/** Reader orientation, independent of governed assessment outcomes and enforcement. */
|
|
4
|
-
|
|
4
|
+
declare const changeSummarySchema: z.ZodObject<{
|
|
5
5
|
overview: z.ZodString;
|
|
6
6
|
reviewFocus: z.ZodArray<z.ZodString>;
|
|
7
7
|
sourceRefs: z.ZodArray<z.ZodObject<{
|
|
@@ -31,3 +31,4 @@ export type ChangeSummaryState = {
|
|
|
31
31
|
export declare function validateChangeSummary(value: unknown, ledger: EvidenceLedger): ChangeSummary;
|
|
32
32
|
/** Older handoffs are arrays; summaries are bound into newer handoff output hashes. */
|
|
33
33
|
export declare function phaseItems<T>(output: unknown, field: 'assessments' | 'entries'): T[];
|
|
34
|
+
export {};
|
package/dist/change-summary.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { citationSchema, selectionSchema } from './evidence.js';
|
|
3
3
|
/** Reader orientation, independent of governed assessment outcomes and enforcement. */
|
|
4
|
-
|
|
4
|
+
const changeSummarySchema = z.strictObject({
|
|
5
5
|
overview: z.string().min(1).max(2000),
|
|
6
6
|
reviewFocus: z.array(z.string().min(1).max(1000)).max(5),
|
|
7
7
|
sourceRefs: z.array(citationSchema).min(1).max(12),
|
package/dist/cli.js
CHANGED
|
@@ -10,6 +10,7 @@ import { loadAssessmentCheckpoint } from './resume.js';
|
|
|
10
10
|
import { runAssessment, DEFAULT_EXECUTION_LIMITS } from './assess.js';
|
|
11
11
|
import { DEFAULT_SOURCE_LIMITS } from './source.js';
|
|
12
12
|
import { providerTransportDiagnostics } from './providers.js';
|
|
13
|
+
import { PROVIDER_SPEEDS } from './speed.js';
|
|
13
14
|
import { DEFAULT_POLICY } from './results.js';
|
|
14
15
|
import { prepareGitHubAssessment, publishGitHubAssessment } from './github.js';
|
|
15
16
|
// SDK warnings are captured and redacted in the audit; avoid uncontrolled CLI logging.
|
|
@@ -19,6 +20,7 @@ const executionFlags = {
|
|
|
19
20
|
'max-output-tokens': 'maxOutputTokens',
|
|
20
21
|
'max-total-tokens': 'maxTotalTokens',
|
|
21
22
|
'max-time-ms': 'maxTimeMs',
|
|
23
|
+
'max-verify-calls': 'maxVerifyCalls',
|
|
22
24
|
};
|
|
23
25
|
const sourceFlags = {
|
|
24
26
|
'source-page-entries': 'pageEntries',
|
|
@@ -44,6 +46,8 @@ run requires --provider openai|openrouter and --model PROVIDER_NATIVE_ID
|
|
|
44
46
|
(or CONTEXTRAIL_ASSESS_PROVIDER / CONTEXTRAIL_ASSESS_MODEL). CLI overrides environment.
|
|
45
47
|
Credentials: OPENAI_API_KEY / OPENROUTER_API_KEY. No implicit provider/model fallback.
|
|
46
48
|
--effort none|minimal|low|medium|high|xhigh (provider/model must support it).
|
|
49
|
+
--speed normal|fast|ultrafast (default normal) requests the provider's priority or ultrafast service tier;
|
|
50
|
+
an unsupported tier is reported and the run continues at the tier the provider serves.
|
|
47
51
|
--phase-models JSON overrides assess/verify with explicit {provider,model,effort?} objects.
|
|
48
52
|
--log-level quiet|info|debug (default info).
|
|
49
53
|
--purpose TEXT supplies caller-stated objective/stage; no inferred objective is treated as approved.
|
|
@@ -56,6 +60,7 @@ Exit 0: allowed by enforcement policy (not proof of compliance); 1: policy block
|
|
|
56
60
|
|
|
57
61
|
Execution settings: --max-turns N, --max-output-tokens N, --max-total-tokens N, --max-time-ms N.
|
|
58
62
|
Default: unset (no package ceiling; provider output/context constraints still apply).
|
|
63
|
+
--max-verify-calls N sets model calls per verification (default 12); raise it for large changes.
|
|
59
64
|
Total-token cutoff is observed after calls, can overshoot by one call; missing usage prevents enforcement.
|
|
60
65
|
Per-call output allowance includes provider reasoning tokens where applicable.
|
|
61
66
|
Source response/operation settings (positive integers, read-bytes >=4):
|
|
@@ -99,6 +104,7 @@ function readArguments() {
|
|
|
99
104
|
provider: { type: 'string' },
|
|
100
105
|
model: { type: 'string' },
|
|
101
106
|
effort: { type: 'string' },
|
|
107
|
+
speed: { type: 'string' },
|
|
102
108
|
'phase-models': { type: 'string' },
|
|
103
109
|
'log-level': { type: 'string' },
|
|
104
110
|
repository: { type: 'string' },
|
|
@@ -256,6 +262,7 @@ function commandSettings(values, isAssessment) {
|
|
|
256
262
|
provider: undefined,
|
|
257
263
|
phaseModels: {},
|
|
258
264
|
logLevel: 'info',
|
|
265
|
+
speed: 'normal',
|
|
259
266
|
limits: { ...DEFAULT_EXECUTION_LIMITS },
|
|
260
267
|
sourceLimits: {},
|
|
261
268
|
policy: { ...DEFAULT_POLICY },
|
|
@@ -264,6 +271,9 @@ function commandSettings(values, isAssessment) {
|
|
|
264
271
|
return { setting, resumePath, feedback, repositoryAttribution, ...defaults };
|
|
265
272
|
const provider = readProvider(setting);
|
|
266
273
|
const phaseModels = readPhaseModels(setting('phase-models'));
|
|
274
|
+
const speed = setting('speed') ?? 'normal';
|
|
275
|
+
if (!PROVIDER_SPEEDS.includes(speed))
|
|
276
|
+
throw new CliInputError('Invalid --speed. See --help.');
|
|
267
277
|
const level = setting('log-level') ?? 'info';
|
|
268
278
|
if (!['quiet', 'info', 'debug'].includes(level))
|
|
269
279
|
throw new CliInputError('Invalid log-level.');
|
|
@@ -277,6 +287,7 @@ function commandSettings(values, isAssessment) {
|
|
|
277
287
|
repositoryAttribution,
|
|
278
288
|
provider,
|
|
279
289
|
phaseModels,
|
|
290
|
+
speed: speed,
|
|
280
291
|
logLevel,
|
|
281
292
|
...limits,
|
|
282
293
|
policy,
|
|
@@ -307,7 +318,7 @@ function commandComparison(values, isAssessment, setting) {
|
|
|
307
318
|
let commandStage = 'arguments';
|
|
308
319
|
async function runCommand(values, isAssessment) {
|
|
309
320
|
commandStage = 'configuration';
|
|
310
|
-
const { setting, resumePath, feedback, repositoryAttribution, provider, phaseModels, logLevel, limits, sourceLimits, policy, } = commandSettings(values, isAssessment);
|
|
321
|
+
const { setting, resumePath, feedback, repositoryAttribution, provider, phaseModels, speed, logLevel, limits, sourceLimits, policy, } = commandSettings(values, isAssessment);
|
|
311
322
|
const timeoutMs = mcpTimeoutMs(values['mcp-timeout-ms'] ?? process.env.CONTEXTRAIL_MCP_TIMEOUT_MS);
|
|
312
323
|
commandStage = 'comparison';
|
|
313
324
|
const { artifact, prContext } = commandComparison(values, isAssessment, setting);
|
|
@@ -342,6 +353,7 @@ async function runCommand(values, isAssessment) {
|
|
|
342
353
|
resume: resumed?.checkpoint,
|
|
343
354
|
provider: provider,
|
|
344
355
|
phaseModels,
|
|
356
|
+
speed,
|
|
345
357
|
logLevel,
|
|
346
358
|
policy,
|
|
347
359
|
limits,
|
package/dist/contracts.d.ts
CHANGED
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
export declare const selectionsSchema: z.ZodObject<{
|
|
3
|
-
contextIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
4
|
-
domains: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
5
|
-
domainId: z.ZodString;
|
|
6
|
-
includeDescendants: z.ZodBoolean;
|
|
7
|
-
}, z.core.$strict>>>;
|
|
8
|
-
}, z.core.$strict>;
|
|
9
2
|
export declare const reviewerSectionsSchema: z.ZodObject<{
|
|
10
3
|
examples: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
11
4
|
references: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
@@ -42,7 +35,7 @@ export declare const publicationSchema: z.ZodObject<{
|
|
|
42
35
|
}, z.core.$loose>;
|
|
43
36
|
}, z.core.$loose>;
|
|
44
37
|
export type Publication = z.infer<typeof publicationSchema>;
|
|
45
|
-
|
|
38
|
+
declare const scopeEntrySchema: z.ZodObject<{
|
|
46
39
|
nodeId: z.ZodString;
|
|
47
40
|
version: z.ZodNullable<z.ZodNumber>;
|
|
48
41
|
subtype: z.ZodNullable<z.ZodEnum<{
|
|
@@ -97,3 +90,4 @@ export declare class PreparationReadError extends Error {
|
|
|
97
90
|
category?: string;
|
|
98
91
|
} | undefined);
|
|
99
92
|
}
|
|
93
|
+
export {};
|
package/dist/contracts.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
|
|
2
|
+
const selectionsSchema = z.strictObject({
|
|
3
3
|
contextIds: z.array(z.string().startsWith('context://').max(500)).optional(),
|
|
4
4
|
domains: z.array(z.strictObject({ domainId: z.string().min(1), includeDescendants: z.boolean() })).optional(),
|
|
5
5
|
});
|
|
@@ -29,7 +29,7 @@ export const publicationSchema = z
|
|
|
29
29
|
.passthrough(),
|
|
30
30
|
})
|
|
31
31
|
.passthrough();
|
|
32
|
-
|
|
32
|
+
const scopeEntrySchema = z.strictObject({
|
|
33
33
|
nodeId: z.string().startsWith('context://'),
|
|
34
34
|
version: z.number().int().positive().nullable(),
|
|
35
35
|
subtype: z.enum(['standard', 'guidance']).nullable(),
|
package/dist/diagnostics.js
CHANGED
package/dist/evidence.d.ts
CHANGED
|
@@ -55,6 +55,10 @@ export declare class ModelIdentifiers {
|
|
|
55
55
|
private register;
|
|
56
56
|
private evidenceHandle;
|
|
57
57
|
private translate;
|
|
58
|
+
private isCitation;
|
|
59
|
+
private isResult;
|
|
60
|
+
/** Inward, a handle maps back to its canonical ID; outward, an ID becomes its model handle. */
|
|
61
|
+
private translateIdentifier;
|
|
58
62
|
toModel<T>(value: T): T;
|
|
59
63
|
/** Audit-only identity map; never includes the source text behind an evidence reference. */
|
|
60
64
|
snapshot(): Record<string, string>;
|
package/dist/evidence.js
CHANGED
|
@@ -78,18 +78,9 @@ export class ModelIdentifiers {
|
|
|
78
78
|
if (Array.isArray(value))
|
|
79
79
|
return value.map((item) => this.translate(item, outward, field));
|
|
80
80
|
if (value && typeof value === 'object') {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
Object.hasOwn(this.evidence, value.ref) &&
|
|
85
|
-
'quote' in value &&
|
|
86
|
-
typeof value.quote === 'string';
|
|
87
|
-
const result = outward &&
|
|
88
|
-
'assessmentId' in value &&
|
|
89
|
-
typeof value.assessmentId === 'string' &&
|
|
90
|
-
this.handles.has(value.assessmentId) &&
|
|
91
|
-
'disposition' in value &&
|
|
92
|
-
'adherence' in value;
|
|
81
|
+
// Outward, a citation loses its quoted span and a result its revision lineage.
|
|
82
|
+
const citation = outward && this.isCitation(value);
|
|
83
|
+
const result = outward && this.isResult(value);
|
|
93
84
|
return Object.fromEntries(Object.entries(value)
|
|
94
85
|
.filter(([key]) => (!citation || !['quote', 'start', 'end'].includes(key)) &&
|
|
95
86
|
(!result || !['revision', 'revisionOf'].includes(key)))
|
|
@@ -97,6 +88,24 @@ export class ModelIdentifiers {
|
|
|
97
88
|
}
|
|
98
89
|
if (typeof value !== 'string' || !MODEL_IDENTIFIER_FIELDS.has(field))
|
|
99
90
|
return value;
|
|
91
|
+
return this.translateIdentifier(value, outward);
|
|
92
|
+
}
|
|
93
|
+
isCitation(value) {
|
|
94
|
+
return ('ref' in value &&
|
|
95
|
+
typeof value.ref === 'string' &&
|
|
96
|
+
Object.hasOwn(this.evidence, value.ref) &&
|
|
97
|
+
'quote' in value &&
|
|
98
|
+
typeof value.quote === 'string');
|
|
99
|
+
}
|
|
100
|
+
isResult(value) {
|
|
101
|
+
return ('assessmentId' in value &&
|
|
102
|
+
typeof value.assessmentId === 'string' &&
|
|
103
|
+
this.handles.has(value.assessmentId) &&
|
|
104
|
+
'disposition' in value &&
|
|
105
|
+
'adherence' in value);
|
|
106
|
+
}
|
|
107
|
+
/** Inward, a handle maps back to its canonical ID; outward, an ID becomes its model handle. */
|
|
108
|
+
translateIdentifier(value, outward) {
|
|
100
109
|
if (!outward)
|
|
101
110
|
return this.canonical.get(value) ?? value;
|
|
102
111
|
if (Object.hasOwn(this.evidence, value))
|
package/dist/feedback.js
CHANGED
|
@@ -23,6 +23,31 @@ function externalEvidenceCounts(state, assessmentId) {
|
|
|
23
23
|
}
|
|
24
24
|
return Object.values(counts).some((count) => count > 0) ? { externalEvidence: counts } : {};
|
|
25
25
|
}
|
|
26
|
+
/** Local stop reasons in the published report vocabulary; anything unlisted reports as an execution error. */
|
|
27
|
+
const REPORTED_STOP_REASONS = {
|
|
28
|
+
finished: 'finished',
|
|
29
|
+
preparation_incomplete: 'preparation_incomplete',
|
|
30
|
+
cancelled: 'cancelled',
|
|
31
|
+
max_time_ms: 'timeout',
|
|
32
|
+
max_turns: 'max_turns',
|
|
33
|
+
model_call_limit: 'max_turns',
|
|
34
|
+
max_total_tokens: 'max_total_tokens',
|
|
35
|
+
usage_unavailable: 'usage_unavailable',
|
|
36
|
+
provider_error: 'provider_error',
|
|
37
|
+
execution_error: 'execution_error',
|
|
38
|
+
report_delivery_failed: 'report_delivery_failed',
|
|
39
|
+
per_call_output_limit: 'output_limit',
|
|
40
|
+
model_stopped: 'model_stopped',
|
|
41
|
+
total_token_limit_unmeasurable: 'usage_unavailable',
|
|
42
|
+
};
|
|
43
|
+
/** Repository identity is reported only with explicit attribution. */
|
|
44
|
+
function reportedRepository(repository, attribution) {
|
|
45
|
+
if (!attribution)
|
|
46
|
+
return { status: 'unavailable', reason: 'withheld' };
|
|
47
|
+
return repository.status === 'reported'
|
|
48
|
+
? { status: 'reported', provider: repository.provider, owner: repository.owner, name: repository.name }
|
|
49
|
+
: { status: 'unavailable', reason: repository.reason };
|
|
50
|
+
}
|
|
26
51
|
/** Construct from the allowlist. Never serialize local state then remove fields. */
|
|
27
52
|
export function projectAssessmentReport(state, repositoryAttribution = false) {
|
|
28
53
|
if (!state.reviewer || !state.decision || state.assessmentStatus === 'running')
|
|
@@ -31,23 +56,6 @@ export function projectAssessmentReport(state, repositoryAttribution = false) {
|
|
|
31
56
|
const phaseProviders = Object.values(e.phaseModels ?? { shared: e.requested });
|
|
32
57
|
const providers = new Set(phaseProviders.map((profile) => profile.provider));
|
|
33
58
|
const models = new Set(phaseProviders.map((profile) => profile.model));
|
|
34
|
-
const stop = {
|
|
35
|
-
finished: 'finished',
|
|
36
|
-
preparation_incomplete: 'preparation_incomplete',
|
|
37
|
-
cancelled: 'cancelled',
|
|
38
|
-
max_time_ms: 'timeout',
|
|
39
|
-
max_turns: 'max_turns',
|
|
40
|
-
model_call_limit: 'max_turns',
|
|
41
|
-
max_total_tokens: 'max_total_tokens',
|
|
42
|
-
usage_unavailable: 'usage_unavailable',
|
|
43
|
-
provider_error: 'provider_error',
|
|
44
|
-
execution_error: 'execution_error',
|
|
45
|
-
report_delivery_failed: 'report_delivery_failed',
|
|
46
|
-
per_call_output_limit: 'output_limit',
|
|
47
|
-
model_stopped: 'model_stopped',
|
|
48
|
-
total_token_limit_unmeasurable: 'usage_unavailable',
|
|
49
|
-
};
|
|
50
|
-
const repository = state.artifact.repositoryEnvironment;
|
|
51
59
|
return reviewReportInputSchema.parse({
|
|
52
60
|
kind: 'review_report',
|
|
53
61
|
contractVersion: 1,
|
|
@@ -56,11 +64,7 @@ export function projectAssessmentReport(state, repositoryAttribution = false) {
|
|
|
56
64
|
packageVersion,
|
|
57
65
|
methodVersion: state.concernValidation ? 'candidate-adherence-v3' : 'artifact-review-v1',
|
|
58
66
|
reviewer: { nodeId: state.reviewer.nodeId, version: state.reviewer.version },
|
|
59
|
-
repository: repositoryAttribution
|
|
60
|
-
? repository.status === 'reported'
|
|
61
|
-
? { status: 'reported', provider: repository.provider, owner: repository.owner, name: repository.name }
|
|
62
|
-
: { status: 'unavailable', reason: repository.reason }
|
|
63
|
-
: { status: 'unavailable', reason: 'withheld' },
|
|
67
|
+
repository: reportedRepository(state.artifact.repositoryEnvironment, repositoryAttribution),
|
|
64
68
|
scope: {
|
|
65
69
|
enumeration: state.preparationStatus === 'prepared' ? 'complete' : 'incomplete',
|
|
66
70
|
expectedCount: state.expectedContextCount,
|
|
@@ -87,7 +91,7 @@ export function projectAssessmentReport(state, repositoryAttribution = false) {
|
|
|
87
91
|
},
|
|
88
92
|
completion: state.assessmentStatus,
|
|
89
93
|
...(state.concernValidation ? { concernValidation: state.concernValidation } : {}),
|
|
90
|
-
stopReason:
|
|
94
|
+
stopReason: REPORTED_STOP_REASONS[e.stopReason ?? ''] ?? 'execution_error',
|
|
91
95
|
telemetry: {
|
|
92
96
|
durationMs: e.elapsedMs,
|
|
93
97
|
provider: providers.size === 1 ? phaseProviders[0].provider : null,
|