@contextrail/assess 1.0.0-alpha.3 → 1.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/assess.d.ts +2 -0
- package/dist/assess.js +177 -136
- package/dist/assessment.d.ts +1 -14
- package/dist/assessment.js +0 -46
- package/dist/budget.d.ts +3 -0
- package/dist/budget.js +2 -0
- package/dist/candidate.js +135 -104
- package/dist/change-summary.d.ts +2 -1
- package/dist/change-summary.js +1 -1
- package/dist/cli.js +13 -1
- package/dist/contracts.d.ts +2 -8
- package/dist/contracts.js +2 -2
- package/dist/diagnostics.js +1 -0
- package/dist/evidence.d.ts +4 -0
- package/dist/evidence.js +21 -12
- package/dist/feedback.js +27 -23
- package/dist/git.d.ts +6 -0
- package/dist/git.js +30 -3
- package/dist/github.js +78 -52
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp.d.ts +2 -0
- package/dist/mcp.js +67 -56
- package/dist/pack.js +8 -1
- package/dist/pr-context.js +26 -16
- package/dist/presentation.js +113 -71
- package/dist/provider-diagnostics.js +17 -12
- package/dist/providers.js +24 -20
- package/dist/report-contract.js +63 -57
- package/dist/results.js +5 -1
- package/dist/source.js +20 -11
- package/dist/speed.d.ts +13 -0
- package/dist/speed.js +72 -0
- package/docs/action-reference.md +2 -0
- package/docs/configuration.md +12 -1
- package/docs/github-actions.md +1 -0
- package/examples/assess-pull-request.yml +1 -1
- package/package.json +1 -1
package/dist/presentation.js
CHANGED
|
@@ -434,51 +434,63 @@ function renderChangeCounts(counts) {
|
|
|
434
434
|
}
|
|
435
435
|
/** Human orientation is separate from the governed result and never controls enforcement. */
|
|
436
436
|
export function renderReaderGuidance(input) {
|
|
437
|
-
const state = object(input)
|
|
438
|
-
const lines = ['#### Changes and reviewer focus'];
|
|
439
|
-
if (summary.status === 'verified' && typeof accepted.overview === 'string') {
|
|
440
|
-
lines.push(markdownText(accepted.overview, 2000));
|
|
441
|
-
const focus = Array.isArray(accepted.reviewFocus)
|
|
442
|
-
? accepted.reviewFocus.filter((item) => typeof item === 'string')
|
|
443
|
-
: [];
|
|
444
|
-
if (focus.length)
|
|
445
|
-
lines.push('Suggested review focus:\n\n' +
|
|
446
|
-
focus
|
|
447
|
-
.slice(0, 5)
|
|
448
|
-
.map((item) => `- ${markdownText(item, 1000)}`)
|
|
449
|
-
.join('\n'));
|
|
450
|
-
const locations = objects(accepted.sourceRefs)
|
|
451
|
-
.map((citation) => `${string(object(citation.provenance).path)}:${object(citation.location).line ?? '?'}`)
|
|
452
|
-
.filter((location) => !location.startsWith(':'))
|
|
453
|
-
.join(', ');
|
|
454
|
-
lines.push('Source-grounded summary, independently checked; separate from adherence.' +
|
|
455
|
-
(locations ? ` Sources: ${markdownText(locations, 1000)}.` : ''));
|
|
456
|
-
}
|
|
457
|
-
else {
|
|
458
|
-
lines.push('No independently checked narrative summary is available. The prepared file inventory below still describes the comparison.');
|
|
459
|
-
}
|
|
437
|
+
const state = object(input);
|
|
438
|
+
const lines = ['#### Changes and reviewer focus', ...changeSummaryLines(object(state.changeSummary))];
|
|
460
439
|
const followUps = objects(state.followUps).filter((item) => item.kind !== 'external_evidence');
|
|
461
|
-
if (followUps.length)
|
|
462
|
-
lines.push(
|
|
463
|
-
for (const item of followUps) {
|
|
464
|
-
const kind = item.kind === 'operational_check' ? 'Operational check' : 'Human check';
|
|
465
|
-
const status = item.verified === true
|
|
466
|
-
? item.required === true
|
|
467
|
-
? 'mandatory — pending'
|
|
468
|
-
: 'suggested'
|
|
469
|
-
: 'unverified proposal';
|
|
470
|
-
const criterion = findingCriteria([{ assessmentIds: [item.assessmentId] }], objects(state.entries));
|
|
471
|
-
const locations = objects(item.sourceRefs)
|
|
472
|
-
.map((citation) => `${string(object(citation.provenance).path)}:${object(citation.location).line ?? '?'}`)
|
|
473
|
-
.join(', ');
|
|
474
|
-
lines.push(`- **${kind}: ${status}.** ${markdownText(item.explanation, 1200)} **Next:** ${markdownText(item.action, 1200)} Criterion: ${criterion}. Source: ${markdownText(locations, 500)}.`);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
440
|
+
if (followUps.length)
|
|
441
|
+
lines.push(...humanFollowUpLines(followUps, objects(state.entries)));
|
|
477
442
|
const external = renderExternalFollowUps(state, false);
|
|
478
443
|
if (external)
|
|
479
444
|
lines.push(external);
|
|
480
445
|
return lines.join('\n\n');
|
|
481
446
|
}
|
|
447
|
+
/** `path:line` for each citation; `?` marks a citation without a line. */
|
|
448
|
+
function citationLocations(citations) {
|
|
449
|
+
return objects(citations).map((citation) => `${string(object(citation.provenance).path)}:${object(citation.location).line ?? '?'}`);
|
|
450
|
+
}
|
|
451
|
+
/** Only an independently verified summary is shown, with its review focus and source locations. */
|
|
452
|
+
function changeSummaryLines(summary) {
|
|
453
|
+
const accepted = object(summary.accepted);
|
|
454
|
+
if (summary.status !== 'verified' || typeof accepted.overview !== 'string')
|
|
455
|
+
return [
|
|
456
|
+
'No independently checked narrative summary is available. The prepared file inventory below still describes the comparison.',
|
|
457
|
+
];
|
|
458
|
+
const lines = [markdownText(accepted.overview, 2000)];
|
|
459
|
+
const focus = Array.isArray(accepted.reviewFocus)
|
|
460
|
+
? accepted.reviewFocus.filter((item) => typeof item === 'string')
|
|
461
|
+
: [];
|
|
462
|
+
if (focus.length)
|
|
463
|
+
lines.push('Suggested review focus:\n\n' +
|
|
464
|
+
focus
|
|
465
|
+
.slice(0, 5)
|
|
466
|
+
.map((item) => `- ${markdownText(item, 1000)}`)
|
|
467
|
+
.join('\n'));
|
|
468
|
+
const locations = citationLocations(accepted.sourceRefs)
|
|
469
|
+
.filter((location) => !location.startsWith(':'))
|
|
470
|
+
.join(', ');
|
|
471
|
+
lines.push('Source-grounded summary, independently checked; separate from adherence.' +
|
|
472
|
+
(locations ? ` Sources: ${markdownText(locations, 1000)}.` : ''));
|
|
473
|
+
return lines;
|
|
474
|
+
}
|
|
475
|
+
/** Human and operational checks stay separate from adherence; an outcome does not complete them. */
|
|
476
|
+
function humanFollowUpLines(followUps, entries) {
|
|
477
|
+
const lines = [
|
|
478
|
+
'#### Human and operational follow-ups',
|
|
479
|
+
'These remain separate from artifact adherence. An assessment outcome does not establish that operational duties were completed.',
|
|
480
|
+
];
|
|
481
|
+
for (const item of followUps) {
|
|
482
|
+
const kind = item.kind === 'operational_check' ? 'Operational check' : 'Human check';
|
|
483
|
+
const status = item.verified === true
|
|
484
|
+
? item.required === true
|
|
485
|
+
? 'mandatory — pending'
|
|
486
|
+
: 'suggested'
|
|
487
|
+
: 'unverified proposal';
|
|
488
|
+
const criterion = findingCriteria([{ assessmentIds: [item.assessmentId] }], entries);
|
|
489
|
+
const locations = citationLocations(item.sourceRefs).join(', ');
|
|
490
|
+
lines.push(`- **${kind}: ${status}.** ${markdownText(item.explanation, 1200)} **Next:** ${markdownText(item.action, 1200)} Criterion: ${criterion}. Source: ${markdownText(locations, 500)}.`);
|
|
491
|
+
}
|
|
492
|
+
return lines;
|
|
493
|
+
}
|
|
482
494
|
/** Group a shared external evidence boundary without merging the underlying assessment records. */
|
|
483
495
|
function renderExternalFollowUps(state, compact) {
|
|
484
496
|
const groups = new Map();
|
|
@@ -637,7 +649,23 @@ function findingCriteria(accounts, entries) {
|
|
|
637
649
|
})
|
|
638
650
|
.join(', ');
|
|
639
651
|
}
|
|
652
|
+
function collapseIdenticalAccounts(accounts) {
|
|
653
|
+
// A shared location alone does not establish duplicate meaning. Collapse only identical accounts.
|
|
654
|
+
const first = accounts[0];
|
|
655
|
+
const sameMeaning = accounts.every((account) => ['classification', 'explanation', 'consequence', 'correction'].every((key) => account[key] === first[key]));
|
|
656
|
+
if (sameMeaning && accounts.length > 1 && developerFindingGroupLabel(accounts) === 'the same source range')
|
|
657
|
+
accounts = [
|
|
658
|
+
{
|
|
659
|
+
...first,
|
|
660
|
+
assessmentIds: [
|
|
661
|
+
...new Set(accounts.flatMap((account) => Array.isArray(account.assessmentIds) ? account.assessmentIds : [])),
|
|
662
|
+
],
|
|
663
|
+
},
|
|
664
|
+
];
|
|
665
|
+
return accounts;
|
|
666
|
+
}
|
|
640
667
|
function findingGroup(accounts, entries) {
|
|
668
|
+
accounts = collapseIdenticalAccounts(accounts);
|
|
641
669
|
const finding = accounts[0];
|
|
642
670
|
const criteria = findingCriteria(accounts, entries);
|
|
643
671
|
const locations = objects(finding.locationRefs)
|
|
@@ -707,27 +735,17 @@ export function renderAssessmentReport(input, options = {}) {
|
|
|
707
735
|
lines.push(renderRunDetails(execution, usage, options));
|
|
708
736
|
return lines.join('\n\n') + '\n';
|
|
709
737
|
}
|
|
710
|
-
|
|
711
|
-
export function renderAssessmentComment(input, options = {}) {
|
|
712
|
-
const state = object(input), entries = objects(state.entries), execution = object(state.execution);
|
|
713
|
-
const progress = assessmentCoverage(state);
|
|
714
|
-
const verified = objects(state.findings).filter((finding) => finding.verified === true);
|
|
715
|
-
const questions = entries.filter((entry) => object(entry.result).disposition === 'unresolved');
|
|
716
|
-
const count = (value) => entries.filter((entry) => object(entry.result).disposition === value).length;
|
|
717
|
-
const observed = observationCounts(state, entries);
|
|
718
|
-
const exit = options.reviewExit ?? object(state.decision).exitCode ?? object(state.decision).reviewExit;
|
|
719
|
-
const failed = Boolean(options.failedStage || exit === 2 || (execution.stopReason && execution.stopReason !== 'finished'));
|
|
738
|
+
function commentHeading(state, options, verified, questions, pending, failed) {
|
|
720
739
|
const violations = verified.filter((finding) => finding.classification === 'binding_violation').length;
|
|
721
740
|
const outcome = violations
|
|
722
741
|
? `${violations} confirmed violation${violations === 1 ? '' : 's'}`
|
|
723
742
|
: 'No confirmed violations';
|
|
724
|
-
const pending = progress.awaitingVerification + progress.needsReassessment + progress.unexamined;
|
|
725
743
|
const headline = failed
|
|
726
744
|
? 'Review could not finish'
|
|
727
745
|
: pending || (state.assessmentStatus !== 'complete' && !questions.length)
|
|
728
746
|
? 'Assessment incomplete'
|
|
729
747
|
: outcome;
|
|
730
|
-
const policy = string(object(execution.policy).enforcement);
|
|
748
|
+
const policy = string(object(object(state.execution).policy).enforcement);
|
|
731
749
|
const lines = [
|
|
732
750
|
assessLogoBaseUrl
|
|
733
751
|
? `### <picture><source media="(prefers-color-scheme: dark)" srcset="${assessLogoBaseUrl}/icon-dark.png"><img src="${assessLogoBaseUrl}/icon-light.png" height="22" alt="ContextRail"></picture> ContextRail Assess`
|
|
@@ -738,6 +756,47 @@ export function renderAssessmentComment(input, options = {}) {
|
|
|
738
756
|
lines.push(options.failedStage
|
|
739
757
|
? `Action ${markdownText(options.failedStage)} failed. Results below are partial; see run details.`
|
|
740
758
|
: 'Results below are partial. See the workflow run for the failure and retry details.');
|
|
759
|
+
return lines;
|
|
760
|
+
}
|
|
761
|
+
function commentQuestions(state, questions, options) {
|
|
762
|
+
if (questions.length) {
|
|
763
|
+
const lines = ['#### Needs your attention'];
|
|
764
|
+
for (const entry of questions) {
|
|
765
|
+
const result = object(entry.result), action = questionAction(state, entry);
|
|
766
|
+
lines.push(`- **${criterionTitle(entry)}:** ${briefText(result.explanation, 2)}${action ? ` **Next:** ${briefText(action, 1)}` : ''}${sourceLinks(result.citations, options)}`);
|
|
767
|
+
}
|
|
768
|
+
return lines;
|
|
769
|
+
}
|
|
770
|
+
return [];
|
|
771
|
+
}
|
|
772
|
+
function commentRunDetails(state, options, exit) {
|
|
773
|
+
const execution = object(state.execution);
|
|
774
|
+
const details = renderRunDetails(execution, summarizeUsage(state), options);
|
|
775
|
+
const identity = `Run ID: ${markdownText(state.runId)}. Reviewer: ${markdownText(object(state.reviewer).title ?? object(state.reviewer).nodeId)} @ ${markdownText(object(state.reviewer).version)}. Exit: ${markdownText(exit ?? 'unavailable')}.`;
|
|
776
|
+
const lines = [details.replace('\n\n</details>', `\n\n${identity}\n\n</details>`)];
|
|
777
|
+
const links = [
|
|
778
|
+
isArtifactUrl(options.artifactUrl) ? `[Full audit](${options.artifactUrl})` : '',
|
|
779
|
+
runUrl(options.runUrl) ? `[Workflow run](${options.runUrl})` : '',
|
|
780
|
+
object(object(state.artifact).head).commit
|
|
781
|
+
? `Reviewed ${markdownText(string(object(object(state.artifact).head).commit).slice(0, 7))}`
|
|
782
|
+
: '',
|
|
783
|
+
].filter(Boolean);
|
|
784
|
+
if (links.length)
|
|
785
|
+
lines.push(links.join(' · '));
|
|
786
|
+
return lines;
|
|
787
|
+
}
|
|
788
|
+
/** Compact developer view; the Action summary retains the complete audit explanation. */
|
|
789
|
+
export function renderAssessmentComment(input, options = {}) {
|
|
790
|
+
const state = object(input), entries = objects(state.entries), execution = object(state.execution);
|
|
791
|
+
const progress = assessmentCoverage(state);
|
|
792
|
+
const verified = objects(state.findings).filter((finding) => finding.verified === true);
|
|
793
|
+
const questions = entries.filter((entry) => object(entry.result).disposition === 'unresolved');
|
|
794
|
+
const count = (value) => entries.filter((entry) => object(entry.result).disposition === value).length;
|
|
795
|
+
const observed = observationCounts(state, entries);
|
|
796
|
+
const exit = options.reviewExit ?? object(state.decision).exitCode ?? object(state.decision).reviewExit;
|
|
797
|
+
const failed = Boolean(options.failedStage || exit === 2 || (execution.stopReason && execution.stopReason !== 'finished'));
|
|
798
|
+
const pending = progress.awaitingVerification + progress.needsReassessment + progress.unexamined;
|
|
799
|
+
const lines = commentHeading(state, options, verified, questions, pending, failed);
|
|
741
800
|
const summary = object(state.changeSummary), accepted = object(summary.accepted);
|
|
742
801
|
if (summary.status === 'verified' && typeof accepted.overview === 'string')
|
|
743
802
|
lines.push(markdownText([...new Intl.Segmenter('en', { granularity: 'sentence' }).segment(accepted.overview)]
|
|
@@ -746,16 +805,10 @@ export function renderAssessmentComment(input, options = {}) {
|
|
|
746
805
|
.join(''), 600));
|
|
747
806
|
if (verified.length) {
|
|
748
807
|
lines.push('#### Governed findings');
|
|
749
|
-
for (const finding of verified)
|
|
808
|
+
for (const finding of groupDeveloperFindings(verified).flatMap(collapseIdenticalAccounts))
|
|
750
809
|
lines.push(compactFinding(finding, entries, options));
|
|
751
810
|
}
|
|
752
|
-
|
|
753
|
-
lines.push('#### Needs your attention');
|
|
754
|
-
for (const entry of questions) {
|
|
755
|
-
const result = object(entry.result), action = questionAction(state, entry);
|
|
756
|
-
lines.push(`- **${criterionTitle(entry)}:** ${briefText(result.explanation, 2)}${action ? ` **Next:** ${briefText(action, 1)}` : ''}${sourceLinks(result.citations, options)}`);
|
|
757
|
-
}
|
|
758
|
-
}
|
|
811
|
+
lines.push(...commentQuestions(state, questions, options));
|
|
759
812
|
const externalFollowUps = renderExternalFollowUps(state, true);
|
|
760
813
|
if (externalFollowUps)
|
|
761
814
|
lines.push(externalFollowUps);
|
|
@@ -768,18 +821,7 @@ export function renderAssessmentComment(input, options = {}) {
|
|
|
768
821
|
'\n\n' +
|
|
769
822
|
compactEvidence(state) +
|
|
770
823
|
'\n\n</details>');
|
|
771
|
-
|
|
772
|
-
const identity = `Run ID: ${markdownText(state.runId)}. Reviewer: ${markdownText(object(state.reviewer).title ?? object(state.reviewer).nodeId)} @ ${markdownText(object(state.reviewer).version)}. Exit: ${markdownText(exit ?? 'unavailable')}.`;
|
|
773
|
-
lines.push(details.replace('\n\n</details>', `\n\n${identity}\n\n</details>`));
|
|
774
|
-
const links = [
|
|
775
|
-
isArtifactUrl(options.artifactUrl) ? `[Full audit](${options.artifactUrl})` : '',
|
|
776
|
-
runUrl(options.runUrl) ? `[Workflow run](${options.runUrl})` : '',
|
|
777
|
-
object(object(state.artifact).head).commit
|
|
778
|
-
? `Reviewed ${markdownText(string(object(object(state.artifact).head).commit).slice(0, 7))}`
|
|
779
|
-
: '',
|
|
780
|
-
].filter(Boolean);
|
|
781
|
-
if (links.length)
|
|
782
|
-
lines.push(links.join(' · '));
|
|
824
|
+
lines.push(...commentRunDetails(state, options, exit));
|
|
783
825
|
return lines.join('\n\n') + '\n';
|
|
784
826
|
}
|
|
785
827
|
function briefText(value, sentences) {
|
|
@@ -150,12 +150,7 @@ function nonnegativeInteger(value) {
|
|
|
150
150
|
}
|
|
151
151
|
/** Count the serialized wire request without retaining input, tool descriptions, schema properties or model IDs. */
|
|
152
152
|
export function providerRequestDiagnostics(body) {
|
|
153
|
-
const text
|
|
154
|
-
const requestBytes = typeof body === 'string'
|
|
155
|
-
? Buffer.byteLength(body, 'utf8')
|
|
156
|
-
: body instanceof Uint8Array
|
|
157
|
-
? body.byteLength
|
|
158
|
-
: null;
|
|
153
|
+
const { text, requestBytes } = decodedBody(body);
|
|
159
154
|
const parsed = parseBody(text, requestBytesLimit);
|
|
160
155
|
const value = parsed.value;
|
|
161
156
|
const messages = value?.messages ?? value?.input;
|
|
@@ -164,16 +159,26 @@ export function providerRequestDiagnostics(body) {
|
|
|
164
159
|
requestBytes,
|
|
165
160
|
messageCount: Array.isArray(messages) ? messages.length : null,
|
|
166
161
|
toolCount: Array.isArray(tools) ? tools.length : null,
|
|
167
|
-
toolSchemaBytes: Array.isArray(tools)
|
|
168
|
-
? Buffer.byteLength(JSON.stringify(tools.map((tool) => {
|
|
169
|
-
const entry = object(tool);
|
|
170
|
-
return object(entry?.function)?.parameters ?? entry?.parameters ?? null;
|
|
171
|
-
})), 'utf8')
|
|
172
|
-
: null,
|
|
162
|
+
toolSchemaBytes: Array.isArray(tools) ? toolSchemaBytes(tools) : null,
|
|
173
163
|
maxOutputTokens: nonnegativeInteger(value?.max_output_tokens ?? value?.max_completion_tokens ?? value?.max_tokens),
|
|
174
164
|
requestFormat: text === null ? 'unavailable' : parsed.format,
|
|
175
165
|
};
|
|
176
166
|
}
|
|
167
|
+
/** Only string and byte bodies are readable; a stream or form body reports neither text nor size. */
|
|
168
|
+
function decodedBody(body) {
|
|
169
|
+
if (typeof body === 'string')
|
|
170
|
+
return { text: body, requestBytes: Buffer.byteLength(body, 'utf8') };
|
|
171
|
+
if (body instanceof Uint8Array)
|
|
172
|
+
return { text: Buffer.from(body).toString('utf8'), requestBytes: body.byteLength };
|
|
173
|
+
return { text: null, requestBytes: null };
|
|
174
|
+
}
|
|
175
|
+
/** Size of the tool parameter schemas alone, in either the chat or the responses tool shape. */
|
|
176
|
+
function toolSchemaBytes(tools) {
|
|
177
|
+
return Buffer.byteLength(JSON.stringify(tools.map((tool) => {
|
|
178
|
+
const entry = object(tool);
|
|
179
|
+
return object(entry?.function)?.parameters ?? entry?.parameters ?? null;
|
|
180
|
+
})), 'utf8');
|
|
181
|
+
}
|
|
177
182
|
/** Transport diagnoses are observed exception codes, never inferred from arbitrary error prose. */
|
|
178
183
|
export function providerTransportDiagnostics(error) {
|
|
179
184
|
const known = {
|
package/dist/providers.js
CHANGED
|
@@ -152,26 +152,7 @@ export function recoverProviderCalls(model, signal, onRecovery) {
|
|
|
152
152
|
params.abortSignal?.throwIfAborted();
|
|
153
153
|
let response;
|
|
154
154
|
try {
|
|
155
|
-
|
|
156
|
-
const timer = setTimeout(() => timeout.abort(new ProviderRequestTimeout()), 180_000);
|
|
157
|
-
const signals = [signal, params.abortSignal, timeout.signal].filter((value) => value !== undefined);
|
|
158
|
-
try {
|
|
159
|
-
response = await underlying.doGenerate({
|
|
160
|
-
...params,
|
|
161
|
-
abortSignal: AbortSignal.any(signals),
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
catch (error) {
|
|
165
|
-
if (timeout.signal.aborted &&
|
|
166
|
-
!signal.aborted &&
|
|
167
|
-
!params.abortSignal?.aborted &&
|
|
168
|
-
!accountingFailure(error))
|
|
169
|
-
throw new ProviderRequestTimeout();
|
|
170
|
-
throw error;
|
|
171
|
-
}
|
|
172
|
-
finally {
|
|
173
|
-
clearTimeout(timer);
|
|
174
|
-
}
|
|
155
|
+
response = await generateOnce(underlying, params, signal);
|
|
175
156
|
}
|
|
176
157
|
catch (error) {
|
|
177
158
|
if (signal.aborted || params.abortSignal?.aborted || !recoverableProviderError(error))
|
|
@@ -194,6 +175,29 @@ export function recoverProviderCalls(model, signal, onRecovery) {
|
|
|
194
175
|
},
|
|
195
176
|
});
|
|
196
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* One provider request bounded at 180 seconds. Only a timeout this bound caused, not caller
|
|
180
|
+
* cancellation or an accounting failure, becomes a `ProviderRequestTimeout`.
|
|
181
|
+
*/
|
|
182
|
+
async function generateOnce(underlying, params, signal) {
|
|
183
|
+
const timeout = new AbortController();
|
|
184
|
+
const timer = setTimeout(() => timeout.abort(new ProviderRequestTimeout()), 180_000);
|
|
185
|
+
const signals = [signal, params.abortSignal, timeout.signal].filter((value) => value !== undefined);
|
|
186
|
+
try {
|
|
187
|
+
return await underlying.doGenerate({
|
|
188
|
+
...params,
|
|
189
|
+
abortSignal: AbortSignal.any(signals),
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
if (timeout.signal.aborted && !signal.aborted && !params.abortSignal?.aborted && !accountingFailure(error))
|
|
194
|
+
throw new ProviderRequestTimeout();
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
clearTimeout(timer);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
197
201
|
function accountingFailure(error) {
|
|
198
202
|
const seen = new Set();
|
|
199
203
|
let cause = error;
|
package/dist/report-contract.js
CHANGED
|
@@ -149,64 +149,11 @@ export const reviewReportInputSchema = z
|
|
|
149
149
|
}),
|
|
150
150
|
})
|
|
151
151
|
.superRefine((v, ctx) => {
|
|
152
|
-
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
v.concernValidation.complete !== (v.concernValidation.validated === v.concernValidation.expected)))
|
|
157
|
-
issue();
|
|
158
|
-
if (v.completion === 'complete' && v.concernValidation?.complete === false)
|
|
159
|
-
issue();
|
|
160
|
-
if (new Set(expected.map((e) => e.entryId)).size !== expected.length ||
|
|
161
|
-
new Set(expected.map((e) => `${e.context.nodeId}@${e.context.version}`)).size !== expected.length)
|
|
162
|
-
issue();
|
|
163
|
-
if (v.scope.expectedCount !== null && v.scope.expectedCount < expected.length)
|
|
164
|
-
issue();
|
|
165
|
-
if (v.scope.enumeration === 'complete' && (v.scope.expectedCount !== expected.length || !expected.length))
|
|
166
|
-
issue();
|
|
167
|
-
if (new Set(v.outcomes.map((e) => e.entryId)).size !== v.outcomes.length)
|
|
168
|
-
issue();
|
|
169
|
-
for (const o of v.outcomes) {
|
|
170
|
-
if (!expected.some((e) => e.entryId === o.entryId))
|
|
171
|
-
issue();
|
|
172
|
-
if (o.disposition === 'evaluated') {
|
|
173
|
-
if (o.adherence === null ||
|
|
174
|
-
(o.confidence === null && !v.concernValidation) ||
|
|
175
|
-
(o.finding === 'none') !== (o.adherence === 2))
|
|
176
|
-
issue();
|
|
177
|
-
}
|
|
178
|
-
else if (o.adherence !== null ||
|
|
179
|
-
(o.disposition !== 'unresolved' && o.finding !== 'none') ||
|
|
180
|
-
(o.disposition !== 'not_applicable' && o.confidence !== null))
|
|
181
|
-
issue();
|
|
182
|
-
if (v.origin === 'agent_skill' ? o.blocking !== null : typeof o.blocking !== 'boolean')
|
|
183
|
-
issue();
|
|
184
|
-
if (v.runner && o.blocking !== policyBlocksOutcome(v.runner.policy, o))
|
|
185
|
-
issue();
|
|
186
|
-
}
|
|
187
|
-
if (v.origin === 'npm' ? !v.runner || v.packageVersion === null : v.runner !== null || v.packageVersion !== null)
|
|
188
|
-
issue();
|
|
189
|
-
if (v.completion === 'complete' && (!isCompleteReviewCoverage(v) || v.stopReason !== 'finished'))
|
|
190
|
-
issue();
|
|
191
|
-
if (v.runner && 'enforcement' in v.runner.policy) {
|
|
192
|
-
if (!validEnforcementDecision(v))
|
|
193
|
-
issue();
|
|
152
|
+
// One issue per failed check, in a stable order.
|
|
153
|
+
for (const consistent of reviewMetadataChecks(v)) {
|
|
154
|
+
if (!consistent)
|
|
155
|
+
ctx.addIssue({ code: 'custom', message: 'Inconsistent structured review metadata.' });
|
|
194
156
|
}
|
|
195
|
-
else if (v.runner && 'advisory' in v.runner.policy) {
|
|
196
|
-
const policy = v.runner.policy;
|
|
197
|
-
const blocking = v.outcomes.some((o) => o.blocking);
|
|
198
|
-
const insufficient = v.outcomes.some((o) => policy.minConfidence !== null && (o.confidence === null || o.confidence < policy.minConfidence));
|
|
199
|
-
const reviewExit = blocking ? 1 : v.completion === 'incomplete' || insufficient ? 2 : 0;
|
|
200
|
-
if (v.completion === 'complete' && insufficient)
|
|
201
|
-
issue();
|
|
202
|
-
if (v.runner.reviewExit !== reviewExit ||
|
|
203
|
-
(v.runner.exitCode !== reviewExit && !(v.runner.exitCode === 2 && v.completion === 'incomplete')))
|
|
204
|
-
issue();
|
|
205
|
-
}
|
|
206
|
-
const usage = v.telemetry.usage;
|
|
207
|
-
if (new Set(usage.map((u) => u.call)).size !== usage.length ||
|
|
208
|
-
(v.telemetry.calls === null ? usage.length > 0 : usage.some((u) => u.call > v.telemetry.calls)))
|
|
209
|
-
issue();
|
|
210
157
|
})
|
|
211
158
|
.refine((v) => new TextEncoder().encode(JSON.stringify(v)).length <= 512 * 1024, 'Review metadata exceeds 512 KiB.');
|
|
212
159
|
/** Declared accounting only; never independent observation of the host review. */
|
|
@@ -246,3 +193,62 @@ function validEnforcementDecision(report) {
|
|
|
246
193
|
const reviewExit = blocking || (strictGap && !executionFailed) ? 1 : 0;
|
|
247
194
|
return runner.reviewExit === reviewExit && runner.exitCode === (executionFailed ? 2 : reviewExit);
|
|
248
195
|
}
|
|
196
|
+
/** Every cross-field rule of structured review metadata; `false` marks an inconsistency. */
|
|
197
|
+
function reviewMetadataChecks(v) {
|
|
198
|
+
const expected = v.scope.expected;
|
|
199
|
+
const validation = v.concernValidation;
|
|
200
|
+
return [
|
|
201
|
+
!validation ||
|
|
202
|
+
(validation.validated <= validation.expected &&
|
|
203
|
+
validation.complete === (validation.validated === validation.expected)),
|
|
204
|
+
!(v.completion === 'complete' && validation?.complete === false),
|
|
205
|
+
new Set(expected.map((e) => e.entryId)).size === expected.length &&
|
|
206
|
+
new Set(expected.map((e) => `${e.context.nodeId}@${e.context.version}`)).size === expected.length,
|
|
207
|
+
v.scope.expectedCount === null || v.scope.expectedCount >= expected.length,
|
|
208
|
+
v.scope.enumeration !== 'complete' || (v.scope.expectedCount === expected.length && expected.length > 0),
|
|
209
|
+
new Set(v.outcomes.map((e) => e.entryId)).size === v.outcomes.length,
|
|
210
|
+
...v.outcomes.flatMap((o) => outcomeChecks(v, o)),
|
|
211
|
+
v.origin === 'npm' ? Boolean(v.runner) && v.packageVersion !== null : !v.runner && v.packageVersion === null,
|
|
212
|
+
v.completion !== 'complete' || (isCompleteReviewCoverage(v) && v.stopReason === 'finished'),
|
|
213
|
+
...runnerDecisionChecks(v),
|
|
214
|
+
telemetryConsistent(v.telemetry),
|
|
215
|
+
];
|
|
216
|
+
}
|
|
217
|
+
function outcomeChecks(v, o) {
|
|
218
|
+
const dispositionConsistent = o.disposition === 'evaluated'
|
|
219
|
+
? o.adherence !== null &&
|
|
220
|
+
(o.confidence !== null || Boolean(v.concernValidation)) &&
|
|
221
|
+
(o.finding === 'none') === (o.adherence === 2)
|
|
222
|
+
: o.adherence === null &&
|
|
223
|
+
(o.disposition === 'unresolved' || o.finding === 'none') &&
|
|
224
|
+
(o.disposition === 'not_applicable' || o.confidence === null);
|
|
225
|
+
return [
|
|
226
|
+
v.scope.expected.some((e) => e.entryId === o.entryId),
|
|
227
|
+
dispositionConsistent,
|
|
228
|
+
v.origin === 'agent_skill' ? o.blocking === null : typeof o.blocking === 'boolean',
|
|
229
|
+
!v.runner || o.blocking === policyBlocksOutcome(v.runner.policy, o),
|
|
230
|
+
];
|
|
231
|
+
}
|
|
232
|
+
/** The runner's exit codes must follow from its policy and the outcomes. */
|
|
233
|
+
function runnerDecisionChecks(v) {
|
|
234
|
+
if (!v.runner)
|
|
235
|
+
return [];
|
|
236
|
+
if ('enforcement' in v.runner.policy)
|
|
237
|
+
return [validEnforcementDecision(v)];
|
|
238
|
+
if (!('advisory' in v.runner.policy))
|
|
239
|
+
return [];
|
|
240
|
+
const policy = v.runner.policy;
|
|
241
|
+
const blocking = v.outcomes.some((o) => o.blocking);
|
|
242
|
+
const insufficient = v.outcomes.some((o) => policy.minConfidence !== null && (o.confidence === null || o.confidence < policy.minConfidence));
|
|
243
|
+
const reviewExit = blocking ? 1 : v.completion === 'incomplete' || insufficient ? 2 : 0;
|
|
244
|
+
return [
|
|
245
|
+
!(v.completion === 'complete' && insufficient),
|
|
246
|
+
v.runner.reviewExit === reviewExit &&
|
|
247
|
+
(v.runner.exitCode === reviewExit || (v.runner.exitCode === 2 && v.completion === 'incomplete')),
|
|
248
|
+
];
|
|
249
|
+
}
|
|
250
|
+
function telemetryConsistent(telemetry) {
|
|
251
|
+
const { calls, usage } = telemetry;
|
|
252
|
+
return (new Set(usage.map((u) => u.call)).size === usage.length &&
|
|
253
|
+
(calls === null ? usage.length === 0 : usage.every((u) => u.call <= calls)));
|
|
254
|
+
}
|
package/dist/results.js
CHANGED
|
@@ -56,6 +56,11 @@ export function validateResult(input, entries, ledger, path = []) {
|
|
|
56
56
|
throw new Error('Pinned publication unavailable; this entry cannot be assessed.');
|
|
57
57
|
validateJudgment(value);
|
|
58
58
|
const citations = value.citations.map((citation, index) => ledger.cite(citation, [...path, 'citations', index]));
|
|
59
|
+
assertEvidenceRequirements(value, citations, entry);
|
|
60
|
+
return { ...value, citations, revision: (currentRevision ?? 0) + 1 };
|
|
61
|
+
}
|
|
62
|
+
/** Cite the entry's own publication; evaluated judgments also need source; findings need consequence and correction. */
|
|
63
|
+
function assertEvidenceRequirements(value, citations, entry) {
|
|
59
64
|
if (!citations.some((c) => c.provenance.kind === 'publication' &&
|
|
60
65
|
c.provenance.nodeId === entry.nodeId &&
|
|
61
66
|
c.provenance.version === entry.version))
|
|
@@ -65,7 +70,6 @@ export function validateResult(input, entries, ledger, path = []) {
|
|
|
65
70
|
throw new Error('Evaluated judgments require concrete pinned source evidence from read_source. Navigation/search alone is not assessment evidence.');
|
|
66
71
|
if (value.finding !== 'none' && (!value.consequence.trim() || !value.correction.trim()))
|
|
67
72
|
throw new Error('Findings need a concrete consequence and proportionate correction.');
|
|
68
|
-
return { ...value, citations, revision: (currentRevision ?? 0) + 1 };
|
|
69
73
|
}
|
|
70
74
|
function resultGate(result) {
|
|
71
75
|
if (result.disposition === 'not_evaluated' || result.disposition === 'unresolved')
|
package/dist/source.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
|
-
import { MAX_PREPARATION_BYTES, passiveGitEnvironment, sensitiveGitPath } from './git.js';
|
|
3
|
+
import { gitExecutable, MAX_PREPARATION_BYTES, passiveGitEnvironment, sensitiveGitPath } from './git.js';
|
|
4
4
|
const sensitiveSourcePath = sensitiveGitPath;
|
|
5
5
|
/** Provisional response/operation defaults, never total-source capacity limits. */
|
|
6
6
|
export const DEFAULT_SOURCE_LIMITS = Object.freeze({
|
|
@@ -30,6 +30,22 @@ function safePath(path) {
|
|
|
30
30
|
path.split('/').some((p) => !p || p === '.' || p === '..'))
|
|
31
31
|
throw new SourceAccessError('invalid_input', 'path must be an exact repository-relative Git path without dot segments.');
|
|
32
32
|
}
|
|
33
|
+
const CHANGE_KINDS = { A: 'addition', D: 'deletion', M: 'modification', R: 'rename', T: 'type_change' };
|
|
34
|
+
/**
|
|
35
|
+
* One `diff-tree --raw` change from its `:baseMode headMode baseObject headObject status` header and
|
|
36
|
+
* path; a rename also has its original path. A side whose mode is all zeros does not exist.
|
|
37
|
+
*/
|
|
38
|
+
function sourceChange(header, path, originalPath, offset, base, head) {
|
|
39
|
+
const [baseMode, headMode, baseObject, headObject, status] = header.slice(1).split(' ');
|
|
40
|
+
const side = (snapshot, mode, object, sidePath) => mode === '000000' ? null : { ...snapshot, mode, object, path: sidePath, kind: kind(mode) };
|
|
41
|
+
return {
|
|
42
|
+
offset,
|
|
43
|
+
kind: CHANGE_KINDS[status[0]],
|
|
44
|
+
base: side(base, baseMode, baseObject, originalPath ?? path),
|
|
45
|
+
head: side(head, headMode, headObject, path),
|
|
46
|
+
similarity: status.startsWith('R') ? Number(status.slice(1)) : null,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
33
49
|
function kind(mode) {
|
|
34
50
|
return mode === '160000' ? 'submodule' : mode === '120000' ? 'symlink' : 'file';
|
|
35
51
|
}
|
|
@@ -185,7 +201,7 @@ export class FixedSnapshotSource {
|
|
|
185
201
|
}
|
|
186
202
|
async *git(args, operation) {
|
|
187
203
|
operation.check();
|
|
188
|
-
const child = spawn(
|
|
204
|
+
const child = spawn(gitExecutable(), [
|
|
189
205
|
'--no-pager',
|
|
190
206
|
'--no-lazy-fetch',
|
|
191
207
|
'--no-replace-objects',
|
|
@@ -312,7 +328,7 @@ export class FixedSnapshotSource {
|
|
|
312
328
|
header = record;
|
|
313
329
|
continue;
|
|
314
330
|
}
|
|
315
|
-
const
|
|
331
|
+
const status = header.slice(1).split(' ')[4];
|
|
316
332
|
if (status.startsWith('R') && firstPath === undefined) {
|
|
317
333
|
firstPath = record;
|
|
318
334
|
continue;
|
|
@@ -327,14 +343,7 @@ export class FixedSnapshotSource {
|
|
|
327
343
|
items,
|
|
328
344
|
next: { offset: offset + items.length, detectRenames },
|
|
329
345
|
};
|
|
330
|
-
|
|
331
|
-
items.push({
|
|
332
|
-
offset: index - 1,
|
|
333
|
-
kind: { A: 'addition', D: 'deletion', M: 'modification', R: 'rename', T: 'type_change' }[status[0]],
|
|
334
|
-
base: source(base, baseMode, baseObject, firstPath ?? record),
|
|
335
|
-
head: source(head, headMode, headObject, record),
|
|
336
|
-
similarity: status.startsWith('R') ? Number(status.slice(1)) : null,
|
|
337
|
-
});
|
|
346
|
+
items.push(sourceChange(header, record, firstPath, index - 1, base, head));
|
|
338
347
|
}
|
|
339
348
|
header = undefined;
|
|
340
349
|
firstPath = undefined;
|
package/dist/speed.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export type ProviderSpeed = 'normal' | 'fast' | 'ultrafast';
|
|
2
|
+
export declare const PROVIDER_SPEEDS: readonly ProviderSpeed[];
|
|
3
|
+
export type SpeedUnavailable = {
|
|
4
|
+
model: string;
|
|
5
|
+
expected: string;
|
|
6
|
+
actual: string | null;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Requests a faster service tier on each provider call. The tier is best effort:
|
|
10
|
+
* a rejected tier is dropped for that model and the same request is resent once,
|
|
11
|
+
* and a request served on another tier is reported, never failed.
|
|
12
|
+
*/
|
|
13
|
+
export declare function speedFetch(fetcher: typeof fetch, speed: ProviderSpeed, unavailable: (event: SpeedUnavailable) => unknown): typeof fetch;
|
package/dist/speed.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export const PROVIDER_SPEEDS = ['normal', 'fast', 'ultrafast'];
|
|
2
|
+
/** OpenAI and OpenRouter both read `service_tier`; "priority" is the tier OpenAI calls Fast. */
|
|
3
|
+
const tiers = { fast: 'priority', ultrafast: 'ultrafast' };
|
|
4
|
+
const served = { fast: ['priority', 'fast'], ultrafast: ['ultrafast'] };
|
|
5
|
+
/**
|
|
6
|
+
* Requests a faster service tier on each provider call. The tier is best effort:
|
|
7
|
+
* a rejected tier is dropped for that model and the same request is resent once,
|
|
8
|
+
* and a request served on another tier is reported, never failed.
|
|
9
|
+
*/
|
|
10
|
+
export function speedFetch(fetcher, speed, unavailable) {
|
|
11
|
+
if (speed === 'normal')
|
|
12
|
+
return fetcher;
|
|
13
|
+
const expected = tiers[speed];
|
|
14
|
+
const dropped = new Set(), reported = new Set();
|
|
15
|
+
const report = (model, actual) => {
|
|
16
|
+
if (reported.has(model))
|
|
17
|
+
return;
|
|
18
|
+
reported.add(model);
|
|
19
|
+
unavailable({ model, expected, actual });
|
|
20
|
+
};
|
|
21
|
+
return async (input, init) => {
|
|
22
|
+
const body = requestBody(init?.body);
|
|
23
|
+
if (!body || dropped.has(body.model))
|
|
24
|
+
return fetcher(input, init);
|
|
25
|
+
const response = await fetcher(input, {
|
|
26
|
+
...init,
|
|
27
|
+
body: JSON.stringify({ ...body.value, service_tier: expected }),
|
|
28
|
+
});
|
|
29
|
+
if (await tierRejected(response)) {
|
|
30
|
+
dropped.add(body.model);
|
|
31
|
+
report(body.model, null);
|
|
32
|
+
await response.body?.cancel().catch(() => undefined);
|
|
33
|
+
return fetcher(input, init);
|
|
34
|
+
}
|
|
35
|
+
const actual = response.ok ? await servedTier(response) : null;
|
|
36
|
+
if (actual !== null && !served[speed].includes(actual))
|
|
37
|
+
report(body.model, actual);
|
|
38
|
+
return response;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function requestBody(body) {
|
|
42
|
+
if (typeof body !== 'string')
|
|
43
|
+
return null;
|
|
44
|
+
try {
|
|
45
|
+
const value = JSON.parse(body);
|
|
46
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
47
|
+
return null;
|
|
48
|
+
const model = value.model;
|
|
49
|
+
return { model: typeof model === 'string' ? model : '', value: value };
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
async function tierRejected(response) {
|
|
56
|
+
if (response.status < 400 || response.status >= 500 || [401, 408, 429].includes(response.status))
|
|
57
|
+
return false;
|
|
58
|
+
const text = await response
|
|
59
|
+
.clone()
|
|
60
|
+
.text()
|
|
61
|
+
.catch(() => '');
|
|
62
|
+
return /service[_ ]tier|ultrafast|priority/i.test(text);
|
|
63
|
+
}
|
|
64
|
+
async function servedTier(response) {
|
|
65
|
+
try {
|
|
66
|
+
const tier = (await response.clone().json()).service_tier;
|
|
67
|
+
return typeof tier === 'string' ? tier : null;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|