coding-artifact-evaluation-shell 0.0.0-stage → 0.1.0-eidolon.a1b37cbcd7c07
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +15 -0
- package/README.md +56 -2
- package/SKILL.md +30 -0
- package/SOP.md +157 -0
- package/adapter/benchmark.ts +28 -0
- package/app/app.ts +15 -0
- package/app/host.ts +30 -0
- package/budget.ts +49 -0
- package/bun.lock +1045 -0
- package/calibrate.ts +81 -0
- package/calibration-history.ts +33 -0
- package/calibration-recheck.ts +74 -0
- package/calibration-report.ts +36 -0
- package/campaign.ts +85 -0
- package/capsule.ts +162 -0
- package/cli.ts +23 -0
- package/coding-scope.ts +13 -0
- package/contract.ts +116 -0
- package/counter-review.ts +74 -0
- package/coverage.ts +74 -0
- package/environment.ts +11 -0
- package/feedback.ts +31 -0
- package/history.ts +11 -0
- package/host.xnl +7 -0
- package/index.ts +6 -0
- package/logic.ts +135 -0
- package/manifest.xnl +10 -0
- package/native-references.ts +21 -0
- package/needs.ts +33 -0
- package/obligation-graph.ts +47 -0
- package/package.json +32 -4
- package/proposal.ts +94 -0
- package/protocol-error.ts +1 -0
- package/replay.ts +39 -0
- package/representation.ts +24 -0
- package/resources/counter-review.md +16 -0
- package/resources/evaluate.md +34 -0
- package/resources/iterate.md +20 -0
- package/resources/review.md +23 -0
- package/resources/submit.md +18 -0
- package/support/agent-runtime.ts +520 -0
- package/support/artifact-inventory.ts +20 -0
- package/support/artifact-links.ts +82 -0
- package/support/browser-channel.ts +209 -0
- package/support/browser-client.ts +33 -0
- package/support/browser-owner.ts +69 -0
- package/support/browser-worker.ts +61 -0
- package/support/campaign-ledger.ts +66 -0
- package/support/code-knowledge.ts +73 -0
- package/support/counter-review.ts +35 -0
- package/support/ego-tools.ts +19 -0
- package/support/evidence-catalog.ts +37 -0
- package/support/evidence-input.ts +70 -0
- package/support/evidence-object.ts +9 -0
- package/support/evidence-read.ts +18 -0
- package/support/execution-profile.ts +65 -0
- package/support/guidance.ts +16 -0
- package/support/history.ts +89 -0
- package/support/http-request.ts +21 -0
- package/support/isolation.ts +94 -0
- package/support/judge.ts +60 -0
- package/support/model-turn.ts +94 -0
- package/support/observation-agent.ts +279 -0
- package/support/observation-protocol.ts +33 -0
- package/support/observers.ts +102 -0
- package/support/process.ts +22 -0
- package/support/proposal-check.ts +82 -0
- package/support/retention.ts +93 -0
- package/support/runtime-binding.ts +39 -0
- package/support/runtime-preflight.ts +16 -0
- package/support/service-evidence.ts +23 -0
- package/support/service-group.ts +45 -0
- package/support/service.ts +84 -0
- package/support/source-identity.ts +22 -0
- package/support/target-group-view.ts +11 -0
- package/target-groups.ts +57 -0
package/contract.ts
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/** Stable data/effect boundary. No product, vendor or IO dependencies. */
|
|
2
|
+
export const POLICY = 'artifact-evaluation/v1';
|
|
3
|
+
export type Judgment = 'pass' | 'fail' | 'indeterminate';
|
|
4
|
+
export type FailureClass = 'business' | 'infrastructure' | 'protocol-cost' | 'scope-unresolved';
|
|
5
|
+
export interface Source {
|
|
6
|
+
id: string; path: string; digest: string; content: string;
|
|
7
|
+
authority: 'original' | 'confirmed' | 'commitment' | 'inferred';
|
|
8
|
+
}
|
|
9
|
+
export interface Obligation {
|
|
10
|
+
id: string; sourceId: string; quote: string; expectation: string; rationale: string;
|
|
11
|
+
relation: 'leaf' | 'all' | 'any' | 'conditional'; children: string[];
|
|
12
|
+
condition: 'applicable' | 'not-applicable' | 'unresolved';
|
|
13
|
+
}
|
|
14
|
+
export interface Observation {
|
|
15
|
+
id: string; runId: string; artifactDigest: string; phase: 'observation';
|
|
16
|
+
channel: 'process' | 'browser' | 'http' | 'file'; method: string; conditions: string;
|
|
17
|
+
content: string; contentDigest: string; bytes: number; elapsedMs: number;
|
|
18
|
+
status: 'completed' | 'interrupted' | 'infrastructure-failed';
|
|
19
|
+
exitCode?: number; argv?: string[]; limitations: string[];
|
|
20
|
+
/** Owner-written provenance; absent on historical delivery observations. */
|
|
21
|
+
purpose?: 'delivery' | 'setup' | 'target';
|
|
22
|
+
environment?: {
|
|
23
|
+
process?: {network:'denied'|'loopback-only'; writes:'scratch-only'|'isolated-derived'; cwd:string;sourceRoot?:string;derivedDirectories?:string[]};
|
|
24
|
+
http?: {origin:string; method:string; path:string; serviceId?:string; authorization:'absent'|'present-redacted'; body?:unknown};
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export interface CoverageItem {id:string;sourceId:string;quote:string;expectation:string;mode:'inspection'|'execution'}
|
|
28
|
+
export interface CoverageReport {
|
|
29
|
+
items:CoverageItem[]; reviewedSourceIds:string[]; checkpointed:boolean;
|
|
30
|
+
links:{itemId:string;observationIds:string[]}[];
|
|
31
|
+
unavailable?:{itemId:string;reason:string}[];
|
|
32
|
+
gaps:{itemId:string;reason:string}[];
|
|
33
|
+
supplement:'not-needed'|'attempted'|'budget-unavailable'|'capability-unavailable'|'fenced'; semanticAuthority:false;
|
|
34
|
+
}
|
|
35
|
+
export interface Claim {
|
|
36
|
+
obligationId: string; judgment: Judgment; semanticExpectation: string;
|
|
37
|
+
method: string; conditions: string; reasoning: string; limitations: string[];
|
|
38
|
+
evidence: { observationId: string; excerpt: string; supports: string; complete?:true }[];
|
|
39
|
+
/** Required for business FAIL from restricted, nonzero process execution. */
|
|
40
|
+
preconditions?: {status:'established'|'unresolved'|'incompatible';reasoning:string;evidenceIds:string[]};
|
|
41
|
+
}
|
|
42
|
+
export interface Finding { id: string; obligationIds: string[]; description: string }
|
|
43
|
+
export interface Proposal {
|
|
44
|
+
judgment: Judgment; obligations: Obligation[]; claims: Claim[]; findings: Finding[];
|
|
45
|
+
sourceReviews: { sourceId: string; complete: boolean; omissions: string[]; ambiguities: string[] }[];
|
|
46
|
+
needReviews?: {needId:string;disposition:'addressed'|'unresolved'|'not-required';obligationIds:string[];reasoning:string}[];
|
|
47
|
+
}
|
|
48
|
+
export interface Diagnostic { rule: string; location: string; message: string; repairable: boolean }
|
|
49
|
+
export interface EvidenceRequest {id:string;sourceId:string;quote:string;expectation:string;missingEvidence:string;observationIds:string[]}
|
|
50
|
+
export interface EvidenceFeedback {format:'evidence-request-v1';requests:EvidenceRequest[]}
|
|
51
|
+
/** Owner handles survive repeated model-local request labels and fresh contexts. */
|
|
52
|
+
export interface EvidenceNeed extends EvidenceRequest {needId:string;round:number}
|
|
53
|
+
export interface Usage { input: number; cached: number; output: number }
|
|
54
|
+
/** Owner-selected window INSIDE the original evaluation clock, never a new budget. */
|
|
55
|
+
export interface ModelWindow {kind:'initial-collection'|'sufficiency-review'|'supplement'|'candidate-review';deadlineAt:number;finalReviewDeadlineAt:number;finalReviewReserveMs:number}
|
|
56
|
+
export interface Invocation {
|
|
57
|
+
id: string; phase: 'judgment' | 'preparation' | 'representation-repair' | 'counter-review';
|
|
58
|
+
elapsedMs: number; usage: Usage | null; completed: boolean; model: string; effort: string | null;
|
|
59
|
+
window?:ModelWindow; outcome?:'phase-cutoff';
|
|
60
|
+
}
|
|
61
|
+
export interface EvaluationInput {
|
|
62
|
+
runId: string; artifactDigest: string; sources: Source[]; observations: Observation[];
|
|
63
|
+
/** Caller-owned targets, never inferred from the artifact's completion statements. */
|
|
64
|
+
obligations?: Obligation[];
|
|
65
|
+
coverage?:CoverageReport;
|
|
66
|
+
needs?:EvidenceNeed[];
|
|
67
|
+
executionContext?: {artifactRoot:string;artifactWrites:'denied';processNetwork:'denied'|'loopback-only';processWrites:'scratch-only'|'isolated-derived';executionRoot?:string;derivedDirectories?:string[];origins:Record<string,string>};
|
|
68
|
+
}
|
|
69
|
+
export interface EvaluationConfig { policyVersion: string; includeCommitments: boolean }
|
|
70
|
+
export interface Decision {
|
|
71
|
+
schemaVersion: 1; policyVersion: string; runId: string; artifactDigest: string;
|
|
72
|
+
execution: 'completed' | 'interrupted' | 'infrastructure-failed'; judgment: Judgment;
|
|
73
|
+
failureClass: FailureClass | null; proposal: Proposal | null; observations: Observation[];
|
|
74
|
+
diagnostics: Diagnostic[]; invocations: Invocation[];
|
|
75
|
+
}
|
|
76
|
+
export interface JudgePort {
|
|
77
|
+
judge(input: EvaluationInput, repair?: { proposal: Proposal; diagnostics: Diagnostic[] }): Promise<Proposal>;
|
|
78
|
+
invocations(): Invocation[];
|
|
79
|
+
/** Independent sufficiency feedback while the native lease is still alive. */
|
|
80
|
+
review?(input:EvaluationInput,canSupplement:boolean,window?:ModelWindow):Promise<Proposal|EvidenceFeedback>;
|
|
81
|
+
}
|
|
82
|
+
export interface EvaluationRuntime { judge: JudgePort; record(event: unknown): void }
|
|
83
|
+
export interface ProcessProbe { argv: string[]; conditions: string; timeoutMs?: number;cwd?:string }
|
|
84
|
+
export interface BrowserProbe { op: string; args: unknown[]; conditions: string }
|
|
85
|
+
/** Trusted caller declares execution capabilities, never business answers. */
|
|
86
|
+
export interface ServiceSpec {
|
|
87
|
+
argv:string[]; portEnv:string; dataEnv:string[]; readyPath:string; readyTimeoutMs?:number;
|
|
88
|
+
cwd?:string; dataFiles?:Record<string,string>;
|
|
89
|
+
}
|
|
90
|
+
/** Native launch facts, not an application claim or semantic verdict. */
|
|
91
|
+
export interface ServiceLaunchReceipt {
|
|
92
|
+
leaseId:string;pid:number|undefined;origin:string;data:string;argv:string[];cwd:string;generation:number;
|
|
93
|
+
environment:Record<string,string>;state:{directory:string;initiallyEmpty:true};
|
|
94
|
+
readiness:{path:string;status:number};semanticPass:false;elapsedMs:number;
|
|
95
|
+
}
|
|
96
|
+
export interface OwnedServiceCommand {serviceId:string;argv:string[];cwd?:string}
|
|
97
|
+
export interface ServiceNode extends ServiceSpec {id:string;links?:Record<string,string>}
|
|
98
|
+
export interface ServiceGraph {entry:string;nodes:ServiceNode[]}
|
|
99
|
+
export interface RuntimeSpec {
|
|
100
|
+
/** Explicit tool aliases and readonly dependency roots. Never inferred by the judge. */
|
|
101
|
+
executables?:Record<string,string>; readPaths?:string[]; environment?:Record<string,string>;
|
|
102
|
+
}
|
|
103
|
+
export interface HttpProbe {method:'GET'|'POST'|'PUT'|'PATCH'|'DELETE';path:string;body?:unknown;headers?:Record<string,string>;conditions:string;serviceId?:string}
|
|
104
|
+
export interface FileProbe {path:string;conditions:string}
|
|
105
|
+
export interface Manifest {
|
|
106
|
+
artifact: string; targets: {id: string; path: string; authority: Source['authority']}[];
|
|
107
|
+
obligations?: Obligation[]; process?: ProcessProbe[];
|
|
108
|
+
browser?: {entry?: string; probes: BrowserProbe[]}; includeCommitments?: boolean;
|
|
109
|
+
service?:ServiceSpec; http?:HttpProbe[];
|
|
110
|
+
services?:ServiceGraph; runtime?:RuntimeSpec;
|
|
111
|
+
/** Native-owned copy; only derived outputs are writable, historical inputs stay protected. */
|
|
112
|
+
execution?:{workspace:'isolated-copy';network:'denied'|'loopback-only';derivedDirectories?:string[]};
|
|
113
|
+
files?:FileProbe[];
|
|
114
|
+
/** A bounded observer chooses actions from originals, never from historical findings. */
|
|
115
|
+
agentObservation?:{maxActions:number;allowServiceRestart?:boolean};
|
|
116
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type {Decision,EvaluationConfig,EvaluationInput,Invocation,Proposal} from './contract';
|
|
2
|
+
import {admit} from './logic';
|
|
3
|
+
import {canonicalIdentity} from './representation';
|
|
4
|
+
import {decodeProposal,JudgeProtocolError} from './proposal';
|
|
5
|
+
import {observationRef,resolveObservation} from './native-references';
|
|
6
|
+
import {isDeliveryFact} from './coverage';
|
|
7
|
+
|
|
8
|
+
/** Experimental second interpretation, never observation/requirement authority. */
|
|
9
|
+
export interface CounterReview {
|
|
10
|
+
disposition:'uphold'|'challenge'|'unresolved';
|
|
11
|
+
reason:string;
|
|
12
|
+
sourceIds:string[];
|
|
13
|
+
observationRefs:string[];
|
|
14
|
+
replacement?:Proposal;
|
|
15
|
+
}
|
|
16
|
+
export interface CounterReviewResult {
|
|
17
|
+
candidate:Decision;
|
|
18
|
+
review:CounterReview|null;
|
|
19
|
+
decision:Decision;
|
|
20
|
+
resolution:'not-required'|'upheld'|'challenged'|'replacement-failure'|'unavailable';
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function eligibleCandidate(input:EvaluationInput,candidate:Decision,config:EvaluationConfig):boolean {
|
|
24
|
+
if(candidate.runId!==input.runId||candidate.artifactDigest!==input.artifactDigest||candidate.policyVersion!==config.policyVersion||canonicalIdentity(candidate.observations)!==canonicalIdentity(input.observations))throw new JudgeProtocolError('Counter-review requires the exact candidate identity, policy and observations');
|
|
25
|
+
return candidate.execution==='completed'&&candidate.diagnostics.length===0&&candidate.judgment!=='indeterminate'&&!!candidate.proposal&&candidate.proposal.judgment===candidate.judgment&&admit(input,candidate.proposal,config).length===0;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Remove only full duplicate bodies; all semantic choices remain explicit. */
|
|
29
|
+
export function counterCandidateView(input:EvaluationInput,candidate:Decision,config:EvaluationConfig) {
|
|
30
|
+
if(!eligibleCandidate(input,candidate,config))throw new JudgeProtocolError('Only an admitted definitive candidate can be counter-reviewed');
|
|
31
|
+
const p=candidate.proposal!;
|
|
32
|
+
const reference=(id:string)=>observationRef(input.observations.findIndex(o=>o.id===id));
|
|
33
|
+
return {...p,obligations:p.obligations.map(({quote,...o})=>({...o,...(quote===input.sources.find(s=>s.id===o.sourceId)?.content?{wholeSource:true}:{quote})})),
|
|
34
|
+
claims:p.claims.map(c=>({...c,...(c.preconditions?{preconditions:{...c.preconditions,evidenceIds:c.preconditions.evidenceIds.map(reference)}}:{}),evidence:c.evidence.map(({observationId,excerpt,...e})=>{
|
|
35
|
+
const index=input.observations.findIndex(o=>o.id===observationId);
|
|
36
|
+
return {...e,observationRef:observationRef(index),...(excerpt===input.observations[index]!.content?{wholeObject:true}:{excerpt})};
|
|
37
|
+
})})),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function decodeCounterReview(input:EvaluationInput,candidate:Decision,value:unknown,config:EvaluationConfig):CounterReview {
|
|
42
|
+
if(!eligibleCandidate(input,candidate,config))throw new JudgeProtocolError('Counter-review candidate is not eligible');
|
|
43
|
+
if(!value||typeof value!=='object'||Array.isArray(value))throw new JudgeProtocolError('Counter-review must be an object');
|
|
44
|
+
const v=value as Record<string,unknown>;
|
|
45
|
+
const strings=(x:unknown):x is string[]=>Array.isArray(x)&&x.every(s=>typeof s==='string'&&!!s.trim())&&new Set(x).size===x.length;
|
|
46
|
+
if(Object.keys(v).some(k=>!['disposition','reason','sourceIds','observationRefs','replacement'].includes(k))||!['uphold','challenge','unresolved'].includes(v.disposition as string)||typeof v.reason!=='string'||!v.reason.trim()||!strings(v.sourceIds)||!v.sourceIds.length||!strings(v.observationRefs))throw new JudgeProtocolError('Counter-review needs disposition, reason, sourceIds and observationRefs');
|
|
47
|
+
const required=input.sources.filter(s=>s.authority==='original'||s.authority==='confirmed'||s.authority==='commitment'&&config.includeCommitments);
|
|
48
|
+
const sourceIds=v.sourceIds;
|
|
49
|
+
if(sourceIds.some(id=>!required.some(s=>s.id===id)))throw new JudgeProtocolError('Counter-review sources must have original policy authority');
|
|
50
|
+
if(v.observationRefs.some(ref=>!isDeliveryFact(resolveObservation(input,ref))))throw new JudgeProtocolError('Counter-review cannot use failed or setup-only facts');
|
|
51
|
+
if(v.disposition==='uphold'&&(!v.observationRefs.length||candidate.judgment==='pass'&&required.some(s=>!sourceIds.includes(s.id))))throw new JudgeProtocolError('Uphold needs native evidence; PASS review must include every authoritative source');
|
|
52
|
+
let replacement:Proposal|undefined;
|
|
53
|
+
if(v.replacement!==undefined){
|
|
54
|
+
if(v.disposition!=='challenge')throw new JudgeProtocolError('Only a challenge may supply a replacement failure');
|
|
55
|
+
replacement=decodeProposal(v.replacement,input);
|
|
56
|
+
if(replacement.judgment!=='fail')throw new JudgeProtocolError('Counter-review never flips a rejected FAIL into automatic PASS');
|
|
57
|
+
const diagnostics=admit(input,replacement,config);
|
|
58
|
+
if(diagnostics.length)throw new JudgeProtocolError('Replacement failure rejected: '+JSON.stringify(diagnostics));
|
|
59
|
+
}
|
|
60
|
+
return {disposition:v.disposition as CounterReview['disposition'],reason:v.reason,sourceIds:v.sourceIds,observationRefs:v.observationRefs,...(replacement?{replacement}:{})};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** No voting, semantic synthesis, opposite-verdict inference or historical write. */
|
|
64
|
+
export function resolveCounterReview(input:EvaluationInput,candidate:Decision,value:unknown,invocations:Invocation[],config:EvaluationConfig):CounterReviewResult {
|
|
65
|
+
const review=decodeCounterReview(input,candidate,value,config);
|
|
66
|
+
if(review.disposition==='uphold')return {candidate,review,decision:{...candidate,invocations},resolution:'upheld'};
|
|
67
|
+
if(review.replacement)return {candidate,review,decision:{...candidate,proposal:review.replacement,judgment:'fail',failureClass:'business',diagnostics:[],invocations},resolution:'replacement-failure'};
|
|
68
|
+
return {candidate,review,decision:{...candidate,proposal:null,judgment:'indeterminate',failureClass:'scope-unresolved',invocations,diagnostics:[{rule:'counter-review-unresolved',location:'$',message:review.reason,repairable:false}]},resolution:'challenged'};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function unavailableCounterReview(candidate:Decision,invocations:Invocation[],reason:string,kind:'budget'|'protocol'|'infrastructure'):CounterReviewResult {
|
|
72
|
+
const failureClasses={budget:'scope-unresolved',protocol:'protocol-cost',infrastructure:'infrastructure'} as const;
|
|
73
|
+
return {candidate,review:null,resolution:'unavailable',decision:{...candidate,proposal:null,judgment:'indeterminate',execution:kind==='infrastructure'?'infrastructure-failed':'completed',failureClass:failureClasses[kind],invocations,diagnostics:[{rule:'counter-review-'+kind,location:'$',message:reason,repairable:false}]}};
|
|
74
|
+
}
|
package/coverage.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type {CoverageItem,CoverageReport,EvaluationInput,Observation,Source} from './contract';
|
|
2
|
+
import {canonicalIdentity} from './representation';
|
|
3
|
+
|
|
4
|
+
const text=(value:unknown):value is string=>typeof value==='string'&&!!value.trim();
|
|
5
|
+
export const isDeliveryFact=(o:Observation)=>o.status==='completed'&&(!o.purpose||o.purpose==='delivery');
|
|
6
|
+
|
|
7
|
+
function requiredSources(input:EvaluationInput,includeCommitments:boolean):Source[] {
|
|
8
|
+
return input.sources.filter(source=>source.authority==='original'||source.authority==='confirmed'||includeCommitments&&source.authority==='commitment');
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** Whitelisted immutable inputs, never observations of the delivery. */
|
|
12
|
+
export function readTarget(input:EvaluationInput,sourceId:unknown):Source {
|
|
13
|
+
const source=input.sources.find(s=>s.id===sourceId);
|
|
14
|
+
if(!source)throw Error('Unknown target source ID; use the immutable input source list');
|
|
15
|
+
return structuredClone(source);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Reference coverage only; never a semantic sufficiency check. */
|
|
19
|
+
export function coveragePlan(input:EvaluationInput,value:unknown,includeCommitments=false):CoverageReport {
|
|
20
|
+
const plan=value as {items:CoverageItem[];reviewedSourceIds:string[]};
|
|
21
|
+
if(!plan||!Array.isArray(plan.items)||!plan.items.length||plan.items.length>200||!Array.isArray(plan.reviewedSourceIds)||!plan.reviewedSourceIds.every(text))throw Error('Coverage plan needs 1..200 source-bound items and reviewedSourceIds');
|
|
22
|
+
const allowed=requiredSources(input,includeCommitments);
|
|
23
|
+
if(plan.reviewedSourceIds.some(id=>!allowed.some(s=>s.id===id)))throw Error('Coverage reviewed source is not requirement authority');
|
|
24
|
+
const ids=new Set<string>();
|
|
25
|
+
for(const item of plan.items){
|
|
26
|
+
if(!item||!text(item.id)||ids.has(item.id)||!text(item.quote)||!text(item.expectation)||!['inspection','execution'].includes(item.mode))throw Error('Invalid or duplicate coverage item');
|
|
27
|
+
ids.add(item.id);
|
|
28
|
+
if(!allowed.some(s=>s.id===item.sourceId&&s.content.includes(item.quote)))throw Error(`Coverage item ${item.id} in source ${item.sourceId} must quote an exact authoritative source`);
|
|
29
|
+
}
|
|
30
|
+
for(const obligation of input.obligations??[]){
|
|
31
|
+
if(obligation.children.length||obligation.condition==='not-applicable')continue;
|
|
32
|
+
const unchanged=plan.items.some(item=>item.id===obligation.id&&item.sourceId===obligation.sourceId&&item.quote===obligation.quote&&item.expectation===obligation.expectation);
|
|
33
|
+
if(!unchanged)throw Error('Coverage plan omitted or changed a caller-owned leaf obligation');
|
|
34
|
+
}
|
|
35
|
+
return {items:structuredClone(plan.items),reviewedSourceIds:[...new Set(plan.reviewedSourceIds)],checkpointed:false,links:[],gaps:[],supplement:'not-needed',semanticAuthority:false};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function coverageCheckpoint(input:EvaluationInput,plan:CoverageReport,value:unknown,unavailable:unknown=[]):CoverageReport {
|
|
39
|
+
const links=value as CoverageReport['links'];
|
|
40
|
+
if(!Array.isArray(links))throw Error('Coverage checkpoint needs links array');
|
|
41
|
+
const seen=new Set<string>();
|
|
42
|
+
for(const link of links){
|
|
43
|
+
const item=plan.items.find(i=>i.id===link?.itemId);
|
|
44
|
+
if(!item||seen.has(link.itemId)||!Array.isArray(link.observationIds)||!link.observationIds.every(text))throw Error('Invalid coverage checkpoint item');
|
|
45
|
+
seen.add(link.itemId);
|
|
46
|
+
for(const id of link.observationIds){
|
|
47
|
+
const fact=input.observations.find(o=>o.id===id);
|
|
48
|
+
if(!fact||fact.runId!==input.runId||fact.artifactDigest!==input.artifactDigest||!isDeliveryFact(fact))throw Error('Coverage reference must bind a completed native delivery fact');
|
|
49
|
+
if(item.mode==='execution'&&fact.channel==='file')throw Error('Static file cannot close execution coverage');
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if(!Array.isArray(unavailable)||unavailable.some(item=>!item||!plan.items.some(i=>i.id===item.itemId)||!text(item.reason)))throw Error('Unavailable coverage needs known items and explicit capability limitations');
|
|
53
|
+
return {...plan,checkpointed:true,links:structuredClone(links),unavailable:structuredClone(unavailable)};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function coverageGaps(input:EvaluationInput,plan?:CoverageReport,includeCommitments=false):CoverageReport {
|
|
57
|
+
const result:CoverageReport=plan?structuredClone(plan):{items:[],reviewedSourceIds:[],checkpointed:false,links:[],gaps:[],supplement:'not-needed',semanticAuthority:false};
|
|
58
|
+
result.gaps=[];
|
|
59
|
+
if(!plan)result.gaps.push({itemId:'$plan',reason:'No source-bound observation plan was submitted'});
|
|
60
|
+
if(!result.checkpointed)result.gaps.push({itemId:'$checkpoint',reason:'No final native-reference coverage checkpoint'});
|
|
61
|
+
for(const source of requiredSources(input,includeCommitments)){
|
|
62
|
+
if(!result.reviewedSourceIds.includes(source.id))result.gaps.push({itemId:`$source:${source.id}`,reason:'Original target source not reviewed in the observation plan'});
|
|
63
|
+
}
|
|
64
|
+
for(const item of result.items){
|
|
65
|
+
if(result.links.find(l=>l.itemId===item.id)?.observationIds.length)continue;
|
|
66
|
+
const limitation=result.unavailable?.find(entry=>entry.itemId===item.id);
|
|
67
|
+
result.gaps.push({itemId:item.id,reason:limitation?.reason??`No native ${item.mode} reference; ${item.expectation}`});
|
|
68
|
+
}
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function samePlan(a:CoverageReport,b:CoverageReport):boolean {
|
|
73
|
+
return canonicalIdentity({items:a.items,sources:a.reviewedSourceIds})===canonicalIdentity({items:b.items,sources:b.reviewedSourceIds});
|
|
74
|
+
}
|
package/environment.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type {HttpProbe,Observation} from './contract';
|
|
2
|
+
|
|
3
|
+
/** Preserve request shape/ordinary values without persisting credentials. */
|
|
4
|
+
export function redactRequest(value:unknown):unknown {
|
|
5
|
+
if(Array.isArray(value))return value.map(redactRequest);
|
|
6
|
+
if(value&&typeof value==='object')return Object.fromEntries(Object.entries(value).map(([key,entry])=>[key,/password|secret|token|authorization|cookie|api.?key/i.test(key)?'[redacted]':redactRequest(entry)]));
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
export function httpEnvironment(origin:string,probe:HttpProbe):NonNullable<Observation['environment']> {
|
|
10
|
+
return {http:{origin,method:probe.method,path:probe.path,serviceId:probe.serviceId,authorization:Object.keys(probe.headers??{}).some(key=>key.toLowerCase()==='authorization')?'present-redacted':'absent',...(probe.body===undefined?{}:{body:redactRequest(probe.body)})}};
|
|
11
|
+
}
|
package/feedback.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type {EvaluationInput,EvidenceFeedback} from './contract';
|
|
2
|
+
import {JudgeProtocolError} from './proposal';
|
|
3
|
+
import {resolveObservation} from './native-references';
|
|
4
|
+
|
|
5
|
+
/** Source/reference authenticity, NOT a completeness or business oracle. */
|
|
6
|
+
export function decodeFeedback(value:unknown,input:EvaluationInput,includeCommitments=false):EvidenceFeedback {
|
|
7
|
+
let v=value as EvidenceFeedback;
|
|
8
|
+
if((value as any)?.format==='evidence-request-v2'){
|
|
9
|
+
const data=value as {format:string;requests:any[]};
|
|
10
|
+
if(Object.keys(data).some(k=>!['format','requests'].includes(k))||!Array.isArray(data.requests))throw new JudgeProtocolError('Malformed native evidence request');
|
|
11
|
+
v={format:'evidence-request-v1',requests:data.requests.map(r=>{
|
|
12
|
+
if(!r||Object.keys(r).some(k=>!['id','sourceId','quote','expectation','missingEvidence','observationRefs'].includes(k))||r.observationRefs!==undefined&&!Array.isArray(r.observationRefs))throw new JudgeProtocolError('Evidence request uses only source binding and optional observationRefs');
|
|
13
|
+
const source=input.sources.find(s=>s.id===r.sourceId);
|
|
14
|
+
return {id:r.id,sourceId:r.sourceId,quote:r.quote??source?.content,expectation:r.expectation,missingEvidence:r.missingEvidence,observationIds:(r.observationRefs??[]).map((ref:unknown)=>resolveObservation(input,ref).id)};
|
|
15
|
+
})};
|
|
16
|
+
}
|
|
17
|
+
if(!v||v.format!=='evidence-request-v1'||!Array.isArray(v.requests)||!v.requests.length||v.requests.length>24)throw new JudgeProtocolError('Evidence feedback requires 1..24 source-bound requests');
|
|
18
|
+
if(Object.keys(v).some(k=>!['format','requests'].includes(k)))throw new JudgeProtocolError('Evidence feedback is a request, never a verdict or capability grant');
|
|
19
|
+
const ids=new Set<string>();
|
|
20
|
+
for(const r of v.requests){
|
|
21
|
+
const s=input.sources.find(s=>s.id===r?.sourceId);
|
|
22
|
+
if(!r||[r.id,r.sourceId,r.quote,r.expectation,r.missingEvidence].some(v=>typeof v!=='string'||!v.trim())||ids.has(r.id))throw new JudgeProtocolError('Malformed or duplicate evidence request');
|
|
23
|
+
ids.add(r.id);
|
|
24
|
+
if(!s||s.authority==='inferred'||s.authority==='commitment'&&!includeCommitments||!s.content.includes(r.quote))throw new JudgeProtocolError('Evidence request must bind an exact authoritative source quote');
|
|
25
|
+
if(!Array.isArray(r.observationIds)||r.observationIds.some(id=>!input.observations.some(o=>o.id===id&&o.runId===input.runId&&o.artifactDigest===input.artifactDigest)))throw new JudgeProtocolError('Evidence request references foreign observations');
|
|
26
|
+
if(Object.keys(r).some(k=>!['id','sourceId','quote','expectation','missingEvidence','observationIds'].includes(k)))throw new JudgeProtocolError('Evidence request cannot contain executable steps or permission overrides');
|
|
27
|
+
}
|
|
28
|
+
return v;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const FEEDBACK_GUIDANCE=`While the native lease is alive, check facts against COMPLETE originals, not just coverage IDs. If a material missing fact can safely be obtained, save {format:"evidence-request-v2",requests:[{id:"need-1",sourceId:"original-id",expectation:"unchanged required behavior",missingEvidence:"specific absent observation and why needed",observationRefs:["obs:N"]}]}. quote is optional: omitting it natively binds the complete original; observationRefs is optional. Use current catalog refs, never digests/UUIDs. This is advisory need, not a verdict/script/permission. Request ALL currently material gaps together; do not demand exhaustive proof or re-exercise sufficiently supported behavior. A supported required defect already permits overall FAIL: submit its finding, preserving other limits rather than collecting until PASS. When canSupplement:false, or nothing material is missing, submit the final Proposal. The owner checks the same immutable bytes.`;
|
package/history.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import {inventoryHistory} from './support/history';
|
|
5
|
+
const [command,list,output]=process.argv.slice(2);
|
|
6
|
+
if(command!=='inventory'||!list||!output||!path.isAbsolute(list)||!path.isAbsolute(output))throw Error('Usage: bun evaluation/history.ts inventory /absolute/roots.json /absolute/new-inventory.json');
|
|
7
|
+
const roots=JSON.parse(fs.readFileSync(list,'utf8'));
|
|
8
|
+
if(!Array.isArray(roots)||roots.some(r=>typeof r!=='string'||!path.isAbsolute(r)))throw Error('Provide an explicit absolute-path list');
|
|
9
|
+
const inventory=inventoryHistory(roots);
|
|
10
|
+
fs.writeFileSync(output,JSON.stringify(inventory,null,2)+'\n',{flag:'wx'});
|
|
11
|
+
console.log(JSON.stringify({samples:inventory.samples.length,excluded:inventory.excluded.length,duplicates:inventory.duplicates.length,output}));
|
package/host.xnl
ADDED
package/index.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Public evaluation surface; IO and product adapters remain outside pure rules. */
|
|
2
|
+
export * from './contract';
|
|
3
|
+
export { evaluate, admit, semanticIdentity, summarize, aggregateUsage, defaultConfig } from './logic';
|
|
4
|
+
export { newBudget } from './budget';
|
|
5
|
+
export {projectTargetGroups,type TargetGroup} from './target-groups';
|
|
6
|
+
export {resolveCounterReview,decodeCounterReview,type CounterReview,type CounterReviewResult} from './counter-review';
|
package/logic.ts
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { POLICY, type Claim, type Decision, type Diagnostic, type EvaluationConfig, type EvaluationInput, type EvaluationRuntime, type Invocation, type Obligation, type Proposal } from './contract';
|
|
2
|
+
import { canonicalIdentity, excerptDiagnostic, repairOnce } from './representation';
|
|
3
|
+
import { JudgeProtocolError } from './proposal';
|
|
4
|
+
import {isDeliveryFact} from './coverage';
|
|
5
|
+
import {admitNeeds} from './needs';
|
|
6
|
+
import {obligationGraph} from './obligation-graph';
|
|
7
|
+
|
|
8
|
+
const issue = (rule: string, location: string, message: string, repairable = false): Diagnostic => ({rule, location, message, repairable});
|
|
9
|
+
const text = (s: unknown): s is string => typeof s === 'string' && s.trim().length > 0;
|
|
10
|
+
const validJudgment = (s: unknown) => typeof s==='string'&&['pass','fail','indeterminate'].includes(s);
|
|
11
|
+
|
|
12
|
+
/** Authenticity/coverage admission, NOT a semantic oracle. Model reasoning remains fallible. */
|
|
13
|
+
export function admit(input: EvaluationInput, proposal: Proposal, config: EvaluationConfig): Diagnostic[] {
|
|
14
|
+
const diagnostics: Diagnostic[] = [];
|
|
15
|
+
if (!proposal || !validJudgment(proposal.judgment) || !Array.isArray(proposal.obligations) || !Array.isArray(proposal.claims) || !Array.isArray(proposal.findings) || !Array.isArray(proposal.sourceReviews)) {
|
|
16
|
+
return [issue('proposal-shape','$','Malformed judgment protocol')];
|
|
17
|
+
}
|
|
18
|
+
if(proposal.obligations.some(o=>!o||!Array.isArray(o.children)||!o.children.every(text))||proposal.claims.some(c=>!c||!Array.isArray(c.evidence)||c.evidence.some(e=>!e))||proposal.findings.some(f=>!f)||proposal.sourceReviews.some(r=>!r||typeof r.complete!=='boolean'))return [issue('proposal-shape','$','Malformed nested protocol data')];
|
|
19
|
+
const obligations = new Map<string,Obligation>();
|
|
20
|
+
for (const [i,o] of proposal.obligations.entries()) {
|
|
21
|
+
const at = `obligations[${i}]`;
|
|
22
|
+
if (!o || !text(o.id) || obligations.has(o.id)) {diagnostics.push(issue('obligation-id',at,'Invalid or duplicate obligation'));continue;}
|
|
23
|
+
obligations.set(o.id,o);
|
|
24
|
+
const source = input.sources.find(s=>s.id===o.sourceId);
|
|
25
|
+
if (!source || !text(o.quote) || !source.content.includes(o.quote)) diagnostics.push(issue('source-binding',at,'Quote must be an exact excerpt of a frozen source'));
|
|
26
|
+
if (source && (source.authority==='inferred' || (source.authority==='commitment' && !config.includeCommitments))) diagnostics.push(issue('source-authority',at,'This source cannot create a required obligation under the trusted policy'));
|
|
27
|
+
if (!text(o.expectation) || !text(o.rationale) || !['leaf','all','any','conditional'].includes(o.relation) || !Array.isArray(o.children) || !['applicable','not-applicable','unresolved'].includes(o.condition)) diagnostics.push(issue('obligation-shape',at,'Expectation, relation, children, condition and rationale required'));
|
|
28
|
+
else if (o.relation==='leaf' && o.children.length || ['all','any'].includes(o.relation) && !o.children.length) diagnostics.push(issue('obligation-relation',at,'All/any need children; leaves cannot have children'));
|
|
29
|
+
}
|
|
30
|
+
for (const o of proposal.obligations) {
|
|
31
|
+
if (!Array.isArray(o?.children)) continue;
|
|
32
|
+
const visit = (id: string, trail: string[]): void => {
|
|
33
|
+
if (trail.includes(id)) {diagnostics.push(issue('obligation-cycle',o.id,'Cyclic obligation graph'));return;}
|
|
34
|
+
const child = obligations.get(id);
|
|
35
|
+
if (!child) {diagnostics.push(issue('obligation-child',o.id,`Unknown child ${id}`));return;}
|
|
36
|
+
for (const c of child.children ?? []) visit(c,[...trail,id]);
|
|
37
|
+
};
|
|
38
|
+
for (const child of o.children) visit(child,[o.id]);
|
|
39
|
+
}
|
|
40
|
+
for (const frozen of input.obligations ?? []) {
|
|
41
|
+
if (canonicalIdentity(obligations.get(frozen.id))!==canonicalIdentity(frozen)) diagnostics.push(issue('target-omission',frozen.id,'Caller-owned obligation omitted or changed'));
|
|
42
|
+
}
|
|
43
|
+
for (const source of input.sources.filter(s=>s.authority==='original'||s.authority==='confirmed'||(s.authority==='commitment'&&config.includeCommitments))) {
|
|
44
|
+
const review = proposal.sourceReviews.filter(r=>r.sourceId===source.id);
|
|
45
|
+
if (review.length!==1 || !Array.isArray(review[0]?.omissions) || !Array.isArray(review[0]?.ambiguities)) diagnostics.push(issue('source-review',source.id,'Full-source review must record completeness, omissions and ambiguities'));
|
|
46
|
+
else if (proposal.judgment==='pass' && (!review[0].complete || review[0].omissions.length || review[0].ambiguities.length)) diagnostics.push(issue('target-unresolved',source.id,'Omissions or ambiguity forbid unconditional PASS'));
|
|
47
|
+
}
|
|
48
|
+
const claims = new Map<string,Claim>();
|
|
49
|
+
for (const [i,c] of proposal.claims.entries()) {
|
|
50
|
+
const at = `claims[${i}]`;
|
|
51
|
+
if (!c || !obligations.has(c.obligationId) || claims.has(c.obligationId) || !validJudgment(c.judgment) || !Array.isArray(c.evidence) || !Array.isArray(c.limitations)) {diagnostics.push(issue('claim-shape',at,'Unique claim must reference an obligation'));continue;}
|
|
52
|
+
claims.set(c.obligationId,c);
|
|
53
|
+
if (c.semanticExpectation !== obligations.get(c.obligationId)!.expectation || !text(c.method) || !text(c.conditions) || !text(c.reasoning)) diagnostics.push(issue('claim-support',at,'Freeze expectation; explain method, conditions and semantic support'));
|
|
54
|
+
if (c.judgment!=='indeterminate' && !c.evidence.length && !obligations.get(c.obligationId)!.children.length) diagnostics.push(issue('evidence-insufficient',at,'A definitive leaf judgment needs native evidence'));
|
|
55
|
+
if(c.preconditions&&(!['established','unresolved','incompatible'].includes(c.preconditions.status)||!text(c.preconditions.reasoning)||!Array.isArray(c.preconditions.evidenceIds)))diagnostics.push(issue('environment-preconditions',at,'Malformed execution prerequisite assessment'));
|
|
56
|
+
if(c.judgment!=='indeterminate'&&c.preconditions&&c.preconditions.status!=='established')diagnostics.push(issue('environment-unresolved',at,'Unresolved or incompatible execution prerequisites cannot support a definitive claim'));
|
|
57
|
+
const restrictedFailure=c.judgment==='fail'&&c.evidence.some(e=>input.observations.some(o=>o.id===e.observationId&&o.environment?.process&&o.exitCode!==undefined&&o.exitCode!==0));
|
|
58
|
+
if(restrictedFailure){
|
|
59
|
+
const p=c.preconditions;
|
|
60
|
+
const established=p?.status==='established'&&text(p.reasoning)&&Array.isArray(p.evidenceIds)&&p.evidenceIds.length>0&&p.evidenceIds.every(id=>c.evidence.some(e=>e.observationId===id)&&input.observations.some(o=>o.id===id&&o.runId===input.runId&&o.artifactDigest===input.artifactDigest&&isDeliveryFact(o)&&(!o.environment?.process||o.exitCode===0)));
|
|
61
|
+
if(!established)diagnostics.push(issue('environment-unresolved',at,'Restricted nonzero execution alone cannot establish business FAIL; prerequisite applicability needs independently cited native delivery evidence'));
|
|
62
|
+
}
|
|
63
|
+
for (const [j,e] of c.evidence.entries()) {
|
|
64
|
+
const location = `${at}.evidence[${j}]`;
|
|
65
|
+
const observation = input.observations.find(o=>o.id===e.observationId);
|
|
66
|
+
if (!observation || observation.runId!==input.runId || observation.artifactDigest!==input.artifactDigest || observation.phase!=='observation' || !isDeliveryFact(observation)) diagnostics.push(issue('observation-binding',location,'Unknown, foreign, failed, input/setup-only or wrong-phase observation',true));
|
|
67
|
+
else if(e.complete===true){if(e.excerpt!==observation.content)diagnostics.push(issue('native-excerpt',location,'Complete evidence must preserve every native byte',true));}
|
|
68
|
+
else {const diagnostic=excerptDiagnostic(observation.content,e.excerpt,location);if(diagnostic)diagnostics.push(diagnostic);}
|
|
69
|
+
if (!text(e.supports)) diagnostics.push(issue('evidence-reasoning',location,'Explain how this fact supports the unchanged claim'));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (proposal.judgment==='pass' && (!obligations.size || proposal.findings.length)) diagnostics.push(issue('pass-shape','$','PASS requires obligations and no findings'));
|
|
73
|
+
const graph=obligationGraph(proposal);
|
|
74
|
+
if(proposal.judgment==='pass')for(const o of graph.roots)if(!['pass','not-applicable'].includes(graph.compare(o.id).judgment))diagnostics.push(issue('leaf-coverage',o.id,'Unsupported/unresolved compound obligation cannot pass via parent coverage ID'));
|
|
75
|
+
if (proposal.judgment==='fail' && (!proposal.findings.length || !proposal.claims.some(c=>c.judgment==='fail'))) diagnostics.push(issue('finding-required','$','Business FAIL needs a supported failed claim and stable finding'));
|
|
76
|
+
const ids = new Set<string>();
|
|
77
|
+
for (const finding of proposal.findings) {
|
|
78
|
+
if (!text(finding.id) || ids.has(finding.id) || !text(finding.description) || !Array.isArray(finding.obligationIds) || !finding.obligationIds.length || finding.obligationIds.some(id=>claims.get(id)?.judgment!=='fail'||graph.compare(id).judgment!=='fail')) diagnostics.push(issue('finding-binding',`findings.${finding.id}`,'Stable finding must bind failed obligations supported by the applicable leaf relation'));
|
|
79
|
+
else if(finding.obligationIds.some(id=>!graph.withinFailedRoot(id)))diagnostics.push(issue('target-contradiction',`findings.${finding.id}`,'A failed optional branch cannot falsify a satisfied or unresolved required root'));
|
|
80
|
+
ids.add(finding.id);
|
|
81
|
+
}
|
|
82
|
+
return [...diagnostics,...admitNeeds(input,proposal)];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Evidence references/excerpts are representations. Business semantics are immutable. */
|
|
86
|
+
export function semanticIdentity(proposal: Proposal): string {
|
|
87
|
+
return canonicalIdentity({...proposal, claims: proposal.claims.map(c=>({...c,evidence:c.evidence.map(e=>({supports:e.supports}))}))});
|
|
88
|
+
}
|
|
89
|
+
export function decisionExitCode(decision:Pick<Decision,'execution'|'judgment'>,noModel=false):number {
|
|
90
|
+
if(decision.execution!=='completed')return 2;
|
|
91
|
+
if(decision.judgment==='fail')return 1;
|
|
92
|
+
if(decision.judgment==='indeterminate'&&!noModel)return 3;
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
export async function evaluate(runtime: EvaluationRuntime, input: EvaluationInput, config: EvaluationConfig): Promise<Decision> {
|
|
96
|
+
const failure = input.observations.find(o=>o.status!=='completed');
|
|
97
|
+
const base = {schemaVersion:1 as const,policyVersion:config.policyVersion,runId:input.runId,artifactDigest:input.artifactDigest,observations:input.observations};
|
|
98
|
+
if (failure) return {...base,execution:failure.status==='interrupted'?'interrupted':'infrastructure-failed',judgment:'indeterminate',failureClass:'infrastructure',proposal:null,diagnostics:[issue('execution-failed',failure.id,'Native observer did not complete')],invocations:runtime.judge.invocations()};
|
|
99
|
+
if(!input.observations.some(isDeliveryFact)){
|
|
100
|
+
runtime.record({phase:'judgment-skipped',reason:'no-delivery-evidence',coverage:input.coverage??null});
|
|
101
|
+
return {...base,execution:'completed',judgment:'indeterminate',failureClass:'scope-unresolved',proposal:null,diagnostics:[issue('no-delivery-evidence','$','No completed native delivery evidence. Target reads and lease setup do not demonstrate the artifact; no model judge was invoked.')],invocations:runtime.judge.invocations()};
|
|
102
|
+
}
|
|
103
|
+
let proposal: Proposal;
|
|
104
|
+
try {proposal=await runtime.judge.judge(input);} catch (error) {
|
|
105
|
+
const protocol=error instanceof JudgeProtocolError;
|
|
106
|
+
return {...base,execution:protocol?'completed':'infrastructure-failed',judgment:'indeterminate',failureClass:protocol?'protocol-cost':'infrastructure',proposal:null,diagnostics:[issue(protocol?'judge-protocol':'judge-unavailable','$',String(error))],invocations:runtime.judge.invocations()};
|
|
107
|
+
}
|
|
108
|
+
let diagnostics = admit(input,proposal,config);
|
|
109
|
+
if (diagnostics.length && diagnostics.every(d=>d.repairable)) {
|
|
110
|
+
runtime.record({phase:'representation-repair',diagnostics});
|
|
111
|
+
try {
|
|
112
|
+
proposal=await repairOnce({identity:semanticIdentity,repairable:()=>true,record:()=>{},admit:p=>{const ds=admit(input,p,config);if(ds.length)throw Object.assign(Error(JSON.stringify(ds)),{diagnostics:ds});},repair:p=>runtime.judge.judge(input,{proposal:p,diagnostics})},proposal);
|
|
113
|
+
diagnostics=admit(input,proposal,config);
|
|
114
|
+
} catch (error) {diagnostics=[issue(String(error).includes('changed business')?'repair-changed-semantics':'repair-unavailable','$',String(error))];}
|
|
115
|
+
}
|
|
116
|
+
const judgment = diagnostics.length ? 'indeterminate' : proposal.judgment;
|
|
117
|
+
let failureClass:Decision['failureClass']=null;
|
|
118
|
+
if(diagnostics.length)failureClass=diagnostics.every(d=>!['target-omission','source-authority','target-unresolved','leaf-coverage','target-contradiction','environment-unresolved','need-unresolved'].includes(d.rule))?'protocol-cost':'scope-unresolved';
|
|
119
|
+
else if(judgment==='fail')failureClass='business';
|
|
120
|
+
else if(judgment==='indeterminate')failureClass='scope-unresolved';
|
|
121
|
+
return {...base,execution:'completed',judgment,failureClass,proposal,diagnostics,invocations:runtime.judge.invocations()};
|
|
122
|
+
}
|
|
123
|
+
export function aggregateUsage(invocations: Invocation[]) {
|
|
124
|
+
const unique = [...new Map(invocations.map(i=>[i.id,i])).values()];
|
|
125
|
+
const valid = unique.filter(i=>i.usage && [i.usage.input,i.usage.cached,i.usage.output].every(n=>Number.isFinite(n)&&n>=0) && i.usage.cached<=i.usage.input);
|
|
126
|
+
return {known:valid.reduce((a,i)=>({input:a.input+i.usage!.input,cached:a.cached+i.usage!.cached,output:a.output+i.usage!.output}),{input:0,cached:0,output:0}),unknownInvocations:unique.length-valid.length,scope:'cached is a subset of input; includes failures; not a bill'};
|
|
127
|
+
}
|
|
128
|
+
export function summarize(decisions: Decision[]) {
|
|
129
|
+
const policies = [...new Set(decisions.map(d=>d.policyVersion))];
|
|
130
|
+
return {policies,groups:policies.map(policy=> {
|
|
131
|
+
const all=decisions.filter(d=>d.policyVersion===policy),eligible=all.filter(d=>d.execution==='completed'&&d.failureClass!=='protocol-cost'&&d.judgment!=='indeterminate');
|
|
132
|
+
return {policyVersion:policy,allSamples:all.length,businessEligible:eligible.length,passes:eligible.filter(d=>d.judgment==='pass').length,indeterminate:all.filter(d=>d.judgment==='indeterminate').length,usage:aggregateUsage(all.flatMap(d=>d.invocations))};
|
|
133
|
+
})};
|
|
134
|
+
}
|
|
135
|
+
export const defaultConfig: EvaluationConfig = {policyVersion:POLICY,includeCommitments:false};
|
package/manifest.xnl
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
<SkillApp #ArtifactEvaluation.App envelopeVersion="halfcode.resource-envelope/v1" specVersion=1 {
|
|
2
|
+
profile = "app-package"
|
|
3
|
+
packageRoot = "vfs://."
|
|
4
|
+
descriptor = "vfs://./app/app.ts"
|
|
5
|
+
} (
|
|
6
|
+
<Catalogs [
|
|
7
|
+
<DirectoryResourceCatalog #host { resourceKind = "HostBundle" root = "vfs://./" entry = "host.xnl" scope = "root" }>
|
|
8
|
+
<FileResourceCatalog #procedures { resourceKind = "SOP" root = "vfs://./resources/" }>
|
|
9
|
+
]>
|
|
10
|
+
)>
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type {EvaluationInput,Observation} from './contract';
|
|
2
|
+
import {JudgeProtocolError} from './protocol-error';
|
|
3
|
+
|
|
4
|
+
/** Explicit owner-assigned catalog addresses, never guesses from a digest/UUID. */
|
|
5
|
+
export const observationRef=(index:number)=>`obs:${index+1}`;
|
|
6
|
+
export function resolveObservation(input:EvaluationInput,ref:unknown):Observation {
|
|
7
|
+
if(typeof ref!=='string'||!/^obs:[1-9]\d*$/.test(ref))throw new JudgeProtocolError('Use an observationRef from the current catalog (obs:N), not a digest or ID');
|
|
8
|
+
const index=Number(ref.slice(4))-1,o=input.observations[index];
|
|
9
|
+
if(!Number.isSafeInteger(index)||!o||o.runId!==input.runId||o.artifactDigest!==input.artifactDigest)throw new JudgeProtocolError('Unknown or foreign observationRef');
|
|
10
|
+
return o;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Model selects a fact; owner supplies its exact bytes and provenance. */
|
|
14
|
+
export function referenceEvidence(input:EvaluationInput,value:unknown) {
|
|
15
|
+
if(!value||typeof value!=='object'||Array.isArray(value))throw new JudgeProtocolError('Evidence reference must be an object');
|
|
16
|
+
const e=value as Record<string,unknown>;
|
|
17
|
+
if(Object.keys(e).some(k=>!['observationRef','supports','excerpt'].includes(k))||typeof e.supports!=='string'||!e.supports.trim())throw new JudgeProtocolError('Evidence needs observationRef and a semantic supports explanation');
|
|
18
|
+
const o=resolveObservation(input,e.observationRef);
|
|
19
|
+
if(e.excerpt!==undefined&&typeof e.excerpt!=='string')throw new JudgeProtocolError('Optional excerpt must be literal native text');
|
|
20
|
+
return {observationId:o.id,excerpt:e.excerpt===undefined?o.content:e.excerpt,supports:e.supports,...(e.excerpt===undefined?{complete:true as const}:{})};
|
|
21
|
+
}
|
package/needs.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type {Diagnostic,EvaluationInput,EvidenceFeedback,EvidenceNeed,Proposal} from './contract';
|
|
2
|
+
import {obligationGraph} from './obligation-graph';
|
|
3
|
+
|
|
4
|
+
/** Append only. A request is advisory, not a new authority or an execution grant. */
|
|
5
|
+
export function retainNeeds(previous:EvidenceNeed[],feedback:EvidenceFeedback,round:number):EvidenceNeed[] {
|
|
6
|
+
return [...previous,...feedback.requests.map((r,index)=>({...r,observationIds:[...r.observationIds],round,needId:`need:${previous.length+index+1}`}))];
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** Known gap conservation, not a semantic completeness oracle. */
|
|
10
|
+
export function admitNeeds(input:EvaluationInput,proposal:Proposal):Diagnostic[] {
|
|
11
|
+
const diagnostics:Diagnostic[]=[],reviews=proposal.needReviews??[],known=new Map((input.needs??[]).map(n=>[n.needId,n])),seen=new Set<string>();
|
|
12
|
+
const graph=obligationGraph(proposal);
|
|
13
|
+
const issue=(location:string,message:string)=>diagnostics.push({rule:'need-unresolved',location,message,repairable:false});
|
|
14
|
+
if(!Array.isArray(reviews))return [{rule:'need-unresolved',location:'needReviews',message:'Known feedback needs require explicit final dispositions',repairable:false}];
|
|
15
|
+
for(const r of reviews){
|
|
16
|
+
if(!r||!known.has(r.needId)||seen.has(r.needId)||!['addressed','unresolved','not-required'].includes(r.disposition)||typeof r.reasoning!=='string'||!r.reasoning.trim()||!Array.isArray(r.obligationIds)) {issue('needReviews','Invalid, unknown or duplicate need disposition');continue;}
|
|
17
|
+
seen.add(r.needId);
|
|
18
|
+
const need=known.get(r.needId)!;
|
|
19
|
+
const obligations=r.obligationIds.map(id=>proposal.obligations.find(o=>o.id===id));
|
|
20
|
+
if(obligations.some(o=>!o||o.sourceId!==need.sourceId||o.expectation!==need.expectation))issue(r.needId,'Need disposition must bind its retained source AND exact expectation; another claim from the same source does not close this need');
|
|
21
|
+
if(r.disposition==='addressed'&&(!obligations.length||!obligations.some(o=>{
|
|
22
|
+
if(!o)return false;
|
|
23
|
+
const outcome=graph.compare(o.id);
|
|
24
|
+
return outcome.compared&&['pass','fail'].includes(outcome.judgment);
|
|
25
|
+
})))issue(r.needId,'Addressed need requires a supported definitive comparison of its exact leaf or compound relation; partial evidence or a parent assertion alone is insufficient');
|
|
26
|
+
if(r.disposition==='not-required'&&(!obligations.length||!obligations.every(o=>o?.relation==='conditional'&&o.condition==='not-applicable'&&o.rationale.trim())))issue(r.needId,'Not-required requires its exact expectation bound as a justified conditional/not-applicable obligation, not an ordinary or unresolved leaf');
|
|
27
|
+
if(r.disposition==='unresolved'&&proposal.judgment==='pass')issue(r.needId,'An unresolved material feedback need forbids unconditional PASS');
|
|
28
|
+
}
|
|
29
|
+
for(const id of known.keys())if(!seen.has(id))issue(id,'Previously identified need omitted from final review');
|
|
30
|
+
return diagnostics;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const NEED_GUIDANCE='Retain EVERY need with needReviews:[{needId:"need:N",disposition:"addressed|unresolved|not-required",obligationIds:[],reasoning:string}]. Addressed binds the EXACT expectation: use obligation.needRef:"need:N" and omit sourceId/expectation/quote; owner supplies retained bytes. For compound needs put needRef on the all/any parent and compare independent source-bound leaves, not a partially supported broad PASS. ALL needs every applicable child; ANY accepts a valid alternative. Unresolved may bind [] with explicit limits and forbids overall PASS. Not-required binds the exact need as conditional/not-applicable with original-source rationale. Advisory needs are fallible, not new user authority; no same-source substitution or duplicate parent score.';
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type {Judgment,Obligation,Proposal} from './contract';
|
|
2
|
+
|
|
3
|
+
interface Outcome {judgment:Judgment|'not-applicable';compared:boolean}
|
|
4
|
+
const unknown:Outcome={judgment:'indeterminate',compared:false};
|
|
5
|
+
|
|
6
|
+
/** Pure relation folding, not natural-language decomposition or a truth oracle. */
|
|
7
|
+
export function obligationGraph(proposal:Pick<Proposal,'obligations'|'claims'>) {
|
|
8
|
+
const obligations=new Map(proposal.obligations.map(o=>[o.id,o]));
|
|
9
|
+
const claims=new Map(proposal.claims.map(c=>[c.obligationId,c]));
|
|
10
|
+
const children=new Set(proposal.obligations.flatMap(o=>o.children));
|
|
11
|
+
const roots=proposal.obligations.filter(o=>!children.has(o.id));
|
|
12
|
+
|
|
13
|
+
function compare(id:string,trail:string[]=[]):Outcome {
|
|
14
|
+
const o=obligations.get(id);
|
|
15
|
+
if(!o||trail.includes(id))return unknown;
|
|
16
|
+
if(o.relation==='conditional'&&o.condition==='not-applicable'&&o.rationale.trim())return {judgment:'not-applicable',compared:false};
|
|
17
|
+
if(o.condition!=='applicable')return unknown;
|
|
18
|
+
if(o.relation==='leaf'&&o.children.length||['all','any'].includes(o.relation)&&!o.children.length)return unknown;
|
|
19
|
+
if(!o.children.length){
|
|
20
|
+
const claim=claims.get(id);
|
|
21
|
+
if(!claim?.evidence.length||claim.judgment==='indeterminate')return unknown;
|
|
22
|
+
return {judgment:claim.judgment,compared:true};
|
|
23
|
+
}
|
|
24
|
+
const active=o.children.map(child=>compare(child,[...trail,id])).filter(c=>c.judgment!=='not-applicable');
|
|
25
|
+
if(!active.length)return {judgment:'not-applicable',compared:false};
|
|
26
|
+
const compared=active.some(c=>c.compared);
|
|
27
|
+
if(o.relation==='any'){
|
|
28
|
+
if(active.some(c=>c.judgment==='pass'))return {judgment:'pass',compared};
|
|
29
|
+
if(active.every(c=>c.judgment==='fail'))return {judgment:'fail',compared};
|
|
30
|
+
}else{
|
|
31
|
+
if(active.some(c=>c.judgment==='fail'))return {judgment:'fail',compared};
|
|
32
|
+
if(active.every(c=>c.judgment==='pass'))return {judgment:'pass',compared};
|
|
33
|
+
}
|
|
34
|
+
return {judgment:'indeterminate',compared};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function containsContradiction(o:Obligation,id:string,trail:string[]=[]):boolean {
|
|
38
|
+
if(trail.includes(o.id)||compare(o.id).judgment!=='fail')return false;
|
|
39
|
+
if(o.id===id)return true;
|
|
40
|
+
return o.children.some(child=>{
|
|
41
|
+
const next=obligations.get(child);
|
|
42
|
+
return !!next&&containsContradiction(next,id,[...trail,o.id]);
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return {roots,compare,withinFailedRoot:(id:string)=>roots.some(root=>containsContradiction(root,id))};
|
|
47
|
+
}
|