coding-artifact-evaluation-shell 0.1.0-eidolon.a40d355425c38 → 0.1.0-eidolon.af1b35bd65ae9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -2
- package/SKILL.md +0 -2
- package/capsule.ts +1 -1
- package/package.json +1 -1
- package/resources/evaluate.md +0 -2
- package/support/judge.ts +9 -13
package/README.md
CHANGED
|
@@ -53,5 +53,3 @@ The next CLI invocation loads updated code; agents reread updated SOP paths. Fro
|
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
For the explicit coding entry, durable evidence contract and legacy identity compatibility, see [SKILL.md](SKILL.md#coding-scope-and-durable-evidence). New package imports use `coding-artifact-evaluation-shell/run`.
|
|
56
|
-
|
|
57
|
-
For current coding runs, explicit nonempty `codingScope.groups` also asks the same judge to compare every applicable caller-owned leaf, including after an overall FAIL. It does not add a role, scoring protocol, evidence or new requirements. An absent claim is unreviewed, distinct from an explicit indeterminate claim. Historical and ungrouped minimal FAIL decisions retain their original meaning; completeness and semantic correctness still require actual evidence.
|
package/SKILL.md
CHANGED
|
@@ -26,5 +26,3 @@ This evaluator assesses code-development artifacts and their explicitly required
|
|
|
26
26
|
New integrations import `runCodingEvaluation` from `coding-artifact-evaluation-shell/run` and pass `codingScope:{kind:'coding',description,groups?}` plus a new absolute `retainTo` directory whose parent exists. Groups reuse the existing root projection and never change strict overall judgment. `verifyRetainedEvaluation(path)` verifies a data-only archive after temporary inputs are removed. The archive contains code, complete target bytes, observations, original model outputs/logs, decisions and costs; it excludes runtime HOME/auth/bin/live channels and never acts as a resume checkpoint. Failed partial closures remain partial.
|
|
27
27
|
|
|
28
28
|
`runEvaluation`, `artifact-evaluation/v1` and `ArtifactEvaluation.*` FQNs remain compatibility identities for existing coding consumers and immutable historical records. They do not broaden the current product scope. Do not rename historical results or infer semantic PASS from retention verification.
|
|
29
|
-
|
|
30
|
-
For current coding runs, explicit nonempty `codingScope.groups` also asks the same judge to compare every applicable caller-owned leaf, including after an overall FAIL. It does not add a role, scoring protocol, evidence or new requirements. An absent claim is unreviewed, distinct from an explicit indeterminate claim. Historical and ungrouped minimal FAIL decisions retain their original meaning; completeness and semantic correctness still require actual evidence.
|
package/capsule.ts
CHANGED
|
@@ -112,7 +112,7 @@ export async function runEvaluation(manifest:Manifest,output:string,options:Eval
|
|
|
112
112
|
let delegate:JudgePort;
|
|
113
113
|
if(options.judge)delegate=options.judge;
|
|
114
114
|
else if(options.noModel)delegate={invocations:()=>[],judge:async()=>({judgment:'indeterminate',obligations:manifest.obligations??[],claims:[],findings:[],sourceReviews:sources.map(source=>({sourceId:source.id,complete:false,omissions:['No-model smoke does not make a semantic judgment'],ambiguities:[]}))})};
|
|
115
|
-
else delegate=createJudge(run,budget,state,manifest.includeCommitments,options.experimentalCounterReview?{reservedInvocations:1}:undefined,judgeAgent
|
|
115
|
+
else delegate=createJudge(run,budget,state,manifest.includeCommitments,options.experimentalCounterReview?{reservedInvocations:1}:undefined,judgeAgent);
|
|
116
116
|
let reviewed:Proposal|undefined,reviewError:JudgeProtocolError|undefined;
|
|
117
117
|
try {
|
|
118
118
|
try {
|
package/package.json
CHANGED
package/resources/evaluate.md
CHANGED
|
@@ -22,5 +22,3 @@ Collect facts through the runner's owned process/HTTP/browser/file capabilities.
|
|
|
22
22
|
The dynamic Evidence, Check and Groups LocalFunctions are read-only tools over explicit input paths. They do not grant execution rights, write verdicts or approve model budgets. Automated isolated model turns use the same implementation compiled into their owned scratch, not a generic host with broader permissions.
|
|
23
23
|
|
|
24
24
|
For grouped results, the caller explicitly supplies `{id,label,rootIds}` entries selecting actual root obligation IDs from the admitted decision. Use `projectTargetGroups(input, decision, groups)` from the package, or discover the Groups LocalFunction and invoke it with absolute native `inputPath`, `decisionPath`, and `groups`. It reuses the original graph, shows ungrouped/shared roots, and preserves the overall result. Non-root/unknown selections are rejected for admitted decisions; unfinished or rejected drafts never yield partial PASS. There are no built-in domains or automatic categories. Labels do not certify unlisted requirements: a group of selected calculations is not proof of an entire product. Missing historical process evidence remains missing; a group view cannot supply it or repair a mistaken original judgment.
|
|
25
|
-
|
|
26
|
-
For current coding runs, explicit nonempty `codingScope.groups` also asks the same judge to compare every applicable caller-owned leaf, including after an overall FAIL. It does not add a role, scoring protocol, evidence or new requirements. An absent claim is unreviewed, distinct from an explicit indeterminate claim. Historical and ungrouped minimal FAIL decisions retain their original meaning; completeness and semantic correctness still require actual evidence.
|
package/support/judge.ts
CHANGED
|
@@ -17,20 +17,16 @@ ${ENVIRONMENT_GUIDANCE}`;
|
|
|
17
17
|
export const SERVICE_FACT_GUIDANCE='Service launch/argv/cwd/port/directory facts are conditions, NOT additional application requirements; directory creation is not application state. Do not demand a specific storage/persistence/restart mechanism unless COMPLETE originals actually require it; assess its actual source and native execution evidence.';
|
|
18
18
|
|
|
19
19
|
/** Existing contracts only; a closed review is not taught a now-unavailable feedback route. */
|
|
20
|
-
export function judgeGuidance(canSupplement:boolean,hasNeeds:boolean
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const coverage=completeClaims
|
|
27
|
-
? 'The caller requested coding dimension views. Compare EVERY applicable leaf in the complete caller-owned obligation graph, even after a decisive FAIL. Submit one supported claim per leaf, or an explicit indeterminate claim explaining its missing material fact. Do not stop at the first contradiction or omit other dimensions; never invent positive claims to fill the table. Preserve all original scope, source reviews and needs. Overall still folds the same original graph; groups cannot change it. Keep each comparison concise; do not repeat the full logs.'
|
|
28
|
-
: 'Keep output proportional to the decision. A supported required FAIL needs its minimal witness/finding; an indeterminate needs explicit missing material scope, not gratuitous positive scores of every other area. Preserve caller obligations, source reviews and every need in either case. This never certifies unassessed scope. PASS still requires all original material conditions. Inspect explicit static deliverables early; a supported omission can be decisive. Use one comparison per claim; no extra planning document.';
|
|
29
|
-
return [JUDGE_GUIDANCE,SERVICE_FACT_GUIDANCE,choice,coverage,
|
|
20
|
+
export function judgeGuidance(canSupplement:boolean,hasNeeds:boolean):string {
|
|
21
|
+
const choice=canSupplement
|
|
22
|
+
? 'Check ALL currently material gaps together FIRST. If safely obtainable, return evidence requests promptly, not a provisional scorecard. A supported required contradiction already permits minimal FAIL.'
|
|
23
|
+
: 'Capabilities are closed; do not request or execute new observations. Submit a final comparison of current native facts.';
|
|
24
|
+
return [JUDGE_GUIDANCE,SERVICE_FACT_GUIDANCE,choice,
|
|
25
|
+
'Keep output proportional to the decision. A supported required FAIL needs its minimal witness/finding; an indeterminate needs explicit missing material scope, not gratuitous positive scores of every other area. Preserve caller obligations, source reviews and every need in either case. This never certifies unassessed scope. PASS still requires all original material conditions. Inspect explicit static deliverables early; a supported omission can be decisive. Use one comparison per claim; no extra planning document.',
|
|
30
26
|
...(canSupplement?[FEEDBACK_GUIDANCE]:[]),PROPOSAL_GUIDANCE,...(hasNeeds?[NEED_GUIDANCE]:[])].join('\n');
|
|
31
27
|
}
|
|
32
28
|
|
|
33
|
-
export function createJudge(run:AgentRun,budget:Budget,state:ProcessState,includeCommitments=false,experiment?:{window?:ModelWindow;reservedInvocations:number},agent?:AgentRuntime
|
|
29
|
+
export function createJudge(run:AgentRun,budget:Budget,state:ProcessState,includeCommitments=false,experiment?:{window?:ModelWindow;reservedInvocations:number},agent?:AgentRuntime):JudgePort {
|
|
34
30
|
const invocations:Invocation[]=[];
|
|
35
31
|
let finalWindow=experiment?.window;
|
|
36
32
|
function assertCandidateSlot():void {
|
|
@@ -44,7 +40,7 @@ export function createJudge(run:AgentRun,budget:Budget,state:ProcessState,includ
|
|
|
44
40
|
assertCandidateSlot();
|
|
45
41
|
const effectiveWindow=canSupplement?window:closedWindow()??window;
|
|
46
42
|
const payload={...input,policy:{includeCommitments},canSupplement,window:effectiveWindow};
|
|
47
|
-
const value=await modelTurn({run,budget,state,invocations,agent},{evaluation:input,payload,guidance:judgeGuidance(canSupplement,!!input.needs?.length
|
|
43
|
+
const value=await modelTurn({run,budget,state,invocations,agent},{evaluation:input,payload,guidance:judgeGuidance(canSupplement,!!input.needs?.length),phase:'judgment',window:effectiveWindow});
|
|
48
44
|
if(['evidence-request-v1','evidence-request-v2'].includes((value as any)?.format)){
|
|
49
45
|
if(!canSupplement)throw new JudgeProtocolError('Judge requested evidence after final bounded review');
|
|
50
46
|
return decodeFeedback(value,input,includeCommitments);
|
|
@@ -53,7 +49,7 @@ export function createJudge(run:AgentRun,budget:Budget,state:ProcessState,includ
|
|
|
53
49
|
},async judge(input:EvaluationInput,repair) {
|
|
54
50
|
assertCandidateSlot();
|
|
55
51
|
const payload={...input,policy:{includeCommitments},repair:repair??null};
|
|
56
|
-
const guidance=repair?EVIDENCE_REPAIR_GUIDANCE:judgeGuidance(false,!!input.needs?.length
|
|
52
|
+
const guidance=repair?EVIDENCE_REPAIR_GUIDANCE:judgeGuidance(false,!!input.needs?.length);
|
|
57
53
|
const value=await modelTurn({run,budget,state,invocations,agent},{evaluation:input,payload,guidance,phase:repair?'representation-repair':'judgment',window:closedWindow()});
|
|
58
54
|
return repair?decodeEvidenceRepair(value,repair):decodeProposal(value,input);
|
|
59
55
|
}};
|