@parcha/agentrun-pi 0.1.0-beta.2 → 0.1.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +123 -32
- package/dist/cli.js +15 -4
- package/dist/demo-cli.d.ts +2 -0
- package/dist/demo-cli.js +25 -0
- package/dist/demo.js +1 -0
- package/dist/durable/decision-recovery.d.ts +23 -0
- package/dist/durable/decision-recovery.js +57 -0
- package/dist/durable/decisions.d.ts +113 -0
- package/dist/durable/decisions.js +71 -0
- package/dist/durable/index.d.ts +20 -0
- package/dist/durable/index.js +14 -0
- package/dist/durable/node-runner.d.ts +106 -0
- package/dist/durable/node-runner.js +253 -0
- package/dist/durable/node.d.ts +88 -0
- package/dist/durable/node.js +98 -0
- package/dist/durable/pi-versions.d.ts +34 -0
- package/dist/durable/pi-versions.js +66 -0
- package/dist/durable/readers.d.ts +28 -0
- package/dist/durable/readers.js +103 -0
- package/dist/durable/receipts.d.ts +54 -0
- package/dist/durable/receipts.js +92 -0
- package/dist/durable/record-file.d.ts +10 -0
- package/dist/durable/record-file.js +56 -0
- package/dist/durable/record-tool.d.ts +76 -0
- package/dist/durable/record-tool.js +123 -0
- package/dist/durable/record.d.ts +157 -0
- package/dist/durable/record.js +402 -0
- package/dist/durable/recording.d.ts +90 -0
- package/dist/durable/recording.js +292 -0
- package/dist/durable/store.d.ts +23 -0
- package/dist/durable/store.js +242 -0
- package/dist/durable/task.d.ts +133 -0
- package/dist/durable/task.js +411 -0
- package/dist/durable/workflow-tool.d.ts +32 -0
- package/dist/durable/workflow-tool.js +188 -0
- package/dist/extension-service.js +63 -22
- package/dist/extension.d.ts +2 -0
- package/dist/extension.js +463 -80
- package/dist/host-addendum.d.ts +4 -0
- package/dist/host-addendum.js +20 -0
- package/dist/host-session.js +5 -2
- package/dist/index.d.ts +1 -2
- package/dist/index.js +1 -2
- package/dist/pi-host.d.ts +7 -0
- package/dist/pi-host.js +22 -0
- package/dist/presentation.js +2 -2
- package/dist/run-error.d.ts +6 -0
- package/dist/run-error.js +13 -0
- package/dist/runner.d.ts +1 -6
- package/dist/runner.js +7 -13
- package/dist/signin/index.d.ts +9 -0
- package/dist/signin/index.js +7 -0
- package/dist/signin/limit.d.ts +10 -0
- package/dist/signin/limit.js +23 -0
- package/dist/signin/resolve.d.ts +47 -0
- package/dist/signin/resolve.js +163 -0
- package/dist/signin/signin.d.ts +44 -0
- package/dist/signin/signin.js +126 -0
- package/dist/signin/spec.d.ts +116 -0
- package/dist/signin/spec.js +181 -0
- package/dist/signin/store.d.ts +40 -0
- package/dist/signin/store.js +206 -0
- package/dist/skills/author/SKILL.md +138 -0
- package/{skills → dist/skills}/author/references/jev-decisions.md +27 -14
- package/dist/skills/author/references/language.md +134 -0
- package/dist/skills/author/references/sdk-authoring.md +11 -0
- package/{skills → dist/skills}/author/references/workflow-format.md +7 -15
- package/dist/sop.d.ts +12 -0
- package/dist/sop.js +29 -0
- package/dist/thinking.d.ts +3 -0
- package/dist/thinking.js +7 -0
- package/dist/tool-context.d.ts +21 -0
- package/dist/tool-context.js +13 -0
- package/dist/triage-demo.d.ts +62 -0
- package/dist/triage-demo.js +153 -0
- package/dist/types.d.ts +7 -0
- package/dist/workflow-inspector.d.ts +47 -0
- package/dist/workflow-inspector.js +235 -0
- package/dist/workflow-session.d.ts +23 -0
- package/dist/workflow-session.js +168 -0
- package/dist/workflow-store.d.ts +36 -0
- package/dist/workflow-store.js +255 -0
- package/dist/workflow-tool.d.ts +81 -0
- package/dist/workflow-tool.js +94 -0
- package/dist/workflow-view.d.ts +64 -0
- package/dist/workflow-view.js +273 -0
- package/dist/workflows.d.ts +1 -0
- package/dist/workflows.js +3 -0
- package/package.json +57 -11
- package/dist/admission.d.ts +0 -6
- package/dist/admission.js +0 -51
- package/dist/author.d.ts +0 -21
- package/dist/author.js +0 -72
- package/dist/benchmark-contract.d.ts +0 -88
- package/dist/benchmark-contract.js +0 -50
- package/dist/benchmark-freeze.d.ts +0 -9
- package/dist/benchmark-freeze.js +0 -9
- package/dist/benchmark.d.ts +0 -127
- package/dist/benchmark.js +0 -260
- package/dist/comparison.d.ts +0 -69
- package/dist/comparison.js +0 -229
- package/dist/skill-bundle.d.ts +0 -5
- package/dist/skill-bundle.js +0 -15
- package/skills/author/SKILL.md +0 -102
- package/skills/author/references/sdk-authoring.md +0 -16
- /package/{skills → dist/skills}/author/examples/evidence-gate.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/evidence-gate.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-file.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-file.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-source-decision.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/read-source-decision.json +0 -0
- /package/{skills → dist/skills}/author/examples/release-note.txt +0 -0
- /package/{skills → dist/skills}/author/examples/review-release-notes.input.json +0 -0
- /package/{skills → dist/skills}/author/examples/review-release-notes.json +0 -0
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { inspectWorkflow } from '@parcha/agentrun-dsl';
|
|
2
|
+
import { workflowGraph } from '@parcha/agentrun-dsl/graph';
|
|
3
|
+
import { cleanText } from './presentation.js';
|
|
4
|
+
import { types as utilTypes } from 'node:util';
|
|
5
|
+
/** Display limits never admit, cancel, or modify execution. */
|
|
6
|
+
export function readable(value, max = 12_000) {
|
|
7
|
+
let text;
|
|
8
|
+
let excerpt = false;
|
|
9
|
+
try {
|
|
10
|
+
if (typeof value === 'string')
|
|
11
|
+
text = value;
|
|
12
|
+
// The controller explicitly requests complete expanded output; its structured
|
|
13
|
+
// report remains authoritative. Normal observation never uses this path.
|
|
14
|
+
else if (max === Number.MAX_SAFE_INTEGER)
|
|
15
|
+
text = JSON.stringify(value, null, 2) ?? '';
|
|
16
|
+
else {
|
|
17
|
+
let remaining = max, values = 0;
|
|
18
|
+
const seen = new Set();
|
|
19
|
+
const visit = (item, depth) => {
|
|
20
|
+
if (++values > 2_000 || remaining <= 0 || depth > 16) {
|
|
21
|
+
excerpt = true;
|
|
22
|
+
return '[Display excerpt]';
|
|
23
|
+
}
|
|
24
|
+
if (typeof item === 'string') {
|
|
25
|
+
const part = item.slice(0, remaining);
|
|
26
|
+
remaining -= part.length;
|
|
27
|
+
if (part.length < item.length)
|
|
28
|
+
excerpt = true;
|
|
29
|
+
return part;
|
|
30
|
+
}
|
|
31
|
+
remaining -= 8;
|
|
32
|
+
if (item === null || typeof item === 'boolean' || typeof item === 'number')
|
|
33
|
+
return item;
|
|
34
|
+
if (!item || typeof item !== 'object' || utilTypes.isProxy(item))
|
|
35
|
+
return '[Non-JSON value]';
|
|
36
|
+
if (seen.has(item))
|
|
37
|
+
return '[Repeated reference]';
|
|
38
|
+
const proto = Object.getPrototypeOf(item);
|
|
39
|
+
if (proto !== null && proto !== Object.prototype && proto !== Array.prototype)
|
|
40
|
+
return '[Non-JSON value]';
|
|
41
|
+
seen.add(item);
|
|
42
|
+
const result = Array.isArray(item) ? [] : Object.create(null);
|
|
43
|
+
if (Array.isArray(item)) {
|
|
44
|
+
const length = Math.min(item.length, 2_000);
|
|
45
|
+
for (let index = 0; index < length; index++) {
|
|
46
|
+
if (remaining <= 0 || values >= 2_000) {
|
|
47
|
+
excerpt = true;
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
const descriptor = Object.getOwnPropertyDescriptor(item, String(index));
|
|
51
|
+
result.push(descriptor && 'value' in descriptor ? visit(descriptor.value, depth + 1) : '[Unavailable element]');
|
|
52
|
+
}
|
|
53
|
+
if (result.length < item.length)
|
|
54
|
+
excerpt = true;
|
|
55
|
+
seen.delete(item);
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
for (const key in item) {
|
|
59
|
+
if (!Object.hasOwn(item, key))
|
|
60
|
+
continue;
|
|
61
|
+
if (remaining <= 0 || values >= 2_000) {
|
|
62
|
+
excerpt = true;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
const descriptor = Object.getOwnPropertyDescriptor(item, key);
|
|
66
|
+
remaining -= key.length;
|
|
67
|
+
Object.defineProperty(result, key, { value: 'value' in descriptor ? visit(descriptor.value, depth + 1) : '[Accessor not read]', enumerable: true, configurable: true });
|
|
68
|
+
}
|
|
69
|
+
seen.delete(item);
|
|
70
|
+
return result;
|
|
71
|
+
};
|
|
72
|
+
text = JSON.stringify(visit(value, 0), null, 2) ?? '';
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return '[Not available as JSON]';
|
|
77
|
+
}
|
|
78
|
+
const clipped = text.length > max || excerpt;
|
|
79
|
+
return cleanText(text.slice(0, max)) + (clipped ? '\n… Display excerpt; additional content omitted.' : '');
|
|
80
|
+
}
|
|
81
|
+
const definitionPath = (path) => path.replace(/\/items\/\d+\/body/g, '/body').replace(/\/iterations\/\d+(?=\/|$)/g, '');
|
|
82
|
+
const at = (value, path) => path.split('/').slice(1).reduce((item, key) => item && Object.hasOwn(item, key.replace(/~1/g, '/').replace(/~0/g, '~')) ? item[key.replace(/~1/g, '/').replace(/~0/g, '~')] : undefined, value);
|
|
83
|
+
export class WorkflowObservation {
|
|
84
|
+
data = { steps: {}, decisions: [], notes: {}, routes: {}, omitted: 0 };
|
|
85
|
+
decisionBytes = 0;
|
|
86
|
+
record(event) {
|
|
87
|
+
try {
|
|
88
|
+
this.recordSafe(event);
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
this.data.omitted++;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
recordSafe(event) {
|
|
95
|
+
const path = event.executionPath ?? `/label/${event.label}`;
|
|
96
|
+
const detail = event.detail;
|
|
97
|
+
if (event.type === 'node.start' || event.type === 'node.end') {
|
|
98
|
+
if (!Object.hasOwn(this.data.steps, path) && Object.keys(this.data.steps).length >= 2_000) {
|
|
99
|
+
this.data.omitted++;
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
this.data.steps[path] = { label: cleanText(event.label), kind: String(detail?.kind ?? 'step'),
|
|
103
|
+
status: event.type === 'node.start' ? 'running' : detail?.status === 'ok' ? 'succeeded' : detail?.status === 'escalated' ? 'needs attention' : 'failed' };
|
|
104
|
+
}
|
|
105
|
+
// Escalation emits an evaluation event, not node.start/node.end. Project that
|
|
106
|
+
// existing interpreter evidence so the selected branch cannot look untouched.
|
|
107
|
+
if (event.type === 'escalate.evaluated') {
|
|
108
|
+
if (!Object.hasOwn(this.data.steps, path) && Object.keys(this.data.steps).length >= 2_000) {
|
|
109
|
+
this.data.omitted++;
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
this.data.steps[path] = { label: cleanText(event.label), kind: 'escalate',
|
|
113
|
+
status: detail?.fired === true ? 'needs attention' : 'condition not met' };
|
|
114
|
+
}
|
|
115
|
+
if (event.type === 'judge.answered' || event.type === 'route.chosen' || event.type === 'ask.evaluated') {
|
|
116
|
+
const decision = this.data.decisions.findLast(item => item.path === path && item.accepted === undefined);
|
|
117
|
+
if (decision)
|
|
118
|
+
decision.accepted = true;
|
|
119
|
+
}
|
|
120
|
+
if (event.type === 'route.chosen' || event.type === 'loop.exited' || event.type === 'map.failed' || event.type === 'map.escalated') {
|
|
121
|
+
if (!Object.hasOwn(this.data.notes, path) && Object.keys(this.data.notes).length >= 200) {
|
|
122
|
+
this.data.omitted++;
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
this.data.notes[path] = event.type === 'route.chosen' ? `Selected branch: ${readable(detail?.value, 500)}`
|
|
126
|
+
: event.type === 'loop.exited' ? `${detail?.iterations} iterations · ${detail?.reason === 'condition_met' ? 'stop condition met' : 'iteration bound reached; not proof of success'}`
|
|
127
|
+
: `${detail?.completed} of ${detail?.of} items completed · ${event.type === 'map.failed' ? 'failed' : 'needs attention'}`;
|
|
128
|
+
if (event.type === 'route.chosen' && typeof detail?.value?.taken === 'string')
|
|
129
|
+
this.data.routes[path] = detail.value.taken;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/** Capture the exact supplied questions/state, not an explanation invented afterward. */
|
|
133
|
+
requested(request) {
|
|
134
|
+
if (this.data.decisions.length >= 32 || this.decisionBytes >= 128_000) {
|
|
135
|
+
this.data.omitted++;
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
const item = { path: request.executionPath ?? `/label/${request.label}`, label: cleanText(request.label),
|
|
139
|
+
questions: readable(request.questions, 8_000), evidence: readable(request.state, 8_000) };
|
|
140
|
+
this.decisionBytes += item.questions.length + item.evidence.length;
|
|
141
|
+
this.data.decisions.push(item);
|
|
142
|
+
return item;
|
|
143
|
+
}
|
|
144
|
+
observe(deps) {
|
|
145
|
+
if (!deps.runJudge)
|
|
146
|
+
return deps;
|
|
147
|
+
const runJudge = deps.runJudge;
|
|
148
|
+
return { ...deps, runJudge: async (request) => {
|
|
149
|
+
let record;
|
|
150
|
+
try {
|
|
151
|
+
record = this.requested(request);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
this.data.omitted++;
|
|
155
|
+
}
|
|
156
|
+
const result = await runJudge(request);
|
|
157
|
+
// A transport response is not acceptance; interpreter events confirm it later.
|
|
158
|
+
try {
|
|
159
|
+
if (record)
|
|
160
|
+
record.answer = readable(result.answers, 8_000);
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
this.data.omitted++;
|
|
164
|
+
}
|
|
165
|
+
return result;
|
|
166
|
+
} };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
export function progressLines(observation, terminalStatus) {
|
|
170
|
+
const steps = Object.values(observation.steps).map(step => terminalStatus && step.status === 'running'
|
|
171
|
+
? { ...step, status: ['interrupted', 'cancelled'].includes(terminalStatus) ? 'interrupted' : 'completion not observed' } : step);
|
|
172
|
+
const count = (status) => steps.filter(step => step.status === status).length;
|
|
173
|
+
const active = steps.filter(step => step.status === 'running');
|
|
174
|
+
return [
|
|
175
|
+
`${count('succeeded')} succeeded · ${active.length} active · ${count('failed')} failed · ${count('needs attention')} need attention`,
|
|
176
|
+
...(count('interrupted') ? [`${count('interrupted')} interrupted; completion not observed.`] : []),
|
|
177
|
+
...(count('completion not observed') ? [`${count('completion not observed')} steps have no observed completion.`] : []),
|
|
178
|
+
...active.slice(0, 3).map(step => `Working: ${step.label}`),
|
|
179
|
+
...(active.length > 3 ? [`… ${active.length - 3} more active steps`] : []),
|
|
180
|
+
...(observation.omitted ? ['Observation is partial; omitted details do not stop execution.'] : []),
|
|
181
|
+
];
|
|
182
|
+
}
|
|
183
|
+
export function workflowView(workflow, options = {}) {
|
|
184
|
+
const inspection = inspectWorkflow(workflow);
|
|
185
|
+
const definition = workflow;
|
|
186
|
+
const observed = options.observation;
|
|
187
|
+
const status = options.running ? 'running' : options.report?.status ?? 'draft';
|
|
188
|
+
const required = definition.input ? definition.schemas[definition.input.schemaId]?.required ?? [] : [];
|
|
189
|
+
const missing = required.filter((key) => !Object.hasOwn(options.input ?? {}, key));
|
|
190
|
+
const tools = inspection.requires.tools;
|
|
191
|
+
const unavailable = options.availableTools ? tools.filter(name => !options.availableTools.includes(name)) : [];
|
|
192
|
+
// The nodes are the graph's (`@parcha/agentrun-dsl/graph`, the one the viewer's graph panel draws); inspection only adds what a step writes.
|
|
193
|
+
const writes = new Map(inspection.nodes.map(node => [node.path, node.writes]));
|
|
194
|
+
const nodes = workflowGraph(workflow).nodes.map(graphNode => {
|
|
195
|
+
const node = { ...graphNode, writes: writes.get(graphNode.path) };
|
|
196
|
+
const source = at(definition, node.path) ?? {};
|
|
197
|
+
const childRoot = node.path.lastIndexOf('/workflow/root');
|
|
198
|
+
const owner = childRoot < 0 ? definition : at(definition, node.path.slice(0, childRoot + '/workflow'.length));
|
|
199
|
+
const instances = Object.entries(observed?.steps ?? {}).filter(([path]) => definitionPath(path) === node.path);
|
|
200
|
+
const events = Object.entries(observed?.notes ?? {}).filter(([path]) => definitionPath(path) === node.path);
|
|
201
|
+
const decisions = observed?.decisions.filter(item => definitionPath(item.path) === node.path) ?? [];
|
|
202
|
+
const descendants = Object.entries(observed?.steps ?? {}).filter(([path]) => definitionPath(path).startsWith(`${node.path}/`));
|
|
203
|
+
const structural = ['chain', 'map', 'parallel', 'loop'].includes(node.kind);
|
|
204
|
+
const statuses = (instances.length ? instances : structural ? descendants : []).map(([, item]) => item.status);
|
|
205
|
+
const final = !!options.report && !options.running;
|
|
206
|
+
const skipped = final && !observed?.omitted && Object.entries(observed?.routes ?? {}).some(([routePath]) => {
|
|
207
|
+
const parent = definitionPath(routePath);
|
|
208
|
+
if (!node.path.startsWith(`${parent}/branches/`))
|
|
209
|
+
return false;
|
|
210
|
+
const branch = node.path.slice(`${parent}/branches/`.length).split('/')[0];
|
|
211
|
+
const choices = Object.entries(observed?.routes ?? {}).filter(([path]) => definitionPath(path) === parent);
|
|
212
|
+
return choices.length > 0 && choices.every(([, taken]) => taken.replace(/~/g, '~0').replace(/\//g, '~1') !== branch);
|
|
213
|
+
});
|
|
214
|
+
const nodeStatus = statuses.includes('running') ? final ? ['interrupted', 'cancelled'].includes(status) ? 'interrupted' : 'completion not observed' : 'running' : statuses.includes('failed') ? 'failed'
|
|
215
|
+
: statuses.some(value => ['interrupted', 'cancelled'].includes(value)) ? 'interrupted'
|
|
216
|
+
: statuses.includes('needs attention') ? 'needs attention'
|
|
217
|
+
: structural && !instances.length && observed?.omitted ? 'completion not observed'
|
|
218
|
+
: statuses.length && statuses.every(value => value === 'condition not met') ? 'condition not met'
|
|
219
|
+
: statuses.length && statuses.every(value => value === 'succeeded' || value === 'condition not met') ? 'succeeded'
|
|
220
|
+
: statuses.length ? 'completion not observed'
|
|
221
|
+
: events.length ? 'observed' : status === 'draft' ? 'planned' : skipped ? 'skipped (branch not selected)' : 'not observed';
|
|
222
|
+
const displayedStatus = structural && !instances.length && nodeStatus === 'succeeded' ? 'observed work succeeded' : nodeStatus;
|
|
223
|
+
const counts = instances.length > 1 ? `${statuses.filter(s => s === 'succeeded').length}/${instances.length} observed executions succeeded` : '';
|
|
224
|
+
const detail = [
|
|
225
|
+
`Step: ${node.label} [${node.kind}]`, `Definition: ${node.path}`, `State: ${displayedStatus}`,
|
|
226
|
+
...(source.requires?.length ? [`Requires: ${source.requires.join(', ')}`] : []),
|
|
227
|
+
...(source.state !== undefined ? ['Input binding:', readable(source.state)] : []),
|
|
228
|
+
...(source.instructions ? ['Instructions:', readable(source.instructions)] : []),
|
|
229
|
+
...(typeof source.valuePath === 'string' ? [`Branch chosen by state at ${readable(source.valuePath, 200)}; no model is asked.`, ...(typeof source.otherwise === 'string' ? [`Otherwise: ${readable(source.otherwise, 200)}`] : [])] : []),
|
|
230
|
+
...(source.tools?.length ? [`Tools: ${source.tools.join(', ')}`] : []),
|
|
231
|
+
...(source.tool ? [`Tool: ${source.tool}`] : []),
|
|
232
|
+
...(source.args ? ['Tool arguments:', readable(source.args)] : []),
|
|
233
|
+
...(source.code ? ['Code (not executed by inspection):', readable(source.code)] : []),
|
|
234
|
+
...(source.until ? ['Stop condition:', readable(source.until), `Maximum iterations: ${source.maxIters ?? 'none configured'}`] : []),
|
|
235
|
+
...(source.when ? ['Condition:', readable(source.when)] : []),
|
|
236
|
+
...(source.out ? [`Output schema: ${source.out}`, readable(owner?.schemas?.[source.out])] : []),
|
|
237
|
+
...(node.writes ? [`Writes: ${node.writes}`] : []),
|
|
238
|
+
...(source.branches && !Array.isArray(source.branches) ? ['Branch rules:', readable(Object.fromEntries(Object.entries(source.branches).map(([name, value]) => [name, value.when ?? value.criteria ?? value.description ?? name])))] : []),
|
|
239
|
+
...events.map(([path, note]) => `${path}: ${note}`),
|
|
240
|
+
...decisions.flatMap((item, index) => [`Decision ${index + 1} · ${item.path}`, 'Actual questions and criteria:', item.questions,
|
|
241
|
+
'Actual evidence supplied:', item.evidence, item.accepted ? 'Interpreter accepted the answer:' : 'Response (acceptance not observed):', item.answer ?? 'No answer observed.',
|
|
242
|
+
'Confidence describes this judgment, not whole-workflow correctness.']),
|
|
243
|
+
];
|
|
244
|
+
return { path: node.path, label: cleanText(node.label), kind: node.kind, status: displayedStatus,
|
|
245
|
+
summary: [counts, ...events.map(([, note]) => note)].filter(Boolean).join(' · '), details: detail.map(cleanText) };
|
|
246
|
+
});
|
|
247
|
+
return {
|
|
248
|
+
title: cleanText(inspection.name), digest: inspection.sha256, status,
|
|
249
|
+
summary: [
|
|
250
|
+
`${options.savedName ? `Saved: ${cleanText(options.savedName)} · ` : ''}Revision: ${inspection.sha256.slice(0, 12)} · ${options.mode ?? 'live adapters'}`,
|
|
251
|
+
...(options.savedName ? [`Load this saved revision: /agentrun load ${cleanText(options.savedName)} ${inspection.sha256}`] : []),
|
|
252
|
+
`Input: ${inspection.inputSchema ?? 'object'} → Output: ${inspection.outputSchema}${inspection.outputPath ? ` (${inspection.outputPath})` : ''}`,
|
|
253
|
+
...(missing.length ? [`Missing input: ${missing.join(', ')}`] : []),
|
|
254
|
+
`Tools: ${tools.join(', ') || 'none'}`,
|
|
255
|
+
...(unavailable.length ? [`Unavailable without host configuration or approval: ${unavailable.join(', ')}`] : []),
|
|
256
|
+
...(inspection.requires.executableCode ? ['Approval: code requires /agentrun run --trusted for each run (unsandboxed).'] : []),
|
|
257
|
+
...(options.model && !options.mode?.startsWith('scripted') && inspection.requires.adapters.includes('runNode') ? [`Agent model: ${options.model}`] : []),
|
|
258
|
+
...(options.limits ? [`Host limits: ${Object.entries(options.limits).map(([key, value]) => `${key}=${value ?? 'disabled'}`).join(', ')}`] : []),
|
|
259
|
+
...(observed ? progressLines(observed, options.report && !options.running ? status : undefined) : ['Preflight checks syntax and admission; it does not prove task correctness.']),
|
|
260
|
+
...(options.report?.error ? [`Failure: ${options.report.error.stage ?? 'workflow'} · ${options.report.error.message}`] : []),
|
|
261
|
+
...(options.report?.uncertainEffects?.length ? ['Unknown tool outcomes: reconcile before restarting.'] : []),
|
|
262
|
+
...(options.report?.traceTruncated ? ['Retained event trace is partial.'] : []),
|
|
263
|
+
].map(cleanText), nodes,
|
|
264
|
+
...(options.report?.output !== undefined ? { output: ['Output:', readable(options.report.output)] } : {}),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
export function formatWorkflowView(view) {
|
|
268
|
+
return cleanText([`${view.title} · ${view.status}`, ...view.summary, '',
|
|
269
|
+
...view.nodes.map(node => `${' '.repeat(Math.min(5, node.path.split('/').length - 2))}${node.label} [${node.kind}] · ${node.status}${node.summary ? ` · ${node.summary}` : ''}`),
|
|
270
|
+
...(view.output ? ['', ...view.output] : []),
|
|
271
|
+
'', 'Inspect a step for criteria, evidence, and observed decisions. Restart reruns the whole workflow; it is not resume.',
|
|
272
|
+
].join('\n'));
|
|
273
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { WorkflowStore, WorkflowStoreError, type StoredWorkflow, type WorkflowRevision } from './workflow-store.js';
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
// Saved workflows for a host: the store the /agentrun extension keeps for its project directory, against a directory the host passes.
|
|
2
|
+
// It loads nothing from pi, so a host that never runs the extension can import it.
|
|
3
|
+
export { WorkflowStore, WorkflowStoreError } from './workflow-store.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@parcha/agentrun-pi",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.21",
|
|
4
4
|
"description": "Pi extension for building, running and evaluating AgentRun workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -15,6 +15,18 @@
|
|
|
15
15
|
"./extension": {
|
|
16
16
|
"types": "./dist/extension.d.ts",
|
|
17
17
|
"import": "./dist/extension.js"
|
|
18
|
+
},
|
|
19
|
+
"./durable": {
|
|
20
|
+
"types": "./dist/durable/index.d.ts",
|
|
21
|
+
"import": "./dist/durable/index.js"
|
|
22
|
+
},
|
|
23
|
+
"./signin": {
|
|
24
|
+
"types": "./dist/signin/index.d.ts",
|
|
25
|
+
"import": "./dist/signin/index.js"
|
|
26
|
+
},
|
|
27
|
+
"./workflows": {
|
|
28
|
+
"types": "./dist/workflows.d.ts",
|
|
29
|
+
"import": "./dist/workflows.js"
|
|
18
30
|
}
|
|
19
31
|
},
|
|
20
32
|
"bin": {
|
|
@@ -22,26 +34,60 @@
|
|
|
22
34
|
},
|
|
23
35
|
"files": [
|
|
24
36
|
"dist",
|
|
25
|
-
"skills",
|
|
26
37
|
"README.md",
|
|
27
38
|
"LICENSE",
|
|
28
39
|
"NOTICE"
|
|
29
40
|
],
|
|
30
41
|
"scripts": {
|
|
31
|
-
"build": "node ../../scripts/build-pi-demo.mjs && tsc -p tsconfig.json",
|
|
42
|
+
"build": "node ../../scripts/build-pi-demo.mjs && tsc -p tsconfig.json && node ../../scripts/render-pi-skill.mjs",
|
|
32
43
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
33
44
|
"test": "node --test test/*.test.mjs"
|
|
34
45
|
},
|
|
35
46
|
"devDependencies": {
|
|
36
|
-
"@earendil-works/
|
|
47
|
+
"@earendil-works/chord": "1.1.0",
|
|
48
|
+
"@earendil-works/pi-agent-core": "1.1.0",
|
|
49
|
+
"@earendil-works/pi-ai": "1.1.0",
|
|
50
|
+
"@earendil-works/pi-coding-agent": "1.1.0",
|
|
51
|
+
"@earendil-works/pi-durable": "1.1.0",
|
|
52
|
+
"@earendil-works/pi-tui": "1.1.0",
|
|
53
|
+
"@parcha/pi-durable-disk": "0.1.0-beta.21",
|
|
54
|
+
"typebox": "1.3.7"
|
|
37
55
|
},
|
|
38
56
|
"dependencies": {
|
|
39
|
-
"@parcha/agentrun-dsl": "0.1.0-beta.
|
|
40
|
-
"@parcha/agentrun-jev": "0.1.0-beta.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
"@earendil-works/pi-
|
|
44
|
-
"
|
|
57
|
+
"@parcha/agentrun-dsl": "0.1.0-beta.21",
|
|
58
|
+
"@parcha/agentrun-jev": "0.1.0-beta.21"
|
|
59
|
+
},
|
|
60
|
+
"peerDependencies": {
|
|
61
|
+
"@earendil-works/pi-agent-core": "*",
|
|
62
|
+
"@earendil-works/pi-ai": "*",
|
|
63
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
64
|
+
"@earendil-works/pi-tui": "*",
|
|
65
|
+
"typebox": "*",
|
|
66
|
+
"@earendil-works/pi-durable": ">=1.0.4 <1.4.0",
|
|
67
|
+
"@earendil-works/chord": ">=1.0.4 <1.4.0"
|
|
68
|
+
},
|
|
69
|
+
"peerDependenciesMeta": {
|
|
70
|
+
"@earendil-works/pi-agent-core": {
|
|
71
|
+
"optional": true
|
|
72
|
+
},
|
|
73
|
+
"@earendil-works/pi-ai": {
|
|
74
|
+
"optional": true
|
|
75
|
+
},
|
|
76
|
+
"@earendil-works/pi-coding-agent": {
|
|
77
|
+
"optional": true
|
|
78
|
+
},
|
|
79
|
+
"@earendil-works/pi-tui": {
|
|
80
|
+
"optional": true
|
|
81
|
+
},
|
|
82
|
+
"typebox": {
|
|
83
|
+
"optional": true
|
|
84
|
+
},
|
|
85
|
+
"@earendil-works/pi-durable": {
|
|
86
|
+
"optional": true
|
|
87
|
+
},
|
|
88
|
+
"@earendil-works/chord": {
|
|
89
|
+
"optional": true
|
|
90
|
+
}
|
|
45
91
|
},
|
|
46
92
|
"repository": {
|
|
47
93
|
"type": "git",
|
|
@@ -67,7 +113,7 @@
|
|
|
67
113
|
"./dist/extension.js"
|
|
68
114
|
],
|
|
69
115
|
"skills": [
|
|
70
|
-
"./skills"
|
|
116
|
+
"./dist/skills"
|
|
71
117
|
]
|
|
72
118
|
}
|
|
73
119
|
}
|
package/dist/admission.d.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export interface WorkflowAdmissionOptions {
|
|
2
|
-
rubricSections?: Record<string, string>;
|
|
3
|
-
allowExecutableCandidates?: boolean;
|
|
4
|
-
allowedEffectTools?: readonly string[];
|
|
5
|
-
}
|
|
6
|
-
export declare function workflowPolicyErrors(candidate: unknown, options: WorkflowAdmissionOptions): string[];
|
package/dist/admission.js
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
export function workflowPolicyErrors(candidate, options) {
|
|
2
|
-
const errors = [];
|
|
3
|
-
const sections = Object.keys(options.rubricSections ?? {});
|
|
4
|
-
const record = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
5
|
-
const visit = (value) => {
|
|
6
|
-
const node = record(value);
|
|
7
|
-
if (!node)
|
|
8
|
-
return;
|
|
9
|
-
const kind = node.node;
|
|
10
|
-
if (!options.allowExecutableCandidates && ["code", "call", "artifact"].includes(String(kind)) && !(kind === "call" && node.via === "tool" && typeof node.tool === "string" && options.allowedEffectTools?.includes(node.tool)))
|
|
11
|
-
errors.push(`${kind} requires allowExecutableCandidates`);
|
|
12
|
-
if (sections.length && ["judge", "pick", "sift", "route"].includes(String(kind)))
|
|
13
|
-
errors.push(`${String(kind)} cannot carry supplied SOP sections; use an LLM node with sopSection`);
|
|
14
|
-
const llm = ["agent", "decide", "extract", "report"].includes(String(kind));
|
|
15
|
-
const predicate = kind === "loop" ? record(node.until) : kind === "escalate" ? record(node.when) : undefined;
|
|
16
|
-
if (sections.length && (predicate?.predicate === "ask" || (llm && node.verify !== undefined)))
|
|
17
|
-
errors.push("Semantic predicates and verify clauses require a separately reviewed question contract when rubric sections are supplied");
|
|
18
|
-
const proseArtifact = kind === "artifact" && (node.type === "markdown" || node.type === "report");
|
|
19
|
-
if (llm || proseArtifact) {
|
|
20
|
-
const included = Array.isArray(node.sopSection) ? node.sopSection : [node.sopSection];
|
|
21
|
-
for (const section of sections)
|
|
22
|
-
if (!included.includes(section))
|
|
23
|
-
errors.push(`${String(node.label)} must include rubric section ${section}`);
|
|
24
|
-
}
|
|
25
|
-
switch (kind) {
|
|
26
|
-
case "chain":
|
|
27
|
-
if (Array.isArray(node.steps))
|
|
28
|
-
node.steps.forEach(visit);
|
|
29
|
-
break;
|
|
30
|
-
case "parallel":
|
|
31
|
-
if (Array.isArray(node.branches))
|
|
32
|
-
node.branches.forEach(visit);
|
|
33
|
-
break;
|
|
34
|
-
case "map":
|
|
35
|
-
case "loop":
|
|
36
|
-
visit(node.body);
|
|
37
|
-
break;
|
|
38
|
-
case "route": {
|
|
39
|
-
const branches = record(node.branches);
|
|
40
|
-
if (branches)
|
|
41
|
-
Object.values(branches).forEach(branch => visit(record(branch)?.body));
|
|
42
|
-
break;
|
|
43
|
-
}
|
|
44
|
-
case "workflow":
|
|
45
|
-
visit(record(node.workflow)?.root);
|
|
46
|
-
break;
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
visit(record(candidate)?.root);
|
|
50
|
-
return errors;
|
|
51
|
-
}
|
package/dist/author.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { type Workflow } from "@parcha/agentrun-dsl";
|
|
2
|
-
import { type PiRunnerOptions } from "./runner.js";
|
|
3
|
-
export declare const AUTHOR_CONTRACT = "You author AgentRun DSL v2 workflows as JSON data.\nDeliver with the submit tool, never prose or a filename. Make the smallest workflow satisfying the request.\nThe document has v:2, name, schemas (JSON Schema catalog), optional input:{schemaId}, output:{schemaId,path?}, root.\nEvery referenced schema must exist. Each LLM node has node:agent|decide|extract, label, instructions, out (schema id), as (state key), optional requires/tools/sopSection.\nA chain has node:chain, steps:[nodes]. A map has label, itemsPath, body, as, maxConcurrency. A loop has label, body, until, maxIters.\nA judge has label,state,out,as. Judge schemas use flat boolean/enum/integer properties; each property's description states its question. A route has label,state,instructions,branches:{name:{criteria,body}}, optional unsure:{branch,gte} and as.\nLLM instructions are literal text, never interpolated. The node receives workflow state as a JSON user message; refer to its keys in ordinary prose and declare required keys in requires. Use {stateKey} placeholders only in interpolated fields such as judge.state and call.args; whole placeholders preserve their value's type. Output.path selects a final state value validated by output.schemaId.\nOnly use tools named by the host. Never invent a tool, capability, evidence source or successful test.\nKeep the full rubric in its original source text. Every LLM node's sopSection must include ALL supplied rubric section names. Merged judgments inherit their parents' union. Do not paraphrase policy into instructions. If rubric sections are supplied, use LLM nodes for judgment; Jev nodes do not carry sopSection and require a separately reviewed question contract.\nKeep independent acceptance checks host-owned; never change their implementation, fixtures or thresholds to pass. A valid candidate is not automatically activated.\nCode, shell/effect calls and artifact delivery require explicit host authorization. Bound loops and parallelism; uncertainty should escalate or select an explicit fallback.\nExample: {\"v\":2,\"name\":\"summarize\",\"schemas\":{\"Result\":{\"type\":\"object\",\"properties\":{\"summary\":{\"type\":\"string\"}},\"required\":[\"summary\"],\"additionalProperties\":false}},\"output\":{\"schemaId\":\"Result\",\"path\":\"result\"},\"root\":{\"node\":\"extract\",\"label\":\"summarize\",\"instructions\":\"Summarize the text field from the JSON input faithfully.\",\"out\":\"Result\",\"as\":\"result\",\"requires\":[\"text\"]}}";
|
|
4
|
-
export interface AuthorWorkflowOptions {
|
|
5
|
-
request: string;
|
|
6
|
-
outputDir: string;
|
|
7
|
-
pi: PiRunnerOptions;
|
|
8
|
-
inputKeys?: string[];
|
|
9
|
-
rubricSections?: Record<string, string>;
|
|
10
|
-
allowExecutableCandidates?: boolean;
|
|
11
|
-
maxCandidates?: number;
|
|
12
|
-
acceptance?: (candidate: Workflow) => Promise<string[]> | string[];
|
|
13
|
-
}
|
|
14
|
-
export interface AuthoredWorkflow {
|
|
15
|
-
workflow: Workflow;
|
|
16
|
-
path: string;
|
|
17
|
-
directory: string;
|
|
18
|
-
candidates: number;
|
|
19
|
-
checks: "structural" | "structural-and-host";
|
|
20
|
-
}
|
|
21
|
-
export declare function authorWorkflow(options: AuthorWorkflowOptions): Promise<AuthoredWorkflow>;
|
package/dist/author.js
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import { workflowPolicyErrors } from "./admission.js";
|
|
2
|
-
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
4
|
-
import { resolve, join } from "node:path";
|
|
5
|
-
import { validateWorkflow } from "@parcha/agentrun-dsl";
|
|
6
|
-
import { createPiRunner } from "./runner.js";
|
|
7
|
-
export const AUTHOR_CONTRACT = `You author AgentRun DSL v2 workflows as JSON data.
|
|
8
|
-
Deliver with the submit tool, never prose or a filename. Make the smallest workflow satisfying the request.
|
|
9
|
-
The document has v:2, name, schemas (JSON Schema catalog), optional input:{schemaId}, output:{schemaId,path?}, root.
|
|
10
|
-
Every referenced schema must exist. Each LLM node has node:agent|decide|extract, label, instructions, out (schema id), as (state key), optional requires/tools/sopSection.
|
|
11
|
-
A chain has node:chain, steps:[nodes]. A map has label, itemsPath, body, as, maxConcurrency. A loop has label, body, until, maxIters.
|
|
12
|
-
A judge has label,state,out,as. Judge schemas use flat boolean/enum/integer properties; each property's description states its question. A route has label,state,instructions,branches:{name:{criteria,body}}, optional unsure:{branch,gte} and as.
|
|
13
|
-
LLM instructions are literal text, never interpolated. The node receives workflow state as a JSON user message; refer to its keys in ordinary prose and declare required keys in requires. Use {stateKey} placeholders only in interpolated fields such as judge.state and call.args; whole placeholders preserve their value's type. Output.path selects a final state value validated by output.schemaId.
|
|
14
|
-
Only use tools named by the host. Never invent a tool, capability, evidence source or successful test.
|
|
15
|
-
Keep the full rubric in its original source text. Every LLM node's sopSection must include ALL supplied rubric section names. Merged judgments inherit their parents' union. Do not paraphrase policy into instructions. If rubric sections are supplied, use LLM nodes for judgment; Jev nodes do not carry sopSection and require a separately reviewed question contract.
|
|
16
|
-
Keep independent acceptance checks host-owned; never change their implementation, fixtures or thresholds to pass. A valid candidate is not automatically activated.
|
|
17
|
-
Code, shell/effect calls and artifact delivery require explicit host authorization. Bound loops and parallelism; uncertainty should escalate or select an explicit fallback.
|
|
18
|
-
Example: {"v":2,"name":"summarize","schemas":{"Result":{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"],"additionalProperties":false}},"output":{"schemaId":"Result","path":"result"},"root":{"node":"extract","label":"summarize","instructions":"Summarize the text field from the JSON input faithfully.","out":"Result","as":"result","requires":["text"]}}`;
|
|
19
|
-
export async function authorWorkflow(options) {
|
|
20
|
-
if (!options.request.trim())
|
|
21
|
-
throw new Error("Workflow request is required");
|
|
22
|
-
const maxCandidates = options.maxCandidates ?? 4;
|
|
23
|
-
if (!Number.isSafeInteger(maxCandidates) || maxCandidates < 1)
|
|
24
|
-
throw new Error("maxCandidates must be a positive integer");
|
|
25
|
-
const directory = resolve(options.outputDir, `candidate-${randomUUID()}`);
|
|
26
|
-
await mkdir(directory, { recursive: true });
|
|
27
|
-
await writeFile(join(directory, "request.json"), JSON.stringify({ request: options.request, inputKeys: options.inputKeys ?? [], rubricSections: options.rubricSections ?? {}, allowExecutableCandidates: options.allowExecutableCandidates ?? false, maxCandidates }, null, 2), { flag: "wx" });
|
|
28
|
-
let candidates = 0;
|
|
29
|
-
let acceptedPath;
|
|
30
|
-
const runner = createPiRunner({ ...options.pi, tools: [], maxSubmissions: maxCandidates });
|
|
31
|
-
try {
|
|
32
|
-
const workflow = await runner({
|
|
33
|
-
kind: "agent", label: "author-workflow", tools: [],
|
|
34
|
-
system: [AUTHOR_CONTRACT, `Executable candidates authorized: ${options.allowExecutableCandidates ?? false}.`, `All supplied rubric sections (authoritative): ${JSON.stringify(options.rubricSections ?? {})}`],
|
|
35
|
-
user: `${options.request}\nAvailable input keys: ${JSON.stringify(options.inputKeys ?? [])}`,
|
|
36
|
-
schema: { type: "object", additionalProperties: true },
|
|
37
|
-
async review(value) {
|
|
38
|
-
candidates++;
|
|
39
|
-
if (candidates > maxCandidates)
|
|
40
|
-
throw new Error("Candidate limit exceeded");
|
|
41
|
-
const candidatePath = join(directory, `${String(candidates).padStart(3, "0")}.json`);
|
|
42
|
-
await writeFile(candidatePath, JSON.stringify(value, null, 2), { flag: "wx" });
|
|
43
|
-
const errors = workflowPolicyErrors(value, options);
|
|
44
|
-
if (!errors.length) {
|
|
45
|
-
try {
|
|
46
|
-
const verdict = validateWorkflow(value, { inputKeys: options.inputKeys });
|
|
47
|
-
if (!verdict.ok)
|
|
48
|
-
errors.push(...verdict.errors);
|
|
49
|
-
}
|
|
50
|
-
catch (error) {
|
|
51
|
-
errors.push(`Invalid workflow: ${error instanceof Error ? error.message : String(error)}`);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
if (!errors.length && options.acceptance)
|
|
55
|
-
errors.push(...await options.acceptance(structuredClone(value)));
|
|
56
|
-
await writeFile(join(directory, `${String(candidates).padStart(3, "0")}.review.json`), JSON.stringify({ accepted: !errors.length, checks: options.acceptance ? "structural-and-host" : "structural", errors }, null, 2), { flag: "wx" });
|
|
57
|
-
if (errors.length)
|
|
58
|
-
return { accepted: false, message: errors.join("\n") };
|
|
59
|
-
acceptedPath = candidatePath;
|
|
60
|
-
return { accepted: true };
|
|
61
|
-
},
|
|
62
|
-
});
|
|
63
|
-
if (!acceptedPath)
|
|
64
|
-
throw new Error("Author returned without a retained accepted candidate");
|
|
65
|
-
await writeFile(join(directory, "result.json"), JSON.stringify({ status: "candidate", path: acceptedPath, candidates }, null, 2), { flag: "wx" });
|
|
66
|
-
return { workflow, path: acceptedPath, directory, candidates, checks: options.acceptance ? "structural-and-host" : "structural" };
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
await writeFile(join(directory, "result.json"), JSON.stringify({ status: "failed", candidates, message: error instanceof Error ? error.message : String(error) }, null, 2), { flag: "wx" });
|
|
70
|
-
throw error;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
export type BenchmarkSplit = 'development' | 'heldout';
|
|
2
|
-
export type BenchmarkFinding = {
|
|
3
|
-
topicId: string;
|
|
4
|
-
status: 'supported' | 'conflicting' | 'insufficient';
|
|
5
|
-
summary: string;
|
|
6
|
-
sourceIds: string[];
|
|
7
|
-
};
|
|
8
|
-
export type BenchmarkCase = {
|
|
9
|
-
id: string;
|
|
10
|
-
split: BenchmarkSplit;
|
|
11
|
-
question: string;
|
|
12
|
-
topics: {
|
|
13
|
-
id: string;
|
|
14
|
-
question: string;
|
|
15
|
-
}[];
|
|
16
|
-
sources: {
|
|
17
|
-
id: string;
|
|
18
|
-
text: string;
|
|
19
|
-
}[];
|
|
20
|
-
expected: Omit<BenchmarkFinding, 'summary'>[];
|
|
21
|
-
};
|
|
22
|
-
export declare const benchmarkHash: (value: unknown) => string;
|
|
23
|
-
export declare const benchmarkInstructions = "Answer using only the fictional source corpus. Produce exactly one finding for each supplied topic. Status supported means concrete evidence establishes a finding, including a negative or qualified answer. Conflicting means incompatible accounts. Insufficient means no concrete information answers the question. Preserve conditions and contrary evidence; cite every material source by ID. Do not invent facts or citations. Submit the final structured report.";
|
|
24
|
-
export declare const benchmarkReportSchema: {
|
|
25
|
-
type: string;
|
|
26
|
-
additionalProperties: boolean;
|
|
27
|
-
required: string[];
|
|
28
|
-
properties: {
|
|
29
|
-
findings: {
|
|
30
|
-
type: string;
|
|
31
|
-
minItems: number;
|
|
32
|
-
items: {
|
|
33
|
-
type: string;
|
|
34
|
-
additionalProperties: boolean;
|
|
35
|
-
required: string[];
|
|
36
|
-
properties: {
|
|
37
|
-
topicId: {
|
|
38
|
-
type: string;
|
|
39
|
-
minLength: number;
|
|
40
|
-
};
|
|
41
|
-
status: {
|
|
42
|
-
enum: string[];
|
|
43
|
-
};
|
|
44
|
-
summary: {
|
|
45
|
-
type: string;
|
|
46
|
-
minLength: number;
|
|
47
|
-
};
|
|
48
|
-
sourceIds: {
|
|
49
|
-
type: string;
|
|
50
|
-
uniqueItems: boolean;
|
|
51
|
-
items: {
|
|
52
|
-
type: string;
|
|
53
|
-
minLength: number;
|
|
54
|
-
};
|
|
55
|
-
};
|
|
56
|
-
};
|
|
57
|
-
};
|
|
58
|
-
};
|
|
59
|
-
};
|
|
60
|
-
};
|
|
61
|
-
export declare const benchmarkEvidenceSchema: {
|
|
62
|
-
type: string;
|
|
63
|
-
additionalProperties: boolean;
|
|
64
|
-
required: string[];
|
|
65
|
-
properties: {
|
|
66
|
-
relevant: {
|
|
67
|
-
type: string;
|
|
68
|
-
description: string;
|
|
69
|
-
criteria: {
|
|
70
|
-
true: string;
|
|
71
|
-
false: string;
|
|
72
|
-
};
|
|
73
|
-
};
|
|
74
|
-
};
|
|
75
|
-
};
|
|
76
|
-
export declare function benchmarkCases(split: BenchmarkSplit): BenchmarkCase[];
|
|
77
|
-
/** Frozen independent checks. Never return these results to an authoring agent. */
|
|
78
|
-
export declare function evaluatePiBenchmark(caseId: string, output: unknown): {
|
|
79
|
-
passed: boolean;
|
|
80
|
-
issues: string[];
|
|
81
|
-
};
|
|
82
|
-
export declare function benchmarkContract(): {
|
|
83
|
-
version: number;
|
|
84
|
-
fixturesSha256: string;
|
|
85
|
-
verifierSha256: string;
|
|
86
|
-
outputSchemaSha256: string;
|
|
87
|
-
instructionsSha256: string;
|
|
88
|
-
};
|