coding-artifact-evaluation-shell 0.0.0-stage → 0.1.0-eidolon.a1b37cbcd7c07
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +15 -0
- package/README.md +56 -2
- package/SKILL.md +30 -0
- package/SOP.md +157 -0
- package/adapter/benchmark.ts +28 -0
- package/app/app.ts +15 -0
- package/app/host.ts +30 -0
- package/budget.ts +49 -0
- package/bun.lock +1045 -0
- package/calibrate.ts +81 -0
- package/calibration-history.ts +33 -0
- package/calibration-recheck.ts +74 -0
- package/calibration-report.ts +36 -0
- package/campaign.ts +85 -0
- package/capsule.ts +162 -0
- package/cli.ts +23 -0
- package/coding-scope.ts +13 -0
- package/contract.ts +116 -0
- package/counter-review.ts +74 -0
- package/coverage.ts +74 -0
- package/environment.ts +11 -0
- package/feedback.ts +31 -0
- package/history.ts +11 -0
- package/host.xnl +7 -0
- package/index.ts +6 -0
- package/logic.ts +135 -0
- package/manifest.xnl +10 -0
- package/native-references.ts +21 -0
- package/needs.ts +33 -0
- package/obligation-graph.ts +47 -0
- package/package.json +32 -4
- package/proposal.ts +94 -0
- package/protocol-error.ts +1 -0
- package/replay.ts +39 -0
- package/representation.ts +24 -0
- package/resources/counter-review.md +16 -0
- package/resources/evaluate.md +34 -0
- package/resources/iterate.md +20 -0
- package/resources/review.md +23 -0
- package/resources/submit.md +18 -0
- package/support/agent-runtime.ts +520 -0
- package/support/artifact-inventory.ts +20 -0
- package/support/artifact-links.ts +82 -0
- package/support/browser-channel.ts +209 -0
- package/support/browser-client.ts +33 -0
- package/support/browser-owner.ts +69 -0
- package/support/browser-worker.ts +61 -0
- package/support/campaign-ledger.ts +66 -0
- package/support/code-knowledge.ts +73 -0
- package/support/counter-review.ts +35 -0
- package/support/ego-tools.ts +19 -0
- package/support/evidence-catalog.ts +37 -0
- package/support/evidence-input.ts +70 -0
- package/support/evidence-object.ts +9 -0
- package/support/evidence-read.ts +18 -0
- package/support/execution-profile.ts +65 -0
- package/support/guidance.ts +16 -0
- package/support/history.ts +89 -0
- package/support/http-request.ts +21 -0
- package/support/isolation.ts +94 -0
- package/support/judge.ts +60 -0
- package/support/model-turn.ts +94 -0
- package/support/observation-agent.ts +279 -0
- package/support/observation-protocol.ts +33 -0
- package/support/observers.ts +102 -0
- package/support/process.ts +22 -0
- package/support/proposal-check.ts +82 -0
- package/support/retention.ts +93 -0
- package/support/runtime-binding.ts +39 -0
- package/support/runtime-preflight.ts +16 -0
- package/support/service-evidence.ts +23 -0
- package/support/service-group.ts +45 -0
- package/support/service.ts +84 -0
- package/support/source-identity.ts +22 -0
- package/support/target-group-view.ts +11 -0
- package/target-groups.ts +57 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared native agent adapter; product installation stays in the consumer.
|
|
3
|
+
*
|
|
4
|
+
* The business gates, acceptance boundary, fingerprints and report semantics are
|
|
5
|
+
* agent-agnostic; codex and eidolon differ only in invocation shape, evidence
|
|
6
|
+
* projection, identity/auth preparation and where sessions/skills live.
|
|
7
|
+
*
|
|
8
|
+
* Nothing here relaxes a gate: a runtime that cannot produce the evidence a gate
|
|
9
|
+
* requires must fail that gate, not skip it.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'node:fs';
|
|
12
|
+
import * as os from 'node:os';
|
|
13
|
+
import * as path from 'node:path';
|
|
14
|
+
|
|
15
|
+
export type AgentId = 'codex' | 'eidolon';
|
|
16
|
+
|
|
17
|
+
/** Structural subset of a run; kept local so this module never imports runtime.ts. */
|
|
18
|
+
export interface AgentRun {
|
|
19
|
+
readonly root: string;
|
|
20
|
+
readonly workspace: string;
|
|
21
|
+
readonly controlWorkspace?: string;
|
|
22
|
+
readonly modelScratch?: string;
|
|
23
|
+
readonly execution?:{root:string;network:'denied'|'loopback-only';protectedFiles:string[];derivedDirectories?:string[]};
|
|
24
|
+
readonly sealedModelContexts?:readonly string[];
|
|
25
|
+
readonly home: string;
|
|
26
|
+
readonly env: NodeJS.ProcessEnv;
|
|
27
|
+
readonly readonlyWorkspace?: boolean;
|
|
28
|
+
readonly runtimeReadPaths?: readonly string[];
|
|
29
|
+
readonly runtimeBindings?: {executables:Record<string,string>;readPaths:string[];environment:Record<string,string>};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface Usage { input: number; cached: number; output: number }
|
|
33
|
+
export interface CommandExecution { command: string; argv?: readonly string[]; exitCode: number | null }
|
|
34
|
+
|
|
35
|
+
export interface AgentEvents {
|
|
36
|
+
usage: Usage | null;
|
|
37
|
+
/** Provider counters were estimated/synthesized: never present as measured usage. */
|
|
38
|
+
usageEstimated: boolean;
|
|
39
|
+
/** Output-token counters were absent from the provider projection. */
|
|
40
|
+
outputTokensUnavailable: boolean;
|
|
41
|
+
failed: boolean;
|
|
42
|
+
completed: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* The turn ended because the provider/transport failed, not because the agent
|
|
45
|
+
* produced a wrong answer. A socket drop or a provider-side reset must not be
|
|
46
|
+
* charged against the business correction budget.
|
|
47
|
+
*/
|
|
48
|
+
transportFailure: boolean;
|
|
49
|
+
failureSummary: string | null;
|
|
50
|
+
reconnects: string[];
|
|
51
|
+
threadId?: string;
|
|
52
|
+
commands: string[];
|
|
53
|
+
executions: CommandExecution[];
|
|
54
|
+
/** Every model identity observed in provider evidence for this turn. */
|
|
55
|
+
observedModels: string[];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface AgentModelContext { file: string; model: string | null; effort: string | null }
|
|
59
|
+
|
|
60
|
+
export interface AgentInvocation {
|
|
61
|
+
argv: string[];
|
|
62
|
+
/** Prompt transport; eidolon reads stdin, codex takes it as the final argument. */
|
|
63
|
+
input: string;
|
|
64
|
+
env: NodeJS.ProcessEnv;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface AgentRuntime {
|
|
68
|
+
readonly id: AgentId;
|
|
69
|
+
/** The model reference used to invoke this runtime (may be a provider/model ref). */
|
|
70
|
+
readonly model: string;
|
|
71
|
+
/**
|
|
72
|
+
* Every identity string that legitimately denotes this model in provider
|
|
73
|
+
* evidence. A runtime invoked by a provider/model ref resolves to a bare
|
|
74
|
+
* model id, so both forms are the same model and neither is drift.
|
|
75
|
+
*/
|
|
76
|
+
readonly modelIdentities: readonly string[];
|
|
77
|
+
readonly effort: string | null;
|
|
78
|
+
/**
|
|
79
|
+
* Multiplier applied to the harness's per-turn calibration bounds. Codex
|
|
80
|
+
* turns were measured against the original bounds; a substitute provider with
|
|
81
|
+
* different latency needs a declared, reviewable factor rather than a silent
|
|
82
|
+
* bound change. Gate semantics and attempt counts are unaffected.
|
|
83
|
+
*/
|
|
84
|
+
readonly timeoutScale: number;
|
|
85
|
+
/**
|
|
86
|
+
* Absolute path to the agent executable, resolved in the parent process
|
|
87
|
+
* before any sandbox is applied. The sandboxed PATH is deliberately minimal
|
|
88
|
+
* and cannot be relied on to find an agent installed outside it. The run's
|
|
89
|
+
* own bin dir takes precedence so harness fixtures can substitute it.
|
|
90
|
+
*/
|
|
91
|
+
executable(run: AgentRun): string;
|
|
92
|
+
/** Model clause for prompts. Effort is omitted only when the runtime has no such control. */
|
|
93
|
+
identityGuidance(): string;
|
|
94
|
+
invocation(run: AgentRun, prompt: string, name: string, outputFile: string, log: string, extraArgs: string[]): AgentInvocation;
|
|
95
|
+
/**
|
|
96
|
+
* Where this runtime's machine-readable evidence for a turn actually lands.
|
|
97
|
+
* The process stdout log is harness-owned and lives outside the writable
|
|
98
|
+
* sandbox roots, so a runtime that writes its own trace must write it
|
|
99
|
+
* somewhere the sandbox permits.
|
|
100
|
+
*/
|
|
101
|
+
evidencePath(run: AgentRun, name: string, log: string): string;
|
|
102
|
+
/** Structured-output contract. Empty args: no chat JSON envelope. */
|
|
103
|
+
schemaArgs(schemaPath: string, schemaJson: string): { args: string[]; promptSuffix: string };
|
|
104
|
+
readEvents(log: string): AgentEvents;
|
|
105
|
+
auditModels(run: AgentRun): { contexts: AgentModelContext[]; note: string };
|
|
106
|
+
initArgs(): string[];
|
|
107
|
+
skillRoot(run: AgentRun): string;
|
|
108
|
+
/** Extra read-only roots the sandbox must allow for this agent's own runtime. */
|
|
109
|
+
sandboxReadPaths(): readonly string[];
|
|
110
|
+
prepare(run: AgentRun, authSource: string | undefined): void;
|
|
111
|
+
cleanup(run: AgentRun): void;
|
|
112
|
+
defaultAuth(): string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Resolve an agent binary: a run-local fixture wins, then the known install dir. */
|
|
116
|
+
function resolveExecutable(name: string, run: AgentRun | undefined, directory: string): string {
|
|
117
|
+
if (run) {
|
|
118
|
+
const local = path.join(run.root, 'bin', name);
|
|
119
|
+
if (fs.existsSync(local)) return fs.realpathSync(local);
|
|
120
|
+
}
|
|
121
|
+
const candidate = path.join(directory, name);
|
|
122
|
+
return fs.existsSync(candidate) ? fs.realpathSync(candidate) : name;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Render a value template for a small JSON Schema so a runtime without schema
|
|
127
|
+
* enforcement can be told the exact shape without being shown the schema text
|
|
128
|
+
* itself. Only the keywords the E2E contracts use are interpreted; anything
|
|
129
|
+
* unrecognised yields null, and the shared JSON-validation gate still decides.
|
|
130
|
+
* Arrays render empty (never a placeholder element, which invites filling a
|
|
131
|
+
* list the contract may require to stay empty); their member type is described
|
|
132
|
+
* in prose instead, so nothing inside the template can be echoed as data.
|
|
133
|
+
*/
|
|
134
|
+
export function skeletonFromSchema(schema: unknown): unknown {
|
|
135
|
+
if (!schema || typeof schema !== 'object') return null;
|
|
136
|
+
const node = schema as Record<string, unknown>;
|
|
137
|
+
if (Array.isArray(node.enum)) return node.enum[0] ?? null;
|
|
138
|
+
if (node.type === 'array') return [];
|
|
139
|
+
if (node.type === 'object' || node.properties) {
|
|
140
|
+
const properties = (node.properties ?? {}) as Record<string, unknown>;
|
|
141
|
+
const required = Array.isArray(node.required) ? node.required as string[] : Object.keys(properties);
|
|
142
|
+
return Object.fromEntries(required.filter(key => key in properties).map(key => [key, skeletonFromSchema(properties[key])]));
|
|
143
|
+
}
|
|
144
|
+
if (node.type === 'string') return '';
|
|
145
|
+
if (node.type === 'number' || node.type === 'integer') return 0;
|
|
146
|
+
if (node.type === 'boolean') return false;
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Describe array member types in prose so the template need not contain one. */
|
|
151
|
+
export function arrayMemberNotes(schema: unknown): string[] {
|
|
152
|
+
const notes: string[] = [];
|
|
153
|
+
const visit = (node: unknown, trail: string): void => {
|
|
154
|
+
if (!node || typeof node !== 'object') return;
|
|
155
|
+
const record = node as Record<string, unknown>;
|
|
156
|
+
const properties = (record.properties ?? {}) as Record<string, unknown>;
|
|
157
|
+
for (const [key, value] of Object.entries(properties)) {
|
|
158
|
+
const path = trail ? `${trail}.${key}` : key;
|
|
159
|
+
const child = value as Record<string, unknown> | null;
|
|
160
|
+
if (child && child.type === 'array') {
|
|
161
|
+
const items = child.items as Record<string, unknown> | undefined;
|
|
162
|
+
const memberType = items && typeof items.type === 'string' ? items.type : 'string';
|
|
163
|
+
notes.push(`"${path}" must be an array of ${memberType}s (never objects).`);
|
|
164
|
+
} else visit(child, path);
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
visit(schema, '');
|
|
168
|
+
return notes;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Provider/transport failures reported by an agent runtime. These are
|
|
173
|
+
* infrastructure: the agent never produced a business answer, so they must not
|
|
174
|
+
* consume the correction budget or be fed back as a business defect.
|
|
175
|
+
*/
|
|
176
|
+
const TRANSPORT_FAILURE_PATTERNS = [
|
|
177
|
+
/socket connection was closed/i,
|
|
178
|
+
/socket hang up/i,
|
|
179
|
+
/ECONNRESET|ECONNREFUSED|EPIPE|ETIMEDOUT|ENOTFOUND|EAI_AGAIN/,
|
|
180
|
+
/fetch failed/i,
|
|
181
|
+
/network error/i,
|
|
182
|
+
/rate limit|429|too many requests/i,
|
|
183
|
+
/overloaded|503|502|500\b/,
|
|
184
|
+
/context length|maximum context/i,
|
|
185
|
+
/insufficient balance/i,
|
|
186
|
+
/fetch error 402\b/i,
|
|
187
|
+
/first event exceeded timeout/i,
|
|
188
|
+
/Timeout after \d+ms/,
|
|
189
|
+
/Model not found under provider/i,
|
|
190
|
+
/Unknown model:/i,
|
|
191
|
+
];
|
|
192
|
+
function isTransportFailure(summary: string | null | undefined): boolean {
|
|
193
|
+
if (!summary) return false;
|
|
194
|
+
return TRANSPORT_FAILURE_PATTERNS.some(pattern => pattern.test(summary));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function walkFiles(root: string): string[] {
|
|
198
|
+
if (!fs.existsSync(root)) return [];
|
|
199
|
+
return fs.readdirSync(root, { withFileTypes: true }).flatMap(entry => {
|
|
200
|
+
const file = path.join(root, entry.name);
|
|
201
|
+
if (entry.isSymbolicLink()) throw new Error(`Unexpected symlink: ${file}`);
|
|
202
|
+
return entry.isDirectory() ? walkFiles(file) : [file];
|
|
203
|
+
}).sort();
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Walk a temp/scratch tree where the agent legitimately creates symlinks (a
|
|
208
|
+
* virtualenv's interpreter links, for example). Symlinks are reported as leaves
|
|
209
|
+
* rather than followed, and never raise: this tree is ephemera, not evidence.
|
|
210
|
+
*/
|
|
211
|
+
function walkFilesAllowingSymlinks(root: string): string[] {
|
|
212
|
+
if (!fs.existsSync(root)) return [];
|
|
213
|
+
return fs.readdirSync(root, { withFileTypes: true }).flatMap(entry => {
|
|
214
|
+
const file = path.join(root, entry.name);
|
|
215
|
+
if (entry.isSymbolicLink() || !entry.isDirectory()) return [file];
|
|
216
|
+
return walkFilesAllowingSymlinks(file);
|
|
217
|
+
}).sort();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function readJsonLines(file: string): Record<string, unknown>[] {
|
|
221
|
+
if (!fs.existsSync(file)) return [];
|
|
222
|
+
return fs.readFileSync(file, 'utf8').split('\n').flatMap(line => {
|
|
223
|
+
try {
|
|
224
|
+
const value = JSON.parse(line) as unknown;
|
|
225
|
+
return value && typeof value === 'object' ? [value as Record<string, unknown>] : [];
|
|
226
|
+
} catch { return []; }
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/* ------------------------------------------------------------------ codex */
|
|
231
|
+
|
|
232
|
+
const CODEX_MODEL = 'gpt-5.6-terra';
|
|
233
|
+
|
|
234
|
+
/** Trusted host dependency, never a choice derived from artifact contents. */
|
|
235
|
+
export function createCodexRuntime(effort: 'medium' | 'high' = 'medium', model: 'gpt-5.6-terra' | 'gpt-6.1-sol' = CODEX_MODEL, executable?: string): AgentRuntime {
|
|
236
|
+
if (effort !== 'medium' && effort !== 'high') throw new Error('Unsupported Codex reasoning effort');
|
|
237
|
+
if (model !== 'gpt-5.6-terra' && model !== 'gpt-6.1-sol') throw new Error('Unsupported Codex model');
|
|
238
|
+
if (executable && (!path.isAbsolute(executable) || !fs.statSync(executable).isFile())) throw new Error('Explicit Codex executable must be an absolute file');
|
|
239
|
+
const pinnedExecutable = executable ? fs.realpathSync(executable) : undefined;
|
|
240
|
+
const runtime: AgentRuntime = {
|
|
241
|
+
id: 'codex',
|
|
242
|
+
model,
|
|
243
|
+
modelIdentities: [model],
|
|
244
|
+
effort,
|
|
245
|
+
timeoutScale: 1,
|
|
246
|
+
executable: (run) => pinnedExecutable ?? resolveExecutable('codex', run, path.join(os.homedir(), '.bun/bin')),
|
|
247
|
+
identityGuidance: () => `Every model invocation including child agents must use ${model}, reasoning ${effort}; do not override to other models.`,
|
|
248
|
+
invocation: (run, prompt, _name, outputFile, _log, extraArgs) => ({
|
|
249
|
+
argv: [runtime.executable(run), 'exec', '--ignore-user-config', '--ignore-rules', '--json',
|
|
250
|
+
'-m', model, '-c', `model_reasoning_effort="${effort}"`,
|
|
251
|
+
'-c', 'approval_policy="never"', '-c', 'sandbox_workspace_write.network_access=true',
|
|
252
|
+
'-c', 'sandbox_workspace_write.exclude_slash_tmp=true', '-c', 'sandbox_workspace_write.exclude_tmpdir_env_var=true',
|
|
253
|
+
'-c', 'shell_environment_policy.inherit="all"',
|
|
254
|
+
// Seatbelt is already applied to the whole process tree; macOS rejects reapplying it.
|
|
255
|
+
'--sandbox', 'danger-full-access', '-C', run.controlWorkspace??run.workspace, '--skip-git-repo-check',
|
|
256
|
+
...extraArgs, '-o', outputFile, prompt],
|
|
257
|
+
input: '',
|
|
258
|
+
env: run.env,
|
|
259
|
+
}),
|
|
260
|
+
evidencePath: (_run, _name, log) => log,
|
|
261
|
+
schemaArgs: (schemaPath) => ({ args: ['--output-schema', schemaPath], promptSuffix: '' }),
|
|
262
|
+
readEvents: (log) => {
|
|
263
|
+
const result: AgentEvents = { usage: null, usageEstimated: false, outputTokensUnavailable: false, failed: false, completed: false, transportFailure: false, failureSummary: null, reconnects: [], commands: [], executions: [], observedModels: [] };
|
|
264
|
+
for (const e of readJsonLines(log) as Record<string, any>[]) {
|
|
265
|
+
if (e.type === 'thread.started') result.threadId = e.thread_id;
|
|
266
|
+
if (e.type === 'turn.started') result.completed = false;
|
|
267
|
+
if (e.type === 'turn.failed') result.failed = true;
|
|
268
|
+
if (e.type === 'error') {
|
|
269
|
+
// Codex emits reconnect notifications as error events even when the same
|
|
270
|
+
// turn subsequently completes. Only this observed nonterminal shape is
|
|
271
|
+
// recoverable; unknown errors and terminal failures remain failures.
|
|
272
|
+
if (typeof e.message === 'string' && /^Reconnecting\.\.\. \d+\/\d+ \(/.test(e.message)) result.reconnects.push(e.message);
|
|
273
|
+
else { result.failed = true; result.failureSummary ??= typeof e.message === 'string' ? e.message : null; }
|
|
274
|
+
}
|
|
275
|
+
if (e.type === 'turn.failed' && typeof e.error === 'string') result.failureSummary ??= e.error;
|
|
276
|
+
if (e.type === 'turn.completed') result.completed = true;
|
|
277
|
+
if (e.type === 'turn.completed' && e.usage) {
|
|
278
|
+
result.usage ??= { input: 0, cached: 0, output: 0 };
|
|
279
|
+
result.usage.input += e.usage.input_tokens;
|
|
280
|
+
result.usage.cached += e.usage.cached_input_tokens ?? 0;
|
|
281
|
+
result.usage.output += e.usage.output_tokens;
|
|
282
|
+
}
|
|
283
|
+
if (e.type === 'item.completed' && e.item?.type === 'command_execution') {
|
|
284
|
+
result.commands.push(e.item.command);
|
|
285
|
+
result.executions.push({ command: e.item.command, exitCode: e.item.exit_code ?? null });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
if (!result.completed) result.failed = true;
|
|
289
|
+
result.transportFailure = result.failed && isTransportFailure(result.failureSummary);
|
|
290
|
+
return result;
|
|
291
|
+
},
|
|
292
|
+
auditModels: (run) => {
|
|
293
|
+
const contexts: AgentModelContext[] = [];
|
|
294
|
+
for (const file of walkFiles(path.join(run.home, '.codex/sessions')).filter(f => f.endsWith('.jsonl'))) {
|
|
295
|
+
for (const e of readJsonLines(file) as Record<string, any>[]) {
|
|
296
|
+
if (e.type === 'turn_context') contexts.push({ file, model: e.payload?.model ?? null, effort: e.payload?.effort ?? null });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return { contexts, note: 'codex turn_context events across parent and child sessions' };
|
|
300
|
+
},
|
|
301
|
+
initArgs: () => ['init', '--agent=codex', '--json'],
|
|
302
|
+
skillRoot: (run) => path.join(run.home, '.agents/skills/depa-codument'),
|
|
303
|
+
sandboxReadPaths: () => [],
|
|
304
|
+
prepare: (run, authSource) => {
|
|
305
|
+
// Copy only the auth file, never config, history, plugins or personal instructions.
|
|
306
|
+
if (!authSource) throw new Error('Codex runtime requires an auth file');
|
|
307
|
+
const auth = path.join(run.home, '.codex/auth.json');
|
|
308
|
+
fs.copyFileSync(authSource, auth); fs.chmodSync(auth, 0o600);
|
|
309
|
+
},
|
|
310
|
+
cleanup: (run) => {
|
|
311
|
+
const auth = path.join(run.home, '.codex/auth.json');
|
|
312
|
+
if (fs.existsSync(auth)) fs.unlinkSync(auth);
|
|
313
|
+
},
|
|
314
|
+
defaultAuth: () => path.join(os.homedir(), '.codex/auth.json'),
|
|
315
|
+
};
|
|
316
|
+
return runtime;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const codexRuntime = createCodexRuntime();
|
|
320
|
+
|
|
321
|
+
/* ---------------------------------------------------------------- eidolon */
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Default provider/model for the eidolon runtime. Keep the provider-qualified
|
|
325
|
+
* form: a bare model id is not resolvable. Official DeepSeek is not the
|
|
326
|
+
* default; iqingwa is the authorized substitute when that balance is empty.
|
|
327
|
+
*/
|
|
328
|
+
const EIDOLON_MODEL = process.env.E2E_EIDOLON_MODEL ?? 'deepseek-iqingwa/deepseek-v4.1-flash';
|
|
329
|
+
/**
|
|
330
|
+
* Per-turn settle point handed to eidolon. Chosen above the observed spread of
|
|
331
|
+
* normal implementation turns (median 21.5m, max 44.8m across 21 recorded runs)
|
|
332
|
+
* and around the point where this provider has been observed to drop the socket
|
|
333
|
+
* (35.8m), so an over-long turn becomes resumable rather than a transport
|
|
334
|
+
* failure. Overridable for a measured re-calibration.
|
|
335
|
+
*/
|
|
336
|
+
const EIDOLON_TURN_SETTLE_SECONDS = Number(process.env.E2E_EIDOLON_TURN_SECONDS ?? 2700);
|
|
337
|
+
/** eidolon exec has no reasoning-effort control; effort stays unobserved, not assumed. */
|
|
338
|
+
const EIDOLON_EFFORT: string | null = null;
|
|
339
|
+
|
|
340
|
+
const EXIT_CODE_PATTERN = /Process exited with exit code (-?\d+)\./;
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Only tokenize commands that carry no shell evaluation. Anything with quoting,
|
|
344
|
+
* expansion, redirection or control operators keeps its raw string form so the
|
|
345
|
+
* shared lexical gate decides on exactly what ran.
|
|
346
|
+
*/
|
|
347
|
+
function plainArgv(command: string): readonly string[] | undefined {
|
|
348
|
+
if (!/^[\w./=:@+,-]+(?: [\w./=:@+,-]+)*$/.test(command)) return undefined;
|
|
349
|
+
return command.split(' ');
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const eidolonRuntime: AgentRuntime = {
|
|
353
|
+
id: 'eidolon',
|
|
354
|
+
model: EIDOLON_MODEL,
|
|
355
|
+
modelIdentities: [EIDOLON_MODEL, EIDOLON_MODEL.slice(EIDOLON_MODEL.lastIndexOf('/') + 1)],
|
|
356
|
+
effort: EIDOLON_EFFORT,
|
|
357
|
+
// Observed: a reviewer calibration turn took ~190-230s where codex finished
|
|
358
|
+
// inside 300s; the factor is declared here rather than hidden in a call site.
|
|
359
|
+
timeoutScale: 3,
|
|
360
|
+
executable: (run) => resolveExecutable('eidolon', run, path.join(os.homedir(), '.local/bin')),
|
|
361
|
+
identityGuidance: () => `Every model invocation including child agents must use ${EIDOLON_MODEL}; do not override to other models.`,
|
|
362
|
+
invocation: (run, prompt, name, outputFile, log, extraArgs) => ({
|
|
363
|
+
argv: [eidolonRuntime.executable(run), 'exec', '--json',
|
|
364
|
+
'-m', EIDOLON_MODEL,
|
|
365
|
+
// The outer Seatbelt boundary owns isolation; eidolon must not re-apply its own.
|
|
366
|
+
'--yolo',
|
|
367
|
+
'-C', run.controlWorkspace??run.workspace,
|
|
368
|
+
// Sessions and traces stay outside the delivered workspace so the source
|
|
369
|
+
// fingerprint only ever covers authored material.
|
|
370
|
+
'--ephemeral',
|
|
371
|
+
'--output-trace', eidolonRuntime.evidencePath(run, name, log),
|
|
372
|
+
'-o', outputFile,
|
|
373
|
+
// eidolon's own long-turn machinery: a turn that is interrupted mid-flight
|
|
374
|
+
// is settled as resumable and continued inside this same invocation rather
|
|
375
|
+
// than being reported as a failed turn. This does not grant another
|
|
376
|
+
// attempt, reset a budget or relax a gate - it only lets one authored turn
|
|
377
|
+
// run to completion.
|
|
378
|
+
'--auto-resume',
|
|
379
|
+
// A per-turn settle point below the observed provider disconnect. Measured
|
|
380
|
+
// basis: 21 recorded implementation turns ran a median of 21.5m and at most
|
|
381
|
+
// 44.8m; this provider dropped the socket at 35.8m in one trial. 45 minutes
|
|
382
|
+
// leaves normal turns untouched while giving a long turn a resumable
|
|
383
|
+
// checkpoint instead of a transport failure.
|
|
384
|
+
'--timeout', String(EIDOLON_TURN_SETTLE_SECONDS),
|
|
385
|
+
// A personal MCP server must never load into an isolated trial.
|
|
386
|
+
'-c', 'mcp_servers={}',
|
|
387
|
+
// A fresh session key per turn keeps threads distinct without reuse.
|
|
388
|
+
'-s', name,
|
|
389
|
+
...extraArgs,
|
|
390
|
+
// '-' makes the positional a stdin sentinel; the prompt never lands in argv.
|
|
391
|
+
'-'],
|
|
392
|
+
input: prompt,
|
|
393
|
+
env: { ...run.env, EIDOLON_GLOBAL_DIR: path.join(run.home, '.eidolon') },
|
|
394
|
+
}),
|
|
395
|
+
evidencePath: (run, name) => path.join(run.home, 'tmp', `${name}-trace.jsonl`),
|
|
396
|
+
schemaArgs: () => ({
|
|
397
|
+
// eidolon has no --output-schema. Do not pretend the chat message is JSON:
|
|
398
|
+
// E2E contracts are files and workspace facts, not the final model turn.
|
|
399
|
+
args: [],
|
|
400
|
+
promptSuffix: '',
|
|
401
|
+
}),
|
|
402
|
+
readEvents: (log) => {
|
|
403
|
+
const result: AgentEvents = { usage: null, usageEstimated: false, outputTokensUnavailable: false, failed: false, completed: false, transportFailure: false, failureSummary: null, reconnects: [], commands: [], executions: [], observedModels: [] };
|
|
404
|
+
const started = new Map<string, { command: string; argv?: readonly string[] }>();
|
|
405
|
+
const seenSessions = new Set<string>();
|
|
406
|
+
let status: string | undefined;
|
|
407
|
+
for (const e of readJsonLines(log) as Record<string, any>[]) {
|
|
408
|
+
if (e.kind === 'eidolon.headlessExecResult') {
|
|
409
|
+
status = e.status;
|
|
410
|
+
if (typeof e.timing?.sessionId === 'string') result.threadId = e.timing.sessionId;
|
|
411
|
+
if (typeof e.failureSummary === 'string' && e.failureSummary) result.failureSummary ??= e.failureSummary;
|
|
412
|
+
}
|
|
413
|
+
if (e.type === 'session_start' && typeof e.model === 'string' && e.model) result.observedModels.push(e.model);
|
|
414
|
+
if (e.type === 'history') {
|
|
415
|
+
const summary = e.summary ?? {};
|
|
416
|
+
if (e.stream === 'tool_call_start' && typeof summary.toolCallId === 'string') {
|
|
417
|
+
let command = '';
|
|
418
|
+
try {
|
|
419
|
+
const parsed = JSON.parse(summary.argumentsText ?? '{}') as Record<string, unknown>;
|
|
420
|
+
if (typeof parsed.command === 'string') command = parsed.command;
|
|
421
|
+
} catch { /* non-bash tool call */ }
|
|
422
|
+
started.set(summary.toolCallId, { command, argv: plainArgv(command) });
|
|
423
|
+
}
|
|
424
|
+
if (e.stream === 'tool_call_result' && typeof summary.toolCallId === 'string') {
|
|
425
|
+
const open = started.get(summary.toolCallId);
|
|
426
|
+
started.delete(summary.toolCallId);
|
|
427
|
+
const text = typeof summary.resultText === 'string' ? summary.resultText : '';
|
|
428
|
+
const matched = EXIT_CODE_PATTERN.exec(text);
|
|
429
|
+
// A tool result carries an exit code only when the process itself
|
|
430
|
+
// reported one; otherwise the code stays unknown rather than 0.
|
|
431
|
+
const exitCode = summary.isError === false ? 0 : matched ? Number(matched[1]) : null;
|
|
432
|
+
if (open?.command) {
|
|
433
|
+
result.commands.push(open.command);
|
|
434
|
+
result.executions.push({ command: open.command, ...(open.argv ? { argv: open.argv } : {}), exitCode });
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
if (e.type === 'session_end') {
|
|
439
|
+
if (typeof e.failureSummary === 'string' && e.failureSummary) result.failureSummary ??= e.failureSummary;
|
|
440
|
+
if (typeof e.timing?.sessionId === 'string') { result.threadId ??= e.timing.sessionId; seenSessions.add(e.timing.sessionId); }
|
|
441
|
+
const usage = e.usage ?? {};
|
|
442
|
+
const isEstimated = usage.is_estimated === true;
|
|
443
|
+
const output = Number(usage.completion_tokens ?? 0);
|
|
444
|
+
result.usage ??= { input: 0, cached: 0, output: 0 };
|
|
445
|
+
result.usage.input += Number(usage.prompt_tokens ?? 0);
|
|
446
|
+
result.usage.cached += Number(usage.cache_read_tokens ?? 0);
|
|
447
|
+
result.usage.output += output;
|
|
448
|
+
result.usageEstimated ||= isEstimated;
|
|
449
|
+
if (output === 0) result.outputTokensUnavailable = true;
|
|
450
|
+
for (const observation of e.providerCacheObservations ?? []) {
|
|
451
|
+
const model = observation?.identity?.model;
|
|
452
|
+
if (typeof model === 'string' && model) result.observedModels.push(model);
|
|
453
|
+
}
|
|
454
|
+
if (status === undefined) status = e.status;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
result.completed = status === 'completed';
|
|
458
|
+
if (status === 'paused_with_progress') result.failed = true;
|
|
459
|
+
if (!result.completed) result.failed = true;
|
|
460
|
+
if (!result.usage) result.usage = null;
|
|
461
|
+
result.transportFailure = result.failed && isTransportFailure(result.failureSummary);
|
|
462
|
+
return result;
|
|
463
|
+
},
|
|
464
|
+
auditModels: (run) => {
|
|
465
|
+
// eidolon has no per-turn context file; provider evidence in each exec trace
|
|
466
|
+
// is the observable identity source. Effort is not exposed by this runtime.
|
|
467
|
+
const contexts: AgentModelContext[] = [];
|
|
468
|
+
// Traces live under home/tmp because the agent process itself writes them
|
|
469
|
+
// inside the sandbox; logs/ holds harness-owned stdout only. That temp root
|
|
470
|
+
// also holds agent-created scratch (e.g. reviewer virtualenvs), so the scan
|
|
471
|
+
// reads regular files and skips symlinked runtime artifacts rather than
|
|
472
|
+
// treating them as evidence.
|
|
473
|
+
for (const file of walkFilesAllowingSymlinks(path.join(run.home, 'tmp'))) {
|
|
474
|
+
if (!/-trace\.jsonl$/.test(file)) continue;
|
|
475
|
+
const models = new Set<string>();
|
|
476
|
+
for (const e of readJsonLines(file) as Record<string, any>[]) {
|
|
477
|
+
if (e.type === 'session_start' && typeof e.model === 'string' && e.model) models.add(e.model);
|
|
478
|
+
if (e.type === 'session_end') {
|
|
479
|
+
for (const observation of e.providerCacheObservations ?? []) {
|
|
480
|
+
const model = observation?.identity?.model;
|
|
481
|
+
if (typeof model === 'string' && model) models.add(model);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
for (const model of models) contexts.push({ file, model, effort: null });
|
|
486
|
+
}
|
|
487
|
+
return { contexts, note: 'eidolon provider-call identities per exec trace; reasoning effort is not observable in this runtime and is not asserted' };
|
|
488
|
+
},
|
|
489
|
+
initArgs: () => ['init', '--agent=eidolon', '--json'],
|
|
490
|
+
skillRoot: (run) => path.join(run.home, '.eidolon/skills/depa-codument'),
|
|
491
|
+
sandboxReadPaths: () => [path.join(os.homedir(), 'ai/eidolon/eidolon-anchor/dist')],
|
|
492
|
+
prepare: (run, authSource) => {
|
|
493
|
+
// Copy only the credential catalog and preset selection, never mcp servers,
|
|
494
|
+
// sessions, skills, projects or personal instructions.
|
|
495
|
+
const target = path.join(run.home, '.eidolon');
|
|
496
|
+
fs.mkdirSync(target, { recursive: true });
|
|
497
|
+
for (const name of ['llm-provider.json', 'agent-present.json']) {
|
|
498
|
+
const source = path.join(authSource ?? path.join(os.homedir(), '.eidolon'), name);
|
|
499
|
+
if (!fs.existsSync(source)) throw new Error(`eidolon runtime requires ${source}`);
|
|
500
|
+
const destination = path.join(target, name);
|
|
501
|
+
fs.copyFileSync(source, destination); fs.chmodSync(destination, 0o600);
|
|
502
|
+
}
|
|
503
|
+
},
|
|
504
|
+
cleanup: (run) => {
|
|
505
|
+
for (const name of ['llm-provider.json', 'agent-present.json']) {
|
|
506
|
+
const file = path.join(run.home, '.eidolon', name);
|
|
507
|
+
if (fs.existsSync(file)) fs.unlinkSync(file);
|
|
508
|
+
}
|
|
509
|
+
},
|
|
510
|
+
defaultAuth: () => path.join(os.homedir(), '.eidolon'),
|
|
511
|
+
};
|
|
512
|
+
|
|
513
|
+
const RUNTIMES: Record<AgentId, AgentRuntime> = { codex: codexRuntime, eidolon: eidolonRuntime };
|
|
514
|
+
|
|
515
|
+
export function resolveAgentRuntime(id: string | undefined): AgentRuntime {
|
|
516
|
+
const key = (id ?? 'codex') as AgentId;
|
|
517
|
+
const runtime = RUNTIMES[key];
|
|
518
|
+
if (!runtime) throw new Error(`Unknown E2E agent runtime: ${id}. Supported: ${Object.keys(RUNTIMES).join(', ')}`);
|
|
519
|
+
return runtime;
|
|
520
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import {createHash} from 'node:crypto';
|
|
4
|
+
|
|
5
|
+
/** Discovery projection only. Never follows links or claims excluded paths are absent. */
|
|
6
|
+
export function artifactInventory(root:string) {
|
|
7
|
+
const canonical=fs.realpathSync(root),excludedNames=['.git','node_modules','.tmp','coverage'];
|
|
8
|
+
const entries:{path:string;absolutePath:string;kind:'file'|'directory'|'link'|'other';bytes:number;digest?:string}[]=[],excluded:string[]=[];
|
|
9
|
+
const visit=(relative:string)=>{
|
|
10
|
+
for(const name of fs.readdirSync(path.join(canonical,relative)).sort()){
|
|
11
|
+
const item=path.join(relative,name),absolutePath=path.join(canonical,item),s=fs.lstatSync(absolutePath);
|
|
12
|
+
if(s.isDirectory()&&excludedNames.includes(name)){excluded.push(item);continue;}
|
|
13
|
+
const kind=s.isSymbolicLink()?'link':s.isDirectory()?'directory':s.isFile()?'file':'other';
|
|
14
|
+
entries.push({path:item,absolutePath,kind,bytes:s.size,...(kind==='file'?{digest:createHash('sha256').update(fs.readFileSync(absolutePath)).digest('hex')}: {})});
|
|
15
|
+
if(kind==='directory')visit(item);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
visit('');
|
|
19
|
+
return {root:canonical,entries,excluded,excludedNames,semanticAuthority:false,limitations:['Metadata is discovery only; use native file observations for static evidence','Links are not traversed; excluded directories are unknown, not absent','Inventory never changes the strict artifact fingerprint']};
|
|
20
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
|
|
4
|
+
export interface ArtifactFileLink {
|
|
5
|
+
target: string;
|
|
6
|
+
resolvedTarget: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/** Only closed, internal file aliases are execution capabilities. Never follow directory aliases. */
|
|
10
|
+
export function artifactFileLink(root: string, file: string): ArtifactFileLink {
|
|
11
|
+
const base = fs.realpathSync(root);
|
|
12
|
+
const lexical = path.resolve(path.dirname(file), fs.readlinkSync(file));
|
|
13
|
+
let target: string;
|
|
14
|
+
try {
|
|
15
|
+
target = path.join(fs.realpathSync(path.dirname(lexical)), path.basename(lexical));
|
|
16
|
+
} catch {
|
|
17
|
+
throw Error(`Artifact symlink is dangling or cyclic: ${file}`);
|
|
18
|
+
}
|
|
19
|
+
if (!target.startsWith(base + path.sep)) throw Error(`Artifact symlink escapes source: ${file}`);
|
|
20
|
+
let resolved: string;
|
|
21
|
+
try {
|
|
22
|
+
resolved = fs.realpathSync(file);
|
|
23
|
+
} catch {
|
|
24
|
+
throw Error(`Artifact symlink is dangling or cyclic: ${file}`);
|
|
25
|
+
}
|
|
26
|
+
if (!resolved.startsWith(base + path.sep)) throw Error(`Artifact symlink escapes source: ${file}`);
|
|
27
|
+
if (!fs.statSync(resolved).isFile()) throw Error(`Artifact directory symlink rejected: ${file}`);
|
|
28
|
+
return {target: path.relative(base, target), resolvedTarget: path.relative(base, resolved)};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Fresh independent copy preserving executable relative-import semantics and inode isolation. */
|
|
32
|
+
export function copyArtifact(input: {source: string; destination: string; excludedNames?: string[]}): {preservedLinks: {path: string; target: string}[]} {
|
|
33
|
+
const source = fs.realpathSync(input.source);
|
|
34
|
+
const destination = path.resolve(input.destination);
|
|
35
|
+
if (!path.isAbsolute(input.destination) || destination !== input.destination) throw Error('Artifact destination must be absolute without traversal');
|
|
36
|
+
if (fs.existsSync(destination) || fs.lstatSync(source).isSymbolicLink()) throw Error('Never overwrite a frozen delivery');
|
|
37
|
+
let parent = path.dirname(destination);
|
|
38
|
+
while (!fs.existsSync(parent)) parent = path.dirname(parent);
|
|
39
|
+
if (fs.realpathSync(parent) !== parent) throw Error('Artifact destination ancestor symlink rejected');
|
|
40
|
+
if (destination.startsWith(source + path.sep)) throw Error('Artifact destination must be disjoint from source');
|
|
41
|
+
|
|
42
|
+
const excluded = new Set(input.excludedNames ?? []);
|
|
43
|
+
const entries: {file: string; relative: string; kind: 'directory' | 'file' | 'link'; mode: number; link?: ArtifactFileLink}[] = [];
|
|
44
|
+
function inspect(directory: string): void {
|
|
45
|
+
for (const name of fs.readdirSync(directory).sort()) {
|
|
46
|
+
if (excluded.has(name)) continue;
|
|
47
|
+
const file = path.join(directory, name);
|
|
48
|
+
const stat = fs.lstatSync(file);
|
|
49
|
+
const relative = path.relative(source, file);
|
|
50
|
+
if (stat.isSymbolicLink()) entries.push({file, relative, kind: 'link', mode: stat.mode, link: artifactFileLink(source, file)});
|
|
51
|
+
else if (stat.isDirectory()) {
|
|
52
|
+
entries.push({file, relative, kind: 'directory', mode: stat.mode});
|
|
53
|
+
inspect(file);
|
|
54
|
+
} else if (stat.isFile()) entries.push({file, relative, kind: 'file', mode: stat.mode});
|
|
55
|
+
else throw Error(`Unsupported artifact file type: ${file}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
inspect(source); // Validate the complete boundary before creating any output.
|
|
59
|
+
const included = new Set(entries.map(entry => entry.relative));
|
|
60
|
+
for (const entry of entries) if (entry.link && (!included.has(entry.link.target) || !included.has(entry.link.resolvedTarget))) {
|
|
61
|
+
throw Error(`Artifact symlink target was excluded: ${entry.file}`);
|
|
62
|
+
}
|
|
63
|
+
fs.mkdirSync(destination, {recursive: true, mode: 0o700});
|
|
64
|
+
for (const entry of entries) {
|
|
65
|
+
const output = path.join(destination, entry.relative);
|
|
66
|
+
if (entry.kind === 'directory') fs.mkdirSync(output, {mode: 0o700});
|
|
67
|
+
else if (entry.kind === 'file') {
|
|
68
|
+
fs.copyFileSync(entry.file, output);
|
|
69
|
+
fs.chmodSync(output, entry.mode & 0o777);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const preservedLinks = entries.filter(entry => entry.link).map(entry => {
|
|
73
|
+
const output = path.join(destination, entry.relative);
|
|
74
|
+
const target = path.join(destination, entry.link!.target);
|
|
75
|
+
fs.symlinkSync(path.relative(path.dirname(output), target), output);
|
|
76
|
+
return {path: entry.relative, target: entry.link!.target};
|
|
77
|
+
});
|
|
78
|
+
for (const entry of [...entries].reverse()) if (entry.kind === 'directory') {
|
|
79
|
+
fs.chmodSync(path.join(destination, entry.relative), entry.mode & 0o777);
|
|
80
|
+
}
|
|
81
|
+
return {preservedLinks};
|
|
82
|
+
}
|