agent-apprenticeship 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -197
- package/bin/agent-apprenticeship.js +85 -37
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/agent_apprenticeship_trace/__init__.py +1 -1
- package/src/agent_apprenticeship_trace/actual_outputs_normalizer.py +104 -28
- package/src/agent_apprenticeship_trace/apprentice_adapters.py +3 -1
- package/src/agent_apprenticeship_trace/artifact_resolver.py +6 -0
- package/src/agent_apprenticeship_trace/bundle_exporter.py +132 -2
- package/src/agent_apprenticeship_trace/cli.py +1229 -269
- package/src/agent_apprenticeship_trace/codex_runner.py +89 -6
- package/src/agent_apprenticeship_trace/config.py +251 -32
- package/src/agent_apprenticeship_trace/ecosystem_r2.py +515 -0
- package/src/agent_apprenticeship_trace/env.py +5 -5
- package/src/agent_apprenticeship_trace/evaluator.py +127 -11
- package/src/agent_apprenticeship_trace/grader.py +12 -11
- package/src/agent_apprenticeship_trace/learning.py +270 -1
- package/src/agent_apprenticeship_trace/llm_output_normalizer.py +8 -7
- package/src/agent_apprenticeship_trace/loop.py +103 -4
- package/src/agent_apprenticeship_trace/loop_review.py +686 -0
- package/src/agent_apprenticeship_trace/mentor_checkpoints.py +49 -21
- package/src/agent_apprenticeship_trace/openai_structured.py +84 -14
- package/src/agent_apprenticeship_trace/package_exporter.py +1 -1
- package/src/agent_apprenticeship_trace/progress.py +2 -1
- package/src/agent_apprenticeship_trace/public_run.py +369 -57
- package/src/agent_apprenticeship_trace/public_sanitizer.py +40 -3
- package/src/agent_apprenticeship_trace/rubric_generation.py +11 -10
- package/src/agent_apprenticeship_trace/schemas.py +378 -3
- package/src/agent_apprenticeship_trace/task_intake.py +43 -9
- package/src/agent_apprenticeship_trace/tdo.py +2304 -0
- package/src/agent_apprenticeship_trace/trace_normalizer.py +351 -17
- package/src/agent_apprenticeship_trace/trace_prompt.py +38 -11
- package/src/agent_apprenticeship_trace/training_signals.py +1 -1
- package/src/agent_apprenticeship_trace/verifier.py +12 -6
package/README.md
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
# Agent Apprenticeship
|
|
2
|
-
|
|
3
1
|
The living ecosystem where AI agents learn from real-world work through iterative workflow loops, reusable experience, and collective training signal exchange.
|
|
4
2
|
|
|
5
3
|
As agents move into long-horizon, economically valuable work, Agent Apprenticeship creates the open infrastructure where real-world tasks generate reusable learning signals and complex workflows advance through agent loops that turn execution into shared improvement.
|
|
@@ -11,198 +9,3 @@ Agent Apprenticeship is built for iterative workflow loops across domains, from
|
|
|
11
9
|
Agent Apprenticeship is now available for anyone to start using with local agents including Codex, Cursor, Claude Code, OpenClaw, OpenCode, Hermes Agent, and custom agents, alongside different model providers. Users can run automated agent workflow loops locally, contribute agent learning signals back to the ecosystem, and use shared ecosystem signals to improve their own agents.
|
|
12
10
|
|
|
13
11
|
Agent Apprenticeship is about the future of work and the economic value of agents. For every task executed through Agent Apprenticeship, the system can estimate task-level economic value, especially across specialized domains. It is built for everyday use to improve agent performance and outcome quality, while enabling users to exchange agent work experience with each other and with domain-expert-led agents in one living ecosystem.
|
|
14
|
-
|
|
15
|
-
## Install
|
|
16
|
-
|
|
17
|
-
```bash
|
|
18
|
-
npx agent-apprenticeship init
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
```bash
|
|
22
|
-
npm install -g agent-apprenticeship
|
|
23
|
-
apprentice init
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
The installed command is:
|
|
27
|
-
|
|
28
|
-
```bash
|
|
29
|
-
apprentice
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
The long-form command also remains available:
|
|
33
|
-
|
|
34
|
-
```bash
|
|
35
|
-
agent-apprenticeship
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
## Quickstart
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
apprentice init
|
|
42
|
-
apprentice settings
|
|
43
|
-
apprentice run "Create a short market map for AI procurement tools."
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
Runs print the artifacts path and Contribution Bundle path.
|
|
47
|
-
|
|
48
|
-
```bash
|
|
49
|
-
apprentice ecosystem contribute <bundle_path>
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
Public ecosystem:
|
|
53
|
-
|
|
54
|
-
https://github.com/Forsy-AI/agent-apprenticeship
|
|
55
|
-
|
|
56
|
-
## Apprentice Agents
|
|
57
|
-
|
|
58
|
-
Selected v0 Apprentice Agents:
|
|
59
|
-
|
|
60
|
-
* Codex
|
|
61
|
-
* Cursor
|
|
62
|
-
* Claude Code
|
|
63
|
-
* OpenClaw
|
|
64
|
-
* OpenCode
|
|
65
|
-
* Hermes Agent
|
|
66
|
-
* Custom
|
|
67
|
-
|
|
68
|
-
Agent Apprenticeship auto-detects installed CLIs. If multiple are detected, choose one during setup.
|
|
69
|
-
|
|
70
|
-
Custom lets you provide a command template:
|
|
71
|
-
|
|
72
|
-
```bash
|
|
73
|
-
apprentice configure agent custom --command-template "my-agent run --workspace {workspace} --prompt-file {prompt_file}"
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
## Mentor Model Providers
|
|
77
|
-
|
|
78
|
-
Store local keys in:
|
|
79
|
-
|
|
80
|
-
```text
|
|
81
|
-
~/.agent-apprenticeship/.env.local
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
Example:
|
|
85
|
-
|
|
86
|
-
```bash
|
|
87
|
-
OPENAI_API_KEY=""
|
|
88
|
-
ANTHROPIC_API_KEY=""
|
|
89
|
-
GEMINI_API_KEY=""
|
|
90
|
-
OPENROUTER_API_KEY=""
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
Configure:
|
|
94
|
-
|
|
95
|
-
```bash
|
|
96
|
-
apprentice configure model
|
|
97
|
-
apprentice doctor
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
## Mentor Modes
|
|
101
|
-
|
|
102
|
-
```bash
|
|
103
|
-
apprentice run "..." --mentor-mode model-assisted
|
|
104
|
-
apprentice run "..." --mentor-mode expert-led
|
|
105
|
-
apprentice run "..." --mentor-mode hybrid
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
* `model-assisted`: Mentor Model Provider handles the mentor loop.
|
|
109
|
-
* `expert-led`: human expert checkpoints guide the mentor loop.
|
|
110
|
-
* `hybrid`: Mentor Model Provider drafts and human expert checkpoints approve or edit.
|
|
111
|
-
|
|
112
|
-
## Seed Dataset
|
|
113
|
-
|
|
114
|
-
The seed dataset is included under:
|
|
115
|
-
|
|
116
|
-
```text
|
|
117
|
-
seed_dataset/
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
Explore seed tasks:
|
|
121
|
-
|
|
122
|
-
```bash
|
|
123
|
-
apprentice ecosystem list
|
|
124
|
-
apprentice ecosystem search cloud
|
|
125
|
-
apprentice ecosystem inspect aa-seed-task-501
|
|
126
|
-
apprentice ecosystem pull aa-seed-task-501
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
## Ecosystem Learning
|
|
130
|
-
|
|
131
|
-
Create Experience Packs from ecosystem experience:
|
|
132
|
-
|
|
133
|
-
```bash
|
|
134
|
-
apprentice learn create aa-seed-task-501
|
|
135
|
-
apprentice learn preview <pack_id>
|
|
136
|
-
apprentice learn replay <pack_id>
|
|
137
|
-
apprentice learn keep <pack_id>
|
|
138
|
-
apprentice run "Create a related incident response checklist." --experience-pack <pack_id>
|
|
139
|
-
apprentice learn revert <pack_id>
|
|
140
|
-
```
|
|
141
|
-
|
|
142
|
-
Use active packs explicitly:
|
|
143
|
-
|
|
144
|
-
```bash
|
|
145
|
-
apprentice run "..." --use-active-experience-packs
|
|
146
|
-
apprentice run "..." --no-experience-packs
|
|
147
|
-
```
|
|
148
|
-
|
|
149
|
-
## Contribution Bundles
|
|
150
|
-
|
|
151
|
-
Runs produce Contribution Bundles.
|
|
152
|
-
|
|
153
|
-
Contribute one to the public ecosystem:
|
|
154
|
-
|
|
155
|
-
```bash
|
|
156
|
-
apprentice ecosystem contribute <bundle_path>
|
|
157
|
-
apprentice bundle contribute <bundle_path>
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
Public ecosystem:
|
|
161
|
-
|
|
162
|
-
https://github.com/Forsy-AI/agent-apprenticeship
|
|
163
|
-
|
|
164
|
-
## Ecosystem Auto-Share
|
|
165
|
-
|
|
166
|
-
Default mode is Manual.
|
|
167
|
-
|
|
168
|
-
```bash
|
|
169
|
-
apprentice ecosystem configure --repo Forsy-AI/agent-apprenticeship
|
|
170
|
-
apprentice ecosystem configure --auto-share manual
|
|
171
|
-
apprentice ecosystem configure --auto-share ask
|
|
172
|
-
apprentice ecosystem configure --auto-share automatic
|
|
173
|
-
apprentice ecosystem status
|
|
174
|
-
```
|
|
175
|
-
|
|
176
|
-
Requirements:
|
|
177
|
-
|
|
178
|
-
* GitHub CLI installed
|
|
179
|
-
* `gh` authenticated
|
|
180
|
-
* ecosystem repo configured
|
|
181
|
-
|
|
182
|
-
## Search, Inspect, Pull
|
|
183
|
-
|
|
184
|
-
Discover and export ecosystem experience:
|
|
185
|
-
|
|
186
|
-
```bash
|
|
187
|
-
apprentice ecosystem search <query>
|
|
188
|
-
apprentice ecosystem inspect <id>
|
|
189
|
-
apprentice ecosystem pull <id>
|
|
190
|
-
```
|
|
191
|
-
|
|
192
|
-
## Public Repo Structure
|
|
193
|
-
|
|
194
|
-
```text
|
|
195
|
-
seed_dataset/
|
|
196
|
-
ecosystem/
|
|
197
|
-
ecosystem/contributions/
|
|
198
|
-
schemas/
|
|
199
|
-
examples/
|
|
200
|
-
```
|
|
201
|
-
|
|
202
|
-
## Development Commands
|
|
203
|
-
|
|
204
|
-
```bash
|
|
205
|
-
.venv/bin/python -m pytest -q tests
|
|
206
|
-
PYTHONPATH=src .venv/bin/python -m compileall -q src tests scripts examples
|
|
207
|
-
bash scripts/export_public_repo.sh
|
|
208
|
-
```
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
const fs = require("fs");
|
|
5
5
|
const os = require("os");
|
|
6
6
|
const path = require("path");
|
|
7
|
-
const { spawnSync } = require("child_process");
|
|
7
|
+
const { spawn, spawnSync } = require("child_process");
|
|
8
8
|
|
|
9
9
|
const packageRoot = path.resolve(__dirname, "..");
|
|
10
10
|
const packageJson = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"));
|
|
@@ -103,26 +103,69 @@ function setupTimeout(defaultMs) {
|
|
|
103
103
|
return Number.isFinite(value) && value > 0 ? value : defaultMs;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
function appendLimited(current, chunk, limit = 16000) {
|
|
107
|
+
const next = current + chunk.toString();
|
|
108
|
+
return next.length > limit ? next.slice(next.length - limit) : next;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function setupHeartbeatMs(defaultMs) {
|
|
112
|
+
const raw = process.env.AA_NPM_SETUP_HEARTBEAT_MS;
|
|
113
|
+
if (!raw) return defaultMs;
|
|
114
|
+
const value = Number(raw);
|
|
115
|
+
return Number.isFinite(value) && value > 0 ? value : defaultMs;
|
|
116
|
+
}
|
|
117
|
+
|
|
106
118
|
function runQuiet(command, args, label, timeoutMs) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
119
|
+
return new Promise((resolve) => {
|
|
120
|
+
let stdout = "";
|
|
121
|
+
let stderr = "";
|
|
122
|
+
let timedOut = false;
|
|
123
|
+
let settled = false;
|
|
124
|
+
const child = spawn(command, args, {
|
|
125
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
126
|
+
env: runtimeEnv()
|
|
127
|
+
});
|
|
128
|
+
const heartbeat = setInterval(() => {
|
|
129
|
+
console.error(`Still working: ${label}...`);
|
|
130
|
+
}, setupHeartbeatMs(30000));
|
|
131
|
+
const timeout = setTimeout(() => {
|
|
132
|
+
timedOut = true;
|
|
133
|
+
child.kill("SIGTERM");
|
|
134
|
+
setTimeout(() => {
|
|
135
|
+
if (!settled) child.kill("SIGKILL");
|
|
136
|
+
}, 3000).unref();
|
|
137
|
+
}, setupTimeout(timeoutMs));
|
|
138
|
+
child.stdout.on("data", (chunk) => {
|
|
139
|
+
stdout = appendLimited(stdout, chunk);
|
|
140
|
+
});
|
|
141
|
+
child.stderr.on("data", (chunk) => {
|
|
142
|
+
stderr = appendLimited(stderr, chunk);
|
|
143
|
+
});
|
|
144
|
+
child.on("error", (error) => {
|
|
145
|
+
settled = true;
|
|
146
|
+
clearInterval(heartbeat);
|
|
147
|
+
clearTimeout(timeout);
|
|
148
|
+
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
149
|
+
console.error(String(error.message || error));
|
|
150
|
+
process.exit(1);
|
|
151
|
+
});
|
|
152
|
+
child.on("close", (code, signal) => {
|
|
153
|
+
settled = true;
|
|
154
|
+
clearInterval(heartbeat);
|
|
155
|
+
clearTimeout(timeout);
|
|
156
|
+
if (code === 0) {
|
|
157
|
+
resolve();
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
161
|
+
if (timedOut || signal) {
|
|
162
|
+
console.error("Runtime setup timed out. Re-run the command, or set AA_PYTHON to a working Python 3.11+ interpreter.");
|
|
163
|
+
}
|
|
164
|
+
const out = [stdout, stderr].filter(Boolean).join("\n").trim();
|
|
165
|
+
if (out) console.error(out.slice(-8000));
|
|
166
|
+
process.exit(code || 1);
|
|
167
|
+
});
|
|
112
168
|
});
|
|
113
|
-
if (result.error || result.status !== 0) {
|
|
114
|
-
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
115
|
-
const timedOut = Boolean(result.signal) || (result.error && String(result.error.message || result.error).includes("ETIMEDOUT"));
|
|
116
|
-
if (timedOut) {
|
|
117
|
-
console.error("Runtime setup timed out. Re-run the command, or set AA_PYTHON to a working Python 3.11+ interpreter.");
|
|
118
|
-
} else if (result.error) {
|
|
119
|
-
const message = String(result.error.message || result.error);
|
|
120
|
-
console.error(message);
|
|
121
|
-
}
|
|
122
|
-
const out = [result.stdout, result.stderr].filter(Boolean).join("\n").trim();
|
|
123
|
-
if (out) console.error(out.slice(-8000));
|
|
124
|
-
process.exit(result.status || 1);
|
|
125
|
-
}
|
|
126
169
|
}
|
|
127
170
|
|
|
128
171
|
function runVisible(command, args, label) {
|
|
@@ -137,7 +180,7 @@ function runVisible(command, args, label) {
|
|
|
137
180
|
}
|
|
138
181
|
}
|
|
139
182
|
|
|
140
|
-
function ensureRuntime(python) {
|
|
183
|
+
async function ensureRuntime(python) {
|
|
141
184
|
const venvDir = path.resolve(process.env.AA_NPM_VENV || path.join(appHome(), "npm-venv", packageJson.version));
|
|
142
185
|
const py = venvPython(venvDir);
|
|
143
186
|
const markerPath = path.join(venvDir, ".agent-apprenticeship-npm.json");
|
|
@@ -154,8 +197,8 @@ function ensureRuntime(python) {
|
|
|
154
197
|
console.error("First run can take a few minutes while Python dependencies are installed.");
|
|
155
198
|
fs.rmSync(venvDir, { recursive: true, force: true });
|
|
156
199
|
fs.mkdirSync(path.dirname(venvDir), { recursive: true });
|
|
157
|
-
runQuiet(python, ["-m", "venv", venvDir], "create the Python environment", 120000);
|
|
158
|
-
runQuiet(py, [
|
|
200
|
+
await runQuiet(python, ["-m", "venv", venvDir], "create the Python environment", 120000);
|
|
201
|
+
await runQuiet(py, [
|
|
159
202
|
"-m", "pip", "install",
|
|
160
203
|
"--disable-pip-version-check",
|
|
161
204
|
"--progress-bar", "off",
|
|
@@ -191,20 +234,25 @@ function runCli(python, args, usePackagedSource) {
|
|
|
191
234
|
process.exit(result.status === null ? 1 : result.status);
|
|
192
235
|
}
|
|
193
236
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
237
|
+
(async () => {
|
|
238
|
+
const python = findPython();
|
|
239
|
+
if (!python) {
|
|
240
|
+
console.error("Agent Apprenticeship requires Python 3.11 or newer.");
|
|
241
|
+
console.error("");
|
|
242
|
+
console.error("macOS Homebrew:");
|
|
243
|
+
console.error("brew install python@3.11");
|
|
244
|
+
console.error("");
|
|
245
|
+
console.error("Or set:");
|
|
246
|
+
console.error("AA_PYTHON=/path/to/python3.11");
|
|
247
|
+
process.exit(1);
|
|
248
|
+
}
|
|
205
249
|
|
|
206
|
-
if (process.env.AA_NPM_USE_SYSTEM_PYTHON === "1") {
|
|
207
|
-
|
|
208
|
-
}
|
|
250
|
+
if (process.env.AA_NPM_USE_SYSTEM_PYTHON === "1") {
|
|
251
|
+
runCli(python, process.argv.slice(2), true);
|
|
252
|
+
}
|
|
209
253
|
|
|
210
|
-
runCli(ensureRuntime(python), process.argv.slice(2), false);
|
|
254
|
+
runCli(await ensureRuntime(python), process.argv.slice(2), false);
|
|
255
|
+
})().catch((error) => {
|
|
256
|
+
console.error(`Failed to start Agent Apprenticeship: ${error.message || error}`);
|
|
257
|
+
process.exit(1);
|
|
258
|
+
});
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agent-apprenticeship"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.7"
|
|
8
8
|
description = "Open framework for turning real agent work into transferable agent experience"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""Agent Apprenticeship framework."""
|
|
2
|
-
__version__ = "0.1.
|
|
2
|
+
__version__ = "0.1.7"
|
|
@@ -5,6 +5,7 @@ from typing import Any
|
|
|
5
5
|
from pydantic import BaseModel, Field
|
|
6
6
|
from .schemas import ActualOutputs
|
|
7
7
|
from .io import write_json
|
|
8
|
+
from .openai_structured import extract_json_object
|
|
8
9
|
|
|
9
10
|
CANONICAL_ACTUAL_FIELDS = set(ActualOutputs.model_fields.keys())
|
|
10
11
|
|
|
@@ -72,7 +73,15 @@ def _validation_errors(exc: Exception) -> list[str]:
|
|
|
72
73
|
|
|
73
74
|
|
|
74
75
|
def _norm_ref(ref: str, attempt_kind: str) -> str:
|
|
75
|
-
ref=str(ref)
|
|
76
|
+
ref=str(ref).replace("\\", "/").strip().lstrip("/")
|
|
77
|
+
while ref.startswith("./"):
|
|
78
|
+
ref = ref[2:]
|
|
79
|
+
if not ref:
|
|
80
|
+
return ref
|
|
81
|
+
if ref in {"actual_outputs.json", "agent_trace.json", "stdout.txt", "stderr.txt", "final_message.txt"}:
|
|
82
|
+
return f"attempts/{attempt_kind}/{ref}"
|
|
83
|
+
if ref.startswith("input/") or ref.startswith("task/"):
|
|
84
|
+
return ref
|
|
76
85
|
if ref.startswith(f'attempts/{attempt_kind}/'):
|
|
77
86
|
return ref
|
|
78
87
|
if ref.startswith('artifacts/'):
|
|
@@ -82,11 +91,50 @@ def _norm_ref(ref: str, attempt_kind: str) -> str:
|
|
|
82
91
|
return ref
|
|
83
92
|
|
|
84
93
|
|
|
94
|
+
def _is_input_ref(ref: str) -> bool:
|
|
95
|
+
return ref.startswith("input/") or ref.startswith("task/") or "/input/" in ref or "/task/" in ref
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _is_attempt_metadata_ref(ref: str, attempt_kind: str) -> bool:
|
|
99
|
+
return ref in {
|
|
100
|
+
f"attempts/{attempt_kind}/actual_outputs.json",
|
|
101
|
+
f"attempts/{attempt_kind}/agent_trace.json",
|
|
102
|
+
f"attempts/{attempt_kind}/stdout.txt",
|
|
103
|
+
f"attempts/{attempt_kind}/stderr.txt",
|
|
104
|
+
f"attempts/{attempt_kind}/final_message.txt",
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _generated_ref_list(value: Any, attempt_kind: str) -> list[str]:
|
|
109
|
+
refs=[]
|
|
110
|
+
for item in _as_list(value):
|
|
111
|
+
if not isinstance(item, str):
|
|
112
|
+
continue
|
|
113
|
+
normalized=_norm_ref(item, attempt_kind)
|
|
114
|
+
if _is_input_ref(normalized) or _is_attempt_metadata_ref(normalized, attempt_kind):
|
|
115
|
+
continue
|
|
116
|
+
if normalized and normalized not in refs:
|
|
117
|
+
refs.append(normalized)
|
|
118
|
+
return refs
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _input_ref_list(value: Any, attempt_kind: str) -> list[str]:
|
|
122
|
+
refs=[]
|
|
123
|
+
for item in _as_list(value):
|
|
124
|
+
if not isinstance(item, str):
|
|
125
|
+
continue
|
|
126
|
+
normalized=_norm_ref(item, attempt_kind)
|
|
127
|
+
if normalized and _is_input_ref(normalized) and normalized not in refs:
|
|
128
|
+
refs.append(normalized)
|
|
129
|
+
return refs
|
|
130
|
+
|
|
131
|
+
|
|
85
132
|
def _artifact_paths_from_raw(raw: dict[str, Any]) -> dict[str, Any]:
|
|
86
133
|
found={}
|
|
87
134
|
for k,v in raw.items():
|
|
88
|
-
if isinstance(k, str) and
|
|
89
|
-
|
|
135
|
+
key = k[2:] if isinstance(k, str) and k.startswith("./") else k
|
|
136
|
+
if isinstance(key, str) and (key.startswith('artifacts/') or '/artifacts/' in key):
|
|
137
|
+
found[key]=v
|
|
90
138
|
return found
|
|
91
139
|
|
|
92
140
|
|
|
@@ -102,35 +150,52 @@ def _scan_existing_artifacts(ctx: ActualOutputsNormalizationContext) -> list[str
|
|
|
102
150
|
return refs
|
|
103
151
|
|
|
104
152
|
|
|
105
|
-
def _extract_refs(raw: dict[str, Any], ctx: ActualOutputsNormalizationContext) -> tuple[list[str], dict[str, Any], list[str]]:
|
|
153
|
+
def _extract_refs(raw: dict[str, Any], ctx: ActualOutputsNormalizationContext) -> tuple[list[str], list[str], dict[str, Any], list[str]]:
|
|
106
154
|
warnings=[]
|
|
107
155
|
refs=[]
|
|
156
|
+
input_refs=[]
|
|
108
157
|
original_path_fields=_artifact_paths_from_raw(raw)
|
|
109
158
|
for k in original_path_fields:
|
|
110
|
-
|
|
159
|
+
normalized = _norm_ref(k, ctx.attempt_kind)
|
|
160
|
+
(input_refs if _is_input_ref(normalized) else refs).append(normalized)
|
|
111
161
|
for key in ['deliverable_refs','artifact_refs','files_created','outputs','artifacts','output_files']:
|
|
112
162
|
val=raw.get(key)
|
|
113
163
|
if isinstance(val, dict):
|
|
114
164
|
for k in val:
|
|
115
165
|
if isinstance(k, str) and (k.startswith('artifacts/') or '/artifacts/' in k or '.' in Path(k).name):
|
|
116
|
-
|
|
166
|
+
normalized = _norm_ref(k, ctx.attempt_kind)
|
|
167
|
+
if _is_input_ref(normalized):
|
|
168
|
+
input_refs.append(normalized)
|
|
169
|
+
elif not _is_attempt_metadata_ref(normalized, ctx.attempt_kind):
|
|
170
|
+
refs.append(normalized)
|
|
117
171
|
else:
|
|
118
172
|
for item in _as_list(val):
|
|
119
173
|
if isinstance(item, str):
|
|
120
|
-
|
|
174
|
+
normalized = _norm_ref(item, ctx.attempt_kind)
|
|
175
|
+
if _is_input_ref(normalized):
|
|
176
|
+
input_refs.append(normalized)
|
|
177
|
+
elif not _is_attempt_metadata_ref(normalized, ctx.attempt_kind):
|
|
178
|
+
refs.append(normalized)
|
|
179
|
+
for item in _as_list(raw.get("input_artifact_refs")):
|
|
180
|
+
if isinstance(item, str):
|
|
181
|
+
input_refs.append(_norm_ref(item, ctx.attempt_kind))
|
|
121
182
|
existing=_scan_existing_artifacts(ctx)
|
|
122
183
|
refs.extend(existing)
|
|
123
184
|
dedup=[]
|
|
124
185
|
for r in refs:
|
|
125
186
|
if r and r not in dedup:
|
|
126
187
|
dedup.append(r)
|
|
127
|
-
|
|
188
|
+
input_dedup=[]
|
|
189
|
+
for r in input_refs:
|
|
190
|
+
if r and r not in input_dedup:
|
|
191
|
+
input_dedup.append(r)
|
|
192
|
+
return dedup, input_dedup, original_path_fields, warnings
|
|
128
193
|
|
|
129
194
|
|
|
130
195
|
def normalize_actual_outputs(raw: dict[str, Any] | None, ctx: ActualOutputsNormalizationContext) -> ActualOutputsNormalizationResult:
|
|
131
196
|
raw = dict(raw or {})
|
|
132
197
|
report=ActualOutputsNormalizationReport(task_id=ctx.task_id, attempt_id=ctx.attempt_id, attempt_kind=ctx.attempt_kind, actual_outputs_raw_count=1)
|
|
133
|
-
refs, original_path_fields, warnings = _extract_refs(raw, ctx)
|
|
198
|
+
refs, input_refs, original_path_fields, warnings = _extract_refs(raw, ctx)
|
|
134
199
|
original_fields={k:v for k,v in raw.items() if k not in CANONICAL_ACTUAL_FIELDS}
|
|
135
200
|
raw_status=str(raw.get('status') or '').lower()
|
|
136
201
|
failure_status=raw_status in {'failed','error','timeout'}
|
|
@@ -156,13 +221,14 @@ def normalize_actual_outputs(raw: dict[str, Any] | None, ctx: ActualOutputsNorma
|
|
|
156
221
|
'attempt_kind': str(raw.get('attempt_kind') or ctx.attempt_kind),
|
|
157
222
|
'status': status,
|
|
158
223
|
'output_summary': str(summary),
|
|
159
|
-
'primary_output_ref': raw.get('primary_output_ref')
|
|
160
|
-
'
|
|
224
|
+
'primary_output_ref': _generated_ref_list(raw.get('primary_output_ref'), ctx.attempt_kind)[0] if _generated_ref_list(raw.get('primary_output_ref'), ctx.attempt_kind) else (refs[0] if refs else None),
|
|
225
|
+
'input_artifact_refs': _input_ref_list(raw.get('input_artifact_refs'), ctx.attempt_kind) or input_refs,
|
|
226
|
+
'deliverable_refs': _generated_ref_list(raw.get('deliverable_refs'), ctx.attempt_kind) or refs,
|
|
161
227
|
'final_message_ref': raw.get('final_message_ref') or f'attempts/{ctx.attempt_kind}/final_message.txt',
|
|
162
|
-
'artifact_refs':
|
|
163
|
-
'files_created':
|
|
164
|
-
'files_modified': [str(x) for x in (raw.get('files_modified') if isinstance(raw.get('files_modified'), list) else [])],
|
|
165
|
-
'files_deleted': [str(x) for x in (raw.get('files_deleted') if isinstance(raw.get('files_deleted'), list) else [])],
|
|
228
|
+
'artifact_refs': _generated_ref_list(raw.get('artifact_refs'), ctx.attempt_kind) or refs,
|
|
229
|
+
'files_created': _generated_ref_list(raw.get('files_created'), ctx.attempt_kind) or refs,
|
|
230
|
+
'files_modified': [_norm_ref(str(x), ctx.attempt_kind) for x in (raw.get('files_modified') if isinstance(raw.get('files_modified'), list) else [])],
|
|
231
|
+
'files_deleted': [_norm_ref(str(x), ctx.attempt_kind) for x in (raw.get('files_deleted') if isinstance(raw.get('files_deleted'), list) else [])],
|
|
166
232
|
'stdout_ref': raw.get('stdout_ref') or f'attempts/{ctx.attempt_kind}/stdout.txt',
|
|
167
233
|
'stderr_ref': raw.get('stderr_ref') or f'attempts/{ctx.attempt_kind}/stderr.txt',
|
|
168
234
|
'raw_log_refs': [str(x) for x in (raw.get('raw_log_refs') if isinstance(raw.get('raw_log_refs'), list) else [])] or [f'attempts/{ctx.attempt_kind}/stdout.txt', f'attempts/{ctx.attempt_kind}/stderr.txt', f'attempts/{ctx.attempt_kind}/final_message.txt'],
|
|
@@ -201,24 +267,32 @@ def repair_actual_outputs_file(path: Path, ctx: ActualOutputsNormalizationContex
|
|
|
201
267
|
text=path.read_text()
|
|
202
268
|
raw_path=path.with_name('actual_outputs.raw.json')
|
|
203
269
|
raw_path.write_text(text)
|
|
270
|
+
json_meta: dict[str, Any] = {}
|
|
271
|
+
parse_warnings: list[str] = []
|
|
204
272
|
try:
|
|
205
273
|
raw=json.loads(text)
|
|
206
274
|
if not isinstance(raw, dict):
|
|
207
275
|
raise ValueError('actual_outputs JSON was not an object')
|
|
208
|
-
except Exception as
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
276
|
+
except Exception as first_exc:
|
|
277
|
+
try:
|
|
278
|
+
raw, json_meta = extract_json_object(text, return_metadata=True)
|
|
279
|
+
if not isinstance(raw, dict):
|
|
280
|
+
raise ValueError('actual_outputs JSON was not an object')
|
|
281
|
+
parse_warnings.append(f"actual_outputs parsed from embedded JSON after raw parse failed: {first_exc}")
|
|
282
|
+
except Exception as exc:
|
|
283
|
+
report.actual_outputs_fallback=True
|
|
284
|
+
report.actual_outputs_fallback_count=1
|
|
285
|
+
report.validation_errors.append(str(first_exc))
|
|
286
|
+
if str(exc) != str(first_exc):
|
|
287
|
+
report.validation_errors.append(str(exc))
|
|
288
|
+
return ActualOutputsNormalizationResult(actual_outputs=None, report=report, fallback_required=True, parse_error=str(first_exc))
|
|
213
289
|
try:
|
|
214
|
-
|
|
215
|
-
result=
|
|
216
|
-
result.report.
|
|
217
|
-
result.report.
|
|
218
|
-
result.report.
|
|
219
|
-
result.report.
|
|
220
|
-
result.report.actual_outputs_normalized_count=0
|
|
221
|
-
result.report.actual_outputs_inferred_artifact_count=len(valid.get('deliverable_refs') or [])
|
|
290
|
+
ActualOutputs.model_validate(raw)
|
|
291
|
+
result=normalize_actual_outputs(raw, ctx)
|
|
292
|
+
result.report.raw_outputs_ref='actual_outputs.raw.json'
|
|
293
|
+
result.report.canonical_outputs_ref='actual_outputs.json'
|
|
294
|
+
result.report.warnings.extend(parse_warnings)
|
|
295
|
+
result.report.metadata_json.update(json_meta)
|
|
222
296
|
return result
|
|
223
297
|
except Exception as exc:
|
|
224
298
|
path.with_name('actual_outputs.invalid.json').write_text(text)
|
|
@@ -228,6 +302,8 @@ def repair_actual_outputs_file(path: Path, ctx: ActualOutputsNormalizationContex
|
|
|
228
302
|
result.report.normalized_outputs_ref='actual_outputs.normalized.json'
|
|
229
303
|
result.report.canonical_outputs_ref='actual_outputs.json'
|
|
230
304
|
result.report.validation_errors.extend(_validation_errors(exc))
|
|
305
|
+
result.report.warnings.extend(parse_warnings)
|
|
306
|
+
result.report.metadata_json.update(json_meta)
|
|
231
307
|
return result
|
|
232
308
|
|
|
233
309
|
|
|
@@ -204,11 +204,13 @@ def build_agent_invocation(agent_id: str, command: str, workspace: Path, prompt_
|
|
|
204
204
|
def classify_agent_failure(agent_id: str, display_name: str, error: object | None, stdout: str = "", stderr: str = "", returncode: int | None = None) -> str | None:
|
|
205
205
|
text = f"{error or ''}\n{stdout or ''}\n{stderr or ''}".lower()
|
|
206
206
|
command = WORKER_AGENT_RECIPES.get(agent_id).command_name if agent_id in WORKER_AGENT_RECIPES else agent_id
|
|
207
|
+
if error is None and returncode == 0:
|
|
208
|
+
return None
|
|
207
209
|
if isinstance(error, FileNotFoundError) or f"no such file or directory: '{command}'" in text or f"no such file or directory: {command}" in text:
|
|
208
210
|
return f"Apprentice Agent command not found: {command}"
|
|
209
211
|
if "headless mode unavailable" in text or "headless execution is unavailable" in text:
|
|
210
212
|
return str(error or stderr).strip()
|
|
211
|
-
if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout" in text:
|
|
213
|
+
if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout expired" in text or "command timed out" in text:
|
|
212
214
|
return f"Apprentice Agent attempt timed out while running {display_name}."
|
|
213
215
|
if _has_any(text, "eperm", "permission denied", "operation not permitted", "read-only file system"):
|
|
214
216
|
return f"Apprentice Agent permission error while running {display_name}: the external CLI could not access a required file or directory."
|
|
@@ -41,6 +41,8 @@ def normalize_artifact_ref(ref: object) -> str:
|
|
|
41
41
|
|
|
42
42
|
value = "" if ref is None else str(ref)
|
|
43
43
|
value = value.strip().strip('"').strip("'").replace("\\", "/").strip()
|
|
44
|
+
while value.startswith("./"):
|
|
45
|
+
value = value[2:]
|
|
44
46
|
|
|
45
47
|
for prefix in (
|
|
46
48
|
"artifact_content_previews_truncated:",
|
|
@@ -101,6 +103,8 @@ def is_artifact_evidence_ref(ref: object) -> bool:
|
|
|
101
103
|
"trace_summary_json:",
|
|
102
104
|
"actual_outputs:",
|
|
103
105
|
"actual_outputs.",
|
|
106
|
+
"actual_outputs.json",
|
|
107
|
+
"agent_trace.json",
|
|
104
108
|
"rubric:",
|
|
105
109
|
"rubric.",
|
|
106
110
|
"score:",
|
|
@@ -113,6 +117,8 @@ def is_artifact_evidence_ref(ref: object) -> bool:
|
|
|
113
117
|
return (
|
|
114
118
|
("attempts/" in normalized and "/artifacts/" in normalized)
|
|
115
119
|
or normalized.startswith("artifacts/")
|
|
120
|
+
or normalized.startswith("input/")
|
|
121
|
+
or normalized.startswith("task/")
|
|
116
122
|
or ("packages/" in normalized and "/attempts/" in normalized and "/artifacts/" in normalized)
|
|
117
123
|
)
|
|
118
124
|
|