agent-apprenticeship 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agent-apprenticeship.js +85 -37
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/agent_apprenticeship_trace/__init__.py +1 -1
- package/src/agent_apprenticeship_trace/actual_outputs_normalizer.py +104 -28
- package/src/agent_apprenticeship_trace/apprentice_adapters.py +3 -1
- package/src/agent_apprenticeship_trace/artifact_resolver.py +6 -0
- package/src/agent_apprenticeship_trace/bundle_exporter.py +132 -2
- package/src/agent_apprenticeship_trace/cli.py +1171 -262
- package/src/agent_apprenticeship_trace/codex_runner.py +89 -6
- package/src/agent_apprenticeship_trace/config.py +251 -32
- package/src/agent_apprenticeship_trace/ecosystem_r2.py +515 -0
- package/src/agent_apprenticeship_trace/env.py +5 -5
- package/src/agent_apprenticeship_trace/evaluator.py +127 -11
- package/src/agent_apprenticeship_trace/grader.py +12 -11
- package/src/agent_apprenticeship_trace/learning.py +270 -1
- package/src/agent_apprenticeship_trace/llm_output_normalizer.py +8 -7
- package/src/agent_apprenticeship_trace/loop.py +103 -4
- package/src/agent_apprenticeship_trace/loop_review.py +686 -0
- package/src/agent_apprenticeship_trace/mentor_checkpoints.py +49 -21
- package/src/agent_apprenticeship_trace/openai_structured.py +84 -14
- package/src/agent_apprenticeship_trace/package_exporter.py +1 -1
- package/src/agent_apprenticeship_trace/progress.py +2 -1
- package/src/agent_apprenticeship_trace/public_run.py +369 -57
- package/src/agent_apprenticeship_trace/public_sanitizer.py +37 -1
- package/src/agent_apprenticeship_trace/rubric_generation.py +11 -10
- package/src/agent_apprenticeship_trace/schemas.py +378 -3
- package/src/agent_apprenticeship_trace/task_intake.py +43 -9
- package/src/agent_apprenticeship_trace/tdo.py +2304 -0
- package/src/agent_apprenticeship_trace/trace_normalizer.py +351 -17
- package/src/agent_apprenticeship_trace/trace_prompt.py +38 -11
- package/src/agent_apprenticeship_trace/training_signals.py +1 -1
- package/src/agent_apprenticeship_trace/verifier.py +12 -6
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
const fs = require("fs");
|
|
5
5
|
const os = require("os");
|
|
6
6
|
const path = require("path");
|
|
7
|
-
const { spawnSync } = require("child_process");
|
|
7
|
+
const { spawn, spawnSync } = require("child_process");
|
|
8
8
|
|
|
9
9
|
const packageRoot = path.resolve(__dirname, "..");
|
|
10
10
|
const packageJson = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"));
|
|
@@ -103,26 +103,69 @@ function setupTimeout(defaultMs) {
|
|
|
103
103
|
return Number.isFinite(value) && value > 0 ? value : defaultMs;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
function appendLimited(current, chunk, limit = 16000) {
|
|
107
|
+
const next = current + chunk.toString();
|
|
108
|
+
return next.length > limit ? next.slice(next.length - limit) : next;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function setupHeartbeatMs(defaultMs) {
|
|
112
|
+
const raw = process.env.AA_NPM_SETUP_HEARTBEAT_MS;
|
|
113
|
+
if (!raw) return defaultMs;
|
|
114
|
+
const value = Number(raw);
|
|
115
|
+
return Number.isFinite(value) && value > 0 ? value : defaultMs;
|
|
116
|
+
}
|
|
117
|
+
|
|
106
118
|
function runQuiet(command, args, label, timeoutMs) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
119
|
+
return new Promise((resolve) => {
|
|
120
|
+
let stdout = "";
|
|
121
|
+
let stderr = "";
|
|
122
|
+
let timedOut = false;
|
|
123
|
+
let settled = false;
|
|
124
|
+
const child = spawn(command, args, {
|
|
125
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
126
|
+
env: runtimeEnv()
|
|
127
|
+
});
|
|
128
|
+
const heartbeat = setInterval(() => {
|
|
129
|
+
console.error(`Still working: ${label}...`);
|
|
130
|
+
}, setupHeartbeatMs(30000));
|
|
131
|
+
const timeout = setTimeout(() => {
|
|
132
|
+
timedOut = true;
|
|
133
|
+
child.kill("SIGTERM");
|
|
134
|
+
setTimeout(() => {
|
|
135
|
+
if (!settled) child.kill("SIGKILL");
|
|
136
|
+
}, 3000).unref();
|
|
137
|
+
}, setupTimeout(timeoutMs));
|
|
138
|
+
child.stdout.on("data", (chunk) => {
|
|
139
|
+
stdout = appendLimited(stdout, chunk);
|
|
140
|
+
});
|
|
141
|
+
child.stderr.on("data", (chunk) => {
|
|
142
|
+
stderr = appendLimited(stderr, chunk);
|
|
143
|
+
});
|
|
144
|
+
child.on("error", (error) => {
|
|
145
|
+
settled = true;
|
|
146
|
+
clearInterval(heartbeat);
|
|
147
|
+
clearTimeout(timeout);
|
|
148
|
+
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
149
|
+
console.error(String(error.message || error));
|
|
150
|
+
process.exit(1);
|
|
151
|
+
});
|
|
152
|
+
child.on("close", (code, signal) => {
|
|
153
|
+
settled = true;
|
|
154
|
+
clearInterval(heartbeat);
|
|
155
|
+
clearTimeout(timeout);
|
|
156
|
+
if (code === 0) {
|
|
157
|
+
resolve();
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
161
|
+
if (timedOut || signal) {
|
|
162
|
+
console.error("Runtime setup timed out. Re-run the command, or set AA_PYTHON to a working Python 3.11+ interpreter.");
|
|
163
|
+
}
|
|
164
|
+
const out = [stdout, stderr].filter(Boolean).join("\n").trim();
|
|
165
|
+
if (out) console.error(out.slice(-8000));
|
|
166
|
+
process.exit(code || 1);
|
|
167
|
+
});
|
|
112
168
|
});
|
|
113
|
-
if (result.error || result.status !== 0) {
|
|
114
|
-
console.error(`Agent Apprenticeship runtime setup failed while trying to ${label}.`);
|
|
115
|
-
const timedOut = Boolean(result.signal) || (result.error && String(result.error.message || result.error).includes("ETIMEDOUT"));
|
|
116
|
-
if (timedOut) {
|
|
117
|
-
console.error("Runtime setup timed out. Re-run the command, or set AA_PYTHON to a working Python 3.11+ interpreter.");
|
|
118
|
-
} else if (result.error) {
|
|
119
|
-
const message = String(result.error.message || result.error);
|
|
120
|
-
console.error(message);
|
|
121
|
-
}
|
|
122
|
-
const out = [result.stdout, result.stderr].filter(Boolean).join("\n").trim();
|
|
123
|
-
if (out) console.error(out.slice(-8000));
|
|
124
|
-
process.exit(result.status || 1);
|
|
125
|
-
}
|
|
126
169
|
}
|
|
127
170
|
|
|
128
171
|
function runVisible(command, args, label) {
|
|
@@ -137,7 +180,7 @@ function runVisible(command, args, label) {
|
|
|
137
180
|
}
|
|
138
181
|
}
|
|
139
182
|
|
|
140
|
-
function ensureRuntime(python) {
|
|
183
|
+
async function ensureRuntime(python) {
|
|
141
184
|
const venvDir = path.resolve(process.env.AA_NPM_VENV || path.join(appHome(), "npm-venv", packageJson.version));
|
|
142
185
|
const py = venvPython(venvDir);
|
|
143
186
|
const markerPath = path.join(venvDir, ".agent-apprenticeship-npm.json");
|
|
@@ -154,8 +197,8 @@ function ensureRuntime(python) {
|
|
|
154
197
|
console.error("First run can take a few minutes while Python dependencies are installed.");
|
|
155
198
|
fs.rmSync(venvDir, { recursive: true, force: true });
|
|
156
199
|
fs.mkdirSync(path.dirname(venvDir), { recursive: true });
|
|
157
|
-
runQuiet(python, ["-m", "venv", venvDir], "create the Python environment", 120000);
|
|
158
|
-
runQuiet(py, [
|
|
200
|
+
await runQuiet(python, ["-m", "venv", venvDir], "create the Python environment", 120000);
|
|
201
|
+
await runQuiet(py, [
|
|
159
202
|
"-m", "pip", "install",
|
|
160
203
|
"--disable-pip-version-check",
|
|
161
204
|
"--progress-bar", "off",
|
|
@@ -191,20 +234,25 @@ function runCli(python, args, usePackagedSource) {
|
|
|
191
234
|
process.exit(result.status === null ? 1 : result.status);
|
|
192
235
|
}
|
|
193
236
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
237
|
+
(async () => {
|
|
238
|
+
const python = findPython();
|
|
239
|
+
if (!python) {
|
|
240
|
+
console.error("Agent Apprenticeship requires Python 3.11 or newer.");
|
|
241
|
+
console.error("");
|
|
242
|
+
console.error("macOS Homebrew:");
|
|
243
|
+
console.error("brew install python@3.11");
|
|
244
|
+
console.error("");
|
|
245
|
+
console.error("Or set:");
|
|
246
|
+
console.error("AA_PYTHON=/path/to/python3.11");
|
|
247
|
+
process.exit(1);
|
|
248
|
+
}
|
|
205
249
|
|
|
206
|
-
if (process.env.AA_NPM_USE_SYSTEM_PYTHON === "1") {
|
|
207
|
-
|
|
208
|
-
}
|
|
250
|
+
if (process.env.AA_NPM_USE_SYSTEM_PYTHON === "1") {
|
|
251
|
+
runCli(python, process.argv.slice(2), true);
|
|
252
|
+
}
|
|
209
253
|
|
|
210
|
-
runCli(ensureRuntime(python), process.argv.slice(2), false);
|
|
254
|
+
runCli(await ensureRuntime(python), process.argv.slice(2), false);
|
|
255
|
+
})().catch((error) => {
|
|
256
|
+
console.error(`Failed to start Agent Apprenticeship: ${error.message || error}`);
|
|
257
|
+
process.exit(1);
|
|
258
|
+
});
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agent-apprenticeship"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.8"
|
|
8
8
|
description = "Open framework for turning real agent work into transferable agent experience"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""Agent Apprenticeship framework."""
|
|
2
|
-
__version__ = "0.1.
|
|
2
|
+
__version__ = "0.1.8"
|
|
@@ -5,6 +5,7 @@ from typing import Any
|
|
|
5
5
|
from pydantic import BaseModel, Field
|
|
6
6
|
from .schemas import ActualOutputs
|
|
7
7
|
from .io import write_json
|
|
8
|
+
from .openai_structured import extract_json_object
|
|
8
9
|
|
|
9
10
|
CANONICAL_ACTUAL_FIELDS = set(ActualOutputs.model_fields.keys())
|
|
10
11
|
|
|
@@ -72,7 +73,15 @@ def _validation_errors(exc: Exception) -> list[str]:
|
|
|
72
73
|
|
|
73
74
|
|
|
74
75
|
def _norm_ref(ref: str, attempt_kind: str) -> str:
|
|
75
|
-
ref=str(ref)
|
|
76
|
+
ref=str(ref).replace("\\", "/").strip().lstrip("/")
|
|
77
|
+
while ref.startswith("./"):
|
|
78
|
+
ref = ref[2:]
|
|
79
|
+
if not ref:
|
|
80
|
+
return ref
|
|
81
|
+
if ref in {"actual_outputs.json", "agent_trace.json", "stdout.txt", "stderr.txt", "final_message.txt"}:
|
|
82
|
+
return f"attempts/{attempt_kind}/{ref}"
|
|
83
|
+
if ref.startswith("input/") or ref.startswith("task/"):
|
|
84
|
+
return ref
|
|
76
85
|
if ref.startswith(f'attempts/{attempt_kind}/'):
|
|
77
86
|
return ref
|
|
78
87
|
if ref.startswith('artifacts/'):
|
|
@@ -82,11 +91,50 @@ def _norm_ref(ref: str, attempt_kind: str) -> str:
|
|
|
82
91
|
return ref
|
|
83
92
|
|
|
84
93
|
|
|
94
|
+
def _is_input_ref(ref: str) -> bool:
|
|
95
|
+
return ref.startswith("input/") or ref.startswith("task/") or "/input/" in ref or "/task/" in ref
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _is_attempt_metadata_ref(ref: str, attempt_kind: str) -> bool:
|
|
99
|
+
return ref in {
|
|
100
|
+
f"attempts/{attempt_kind}/actual_outputs.json",
|
|
101
|
+
f"attempts/{attempt_kind}/agent_trace.json",
|
|
102
|
+
f"attempts/{attempt_kind}/stdout.txt",
|
|
103
|
+
f"attempts/{attempt_kind}/stderr.txt",
|
|
104
|
+
f"attempts/{attempt_kind}/final_message.txt",
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _generated_ref_list(value: Any, attempt_kind: str) -> list[str]:
|
|
109
|
+
refs=[]
|
|
110
|
+
for item in _as_list(value):
|
|
111
|
+
if not isinstance(item, str):
|
|
112
|
+
continue
|
|
113
|
+
normalized=_norm_ref(item, attempt_kind)
|
|
114
|
+
if _is_input_ref(normalized) or _is_attempt_metadata_ref(normalized, attempt_kind):
|
|
115
|
+
continue
|
|
116
|
+
if normalized and normalized not in refs:
|
|
117
|
+
refs.append(normalized)
|
|
118
|
+
return refs
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _input_ref_list(value: Any, attempt_kind: str) -> list[str]:
|
|
122
|
+
refs=[]
|
|
123
|
+
for item in _as_list(value):
|
|
124
|
+
if not isinstance(item, str):
|
|
125
|
+
continue
|
|
126
|
+
normalized=_norm_ref(item, attempt_kind)
|
|
127
|
+
if normalized and _is_input_ref(normalized) and normalized not in refs:
|
|
128
|
+
refs.append(normalized)
|
|
129
|
+
return refs
|
|
130
|
+
|
|
131
|
+
|
|
85
132
|
def _artifact_paths_from_raw(raw: dict[str, Any]) -> dict[str, Any]:
|
|
86
133
|
found={}
|
|
87
134
|
for k,v in raw.items():
|
|
88
|
-
if isinstance(k, str) and
|
|
89
|
-
|
|
135
|
+
key = k[2:] if isinstance(k, str) and k.startswith("./") else k
|
|
136
|
+
if isinstance(key, str) and (key.startswith('artifacts/') or '/artifacts/' in key):
|
|
137
|
+
found[key]=v
|
|
90
138
|
return found
|
|
91
139
|
|
|
92
140
|
|
|
@@ -102,35 +150,52 @@ def _scan_existing_artifacts(ctx: ActualOutputsNormalizationContext) -> list[str
|
|
|
102
150
|
return refs
|
|
103
151
|
|
|
104
152
|
|
|
105
|
-
def _extract_refs(raw: dict[str, Any], ctx: ActualOutputsNormalizationContext) -> tuple[list[str], dict[str, Any], list[str]]:
|
|
153
|
+
def _extract_refs(raw: dict[str, Any], ctx: ActualOutputsNormalizationContext) -> tuple[list[str], list[str], dict[str, Any], list[str]]:
|
|
106
154
|
warnings=[]
|
|
107
155
|
refs=[]
|
|
156
|
+
input_refs=[]
|
|
108
157
|
original_path_fields=_artifact_paths_from_raw(raw)
|
|
109
158
|
for k in original_path_fields:
|
|
110
|
-
|
|
159
|
+
normalized = _norm_ref(k, ctx.attempt_kind)
|
|
160
|
+
(input_refs if _is_input_ref(normalized) else refs).append(normalized)
|
|
111
161
|
for key in ['deliverable_refs','artifact_refs','files_created','outputs','artifacts','output_files']:
|
|
112
162
|
val=raw.get(key)
|
|
113
163
|
if isinstance(val, dict):
|
|
114
164
|
for k in val:
|
|
115
165
|
if isinstance(k, str) and (k.startswith('artifacts/') or '/artifacts/' in k or '.' in Path(k).name):
|
|
116
|
-
|
|
166
|
+
normalized = _norm_ref(k, ctx.attempt_kind)
|
|
167
|
+
if _is_input_ref(normalized):
|
|
168
|
+
input_refs.append(normalized)
|
|
169
|
+
elif not _is_attempt_metadata_ref(normalized, ctx.attempt_kind):
|
|
170
|
+
refs.append(normalized)
|
|
117
171
|
else:
|
|
118
172
|
for item in _as_list(val):
|
|
119
173
|
if isinstance(item, str):
|
|
120
|
-
|
|
174
|
+
normalized = _norm_ref(item, ctx.attempt_kind)
|
|
175
|
+
if _is_input_ref(normalized):
|
|
176
|
+
input_refs.append(normalized)
|
|
177
|
+
elif not _is_attempt_metadata_ref(normalized, ctx.attempt_kind):
|
|
178
|
+
refs.append(normalized)
|
|
179
|
+
for item in _as_list(raw.get("input_artifact_refs")):
|
|
180
|
+
if isinstance(item, str):
|
|
181
|
+
input_refs.append(_norm_ref(item, ctx.attempt_kind))
|
|
121
182
|
existing=_scan_existing_artifacts(ctx)
|
|
122
183
|
refs.extend(existing)
|
|
123
184
|
dedup=[]
|
|
124
185
|
for r in refs:
|
|
125
186
|
if r and r not in dedup:
|
|
126
187
|
dedup.append(r)
|
|
127
|
-
|
|
188
|
+
input_dedup=[]
|
|
189
|
+
for r in input_refs:
|
|
190
|
+
if r and r not in input_dedup:
|
|
191
|
+
input_dedup.append(r)
|
|
192
|
+
return dedup, input_dedup, original_path_fields, warnings
|
|
128
193
|
|
|
129
194
|
|
|
130
195
|
def normalize_actual_outputs(raw: dict[str, Any] | None, ctx: ActualOutputsNormalizationContext) -> ActualOutputsNormalizationResult:
|
|
131
196
|
raw = dict(raw or {})
|
|
132
197
|
report=ActualOutputsNormalizationReport(task_id=ctx.task_id, attempt_id=ctx.attempt_id, attempt_kind=ctx.attempt_kind, actual_outputs_raw_count=1)
|
|
133
|
-
refs, original_path_fields, warnings = _extract_refs(raw, ctx)
|
|
198
|
+
refs, input_refs, original_path_fields, warnings = _extract_refs(raw, ctx)
|
|
134
199
|
original_fields={k:v for k,v in raw.items() if k not in CANONICAL_ACTUAL_FIELDS}
|
|
135
200
|
raw_status=str(raw.get('status') or '').lower()
|
|
136
201
|
failure_status=raw_status in {'failed','error','timeout'}
|
|
@@ -156,13 +221,14 @@ def normalize_actual_outputs(raw: dict[str, Any] | None, ctx: ActualOutputsNorma
|
|
|
156
221
|
'attempt_kind': str(raw.get('attempt_kind') or ctx.attempt_kind),
|
|
157
222
|
'status': status,
|
|
158
223
|
'output_summary': str(summary),
|
|
159
|
-
'primary_output_ref': raw.get('primary_output_ref')
|
|
160
|
-
'
|
|
224
|
+
'primary_output_ref': _generated_ref_list(raw.get('primary_output_ref'), ctx.attempt_kind)[0] if _generated_ref_list(raw.get('primary_output_ref'), ctx.attempt_kind) else (refs[0] if refs else None),
|
|
225
|
+
'input_artifact_refs': _input_ref_list(raw.get('input_artifact_refs'), ctx.attempt_kind) or input_refs,
|
|
226
|
+
'deliverable_refs': _generated_ref_list(raw.get('deliverable_refs'), ctx.attempt_kind) or refs,
|
|
161
227
|
'final_message_ref': raw.get('final_message_ref') or f'attempts/{ctx.attempt_kind}/final_message.txt',
|
|
162
|
-
'artifact_refs':
|
|
163
|
-
'files_created':
|
|
164
|
-
'files_modified': [str(x) for x in (raw.get('files_modified') if isinstance(raw.get('files_modified'), list) else [])],
|
|
165
|
-
'files_deleted': [str(x) for x in (raw.get('files_deleted') if isinstance(raw.get('files_deleted'), list) else [])],
|
|
228
|
+
'artifact_refs': _generated_ref_list(raw.get('artifact_refs'), ctx.attempt_kind) or refs,
|
|
229
|
+
'files_created': _generated_ref_list(raw.get('files_created'), ctx.attempt_kind) or refs,
|
|
230
|
+
'files_modified': [_norm_ref(str(x), ctx.attempt_kind) for x in (raw.get('files_modified') if isinstance(raw.get('files_modified'), list) else [])],
|
|
231
|
+
'files_deleted': [_norm_ref(str(x), ctx.attempt_kind) for x in (raw.get('files_deleted') if isinstance(raw.get('files_deleted'), list) else [])],
|
|
166
232
|
'stdout_ref': raw.get('stdout_ref') or f'attempts/{ctx.attempt_kind}/stdout.txt',
|
|
167
233
|
'stderr_ref': raw.get('stderr_ref') or f'attempts/{ctx.attempt_kind}/stderr.txt',
|
|
168
234
|
'raw_log_refs': [str(x) for x in (raw.get('raw_log_refs') if isinstance(raw.get('raw_log_refs'), list) else [])] or [f'attempts/{ctx.attempt_kind}/stdout.txt', f'attempts/{ctx.attempt_kind}/stderr.txt', f'attempts/{ctx.attempt_kind}/final_message.txt'],
|
|
@@ -201,24 +267,32 @@ def repair_actual_outputs_file(path: Path, ctx: ActualOutputsNormalizationContex
|
|
|
201
267
|
text=path.read_text()
|
|
202
268
|
raw_path=path.with_name('actual_outputs.raw.json')
|
|
203
269
|
raw_path.write_text(text)
|
|
270
|
+
json_meta: dict[str, Any] = {}
|
|
271
|
+
parse_warnings: list[str] = []
|
|
204
272
|
try:
|
|
205
273
|
raw=json.loads(text)
|
|
206
274
|
if not isinstance(raw, dict):
|
|
207
275
|
raise ValueError('actual_outputs JSON was not an object')
|
|
208
|
-
except Exception as
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
276
|
+
except Exception as first_exc:
|
|
277
|
+
try:
|
|
278
|
+
raw, json_meta = extract_json_object(text, return_metadata=True)
|
|
279
|
+
if not isinstance(raw, dict):
|
|
280
|
+
raise ValueError('actual_outputs JSON was not an object')
|
|
281
|
+
parse_warnings.append(f"actual_outputs parsed from embedded JSON after raw parse failed: {first_exc}")
|
|
282
|
+
except Exception as exc:
|
|
283
|
+
report.actual_outputs_fallback=True
|
|
284
|
+
report.actual_outputs_fallback_count=1
|
|
285
|
+
report.validation_errors.append(str(first_exc))
|
|
286
|
+
if str(exc) != str(first_exc):
|
|
287
|
+
report.validation_errors.append(str(exc))
|
|
288
|
+
return ActualOutputsNormalizationResult(actual_outputs=None, report=report, fallback_required=True, parse_error=str(first_exc))
|
|
213
289
|
try:
|
|
214
|
-
|
|
215
|
-
result=
|
|
216
|
-
result.report.
|
|
217
|
-
result.report.
|
|
218
|
-
result.report.
|
|
219
|
-
result.report.
|
|
220
|
-
result.report.actual_outputs_normalized_count=0
|
|
221
|
-
result.report.actual_outputs_inferred_artifact_count=len(valid.get('deliverable_refs') or [])
|
|
290
|
+
ActualOutputs.model_validate(raw)
|
|
291
|
+
result=normalize_actual_outputs(raw, ctx)
|
|
292
|
+
result.report.raw_outputs_ref='actual_outputs.raw.json'
|
|
293
|
+
result.report.canonical_outputs_ref='actual_outputs.json'
|
|
294
|
+
result.report.warnings.extend(parse_warnings)
|
|
295
|
+
result.report.metadata_json.update(json_meta)
|
|
222
296
|
return result
|
|
223
297
|
except Exception as exc:
|
|
224
298
|
path.with_name('actual_outputs.invalid.json').write_text(text)
|
|
@@ -228,6 +302,8 @@ def repair_actual_outputs_file(path: Path, ctx: ActualOutputsNormalizationContex
|
|
|
228
302
|
result.report.normalized_outputs_ref='actual_outputs.normalized.json'
|
|
229
303
|
result.report.canonical_outputs_ref='actual_outputs.json'
|
|
230
304
|
result.report.validation_errors.extend(_validation_errors(exc))
|
|
305
|
+
result.report.warnings.extend(parse_warnings)
|
|
306
|
+
result.report.metadata_json.update(json_meta)
|
|
231
307
|
return result
|
|
232
308
|
|
|
233
309
|
|
|
@@ -204,11 +204,13 @@ def build_agent_invocation(agent_id: str, command: str, workspace: Path, prompt_
|
|
|
204
204
|
def classify_agent_failure(agent_id: str, display_name: str, error: object | None, stdout: str = "", stderr: str = "", returncode: int | None = None) -> str | None:
|
|
205
205
|
text = f"{error or ''}\n{stdout or ''}\n{stderr or ''}".lower()
|
|
206
206
|
command = WORKER_AGENT_RECIPES.get(agent_id).command_name if agent_id in WORKER_AGENT_RECIPES else agent_id
|
|
207
|
+
if error is None and returncode == 0:
|
|
208
|
+
return None
|
|
207
209
|
if isinstance(error, FileNotFoundError) or f"no such file or directory: '{command}'" in text or f"no such file or directory: {command}" in text:
|
|
208
210
|
return f"Apprentice Agent command not found: {command}"
|
|
209
211
|
if "headless mode unavailable" in text or "headless execution is unavailable" in text:
|
|
210
212
|
return str(error or stderr).strip()
|
|
211
|
-
if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout" in text:
|
|
213
|
+
if isinstance(error, subprocess.TimeoutExpired) or "timed out" in text or "timeout expired" in text or "command timed out" in text:
|
|
212
214
|
return f"Apprentice Agent attempt timed out while running {display_name}."
|
|
213
215
|
if _has_any(text, "eperm", "permission denied", "operation not permitted", "read-only file system"):
|
|
214
216
|
return f"Apprentice Agent permission error while running {display_name}: the external CLI could not access a required file or directory."
|
|
@@ -41,6 +41,8 @@ def normalize_artifact_ref(ref: object) -> str:
|
|
|
41
41
|
|
|
42
42
|
value = "" if ref is None else str(ref)
|
|
43
43
|
value = value.strip().strip('"').strip("'").replace("\\", "/").strip()
|
|
44
|
+
while value.startswith("./"):
|
|
45
|
+
value = value[2:]
|
|
44
46
|
|
|
45
47
|
for prefix in (
|
|
46
48
|
"artifact_content_previews_truncated:",
|
|
@@ -101,6 +103,8 @@ def is_artifact_evidence_ref(ref: object) -> bool:
|
|
|
101
103
|
"trace_summary_json:",
|
|
102
104
|
"actual_outputs:",
|
|
103
105
|
"actual_outputs.",
|
|
106
|
+
"actual_outputs.json",
|
|
107
|
+
"agent_trace.json",
|
|
104
108
|
"rubric:",
|
|
105
109
|
"rubric.",
|
|
106
110
|
"score:",
|
|
@@ -113,6 +117,8 @@ def is_artifact_evidence_ref(ref: object) -> bool:
|
|
|
113
117
|
return (
|
|
114
118
|
("attempts/" in normalized and "/artifacts/" in normalized)
|
|
115
119
|
or normalized.startswith("artifacts/")
|
|
120
|
+
or normalized.startswith("input/")
|
|
121
|
+
or normalized.startswith("task/")
|
|
116
122
|
or ("packages/" in normalized and "/attempts/" in normalized and "/artifacts/" in normalized)
|
|
117
123
|
)
|
|
118
124
|
|
|
@@ -6,10 +6,12 @@ import re
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Any
|
|
8
8
|
|
|
9
|
-
from .config import Settings, get_settings, normalize_mentor_mode
|
|
9
|
+
from .config import Settings, apprenticeship_mode_display, get_settings, normalize_apprenticeship_mode, normalize_mentor_mode
|
|
10
10
|
from .io import append_jsonl, read_json, read_jsonl, write_json
|
|
11
11
|
from .public_sanitizer import sanitize_public_obj, sanitize_public_text
|
|
12
12
|
from .release_exporter import create_release
|
|
13
|
+
from .tdo import tdo_manifest_fields_from_report
|
|
14
|
+
from .loop_review import summarize_loop_reviews
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
def _count_jsonl(path: Path) -> int:
|
|
@@ -96,7 +98,7 @@ def contribution_counts(release_root: Path) -> dict[str, int]:
|
|
|
96
98
|
|
|
97
99
|
def _write_card(bundle_root: Path) -> None:
|
|
98
100
|
(bundle_root / "contribution_card.md").write_text(
|
|
99
|
-
"# Agent Apprenticeship
|
|
101
|
+
"# Agent Apprenticeship Agent Experience Package\n\n"
|
|
100
102
|
"This local bundle packages an apprenticeship session for public ecosystem contribution. "
|
|
101
103
|
"It includes session events, task instructions, traces, actual outputs, evaluation results, "
|
|
102
104
|
"and learning-data views when available.\n\n"
|
|
@@ -110,6 +112,15 @@ def _session_mentor_mode(session: dict[str, Any], settings: Settings) -> str:
|
|
|
110
112
|
return normalize_mentor_mode(session.get("mentor_mode") or session.get("evaluation_mode") or settings.mentor_mode)
|
|
111
113
|
|
|
112
114
|
|
|
115
|
+
def _session_apprenticeship_mode(session: dict[str, Any], settings: Settings) -> str:
|
|
116
|
+
return normalize_apprenticeship_mode(
|
|
117
|
+
session.get("apprenticeship_mode")
|
|
118
|
+
or session.get("mentor_mode")
|
|
119
|
+
or session.get("evaluation_mode")
|
|
120
|
+
or settings.apprenticeship_mode
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
113
124
|
def _clean_session_metadata(session: dict[str, Any], settings: Settings) -> dict[str, Any]:
|
|
114
125
|
allowed = {
|
|
115
126
|
"run_id",
|
|
@@ -128,6 +139,8 @@ def _clean_session_metadata(session: dict[str, Any], settings: Settings) -> dict
|
|
|
128
139
|
"session_status",
|
|
129
140
|
}
|
|
130
141
|
clean = {k: v for k, v in session.items() if k in allowed and v is not None}
|
|
142
|
+
clean["apprenticeship_mode"] = _session_apprenticeship_mode(session, settings)
|
|
143
|
+
clean["apprenticeship_mode_display"] = apprenticeship_mode_display(clean["apprenticeship_mode"])
|
|
131
144
|
clean["mentor_mode"] = _session_mentor_mode(session, settings)
|
|
132
145
|
return clean
|
|
133
146
|
|
|
@@ -141,6 +154,8 @@ def _copy_session_files(run_root: Path, bundle_root: Path, settings: Settings) -
|
|
|
141
154
|
shutil.copytree(run_root / "task", bundle_root / "task", dirs_exist_ok=True)
|
|
142
155
|
if (run_root / "mentor_checkpoints").exists():
|
|
143
156
|
shutil.copytree(run_root / "mentor_checkpoints", bundle_root / "mentor_checkpoints", dirs_exist_ok=True)
|
|
157
|
+
if (run_root / "loops").exists():
|
|
158
|
+
shutil.copytree(run_root / "loops", bundle_root / "loops", dirs_exist_ok=True)
|
|
144
159
|
|
|
145
160
|
|
|
146
161
|
def _copy_clean_bundle_files(release_root: Path, bundle_root: Path, include_debug: bool) -> None:
|
|
@@ -153,6 +168,9 @@ def _copy_clean_bundle_files(release_root: Path, bundle_root: Path, include_debu
|
|
|
153
168
|
_copy_jsonl_if_nonempty(release_root / "actual_outputs.jsonl", bundle_root / "outputs" / "actual_outputs.jsonl")
|
|
154
169
|
_copy_json_if_exists(release_root / "artifacts_index.json", bundle_root / "outputs" / "artifacts_index.json")
|
|
155
170
|
_copy_indexed_package_files(release_root, bundle_root)
|
|
171
|
+
_copy_tdo_outputs(release_root, bundle_root)
|
|
172
|
+
_copy_experience_compiler_outputs(release_root, bundle_root)
|
|
173
|
+
_copy_loop_review_outputs(release_root, bundle_root)
|
|
156
174
|
|
|
157
175
|
for name in [
|
|
158
176
|
"grader_results.jsonl",
|
|
@@ -214,6 +232,42 @@ def _copy_indexed_package_files(release_root: Path, bundle_root: Path) -> None:
|
|
|
214
232
|
write_json(bundle_root / "outputs" / "artifacts_index.json", rewritten)
|
|
215
233
|
|
|
216
234
|
|
|
235
|
+
def _copy_tdo_outputs(release_root: Path, bundle_root: Path) -> None:
|
|
236
|
+
package_tdo_dirs = sorted(path for path in (release_root / "packages").glob("*/tdo") if path.is_dir())
|
|
237
|
+
if not package_tdo_dirs:
|
|
238
|
+
return
|
|
239
|
+
if len(package_tdo_dirs) == 1:
|
|
240
|
+
shutil.copytree(package_tdo_dirs[0], bundle_root / "tdo", dirs_exist_ok=True)
|
|
241
|
+
return
|
|
242
|
+
for src in package_tdo_dirs:
|
|
243
|
+
target = bundle_root / "tdo" / src.parent.name
|
|
244
|
+
shutil.copytree(src, target, dirs_exist_ok=True)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _copy_experience_compiler_outputs(release_root: Path, bundle_root: Path) -> None:
|
|
248
|
+
package_dirs = sorted(path for path in (release_root / "packages").glob("*/experience_compiler") if path.is_dir())
|
|
249
|
+
if not package_dirs:
|
|
250
|
+
return
|
|
251
|
+
if len(package_dirs) == 1:
|
|
252
|
+
shutil.copytree(package_dirs[0], bundle_root / "experience_compiler", dirs_exist_ok=True)
|
|
253
|
+
return
|
|
254
|
+
for src in package_dirs:
|
|
255
|
+
target = bundle_root / "experience_compiler" / src.parent.name
|
|
256
|
+
shutil.copytree(src, target, dirs_exist_ok=True)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _copy_loop_review_outputs(release_root: Path, bundle_root: Path) -> None:
|
|
260
|
+
package_dirs = sorted(path for path in (release_root / "packages").glob("*/loops") if path.is_dir())
|
|
261
|
+
if not package_dirs:
|
|
262
|
+
return
|
|
263
|
+
if len(package_dirs) == 1:
|
|
264
|
+
shutil.copytree(package_dirs[0], bundle_root / "loops", dirs_exist_ok=True)
|
|
265
|
+
return
|
|
266
|
+
for src in package_dirs:
|
|
267
|
+
target = bundle_root / "loops" / src.parent.name
|
|
268
|
+
shutil.copytree(src, target, dirs_exist_ok=True)
|
|
269
|
+
|
|
270
|
+
|
|
217
271
|
def is_unsafe_bundle_ref(ref: str) -> bool:
|
|
218
272
|
path = Path(ref)
|
|
219
273
|
return path.is_absolute() or ".." in path.parts
|
|
@@ -263,6 +317,8 @@ def _bundle_metadata_shape(bundle_root: Path, release_root: Path, settings: Sett
|
|
|
263
317
|
"agent_apprentice_role": role,
|
|
264
318
|
"expected_economic_value": task.get("expected_economic_value"),
|
|
265
319
|
"expected_economic_value_for_agent_apprentice": task.get("expected_economic_value_for_agent_apprentice"),
|
|
320
|
+
"apprenticeship_mode": _session_apprenticeship_mode(session, settings),
|
|
321
|
+
"apprenticeship_mode_display": apprenticeship_mode_display(_session_apprenticeship_mode(session, settings)),
|
|
266
322
|
"mentor_mode": _session_mentor_mode(session, settings),
|
|
267
323
|
"task_status": task_status,
|
|
268
324
|
"run_status": run_status,
|
|
@@ -280,9 +336,83 @@ def _write_manifest(bundle_root: Path, release_root: Path, settings: Settings, i
|
|
|
280
336
|
manifest["status_reason"] = session["status_reason"]
|
|
281
337
|
if include_internal_schema:
|
|
282
338
|
manifest["source_release_schema_version"] = release_manifest.get("schema_version")
|
|
339
|
+
manifest.update(_tdo_manifest_shape(release_root))
|
|
340
|
+
manifest.update(_loop_manifest_shape(release_root))
|
|
283
341
|
write_json(bundle_root / "contribution_manifest.json", manifest)
|
|
284
342
|
|
|
285
343
|
|
|
344
|
+
def _loop_manifest_shape(release_root: Path) -> dict[str, Any]:
|
|
345
|
+
summary = summarize_loop_reviews(release_root)
|
|
346
|
+
return {
|
|
347
|
+
"has_loop_review_packets": summary.get("has_loop_review_packets"),
|
|
348
|
+
"loop_iteration_count": summary.get("loop_iteration_count"),
|
|
349
|
+
"loop_reviewer_types": summary.get("reviewer_types") or [],
|
|
350
|
+
"final_loop_verdict": summary.get("final_loop_verdict"),
|
|
351
|
+
"loop_review_refs": summary.get("loop_review_refs") or [],
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _tdo_manifest_shape(release_root: Path) -> dict[str, Any]:
|
|
356
|
+
reports = []
|
|
357
|
+
for path in sorted((release_root / "packages").glob("*/tdo/tdo_report.json")):
|
|
358
|
+
try:
|
|
359
|
+
reports.append(read_json(path))
|
|
360
|
+
except Exception:
|
|
361
|
+
continue
|
|
362
|
+
if not reports:
|
|
363
|
+
return {
|
|
364
|
+
"tdo_enabled": False,
|
|
365
|
+
"tdo_status": "not_run",
|
|
366
|
+
"tdo_output_refs": [],
|
|
367
|
+
"experience_compiler_enabled": False,
|
|
368
|
+
"experience_compiler_status": "not_run",
|
|
369
|
+
"experience_compiler_output_refs": [],
|
|
370
|
+
}
|
|
371
|
+
latest = reports[-1]
|
|
372
|
+
output_refs: list[str] = []
|
|
373
|
+
package_tdo_dirs = sorted(path for path in (release_root / "packages").glob("*/tdo") if path.is_dir())
|
|
374
|
+
if len(package_tdo_dirs) == 1:
|
|
375
|
+
for child in sorted(package_tdo_dirs[0].iterdir()):
|
|
376
|
+
if child.is_file():
|
|
377
|
+
output_refs.append(f"tdo/{child.name}")
|
|
378
|
+
else:
|
|
379
|
+
for src in package_tdo_dirs:
|
|
380
|
+
for child in sorted(src.iterdir()):
|
|
381
|
+
if child.is_file():
|
|
382
|
+
output_refs.append(f"tdo/{src.parent.name}/{child.name}")
|
|
383
|
+
fields = tdo_manifest_fields_from_report(latest)
|
|
384
|
+
fields["tdo_output_refs"] = output_refs
|
|
385
|
+
fields.update(_experience_compiler_manifest_shape(release_root, latest))
|
|
386
|
+
if len(reports) > 1:
|
|
387
|
+
fields["tdo_package_count"] = len(reports)
|
|
388
|
+
return fields
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _experience_compiler_manifest_shape(release_root: Path, latest_report: dict[str, Any]) -> dict[str, Any]:
|
|
392
|
+
compiler_dirs = sorted(path for path in (release_root / "packages").glob("*/experience_compiler") if path.is_dir())
|
|
393
|
+
refs: list[str] = []
|
|
394
|
+
if len(compiler_dirs) == 1:
|
|
395
|
+
for child in sorted(compiler_dirs[0].rglob("*")):
|
|
396
|
+
if child.is_file():
|
|
397
|
+
refs.append(f"experience_compiler/{child.relative_to(compiler_dirs[0]).as_posix()}")
|
|
398
|
+
else:
|
|
399
|
+
for src in compiler_dirs:
|
|
400
|
+
for child in sorted(src.rglob("*")):
|
|
401
|
+
if child.is_file():
|
|
402
|
+
refs.append(f"experience_compiler/{src.parent.name}/{child.relative_to(src).as_posix()}")
|
|
403
|
+
return {
|
|
404
|
+
"experience_compiler_enabled": bool(compiler_dirs),
|
|
405
|
+
"experience_compiler_status": latest_report.get("tdo_status"),
|
|
406
|
+
"experience_compiler_id": str(latest_report.get("tdo_id") or "").replace("tdo_", "compiler_", 1),
|
|
407
|
+
"experience_compiler_output_refs": refs,
|
|
408
|
+
"experience_compiler_rows_by_type": latest_report.get("rows_accepted_by_type") or {},
|
|
409
|
+
"experience_compiler_rounds_run": latest_report.get("rounds_run"),
|
|
410
|
+
"experience_compiler_average_quality_score": latest_report.get("average_quality_score"),
|
|
411
|
+
"experience_compiler_average_reuse_value_score": latest_report.get("average_reuse_value_score"),
|
|
412
|
+
"experience_compiler_judge_source": latest_report.get("tdo_judge_source"),
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
|
|
286
416
|
def create_contribution_bundle(
|
|
287
417
|
run_root: Path,
|
|
288
418
|
bundle_root: Path | None = None,
|