agent-apprenticeship 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +217 -0
- package/bin/agent-apprenticeship.js +131 -0
- package/package.json +30 -0
- package/pyproject.toml +23 -0
- package/src/agent_apprenticeship_trace/__init__.py +2 -0
- package/src/agent_apprenticeship_trace/actual_outputs_normalizer.py +240 -0
- package/src/agent_apprenticeship_trace/apprentice_adapters.py +348 -0
- package/src/agent_apprenticeship_trace/artifact_capture.py +23 -0
- package/src/agent_apprenticeship_trace/artifact_previews.py +80 -0
- package/src/agent_apprenticeship_trace/artifact_resolver.py +142 -0
- package/src/agent_apprenticeship_trace/batch_runner.py +116 -0
- package/src/agent_apprenticeship_trace/bundle_exporter.py +254 -0
- package/src/agent_apprenticeship_trace/certification.py +580 -0
- package/src/agent_apprenticeship_trace/cli.py +2979 -0
- package/src/agent_apprenticeship_trace/codex_runner.py +428 -0
- package/src/agent_apprenticeship_trace/command_discovery.py +94 -0
- package/src/agent_apprenticeship_trace/config.py +609 -0
- package/src/agent_apprenticeship_trace/contract_diagnostics.py +69 -0
- package/src/agent_apprenticeship_trace/env.py +46 -0
- package/src/agent_apprenticeship_trace/evaluator.py +64 -0
- package/src/agent_apprenticeship_trace/grader.py +194 -0
- package/src/agent_apprenticeship_trace/integration_status.py +193 -0
- package/src/agent_apprenticeship_trace/io.py +20 -0
- package/src/agent_apprenticeship_trace/learning.py +627 -0
- package/src/agent_apprenticeship_trace/lesson_extractor.py +5 -0
- package/src/agent_apprenticeship_trace/llm_output_normalizer.py +467 -0
- package/src/agent_apprenticeship_trace/loop.py +111 -0
- package/src/agent_apprenticeship_trace/mentor_checkpoints.py +354 -0
- package/src/agent_apprenticeship_trace/openai_structured.py +783 -0
- package/src/agent_apprenticeship_trace/package_exporter.py +303 -0
- package/src/agent_apprenticeship_trace/progress.py +223 -0
- package/src/agent_apprenticeship_trace/public_run.py +1109 -0
- package/src/agent_apprenticeship_trace/public_sanitizer.py +139 -0
- package/src/agent_apprenticeship_trace/recipes.py +129 -0
- package/src/agent_apprenticeship_trace/release_exporter.py +259 -0
- package/src/agent_apprenticeship_trace/revision.py +21 -0
- package/src/agent_apprenticeship_trace/role_runners.py +7 -0
- package/src/agent_apprenticeship_trace/rubric_generation.py +75 -0
- package/src/agent_apprenticeship_trace/schemas.py +273 -0
- package/src/agent_apprenticeship_trace/session_events.py +99 -0
- package/src/agent_apprenticeship_trace/task_intake.py +112 -0
- package/src/agent_apprenticeship_trace/trace_normalizer.py +669 -0
- package/src/agent_apprenticeship_trace/trace_prompt.py +51 -0
- package/src/agent_apprenticeship_trace/training_signals.py +30 -0
- package/src/agent_apprenticeship_trace/validation.py +210 -0
- package/src/agent_apprenticeship_trace/verifier.py +55 -0
|
@@ -0,0 +1,2979 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import base64
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import shlex
|
|
6
|
+
import sys
|
|
7
|
+
import subprocess
|
|
8
|
+
import shutil, json
|
|
9
|
+
import zipfile
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import typer
|
|
14
|
+
from .schemas import RawTaskRecord, AgentTrace
|
|
15
|
+
from .io import read_json, read_jsonl, write_json
|
|
16
|
+
from .loop import run_task as run_one
|
|
17
|
+
from .batch_runner import run_batch as run_batch_impl
|
|
18
|
+
from .bundle_exporter import create_contribution_bundle
|
|
19
|
+
from .release_exporter import create_release as create_release_impl
|
|
20
|
+
from .validation import validate_release as validate_release_impl, format_counters
|
|
21
|
+
from .config import (
|
|
22
|
+
DATA_SHARING_LEVELS,
|
|
23
|
+
DEFAULT_PUBLIC_ECOSYSTEM_REPO,
|
|
24
|
+
ECOSYSTEM_AUTO_SHARE_MODES,
|
|
25
|
+
EVALUATION_MODES,
|
|
26
|
+
MENTOR_MODES,
|
|
27
|
+
SENSITIVE_INFO_MASKING_LEVELS,
|
|
28
|
+
ecosystem_auto_share_display,
|
|
29
|
+
get_settings,
|
|
30
|
+
init_settings,
|
|
31
|
+
model_provider_ready,
|
|
32
|
+
configured_model_provider_ready,
|
|
33
|
+
apprentice_agent_readiness_status,
|
|
34
|
+
mentor_model_provider_readiness,
|
|
35
|
+
normalize_mentor_mode,
|
|
36
|
+
normalize_ecosystem_auto_share,
|
|
37
|
+
normalize_sensitive_info_masking,
|
|
38
|
+
debug_settings,
|
|
39
|
+
public_settings,
|
|
40
|
+
public_settings_text,
|
|
41
|
+
update_settings,
|
|
42
|
+
)
|
|
43
|
+
from .openai_structured import run_llm_smoke, format_smoke_counters
|
|
44
|
+
from .env import contains_secret, redact_secrets
|
|
45
|
+
from .command_discovery import AGENT_COMMAND_CANDIDATES, resolve_command, resolve_agent_command
|
|
46
|
+
from .integration_status import integrations_report
|
|
47
|
+
from .certification import certification_exit_code, run_certification_matrix
|
|
48
|
+
from .learning import (
|
|
49
|
+
active_packs,
|
|
50
|
+
compare_replay,
|
|
51
|
+
compile_experience_pack,
|
|
52
|
+
list_packs,
|
|
53
|
+
load_pack,
|
|
54
|
+
pack_run_refs,
|
|
55
|
+
replay_instruction_for_pack,
|
|
56
|
+
remove_pack,
|
|
57
|
+
resolve_learning_source,
|
|
58
|
+
resolve_packs_for_run,
|
|
59
|
+
search_learning_sources,
|
|
60
|
+
slugify,
|
|
61
|
+
update_pack_status,
|
|
62
|
+
write_before_after_result,
|
|
63
|
+
)
|
|
64
|
+
from .public_run import apprentice_agent_readiness, continue_session, finish_session, run_prompt_task, run_root_for
|
|
65
|
+
from .progress import format_progress_event, format_run_status, read_run_status, watch_progress
|
|
66
|
+
from .recipes import MODEL_PROVIDER_RECIPES, REMOVED_V0_MODEL_PROVIDER_IDS, WORKER_AGENT_RECIPES
|
|
67
|
+
app=typer.Typer(add_completion=False)
|
|
68
|
+
configure_app=typer.Typer(add_completion=False, help='Configure Apprentice Agents and Mentor Model Providers.')
|
|
69
|
+
ecosystem_app=typer.Typer(add_completion=False, help='Explore and contribute Agent Apprenticeship ecosystem bundles.')
|
|
70
|
+
bundle_app=typer.Typer(add_completion=False, help='Inspect local Contribution Bundles.')
|
|
71
|
+
learn_app=typer.Typer(add_completion=False, help='Create and apply reversible Experience Packs from ecosystem experience.')
|
|
72
|
+
app.add_typer(configure_app, name='configure')
|
|
73
|
+
app.add_typer(ecosystem_app, name='ecosystem')
|
|
74
|
+
app.add_typer(bundle_app, name='bundle')
|
|
75
|
+
app.add_typer(learn_app, name='learn')
|
|
76
|
+
|
|
77
|
+
SLACK_LINK='https://join.slack.com/t/fsycommunity/shared_invite/zt-37417grrb-jFD6BQIYgC5wEMrW2bHssw'
|
|
78
|
+
|
|
79
|
+
MODEL_KEY_CANDIDATES = {
|
|
80
|
+
"openai": ["OPENAI_API_KEY"],
|
|
81
|
+
"anthropic": ["ANTHROPIC_API_KEY"],
|
|
82
|
+
"google": ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
|
|
83
|
+
"openrouter": ["OPENROUTER_API_KEY"],
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
def _split_asset_prompt(text: str) -> list[Path]:
|
|
87
|
+
return [Path(p.strip()) for p in text.split(',') if p.strip()]
|
|
88
|
+
|
|
89
|
+
def _print_bundle_ready(bundle: Path | None) -> None:
|
|
90
|
+
if not bundle:
|
|
91
|
+
return
|
|
92
|
+
typer.echo(f'Contribution Bundle path: {bundle}')
|
|
93
|
+
typer.echo(f'Contribute to public ecosystem: apprentice ecosystem contribute {bundle}')
|
|
94
|
+
typer.echo(f'Public ecosystem: {_ecosystem_repo_url()}')
|
|
95
|
+
typer.echo(f'View files: open {bundle}')
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _print_public_ecosystem_contribution_help(bundle: str | Path) -> None:
|
|
99
|
+
typer.echo('Contribute to public ecosystem:')
|
|
100
|
+
typer.echo(f'apprentice ecosystem contribute {bundle}')
|
|
101
|
+
typer.echo('')
|
|
102
|
+
typer.echo('Public ecosystem:')
|
|
103
|
+
typer.echo(_ecosystem_repo_url())
|
|
104
|
+
|
|
105
|
+
def _print_task_result(
|
|
106
|
+
status: dict,
|
|
107
|
+
*,
|
|
108
|
+
followup: bool=False,
|
|
109
|
+
record_only: bool=False,
|
|
110
|
+
include_contribution_help: bool=True,
|
|
111
|
+
) -> None:
|
|
112
|
+
task_status = status.get('task_status')
|
|
113
|
+
bundle = status.get('contribution_bundle_path')
|
|
114
|
+
artifacts = status.get('artifacts_path')
|
|
115
|
+
error = status.get('last_operational_error') or status.get('latest_message')
|
|
116
|
+
if followup:
|
|
117
|
+
typer.echo('')
|
|
118
|
+
if record_only:
|
|
119
|
+
typer.echo('Follow-up recorded.')
|
|
120
|
+
typer.echo('No Apprentice Agent loop was run. Use --run-loop to continue work.')
|
|
121
|
+
else:
|
|
122
|
+
typer.echo('Task updated.' if task_status != 'failed' else 'Task update failed.')
|
|
123
|
+
elif task_status == 'completed':
|
|
124
|
+
typer.echo('')
|
|
125
|
+
typer.echo('Task completed.')
|
|
126
|
+
elif task_status == 'failed':
|
|
127
|
+
typer.echo('')
|
|
128
|
+
typer.echo('Task failed.')
|
|
129
|
+
if error:
|
|
130
|
+
typer.echo(f'Reason: {error}')
|
|
131
|
+
elif task_status == 'partial':
|
|
132
|
+
typer.echo('')
|
|
133
|
+
typer.echo('Task partially completed.')
|
|
134
|
+
if error:
|
|
135
|
+
typer.echo(f'Reason: {error}')
|
|
136
|
+
if artifacts:
|
|
137
|
+
label = 'Artifacts/partial files' if task_status == 'failed' else 'Artifacts'
|
|
138
|
+
typer.echo(f'{label}:')
|
|
139
|
+
typer.echo(str(artifacts))
|
|
140
|
+
if bundle:
|
|
141
|
+
typer.echo('Contribution Bundle:')
|
|
142
|
+
typer.echo(str(bundle))
|
|
143
|
+
if include_contribution_help:
|
|
144
|
+
typer.echo('')
|
|
145
|
+
mode = get_settings().ecosystem_auto_share
|
|
146
|
+
if mode == "manual":
|
|
147
|
+
_print_public_ecosystem_contribution_help(bundle)
|
|
148
|
+
else:
|
|
149
|
+
typer.echo(f'Ecosystem Auto-Share: {ecosystem_auto_share_display(mode)}')
|
|
150
|
+
typer.echo('')
|
|
151
|
+
typer.echo('View files:')
|
|
152
|
+
typer.echo(f'open {bundle}')
|
|
153
|
+
if not followup and status.get('run_id'):
|
|
154
|
+
typer.echo('')
|
|
155
|
+
typer.echo('Next:')
|
|
156
|
+
typer.echo(f'- Add follow-up: apprentice continue {status["run_id"]}')
|
|
157
|
+
typer.echo(f'- Finish session: apprentice finish {status["run_id"]}')
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _print_session_finished(status: dict, *, include_contribution_help: bool=True) -> None:
|
|
161
|
+
typer.echo('Session finished.')
|
|
162
|
+
typer.echo(f"Task Status: {status.get('task_status')}")
|
|
163
|
+
if status.get('last_operational_error'):
|
|
164
|
+
typer.echo(f"Reason: {status.get('last_operational_error')}")
|
|
165
|
+
if status.get('artifacts_path'):
|
|
166
|
+
typer.echo("Artifacts:")
|
|
167
|
+
typer.echo(str(status.get('artifacts_path')))
|
|
168
|
+
if status.get('contribution_bundle_path'):
|
|
169
|
+
typer.echo("Contribution Bundle:")
|
|
170
|
+
typer.echo(str(status.get('contribution_bundle_path')))
|
|
171
|
+
if include_contribution_help:
|
|
172
|
+
typer.echo('')
|
|
173
|
+
mode = get_settings().ecosystem_auto_share
|
|
174
|
+
if mode == "manual":
|
|
175
|
+
_print_public_ecosystem_contribution_help(status.get('contribution_bundle_path'))
|
|
176
|
+
else:
|
|
177
|
+
typer.echo(f'Ecosystem Auto-Share: {ecosystem_auto_share_display(mode)}')
|
|
178
|
+
typer.echo('')
|
|
179
|
+
typer.echo('View files:')
|
|
180
|
+
typer.echo(f"open {status.get('contribution_bundle_path')}")
|
|
181
|
+
|
|
182
|
+
def _yes_no(value: bool) -> str:
|
|
183
|
+
return "yes" if value else "no"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _status_label(status: str | None) -> str:
|
|
187
|
+
if status == "ready":
|
|
188
|
+
return "Ready"
|
|
189
|
+
if status in {"not_ready", "missing_api_key", "missing_command"}:
|
|
190
|
+
return "Not ready"
|
|
191
|
+
if status == "untested":
|
|
192
|
+
return "Untested"
|
|
193
|
+
return (status or "not_ready").replace("_", " ").title()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _status_line(label: str, status: str | None, reason: str | None = None) -> str:
|
|
197
|
+
return f"{label}: {_status_label(status)}" + (f" - {reason}" if reason else "")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _env_next_step(env_var: str | None) -> str:
|
|
201
|
+
env = env_var or "YOUR_PROVIDER_API_KEY"
|
|
202
|
+
return f"Next: export {env}=... or add {env}=... to ~/.agent-apprenticeship/.env.local"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _detect_apprentice_agents() -> list[dict]:
|
|
206
|
+
detected = []
|
|
207
|
+
for agent_id, commands in AGENT_COMMAND_CANDIDATES.items():
|
|
208
|
+
recipe = WORKER_AGENT_RECIPES[agent_id]
|
|
209
|
+
configured = next((command for command in commands if shutil.which(command)), None)
|
|
210
|
+
resolved = shutil.which(configured) if configured else None
|
|
211
|
+
if not resolved and os.getenv("AA_DISABLE_LOCAL_ENV") != "1":
|
|
212
|
+
configured, resolved = resolve_agent_command(agent_id)
|
|
213
|
+
if resolved:
|
|
214
|
+
detected.append(
|
|
215
|
+
{
|
|
216
|
+
"id": agent_id,
|
|
217
|
+
"display_name": recipe.display_name,
|
|
218
|
+
"command": configured,
|
|
219
|
+
"command_resolved": resolved,
|
|
220
|
+
"command_found": True,
|
|
221
|
+
}
|
|
222
|
+
)
|
|
223
|
+
return detected
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _detect_model_provider_keys() -> list[dict]:
|
|
227
|
+
# get_settings() loads .env.local from cwd and app home before we inspect env.
|
|
228
|
+
get_settings()
|
|
229
|
+
detected = []
|
|
230
|
+
for provider_id, env_vars in MODEL_KEY_CANDIDATES.items():
|
|
231
|
+
visible = next((name for name in env_vars if os.getenv(name)), None)
|
|
232
|
+
if visible:
|
|
233
|
+
recipe = MODEL_PROVIDER_RECIPES[provider_id]
|
|
234
|
+
detected.append(
|
|
235
|
+
{
|
|
236
|
+
"id": provider_id,
|
|
237
|
+
"display_name": recipe.display_name,
|
|
238
|
+
"api_key_env_var": visible,
|
|
239
|
+
"model": recipe.default_model,
|
|
240
|
+
}
|
|
241
|
+
)
|
|
242
|
+
return detected
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _print_key_storage_guidance() -> None:
|
|
246
|
+
typer.echo("Store Mentor Model Provider keys in ~/.agent-apprenticeship/.env.local, a gitignored repo .env.local, or exported shell env vars.")
|
|
247
|
+
typer.echo("Example:")
|
|
248
|
+
for name in ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", "OPENROUTER_API_KEY"]:
|
|
249
|
+
typer.echo(f'{name}="..."')
|
|
250
|
+
typer.echo("Agent Apprenticeship records env var names only; it never stores or prints raw keys.")
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _choose_detected_index(count: int, default: int = 1) -> int | None:
|
|
254
|
+
raw = typer.prompt("Choose", default=str(default)).strip().lower()
|
|
255
|
+
if raw in {"y", "yes"}:
|
|
256
|
+
return default if 1 <= default <= count else None
|
|
257
|
+
if raw in {"", "skip", "none", "cancel"}:
|
|
258
|
+
return None
|
|
259
|
+
try:
|
|
260
|
+
idx = int(raw)
|
|
261
|
+
except ValueError:
|
|
262
|
+
return None
|
|
263
|
+
return idx if 1 <= idx <= count else None
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _configure_detected_apprentice_agent(row: dict) -> None:
|
|
267
|
+
agent_id = str(row["id"])
|
|
268
|
+
command = str(row["command"])
|
|
269
|
+
runner = "codex" if agent_id == "codex" else agent_id
|
|
270
|
+
update_settings(
|
|
271
|
+
worker_agent=agent_id,
|
|
272
|
+
worker_agent_command=command,
|
|
273
|
+
worker_runner=runner,
|
|
274
|
+
reviser_runner=runner,
|
|
275
|
+
apprentice_agent_readiness_status="untested",
|
|
276
|
+
apprentice_agent_readiness_reason="Command was auto-detected; run a live smoke before release certification.",
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _configure_detected_model_provider(row: dict) -> None:
|
|
281
|
+
provider_id = str(row["id"])
|
|
282
|
+
recipe = MODEL_PROVIDER_RECIPES[provider_id]
|
|
283
|
+
update_settings(
|
|
284
|
+
model_provider=provider_id,
|
|
285
|
+
model_provider_api_key_env=str(row["api_key_env_var"]),
|
|
286
|
+
model_provider_model=str(row.get("model") or recipe.default_model),
|
|
287
|
+
mentor_model_provider_readiness_status="untested",
|
|
288
|
+
mentor_model_provider_readiness_reason="API key env var was auto-detected; run a live smoke before release certification.",
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _first_run_setup(*, interactive: bool) -> None:
|
|
293
|
+
detected_agents = _detect_apprentice_agents()
|
|
294
|
+
typer.echo("")
|
|
295
|
+
if detected_agents:
|
|
296
|
+
typer.echo("Detected Apprentice Agents:")
|
|
297
|
+
for idx, row in enumerate(detected_agents, 1):
|
|
298
|
+
typer.echo(f"{idx}. {row['display_name']} - command found ({row['command']})")
|
|
299
|
+
custom_index = len(detected_agents) + 1
|
|
300
|
+
typer.echo(f"{custom_index}. Custom - use a custom command template")
|
|
301
|
+
if interactive:
|
|
302
|
+
choice = _choose_detected_index(custom_index, default=1)
|
|
303
|
+
if choice and choice <= len(detected_agents):
|
|
304
|
+
_configure_detected_apprentice_agent(detected_agents[choice - 1])
|
|
305
|
+
elif choice == custom_index:
|
|
306
|
+
typer.echo("Run: apprentice configure agent custom --command-template \"my-agent run --workspace {workspace} --prompt-file {prompt_file}\"")
|
|
307
|
+
else:
|
|
308
|
+
typer.echo("Run `apprentice configure agent <id>` or rerun `apprentice init --setup` to choose one.")
|
|
309
|
+
else:
|
|
310
|
+
typer.echo("No supported Apprentice Agent CLI was detected.")
|
|
311
|
+
typer.echo("Install one of: Codex, Cursor, Claude Code, OpenClaw, OpenCode, Hermes Agent.")
|
|
312
|
+
typer.echo("Or choose Custom to provide a command template.")
|
|
313
|
+
typer.echo("Then rerun: apprentice init --setup")
|
|
314
|
+
|
|
315
|
+
detected_providers = _detect_model_provider_keys()
|
|
316
|
+
typer.echo("")
|
|
317
|
+
if detected_providers:
|
|
318
|
+
typer.echo("Detected Mentor Model Provider keys:")
|
|
319
|
+
for idx, row in enumerate(detected_providers, 1):
|
|
320
|
+
typer.echo(f"{idx}. {row['display_name']} - {row['api_key_env_var']} visible")
|
|
321
|
+
if interactive:
|
|
322
|
+
choice = _choose_detected_index(len(detected_providers), default=1)
|
|
323
|
+
if choice:
|
|
324
|
+
_configure_detected_model_provider(detected_providers[choice - 1])
|
|
325
|
+
else:
|
|
326
|
+
typer.echo("Run `apprentice configure model <id>` or rerun `apprentice init --setup` to choose one.")
|
|
327
|
+
else:
|
|
328
|
+
typer.echo("No Mentor Model Provider API key was detected.")
|
|
329
|
+
_print_key_storage_guidance()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _progress_callback(quiet: bool=False, verbose: bool=False, json_progress: bool=False):
|
|
333
|
+
if quiet:
|
|
334
|
+
return None
|
|
335
|
+
def _emit(event: dict, status: dict) -> None:
|
|
336
|
+
if json_progress:
|
|
337
|
+
typer.echo(json.dumps(event, sort_keys=True))
|
|
338
|
+
return
|
|
339
|
+
event_type = event.get('event_type')
|
|
340
|
+
if event_type == 'run_started':
|
|
341
|
+
typer.echo('Agent Apprenticeship run started')
|
|
342
|
+
typer.echo('')
|
|
343
|
+
typer.echo(f'Run: {status.get("run_id")}')
|
|
344
|
+
typer.echo(f'Apprentice Agent: {status.get("apprentice_agent") or status.get("worker_agent")}')
|
|
345
|
+
typer.echo(f'Mentor Mode: {status.get("mentor_mode")}')
|
|
346
|
+
typer.echo(f'Maximum Improvement Loops: {status.get("maximum_improvement_loops")}')
|
|
347
|
+
typer.echo(f'Task workspace: {status.get("task_workspace_path")}')
|
|
348
|
+
typer.echo(f'Artifacts: {status.get("artifacts_path")}')
|
|
349
|
+
typer.echo('')
|
|
350
|
+
return
|
|
351
|
+
if event_type in {'run_completed'}:
|
|
352
|
+
_print_task_result(status)
|
|
353
|
+
return
|
|
354
|
+
if event_type in {'followup_started'}:
|
|
355
|
+
typer.echo(format_progress_event(event))
|
|
356
|
+
return
|
|
357
|
+
if event_type in {'followup_completed'}:
|
|
358
|
+
_print_task_result(status, followup=True, record_only=bool((event.get('metadata_json') or {}).get('record_only')))
|
|
359
|
+
return
|
|
360
|
+
if event_type in {'apprentice_attempt_started','apprentice_attempt_completed','worker_attempt_started','worker_attempt_completed','revision_started','revision_completed','mentor_review_started','mentor_review_completed','task_workspace_prepared','contribution_bundle_completed','operational_error'} or verbose:
|
|
361
|
+
typer.echo(format_progress_event(event))
|
|
362
|
+
return _emit
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _interactive_checkpoint_requested(
|
|
366
|
+
settings,
|
|
367
|
+
*,
|
|
368
|
+
expert_auto_approve: bool = False,
|
|
369
|
+
hybrid_auto_approve: bool = False,
|
|
370
|
+
expert_interactive: bool = False,
|
|
371
|
+
hybrid_interactive: bool = False,
|
|
372
|
+
) -> bool:
|
|
373
|
+
if settings.mentor_mode == "expert_led":
|
|
374
|
+
if expert_auto_approve or os.getenv("AA_EXPERT_AUTO_APPROVE") == "1":
|
|
375
|
+
return False
|
|
376
|
+
return expert_interactive or os.getenv("AA_EXPERT_INTERACTIVE") == "1" or sys.stdin.isatty()
|
|
377
|
+
if settings.mentor_mode == "hybrid":
|
|
378
|
+
if hybrid_auto_approve or os.getenv("AA_HYBRID_AUTO_APPROVE") == "1":
|
|
379
|
+
return False
|
|
380
|
+
return hybrid_interactive or os.getenv("AA_HYBRID_INTERACTIVE") == "1" or sys.stdin.isatty()
|
|
381
|
+
return False
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
@contextmanager
|
|
385
|
+
def _temporary_auto_approve_env(
|
|
386
|
+
*,
|
|
387
|
+
expert_auto_approve: bool = False,
|
|
388
|
+
hybrid_auto_approve: bool = False,
|
|
389
|
+
mentor_interactive_checkpoints: bool = False,
|
|
390
|
+
):
|
|
391
|
+
old_expert = os.environ.get("AA_EXPERT_AUTO_APPROVE")
|
|
392
|
+
old_hybrid = os.environ.get("AA_HYBRID_AUTO_APPROVE")
|
|
393
|
+
old_interactive = os.environ.get("AA_MENTOR_INTERACTIVE_CHECKPOINTS")
|
|
394
|
+
if expert_auto_approve:
|
|
395
|
+
os.environ["AA_EXPERT_AUTO_APPROVE"] = "1"
|
|
396
|
+
if hybrid_auto_approve:
|
|
397
|
+
os.environ["AA_HYBRID_AUTO_APPROVE"] = "1"
|
|
398
|
+
if mentor_interactive_checkpoints:
|
|
399
|
+
os.environ["AA_MENTOR_INTERACTIVE_CHECKPOINTS"] = "1"
|
|
400
|
+
try:
|
|
401
|
+
yield
|
|
402
|
+
finally:
|
|
403
|
+
if old_expert is None:
|
|
404
|
+
os.environ.pop("AA_EXPERT_AUTO_APPROVE", None)
|
|
405
|
+
else:
|
|
406
|
+
os.environ["AA_EXPERT_AUTO_APPROVE"] = old_expert
|
|
407
|
+
if old_hybrid is None:
|
|
408
|
+
os.environ.pop("AA_HYBRID_AUTO_APPROVE", None)
|
|
409
|
+
else:
|
|
410
|
+
os.environ["AA_HYBRID_AUTO_APPROVE"] = old_hybrid
|
|
411
|
+
if old_interactive is None:
|
|
412
|
+
os.environ.pop("AA_MENTOR_INTERACTIVE_CHECKPOINTS", None)
|
|
413
|
+
else:
|
|
414
|
+
os.environ["AA_MENTOR_INTERACTIVE_CHECKPOINTS"] = old_interactive
|
|
415
|
+
|
|
416
|
+
def _normalize_evaluation_mode(value: str | None) -> str | None:
|
|
417
|
+
if value is None:
|
|
418
|
+
return None
|
|
419
|
+
normalized=value.strip().lower().replace('_','-')
|
|
420
|
+
if normalized not in EVALUATION_MODES:
|
|
421
|
+
raise typer.BadParameter(f'mentor mode must be one of: {", ".join(MENTOR_MODES)}')
|
|
422
|
+
return normalized
|
|
423
|
+
|
|
424
|
+
def _normalize_mentor_mode(value: str | None) -> str | None:
|
|
425
|
+
if value is None:
|
|
426
|
+
return None
|
|
427
|
+
try:
|
|
428
|
+
return normalize_mentor_mode(value)
|
|
429
|
+
except ValueError as exc:
|
|
430
|
+
raise typer.BadParameter(f'mentor mode must be one of: {", ".join(MENTOR_MODES)}') from exc
|
|
431
|
+
|
|
432
|
+
def _normalize_data_sharing(value: str | None) -> str | None:
|
|
433
|
+
if value is None:
|
|
434
|
+
return None
|
|
435
|
+
normalized=value.strip().lower().replace('_','-').replace(' ', '-')
|
|
436
|
+
if normalized == 'full':
|
|
437
|
+
normalized='full-context'
|
|
438
|
+
if normalized not in DATA_SHARING_LEVELS:
|
|
439
|
+
raise typer.BadParameter(
|
|
440
|
+
f'sensitive info masking must be one of: {", ".join(SENSITIVE_INFO_MASKING_LEVELS)}'
|
|
441
|
+
)
|
|
442
|
+
return normalized
|
|
443
|
+
|
|
444
|
+
def _normalize_masking(value: str | None) -> str | None:
|
|
445
|
+
if value is None:
|
|
446
|
+
return None
|
|
447
|
+
try:
|
|
448
|
+
return normalize_sensitive_info_masking(value)
|
|
449
|
+
except ValueError as exc:
|
|
450
|
+
raise typer.BadParameter(
|
|
451
|
+
f'sensitive info masking must be one of: {", ".join(SENSITIVE_INFO_MASKING_LEVELS)}'
|
|
452
|
+
) from exc
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _normalize_auto_share(value: str | None) -> str | None:
|
|
456
|
+
if value is None:
|
|
457
|
+
return None
|
|
458
|
+
try:
|
|
459
|
+
return normalize_ecosystem_auto_share(value)
|
|
460
|
+
except ValueError as exc:
|
|
461
|
+
raise typer.BadParameter(
|
|
462
|
+
f'ecosystem auto-share must be one of: {", ".join(ECOSYSTEM_AUTO_SHARE_MODES)}'
|
|
463
|
+
) from exc
|
|
464
|
+
|
|
465
|
+
def _settings_for_run(
|
|
466
|
+
mentor_mode: str | None=None,
|
|
467
|
+
sensitive_info_masking: str | None=None,
|
|
468
|
+
max_loops: int | None=None,
|
|
469
|
+
evaluation_mode: str | None=None,
|
|
470
|
+
data_sharing_level: str | None=None,
|
|
471
|
+
):
|
|
472
|
+
settings=get_settings()
|
|
473
|
+
updates={}
|
|
474
|
+
if mentor_mode is not None:
|
|
475
|
+
updates['mentor_mode']=_normalize_mentor_mode(mentor_mode)
|
|
476
|
+
if evaluation_mode is not None:
|
|
477
|
+
updates['evaluation_mode']=_normalize_evaluation_mode(evaluation_mode)
|
|
478
|
+
if mentor_mode is None:
|
|
479
|
+
updates['mentor_mode']=_normalize_mentor_mode(evaluation_mode)
|
|
480
|
+
if sensitive_info_masking is not None:
|
|
481
|
+
updates['sensitive_info_masking']=_normalize_masking(sensitive_info_masking)
|
|
482
|
+
if data_sharing_level is not None:
|
|
483
|
+
updates['data_sharing_level']=_normalize_data_sharing(data_sharing_level)
|
|
484
|
+
if sensitive_info_masking is None:
|
|
485
|
+
updates['sensitive_info_masking']=_normalize_masking(data_sharing_level)
|
|
486
|
+
if max_loops is not None:
|
|
487
|
+
if max_loops < 1:
|
|
488
|
+
raise typer.BadParameter('maximum improvement loops must be at least 1')
|
|
489
|
+
updates['max_improvement_loops']=max_loops
|
|
490
|
+
updates['max_iterations']=max_loops
|
|
491
|
+
return settings.model_copy(update=updates)
|
|
492
|
+
|
|
493
|
+
def _configure_model_impl(provider: str | None, model: str | None, api_key_env_var: str | None, test_connection: bool):
|
|
494
|
+
if provider is None:
|
|
495
|
+
detected = _detect_model_provider_keys()
|
|
496
|
+
if detected:
|
|
497
|
+
typer.echo('Detected Mentor Model Provider keys:')
|
|
498
|
+
for idx, row in enumerate(detected, 1):
|
|
499
|
+
typer.echo(f"{idx}. {row['display_name']} - {row['api_key_env_var']} visible")
|
|
500
|
+
choice = typer.prompt('Choose Mentor Model Provider or press Enter to list all', default='', show_default=False).strip()
|
|
501
|
+
if choice:
|
|
502
|
+
try:
|
|
503
|
+
idx = int(choice)
|
|
504
|
+
except ValueError:
|
|
505
|
+
provider = choice
|
|
506
|
+
else:
|
|
507
|
+
if 1 <= idx <= len(detected):
|
|
508
|
+
provider = str(detected[idx - 1]["id"])
|
|
509
|
+
api_key_env_var = api_key_env_var or str(detected[idx - 1]["api_key_env_var"])
|
|
510
|
+
if provider is not None:
|
|
511
|
+
return _configure_model_impl(provider, model, api_key_env_var, test_connection)
|
|
512
|
+
typer.echo('Mentor Model Provider options:')
|
|
513
|
+
for key, recipe in MODEL_PROVIDER_RECIPES.items():
|
|
514
|
+
typer.echo(f'- {key}: {recipe.display_name}')
|
|
515
|
+
provider=typer.prompt('Mentor Model Provider')
|
|
516
|
+
provider=provider.strip().lower()
|
|
517
|
+
if provider not in MODEL_PROVIDER_RECIPES:
|
|
518
|
+
raise typer.BadParameter(f'Mentor Model Provider must be one of: {", ".join(MODEL_PROVIDER_RECIPES)}')
|
|
519
|
+
recipe=MODEL_PROVIDER_RECIPES[provider]
|
|
520
|
+
env_var=api_key_env_var or recipe.api_key_env_var
|
|
521
|
+
selected_model=model or recipe.default_model
|
|
522
|
+
get_settings()
|
|
523
|
+
key_visible=bool(os.getenv(env_var))
|
|
524
|
+
if provider == "google" and not key_visible and os.getenv("GOOGLE_API_KEY"):
|
|
525
|
+
key_visible=True
|
|
526
|
+
env_var="GOOGLE_API_KEY"
|
|
527
|
+
readiness_status="untested" if key_visible else "missing_api_key"
|
|
528
|
+
readiness_reason=None if key_visible else f"{env_var} is not visible."
|
|
529
|
+
settings=update_settings(
|
|
530
|
+
model_provider=provider,
|
|
531
|
+
model_provider_api_key_env=env_var,
|
|
532
|
+
model_provider_model=selected_model,
|
|
533
|
+
mentor_model_provider_readiness_status=readiness_status,
|
|
534
|
+
mentor_model_provider_readiness_reason=readiness_reason,
|
|
535
|
+
)
|
|
536
|
+
if test_connection and key_visible:
|
|
537
|
+
out_dir=settings.app_home / "model_provider_smoke" / provider
|
|
538
|
+
counters=run_llm_smoke(out_dir, provider_id=provider)
|
|
539
|
+
role_names=("intake","rubric","grader","verifier","evaluator")
|
|
540
|
+
ready=bool(counters.get("secret_scan_ok")) and all(
|
|
541
|
+
counters.get(f"{name}_live_call_ok") and counters.get(f"{name}_structured_output_validation_ok")
|
|
542
|
+
for name in role_names
|
|
543
|
+
)
|
|
544
|
+
readiness_status="ready" if ready else "failed"
|
|
545
|
+
readiness_reason=None if ready else "Live Mentor Model Provider check failed; inspect smoke artifacts under " + str(out_dir)
|
|
546
|
+
settings=update_settings(
|
|
547
|
+
mentor_model_provider_readiness_status=readiness_status,
|
|
548
|
+
mentor_model_provider_readiness_reason=readiness_reason,
|
|
549
|
+
)
|
|
550
|
+
typer.echo('Mentor Model Provider configured')
|
|
551
|
+
typer.echo(f'Provider: {recipe.display_name}')
|
|
552
|
+
typer.echo(f'Model: {selected_model}')
|
|
553
|
+
typer.echo(f'API key env var: {env_var}')
|
|
554
|
+
typer.echo(f'API key visible: {_yes_no(key_visible)}')
|
|
555
|
+
typer.echo(f'Live test run: {"yes" if test_connection else "no"}')
|
|
556
|
+
typer.echo(_status_line('Mentor Model Provider Status', readiness_status, readiness_reason))
|
|
557
|
+
if not key_visible:
|
|
558
|
+
typer.echo(_env_next_step(env_var))
|
|
559
|
+
else:
|
|
560
|
+
typer.echo(f'Next: AA_RUN_LIVE_MODEL_PROVIDER_SMOKE={provider} bash scripts/live_model_provider_smoke.sh {provider}')
|
|
561
|
+
if test_connection:
|
|
562
|
+
if not key_visible:
|
|
563
|
+
raise typer.Exit(1)
|
|
564
|
+
return settings
|
|
565
|
+
|
|
566
|
+
def _ensure_evaluation_ready(settings, interactive: bool=True):
|
|
567
|
+
if settings.mentor_mode == 'expert_led' or model_provider_ready(settings):
|
|
568
|
+
return settings
|
|
569
|
+
if not interactive:
|
|
570
|
+
raise typer.BadParameter('model-assisted/hybrid Mentor Mode requires a ready Mentor Model Provider; use configure model or --mentor-mode expert-led')
|
|
571
|
+
typer.echo('Model-assisted Mentor Mode needs a ready Mentor Model Provider.')
|
|
572
|
+
readiness=mentor_model_provider_readiness(settings)
|
|
573
|
+
if readiness.get("provider"):
|
|
574
|
+
typer.echo(_status_line('Mentor Model Provider Status', readiness.get("status"), readiness.get("reason")))
|
|
575
|
+
typer.echo('')
|
|
576
|
+
typer.echo('Choose:')
|
|
577
|
+
typer.echo('1. Configure Mentor Model Provider')
|
|
578
|
+
typer.echo('2. Run this session in expert-led mode')
|
|
579
|
+
typer.echo('3. Cancel')
|
|
580
|
+
typer.echo('')
|
|
581
|
+
choice=typer.prompt('Default', default='expert-led').strip().lower()
|
|
582
|
+
if choice in {'1','setup','set up','configure','configure mentor model provider','set up model provider now'}:
|
|
583
|
+
configured=_configure_model_impl(None, None, None, test_connection=False)
|
|
584
|
+
return configured
|
|
585
|
+
if choice in {'2','expert-led','expert','manual','run this session in expert-led mode'}:
|
|
586
|
+
return settings.model_copy(update={'mentor_mode':'expert_led', 'evaluation_mode':'expert-led'})
|
|
587
|
+
raise typer.Exit(1)
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _ensure_apprentice_agent_ready(settings, runner: str | None=None, interactive: bool=True):
|
|
591
|
+
if runner == "deterministic":
|
|
592
|
+
return
|
|
593
|
+
status = apprentice_agent_readiness_status(settings)
|
|
594
|
+
if status.get("status") == "ready":
|
|
595
|
+
return
|
|
596
|
+
reason = status.get("reason")
|
|
597
|
+
if not interactive:
|
|
598
|
+
raise typer.BadParameter(reason or 'Apprentice Agent is not ready.')
|
|
599
|
+
if status.get("status") == "untested":
|
|
600
|
+
typer.echo('Apprentice Agent Status: Untested')
|
|
601
|
+
if reason:
|
|
602
|
+
typer.echo(f'Reason: {reason}')
|
|
603
|
+
typer.echo('Run a quick readiness check now?')
|
|
604
|
+
typer.echo('')
|
|
605
|
+
typer.echo('1. Run check')
|
|
606
|
+
typer.echo('2. Run anyway')
|
|
607
|
+
typer.echo('3. Configure Apprentice Agent')
|
|
608
|
+
typer.echo('4. Cancel')
|
|
609
|
+
typer.echo('')
|
|
610
|
+
choice=typer.prompt('Default', default='run anyway').strip().lower()
|
|
611
|
+
if choice in {'1','run check','check'}:
|
|
612
|
+
if status.get("command_found"):
|
|
613
|
+
update_settings(apprentice_agent_readiness_status="ready", apprentice_agent_readiness_reason="Command availability check passed.")
|
|
614
|
+
typer.echo('Apprentice Agent Status: Ready')
|
|
615
|
+
return
|
|
616
|
+
typer.echo(f'Reason: {reason}')
|
|
617
|
+
raise typer.Exit(1)
|
|
618
|
+
if choice in {'2','run anyway','continue','continue anyway'}:
|
|
619
|
+
return
|
|
620
|
+
if choice in {'3','configure','configure apprentice agent','configure agent'}:
|
|
621
|
+
typer.echo('Run: agent-apprenticeship configure agent')
|
|
622
|
+
raise typer.Exit(1)
|
|
623
|
+
raise typer.Exit(1)
|
|
624
|
+
typer.echo('Apprentice Agent is not ready.')
|
|
625
|
+
typer.echo(f'Reason: {reason}')
|
|
626
|
+
typer.echo('')
|
|
627
|
+
typer.echo('Choose:')
|
|
628
|
+
typer.echo('1. Configure Apprentice Agent')
|
|
629
|
+
typer.echo('2. Continue anyway')
|
|
630
|
+
typer.echo('3. Cancel')
|
|
631
|
+
typer.echo('')
|
|
632
|
+
choice=typer.prompt('Default', default='cancel').strip().lower()
|
|
633
|
+
if choice in {'1','configure','configure apprentice agent','configure agent'}:
|
|
634
|
+
typer.echo('Run: agent-apprenticeship configure agent')
|
|
635
|
+
raise typer.Exit(1)
|
|
636
|
+
if choice in {'2','continue','continue anyway'}:
|
|
637
|
+
return
|
|
638
|
+
raise typer.Exit(1)
|
|
639
|
+
|
|
640
|
+
@app.command('init')
|
|
641
|
+
def init(
|
|
642
|
+
overwrite: bool=typer.Option(False, help='Replace existing settings.json with defaults.'),
|
|
643
|
+
setup: bool=typer.Option(False, '--setup', help='Detect installed Apprentice Agents and Mentor Model Provider keys.'),
|
|
644
|
+
):
|
|
645
|
+
path=init_settings(overwrite=overwrite)
|
|
646
|
+
typer.echo(f'initialized Agent Apprenticeship settings at {path}')
|
|
647
|
+
if setup or sys.stdin.isatty():
|
|
648
|
+
_first_run_setup(interactive=bool(setup or sys.stdin.isatty()))
|
|
649
|
+
|
|
650
|
+
@app.command('settings')
|
|
651
|
+
def settings(
|
|
652
|
+
as_json: bool=typer.Option(False, '--json', help='Print settings as JSON.'),
|
|
653
|
+
debug: bool=typer.Option(False, '--debug', help='Include raw/internal compatibility settings.'),
|
|
654
|
+
):
|
|
655
|
+
if debug:
|
|
656
|
+
typer.echo(json.dumps(debug_settings(), indent=2, sort_keys=True))
|
|
657
|
+
elif as_json:
|
|
658
|
+
typer.echo(json.dumps(public_settings(), indent=2, sort_keys=True))
|
|
659
|
+
else:
|
|
660
|
+
typer.echo(_public_settings_text_with_ecosystem())
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def _latest_run_path(settings) -> Path | None:
|
|
664
|
+
runs = settings.app_home / "runs"
|
|
665
|
+
if not runs.exists():
|
|
666
|
+
return None
|
|
667
|
+
candidates = [p for p in runs.iterdir() if p.is_dir()]
|
|
668
|
+
if not candidates:
|
|
669
|
+
return None
|
|
670
|
+
return max(candidates, key=lambda p: p.stat().st_mtime)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
@app.command('doctor')
|
|
674
|
+
def doctor(
|
|
675
|
+
live: bool=typer.Option(False, '--live', help='Run live readiness checks when provider/tools are configured.'),
|
|
676
|
+
integrations: bool=typer.Option(False, '--integrations', help='Print selected integration health matrix.'),
|
|
677
|
+
):
|
|
678
|
+
if integrations:
|
|
679
|
+
_print_integrations_report()
|
|
680
|
+
return
|
|
681
|
+
settings=get_settings()
|
|
682
|
+
agent=apprentice_agent_readiness_status(settings)
|
|
683
|
+
provider=mentor_model_provider_readiness(settings)
|
|
684
|
+
typer.echo('Agent Apprenticeship Doctor')
|
|
685
|
+
typer.echo('')
|
|
686
|
+
typer.echo(f'App Home: {settings.app_home}')
|
|
687
|
+
typer.echo(f'Apprentice Agent: {public_settings(settings)["apprentice_agent"]}')
|
|
688
|
+
typer.echo(f'Apprentice Agent command: {agent.get("command") or "Not configured"}')
|
|
689
|
+
typer.echo(f'Apprentice Agent command found: {_yes_no(bool(agent.get("command_found")))}')
|
|
690
|
+
typer.echo(_status_line('Apprentice Agent readiness', agent.get('status'), agent.get('reason')))
|
|
691
|
+
typer.echo(f'Mentor Mode: {settings.mentor_mode}')
|
|
692
|
+
typer.echo(f'Mentor Model Provider: {provider.get("provider_display")}')
|
|
693
|
+
typer.echo(f'Mentor model: {provider.get("model") or "Not configured"}')
|
|
694
|
+
typer.echo(f'API key env var: {provider.get("api_key_env_var") or "Not configured"}')
|
|
695
|
+
typer.echo(f'API key visible: {_yes_no(bool(provider.get("api_key_visible")))}')
|
|
696
|
+
typer.echo(_status_line('Mentor Model Provider readiness', provider.get('status'), provider.get('reason')))
|
|
697
|
+
typer.echo(f'Maximum Improvement Loops: {settings.max_improvement_loops}')
|
|
698
|
+
typer.echo(f'Sensitive Info Masking: {settings.sensitive_info_masking}')
|
|
699
|
+
typer.echo(f'Public Ecosystem Repo: {settings.ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO}')
|
|
700
|
+
typer.echo(f'GitHub URL: {_ecosystem_repo_url(settings.ecosystem_repo)}')
|
|
701
|
+
typer.echo(_ecosystem_auto_share_status_line(settings).replace("\n", "\n"))
|
|
702
|
+
latest=_latest_run_path(settings)
|
|
703
|
+
typer.echo(f'Latest run: {latest if latest else "None"}')
|
|
704
|
+
if not live:
|
|
705
|
+
return
|
|
706
|
+
typer.echo('')
|
|
707
|
+
typer.echo('Live Checks')
|
|
708
|
+
if not provider.get('provider') or not provider.get('api_key_visible'):
|
|
709
|
+
typer.echo('Mentor Model Provider live check: SKIPPED - provider/API key is not ready.')
|
|
710
|
+
else:
|
|
711
|
+
out_dir=settings.app_home / 'doctor_live_model' / str(provider.get('provider'))
|
|
712
|
+
counters=run_llm_smoke(out_dir, provider_id=str(provider.get('provider')))
|
|
713
|
+
roles=['intake','rubric','grader','verifier','evaluator']
|
|
714
|
+
ok=bool(counters.get('mentor_model_provider_available')) and all(
|
|
715
|
+
counters.get(f'{r}_live_call_ok') and counters.get(f'{r}_structured_output_validation_ok')
|
|
716
|
+
for r in roles
|
|
717
|
+
) and bool(counters.get('secret_scan_ok'))
|
|
718
|
+
typer.echo(f'Mentor Model Provider live check: {"PASSED" if ok else "FAILED"}')
|
|
719
|
+
if ok:
|
|
720
|
+
update_settings(
|
|
721
|
+
mentor_model_provider_readiness_status="ready",
|
|
722
|
+
mentor_model_provider_readiness_reason="doctor --live passed.",
|
|
723
|
+
)
|
|
724
|
+
if not ok:
|
|
725
|
+
for key in sorted(k for k in counters if k.endswith('_error_message') or k.endswith('_error_type')):
|
|
726
|
+
typer.echo(f'{key}: {counters[key]}')
|
|
727
|
+
if agent.get('status') == 'missing_command':
|
|
728
|
+
typer.echo('Apprentice Agent live check: SKIPPED - command not found.')
|
|
729
|
+
elif agent.get('command_found'):
|
|
730
|
+
typer.echo('Apprentice Agent live check: PASSED - command availability check passed.')
|
|
731
|
+
else:
|
|
732
|
+
typer.echo('Apprentice Agent live check: SKIPPED - Apprentice Agent is not configured.')
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def _print_integrations_report() -> None:
|
|
736
|
+
report = integrations_report()
|
|
737
|
+
typer.echo("Agent Apprenticeship Integrations")
|
|
738
|
+
typer.echo("")
|
|
739
|
+
_print_integration_groups(report)
|
|
740
|
+
typer.echo("")
|
|
741
|
+
typer.echo("Apprentice Agents")
|
|
742
|
+
for row in report["apprentice_agents"]:
|
|
743
|
+
typer.echo(f"- {row['id']}: {row['display_name']}")
|
|
744
|
+
typer.echo(f" adapter implemented: {_yes_no(row['adapter_implemented'])}")
|
|
745
|
+
typer.echo(f" command expected: {row['command_expected']}")
|
|
746
|
+
typer.echo(f" command found: {_yes_no(row['command_found'])}")
|
|
747
|
+
typer.echo(f" fake adapter test covered: {_yes_no(row['fake_adapter_test_covered'])}")
|
|
748
|
+
typer.echo(f" live smoke script available: {_yes_no(row['live_smoke_script_available'])}")
|
|
749
|
+
typer.echo(f" latest local live smoke result: {row['latest_local_live_smoke_result']}")
|
|
750
|
+
typer.echo(f" latest full E2E result: {row['latest_full_e2e_result']}")
|
|
751
|
+
if row.get("last_tested_at"):
|
|
752
|
+
typer.echo(f" last tested: {row['last_tested_at']}")
|
|
753
|
+
if row["id"] == "custom":
|
|
754
|
+
typer.echo(f" custom fixture live smoke result: {row.get('custom_fixture_live_smoke_result') or 'not_run'}")
|
|
755
|
+
typer.echo(f" custom user-configured live smoke result: {row.get('custom_user_live_smoke_result') or 'not_run'}")
|
|
756
|
+
if row.get("latest_error_summary"):
|
|
757
|
+
typer.echo(f" latest error summary: {_concise_integration_error(row['latest_error_summary'], provider_id=row['id'])}")
|
|
758
|
+
if row.get("latest_full_e2e_error_summary"):
|
|
759
|
+
typer.echo(f" latest full E2E error summary: {_concise_integration_error(row['latest_full_e2e_error_summary'], provider_id=row['id'])}")
|
|
760
|
+
next_action = _integration_next_action(row, provider_type="apprentice_agent")
|
|
761
|
+
if next_action:
|
|
762
|
+
typer.echo(f" next action: {next_action}")
|
|
763
|
+
typer.echo("")
|
|
764
|
+
typer.echo("Mentor Model Providers")
|
|
765
|
+
for row in report["mentor_model_providers"]:
|
|
766
|
+
typer.echo(f"- {row['id']}: {row['display_name']}")
|
|
767
|
+
typer.echo(f" adapter implemented: {_yes_no(row['adapter_implemented'])}")
|
|
768
|
+
typer.echo(f" key env var: {row['key_env_var']}")
|
|
769
|
+
typer.echo(f" key visible: {_yes_no(row['key_visible'])}")
|
|
770
|
+
typer.echo(f" fake provider test covered: {_yes_no(row['fake_provider_test_covered'])}")
|
|
771
|
+
typer.echo(f" live smoke script available: {_yes_no(row['live_smoke_script_available'])}")
|
|
772
|
+
typer.echo(f" latest local live smoke result: {row['latest_local_live_smoke_result']}")
|
|
773
|
+
typer.echo(f" latest full E2E result: {row['latest_full_e2e_result']}")
|
|
774
|
+
if row.get("last_tested_at"):
|
|
775
|
+
typer.echo(f" last tested: {row['last_tested_at']}")
|
|
776
|
+
if row.get("latest_error_summary"):
|
|
777
|
+
typer.echo(f" latest error summary: {_concise_integration_error(row['latest_error_summary'], provider_id=row['id'])}")
|
|
778
|
+
if row.get("latest_full_e2e_error_summary"):
|
|
779
|
+
typer.echo(f" latest full E2E error summary: {_concise_integration_error(row['latest_full_e2e_error_summary'], provider_id=row['id'])}")
|
|
780
|
+
next_action = _integration_next_action(row, provider_type="mentor_model_provider")
|
|
781
|
+
if next_action:
|
|
782
|
+
typer.echo(f" next action: {next_action}")
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _status_is_passed(value: str | None) -> bool:
|
|
786
|
+
return value == "passed"
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _status_is_external_blocker(value: str | None) -> bool:
|
|
790
|
+
return value in {
|
|
791
|
+
"failed_auth",
|
|
792
|
+
"failed_quota",
|
|
793
|
+
"failed_insufficient_balance",
|
|
794
|
+
"skipped_missing_key",
|
|
795
|
+
"skipped_missing_command",
|
|
796
|
+
"skipped_not_configured",
|
|
797
|
+
"not_certified_due_to_provider_failure",
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def _status_is_framework_failure(value: str | None) -> bool:
|
|
802
|
+
return value in {"failed", "failed_output_contract", "failed_timeout", "failed_provider_error"}
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _concise_integration_error(text: object, *, provider_id: str | None = None) -> str:
|
|
806
|
+
raw = redact_secrets(str(text or "")).strip()
|
|
807
|
+
low = raw.lower()
|
|
808
|
+
if provider_id == "google" or "generativelanguage.googleapis.com" in low or "gemini" in low:
|
|
809
|
+
if "quota" in low or "rate" in low or "resource_exhausted" in low:
|
|
810
|
+
return "Google Gemini quota/rate limit reached."
|
|
811
|
+
if "structured" in low or "json" in low:
|
|
812
|
+
return "Google Gemini structured-output response could not be validated."
|
|
813
|
+
if provider_id == "openrouter" or "openrouter" in low:
|
|
814
|
+
if "requires more credits" in low or "can only afford" in low or "credit" in low or "402" in low:
|
|
815
|
+
return "OpenRouter credits/request-size limit reached."
|
|
816
|
+
if "parse" in low or "json" in low:
|
|
817
|
+
return "OpenRouter provider response could not be parsed."
|
|
818
|
+
if provider_id == "opencode" or "opencode setup required" in low:
|
|
819
|
+
return "OpenCode needs provider/API-key setup."
|
|
820
|
+
if provider_id == "cursor" or "cursor provider quota" in low:
|
|
821
|
+
return "Cursor-side provider quota/credit limit reached."
|
|
822
|
+
if "insufficient balance" in low:
|
|
823
|
+
return "Provider account balance is insufficient."
|
|
824
|
+
if "api key" in low or "auth" in low or "login" in low or "setup required" in low:
|
|
825
|
+
return raw[:240]
|
|
826
|
+
if "quota" in low or "credit" in low or "rate limit" in low:
|
|
827
|
+
return "Provider quota/credit limit reached."
|
|
828
|
+
return raw[:240]
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _integration_next_action(row: dict, *, provider_type: str) -> str | None:
|
|
832
|
+
result_values = {
|
|
833
|
+
str(row.get("latest_local_live_smoke_result") or ""),
|
|
834
|
+
str(row.get("latest_full_e2e_result") or ""),
|
|
835
|
+
}
|
|
836
|
+
if not any(_status_is_external_blocker(value) for value in result_values):
|
|
837
|
+
return None
|
|
838
|
+
ident = str(row.get("id") or "")
|
|
839
|
+
if provider_type == "mentor_model_provider":
|
|
840
|
+
if ident == "google":
|
|
841
|
+
return "Wait for Gemini quota reset or add billing/credits, then rerun `bash scripts/live_model_provider_smoke.sh google`."
|
|
842
|
+
if ident == "openrouter":
|
|
843
|
+
return "Add OpenRouter credits or choose a smaller model/request size, then rerun `bash scripts/live_model_provider_smoke.sh openrouter`."
|
|
844
|
+
if str(row.get("latest_local_live_smoke_result")) == "skipped_missing_key":
|
|
845
|
+
env_var = row.get("key_env_var") or "PROVIDER_API_KEY"
|
|
846
|
+
return f"Add {env_var}=... to ~/.agent-apprenticeship/.env.local, then rerun the provider smoke."
|
|
847
|
+
if ident == "opencode":
|
|
848
|
+
return "Configure OpenCode's own provider/API key, then rerun `bash scripts/live_agent_provider_smoke.sh opencode`."
|
|
849
|
+
if ident == "cursor":
|
|
850
|
+
return "Resolve the Cursor-side quota/credit limit, then rerun `bash scripts/live_agent_provider_smoke.sh cursor`."
|
|
851
|
+
if ident == "custom" and row.get("custom_user_live_smoke_result") == "skipped_not_configured":
|
|
852
|
+
return "Configure a Custom command template if you want to certify your own custom agent command."
|
|
853
|
+
if str(row.get("latest_local_live_smoke_result")).startswith("skipped_missing_command"):
|
|
854
|
+
return "Install the CLI or expose it on PATH, then rerun the agent smoke."
|
|
855
|
+
return None
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _join_or_none(items: list[str]) -> str:
|
|
859
|
+
return ", ".join(items) if items else "none"
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _print_integration_groups(report: dict) -> None:
|
|
863
|
+
agents = report["apprentice_agents"]
|
|
864
|
+
providers = report["mentor_model_providers"]
|
|
865
|
+
live_agents = [row["id"] for row in agents if _status_is_passed(row.get("latest_local_live_smoke_result"))]
|
|
866
|
+
live_providers = [row["id"] for row in providers if _status_is_passed(row.get("latest_local_live_smoke_result"))]
|
|
867
|
+
live_pairs = []
|
|
868
|
+
blocked = []
|
|
869
|
+
not_configured = []
|
|
870
|
+
framework_failures = []
|
|
871
|
+
for row in agents:
|
|
872
|
+
full = row.get("latest_full_e2e_result")
|
|
873
|
+
smoke = row.get("latest_local_live_smoke_result")
|
|
874
|
+
if _status_is_passed(full):
|
|
875
|
+
live_pairs.append(f"{row['id']}+configured-provider")
|
|
876
|
+
if _status_is_external_blocker(smoke) or _status_is_external_blocker(full):
|
|
877
|
+
target = not_configured if "skipped" in str(smoke) or "skipped" in str(full) else blocked
|
|
878
|
+
reason = row.get("latest_error_summary") or row.get("latest_full_e2e_error_summary") or smoke or full
|
|
879
|
+
target.append(f"{row['id']}: {_concise_integration_error(reason, provider_id=row['id'])}")
|
|
880
|
+
if _status_is_framework_failure(smoke) or _status_is_framework_failure(full):
|
|
881
|
+
reason = row.get("latest_error_summary") or row.get("latest_full_e2e_error_summary") or smoke or full
|
|
882
|
+
framework_failures.append(f"{row['id']}: {_concise_integration_error(reason, provider_id=row['id'])}")
|
|
883
|
+
for row in providers:
|
|
884
|
+
full = row.get("latest_full_e2e_result")
|
|
885
|
+
smoke = row.get("latest_local_live_smoke_result")
|
|
886
|
+
if _status_is_passed(full):
|
|
887
|
+
live_pairs.append(f"custom-or-agent+{row['id']}")
|
|
888
|
+
if _status_is_external_blocker(smoke) or _status_is_external_blocker(full):
|
|
889
|
+
target = not_configured if "skipped" in str(smoke) or "skipped" in str(full) else blocked
|
|
890
|
+
reason = row.get("latest_error_summary") or row.get("latest_full_e2e_error_summary") or smoke or full
|
|
891
|
+
target.append(f"{row['id']}: {_concise_integration_error(reason, provider_id=row['id'])}")
|
|
892
|
+
if _status_is_framework_failure(smoke) or _status_is_framework_failure(full):
|
|
893
|
+
reason = row.get("latest_error_summary") or row.get("latest_full_e2e_error_summary") or smoke or full
|
|
894
|
+
framework_failures.append(f"{row['id']}: {_concise_integration_error(reason, provider_id=row['id'])}")
|
|
895
|
+
typer.echo("Live Verified")
|
|
896
|
+
typer.echo(f"- Apprentice Agents: {_join_or_none(live_agents)}")
|
|
897
|
+
typer.echo(f"- Mentor Model Providers: {_join_or_none(live_providers)}")
|
|
898
|
+
typer.echo(f"- Full E2E: {_join_or_none(live_pairs)}")
|
|
899
|
+
typer.echo("")
|
|
900
|
+
typer.echo("External Setup/Account Blocked")
|
|
901
|
+
if blocked:
|
|
902
|
+
for item in blocked:
|
|
903
|
+
typer.echo(f"- {item}")
|
|
904
|
+
else:
|
|
905
|
+
typer.echo("- none")
|
|
906
|
+
typer.echo("")
|
|
907
|
+
typer.echo("Not Configured")
|
|
908
|
+
if not_configured:
|
|
909
|
+
for item in not_configured:
|
|
910
|
+
typer.echo(f"- {item}")
|
|
911
|
+
else:
|
|
912
|
+
typer.echo("- none")
|
|
913
|
+
typer.echo("")
|
|
914
|
+
typer.echo("Framework/Contract Failures")
|
|
915
|
+
if framework_failures:
|
|
916
|
+
for item in framework_failures:
|
|
917
|
+
typer.echo(f"- {item}")
|
|
918
|
+
else:
|
|
919
|
+
typer.echo("- none")
|
|
920
|
+
if REMOVED_V0_MODEL_PROVIDER_IDS:
|
|
921
|
+
typer.echo("")
|
|
922
|
+
typer.echo("Removed From V0")
|
|
923
|
+
for provider_id in REMOVED_V0_MODEL_PROVIDER_IDS:
|
|
924
|
+
typer.echo(f"- {provider_id}")
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
@app.command('integrations')
|
|
928
|
+
def integrations_command():
|
|
929
|
+
_print_integrations_report()
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def _print_certification_rows(title: str, rows: list[dict]) -> None:
|
|
933
|
+
typer.echo(title)
|
|
934
|
+
if not rows:
|
|
935
|
+
typer.echo("- not run")
|
|
936
|
+
return
|
|
937
|
+
for row in rows:
|
|
938
|
+
if "provider_id" in row and "agent_id" in row and row.get("certification_kind") == "full_e2e":
|
|
939
|
+
label = f"{row['agent_id']} + {row['provider_id']}"
|
|
940
|
+
else:
|
|
941
|
+
label = row.get("agent_id") or row.get("provider_id") or "unknown"
|
|
942
|
+
if row.get("mode"):
|
|
943
|
+
label = f"{label} ({row['mode']})"
|
|
944
|
+
provider_id = row.get("provider_id") or row.get("agent_id")
|
|
945
|
+
reason = f" - {_concise_integration_error(row['error_summary'], provider_id=provider_id)}" if row.get("error_summary") else ""
|
|
946
|
+
typer.echo(f"- {label}: {row['result']}{reason}")
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def _print_certification_groups(report: dict) -> None:
|
|
950
|
+
rows = [*(report.get("agents") or []), *(report.get("models") or []), *(report.get("full_e2e") or [])]
|
|
951
|
+
live = []
|
|
952
|
+
blocked = []
|
|
953
|
+
not_configured = []
|
|
954
|
+
framework_failures = []
|
|
955
|
+
for row in rows:
|
|
956
|
+
if "agent_id" in row and "provider_id" in row and row.get("certification_kind") == "full_e2e":
|
|
957
|
+
label = f"{row['agent_id']}+{row['provider_id']}"
|
|
958
|
+
else:
|
|
959
|
+
label = row.get("agent_id") or row.get("provider_id") or "unknown"
|
|
960
|
+
if row.get("mode"):
|
|
961
|
+
label = f"{label} ({row['mode']})"
|
|
962
|
+
result = str(row.get("result") or "")
|
|
963
|
+
provider_id = row.get("provider_id") or row.get("agent_id")
|
|
964
|
+
reason = _concise_integration_error(row.get("error_summary") or result, provider_id=provider_id)
|
|
965
|
+
if result == "passed":
|
|
966
|
+
live.append(label)
|
|
967
|
+
elif result.startswith("skipped"):
|
|
968
|
+
not_configured.append(f"{label}: {reason}")
|
|
969
|
+
elif _status_is_external_blocker(result):
|
|
970
|
+
blocked.append(f"{label}: {reason}")
|
|
971
|
+
elif _status_is_framework_failure(result):
|
|
972
|
+
framework_failures.append(f"{label}: {reason}")
|
|
973
|
+
typer.echo("Certification Groups")
|
|
974
|
+
typer.echo(f"- Live verified: {_join_or_none(live)}")
|
|
975
|
+
typer.echo("- External setup/account blocked:")
|
|
976
|
+
if blocked:
|
|
977
|
+
for item in blocked:
|
|
978
|
+
typer.echo(f" - {item}")
|
|
979
|
+
else:
|
|
980
|
+
typer.echo(" - none")
|
|
981
|
+
typer.echo("- Not configured:")
|
|
982
|
+
if not_configured:
|
|
983
|
+
for item in not_configured:
|
|
984
|
+
typer.echo(f" - {item}")
|
|
985
|
+
else:
|
|
986
|
+
typer.echo(" - none")
|
|
987
|
+
typer.echo("- Framework/contract failures:")
|
|
988
|
+
if framework_failures:
|
|
989
|
+
for item in framework_failures:
|
|
990
|
+
typer.echo(f" - {item}")
|
|
991
|
+
else:
|
|
992
|
+
typer.echo(" - none")
|
|
993
|
+
if REMOVED_V0_MODEL_PROVIDER_IDS:
|
|
994
|
+
typer.echo(f"- Removed from v0: {', '.join(REMOVED_V0_MODEL_PROVIDER_IDS)}")
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
@app.command('certify-integrations')
|
|
998
|
+
def certify_integrations(
|
|
999
|
+
agents: bool=typer.Option(False, '--agents', help='Run live certification for selected Apprentice Agents.'),
|
|
1000
|
+
models: bool=typer.Option(False, '--models', help='Run live certification for selected Mentor Model Providers.'),
|
|
1001
|
+
full_e2e: bool=typer.Option(False, '--full-e2e', help='Run bounded full E2E integration certification.'),
|
|
1002
|
+
all_: bool=typer.Option(False, '--all', help='Run agents, models, and bounded full E2E certification.'),
|
|
1003
|
+
all_combinations: bool=typer.Option(False, '--all-combinations', help='Run every Apprentice Agent and Mentor Model Provider pair.'),
|
|
1004
|
+
agent: list[str] | None=typer.Option(None, '--agent', help='Limit certification to one Apprentice Agent id. Can be repeated.'),
|
|
1005
|
+
model_provider: list[str] | None=typer.Option(None, '--model-provider', help='Limit certification to one Mentor Model Provider id. Can be repeated.'),
|
|
1006
|
+
pair: list[str] | None=typer.Option(None, '--pair', help='Limit full E2E to agent+provider. Can be repeated.'),
|
|
1007
|
+
as_json: bool=typer.Option(False, '--json', help='Emit certification results as JSON.'),
|
|
1008
|
+
strict: bool=typer.Option(False, '--strict', help='Treat skipped integrations as failures.'),
|
|
1009
|
+
):
|
|
1010
|
+
strict = strict or os.getenv("AA_CERTIFY_STRICT") == "1"
|
|
1011
|
+
include_agents = agents or all_
|
|
1012
|
+
include_models = models or all_
|
|
1013
|
+
include_full_e2e = full_e2e or all_
|
|
1014
|
+
if not (include_agents or include_models or include_full_e2e):
|
|
1015
|
+
include_agents = include_models = include_full_e2e = True
|
|
1016
|
+
agent_ids = agent or None
|
|
1017
|
+
provider_ids = model_provider or None
|
|
1018
|
+
if agent_ids:
|
|
1019
|
+
bad = [value for value in agent_ids if value not in WORKER_AGENT_RECIPES]
|
|
1020
|
+
if bad:
|
|
1021
|
+
raise typer.BadParameter(f'Apprentice Agent must be one of: {", ".join(WORKER_AGENT_RECIPES)}')
|
|
1022
|
+
if provider_ids:
|
|
1023
|
+
bad = [value for value in provider_ids if value not in MODEL_PROVIDER_RECIPES]
|
|
1024
|
+
if bad:
|
|
1025
|
+
raise typer.BadParameter(f'Mentor Model Provider must be one of: {", ".join(MODEL_PROVIDER_RECIPES)}')
|
|
1026
|
+
full_pairs = []
|
|
1027
|
+
for raw_pair in pair or []:
|
|
1028
|
+
if "+" not in raw_pair:
|
|
1029
|
+
raise typer.BadParameter('E2E pair must use agent+provider, for example codex+openai')
|
|
1030
|
+
agent_id, provider_id = [part.strip() for part in raw_pair.split("+", 1)]
|
|
1031
|
+
if agent_id not in WORKER_AGENT_RECIPES:
|
|
1032
|
+
raise typer.BadParameter(f'Apprentice Agent must be one of: {", ".join(WORKER_AGENT_RECIPES)}')
|
|
1033
|
+
if provider_id not in MODEL_PROVIDER_RECIPES:
|
|
1034
|
+
raise typer.BadParameter(f'Mentor Model Provider must be one of: {", ".join(MODEL_PROVIDER_RECIPES)}')
|
|
1035
|
+
full_pairs.append((agent_id, provider_id))
|
|
1036
|
+
report = run_certification_matrix(
|
|
1037
|
+
include_agents=include_agents,
|
|
1038
|
+
include_models=include_models,
|
|
1039
|
+
include_full_e2e=include_full_e2e,
|
|
1040
|
+
all_combinations=all_combinations,
|
|
1041
|
+
strict=strict,
|
|
1042
|
+
agent_ids=agent_ids,
|
|
1043
|
+
provider_ids=provider_ids,
|
|
1044
|
+
full_e2e_pairs=full_pairs or None,
|
|
1045
|
+
)
|
|
1046
|
+
if as_json:
|
|
1047
|
+
typer.echo(json.dumps(report, indent=2, sort_keys=True))
|
|
1048
|
+
else:
|
|
1049
|
+
typer.echo("Agent Apprenticeship Live Integration Certification")
|
|
1050
|
+
typer.echo("")
|
|
1051
|
+
_print_certification_groups(report)
|
|
1052
|
+
typer.echo("")
|
|
1053
|
+
_print_certification_rows("Apprentice Agents", report.get("agents") or [])
|
|
1054
|
+
typer.echo("")
|
|
1055
|
+
_print_certification_rows("Mentor Model Providers", report.get("models") or [])
|
|
1056
|
+
typer.echo("")
|
|
1057
|
+
_print_certification_rows("Full E2E", report.get("full_e2e") or [])
|
|
1058
|
+
typer.echo("")
|
|
1059
|
+
summary = report.get("summary") or {}
|
|
1060
|
+
typer.echo(
|
|
1061
|
+
f"Summary: passed={summary.get('passed', 0)} "
|
|
1062
|
+
f"failed={summary.get('failed', 0)} skipped={summary.get('skipped', 0)}"
|
|
1063
|
+
)
|
|
1064
|
+
code = certification_exit_code(report, strict=strict)
|
|
1065
|
+
if code:
|
|
1066
|
+
raise typer.Exit(code)
|
|
1067
|
+
|
|
1068
|
+
@configure_app.command('agent')
|
|
1069
|
+
def configure_agent(
|
|
1070
|
+
agent: str | None=typer.Argument(None, help='Known Apprentice Agent id.'),
|
|
1071
|
+
command_path: str | None=typer.Option(None, '--command-path', help='Command path/name for the selected agent.'),
|
|
1072
|
+
model: str | None=typer.Option(None, '--model', help='Optional model name passed through to the agent recipe.'),
|
|
1073
|
+
display_name: str | None=typer.Option(None, '--display-name', help='Custom Apprentice Agent display name.'),
|
|
1074
|
+
command_template: str | None=typer.Option(None, '--command-template', help='Custom command template using {workspace} and {prompt_file}.'),
|
|
1075
|
+
can_write_files: bool=typer.Option(True, '--can-write-files/--read-only', help='Whether the custom Apprentice Agent can write files in the workspace.'),
|
|
1076
|
+
smoke_test: bool=typer.Option(False, '--smoke-test/--no-smoke-test', help='Check whether the configured command is available.'),
|
|
1077
|
+
):
|
|
1078
|
+
if agent is None:
|
|
1079
|
+
typer.echo('Apprentice Agent options:')
|
|
1080
|
+
for key, recipe in WORKER_AGENT_RECIPES.items():
|
|
1081
|
+
typer.echo(f'- {key}: {recipe.display_name}')
|
|
1082
|
+
agent=typer.prompt('Apprentice Agent')
|
|
1083
|
+
agent=agent.strip().lower()
|
|
1084
|
+
if agent not in WORKER_AGENT_RECIPES:
|
|
1085
|
+
raise typer.BadParameter(f'Apprentice Agent must be one of: {", ".join(WORKER_AGENT_RECIPES)}')
|
|
1086
|
+
recipe=WORKER_AGENT_RECIPES[agent]
|
|
1087
|
+
command=command_path or recipe.command_name
|
|
1088
|
+
runner='codex' if agent == 'codex' else agent
|
|
1089
|
+
updates=dict(worker_agent=agent, worker_agent_command=command, worker_agent_model=model, worker_runner=runner, reviser_runner=runner)
|
|
1090
|
+
if agent == 'custom':
|
|
1091
|
+
if command_template is None:
|
|
1092
|
+
command_template=typer.prompt('Command template')
|
|
1093
|
+
missing=[token for token in ('{workspace}','{prompt_file}') if token not in command_template]
|
|
1094
|
+
if missing:
|
|
1095
|
+
raise typer.BadParameter('custom Apprentice Agent command template must include placeholders: '+', '.join(missing))
|
|
1096
|
+
updates.update(
|
|
1097
|
+
custom_worker_display_name=display_name or typer.prompt('Display name', default='Custom'),
|
|
1098
|
+
custom_worker_command_template=command_template,
|
|
1099
|
+
custom_worker_can_write_files=can_write_files,
|
|
1100
|
+
worker_agent_command=command_template.split()[0] if command_path is None else command_path,
|
|
1101
|
+
)
|
|
1102
|
+
check_command=command
|
|
1103
|
+
if agent == 'custom' and command_template:
|
|
1104
|
+
try:
|
|
1105
|
+
parts=shlex.split(command_template)
|
|
1106
|
+
except ValueError:
|
|
1107
|
+
parts=command_template.split()
|
|
1108
|
+
check_command=parts[0] if parts else command
|
|
1109
|
+
update_settings(**updates)
|
|
1110
|
+
typer.echo(f'Configured Apprentice Agent: {recipe.display_name if agent != "custom" else (display_name or "Custom")}')
|
|
1111
|
+
typer.echo(f'Command: {command}')
|
|
1112
|
+
resolved = resolve_command(check_command)
|
|
1113
|
+
typer.echo(f'Command found: {_yes_no(bool(resolved))}')
|
|
1114
|
+
if resolved and resolved != check_command:
|
|
1115
|
+
typer.echo(f'Resolved command: {resolved}')
|
|
1116
|
+
typer.echo('Readiness: not tested' if not smoke_test else 'Readiness check: command availability')
|
|
1117
|
+
typer.echo(f'Next: AA_RUN_LIVE_AGENT_PROVIDER_SMOKE={agent} bash scripts/live_agent_provider_smoke.sh {agent}')
|
|
1118
|
+
if smoke_test:
|
|
1119
|
+
ok=bool(resolve_command(check_command))
|
|
1120
|
+
update_settings(
|
|
1121
|
+
apprentice_agent_readiness_status=("ready" if ok else "missing_command"),
|
|
1122
|
+
apprentice_agent_readiness_reason=("Command availability check passed." if ok else f"Apprentice Agent command not found: {check_command}"),
|
|
1123
|
+
)
|
|
1124
|
+
typer.echo(f'apprentice_agent_command_available={str(ok).lower()}')
|
|
1125
|
+
typer.echo(_status_line('Apprentice Agent Status', 'ready' if ok else 'missing_command', None if ok else f"Apprentice Agent command not found: {check_command}"))
|
|
1126
|
+
if not ok:
|
|
1127
|
+
raise typer.Exit(1)
|
|
1128
|
+
else:
|
|
1129
|
+
update_settings(apprentice_agent_readiness_status=None, apprentice_agent_readiness_reason=None)
|
|
1130
|
+
|
|
1131
|
+
@configure_app.command('settings')
|
|
1132
|
+
def configure_settings(
|
|
1133
|
+
mentor_mode: str | None=typer.Option(None, '--mentor-mode', help='model-assisted, expert-led, or hybrid.'),
|
|
1134
|
+
sensitive_info_masking: str | None=typer.Option(None, '--sensitive-info-masking', help='standard or no-masking.'),
|
|
1135
|
+
max_loops: int | None=typer.Option(None, '--max-loops', help='Maximum improvement loops.'),
|
|
1136
|
+
):
|
|
1137
|
+
updates={}
|
|
1138
|
+
if mentor_mode is not None:
|
|
1139
|
+
updates['mentor_mode']=_normalize_mentor_mode(mentor_mode)
|
|
1140
|
+
if sensitive_info_masking is not None:
|
|
1141
|
+
updates['sensitive_info_masking']=_normalize_masking(sensitive_info_masking)
|
|
1142
|
+
if max_loops is not None:
|
|
1143
|
+
if max_loops < 1:
|
|
1144
|
+
raise typer.BadParameter('maximum improvement loops must be at least 1')
|
|
1145
|
+
updates['max_improvement_loops']=max_loops
|
|
1146
|
+
if not updates:
|
|
1147
|
+
typer.echo(public_settings_text())
|
|
1148
|
+
return
|
|
1149
|
+
update_settings(**updates)
|
|
1150
|
+
typer.echo(public_settings_text())
|
|
1151
|
+
|
|
1152
|
+
@configure_app.command('model')
|
|
1153
|
+
def configure_model(
|
|
1154
|
+
provider: str | None=typer.Argument(None, help='Known Mentor Model Provider id.'),
|
|
1155
|
+
model: str | None=typer.Option(None, '--model', help='Model name for evaluator/grader/verifier roles.'),
|
|
1156
|
+
api_key_env_var: str | None=typer.Option(None, '--api-key-env-var', help='Environment variable holding the API key.'),
|
|
1157
|
+
test_connection: bool=typer.Option(False, '--test-connection/--no-test-connection', help='Check whether the Mentor Model Provider API key env var is visible.'),
|
|
1158
|
+
):
|
|
1159
|
+
_configure_model_impl(provider, model, api_key_env_var, test_connection)
|
|
1160
|
+
|
|
1161
|
+
@app.command('start')
|
|
1162
|
+
def start(
|
|
1163
|
+
asset: list[Path]=typer.Option([], '--asset', '-a', help='Optional task asset file or directory.'),
|
|
1164
|
+
runner: str | None=typer.Option(None, '--runner', help='Advanced compatibility runner override.'),
|
|
1165
|
+
mentor_mode: str | None=typer.Option(None, '--mentor-mode', help='model-assisted, expert-led, or hybrid.'),
|
|
1166
|
+
sensitive_info_masking: str | None=typer.Option(None, '--sensitive-info-masking', help='standard or no-masking.'),
|
|
1167
|
+
evaluation_mode: str | None=typer.Option(None, '--evaluation-mode', help='Backward-compatible alias for --mentor-mode.'),
|
|
1168
|
+
data_sharing_level: str | None=typer.Option(None, '--data-sharing-level', help='Backward-compatible alias for --sensitive-info-masking.'),
|
|
1169
|
+
max_loops: int | None=typer.Option(None, '--max-loops', help='Maximum improvement loops for this run.'),
|
|
1170
|
+
quiet: bool=typer.Option(False, '--quiet', help='Print only the final task result.'),
|
|
1171
|
+
verbose: bool=typer.Option(False, '--verbose', help='Print additional progress details when available.'),
|
|
1172
|
+
json_progress: bool=typer.Option(False, '--json-progress', help='Emit progress events as JSONL.'),
|
|
1173
|
+
expert_auto_approve: bool=typer.Option(False, '--expert-auto-approve', help='Test/dev helper: auto-approve expert-led checkpoints.'),
|
|
1174
|
+
hybrid_auto_approve: bool=typer.Option(False, '--hybrid-auto-approve', help='Test/dev helper: auto-approve hybrid checkpoints.'),
|
|
1175
|
+
expert_interactive: bool=typer.Option(False, '--expert-interactive', help='Prompt for expert-led checkpoint inputs even in non-TTY test harnesses.'),
|
|
1176
|
+
hybrid_interactive: bool=typer.Option(False, '--hybrid-interactive', help='Prompt for hybrid approval inputs even in non-TTY test harnesses.'),
|
|
1177
|
+
):
|
|
1178
|
+
current_settings = get_settings()
|
|
1179
|
+
current_agent = apprentice_agent_readiness_status(current_settings)
|
|
1180
|
+
if sys.stdin.isatty() and current_agent.get("status") in {"missing_command", "not_ready"}:
|
|
1181
|
+
_first_run_setup(interactive=True)
|
|
1182
|
+
instruction=typer.prompt('What should your agent work on?')
|
|
1183
|
+
extra=typer.prompt('Add assets? optional', default='', show_default=False)
|
|
1184
|
+
assets=list(asset)+_split_asset_prompt(extra)
|
|
1185
|
+
settings=_ensure_evaluation_ready(_settings_for_run(mentor_mode, sensitive_info_masking, max_loops, evaluation_mode, data_sharing_level), interactive=True)
|
|
1186
|
+
_ensure_apprentice_agent_ready(settings, runner, interactive=True)
|
|
1187
|
+
interactive_checkpoints = _interactive_checkpoint_requested(
|
|
1188
|
+
settings,
|
|
1189
|
+
expert_auto_approve=expert_auto_approve,
|
|
1190
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1191
|
+
expert_interactive=expert_interactive,
|
|
1192
|
+
hybrid_interactive=hybrid_interactive,
|
|
1193
|
+
)
|
|
1194
|
+
with _temporary_auto_approve_env(
|
|
1195
|
+
expert_auto_approve=expert_auto_approve,
|
|
1196
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1197
|
+
mentor_interactive_checkpoints=interactive_checkpoints,
|
|
1198
|
+
):
|
|
1199
|
+
run_root,bundle=run_prompt_task(instruction, assets=assets, settings=settings, runner=runner, progress_callback=_progress_callback(quiet, verbose, json_progress))
|
|
1200
|
+
status = read_run_status(run_root)
|
|
1201
|
+
if quiet:
|
|
1202
|
+
_print_task_result(status, include_contribution_help=False)
|
|
1203
|
+
_handle_ecosystem_auto_share(status, quiet=quiet, json_progress=json_progress)
|
|
1204
|
+
|
|
1205
|
+
@app.command('run')
|
|
1206
|
+
def run_prompt(
|
|
1207
|
+
instruction: str=typer.Argument(..., help='Task instruction for the Apprentice Agent.'),
|
|
1208
|
+
asset: list[Path]=typer.Option([], '--asset', '-a', help='Optional task asset file or directory.'),
|
|
1209
|
+
run_id: str | None=typer.Option(None, '--run-id', help='Optional run id/slug.'),
|
|
1210
|
+
runner: str | None=typer.Option(None, '--runner', help='Advanced compatibility runner override.'),
|
|
1211
|
+
mentor_mode: str | None=typer.Option(None, '--mentor-mode', help='model-assisted, expert-led, or hybrid.'),
|
|
1212
|
+
sensitive_info_masking: str | None=typer.Option(None, '--sensitive-info-masking', help='standard or no-masking.'),
|
|
1213
|
+
evaluation_mode: str | None=typer.Option(None, '--evaluation-mode', help='Backward-compatible alias for --mentor-mode.'),
|
|
1214
|
+
data_sharing_level: str | None=typer.Option(None, '--data-sharing-level', help='Backward-compatible alias for --sensitive-info-masking.'),
|
|
1215
|
+
max_loops: int | None=typer.Option(None, '--max-loops', help='Maximum improvement loops for this run.'),
|
|
1216
|
+
quiet: bool=typer.Option(False, '--quiet', help='Print only the final task result.'),
|
|
1217
|
+
verbose: bool=typer.Option(False, '--verbose', help='Print additional progress details when available.'),
|
|
1218
|
+
json_progress: bool=typer.Option(False, '--json-progress', help='Emit progress events as JSONL.'),
|
|
1219
|
+
experience_pack: list[str]=typer.Option([], '--experience-pack', help='Experience Pack id/path to apply to this run.'),
|
|
1220
|
+
no_experience_packs: bool=typer.Option(False, '--no-experience-packs', help='Do not apply Experience Packs to this run.'),
|
|
1221
|
+
use_active_experience_packs: bool=typer.Option(False, '--use-active-experience-packs', help='Apply active Experience Packs to this run.'),
|
|
1222
|
+
expert_auto_approve: bool=typer.Option(False, '--expert-auto-approve', help='Test/dev helper: auto-approve expert-led checkpoints.'),
|
|
1223
|
+
hybrid_auto_approve: bool=typer.Option(False, '--hybrid-auto-approve', help='Test/dev helper: auto-approve hybrid checkpoints.'),
|
|
1224
|
+
expert_interactive: bool=typer.Option(False, '--expert-interactive', help='Prompt for expert-led checkpoint inputs even in non-TTY test harnesses.'),
|
|
1225
|
+
hybrid_interactive: bool=typer.Option(False, '--hybrid-interactive', help='Prompt for hybrid approval inputs even in non-TTY test harnesses.'),
|
|
1226
|
+
):
|
|
1227
|
+
settings=_ensure_evaluation_ready(_settings_for_run(mentor_mode, sensitive_info_masking, max_loops, evaluation_mode, data_sharing_level), interactive=True)
|
|
1228
|
+
_ensure_apprentice_agent_ready(settings, runner, interactive=True)
|
|
1229
|
+
for pack_id in experience_pack:
|
|
1230
|
+
try:
|
|
1231
|
+
_, pack = load_pack(pack_id, settings)
|
|
1232
|
+
except Exception:
|
|
1233
|
+
continue
|
|
1234
|
+
status = pack.get("status")
|
|
1235
|
+
if status in {"reverted", "removed"}:
|
|
1236
|
+
typer.echo(f"Experience Pack {pack.get('pack_id') or pack_id} is {status} and will not be applied.")
|
|
1237
|
+
selected_packs, pack_guidance = resolve_packs_for_run(
|
|
1238
|
+
experience_pack,
|
|
1239
|
+
use_active=use_active_experience_packs,
|
|
1240
|
+
no_packs=no_experience_packs,
|
|
1241
|
+
settings=settings,
|
|
1242
|
+
)
|
|
1243
|
+
effective_instruction = instruction.rstrip() + pack_guidance if pack_guidance else instruction
|
|
1244
|
+
experience_refs = pack_run_refs(selected_packs)
|
|
1245
|
+
interactive_checkpoints = _interactive_checkpoint_requested(
|
|
1246
|
+
settings,
|
|
1247
|
+
expert_auto_approve=expert_auto_approve,
|
|
1248
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1249
|
+
expert_interactive=expert_interactive,
|
|
1250
|
+
hybrid_interactive=hybrid_interactive,
|
|
1251
|
+
)
|
|
1252
|
+
with _temporary_auto_approve_env(
|
|
1253
|
+
expert_auto_approve=expert_auto_approve,
|
|
1254
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1255
|
+
mentor_interactive_checkpoints=interactive_checkpoints,
|
|
1256
|
+
):
|
|
1257
|
+
run_root,bundle=run_prompt_task(
|
|
1258
|
+
effective_instruction,
|
|
1259
|
+
assets=asset,
|
|
1260
|
+
run_id=run_id,
|
|
1261
|
+
settings=settings,
|
|
1262
|
+
runner=runner,
|
|
1263
|
+
progress_callback=_progress_callback(quiet, verbose, json_progress),
|
|
1264
|
+
experience_pack_refs=experience_refs,
|
|
1265
|
+
)
|
|
1266
|
+
status = read_run_status(run_root)
|
|
1267
|
+
if quiet:
|
|
1268
|
+
_print_task_result(status, include_contribution_help=False)
|
|
1269
|
+
_handle_ecosystem_auto_share(status, quiet=quiet, json_progress=json_progress)
|
|
1270
|
+
|
|
1271
|
+
@app.command('continue')
|
|
1272
|
+
def continue_run(
|
|
1273
|
+
run_id: str=typer.Argument(..., help='Run id or path to continue.'),
|
|
1274
|
+
followup_instruction: str | None=typer.Argument(None, help='Optional direct follow-up instruction.'),
|
|
1275
|
+
asset: list[Path]=typer.Option([], '--asset', '-a', help='Optional follow-up asset file or directory.'),
|
|
1276
|
+
run_loop: bool=typer.Option(False, '--run-loop', help='Run another improvement loop after recording the follow-up.'),
|
|
1277
|
+
record_only: bool=typer.Option(False, '--record-only', '--no-run-loop', help='Only record the follow-up and update the Contribution Bundle.'),
|
|
1278
|
+
runner: str | None=typer.Option(None, '--runner', help='Advanced compatibility runner override.'),
|
|
1279
|
+
quiet: bool=typer.Option(False, '--quiet', help='Print only the final task result.'),
|
|
1280
|
+
verbose: bool=typer.Option(False, '--verbose', help='Print additional progress details when available.'),
|
|
1281
|
+
json_progress: bool=typer.Option(False, '--json-progress', help='Emit progress events as JSONL.'),
|
|
1282
|
+
expert_interactive: bool=typer.Option(False, '--expert-interactive', help='Prompt for expert-led checkpoint inputs even in non-TTY test harnesses.'),
|
|
1283
|
+
hybrid_interactive: bool=typer.Option(False, '--hybrid-interactive', help='Prompt for hybrid approval inputs even in non-TTY test harnesses.'),
|
|
1284
|
+
expert_auto_approve: bool=typer.Option(False, '--expert-auto-approve', help='Test/dev helper: auto-approve expert-led checkpoints.'),
|
|
1285
|
+
hybrid_auto_approve: bool=typer.Option(False, '--hybrid-auto-approve', help='Test/dev helper: auto-approve hybrid checkpoints.'),
|
|
1286
|
+
):
|
|
1287
|
+
settings=get_settings()
|
|
1288
|
+
run_root=run_root_for(run_id, settings)
|
|
1289
|
+
if not quiet and not json_progress:
|
|
1290
|
+
typer.echo(f'Continuing run: {run_root.name}')
|
|
1291
|
+
if followup_instruction is None:
|
|
1292
|
+
followup_instruction=typer.prompt('Follow-up instruction')
|
|
1293
|
+
extra=typer.prompt('Add assets? optional', default='', show_default=False)
|
|
1294
|
+
asset=list(asset)+_split_asset_prompt(extra)
|
|
1295
|
+
if not run_loop and not record_only:
|
|
1296
|
+
run_loop=typer.confirm('Run another improvement loop?', default=False)
|
|
1297
|
+
elif not record_only and not run_loop:
|
|
1298
|
+
run_loop=True
|
|
1299
|
+
if record_only:
|
|
1300
|
+
run_loop=False
|
|
1301
|
+
if run_loop:
|
|
1302
|
+
_ensure_apprentice_agent_ready(settings, runner, interactive=True)
|
|
1303
|
+
if not quiet and not json_progress:
|
|
1304
|
+
typer.echo(f'Follow-up received: {followup_instruction[:80]}')
|
|
1305
|
+
typer.echo(f'Optional assets: {len(asset)}')
|
|
1306
|
+
interactive_checkpoints = _interactive_checkpoint_requested(
|
|
1307
|
+
settings,
|
|
1308
|
+
expert_auto_approve=expert_auto_approve,
|
|
1309
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1310
|
+
expert_interactive=expert_interactive,
|
|
1311
|
+
hybrid_interactive=hybrid_interactive,
|
|
1312
|
+
)
|
|
1313
|
+
with _temporary_auto_approve_env(
|
|
1314
|
+
expert_auto_approve=expert_auto_approve,
|
|
1315
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1316
|
+
mentor_interactive_checkpoints=interactive_checkpoints,
|
|
1317
|
+
):
|
|
1318
|
+
run_root,bundle=continue_session(run_id, followup_instruction, assets=asset, run_loop=run_loop, settings=settings, runner=runner, progress_callback=_progress_callback(quiet, verbose, json_progress))
|
|
1319
|
+
status = read_run_status(run_root)
|
|
1320
|
+
if quiet:
|
|
1321
|
+
_print_task_result(status, followup=True, record_only=not run_loop, include_contribution_help=False)
|
|
1322
|
+
_handle_ecosystem_auto_share(status, quiet=quiet, json_progress=json_progress)
|
|
1323
|
+
|
|
1324
|
+
@app.command('finish')
|
|
1325
|
+
def finish(
|
|
1326
|
+
run_id: str=typer.Argument(..., help='Run id or path to finish.'),
|
|
1327
|
+
quiet: bool=typer.Option(False, '--quiet', help='Print only the final task result.'),
|
|
1328
|
+
json_progress: bool=typer.Option(False, '--json-progress', help='Emit progress events as JSONL.'),
|
|
1329
|
+
expert_interactive: bool=typer.Option(False, '--expert-interactive', help='Prompt for expert-led final checkpoint inputs even in non-TTY test harnesses.'),
|
|
1330
|
+
hybrid_interactive: bool=typer.Option(False, '--hybrid-interactive', help='Prompt for hybrid final checkpoint inputs even in non-TTY test harnesses.'),
|
|
1331
|
+
expert_auto_approve: bool=typer.Option(False, '--expert-auto-approve', help='Test/dev helper: auto-approve expert-led checkpoints.'),
|
|
1332
|
+
hybrid_auto_approve: bool=typer.Option(False, '--hybrid-auto-approve', help='Test/dev helper: auto-approve hybrid checkpoints.'),
|
|
1333
|
+
):
|
|
1334
|
+
settings = get_settings()
|
|
1335
|
+
interactive_checkpoints = _interactive_checkpoint_requested(
|
|
1336
|
+
settings,
|
|
1337
|
+
expert_auto_approve=expert_auto_approve,
|
|
1338
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1339
|
+
expert_interactive=expert_interactive,
|
|
1340
|
+
hybrid_interactive=hybrid_interactive,
|
|
1341
|
+
)
|
|
1342
|
+
with _temporary_auto_approve_env(
|
|
1343
|
+
expert_auto_approve=expert_auto_approve,
|
|
1344
|
+
hybrid_auto_approve=hybrid_auto_approve,
|
|
1345
|
+
mentor_interactive_checkpoints=interactive_checkpoints,
|
|
1346
|
+
):
|
|
1347
|
+
run_root,bundle=finish_session(run_id, settings=settings)
|
|
1348
|
+
status=read_run_status(run_root)
|
|
1349
|
+
if json_progress:
|
|
1350
|
+
for event in read_jsonl(run_root/'progress_events.jsonl'):
|
|
1351
|
+
typer.echo(json.dumps(event, sort_keys=True))
|
|
1352
|
+
elif quiet:
|
|
1353
|
+
_print_session_finished(status, include_contribution_help=False)
|
|
1354
|
+
else:
|
|
1355
|
+
_print_session_finished(status)
|
|
1356
|
+
_handle_ecosystem_auto_share(status, quiet=quiet, json_progress=json_progress)
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
@app.command('status')
|
|
1360
|
+
def status(run_id: str=typer.Argument(..., help='Run id or path to inspect.')):
|
|
1361
|
+
run_root=run_root_for(run_id, get_settings())
|
|
1362
|
+
typer.echo(format_run_status(read_run_status(run_root)))
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
@app.command('watch')
|
|
1366
|
+
def watch(
|
|
1367
|
+
run_id: str=typer.Argument(..., help='Run id or path to watch.'),
|
|
1368
|
+
interval: float=typer.Option(1.0, '--interval', help='Polling interval in seconds.'),
|
|
1369
|
+
timeout: float | None=typer.Option(None, '--timeout', help='Optional maximum watch time in seconds.'),
|
|
1370
|
+
):
|
|
1371
|
+
run_root=run_root_for(run_id, get_settings())
|
|
1372
|
+
watch_progress(run_root, interval_seconds=interval, timeout_seconds=timeout, emit=typer.echo)
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _default_registry_path() -> Path:
|
|
1376
|
+
import os
|
|
1377
|
+
configured=os.getenv('AA_ECOSYSTEM_REGISTRY')
|
|
1378
|
+
if configured:
|
|
1379
|
+
return Path(configured).expanduser()
|
|
1380
|
+
return get_settings().app_home / 'ecosystem' / 'cache' / 'ecosystem_index.json'
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
def _configured_ecosystem_repo() -> str | None:
|
|
1384
|
+
return os.getenv("AA_ECOSYSTEM_REPO") or get_settings().ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _ecosystem_repo_url(repo: str | None = None) -> str:
|
|
1388
|
+
return f"https://github.com/{repo or _configured_ecosystem_repo() or DEFAULT_PUBLIC_ECOSYSTEM_REPO}"
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _configured_ecosystem_repo_path() -> Path | None:
|
|
1392
|
+
value = os.getenv("AA_ECOSYSTEM_REPO_PATH")
|
|
1393
|
+
if value:
|
|
1394
|
+
return Path(value).expanduser()
|
|
1395
|
+
return get_settings().ecosystem_repo_path
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _gh_path() -> str | None:
|
|
1399
|
+
return shutil.which("gh")
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def _run_gh(args: list[str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
|
1403
|
+
return subprocess.run(["gh", *args], text=True, capture_output=True, timeout=timeout)
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _gh_authenticated() -> bool:
|
|
1407
|
+
if not _gh_path():
|
|
1408
|
+
return False
|
|
1409
|
+
try:
|
|
1410
|
+
return _run_gh(["auth", "status"], timeout=10).returncode == 0
|
|
1411
|
+
except Exception:
|
|
1412
|
+
return False
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def _ecosystem_status() -> dict:
|
|
1416
|
+
repo = _configured_ecosystem_repo()
|
|
1417
|
+
repo_path = _configured_ecosystem_repo_path()
|
|
1418
|
+
gh_installed = bool(_gh_path())
|
|
1419
|
+
gh_authed = _gh_authenticated() if gh_installed else False
|
|
1420
|
+
return {
|
|
1421
|
+
"repo": repo,
|
|
1422
|
+
"repo_url": _ecosystem_repo_url(repo),
|
|
1423
|
+
"repo_path": str(repo_path) if repo_path else None,
|
|
1424
|
+
"repo_path_exists": bool(repo_path and repo_path.exists()),
|
|
1425
|
+
"gh_installed": gh_installed,
|
|
1426
|
+
"gh_authenticated": gh_authed,
|
|
1427
|
+
"automatic_contribution_ready": bool(repo and gh_installed and gh_authed),
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def _ecosystem_auto_share_readiness(repo: str | None = None) -> dict:
|
|
1432
|
+
configured_repo = repo or _configured_ecosystem_repo() or DEFAULT_PUBLIC_ECOSYSTEM_REPO
|
|
1433
|
+
gh_installed = bool(_gh_path())
|
|
1434
|
+
gh_authed = _gh_authenticated() if gh_installed else False
|
|
1435
|
+
if not configured_repo:
|
|
1436
|
+
return {
|
|
1437
|
+
"ready": False,
|
|
1438
|
+
"reason": "ecosystem repo is not configured",
|
|
1439
|
+
"next_action": "apprentice ecosystem configure --repo <owner>/<repo>",
|
|
1440
|
+
"repo": configured_repo,
|
|
1441
|
+
"repo_url": _ecosystem_repo_url(configured_repo),
|
|
1442
|
+
"gh_installed": gh_installed,
|
|
1443
|
+
"gh_authenticated": gh_authed,
|
|
1444
|
+
}
|
|
1445
|
+
if not gh_installed:
|
|
1446
|
+
return {
|
|
1447
|
+
"ready": False,
|
|
1448
|
+
"reason": "GitHub CLI is not installed",
|
|
1449
|
+
"next_action": "install GitHub CLI (`gh`)",
|
|
1450
|
+
"repo": configured_repo,
|
|
1451
|
+
"repo_url": _ecosystem_repo_url(configured_repo),
|
|
1452
|
+
"gh_installed": gh_installed,
|
|
1453
|
+
"gh_authenticated": gh_authed,
|
|
1454
|
+
}
|
|
1455
|
+
if not gh_authed:
|
|
1456
|
+
return {
|
|
1457
|
+
"ready": False,
|
|
1458
|
+
"reason": "GitHub CLI is not authenticated",
|
|
1459
|
+
"next_action": "gh auth login",
|
|
1460
|
+
"repo": configured_repo,
|
|
1461
|
+
"repo_url": _ecosystem_repo_url(configured_repo),
|
|
1462
|
+
"gh_installed": gh_installed,
|
|
1463
|
+
"gh_authenticated": gh_authed,
|
|
1464
|
+
}
|
|
1465
|
+
return {
|
|
1466
|
+
"ready": True,
|
|
1467
|
+
"reason": None,
|
|
1468
|
+
"next_action": None,
|
|
1469
|
+
"repo": configured_repo,
|
|
1470
|
+
"repo_url": _ecosystem_repo_url(configured_repo),
|
|
1471
|
+
"gh_installed": gh_installed,
|
|
1472
|
+
"gh_authenticated": gh_authed,
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
def _ecosystem_auto_share_status_line(settings=None) -> str:
|
|
1477
|
+
s = settings or get_settings()
|
|
1478
|
+
readiness = _ecosystem_auto_share_readiness()
|
|
1479
|
+
label = ecosystem_auto_share_display(s.ecosystem_auto_share)
|
|
1480
|
+
if s.ecosystem_auto_share == "manual":
|
|
1481
|
+
return f"Ecosystem Auto-Share: {label}"
|
|
1482
|
+
if readiness["ready"]:
|
|
1483
|
+
return f"Ecosystem Auto-Share: {label}\nEcosystem Auto-Share Status: Ready"
|
|
1484
|
+
reason = readiness.get("reason") or "not ready"
|
|
1485
|
+
return f"Ecosystem Auto-Share: {label}\nEcosystem Auto-Share Status: Not ready - {reason}"
|
|
1486
|
+
|
|
1487
|
+
|
|
1488
|
+
def _public_settings_text_with_ecosystem() -> str:
|
|
1489
|
+
settings = get_settings()
|
|
1490
|
+
lines = public_settings_text(settings).splitlines()
|
|
1491
|
+
status = _ecosystem_status()
|
|
1492
|
+
git_line = f"GitHub CLI: {'ready' if status['gh_installed'] and status['gh_authenticated'] else 'not ready'}"
|
|
1493
|
+
auto_status = _ecosystem_auto_share_status_line(settings).splitlines()
|
|
1494
|
+
out = []
|
|
1495
|
+
inserted = False
|
|
1496
|
+
for line in lines:
|
|
1497
|
+
out.append(line)
|
|
1498
|
+
if line.startswith("Ecosystem Repo:"):
|
|
1499
|
+
out.append(git_line)
|
|
1500
|
+
inserted = True
|
|
1501
|
+
if line.startswith("Public Ecosystem Repo:"):
|
|
1502
|
+
out.append(f"GitHub URL: {_ecosystem_repo_url(settings.ecosystem_repo)}")
|
|
1503
|
+
out.append(git_line)
|
|
1504
|
+
inserted = True
|
|
1505
|
+
if line.startswith("Ecosystem Auto-Share:"):
|
|
1506
|
+
out.pop()
|
|
1507
|
+
out.extend(auto_status)
|
|
1508
|
+
if not inserted:
|
|
1509
|
+
out.append(f"Public Ecosystem Repo: {settings.ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO}")
|
|
1510
|
+
out.append(f"GitHub URL: {_ecosystem_repo_url(settings.ecosystem_repo)}")
|
|
1511
|
+
out.append(git_line)
|
|
1512
|
+
out.extend(auto_status)
|
|
1513
|
+
return "\n".join(out)
|
|
1514
|
+
|
|
1515
|
+
|
|
1516
|
+
def _index_from_gh(repo: str) -> list[dict]:
|
|
1517
|
+
if not _gh_path():
|
|
1518
|
+
raise FileNotFoundError("GitHub CLI `gh` is not installed.")
|
|
1519
|
+
if not _gh_authenticated():
|
|
1520
|
+
raise FileNotFoundError("GitHub CLI `gh` is not authenticated.")
|
|
1521
|
+
last_error = ""
|
|
1522
|
+
for index_path in ["ecosystem_index.json", "bundles/index.json"]:
|
|
1523
|
+
cp = _run_gh(["api", f"repos/{repo}/contents/{index_path}"], timeout=30)
|
|
1524
|
+
if cp.returncode != 0:
|
|
1525
|
+
last_error = redact_secrets(cp.stderr or cp.stdout or "")
|
|
1526
|
+
continue
|
|
1527
|
+
payload = json.loads(cp.stdout or "{}")
|
|
1528
|
+
content = payload.get("content") or ""
|
|
1529
|
+
encoding = payload.get("encoding")
|
|
1530
|
+
if encoding == "base64":
|
|
1531
|
+
text = base64.b64decode(content).decode()
|
|
1532
|
+
else:
|
|
1533
|
+
text = content
|
|
1534
|
+
data = json.loads(text or "{}")
|
|
1535
|
+
rows = data.get("bundles") if isinstance(data, dict) else data
|
|
1536
|
+
return [row for row in rows or [] if isinstance(row, dict)]
|
|
1537
|
+
raise FileNotFoundError(
|
|
1538
|
+
f"Could not read public ecosystem index from {repo}. "
|
|
1539
|
+
f"Expected ecosystem_index.json or bundles/index.json. {last_error}".strip()
|
|
1540
|
+
)
|
|
1541
|
+
|
|
1542
|
+
|
|
1543
|
+
def _load_json_rows(path: Path) -> list[dict]:
|
|
1544
|
+
if not path.exists():
|
|
1545
|
+
return []
|
|
1546
|
+
if path.suffix == ".jsonl":
|
|
1547
|
+
return [row for row in read_jsonl(path) if isinstance(row, dict)]
|
|
1548
|
+
data = read_json(path)
|
|
1549
|
+
if isinstance(data, list):
|
|
1550
|
+
return [row for row in data if isinstance(row, dict)]
|
|
1551
|
+
if isinstance(data, dict):
|
|
1552
|
+
rows = (
|
|
1553
|
+
data.get("bundles")
|
|
1554
|
+
or data.get("items")
|
|
1555
|
+
or data.get("entries")
|
|
1556
|
+
or data.get("contributions")
|
|
1557
|
+
or []
|
|
1558
|
+
)
|
|
1559
|
+
return [row for row in rows if isinstance(row, dict)]
|
|
1560
|
+
return []
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
def _absolutize_public_repo_paths(row: dict, repo_path: Path) -> dict:
|
|
1564
|
+
out = dict(row)
|
|
1565
|
+
path_keys = {
|
|
1566
|
+
"task_path",
|
|
1567
|
+
"task_packet_path",
|
|
1568
|
+
"rubric_path",
|
|
1569
|
+
"worker_visible_rubric_path",
|
|
1570
|
+
"verifier_private_rubric_path",
|
|
1571
|
+
"attempts_path",
|
|
1572
|
+
"evaluation_path",
|
|
1573
|
+
"learning_signals_path",
|
|
1574
|
+
"local_bundle_path",
|
|
1575
|
+
"bundle_path",
|
|
1576
|
+
"bundle_path_or_url",
|
|
1577
|
+
}
|
|
1578
|
+
for key in path_keys:
|
|
1579
|
+
value = out.get(key)
|
|
1580
|
+
if value:
|
|
1581
|
+
candidate = Path(str(value)).expanduser()
|
|
1582
|
+
out[key] = str(candidate if candidate.is_absolute() else repo_path / candidate)
|
|
1583
|
+
if isinstance(out.get("trace_paths"), list):
|
|
1584
|
+
traces = []
|
|
1585
|
+
for value in out["trace_paths"]:
|
|
1586
|
+
candidate = Path(str(value)).expanduser()
|
|
1587
|
+
traces.append(str(candidate if candidate.is_absolute() else repo_path / candidate))
|
|
1588
|
+
out["trace_paths"] = traces
|
|
1589
|
+
return out
|
|
1590
|
+
|
|
1591
|
+
|
|
1592
|
+
def _index_from_public_repo_path(repo_path: Path) -> list[dict]:
|
|
1593
|
+
repo_path = repo_path.expanduser()
|
|
1594
|
+
if not repo_path.exists():
|
|
1595
|
+
raise FileNotFoundError(f"Public ecosystem repo path not found: {repo_path}")
|
|
1596
|
+
rows: list[dict] = []
|
|
1597
|
+
seed_root = repo_path / "seed_dataset"
|
|
1598
|
+
if seed_root.exists():
|
|
1599
|
+
for path in [seed_root / "ecosystem_registry.jsonl", seed_root / "ecosystem_registry.json"]:
|
|
1600
|
+
rows.extend(_absolutize_public_repo_paths(row, repo_path) for row in _load_json_rows(path))
|
|
1601
|
+
contributions_root = repo_path / "ecosystem" / "contributions"
|
|
1602
|
+
if contributions_root.exists():
|
|
1603
|
+
for path in [contributions_root / "index.json", contributions_root / "index.jsonl"]:
|
|
1604
|
+
for row in _load_json_rows(path):
|
|
1605
|
+
bundle_id = row.get("bundle_id")
|
|
1606
|
+
out = dict(row)
|
|
1607
|
+
out.setdefault("kind", "contribution_bundle")
|
|
1608
|
+
out.setdefault("experience_source_type", "contribution_bundle")
|
|
1609
|
+
if bundle_id and not any(out.get(key) for key in ["local_bundle_path", "bundle_path", "bundle_path_or_url"]):
|
|
1610
|
+
bundle_path = contributions_root / "bundles" / str(bundle_id)
|
|
1611
|
+
if bundle_path.exists():
|
|
1612
|
+
out["local_bundle_path"] = str(bundle_path)
|
|
1613
|
+
rows.append(_absolutize_public_repo_paths(out, repo_path))
|
|
1614
|
+
deduped: list[dict] = []
|
|
1615
|
+
seen: set[str] = set()
|
|
1616
|
+
for row in rows:
|
|
1617
|
+
key = str(row.get("bundle_id") or row.get("seed_task_id") or row.get("task_id") or "")
|
|
1618
|
+
if key and key in seen:
|
|
1619
|
+
continue
|
|
1620
|
+
if key:
|
|
1621
|
+
seen.add(key)
|
|
1622
|
+
deduped.append(row)
|
|
1623
|
+
return deduped
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
def _load_registry_file(path: Path) -> list[dict]:
|
|
1627
|
+
data = read_json(path)
|
|
1628
|
+
if isinstance(data, dict) and data.get("kind") == "public_ecosystem_index":
|
|
1629
|
+
repo_path = path.parent.parent if path.parent.name == "ecosystem" else path.parent
|
|
1630
|
+
return _index_from_public_repo_path(repo_path)
|
|
1631
|
+
if isinstance(data, dict):
|
|
1632
|
+
rows = (
|
|
1633
|
+
data.get('bundles')
|
|
1634
|
+
or data.get('items')
|
|
1635
|
+
or data.get('entries')
|
|
1636
|
+
or data.get('contributions')
|
|
1637
|
+
or []
|
|
1638
|
+
)
|
|
1639
|
+
else:
|
|
1640
|
+
rows = data
|
|
1641
|
+
return [row for row in rows if isinstance(row, dict)]
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
def _load_registry(registry: Path | None=None) -> list[dict]:
|
|
1645
|
+
if registry is None:
|
|
1646
|
+
repo_path = _configured_ecosystem_repo_path()
|
|
1647
|
+
if repo_path:
|
|
1648
|
+
return _index_from_public_repo_path(repo_path)
|
|
1649
|
+
repo = _configured_ecosystem_repo()
|
|
1650
|
+
if repo:
|
|
1651
|
+
try:
|
|
1652
|
+
return _index_from_gh(repo)
|
|
1653
|
+
except FileNotFoundError as exc:
|
|
1654
|
+
seed_rows = search_learning_sources(None)
|
|
1655
|
+
if repo == DEFAULT_PUBLIC_ECOSYSTEM_REPO and seed_rows:
|
|
1656
|
+
return seed_rows
|
|
1657
|
+
raise FileNotFoundError(
|
|
1658
|
+
f"Public ecosystem repo is configured as {repo}, but the index could not be read. {exc}"
|
|
1659
|
+
)
|
|
1660
|
+
path=_default_registry_path()
|
|
1661
|
+
if not path.exists():
|
|
1662
|
+
seed_rows = search_learning_sources(None)
|
|
1663
|
+
if seed_rows:
|
|
1664
|
+
return seed_rows
|
|
1665
|
+
raise FileNotFoundError(
|
|
1666
|
+
f"Public ecosystem repo is {DEFAULT_PUBLIC_ECOSYSTEM_REPO}, but no readable public index cache was found. "
|
|
1667
|
+
"Run `apprentice ecosystem configure --repo <owner>/<repo>` to override it, or pass "
|
|
1668
|
+
"`--registry <ecosystem_index.json>` for a local public-index file."
|
|
1669
|
+
)
|
|
1670
|
+
else:
|
|
1671
|
+
path=registry
|
|
1672
|
+
if not path.exists():
|
|
1673
|
+
raise FileNotFoundError(
|
|
1674
|
+
f'Public ecosystem index not found: {path}. '
|
|
1675
|
+
'Run `apprentice ecosystem guide` for setup steps.'
|
|
1676
|
+
)
|
|
1677
|
+
return _load_registry_file(path)
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
def _matches_registry_row(row: dict, query: str | None, filters: dict) -> bool:
|
|
1681
|
+
if query:
|
|
1682
|
+
haystack=' '.join(str(row.get(k) or '') for k in ['bundle_id','title','agent_apprentice_role','expected_economic_value'])
|
|
1683
|
+
haystack+=' '+' '.join(str(v) for v in row.get('domains') or [])
|
|
1684
|
+
haystack+=' '+' '.join(str(v) for v in row.get('subdomains') or [])
|
|
1685
|
+
if query.lower() not in haystack.lower():
|
|
1686
|
+
return False
|
|
1687
|
+
for key in ['title','mentor_mode','task_status','run_status']:
|
|
1688
|
+
value=filters.get(key)
|
|
1689
|
+
if value and str(row.get(key) or '').lower() != str(value).lower():
|
|
1690
|
+
return False
|
|
1691
|
+
for key, field in [('domain','domains'), ('subdomain','subdomains')]:
|
|
1692
|
+
value=filters.get(key)
|
|
1693
|
+
if value and value.lower() not in [str(v).lower() for v in row.get(field) or []]:
|
|
1694
|
+
return False
|
|
1695
|
+
if filters.get('min_traced_steps') is not None and int(row.get('traced_steps') or 0) < filters['min_traced_steps']:
|
|
1696
|
+
return False
|
|
1697
|
+
if filters.get('max_traced_steps') is not None and int(row.get('traced_steps') or 0) > filters['max_traced_steps']:
|
|
1698
|
+
return False
|
|
1699
|
+
if filters.get('expected_economic_value') and str(filters['expected_economic_value']).lower() not in str(row.get('expected_economic_value') or '').lower():
|
|
1700
|
+
return False
|
|
1701
|
+
if filters.get('artifact_type'):
|
|
1702
|
+
types=[str(v).lower() for v in row.get('artifact_types') or row.get('artifacts') or []]
|
|
1703
|
+
if str(filters['artifact_type']).lower() not in types:
|
|
1704
|
+
return False
|
|
1705
|
+
if filters.get('apprentice_agent'):
|
|
1706
|
+
values=[
|
|
1707
|
+
row.get('apprentice_agent'),
|
|
1708
|
+
row.get('agent'),
|
|
1709
|
+
row.get('worker_agent'),
|
|
1710
|
+
]
|
|
1711
|
+
if str(filters['apprentice_agent']).lower() not in [str(v).lower() for v in values if v]:
|
|
1712
|
+
return False
|
|
1713
|
+
if filters.get('mentor_model_provider'):
|
|
1714
|
+
values=[
|
|
1715
|
+
row.get('mentor_model_provider'),
|
|
1716
|
+
row.get('model_provider'),
|
|
1717
|
+
row.get('provider'),
|
|
1718
|
+
]
|
|
1719
|
+
if str(filters['mentor_model_provider']).lower() not in [str(v).lower() for v in values if v]:
|
|
1720
|
+
return False
|
|
1721
|
+
return True
|
|
1722
|
+
|
|
1723
|
+
|
|
1724
|
+
def _print_registry_rows(rows: list[dict]) -> None:
|
|
1725
|
+
if not rows:
|
|
1726
|
+
typer.echo('No ecosystem bundles matched.')
|
|
1727
|
+
return
|
|
1728
|
+
for row in rows:
|
|
1729
|
+
typer.echo(_format_registry_row(row))
|
|
1730
|
+
|
|
1731
|
+
|
|
1732
|
+
def _is_seed_registry_row(row: dict) -> bool:
|
|
1733
|
+
return bool(row.get("seed_task_id") or row.get("task_packet_path") or row.get("learning_signals_path"))
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def _format_count(value) -> str:
|
|
1737
|
+
return str(int(value)) if isinstance(value, (int, float)) or str(value).isdigit() else "0"
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
def _format_seed_registry_row(row: dict) -> str:
|
|
1741
|
+
bundle_id = row.get("bundle_id") or row.get("seed_task_id") or row.get("task_id")
|
|
1742
|
+
status = row.get("task_status") or "ready_to_run"
|
|
1743
|
+
run_status = row.get("run_status") or "not_run"
|
|
1744
|
+
trace_count = row.get("trace_count")
|
|
1745
|
+
if trace_count is None:
|
|
1746
|
+
trace_count = len(row.get("trace_paths") or [])
|
|
1747
|
+
attempt_count = row.get("attempt_count") or row.get("attempts") or 0
|
|
1748
|
+
learning_count = row.get("training_signal_rows") or row.get("lessons_count") or 0
|
|
1749
|
+
learning_label = f"learning signals: {_format_count(learning_count)}" if learning_count else "learning signals: available"
|
|
1750
|
+
details = [
|
|
1751
|
+
f"{status} / {run_status}",
|
|
1752
|
+
f"traces: {_format_count(trace_count)}",
|
|
1753
|
+
f"attempts: {_format_count(attempt_count)}",
|
|
1754
|
+
]
|
|
1755
|
+
if row.get("process_supervision_rows") is not None:
|
|
1756
|
+
details.append(f"process rows: {_format_count(row.get('process_supervision_rows'))}")
|
|
1757
|
+
if row.get("reward_modeling_rows") is not None:
|
|
1758
|
+
details.append(f"reward rows: {_format_count(row.get('reward_modeling_rows'))}")
|
|
1759
|
+
if row.get("revision_preference_pairs") is not None:
|
|
1760
|
+
details.append(f"preferences: {_format_count(row.get('revision_preference_pairs'))}")
|
|
1761
|
+
details.append(learning_label)
|
|
1762
|
+
return f"{bundle_id}: {row.get('title')} [{' · '.join(details)}]"
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
def _format_registry_row(row: dict) -> str:
|
|
1766
|
+
if _is_seed_registry_row(row):
|
|
1767
|
+
return _format_seed_registry_row(row)
|
|
1768
|
+
details = [
|
|
1769
|
+
str(row.get("task_status") or row.get("run_status") or "status_unknown"),
|
|
1770
|
+
]
|
|
1771
|
+
if row.get("mentor_mode"):
|
|
1772
|
+
details.append(f"mentor_mode={row.get('mentor_mode')}")
|
|
1773
|
+
traces = row.get("traced_steps")
|
|
1774
|
+
if traces is not None:
|
|
1775
|
+
details.append(f"traces: {_format_count(traces)}")
|
|
1776
|
+
artifacts = row.get("artifact_count")
|
|
1777
|
+
if artifacts is not None:
|
|
1778
|
+
details.append(f"artifacts: {_format_count(artifacts)}")
|
|
1779
|
+
return (
|
|
1780
|
+
f"{row.get('bundle_id')}: {row.get('title')} "
|
|
1781
|
+
f"[{' · '.join(details)}]"
|
|
1782
|
+
)
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
def _safe_bundle_path(path: Path) -> Path:
|
|
1786
|
+
bundle = path.expanduser()
|
|
1787
|
+
manifest = bundle / "contribution_manifest.json"
|
|
1788
|
+
if not manifest.exists():
|
|
1789
|
+
raise FileNotFoundError(f"Contribution Bundle manifest not found: {manifest}")
|
|
1790
|
+
return bundle
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def _artifact_index(bundle: Path) -> list[dict]:
|
|
1794
|
+
path = bundle / "outputs" / "artifacts_index.json"
|
|
1795
|
+
if not path.exists():
|
|
1796
|
+
return []
|
|
1797
|
+
data = read_json(path)
|
|
1798
|
+
return data if isinstance(data, list) else []
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def _session_events(bundle: Path) -> list[dict]:
|
|
1802
|
+
return read_jsonl(bundle / "session_events.jsonl")
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def _session_metadata(bundle: Path) -> dict:
|
|
1806
|
+
path = bundle / "session_metadata.json"
|
|
1807
|
+
return read_json(path) if path.exists() else {}
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def _text_file_has_secret(path: Path) -> bool:
|
|
1811
|
+
if path.stat().st_size > 5_000_000:
|
|
1812
|
+
return False
|
|
1813
|
+
try:
|
|
1814
|
+
return contains_secret(path.read_text(errors="ignore"))
|
|
1815
|
+
except UnicodeDecodeError:
|
|
1816
|
+
return False
|
|
1817
|
+
|
|
1818
|
+
|
|
1819
|
+
def _bundle_secret_hits(bundle: Path) -> list[str]:
|
|
1820
|
+
hits = []
|
|
1821
|
+
for path in bundle.rglob("*"):
|
|
1822
|
+
if path.is_file() and path.name not in {".env", ".env.local"} and _text_file_has_secret(path):
|
|
1823
|
+
hits.append(str(path.relative_to(bundle)))
|
|
1824
|
+
return hits
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def _bundle_submission_metadata(bundle: Path, package_name: str | None = None, package_hash: str | None = None) -> dict:
|
|
1828
|
+
manifest = read_json(bundle / "contribution_manifest.json")
|
|
1829
|
+
session = _session_metadata(bundle)
|
|
1830
|
+
artifacts = _artifact_index(bundle)
|
|
1831
|
+
events = _session_events(bundle)
|
|
1832
|
+
followups = [event for event in events if event.get("event_type") == "user_followup"]
|
|
1833
|
+
created_at = datetime.now(timezone.utc).isoformat()
|
|
1834
|
+
metadata = {
|
|
1835
|
+
"bundle_id": manifest.get("bundle_id"),
|
|
1836
|
+
"title": manifest.get("title"),
|
|
1837
|
+
"task_status": manifest.get("task_status"),
|
|
1838
|
+
"run_status": manifest.get("run_status"),
|
|
1839
|
+
"apprentice_agent": session.get("apprentice_agent"),
|
|
1840
|
+
"mentor_mode": manifest.get("mentor_mode"),
|
|
1841
|
+
"mentor_model_provider": session.get("model_provider"),
|
|
1842
|
+
"domains": manifest.get("domains") or [],
|
|
1843
|
+
"subdomains": manifest.get("subdomains") or [],
|
|
1844
|
+
"attempts": manifest.get("attempts"),
|
|
1845
|
+
"traced_steps": manifest.get("traced_steps"),
|
|
1846
|
+
"artifact_count": len(artifacts),
|
|
1847
|
+
"follow_up_count": len(followups),
|
|
1848
|
+
"process_supervision_rows": manifest.get("process_supervision_rows"),
|
|
1849
|
+
"reward_modeling_rows": manifest.get("reward_modeling_rows"),
|
|
1850
|
+
"revision_preference_pairs": manifest.get("revision_preference_pairs"),
|
|
1851
|
+
"expected_economic_value": manifest.get("expected_economic_value"),
|
|
1852
|
+
"expected_economic_value_for_agent_apprentice": manifest.get("expected_economic_value_for_agent_apprentice"),
|
|
1853
|
+
"created_at": created_at,
|
|
1854
|
+
"package_name": package_name,
|
|
1855
|
+
"package_hash": package_hash,
|
|
1856
|
+
}
|
|
1857
|
+
forbidden = {
|
|
1858
|
+
"reviewer",
|
|
1859
|
+
"review_decision",
|
|
1860
|
+
"accepted",
|
|
1861
|
+
"rejected",
|
|
1862
|
+
"quality_score",
|
|
1863
|
+
"featured",
|
|
1864
|
+
"curated",
|
|
1865
|
+
"source_url_or_ref",
|
|
1866
|
+
"source_kind",
|
|
1867
|
+
"source_url",
|
|
1868
|
+
"source_ref",
|
|
1869
|
+
"source_license",
|
|
1870
|
+
"source_release_schema_version",
|
|
1871
|
+
"trace_context",
|
|
1872
|
+
}
|
|
1873
|
+
return {key: value for key, value in metadata.items() if key not in forbidden}
|
|
1874
|
+
|
|
1875
|
+
|
|
1876
|
+
def _submission_summary(metadata: dict, bundle: Path, package_zip: Path | None = None) -> str:
|
|
1877
|
+
lines = [
|
|
1878
|
+
"# Agent Apprenticeship Ecosystem Submission",
|
|
1879
|
+
"",
|
|
1880
|
+
"## ecosystem_submission.json summary",
|
|
1881
|
+
"",
|
|
1882
|
+
f"Bundle ID: {metadata.get('bundle_id')}",
|
|
1883
|
+
f"Title: {metadata.get('title')}",
|
|
1884
|
+
f"Task Status: {metadata.get('task_status')}",
|
|
1885
|
+
f"Run Status: {metadata.get('run_status')}",
|
|
1886
|
+
f"Apprentice Agent: {metadata.get('apprentice_agent') or 'unknown'}",
|
|
1887
|
+
f"Mentor Mode: {metadata.get('mentor_mode')}",
|
|
1888
|
+
f"Mentor Model Provider: {metadata.get('mentor_model_provider') or 'none'}",
|
|
1889
|
+
f"Attempts: {metadata.get('attempts')}",
|
|
1890
|
+
f"Traced Steps: {metadata.get('traced_steps')}",
|
|
1891
|
+
f"Process Supervision Rows: {metadata.get('process_supervision_rows')}",
|
|
1892
|
+
f"Reward Modeling Rows: {metadata.get('reward_modeling_rows')}",
|
|
1893
|
+
f"Revision Preference Pairs: {metadata.get('revision_preference_pairs')}",
|
|
1894
|
+
f"Artifacts: {metadata.get('artifact_count')}",
|
|
1895
|
+
f"Follow-ups: {metadata.get('follow_up_count')}",
|
|
1896
|
+
f"Domains: {', '.join(metadata.get('domains') or []) or 'none'}",
|
|
1897
|
+
f"Subdomains: {', '.join(metadata.get('subdomains') or []) or 'none'}",
|
|
1898
|
+
f"Expected Economic Value: {metadata.get('expected_economic_value') or 'unknown'}",
|
|
1899
|
+
f"Expected Economic Value for Agent Apprentice: {metadata.get('expected_economic_value_for_agent_apprentice') or 'unknown'}",
|
|
1900
|
+
"",
|
|
1901
|
+
"## Bundle sharing",
|
|
1902
|
+
"",
|
|
1903
|
+
"This issue records the public ecosystem contribution metadata. Bundle files are packaged locally by Agent Apprenticeship v0.",
|
|
1904
|
+
f"Add the bundle under ecosystem/contributions/bundles/{metadata.get('bundle_id')}/.",
|
|
1905
|
+
"Update ecosystem/contributions/index.json or ecosystem/contributions/index.jsonl with the public metadata entry.",
|
|
1906
|
+
"Attach or share the generated submission package when using the GitHub issue path.",
|
|
1907
|
+
]
|
|
1908
|
+
if package_zip:
|
|
1909
|
+
lines.append(f"Submission Package Name: {metadata.get('package_name') or package_zip.name}")
|
|
1910
|
+
if metadata.get("package_hash"):
|
|
1911
|
+
lines.append(f"Submission Package Hash: {metadata.get('package_hash')}")
|
|
1912
|
+
return "\n".join(lines) + "\n"
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _zip_dir(src: Path, dst: Path) -> str:
|
|
1916
|
+
if dst.exists():
|
|
1917
|
+
dst.unlink()
|
|
1918
|
+
with zipfile.ZipFile(dst, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
1919
|
+
for path in sorted(src.rglob("*")):
|
|
1920
|
+
if path.is_file():
|
|
1921
|
+
zf.write(path, path.relative_to(src.parent))
|
|
1922
|
+
digest = hashlib.sha256(dst.read_bytes()).hexdigest()
|
|
1923
|
+
return f"sha256:{digest}"
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
def _create_ecosystem_submission(bundle: Path) -> tuple[Path, Path, dict]:
|
|
1927
|
+
bundle = _safe_bundle_path(bundle)
|
|
1928
|
+
secret_hits = _bundle_secret_hits(bundle)
|
|
1929
|
+
if secret_hits:
|
|
1930
|
+
raise RuntimeError(
|
|
1931
|
+
"Obvious secret-like values were found in the Contribution Bundle. "
|
|
1932
|
+
"Remove or regenerate the affected files before public contribution: "
|
|
1933
|
+
+ ", ".join(secret_hits[:10])
|
|
1934
|
+
)
|
|
1935
|
+
submission_dir = bundle.parent / "ecosystem_submission"
|
|
1936
|
+
if submission_dir.exists():
|
|
1937
|
+
shutil.rmtree(submission_dir)
|
|
1938
|
+
submission_dir.mkdir(parents=True)
|
|
1939
|
+
shutil.copytree(
|
|
1940
|
+
bundle,
|
|
1941
|
+
submission_dir / "contribution_bundle",
|
|
1942
|
+
ignore=shutil.ignore_patterns(".env", ".env.local", "__pycache__", "*.pyc"),
|
|
1943
|
+
)
|
|
1944
|
+
package_zip = bundle.parent / f"{bundle.name}_ecosystem_submission.zip"
|
|
1945
|
+
metadata = _bundle_submission_metadata(bundle, package_name=package_zip.name)
|
|
1946
|
+
write_json(submission_dir / "ecosystem_submission.json", metadata)
|
|
1947
|
+
(submission_dir / "SUMMARY.md").write_text(_submission_summary(metadata, bundle))
|
|
1948
|
+
package_hash = _zip_dir(submission_dir, package_zip)
|
|
1949
|
+
metadata = _bundle_submission_metadata(bundle, package_name=package_zip.name, package_hash=package_hash)
|
|
1950
|
+
write_json(submission_dir / "ecosystem_submission.json", metadata)
|
|
1951
|
+
(submission_dir / "SUMMARY.md").write_text(_submission_summary(metadata, bundle, package_zip))
|
|
1952
|
+
return submission_dir, package_zip, metadata
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def _ecosystem_contribution_record(
|
|
1956
|
+
*,
|
|
1957
|
+
metadata: dict,
|
|
1958
|
+
auto_share_mode: str,
|
|
1959
|
+
contribution_created: bool,
|
|
1960
|
+
contribution_url: str | None,
|
|
1961
|
+
contribution_method: str,
|
|
1962
|
+
skipped_reason: str | None,
|
|
1963
|
+
) -> dict:
|
|
1964
|
+
return {
|
|
1965
|
+
"bundle_id": metadata.get("bundle_id"),
|
|
1966
|
+
"auto_share_mode": auto_share_mode,
|
|
1967
|
+
"contribution_created": contribution_created,
|
|
1968
|
+
"contribution_url": contribution_url,
|
|
1969
|
+
"contribution_method": contribution_method,
|
|
1970
|
+
"skipped_reason": skipped_reason,
|
|
1971
|
+
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
|
|
1975
|
+
def _write_ecosystem_contribution_record(bundle: Path, record: dict) -> None:
|
|
1976
|
+
bundle = Path(bundle)
|
|
1977
|
+
write_json(bundle / "ecosystem_contribution.json", record)
|
|
1978
|
+
if bundle.name == "contribution_bundle":
|
|
1979
|
+
write_json(bundle.parent / "ecosystem_contribution.json", record)
|
|
1980
|
+
|
|
1981
|
+
|
|
1982
|
+
def _ecosystem_contribute_impl(
|
|
1983
|
+
bundle_path: Path,
|
|
1984
|
+
*,
|
|
1985
|
+
repo: str | None = None,
|
|
1986
|
+
dry_run: bool = False,
|
|
1987
|
+
auto_share_mode: str = "manual",
|
|
1988
|
+
) -> dict:
|
|
1989
|
+
bundle = _safe_bundle_path(bundle_path)
|
|
1990
|
+
submission_dir, package_zip, metadata = _create_ecosystem_submission(bundle)
|
|
1991
|
+
target_repo = repo or _configured_ecosystem_repo()
|
|
1992
|
+
issue_body = _gh_issue_body(metadata, submission_dir, package_zip)
|
|
1993
|
+
result = {
|
|
1994
|
+
"bundle": bundle,
|
|
1995
|
+
"submission_dir": submission_dir,
|
|
1996
|
+
"package_zip": package_zip,
|
|
1997
|
+
"metadata": metadata,
|
|
1998
|
+
"issue_body": issue_body,
|
|
1999
|
+
"target_repo": target_repo,
|
|
2000
|
+
"contribution_created": False,
|
|
2001
|
+
"contribution_url": None,
|
|
2002
|
+
"contribution_method": "manual_package",
|
|
2003
|
+
"skipped_reason": None,
|
|
2004
|
+
"gh_error": None,
|
|
2005
|
+
}
|
|
2006
|
+
if target_repo and _gh_path() and _gh_authenticated() and not dry_run:
|
|
2007
|
+
title = f"Contribution Bundle: {metadata.get('bundle_id')}"
|
|
2008
|
+
cp = _run_gh(["issue", "create", "--repo", target_repo, "--title", title, "--body-file", str(issue_body)], timeout=60)
|
|
2009
|
+
if cp.returncode == 0:
|
|
2010
|
+
result["contribution_created"] = True
|
|
2011
|
+
result["contribution_url"] = (cp.stdout or "").strip()
|
|
2012
|
+
result["contribution_method"] = "github_issue"
|
|
2013
|
+
else:
|
|
2014
|
+
result["skipped_reason"] = "GitHub issue creation failed"
|
|
2015
|
+
result["gh_error"] = redact_secrets(cp.stderr or cp.stdout or "")
|
|
2016
|
+
elif target_repo and _gh_path() and _gh_authenticated() and dry_run:
|
|
2017
|
+
result["skipped_reason"] = "dry run"
|
|
2018
|
+
else:
|
|
2019
|
+
readiness = _ecosystem_auto_share_readiness(target_repo)
|
|
2020
|
+
result["skipped_reason"] = readiness.get("reason")
|
|
2021
|
+
record = _ecosystem_contribution_record(
|
|
2022
|
+
metadata=metadata,
|
|
2023
|
+
auto_share_mode=auto_share_mode,
|
|
2024
|
+
contribution_created=bool(result["contribution_created"]),
|
|
2025
|
+
contribution_url=result.get("contribution_url"),
|
|
2026
|
+
contribution_method=str(result["contribution_method"]),
|
|
2027
|
+
skipped_reason=result.get("skipped_reason"),
|
|
2028
|
+
)
|
|
2029
|
+
result["record"] = record
|
|
2030
|
+
_write_ecosystem_contribution_record(bundle, record)
|
|
2031
|
+
return result
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
def _bundle_from_status(status: dict) -> Path | None:
|
|
2035
|
+
raw = status.get("contribution_bundle_path")
|
|
2036
|
+
if not raw:
|
|
2037
|
+
return None
|
|
2038
|
+
bundle = Path(str(raw)).expanduser()
|
|
2039
|
+
return bundle if (bundle / "contribution_manifest.json").exists() else None
|
|
2040
|
+
|
|
2041
|
+
|
|
2042
|
+
def _print_auto_share_summary(metadata: dict) -> None:
|
|
2043
|
+
typer.echo("Contribution Bundle summary")
|
|
2044
|
+
typer.echo(f"bundle_id: {metadata.get('bundle_id')}")
|
|
2045
|
+
typer.echo(f"title: {metadata.get('title')}")
|
|
2046
|
+
typer.echo(f"task_status: {metadata.get('task_status')}")
|
|
2047
|
+
typer.echo(f"Apprentice Agent: {metadata.get('apprentice_agent') or 'unknown'}")
|
|
2048
|
+
typer.echo(f"Mentor Mode: {metadata.get('mentor_mode')}")
|
|
2049
|
+
typer.echo(f"Mentor Model Provider: {metadata.get('mentor_model_provider') or 'none'}")
|
|
2050
|
+
typer.echo(f"artifact_count: {metadata.get('artifact_count')}")
|
|
2051
|
+
|
|
2052
|
+
|
|
2053
|
+
def _record_auto_share_skip(bundle: Path, *, mode: str, reason: str) -> None:
|
|
2054
|
+
metadata = _bundle_submission_metadata(bundle)
|
|
2055
|
+
record = _ecosystem_contribution_record(
|
|
2056
|
+
metadata=metadata,
|
|
2057
|
+
auto_share_mode=mode,
|
|
2058
|
+
contribution_created=False,
|
|
2059
|
+
contribution_url=None,
|
|
2060
|
+
contribution_method="skipped",
|
|
2061
|
+
skipped_reason=reason,
|
|
2062
|
+
)
|
|
2063
|
+
_write_ecosystem_contribution_record(bundle, record)
|
|
2064
|
+
|
|
2065
|
+
|
|
2066
|
+
def _handle_ecosystem_auto_share(status: dict, *, quiet: bool = False, json_progress: bool = False) -> None:
|
|
2067
|
+
settings = get_settings()
|
|
2068
|
+
mode = settings.ecosystem_auto_share
|
|
2069
|
+
if mode == "manual":
|
|
2070
|
+
return
|
|
2071
|
+
bundle = _bundle_from_status(status)
|
|
2072
|
+
if bundle is None:
|
|
2073
|
+
return
|
|
2074
|
+
emit = not quiet and not json_progress
|
|
2075
|
+
if mode == "automatic":
|
|
2076
|
+
readiness = _ecosystem_auto_share_readiness()
|
|
2077
|
+
if not readiness["ready"]:
|
|
2078
|
+
reason = str(readiness.get("reason") or "not ready")
|
|
2079
|
+
_record_auto_share_skip(bundle, mode=mode, reason=reason)
|
|
2080
|
+
if emit:
|
|
2081
|
+
typer.echo("")
|
|
2082
|
+
typer.echo(f"Ecosystem Auto-Share skipped - {reason}")
|
|
2083
|
+
typer.echo("Ecosystem Auto-Share is configured for:")
|
|
2084
|
+
typer.echo(str(readiness.get("repo_url") or _ecosystem_repo_url()))
|
|
2085
|
+
if readiness.get("next_action"):
|
|
2086
|
+
typer.echo("")
|
|
2087
|
+
typer.echo("Run:")
|
|
2088
|
+
typer.echo(str(readiness["next_action"]))
|
|
2089
|
+
return
|
|
2090
|
+
try:
|
|
2091
|
+
result = _ecosystem_contribute_impl(bundle, auto_share_mode=mode)
|
|
2092
|
+
except Exception as exc:
|
|
2093
|
+
reason = redact_secrets(str(exc))
|
|
2094
|
+
_record_auto_share_skip(bundle, mode=mode, reason=reason)
|
|
2095
|
+
if emit:
|
|
2096
|
+
typer.echo("")
|
|
2097
|
+
typer.echo("Ecosystem Auto-Share skipped - safety check failed")
|
|
2098
|
+
typer.echo(f"Reason: {reason}")
|
|
2099
|
+
typer.echo(f"Bundle: {bundle}")
|
|
2100
|
+
return
|
|
2101
|
+
if emit:
|
|
2102
|
+
typer.echo("")
|
|
2103
|
+
if result["contribution_created"]:
|
|
2104
|
+
typer.echo(f"Public ecosystem contribution created: {result['contribution_url']}")
|
|
2105
|
+
else:
|
|
2106
|
+
reason = result.get("skipped_reason") or "public contribution was not created"
|
|
2107
|
+
typer.echo(f"Ecosystem Auto-Share skipped - {reason}")
|
|
2108
|
+
typer.echo(f"Bundle: {bundle}")
|
|
2109
|
+
return
|
|
2110
|
+
if mode == "ask":
|
|
2111
|
+
try:
|
|
2112
|
+
submission_dir, package_zip, metadata = _create_ecosystem_submission(bundle)
|
|
2113
|
+
_gh_issue_body(metadata, submission_dir, package_zip)
|
|
2114
|
+
except Exception as exc:
|
|
2115
|
+
reason = redact_secrets(str(exc))
|
|
2116
|
+
_record_auto_share_skip(bundle, mode=mode, reason=reason)
|
|
2117
|
+
if emit:
|
|
2118
|
+
typer.echo("")
|
|
2119
|
+
typer.echo("Ecosystem Auto-Share skipped - safety check failed")
|
|
2120
|
+
typer.echo(f"Reason: {reason}")
|
|
2121
|
+
typer.echo(f"Bundle: {bundle}")
|
|
2122
|
+
return
|
|
2123
|
+
if not emit:
|
|
2124
|
+
_record_auto_share_skip(bundle, mode=mode, reason="ask mode requires interactive output")
|
|
2125
|
+
return
|
|
2126
|
+
while True:
|
|
2127
|
+
typer.echo("")
|
|
2128
|
+
typer.echo("Share this Contribution Bundle to the public ecosystem now?")
|
|
2129
|
+
typer.echo("")
|
|
2130
|
+
typer.echo("1. Share now")
|
|
2131
|
+
typer.echo("2. Skip")
|
|
2132
|
+
typer.echo("3. Show submission summary")
|
|
2133
|
+
choice = typer.prompt("Default", default="skip").strip().lower()
|
|
2134
|
+
if choice in {"1", "share", "share now", "yes", "y"}:
|
|
2135
|
+
readiness = _ecosystem_auto_share_readiness()
|
|
2136
|
+
if not readiness["ready"]:
|
|
2137
|
+
reason = str(readiness.get("reason") or "not ready")
|
|
2138
|
+
_record_auto_share_skip(bundle, mode=mode, reason=reason)
|
|
2139
|
+
typer.echo(f"Ecosystem Auto-Share skipped - {reason}")
|
|
2140
|
+
typer.echo("Ecosystem Auto-Share is configured for:")
|
|
2141
|
+
typer.echo(str(readiness.get("repo_url") or _ecosystem_repo_url()))
|
|
2142
|
+
if readiness.get("next_action"):
|
|
2143
|
+
typer.echo("")
|
|
2144
|
+
typer.echo("Run:")
|
|
2145
|
+
typer.echo(str(readiness["next_action"]))
|
|
2146
|
+
return
|
|
2147
|
+
try:
|
|
2148
|
+
result = _ecosystem_contribute_impl(bundle, auto_share_mode=mode)
|
|
2149
|
+
except Exception as exc:
|
|
2150
|
+
reason = redact_secrets(str(exc))
|
|
2151
|
+
_record_auto_share_skip(bundle, mode=mode, reason=reason)
|
|
2152
|
+
typer.echo("Ecosystem Auto-Share skipped - safety check failed")
|
|
2153
|
+
typer.echo(f"Reason: {reason}")
|
|
2154
|
+
typer.echo(f"Bundle: {bundle}")
|
|
2155
|
+
return
|
|
2156
|
+
if result["contribution_created"]:
|
|
2157
|
+
typer.echo(f"Public ecosystem contribution created: {result['contribution_url']}")
|
|
2158
|
+
else:
|
|
2159
|
+
typer.echo(f"Ecosystem Auto-Share skipped - {result.get('skipped_reason') or 'public contribution was not created'}")
|
|
2160
|
+
typer.echo(f"Share later: apprentice ecosystem contribute {bundle}")
|
|
2161
|
+
return
|
|
2162
|
+
if choice in {"3", "summary", "show", "show submission summary"}:
|
|
2163
|
+
_print_auto_share_summary(metadata)
|
|
2164
|
+
continue
|
|
2165
|
+
_record_auto_share_skip(bundle, mode=mode, reason="user skipped")
|
|
2166
|
+
typer.echo(f"Share later: apprentice ecosystem contribute {bundle}")
|
|
2167
|
+
return
|
|
2168
|
+
|
|
2169
|
+
|
|
2170
|
+
def _gh_issue_body(metadata: dict, submission_dir: Path, package_zip: Path) -> Path:
|
|
2171
|
+
body = submission_dir / "GITHUB_ISSUE_BODY.md"
|
|
2172
|
+
body.write_text(_submission_summary(metadata, submission_dir / "contribution_bundle", package_zip))
|
|
2173
|
+
return body
|
|
2174
|
+
|
|
2175
|
+
|
|
2176
|
+
def _registry_entry_from_bundle(bundle: Path, metadata: dict | None = None) -> dict:
|
|
2177
|
+
metadata = metadata or _bundle_submission_metadata(bundle)
|
|
2178
|
+
entry = {
|
|
2179
|
+
**metadata,
|
|
2180
|
+
"bundle_path_or_url": metadata.get("bundle_path_or_url"),
|
|
2181
|
+
"local_bundle_path": str(bundle),
|
|
2182
|
+
}
|
|
2183
|
+
return {k: v for k, v in entry.items() if v is not None}
|
|
2184
|
+
|
|
2185
|
+
|
|
2186
|
+
def _print_learning_sources(rows: list[dict]) -> None:
|
|
2187
|
+
if not rows:
|
|
2188
|
+
typer.echo("No ecosystem experience matched.")
|
|
2189
|
+
return
|
|
2190
|
+
for row in rows:
|
|
2191
|
+
bundle_id = row.get("bundle_id") or row.get("seed_task_id") or row.get("task_id")
|
|
2192
|
+
typer.echo(
|
|
2193
|
+
f"{bundle_id}: {row.get('title')} "
|
|
2194
|
+
f"[domain={','.join(map(str, row.get('domains') or [])) or 'unknown'}]"
|
|
2195
|
+
)
|
|
2196
|
+
|
|
2197
|
+
|
|
2198
|
+
@learn_app.command("search")
|
|
2199
|
+
def learn_search(query: str | None = typer.Argument(None, help="Search ecosystem experience for learning.")):
|
|
2200
|
+
"""Search ecosystem entries that can become Experience Packs."""
|
|
2201
|
+
_print_learning_sources(search_learning_sources(query))
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
@learn_app.command("create")
|
|
2205
|
+
def learn_create(
|
|
2206
|
+
sources: list[str] = typer.Argument(..., help="Ecosystem ids or local Contribution Bundle paths."),
|
|
2207
|
+
title: str | None = typer.Option(None, "--title", help="Optional Experience Pack title."),
|
|
2208
|
+
replay: bool = typer.Option(False, "--replay", help="Run before/after replay after creating the pack."),
|
|
2209
|
+
runner: str | None = typer.Option(None, "--runner", help="Advanced compatibility runner override for replay."),
|
|
2210
|
+
mentor_mode: str = typer.Option("expert-led", "--mentor-mode", help="Mentor Mode for replay, default expert-led."),
|
|
2211
|
+
):
|
|
2212
|
+
settings = get_settings()
|
|
2213
|
+
resolved = [resolve_learning_source(source, settings) for source in sources]
|
|
2214
|
+
pack_path = compile_experience_pack(resolved, title=title, settings=settings)
|
|
2215
|
+
pack = read_json(pack_path / "experience_pack.json")
|
|
2216
|
+
typer.echo("Experience Pack created.")
|
|
2217
|
+
typer.echo(f"pack_id: {pack.get('pack_id')}")
|
|
2218
|
+
typer.echo(f"path: {pack_path}")
|
|
2219
|
+
typer.echo(f"title: {pack.get('title')}")
|
|
2220
|
+
typer.echo("Next:")
|
|
2221
|
+
typer.echo(f"- apprentice learn preview {pack.get('pack_id')}")
|
|
2222
|
+
typer.echo(f"- apprentice learn replay {pack.get('pack_id')}")
|
|
2223
|
+
if replay:
|
|
2224
|
+
_learn_replay_impl(str(pack.get("pack_id")), runner=runner, mentor_mode=mentor_mode)
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
@learn_app.command("preview")
|
|
2228
|
+
def learn_preview(pack_id: str = typer.Argument(..., help="Experience Pack id or path.")):
|
|
2229
|
+
path, pack = load_pack(pack_id)
|
|
2230
|
+
typer.echo(f"Experience Pack: {pack.get('title')}")
|
|
2231
|
+
typer.echo(f"pack_id: {pack.get('pack_id')}")
|
|
2232
|
+
typer.echo(f"status: {pack.get('status')}")
|
|
2233
|
+
typer.echo(f"path: {path}")
|
|
2234
|
+
typer.echo(f"sources: {', '.join(pack.get('source_ecosystem_ids') or [])}")
|
|
2235
|
+
typer.echo(f"domains: {', '.join(pack.get('domains') or []) or 'not specified'}")
|
|
2236
|
+
typer.echo("")
|
|
2237
|
+
typer.echo("Key lessons:")
|
|
2238
|
+
lessons = (pack.get("strategy_lessons") or [])[:5]
|
|
2239
|
+
if lessons:
|
|
2240
|
+
for lesson in lessons:
|
|
2241
|
+
typer.echo(f"- {lesson}")
|
|
2242
|
+
else:
|
|
2243
|
+
typer.echo("- No strategy lessons captured.")
|
|
2244
|
+
artifact_requirements = (pack.get("artifact_requirements") or [])[:5]
|
|
2245
|
+
if artifact_requirements:
|
|
2246
|
+
typer.echo("")
|
|
2247
|
+
typer.echo("Artifact requirements:")
|
|
2248
|
+
for requirement in artifact_requirements:
|
|
2249
|
+
typer.echo(f"- {requirement}")
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
def _learn_replay_impl(pack_id: str, *, runner: str | None, mentor_mode: str) -> dict:
|
|
2253
|
+
settings = _settings_for_run(mentor_mode, None, 1, None, None)
|
|
2254
|
+
_ensure_apprentice_agent_ready(settings, runner, interactive=False)
|
|
2255
|
+
pack_path, pack = load_pack(pack_id, settings)
|
|
2256
|
+
instruction = replay_instruction_for_pack(pack, settings)
|
|
2257
|
+
replay_slug = slugify(str(pack.get("pack_id") or "experience-pack"))[:48]
|
|
2258
|
+
before_id = f"learning-{replay_slug}-before"
|
|
2259
|
+
after_id = f"learning-{replay_slug}-after"
|
|
2260
|
+
typer.echo("Before/after replay started.")
|
|
2261
|
+
typer.echo(f"Experience Pack: {pack.get('title')}")
|
|
2262
|
+
with _temporary_auto_approve_env(expert_auto_approve=True, hybrid_auto_approve=True, mentor_interactive_checkpoints=False):
|
|
2263
|
+
before_root, _ = run_prompt_task(
|
|
2264
|
+
instruction,
|
|
2265
|
+
run_id=before_id,
|
|
2266
|
+
settings=settings,
|
|
2267
|
+
runner=runner,
|
|
2268
|
+
progress_callback=None,
|
|
2269
|
+
)
|
|
2270
|
+
selected, guidance = resolve_packs_for_run([str(pack.get("pack_id"))], settings=settings)
|
|
2271
|
+
after_instruction = instruction.rstrip() + guidance if guidance else instruction
|
|
2272
|
+
after_root, _ = run_prompt_task(
|
|
2273
|
+
after_instruction,
|
|
2274
|
+
run_id=after_id,
|
|
2275
|
+
settings=settings,
|
|
2276
|
+
runner=runner,
|
|
2277
|
+
progress_callback=None,
|
|
2278
|
+
experience_pack_refs=pack_run_refs(selected),
|
|
2279
|
+
)
|
|
2280
|
+
before_status = read_run_status(before_root)
|
|
2281
|
+
after_status = read_run_status(after_root)
|
|
2282
|
+
comparison = compare_replay(before_status, after_status)
|
|
2283
|
+
result = {
|
|
2284
|
+
**comparison,
|
|
2285
|
+
"pack_id": pack.get("pack_id"),
|
|
2286
|
+
"before_run_id": before_root.name,
|
|
2287
|
+
"after_run_id": after_root.name,
|
|
2288
|
+
"before_run_path": str(before_root),
|
|
2289
|
+
"after_run_path": str(after_root),
|
|
2290
|
+
}
|
|
2291
|
+
write_before_after_result(pack_path, result)
|
|
2292
|
+
typer.echo(f"before_status: {comparison.get('before_status')}")
|
|
2293
|
+
typer.echo(f"after_status: {comparison.get('after_status')}")
|
|
2294
|
+
typer.echo(f"before_after_result: {comparison.get('result')}")
|
|
2295
|
+
if comparison.get("result") == "improved":
|
|
2296
|
+
typer.echo(f"Suggested action: apprentice learn keep {pack.get('pack_id')}")
|
|
2297
|
+
elif comparison.get("result") == "regressed":
|
|
2298
|
+
typer.echo(f"Suggested action: apprentice learn revert {pack.get('pack_id')}")
|
|
2299
|
+
else:
|
|
2300
|
+
typer.echo("Suggested action: inspect the replay, then keep or revert learning.")
|
|
2301
|
+
return result
|
|
2302
|
+
|
|
2303
|
+
|
|
2304
|
+
@learn_app.command("replay")
|
|
2305
|
+
def learn_replay(
|
|
2306
|
+
pack_id: str = typer.Argument(..., help="Experience Pack id or path."),
|
|
2307
|
+
runner: str | None = typer.Option(None, "--runner", help="Advanced compatibility runner override."),
|
|
2308
|
+
mentor_mode: str = typer.Option("expert-led", "--mentor-mode", help="Mentor Mode for replay, default expert-led."),
|
|
2309
|
+
):
|
|
2310
|
+
_learn_replay_impl(pack_id, runner=runner, mentor_mode=mentor_mode)
|
|
2311
|
+
|
|
2312
|
+
|
|
2313
|
+
@learn_app.command("keep")
|
|
2314
|
+
def learn_keep(pack_id: str = typer.Argument(..., help="Experience Pack id or path.")):
|
|
2315
|
+
path = update_pack_status(pack_id, "active")
|
|
2316
|
+
typer.echo("Keep learning: Experience Pack is active.")
|
|
2317
|
+
typer.echo(f"path: {path}")
|
|
2318
|
+
|
|
2319
|
+
|
|
2320
|
+
@learn_app.command("revert")
|
|
2321
|
+
def learn_revert(pack_id: str = typer.Argument(..., help="Experience Pack id or path.")):
|
|
2322
|
+
path = update_pack_status(pack_id, "reverted")
|
|
2323
|
+
typer.echo("Revert learning: Experience Pack is inactive.")
|
|
2324
|
+
typer.echo(f"path: {path}")
|
|
2325
|
+
|
|
2326
|
+
|
|
2327
|
+
@learn_app.command("remove")
|
|
2328
|
+
def learn_remove(pack_id: str = typer.Argument(..., help="Experience Pack id or path.")):
|
|
2329
|
+
path = remove_pack(pack_id)
|
|
2330
|
+
typer.echo("Experience Pack removed.")
|
|
2331
|
+
typer.echo(f"path: {path}")
|
|
2332
|
+
|
|
2333
|
+
|
|
2334
|
+
@learn_app.command("list")
|
|
2335
|
+
def learn_list():
|
|
2336
|
+
packs = list_packs()
|
|
2337
|
+
if not packs:
|
|
2338
|
+
typer.echo("No Experience Packs yet.")
|
|
2339
|
+
return
|
|
2340
|
+
for pack in packs:
|
|
2341
|
+
typer.echo(f"{pack.get('pack_id')}: {pack.get('title')} [status={pack.get('status')}]")
|
|
2342
|
+
|
|
2343
|
+
|
|
2344
|
+
@learn_app.command("status")
|
|
2345
|
+
def learn_status():
|
|
2346
|
+
packs = active_packs()
|
|
2347
|
+
if not packs:
|
|
2348
|
+
typer.echo("Experience Packs: Manual")
|
|
2349
|
+
typer.echo("Active Experience Packs: none")
|
|
2350
|
+
return
|
|
2351
|
+
typer.echo("Experience Packs: Manual")
|
|
2352
|
+
typer.echo("Active Experience Packs:")
|
|
2353
|
+
for pack in packs:
|
|
2354
|
+
typer.echo(f"- {pack.get('pack_id')}: {pack.get('title')}")
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
@ecosystem_app.command('guide')
|
|
2358
|
+
def ecosystem_guide():
|
|
2359
|
+
typer.echo('Agent Apprenticeship Ecosystem')
|
|
2360
|
+
typer.echo('')
|
|
2361
|
+
typer.echo('The public ecosystem is the open-source Agent Apprenticeship ecosystem repo.')
|
|
2362
|
+
typer.echo('Run a task, then contribute the generated Contribution Bundle:')
|
|
2363
|
+
typer.echo(' apprentice ecosystem contribute <bundle_path>')
|
|
2364
|
+
typer.echo('')
|
|
2365
|
+
typer.echo('Configure the public ecosystem repo:')
|
|
2366
|
+
typer.echo(f' default: {DEFAULT_PUBLIC_ECOSYSTEM_REPO}')
|
|
2367
|
+
typer.echo(' apprentice ecosystem configure --repo <owner>/<repo>')
|
|
2368
|
+
typer.echo(' apprentice ecosystem configure --repo-path <public-ecosystem-repo-checkout>')
|
|
2369
|
+
typer.echo('')
|
|
2370
|
+
typer.echo('Ecosystem Auto-Share is Manual by default.')
|
|
2371
|
+
typer.echo(' apprentice ecosystem configure --auto-share ask')
|
|
2372
|
+
typer.echo(' apprentice ecosystem configure --auto-share automatic')
|
|
2373
|
+
typer.echo('')
|
|
2374
|
+
typer.echo('If GitHub CLI automation is unavailable, the contribute command prepares a local submission package and copy-paste-ready public issue text.')
|
|
2375
|
+
typer.echo('Remote ecosystem sync is not included in v0. Public repo discovery works through the configured repo when `gh` can read its index; `--registry` can point at an exported public index JSON for tests or offline use.')
|
|
2376
|
+
typer.echo(f'Community Slack: {SLACK_LINK}')
|
|
2377
|
+
|
|
2378
|
+
|
|
2379
|
+
@ecosystem_app.command('configure')
|
|
2380
|
+
def ecosystem_configure(
|
|
2381
|
+
repo: str | None=typer.Option(None, '--repo', help='Public ecosystem repo in owner/name form.'),
|
|
2382
|
+
repo_path: Path | None=typer.Option(None, '--repo-path', help='Public ecosystem repo path for a local checkout/export.'),
|
|
2383
|
+
auto_share: str | None=typer.Option(None, '--auto-share', help='Ecosystem Auto-Share mode: manual, ask, or automatic.'),
|
|
2384
|
+
):
|
|
2385
|
+
settings = get_settings()
|
|
2386
|
+
if repo is None and repo_path is None and auto_share is None:
|
|
2387
|
+
status = _ecosystem_status()
|
|
2388
|
+
typer.echo("Public Ecosystem Configuration")
|
|
2389
|
+
typer.echo(f"Current repo: {settings.ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO}")
|
|
2390
|
+
typer.echo(f"GitHub URL: {_ecosystem_repo_url(settings.ecosystem_repo)}")
|
|
2391
|
+
typer.echo(f"Current public ecosystem repo path: {settings.ecosystem_repo_path or 'Not configured'}")
|
|
2392
|
+
typer.echo(f"GitHub CLI installed: {_yes_no(status['gh_installed'])}")
|
|
2393
|
+
typer.echo(f"GitHub CLI authenticated: {_yes_no(status['gh_authenticated'])}")
|
|
2394
|
+
typer.echo(f"Current Ecosystem Auto-Share: {ecosystem_auto_share_display(settings.ecosystem_auto_share)}")
|
|
2395
|
+
entered_repo = typer.prompt("Public ecosystem repo, owner/name, optional", default=settings.ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO, show_default=False).strip()
|
|
2396
|
+
if entered_repo:
|
|
2397
|
+
repo = entered_repo
|
|
2398
|
+
typer.echo("")
|
|
2399
|
+
typer.echo("Choose Ecosystem Auto-Share mode:")
|
|
2400
|
+
typer.echo("1. Manual")
|
|
2401
|
+
typer.echo("2. Ask before sharing")
|
|
2402
|
+
typer.echo("3. Share automatically")
|
|
2403
|
+
choice = typer.prompt("Default", default=settings.ecosystem_auto_share).strip().lower()
|
|
2404
|
+
auto_share = {"1": "manual", "2": "ask", "3": "automatic"}.get(choice, choice)
|
|
2405
|
+
updates = {}
|
|
2406
|
+
candidate_repo = repo or settings.ecosystem_repo or DEFAULT_PUBLIC_ECOSYSTEM_REPO
|
|
2407
|
+
if repo is not None:
|
|
2408
|
+
if "/" not in repo:
|
|
2409
|
+
typer.echo("Repository must be in owner/name form.")
|
|
2410
|
+
raise typer.Exit(1)
|
|
2411
|
+
updates["ecosystem_repo"] = repo
|
|
2412
|
+
if repo_path is not None:
|
|
2413
|
+
expanded = repo_path.expanduser()
|
|
2414
|
+
if not expanded.exists():
|
|
2415
|
+
typer.echo(f"Public ecosystem repo path not found: {expanded}")
|
|
2416
|
+
raise typer.Exit(1)
|
|
2417
|
+
if not (expanded / "seed_dataset").exists() and not (expanded / "ecosystem").exists():
|
|
2418
|
+
typer.echo(f"Public ecosystem repo path does not look like an Agent Apprenticeship ecosystem repo: {expanded}")
|
|
2419
|
+
raise typer.Exit(1)
|
|
2420
|
+
updates["ecosystem_repo_path"] = expanded
|
|
2421
|
+
if auto_share is not None:
|
|
2422
|
+
mode = _normalize_auto_share(auto_share)
|
|
2423
|
+
if mode in {"ask", "automatic"}:
|
|
2424
|
+
readiness = _ecosystem_auto_share_readiness(candidate_repo)
|
|
2425
|
+
if not readiness["ready"]:
|
|
2426
|
+
typer.echo("Ecosystem Auto-Share cannot be enabled yet.")
|
|
2427
|
+
typer.echo("")
|
|
2428
|
+
typer.echo("Ecosystem Auto-Share is configured for:")
|
|
2429
|
+
typer.echo(str(readiness.get("repo_url") or _ecosystem_repo_url(candidate_repo)))
|
|
2430
|
+
typer.echo("")
|
|
2431
|
+
typer.echo("Required:")
|
|
2432
|
+
next_action = str(readiness.get("next_action") or "gh auth login")
|
|
2433
|
+
typer.echo(f"- {next_action}")
|
|
2434
|
+
if readiness.get("reason"):
|
|
2435
|
+
typer.echo(f"Reason: {readiness['reason']}")
|
|
2436
|
+
raise typer.Exit(1)
|
|
2437
|
+
updates["ecosystem_auto_share"] = mode
|
|
2438
|
+
if not updates:
|
|
2439
|
+
typer.echo("No ecosystem settings changed.")
|
|
2440
|
+
return
|
|
2441
|
+
updated = update_settings(**updates)
|
|
2442
|
+
if repo is not None:
|
|
2443
|
+
typer.echo(f"Public ecosystem repo configured: {repo}")
|
|
2444
|
+
if repo_path is not None:
|
|
2445
|
+
typer.echo(f"Public ecosystem repo path configured: {Path(repo_path).expanduser()}")
|
|
2446
|
+
if auto_share is not None:
|
|
2447
|
+
typer.echo(f"Ecosystem Auto-Share: {ecosystem_auto_share_display(updated.ecosystem_auto_share)}")
|
|
2448
|
+
typer.echo("Next: apprentice ecosystem status")
|
|
2449
|
+
|
|
2450
|
+
|
|
2451
|
+
@ecosystem_app.command('status')
|
|
2452
|
+
def ecosystem_status_command():
|
|
2453
|
+
status = _ecosystem_status()
|
|
2454
|
+
settings = get_settings()
|
|
2455
|
+
typer.echo("Public Ecosystem Status")
|
|
2456
|
+
typer.echo(f"Public Ecosystem Repo: {status['repo'] or DEFAULT_PUBLIC_ECOSYSTEM_REPO}")
|
|
2457
|
+
typer.echo(f"GitHub URL: {status['repo_url']}")
|
|
2458
|
+
typer.echo(f"Public ecosystem repo path: {status.get('repo_path') or 'Not configured'}")
|
|
2459
|
+
if status.get("repo_path"):
|
|
2460
|
+
typer.echo(f"Public ecosystem repo path exists: {_yes_no(status.get('repo_path_exists'))}")
|
|
2461
|
+
typer.echo(f"GitHub CLI installed: {_yes_no(status['gh_installed'])}")
|
|
2462
|
+
typer.echo(f"GitHub CLI authenticated: {_yes_no(status['gh_authenticated'])}")
|
|
2463
|
+
typer.echo(f"Automatic public contribution: {_yes_no(status['automatic_contribution_ready'])}")
|
|
2464
|
+
typer.echo(f"Ecosystem Auto-Share: {ecosystem_auto_share_display(settings.ecosystem_auto_share)}")
|
|
2465
|
+
if settings.ecosystem_auto_share != "manual":
|
|
2466
|
+
readiness = _ecosystem_auto_share_readiness()
|
|
2467
|
+
typer.echo(f"Ecosystem Auto-Share Status: {'Ready' if readiness['ready'] else 'Not ready - ' + str(readiness.get('reason') or 'not ready')}")
|
|
2468
|
+
if not status["gh_installed"]:
|
|
2469
|
+
typer.echo("Next: install GitHub CLI (`gh`) or use `apprentice ecosystem contribute <bundle>` for manual submission files.")
|
|
2470
|
+
elif not status["gh_authenticated"]:
|
|
2471
|
+
typer.echo("Next: run `gh auth login`, or use the manual submission package generated by `apprentice ecosystem contribute <bundle>`.")
|
|
2472
|
+
|
|
2473
|
+
|
|
2474
|
+
@ecosystem_app.command('contribute')
|
|
2475
|
+
def ecosystem_contribute(
|
|
2476
|
+
bundle_path: Path=typer.Argument(..., help='Contribution Bundle folder.'),
|
|
2477
|
+
repo: str | None=typer.Option(None, '--repo', help='Public ecosystem repo in owner/name form.'),
|
|
2478
|
+
dry_run: bool=typer.Option(False, '--dry-run', help='Prepare files and show the GitHub command without creating an issue.'),
|
|
2479
|
+
):
|
|
2480
|
+
try:
|
|
2481
|
+
result = _ecosystem_contribute_impl(bundle_path, repo=repo, dry_run=dry_run, auto_share_mode="manual")
|
|
2482
|
+
except Exception as exc:
|
|
2483
|
+
typer.echo(f"Could not prepare public ecosystem contribution: {redact_secrets(str(exc))}")
|
|
2484
|
+
raise typer.Exit(1)
|
|
2485
|
+
bundle = result["bundle"]
|
|
2486
|
+
submission_dir = result["submission_dir"]
|
|
2487
|
+
package_zip = result["package_zip"]
|
|
2488
|
+
metadata = result["metadata"]
|
|
2489
|
+
target_repo = result["target_repo"]
|
|
2490
|
+
issue_body = result["issue_body"]
|
|
2491
|
+
typer.echo("Public ecosystem contribution prepared.")
|
|
2492
|
+
typer.echo("")
|
|
2493
|
+
typer.echo(f"Bundle ID: {metadata.get('bundle_id')}")
|
|
2494
|
+
typer.echo(f"Title: {metadata.get('title')}")
|
|
2495
|
+
typer.echo(f"Bundle: {bundle}")
|
|
2496
|
+
typer.echo(f"Submission package: {package_zip}")
|
|
2497
|
+
typer.echo(f"Submission metadata: {submission_dir / 'ecosystem_submission.json'}")
|
|
2498
|
+
if result["contribution_created"]:
|
|
2499
|
+
typer.echo(f"Public contribution URL: {result['contribution_url']}")
|
|
2500
|
+
typer.echo("No bundle files were uploaded automatically. Attach or submit the generated package according to the repo instructions.")
|
|
2501
|
+
return
|
|
2502
|
+
if target_repo and _gh_path() and _gh_authenticated() and not dry_run and result.get("gh_error"):
|
|
2503
|
+
typer.echo("GitHub issue creation failed; manual submission package is ready.")
|
|
2504
|
+
typer.echo(str(result.get("gh_error") or ""))
|
|
2505
|
+
elif target_repo and _gh_path() and _gh_authenticated() and dry_run:
|
|
2506
|
+
typer.echo("Dry run: no GitHub issue was created.")
|
|
2507
|
+
typer.echo(f"Would run: gh issue create --repo {target_repo} --title \"Contribution Bundle: {metadata.get('bundle_id')}\" --body-file {issue_body}")
|
|
2508
|
+
else:
|
|
2509
|
+
typer.echo("No upload was performed in v0.")
|
|
2510
|
+
typer.echo(f"Public ecosystem: {_ecosystem_repo_url(target_repo)}")
|
|
2511
|
+
if not _gh_path():
|
|
2512
|
+
typer.echo("GitHub CLI `gh` is not installed.")
|
|
2513
|
+
elif not _gh_authenticated():
|
|
2514
|
+
typer.echo("GitHub CLI `gh` is not authenticated.")
|
|
2515
|
+
typer.echo("")
|
|
2516
|
+
typer.echo("Manual public submission steps:")
|
|
2517
|
+
typer.echo(f"1. Open {_ecosystem_repo_url(target_repo)} in GitHub.")
|
|
2518
|
+
typer.echo(f"2. Create a new issue using: {issue_body}")
|
|
2519
|
+
typer.echo(f"3. Attach or share the submission package: {package_zip}")
|
|
2520
|
+
typer.echo(f"Community Slack: {SLACK_LINK}")
|
|
2521
|
+
|
|
2522
|
+
|
|
2523
|
+
@ecosystem_app.command('list')
|
|
2524
|
+
def ecosystem_list(registry: Path | None=typer.Option(None, '--registry', help='Public ecosystem index JSON path for offline/test use.')):
|
|
2525
|
+
try:
|
|
2526
|
+
rows=_load_registry(registry)
|
|
2527
|
+
except FileNotFoundError as exc:
|
|
2528
|
+
typer.echo(str(exc))
|
|
2529
|
+
raise typer.Exit(1)
|
|
2530
|
+
_print_registry_rows(rows)
|
|
2531
|
+
|
|
2532
|
+
|
|
2533
|
+
@ecosystem_app.command('search')
|
|
2534
|
+
def ecosystem_search(
|
|
2535
|
+
query_arg: str | None=typer.Argument(None, help='Free-text query.'),
|
|
2536
|
+
query: str | None=typer.Option(None, '--query', help='Free-text query.'),
|
|
2537
|
+
registry: Path | None=typer.Option(None, '--registry', help='Public ecosystem index JSON path for offline/test use.'),
|
|
2538
|
+
title: str | None=typer.Option(None, '--title'),
|
|
2539
|
+
domain: str | None=typer.Option(None, '--domain'),
|
|
2540
|
+
subdomain: str | None=typer.Option(None, '--subdomain'),
|
|
2541
|
+
mentor_mode: str | None=typer.Option(None, '--mentor-mode'),
|
|
2542
|
+
task_status: str | None=typer.Option(None, '--task-status'),
|
|
2543
|
+
run_status: str | None=typer.Option(None, '--run-status'),
|
|
2544
|
+
min_traced_steps: int | None=typer.Option(None, '--min-traced-steps'),
|
|
2545
|
+
max_traced_steps: int | None=typer.Option(None, '--max-traced-steps'),
|
|
2546
|
+
expected_economic_value: str | None=typer.Option(None, '--expected-economic-value'),
|
|
2547
|
+
artifact_type: str | None=typer.Option(None, '--artifact-type'),
|
|
2548
|
+
apprentice_agent: str | None=typer.Option(None, '--apprentice-agent'),
|
|
2549
|
+
mentor_model_provider: str | None=typer.Option(None, '--mentor-model-provider'),
|
|
2550
|
+
):
|
|
2551
|
+
try:
|
|
2552
|
+
rows=_load_registry(registry)
|
|
2553
|
+
except FileNotFoundError as exc:
|
|
2554
|
+
typer.echo(str(exc))
|
|
2555
|
+
raise typer.Exit(1)
|
|
2556
|
+
effective_query = query or query_arg
|
|
2557
|
+
filters=locals()
|
|
2558
|
+
filters.pop('query_arg', None)
|
|
2559
|
+
filters.pop('query', None)
|
|
2560
|
+
filters.pop('registry', None)
|
|
2561
|
+
filters.pop('effective_query', None)
|
|
2562
|
+
_print_registry_rows([row for row in rows if _matches_registry_row(row, effective_query, filters)])
|
|
2563
|
+
|
|
2564
|
+
|
|
2565
|
+
def _find_registry_row(bundle_id: str, registry: Path | None=None) -> dict:
|
|
2566
|
+
rows=_load_registry(registry)
|
|
2567
|
+
for row in rows:
|
|
2568
|
+
if row.get('bundle_id') == bundle_id:
|
|
2569
|
+
return row
|
|
2570
|
+
raise FileNotFoundError(f'Bundle id not found in registry: {bundle_id}')
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
def _prepare_ecosystem_pull_destination(bundle_id: str) -> Path:
|
|
2574
|
+
dest=get_settings().app_home / 'ecosystem' / 'bundles' / bundle_id
|
|
2575
|
+
try:
|
|
2576
|
+
if dest.exists():
|
|
2577
|
+
shutil.rmtree(dest)
|
|
2578
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
2579
|
+
except OSError as exc:
|
|
2580
|
+
typer.echo('Could not prepare local ecosystem pull destination.')
|
|
2581
|
+
typer.echo(f'Path: {dest}')
|
|
2582
|
+
typer.echo(f'Reason: {exc}')
|
|
2583
|
+
typer.echo('Next action: set AA_HOME to a writable directory or remove the existing pulled item and try again.')
|
|
2584
|
+
raise typer.Exit(1)
|
|
2585
|
+
return dest
|
|
2586
|
+
|
|
2587
|
+
|
|
2588
|
+
@ecosystem_app.command('pull')
|
|
2589
|
+
def ecosystem_pull(
|
|
2590
|
+
bundle_id: str=typer.Argument(...),
|
|
2591
|
+
registry: Path | None=typer.Option(None, '--registry', help='Public ecosystem index JSON path for offline/test use.'),
|
|
2592
|
+
):
|
|
2593
|
+
try:
|
|
2594
|
+
row=_find_registry_row(bundle_id, registry)
|
|
2595
|
+
except FileNotFoundError as exc:
|
|
2596
|
+
typer.echo(str(exc))
|
|
2597
|
+
raise typer.Exit(1)
|
|
2598
|
+
src=row.get('local_bundle_path') or row.get('path') or row.get('bundle_path') or row.get('bundle_path_or_url')
|
|
2599
|
+
if not src:
|
|
2600
|
+
task_path = row.get("task_path") or row.get("task_packet_path")
|
|
2601
|
+
if not task_path:
|
|
2602
|
+
typer.echo('This public index entry has no downloadable bundle path or URL.')
|
|
2603
|
+
raise typer.Exit(1)
|
|
2604
|
+
dest=_prepare_ecosystem_pull_destination(bundle_id)
|
|
2605
|
+
row = {**row, "experience_source_type": row.get("experience_source_type") or "seed_task", "pulled_item_type": "seed_task"}
|
|
2606
|
+
try:
|
|
2607
|
+
write_json(dest / "ecosystem_item.json", row)
|
|
2608
|
+
|
|
2609
|
+
def resolve_seed_path(raw: str | None) -> Path | None:
|
|
2610
|
+
if not raw:
|
|
2611
|
+
return None
|
|
2612
|
+
path = Path(str(raw)).expanduser()
|
|
2613
|
+
return path if path.is_absolute() else Path.cwd() / path
|
|
2614
|
+
|
|
2615
|
+
def copy_seed_path(raw: str | None, target_rel: str) -> None:
|
|
2616
|
+
source_path = resolve_seed_path(raw)
|
|
2617
|
+
if not source_path or not source_path.exists():
|
|
2618
|
+
return
|
|
2619
|
+
target = dest / target_rel
|
|
2620
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
2621
|
+
if source_path.is_dir():
|
|
2622
|
+
shutil.copytree(source_path, target, dirs_exist_ok=True)
|
|
2623
|
+
else:
|
|
2624
|
+
shutil.copy2(source_path, target)
|
|
2625
|
+
|
|
2626
|
+
copy_seed_path(row.get("task_path") or row.get("task_packet_path"), "task")
|
|
2627
|
+
copy_seed_path(row.get("rubric_path"), "rubric/rubric.json")
|
|
2628
|
+
copy_seed_path(row.get("worker_visible_rubric_path"), "rubric/worker_visible_rubric.md")
|
|
2629
|
+
copy_seed_path(row.get("verifier_private_rubric_path"), "rubric/verifier_private_rubric.json")
|
|
2630
|
+
copy_seed_path(row.get("attempts_path"), "attempts")
|
|
2631
|
+
copy_seed_path(row.get("evaluation_path"), "evaluation")
|
|
2632
|
+
copy_seed_path(row.get("learning_signals_path"), "learning_signals")
|
|
2633
|
+
for idx, trace_path in enumerate(row.get("trace_paths") or [], 1):
|
|
2634
|
+
trace_source = resolve_seed_path(str(trace_path))
|
|
2635
|
+
if trace_source and trace_source.exists():
|
|
2636
|
+
target_name = Path(trace_path).parent.name or f"trace_{idx}"
|
|
2637
|
+
copy_seed_path(str(trace_path), f"traces/{target_name}/{Path(trace_path).name}")
|
|
2638
|
+
except OSError as exc:
|
|
2639
|
+
typer.echo('Could not pull ecosystem seed task.')
|
|
2640
|
+
typer.echo(f'Path: {dest}')
|
|
2641
|
+
typer.echo(f'Reason: {exc}')
|
|
2642
|
+
typer.echo('Next action: set AA_HOME to a writable directory and try again.')
|
|
2643
|
+
raise typer.Exit(1)
|
|
2644
|
+
typer.echo(f'pulled_bundle={dest}')
|
|
2645
|
+
return
|
|
2646
|
+
source=Path(src).expanduser()
|
|
2647
|
+
if not source.exists():
|
|
2648
|
+
typer.echo(f'Bundle path not found locally: {source}')
|
|
2649
|
+
typer.echo('If this is a public URL, download support depends on the ecosystem repo packaging convention for v0.')
|
|
2650
|
+
raise typer.Exit(1)
|
|
2651
|
+
dest=_prepare_ecosystem_pull_destination(bundle_id)
|
|
2652
|
+
try:
|
|
2653
|
+
shutil.rmtree(dest)
|
|
2654
|
+
shutil.copytree(source, dest)
|
|
2655
|
+
except OSError as exc:
|
|
2656
|
+
typer.echo('Could not pull ecosystem bundle.')
|
|
2657
|
+
typer.echo(f'Path: {dest}')
|
|
2658
|
+
typer.echo(f'Reason: {exc}')
|
|
2659
|
+
typer.echo('Next action: set AA_HOME to a writable directory and try again.')
|
|
2660
|
+
raise typer.Exit(1)
|
|
2661
|
+
typer.echo(f'pulled_bundle={dest}')
|
|
2662
|
+
|
|
2663
|
+
|
|
2664
|
+
def _summarize_bundle_manifest(manifest: dict) -> str:
|
|
2665
|
+
keys=[
|
|
2666
|
+
'bundle_id','title','attempts','traced_steps','process_supervision_rows',
|
|
2667
|
+
'reward_modeling_rows','revision_preference_pairs','domains','subdomains',
|
|
2668
|
+
'agent_apprentice_role','expected_economic_value',
|
|
2669
|
+
'expected_economic_value_for_agent_apprentice','mentor_mode','task_status','run_status'
|
|
2670
|
+
]
|
|
2671
|
+
return '\n'.join(f'{key}: {manifest.get(key)}' for key in keys if key in manifest)
|
|
2672
|
+
|
|
2673
|
+
|
|
2674
|
+
def _summarize_seed_registry_row(row: dict) -> str:
|
|
2675
|
+
trace_count = row.get("trace_count")
|
|
2676
|
+
if trace_count is None:
|
|
2677
|
+
trace_count = len(row.get("trace_paths") or [])
|
|
2678
|
+
learning_count = row.get("training_signal_rows") or row.get("lessons_count") or 0
|
|
2679
|
+
lines = [
|
|
2680
|
+
"kind: seed_task",
|
|
2681
|
+
f"bundle_id: {row.get('bundle_id') or row.get('seed_task_id')}",
|
|
2682
|
+
f"title: {row.get('title')}",
|
|
2683
|
+
f"status: {row.get('task_status') or 'ready_to_run'} / {row.get('run_status') or 'not_run'}",
|
|
2684
|
+
f"traces: {_format_count(trace_count)}",
|
|
2685
|
+
f"attempts: {_format_count(row.get('attempt_count') or row.get('attempts') or 0)}",
|
|
2686
|
+
f"process_supervision_rows: {_format_count(row.get('process_supervision_rows') or 0)}",
|
|
2687
|
+
f"reward_modeling_rows: {_format_count(row.get('reward_modeling_rows') or 0)}",
|
|
2688
|
+
f"revision_preference_pairs: {_format_count(row.get('revision_preference_pairs') or 0)}",
|
|
2689
|
+
f"learning_signals: {_format_count(learning_count) if learning_count else 'available'}",
|
|
2690
|
+
]
|
|
2691
|
+
if row.get("domains"):
|
|
2692
|
+
lines.append(f"domains: {row.get('domains')}")
|
|
2693
|
+
if row.get("subdomains"):
|
|
2694
|
+
lines.append(f"subdomains: {row.get('subdomains')}")
|
|
2695
|
+
lines.append("note: this is a seed task, not a completed Contribution Bundle")
|
|
2696
|
+
return "\n".join(lines)
|
|
2697
|
+
|
|
2698
|
+
|
|
2699
|
+
def _summarize_contribution_registry_row(row: dict) -> str:
|
|
2700
|
+
lines = [
|
|
2701
|
+
"kind: contribution_bundle",
|
|
2702
|
+
f"bundle_id: {row.get('bundle_id')}",
|
|
2703
|
+
f"title: {row.get('title')}",
|
|
2704
|
+
f"task_status: {row.get('task_status') or 'unknown'}",
|
|
2705
|
+
f"run_status: {row.get('run_status') or 'unknown'}",
|
|
2706
|
+
]
|
|
2707
|
+
for key in [
|
|
2708
|
+
"apprentice_agent",
|
|
2709
|
+
"mentor_mode",
|
|
2710
|
+
"mentor_model_provider",
|
|
2711
|
+
"attempts",
|
|
2712
|
+
"traced_steps",
|
|
2713
|
+
"artifact_count",
|
|
2714
|
+
"process_supervision_rows",
|
|
2715
|
+
"reward_modeling_rows",
|
|
2716
|
+
"revision_preference_pairs",
|
|
2717
|
+
"expected_economic_value",
|
|
2718
|
+
"expected_economic_value_for_agent_apprentice",
|
|
2719
|
+
]:
|
|
2720
|
+
if row.get(key) is not None:
|
|
2721
|
+
lines.append(f"{key}: {row.get(key)}")
|
|
2722
|
+
if row.get("domains"):
|
|
2723
|
+
lines.append(f"domains: {row.get('domains')}")
|
|
2724
|
+
if row.get("subdomains"):
|
|
2725
|
+
lines.append(f"subdomains: {row.get('subdomains')}")
|
|
2726
|
+
return "\n".join(lines)
|
|
2727
|
+
|
|
2728
|
+
|
|
2729
|
+
@ecosystem_app.command('inspect')
|
|
2730
|
+
def ecosystem_inspect(
|
|
2731
|
+
bundle_id: str=typer.Argument(...),
|
|
2732
|
+
registry: Path | None=typer.Option(None, '--registry', help='Public ecosystem index JSON path for offline/test use.'),
|
|
2733
|
+
):
|
|
2734
|
+
bundle_path=get_settings().app_home / 'ecosystem' / 'bundles' / bundle_id
|
|
2735
|
+
if bundle_path.exists() and (bundle_path/'contribution_manifest.json').exists():
|
|
2736
|
+
typer.echo(_summarize_bundle_manifest(read_json(bundle_path/'contribution_manifest.json')))
|
|
2737
|
+
return
|
|
2738
|
+
try:
|
|
2739
|
+
row=_find_registry_row(bundle_id, registry)
|
|
2740
|
+
except FileNotFoundError as exc:
|
|
2741
|
+
typer.echo(str(exc))
|
|
2742
|
+
raise typer.Exit(1)
|
|
2743
|
+
if _is_seed_registry_row(row):
|
|
2744
|
+
typer.echo(_summarize_seed_registry_row(row))
|
|
2745
|
+
return
|
|
2746
|
+
typer.echo(_summarize_contribution_registry_row(row))
|
|
2747
|
+
|
|
2748
|
+
|
|
2749
|
+
@bundle_app.command('inspect')
|
|
2750
|
+
def bundle_inspect(path: Path=typer.Argument(..., help='Contribution Bundle folder.')):
|
|
2751
|
+
manifest=path/'contribution_manifest.json'
|
|
2752
|
+
if not manifest.exists():
|
|
2753
|
+
typer.echo(f'Contribution Bundle manifest not found: {manifest}')
|
|
2754
|
+
raise typer.Exit(1)
|
|
2755
|
+
manifest_data=read_json(manifest)
|
|
2756
|
+
artifacts=_artifact_index(path)
|
|
2757
|
+
followups=[event for event in _session_events(path) if event.get("event_type") == "user_followup"]
|
|
2758
|
+
checkpoints=list((path / "mentor_checkpoints").glob("*.json")) if (path / "mentor_checkpoints").exists() else []
|
|
2759
|
+
typer.echo(_summarize_bundle_manifest(manifest_data))
|
|
2760
|
+
session=_session_metadata(path)
|
|
2761
|
+
if session.get("apprentice_agent"):
|
|
2762
|
+
typer.echo(f"apprentice_agent: {session.get('apprentice_agent')}")
|
|
2763
|
+
if session.get("model_provider"):
|
|
2764
|
+
typer.echo(f"mentor_model_provider: {session.get('model_provider')}")
|
|
2765
|
+
typer.echo(f"artifact_count: {len(artifacts)}")
|
|
2766
|
+
typer.echo(f"follow_up_count: {len(followups)}")
|
|
2767
|
+
if checkpoints:
|
|
2768
|
+
typer.echo(f"mentor_checkpoint_count: {len(checkpoints)}")
|
|
2769
|
+
top=[row.get("package_relative_path") or row.get("artifact_ref") for row in artifacts[:5]]
|
|
2770
|
+
if top:
|
|
2771
|
+
typer.echo("top_artifacts:")
|
|
2772
|
+
for ref in top:
|
|
2773
|
+
typer.echo(f"- {ref}")
|
|
2774
|
+
typer.echo("")
|
|
2775
|
+
typer.echo(f"Next: apprentice ecosystem contribute {path}")
|
|
2776
|
+
|
|
2777
|
+
|
|
2778
|
+
@bundle_app.command('contribute')
|
|
2779
|
+
def bundle_contribute(path: Path=typer.Argument(..., help='Contribution Bundle folder.')):
|
|
2780
|
+
manifest=path/'contribution_manifest.json'
|
|
2781
|
+
if not manifest.exists():
|
|
2782
|
+
typer.echo(f'Contribution Bundle manifest not found: {manifest}')
|
|
2783
|
+
raise typer.Exit(1)
|
|
2784
|
+
try:
|
|
2785
|
+
result = _ecosystem_contribute_impl(path, auto_share_mode="manual")
|
|
2786
|
+
except Exception as exc:
|
|
2787
|
+
typer.echo(f"Could not prepare public ecosystem contribution: {redact_secrets(str(exc))}")
|
|
2788
|
+
raise typer.Exit(1)
|
|
2789
|
+
metadata = result["metadata"]
|
|
2790
|
+
typer.echo('Contribution Bundle ready.')
|
|
2791
|
+
typer.echo('')
|
|
2792
|
+
typer.echo(f"Bundle ID: {metadata.get('bundle_id')}")
|
|
2793
|
+
typer.echo(f"Title: {metadata.get('title')}")
|
|
2794
|
+
typer.echo('Bundle:')
|
|
2795
|
+
typer.echo(str(path))
|
|
2796
|
+
typer.echo('')
|
|
2797
|
+
if result["contribution_created"]:
|
|
2798
|
+
typer.echo(f"Public contribution URL: {result['contribution_url']}")
|
|
2799
|
+
typer.echo("No bundle files were uploaded automatically. Attach or submit the generated package according to the repo instructions.")
|
|
2800
|
+
return
|
|
2801
|
+
typer.echo('No upload was performed by this command.')
|
|
2802
|
+
reason = result.get("skipped_reason")
|
|
2803
|
+
if reason:
|
|
2804
|
+
typer.echo(f"Reason: {reason}")
|
|
2805
|
+
typer.echo('')
|
|
2806
|
+
typer.echo('Contribute to public ecosystem:')
|
|
2807
|
+
typer.echo(f'apprentice ecosystem contribute {path}')
|
|
2808
|
+
typer.echo('')
|
|
2809
|
+
typer.echo('Public ecosystem:')
|
|
2810
|
+
typer.echo(_ecosystem_repo_url())
|
|
2811
|
+
typer.echo('')
|
|
2812
|
+
typer.echo('View files:')
|
|
2813
|
+
typer.echo(f'open {path}')
|
|
2814
|
+
|
|
2815
|
+
@app.command('init-env')
|
|
2816
|
+
def init_env():
|
|
2817
|
+
dst=Path('.env.local')
|
|
2818
|
+
if not dst.exists(): shutil.copyfile('.env.example', dst); typer.echo('created .env.local from .env.example')
|
|
2819
|
+
else: typer.echo('.env.local already exists')
|
|
2820
|
+
|
|
2821
|
+
@app.command('run-task')
|
|
2822
|
+
def run_task(input: Path=typer.Option(Path('data/seed_tasks/hard_finance_reconciliation.jsonl')), output_root: Path=Path('outputs'), runner: str='deterministic'):
|
|
2823
|
+
raw=RawTaskRecord.model_validate(read_jsonl(input)[0])
|
|
2824
|
+
run_root=output_root/'runs'/'single'
|
|
2825
|
+
pkg=run_one(raw, run_root, runner=runner); typer.echo(str(pkg))
|
|
2826
|
+
|
|
2827
|
+
@app.command('run-batch')
|
|
2828
|
+
def run_batch(input: Path=typer.Option(...), limit: int|None=None, resume: bool=False, max_parallel: int=1, retry_limit: int=0, task_timeout_seconds: int=900, runner: str='deterministic', release_id: str|None=None, output_root: Path=Path('outputs'), max_iterations: int|None=None):
|
|
2829
|
+
typer.echo(str(run_batch_impl(input, output_root, limit, resume, max_parallel, retry_limit, task_timeout_seconds, runner, release_id, max_iterations=max_iterations)))
|
|
2830
|
+
|
|
2831
|
+
@app.command('run-many')
|
|
2832
|
+
def run_many(input: Path=typer.Option(...), limit: int|None=None, resume: bool=False, max_parallel: int=1, retry_limit: int=0, task_timeout_seconds: int=900, runner: str='deterministic', release_id: str|None=None, output_root: Path=Path('outputs'), max_iterations: int|None=None):
|
|
2833
|
+
typer.echo(str(run_batch_impl(input, output_root, limit, resume, max_parallel, retry_limit, task_timeout_seconds, runner, release_id, max_iterations=max_iterations)))
|
|
2834
|
+
|
|
2835
|
+
@app.command('create-bundle')
|
|
2836
|
+
def create_bundle(
|
|
2837
|
+
run_root: Path=typer.Option(...),
|
|
2838
|
+
bundle_root: Path|None=typer.Option(None),
|
|
2839
|
+
include_debug: bool=typer.Option(False, '--include-debug', help='Include debug validation reports in the bundle.'),
|
|
2840
|
+
release_style: bool=typer.Option(False, '--release-style', help='Create the full internal release-style export.'),
|
|
2841
|
+
):
|
|
2842
|
+
typer.echo(str(create_contribution_bundle(run_root, bundle_root, include_debug=include_debug, release_style=release_style)))
|
|
2843
|
+
|
|
2844
|
+
@app.command('create-release')
|
|
2845
|
+
def create_release(run_root: Path=typer.Option(Path('outputs/runs/single')), release_root: Path=typer.Option(Path('outputs/releases/manual'))):
|
|
2846
|
+
typer.echo(str(create_release_impl(run_root, release_root)))
|
|
2847
|
+
|
|
2848
|
+
@app.command('validate-release')
|
|
2849
|
+
def validate_release(release_root: Path=typer.Option(...)):
|
|
2850
|
+
c=validate_release_impl(release_root); typer.echo(format_counters(c)); raise typer.Exit(0 if c['release_valid'] else 1)
|
|
2851
|
+
|
|
2852
|
+
@app.command('validate-public-release')
|
|
2853
|
+
def validate_public_release(release_root: Path=typer.Option(...)):
|
|
2854
|
+
public=release_root/'public' if (release_root/'public').exists() else release_root
|
|
2855
|
+
c=validate_release_impl(release_root if (release_root/'public').exists() else release_root.parent) if public.name == 'public' else validate_release_impl(release_root)
|
|
2856
|
+
typer.echo(format_counters({k:v for k,v in c.items() if k.startswith('public_') or k in ['secret_scan_ok']}))
|
|
2857
|
+
raise typer.Exit(0 if c.get('public_release_valid') else 1)
|
|
2858
|
+
|
|
2859
|
+
|
|
2860
|
+
@app.command('repair-roles')
|
|
2861
|
+
def repair_roles(run_root: Path=typer.Option(...), task_id: str=typer.Option(...), roles: str=typer.Option('evaluator_agent,grader_agent,verifier_agent'), attempts: str=typer.Option('baseline,revised'), rebuild_release: Path|None=typer.Option(None)):
|
|
2862
|
+
"""Re-run model evaluation roles from existing task package artifacts/traces."""
|
|
2863
|
+
from .schemas import RubricSpec, ActualOutputs, AgentTrace, GraderResult, VerifierResult
|
|
2864
|
+
from .grader import grade_attempt, apply_score_reliability
|
|
2865
|
+
from .verifier import verify_attempt
|
|
2866
|
+
from .evaluator import evaluate_attempt
|
|
2867
|
+
from .io import read_json, write_json
|
|
2868
|
+
from datetime import datetime, timezone
|
|
2869
|
+
import shutil
|
|
2870
|
+
pkg=run_root/'packages'/task_id
|
|
2871
|
+
rubric=RubricSpec.model_validate(read_json(pkg/'rubric/rubric.json'))
|
|
2872
|
+
selected={x.strip() for x in roles.split(',') if x.strip()}
|
|
2873
|
+
selected_attempts=[x.strip() for x in attempts.split(',') if x.strip()]
|
|
2874
|
+
role_root=run_root/'roles'/task_id
|
|
2875
|
+
archive_root=role_root/'repair_archive'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
2876
|
+
for attempt in selected_attempts:
|
|
2877
|
+
actual_path=pkg/'attempts'/attempt/'actual_outputs.json'
|
|
2878
|
+
trace_path=pkg/'attempts'/attempt/'agent_trace.json'
|
|
2879
|
+
if not actual_path.exists() or not trace_path.exists():
|
|
2880
|
+
typer.echo(f'skipping {attempt}: missing existing attempt outputs')
|
|
2881
|
+
continue
|
|
2882
|
+
actual=ActualOutputs.model_validate(read_json(actual_path))
|
|
2883
|
+
trace=AgentTrace.model_validate(read_json(trace_path))
|
|
2884
|
+
grade_path=pkg/'grading'/f'{attempt}_grader_result.json'
|
|
2885
|
+
ver_path=pkg/'grading'/f'{attempt}_verifier_result.json'
|
|
2886
|
+
grader=GraderResult.model_validate(read_json(grade_path)) if grade_path.exists() else None
|
|
2887
|
+
verifier=VerifierResult.model_validate(read_json(ver_path)) if ver_path.exists() else None
|
|
2888
|
+
if 'grader_agent' in selected or grader is None:
|
|
2889
|
+
old=role_root/'grader_agent'/attempt
|
|
2890
|
+
if old.exists():
|
|
2891
|
+
archive_root.mkdir(parents=True, exist_ok=True); shutil.copytree(old, archive_root/f'grader_agent_{attempt}', dirs_exist_ok=True)
|
|
2892
|
+
grader=grade_attempt(rubric, actual, attempt, trace, role_root, pkg)
|
|
2893
|
+
if 'verifier_agent' in selected or verifier is None:
|
|
2894
|
+
old=role_root/'verifier_agent'/attempt
|
|
2895
|
+
if old.exists():
|
|
2896
|
+
archive_root.mkdir(parents=True, exist_ok=True); shutil.copytree(old, archive_root/f'verifier_agent_{attempt}', dirs_exist_ok=True)
|
|
2897
|
+
verifier=verify_attempt(grader, actual, trace, role_root, pkg)
|
|
2898
|
+
grader=apply_score_reliability(grader, verifier)
|
|
2899
|
+
grader.metadata_json['repair_run']=True; verifier.metadata_json['repair_run']=True
|
|
2900
|
+
write_json(grade_path, grader); write_json(ver_path, verifier)
|
|
2901
|
+
if 'evaluator_agent' in selected:
|
|
2902
|
+
old=role_root/'evaluator_agent'/attempt
|
|
2903
|
+
if old.exists():
|
|
2904
|
+
archive_root.mkdir(parents=True, exist_ok=True); shutil.copytree(old, archive_root/f'evaluator_agent_{attempt}', dirs_exist_ok=True)
|
|
2905
|
+
target=f'{task_id}_revised' if attempt == 'baseline' else f'{task_id}_{attempt}_followup'
|
|
2906
|
+
fb,rp=evaluate_attempt(grader, verifier, actual, trace, target, role_root, pkg)
|
|
2907
|
+
fb.metadata_json['repair_run']=True; rp.metadata_json['repair_run']=True
|
|
2908
|
+
if attempt == 'baseline':
|
|
2909
|
+
write_json(pkg/'feedback/baseline_evaluator_feedback.json', fb); write_json(pkg/'feedback/revision_plan.json', rp)
|
|
2910
|
+
else:
|
|
2911
|
+
write_json(pkg/'feedback'/f'{attempt}_evaluator_feedback.json', fb)
|
|
2912
|
+
typer.echo(f'repaired roles for task_id={task_id} attempt={attempt}')
|
|
2913
|
+
if rebuild_release is not None:
|
|
2914
|
+
from .release_exporter import create_release
|
|
2915
|
+
create_release(run_root, rebuild_release)
|
|
2916
|
+
typer.echo(f'rebuilt release={rebuild_release}')
|
|
2917
|
+
|
|
2918
|
+
|
|
2919
|
+
@app.command('summarize-releases')
|
|
2920
|
+
def summarize_releases(release_root: Path=typer.Option(Path('outputs/releases')), pattern: str=typer.Option('tasks-*')):
|
|
2921
|
+
"""Summarize release readiness across a release directory."""
|
|
2922
|
+
from .validation import validate_release
|
|
2923
|
+
totals={'total_releases':0,'green_scale_ready_count':0,'release_valid_false_count':0,'fallback_only_count':0,'model_role_incomplete_count':0,'model_score_count_lt_2_count':0,'failed_verification_count':0}
|
|
2924
|
+
needing=[]; warnings=[]; clean=[]
|
|
2925
|
+
for rel in sorted(release_root.glob(pattern)):
|
|
2926
|
+
if not rel.is_dir():
|
|
2927
|
+
continue
|
|
2928
|
+
totals['total_releases'] += 1
|
|
2929
|
+
c=validate_release(rel)
|
|
2930
|
+
task_ids=[]
|
|
2931
|
+
try:
|
|
2932
|
+
task_ids=[str(r.get('task_id') or r.get('raw_task_id')) for r in read_jsonl(rel/'tasks.jsonl')]
|
|
2933
|
+
except Exception:
|
|
2934
|
+
task_ids=[rel.name]
|
|
2935
|
+
label=','.join(task_ids) or rel.name
|
|
2936
|
+
if c.get('scale_ready'):
|
|
2937
|
+
totals['green_scale_ready_count'] += 1
|
|
2938
|
+
if c.get('scale_warnings'):
|
|
2939
|
+
warnings.append(label)
|
|
2940
|
+
else:
|
|
2941
|
+
clean.append(label)
|
|
2942
|
+
else:
|
|
2943
|
+
needing.append(label)
|
|
2944
|
+
if not c.get('release_valid'): totals['release_valid_false_count'] += 1
|
|
2945
|
+
if c.get('fallback_only_task_count'): totals['fallback_only_count'] += 1
|
|
2946
|
+
if not c.get('model_role_completeness_ok'): totals['model_role_incomplete_count'] += 1
|
|
2947
|
+
if int(c.get('model_score_count') or 0) < 2: totals['model_score_count_lt_2_count'] += 1
|
|
2948
|
+
if c.get('verifier_failed_count'): totals['failed_verification_count'] += 1
|
|
2949
|
+
for k,v in totals.items():
|
|
2950
|
+
typer.echo(f'{k}={v}')
|
|
2951
|
+
typer.echo('task_ids_needing_rerun=' + ','.join(needing))
|
|
2952
|
+
typer.echo('task_ids_publishable_with_warnings=' + ','.join(warnings))
|
|
2953
|
+
typer.echo('task_ids_clean_publishable=' + ','.join(clean))
|
|
2954
|
+
|
|
2955
|
+
@app.command('inspect-trace')
|
|
2956
|
+
def inspect_trace(trace_path: Path):
|
|
2957
|
+
t=AgentTrace.model_validate_json(trace_path.read_text()); typer.echo(f'trace_id={t.trace_id}\ntask_id={t.task_id}\nsteps={len(t.steps)}')
|
|
2958
|
+
|
|
2959
|
+
@app.command('codex-smoke')
|
|
2960
|
+
def codex_smoke():
|
|
2961
|
+
if not shutil.which('codex'):
|
|
2962
|
+
typer.echo('codex_available=false'); raise typer.Exit(1)
|
|
2963
|
+
typer.echo('codex_available=true')
|
|
2964
|
+
|
|
2965
|
+
@app.command('llm-smoke')
|
|
2966
|
+
def llm_smoke(
|
|
2967
|
+
output_dir: Path=Path('outputs/llm_smoke'),
|
|
2968
|
+
provider: str | None=typer.Option(None, '--provider', help='Mentor Model Provider id to test. Defaults to configured provider.'),
|
|
2969
|
+
):
|
|
2970
|
+
if provider is not None and provider not in MODEL_PROVIDER_RECIPES:
|
|
2971
|
+
raise typer.BadParameter(f'Mentor Model Provider must be one of: {", ".join(MODEL_PROVIDER_RECIPES)}')
|
|
2972
|
+
counters=run_llm_smoke(output_dir, provider_id=provider)
|
|
2973
|
+
typer.echo(format_smoke_counters(counters))
|
|
2974
|
+
roles=['intake','rubric','grader','verifier','evaluator']
|
|
2975
|
+
ok=counters.get('mentor_model_provider_available') and all(counters.get(f'{r}_live_call_ok') and counters.get(f'{r}_structured_output_validation_ok') for r in roles) and counters.get('secret_scan_ok')
|
|
2976
|
+
raise typer.Exit(0 if ok else 1)
|
|
2977
|
+
|
|
2978
|
+
def main(): app()
|
|
2979
|
+
if __name__=='__main__': main()
|