agent-apprenticeship 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +261 -1
  2. package/bin/agent-apprenticeship.js +1 -2
  3. package/package.json +4 -4
  4. package/pyproject.toml +10 -4
  5. package/schemas/environment_replay.schema.json +41 -0
  6. package/schemas/eval_package.schema.json +37 -0
  7. package/schemas/experience_compilation.schema.json +53 -0
  8. package/schemas/runtime_training.schema.json +33 -0
  9. package/schemas/source_evidence_map.schema.json +25 -0
  10. package/schemas/trace.schema.json +904 -0
  11. package/schemas/trace_step.schema.json +622 -0
  12. package/schemas/training_rows.schema.json +29 -0
  13. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/__init__.py +1 -1
  14. package/src/agent_apprenticeship/agent_env_exporter.py +53 -0
  15. package/src/agent_apprenticeship/agent_env_verify.py +32 -0
  16. package/src/agent_apprenticeship/atif.py +152 -0
  17. package/src/agent_apprenticeship/atif_models/LICENSE +201 -0
  18. package/src/agent_apprenticeship/atif_models/__init__.py +38 -0
  19. package/src/agent_apprenticeship/atif_models/agent.py +35 -0
  20. package/src/agent_apprenticeship/atif_models/content.py +159 -0
  21. package/src/agent_apprenticeship/atif_models/final_metrics.py +43 -0
  22. package/src/agent_apprenticeship/atif_models/metrics.py +47 -0
  23. package/src/agent_apprenticeship/atif_models/observation.py +19 -0
  24. package/src/agent_apprenticeship/atif_models/observation_result.py +45 -0
  25. package/src/agent_apprenticeship/atif_models/step.py +142 -0
  26. package/src/agent_apprenticeship/atif_models/subagent_trajectory_ref.py +88 -0
  27. package/src/agent_apprenticeship/atif_models/tool_call.py +34 -0
  28. package/src/agent_apprenticeship/atif_models/trajectory.py +196 -0
  29. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/batch_runner.py +1 -1
  30. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/bundle_exporter.py +8 -1
  31. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/certification.py +1 -1
  32. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/cli.py +26 -7
  33. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/codex_runner.py +17 -17
  34. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/config.py +1 -1
  35. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/ecosystem_r2.py +3 -1
  36. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/evaluator.py +8 -27
  37. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/grader.py +6 -5
  38. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/integration_status.py +1 -1
  39. package/src/agent_apprenticeship/io.py +104 -0
  40. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/learning.py +7 -1
  41. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/loop.py +5 -5
  42. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/loop_review.py +1 -1
  43. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/mentor_checkpoints.py +1 -1
  44. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/package_exporter.py +5 -3
  45. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/progress.py +1 -1
  46. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/public_run.py +1 -1
  47. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/public_sanitizer.py +2 -2
  48. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/release_exporter.py +7 -4
  49. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/rubric_generation.py +1 -1
  50. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/schemas.py +17 -4
  51. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/session_events.py +4 -4
  52. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/task_intake.py +1 -1
  53. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/tdo.py +5 -1
  54. package/src/agent_apprenticeship/trace_normalizer.py +103 -0
  55. package/src/agent_apprenticeship/trace_prompt.py +48 -0
  56. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/training_signals.py +1 -1
  57. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/validation.py +3 -3
  58. package/src/{agent_apprenticeship_trace → agent_apprenticeship}/verifier.py +6 -5
  59. package/docs/bioactive_evidence_mapping_schema.md +0 -113
  60. package/docs/dataset_schema.md +0 -160
  61. package/docs/local_usage.md +0 -191
  62. package/docs/scaling_to_1000_tasks.md +0 -3
  63. package/docs/trace_schema.md +0 -192
  64. package/docs/v0.2_release_notes.md +0 -63
  65. package/src/agent_apprenticeship_trace/io.py +0 -20
  66. package/src/agent_apprenticeship_trace/trace_normalizer.py +0 -1026
  67. package/src/agent_apprenticeship_trace/trace_prompt.py +0 -78
  68. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/actual_outputs_normalizer.py +0 -0
  69. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/apprentice_adapters.py +0 -0
  70. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/artifact_capture.py +0 -0
  71. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/artifact_previews.py +0 -0
  72. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/artifact_resolver.py +0 -0
  73. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/command_discovery.py +0 -0
  74. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/contract_diagnostics.py +0 -0
  75. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/contract_verifier.py +0 -0
  76. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/env.py +0 -0
  77. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/lesson_extractor.py +0 -0
  78. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/llm_output_normalizer.py +0 -0
  79. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/openai_structured.py +0 -0
  80. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/recipes.py +0 -0
  81. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/revision.py +0 -0
  82. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/role_runners.py +0 -0
  83. /package/src/{agent_apprenticeship_trace → agent_apprenticeship}/version.py +0 -0
@@ -1,160 +0,0 @@
1
- # Agent Apprenticeship Dataset And Package Schema
2
-
3
- Agent Apprenticeship packages capture real agent workflows as reusable experience. A package may include task packets, Apprentice-generated task specs, audited rubrics, traces, actual outputs, artifacts, evaluator/verifier/grader records, mentor or expert checkpoint records, follow-ups, Experience Compiler outputs, and public ecosystem storage references.
4
-
5
- ## Apprenticeship Modes
6
-
7
- New package metadata uses:
8
-
9
- - `apprenticeship_mode = autonomous`
10
- - `apprenticeship_mode = expert_led`
11
- - `apprenticeship_mode = org_custom`
12
-
13
- Legacy packages may include `mentor_mode` for internal compatibility. New public indexes, R2 ecosystem items, compiler manifests, and contribution manifests should include `apprenticeship_mode`.
14
-
15
- ## Agent Training Contribution Mode
16
-
17
- Local settings support:
18
-
19
- - Public Ecosystem: completed Experience Compilations are shared through the Forsy R2 ecosystem backend by default.
20
- - Private Internal Only: completed packages remain local.
21
-
22
- This setting is local user preference. It is not written as a public ecosystem item field.
23
-
24
- ## Core Package Layout
25
-
26
- Typical local package content:
27
-
28
- - `task/` for task packet, task brief, inputs, and task-intake spec
29
- - `rubric/` for Apprentice-generated rubric plus audit metadata
30
- - `attempts/<kind>/agent_trace.json`
31
- - `attempts/<kind>/actual_outputs.json`
32
- - `attempts/<kind>/artifacts/`
33
- - `grading/`, `feedback/`, and `signals/`
34
- - `mentor_checkpoints/` when expert or organization workflows record checkpoints
35
- - `experience_compiler/`
36
- - optional legacy compiler compatibility outputs
37
-
38
- `actual_outputs.json` remains the canonical output contract. Final/static outputs are represented by actual outputs, deliverable refs, artifact refs, and copied files rather than large top-level trace fields.
39
-
40
- ## Experience Compiler Outputs
41
-
42
- The Experience Compiler turns completed workflows into reusable skills, evals, verifier specs, reward signals, environment-response rows, runtime learning packages, and training-ready datasets.
43
-
44
- Public compiler layout:
45
-
46
- - `experience_compiler/compiler_manifest.json`
47
- - `experience_compiler/compiler_report.json`
48
- - `experience_compiler/compiler_trace.jsonl`
49
- - `experience_compiler/compiler_rounds.jsonl`
50
- - `experience_compiler/compiler_recipe.json`
51
- - `experience_compiler/compiler_judge_results.jsonl`
52
- - `experience_compiler/learning_objects_index.json`
53
- - `experience_compiler/experience_pack/`
54
- - `experience_compiler/skill_pack/skill.md`
55
- - `experience_compiler/runtime_learning_package/`
56
- - `experience_compiler/training_time_package/`
57
- - `experience_compiler/evals/`
58
- - `experience_compiler/verifier/`
59
- - `experience_compiler/reward/`
60
- - `experience_compiler/environment/`
61
- - `experience_compiler/reports/`
62
-
63
- `skill_pack/skill.md` should be task-specific and evidence-grounded, with best-use guidance, prerequisites, workflow steps, decision points, common failure patterns, recovery strategies, evidence and artifact expectations, verification checklist, examples, and transfer notes.
64
-
65
- ## Loop Review Evidence
66
-
67
- Current Experience Compilations may include per-iteration loop evidence under `loops/`. These artifacts make autonomous model-review loops and Expert-Led review loops inspectable before feedback or continuation decisions.
68
-
69
- Required loop review artifacts, when a loop iteration runs:
70
-
71
- - `loops/loop_manifest.json`
72
- - `loops/iterations/<nnn>/review_packet.json`
73
- - `loops/iterations/<nnn>/review_packet.md`
74
- - `loops/iterations/<nnn>/output_snapshot.json`
75
- - `loops/iterations/<nnn>/artifact_index.json`
76
- - `loops/iterations/<nnn>/trace_refs.json`
77
- - `loops/iterations/<nnn>/rubric_status.json`
78
- - `loops/iterations/<nnn>/verifier_status.json`
79
- - `loops/iterations/<nnn>/evaluator_feedback.json`
80
- - `loops/iterations/<nnn>/reviewer_feedback.json`
81
- - `loops/iterations/<nnn>/revision_plan.json`
82
- - `loops/iterations/<nnn>/diff_report.json`
83
- - `loops/iterations/<nnn>/loop_decision.json`
84
-
85
- Review packets should summarize the current goal, current outputs, actual output contract, artifacts, trace refs, key trace evidence, rubric/verifier status, known failures, missing outputs, surface and environment summaries, diffs, review instructions, and stop/continue choices. `reviewer_feedback.json` should record whether feedback came from a Mentor Model Provider, human expert, organization/custom reviewer, or simulated test checkpoint. Simulated feedback must be labelled as simulated.
86
-
87
- Experience Compiler evidence maps can reference `review_packet.json`, `reviewer_feedback.json`, `revision_plan.json`, `loop_decision.json`, and `diff_report.json`. Loop evidence can support critique/revision pairs, reward-model rows, verifier rows, process-supervision rows, failure modes, recovery strategies, and runtime decision rules. Older seed packages remain valid when loop review evidence is absent.
88
-
89
- ## Runtime Learning Package
90
-
91
- Runtime learning material can be injected into future local runs as reusable experience. It may include runtime memory rows, tool-use hints, correction patterns, and Experience Pack references. It must not claim model weights were updated.
92
-
93
- ## Ecosystem Learning And Experience Packs
94
-
95
- Ecosystem Learning creates reversible Experience Packs from seed tasks, Experience Compilations, or pulled ecosystem items. Each Experience Pack contains `skill.md`, compact source refs, replay results, status records, and activation metadata.
96
-
97
- ## Training-Time Package
98
-
99
- Training-time material is ready for future supervised fine-tuning, eval building, verifier training, reward modeling, preference optimization, and environment-response training. It may include:
100
-
101
- - `training_examples.jsonl`
102
- - `evaluation_examples.jsonl`
103
- - `verifier_examples.jsonl`
104
- - `reward_model_rows.jsonl`
105
- - `preference_pairs.jsonl`
106
- - `failure_cases.jsonl`
107
- - `task_variants.jsonl`
108
- - `transfer_eval_tasks.jsonl`
109
-
110
- Rows should preserve `source_package_id`, `source_task_id`, `source_trace_id`, `source_step_ids`, `source_artifact_refs`, `source_evaluation_refs`, provenance, quality/fidelity/difficulty/learnability/reuse scores, judge source, audit status, and acceptance status.
111
-
112
- ## Environment And Surface Data
113
-
114
- The environment folder can include:
115
-
116
- - `environment_spec.json`
117
- - `environment_response_pairs.jsonl`
118
- - `multi_agent_handoffs.jsonl`
119
- - `multimodal_artifacts.jsonl`
120
- - `ui_interaction_events.jsonl`
121
- - `physical_environment_events.jsonl`
122
- - `api_interaction_events.jsonl`
123
- - `mcp_interaction_events.jsonl`
124
- - `database_events.jsonl`
125
- - `structured_table_events.jsonl`
126
- - `software_execution_events.jsonl`
127
- - `search_retrieval_events.jsonl`
128
- - `enterprise_app_events.jsonl`
129
- - `async_workflow_events.jsonl`
130
- - `collaboration_events.jsonl`
131
- - `document_events.jsonl`
132
- - `cross_system_events.jsonl`
133
-
134
- Empty optional files are valid when a surface is absent.
135
-
136
- ## Environment Spec
137
-
138
- `environment_spec.json` describes replay and world-model readiness. It may include environment type/domain, interaction surfaces, tools, runtime dependencies, input artifacts, setup commands, initial-state refs, allowed actions, reset strategy, verification commands, scoring refs, expected outputs, replay smoke-test details, and sections for API, MCP, database, spreadsheet, software execution, async workflows, collaboration, documents, cross-system dependencies, multi-agent, multimodal, UI/software, and physical/industrial workflows.
139
-
140
- ## Evaluation Authority
141
-
142
- Goal achievement and outcome quality belong to mentor, expert, evaluator, verifier, and grader records. Apprentice Agent self-assessment is not the source of truth. Structural guardrails may check schemas, paths, refs, secrets, and package completeness, but they are not semantic judges.
143
-
144
- ## R2 Public Ecosystem Records
145
-
146
- R2 ecosystem index records include item id, kind, title, summary, domain, subdomain, tags, timestamps, source, package id, task id, `apprenticeship_mode`, Apprentice Agent, Mentor Model Provider, status, counts, Experience Compiler row counts, contributor pseudonymous user id, and storage refs.
147
-
148
- Records do not include Agent Training Contribution Mode, local OS username, local hostname, secrets, Cloudflare credentials, or private local paths.
149
-
150
- ## Backward Compatibility
151
-
152
- Raw seed traces and older local packages remain compatible even if new fields are missing. Normalizers should preserve new fields when present and tolerate missing optional fields in older traces. Derived/enriched outputs can be added later without mutating the original seed dataset.
153
-
154
- ## Future Central Dataset Readiness
155
-
156
- Future enrichment can build environment specs, environment-response rows, training examples, eval examples, verifier examples, reward rows, preference pairs, failure cases, task variants, compiler reports, and meta-optimization records across the full seed dataset. This should be versioned clearly and uploaded through central infrastructure, not by mutating local seed data by default.
157
-
158
- ## Exclusions
159
-
160
- Dataset and public package metadata must not include commerce metadata, public ranking language, Apprentice Agent self-eval as outcome truth, API keys, auth tokens, cookies, `.env`, `.env.local`, private local paths, source scrape internals, or huge embedded binaries.
@@ -1,191 +0,0 @@
1
- # Local Usage
2
-
3
- Initialize Agent Apprenticeship:
4
-
5
- ```bash
6
- apprentice init --setup
7
- ```
8
-
9
- `apprentice` is the short command used in normal docs. `agent-apprenticeship` remains available as the long-form compatibility command.
10
-
11
- Run a task:
12
-
13
- ```bash
14
- apprentice run "Analyze this CSV and produce an executive summary" --asset ./data.csv
15
- ```
16
-
17
- Start interactively:
18
-
19
- ```bash
20
- apprentice start
21
- ```
22
-
23
- Continue or finish a run:
24
-
25
- ```bash
26
- apprentice continue <run_id> "Add sensitivity analysis" --asset ./updated_assumptions.csv
27
- apprentice finish <run_id>
28
- ```
29
-
30
- Each run is written under `~/.agent-apprenticeship/runs/<run-id>/` with progress files, task outputs, traces, Experience Compiler outputs, and an Experience Compilation.
31
-
32
- ```text
33
- run_status.json
34
- progress_events.jsonl
35
- actual_outputs.json
36
- experience_compiler/
37
- contribution_bundle/
38
- ```
39
-
40
- Check or watch progress:
41
-
42
- ```bash
43
- apprentice status <run_id>
44
- apprentice watch <run_id>
45
- ```
46
-
47
- Use output modes when scripting:
48
-
49
- ```bash
50
- apprentice run "Prepare a market brief" --quiet
51
- apprentice run "Prepare a market brief" --verbose
52
- apprentice run "Prepare a market brief" --json-progress
53
- ```
54
-
55
- ## Configuration
56
-
57
- Configure the Apprentice Agent, Mentor Model Provider, Apprenticeship Mode, and Agent Training Contribution Mode:
58
-
59
- ```bash
60
- apprentice configure agent
61
- apprentice configure model
62
- apprentice configure apprenticeship
63
- apprentice configure contribution
64
- apprentice settings
65
- apprentice doctor
66
- ```
67
-
68
- Apprenticeship Modes:
69
-
70
- - `autonomous`: Autonomous Apprenticeship. The agent works through the task with model guidance, iterative improvement loops, evaluation, and Experience Compiler outputs.
71
- - `expert-led`: Expert-Led Apprenticeship. The agent works with human expert checkpoints until acceptance criteria are met, expert sign-off is recorded, or the configured loop limit is reached.
72
- - `org-custom`: Organization Custom. For teams that need private deployment, custom integrations, internal storage, workflow-specific capture, or forward-deployed setup.
73
-
74
- Autonomous Apprenticeship needs a ready Mentor Model Provider. API keys are read from exported environment variables first and then from `.env.local` in the project or app home. `.env.local` is gitignored and key values are never printed.
75
-
76
- Agent Training Contribution Modes:
77
-
78
- - `public-ecosystem`: completed task and follow-up packages are shared to the Forsy ecosystem through Cloudflare R2.
79
- - `private-internal-only`: packages stay local unless explicitly contributed later.
80
-
81
- ```bash
82
- apprentice ecosystem configure --training-contribution public-ecosystem
83
- apprentice ecosystem configure --training-contribution private-internal-only
84
- ```
85
-
86
- ## Integrations
87
-
88
- Apprentice Agents:
89
-
90
- - `codex`
91
- - `cursor`
92
- - `claude-code`
93
- - `openclaw`
94
- - `opencode`
95
- - `hermes-agent`
96
- - `custom`
97
-
98
- Mentor Model Providers:
99
-
100
- - `openai`
101
- - `anthropic`
102
- - `google`
103
- - `openrouter`
104
-
105
- Run:
106
-
107
- ```bash
108
- apprentice integrations
109
- apprentice certify-integrations --all
110
- ```
111
-
112
- Use `--all-combinations` only when you intentionally want every Apprentice Agent and Mentor Model Provider pair; it is slower and can consume provider quota.
113
-
114
- ## Experience Compiler
115
-
116
- The Experience Compiler turns each completed workflow into reusable skills, evals, verifier specs, reward signals, environment-response data, runtime learning packages, and training-ready datasets.
117
-
118
- Public packages include:
119
-
120
- ```text
121
- experience_compiler/
122
- experience_pack/
123
- skill_pack/
124
- runtime_learning_package/
125
- training_time_package/
126
- evals/
127
- verifier/
128
- reward/
129
- environment/
130
- reports/
131
- ```
132
-
133
- `actual_outputs.json` remains the canonical output contract for completed task artifacts.
134
-
135
- ## Ecosystem
136
-
137
- Cloudflare R2 is the default ecosystem storage backend.
138
-
139
- ```bash
140
- apprentice ecosystem status
141
- apprentice ecosystem list
142
- apprentice ecosystem search "finance"
143
- apprentice ecosystem inspect <item_id>
144
- apprentice ecosystem pull <item_id>
145
- ```
146
-
147
- Share a package explicitly:
148
-
149
- ```bash
150
- apprentice ecosystem contribute ~/.agent-apprenticeship/runs/<run-id>/contribution_bundle
151
- ```
152
-
153
- Dry-run a share:
154
-
155
- ```bash
156
- apprentice ecosystem contribute ~/.agent-apprenticeship/runs/<run-id>/contribution_bundle --dry-run
157
- ```
158
-
159
- Legacy GitHub issue submission remains available only when explicitly selected:
160
-
161
- ```bash
162
- apprentice ecosystem contribute <bundle_path> --storage github
163
- ```
164
-
165
- Inspect local sharing history:
166
-
167
- ```bash
168
- apprentice ecosystem identity
169
- apprentice ecosystem shared
170
- apprentice ecosystem shared --json
171
- apprentice ecosystem list --mine
172
- ```
173
-
174
- ## Ecosystem Learning
175
-
176
- Use Ecosystem Learning when you want your Apprentice Agent to learn from an ecosystem task, trace, or package through a reversible Experience Pack.
177
-
178
- ```bash
179
- apprentice ecosystem search cloud
180
- apprentice ecosystem pull aa-seed-task-501
181
- apprentice learn create aa-seed-task-501
182
- apprentice learn preview <pack_id>
183
- apprentice learn replay <pack_id>
184
- apprentice learn keep <pack_id>
185
- apprentice run "Create a related incident response checklist" --experience-pack <pack_id>
186
- apprentice learn revert <pack_id>
187
- ```
188
-
189
- Experience Packs live in `~/.agent-apprenticeship/experience_packs/<pack_id>/`.
190
-
191
- Rich artifacts are included through indexing rather than special folder rules. Reports, spreadsheets, CSV/JSON/Markdown, screenshots, browser logs, MCP/tool logs, notebooks, code, and generated files are packaged when they appear in the run/package artifact areas and are linked to trace steps.
@@ -1,3 +0,0 @@
1
- # Scaling to 1000 tasks
2
-
3
- Use `run-batch` with checkpointing, resume, progress logging, local packages, and release export. Start with 1, then 3, then 10 tasks before increasing parallelism. Each task packet preserves workflow traces, artifacts, model-judged evaluation, verifier-backed reliability, and revision signals for downstream research use.
@@ -1,192 +0,0 @@
1
- # Agent Apprenticeship Trace Schema
2
-
3
- ## Purpose
4
-
5
- Agent Apprenticeship traces record authentic task work so completed workflows can become reusable agent experience. AgentTrace records observable workflow steps: task state, Apprentice Agent action, observed environment response, next state, artifacts produced, and evidence used by mentors, evaluators, verifiers, and graders.
6
-
7
- Official quality labels come from mentor, expert, evaluator, verifier, and grader records, not from Apprentice Agent self-evaluation. This separation keeps the raw trace focused on what happened while outcome labels remain grounded in external evidence.
8
-
9
- ## Public Product Model
10
-
11
- Public task metadata uses `apprenticeship_mode`:
12
-
13
- - `autonomous` for Autonomous Apprenticeship
14
- - `expert_led` for Expert-Led Apprenticeship
15
- - `org_custom` for Organization Custom
16
-
17
- The older internal `mentor_mode` field may remain in legacy packages for compatibility, but public docs and new package metadata should prefer `apprenticeship_mode`. Mentor Model Providers, experts, evaluators, verifiers, and graders remain outcome evidence sources. Apprentice Agent self-assessment is not outcome truth.
18
-
19
- ## Authentic Workflow Tracing
20
-
21
- Trace real task work only. Do not trace Agent Apprenticeship packaging, public upload mechanics, or ecosystem storage steps as task work. Keep one meaningful action per step. Do not compress multiple actions into a vague paragraph. Preserve causality through `caused_by`, `causal_type`, and `causal_note`.
22
-
23
- ## Canonical Outputs
24
-
25
- `actual_outputs.json` is the canonical output contract. Traces should include `actual_outputs_ref` when known, plus `artifact_refs`, `deliverable_refs`, `files_created`, `output_summary`, and `final_output_summary` derived from actual outputs rather than copied raw deliverables.
26
-
27
- Large files, screenshots, PDFs, tables, logs, audio, video, workbooks, and binary artifacts must be referenced through package-relative artifact refs. Do not embed huge binaries in JSON.
28
-
29
- ## Observation, Input, Output, State Change
30
-
31
- `observation` records what the agent saw before acting. `input` records the exact command, tool request, user instruction, artifact, or feedback used. `output` records the observed tool/environment response. `state_change` records what changed after the action.
32
-
33
- ## Step Core Fields
34
-
35
- Meaningful non-user steps should populate:
36
-
37
- - `step`, `turn`, `actor`, `action`
38
- - `operation`, `tool`
39
- - `observation`, `input`, `output`, `state_change`
40
- - `reasoning`
41
- - `caused_by`, `causal_type`, `causal_note`
42
- - `success`, `step_outcome`
43
- - `artifact_refs` when evidence or outputs are created, read, or validated
44
-
45
- ## Conditional Fields
46
-
47
- Populate conditional fields when the event calls for them:
48
-
49
- - `input_source` when input comes from the user, a prior step, a tool result, an artifact, mentor feedback, verifier feedback, expert feedback, a follow-up, or retry context.
50
- - `feedback_type` and `feedback_content` when the step records feedback, correction, approval, clarification, revision guidance, or follow-up instructions.
51
- - `retry_of` when a step repairs or revises a prior step.
52
- - `alternatives_considered` for meaningful strategy, tool, remediation, or design choices.
53
- - `execution_mode` defaults to `serial`; use `parallel` and `parallel_group` only for real fanout work.
54
- - `started_at` and `ended_at` only when measured by runtime.
55
-
56
- ## Environment Domain Taxonomy And Interaction Surface Taxonomy
57
-
58
- Every meaningful operation should identify its direct interface with `interaction_surface`, broader category with `environment_domain`, and a subtype with `surface_type` when useful.
59
-
60
- Supported surfaces:
61
-
62
- - `api`
63
- - `mcp_tool`, `mcp_resource`, `mcp_prompt`
64
- - `browser`, `computer_use`
65
- - `cli`, `file`, `codebase`
66
- - `database`
67
- - `spreadsheet`, `table`
68
- - `search`
69
- - `cloud_console`, `saas_app`
70
- - `async_job`, `queue`, `webhook`, `scheduler`
71
- - `messaging`, `email`, `collaboration`
72
- - `document`, `pdf`
73
- - `multimodal`
74
- - `device`, `industrial`, `physical`
75
- - `mixed`, `other_tool`, `other_surface`, `unknown`
76
-
77
- If a workflow crosses systems, use `environment_domain = mixed`, record `connected_surfaces`, split actions by system, and preserve `cross_system_dependencies`, `handoff_refs`, `state_sync_refs`, `consistency_checks`, and `system_of_record`.
78
-
79
- ## Surface-Specific Evidence
80
-
81
- When an operation clearly uses a surface, do not leave the relevant fields silently empty. If data is unavailable, set `surface_capture_status`, `surface_capture_notes`, and `missing_surface_fields`.
82
-
83
- API steps should capture endpoint, method, version, status, request/response refs, resource IDs, auth mode without secrets, pagination, rate limits, retries, and errors.
84
-
85
- MCP steps should capture server, capability type, tool/resource/prompt name, argument refs, result refs, schemas, call IDs, and errors.
86
-
87
- Database steps should capture database type, schema/table, query type, query ref or hash, row counts, affected rows, transaction IDs, snapshots, explain plans, rollback refs, and errors.
88
-
89
- Spreadsheet and table steps should capture workbook refs, sheet names, ranges, row/column counts, formula refs, pivots, charts, validation refs, before/after refs, and diffs.
90
-
91
- CLI, file, and codebase steps should capture command, cwd ref, exit code, stdout/stderr refs, file operations, paths, diffs, patches, git state, test/build/lint refs, process IDs, and timeouts.
92
-
93
- Search steps should capture query, provider, scope, filters, result count, selected result refs, citation refs, ranking signals, time range, and errors.
94
-
95
- Browser, computer-use, and software steps should capture UI surface, app name/version, OS, device, window/page title, URL or resource ID, visible-state refs, screenshot refs, DOM/accessibility refs, selectors, coordinates, action type, input text, before/after refs, network/API refs, and software state refs.
96
-
97
- Cloud, SaaS, and enterprise app steps should capture service, workspace/project/tenant identifiers when safe, resource or ticket IDs, object state refs, permission context, region, environment, deployment IDs, audit logs, and change-request refs.
98
-
99
- Async, queue, webhook, and scheduler steps should capture job/message/webhook/schedule IDs, trigger type, enqueue/start/completion times, retry count, delivery/polling/callback status, result refs, and dead-letter refs.
100
-
101
- Messaging, email, and collaboration steps should capture platform, channel/thread/message IDs, sender/recipient roles, message refs, attachment refs, decisions, approval state, and follow-up requirements.
102
-
103
- Document, PDF, report, and extraction steps should capture document refs, document type, page/section refs, text/OCR/table/figure/citation/annotation refs, summaries, and document diffs.
104
-
105
- Device, industrial, and physical workflow steps should capture physical environment type, device details when safe, sensor/telemetry refs, actuator command summaries, control mode, before/after physical state refs, calibration refs, safety state, safety limits, operator override, human supervision, simulation or real-world status, latency, tolerance, rollback action, emergency stop, and compliance refs.
106
-
107
- ## Multi-Agent And Multimodal Support
108
-
109
- Multi-agent workflows should preserve `actor_kind`, `actor_role`, `agent_id`, `agent_name`, `subagent_id`, `subagent_name`, `subagent_role`, `parent_agent_id`, `delegation_id`, `delegation_source_step`, `handoff_reason`, `handoff_payload_ref`, `handoff_output_refs`, `subagent_output_refs`, `coordination_pattern`, `communication_channel`, and `depends_on_agent_outputs`.
110
-
111
- Multimodal workflows should preserve `modality`, `modalities`, `media_type`, `mime_type`, `artifact_role`, `artifact_hash`, `artifact_preview_ref`, `ocr_text_ref`, `transcript_ref`, `annotation_ref`, `bbox_refs`, `frame_refs`, `timecode_refs`, `table_refs`, `sheet_refs`, and `derived_text_summary`.
112
-
113
- ## Completeness Warnings
114
-
115
- New traces should record warnings, not fabricate missing values:
116
-
117
- - `surface_capture_status = complete | partial | unavailable | not_applicable`
118
- - `surface_capture_notes`
119
- - `missing_surface_fields`
120
- - `trace_quality_warnings`
121
-
122
- Old seed traces remain compatible when these fields are absent.
123
-
124
- ## Loop Review Packets
125
-
126
- Each new iterative workflow loop should create review artifacts after an Apprentice Agent iteration and before model or expert review. The package-level structure is:
127
-
128
- ```text
129
- loops/
130
- loop_manifest.json
131
- iterations/
132
- 001/
133
- review_packet.md
134
- review_packet.json
135
- output_snapshot.json
136
- artifact_index.json
137
- trace_refs.json
138
- rubric_status.json
139
- verifier_status.json
140
- evaluator_feedback.json
141
- reviewer_feedback.json
142
- revision_plan.json
143
- diff_report.json
144
- loop_decision.json
145
- ```
146
-
147
- `review_packet.json` is the current iteration dossier. It should include `loop_id`, `loop_iteration`, mode, task/package/run ids, the current goal, output refs, `actual_outputs_ref`, artifact refs and previews, trace refs, key trace steps, rubric and verifier status, evaluator status, known failures, missing outputs, open questions, surface/environment summaries, previous iteration refs, diff from the previous iteration, reviewer instructions, recommended review focus, stop/continue options, and `created_at`.
148
-
149
- Autonomous model review should use the review packet as the review source before judging the attempt. Expert-Led review should show or link the packet before collecting expert feedback. Review feedback records should say whether the reviewer was `mentor_model_provider`, `human_expert`, `org_custom_reviewer`, or `simulated_for_test`; simulated or auto-test checkpoints must not be presented as real human review.
150
-
151
- Revision plans should cite feedback refs, list issues to fix, artifacts and outputs to update, verification steps, stop conditions, and creation time. The next iteration should record feedback refs used, changes made, changed artifacts/outputs, and whether feedback was applied. Record-only follow-ups may create a follow-up review update without inventing new execution data.
152
-
153
- ## Experience Compiler Use
154
-
155
- The Experience Compiler runs after completed task work and follow-ups. It derives reusable runtime learning and training-time data from package evidence: task packets, rubrics, traces, `actual_outputs.json`, artifacts, attempt manifests, evaluator feedback, grader results, verifier results, revision plans, process supervision, reward modeling, revision preference pairs, lessons, follow-up records, mentor checkpoints, loop review packets, reviewer feedback, loop decisions, and diff reports.
156
-
157
- The compiler emits:
158
-
159
- - Experience Pack material
160
- - Skill Pack material, including `skill_pack/skill.md`
161
- - Runtime Learning Package material
162
- - Training-Time Package material
163
- - eval, verifier, reward, and preference rows
164
- - observed environment-response rows
165
- - surface event rows
166
- - `environment_spec.json`
167
- - reports and transfer-readiness records
168
-
169
- It does not claim model training happened. It produces training-ready and runtime-ready material grounded in observed workflow evidence.
170
-
171
- ## World-Model-Ready Fields
172
-
173
- World-model-ready fields include `environment_domain`, `interaction_surface`, `state_before`, `agent_action`, `observed_environment_response`, `state_after`, `turn_idx`, `total_turns`, and environment-response quality scores.
174
-
175
- ## World-Model-Ready Observed Rows
176
-
177
- Observed environment-response rows should include state/action/next-state style data:
178
-
179
- - `environment_domain`
180
- - `interaction_surface`
181
- - `state_before`
182
- - `agent_action`
183
- - `observed_environment_response`
184
- - `state_after`
185
- - `turn_idx`, `total_turns`
186
- - `format_score`, `factuality_score`, `consistency_score`, `realism_score`, `training_quality_score`, `environment_response_quality_score`
187
-
188
- These rows are saved as `environment_response_pairs.jsonl` in Experience Compiler outputs. `environment_spec.json` describes replay and environment readiness. No simulated environment responses are part of the product flow. There is no Qwen-AgentWorld runtime or simulator config in the product flow, and traces must not emit `simulated_environment_response`.
189
-
190
- ## Not Included
191
-
192
- Do not add commerce metadata, public ranking language, or Apprentice Agent self-eval as outcome truth. Do not include secrets, API keys, auth headers, cookies, private local paths, `.env`, `.env.local`, source scrape internals, or huge binaries in JSON.
@@ -1,63 +0,0 @@
1
- # Agent Apprenticeship v0.2.1 Technical Release Notes
2
-
3
- Agent Apprenticeship v0.2.1 includes the v0.2 Experience Compilation system plus a small npm-package polish patch for the public README intro, first-run npm runtime setup, and installed Runtime Training metadata.
4
-
5
- ## Core Product Changes
6
-
7
- - Full Experience Compilation packages now include traces, attempts, artifacts, actual outputs, loop review packets, evaluation/verifier/grader records, Runtime Training, training-time packages, eval packages, environment/replay packages, manifests, and source evidence maps where available.
8
- - Public Ecosystem mode defaults to automatic upload of the full Experience Compilation after completed tasks.
9
- - Private Internal Only mode records a no-upload path and does not upload completed tasks.
10
- - Ecosystem consumption is centered on two actions: Export Full Experience Compilation and Install Runtime Training.
11
- - Runtime Training install/use records installed skill refs, selected Runtime Training refs, confidence metadata, injected sections, and compact contract-focused context.
12
- - `apprentice bundle check` accepts local Experience Compilation folders that use task/package/compiler manifests, including regenerated seed task packages.
13
- - v0.2.1 adds a first-run npm runtime setup lock so concurrent CLI invocations share one Python runtime bootstrap safely.
14
- - v0.2.1 records `package_version` and `runtime_training_confidence` in installed Runtime Training metadata.
15
-
16
- ## Experience Compiler And Training Packages
17
-
18
- - Experience Compiler outputs now include stronger package metadata, schema versions, compiler versions, package versions from `get_package_version()`, row counts, public/private mode, and provenance fields.
19
- - Generated package material is no longer treated as only prompt/completion data. The package supports SFT and LoRA-ready instruction rows, process supervision, reward model rows, verifier model rows, preference pairs, critique/revision pairs, failure cases, task variants, eval/rubric rows, environment/replay rows, and agentic rollout material.
20
- - Source evidence maps now include stable evidence IDs, evidence types, visibility, confidence, content hashes where available, row evidence links, and validation for broken refs.
21
- - Eval and environment packages now have first-class manifests and replay/scoring-oriented metadata.
22
-
23
- ## Demo B Local Proof
24
-
25
- Demo B is a local ALE workflow-family mirrored transfer proof, not an official ALE benchmark score.
26
-
27
- - Completed workflow families: 7
28
- - Improved: 7
29
- - Same: 0
30
- - Worse: 0
31
- - Severe regressions: 0
32
- - Average baseline score: 0.3429
33
- - Average treatment score: 0.8929
34
- - Average delta: +0.5500
35
- - Median delta: +0.4500
36
- - Leakage check: passed
37
-
38
- The supported claim is that Agent Apprenticeship Runtime Training from source Experience Compilations improved related held-out mirrored workflow-family tasks in this local proof set.
39
-
40
- ## Seed Dataset v0.2
41
-
42
- Seed Dataset v0.2 is an Experience Compilation dataset generated from seed tasks. It is not the specialized bioactive/wellness model-training dataset.
43
-
44
- - Full Experience Compilations: 505
45
- - Skill packs: 505
46
- - Runtime Training packages: 505
47
- - Training-time packages: 505
48
- - Eval packages: 505
49
- - Environment/replay packages: 505
50
- - Source evidence maps: 505
51
- - Source evidence rows: 6,483
52
- - Row evidence links: 15,004
53
- - Viewer/export configs: 15
54
- - Normalized Data Viewer rows: 39,835
55
-
56
- Use the wording “505 Experience Compilations plus 39,835 normalized Data Viewer rows.” Do not describe the normalized viewer rows as tasks.
57
-
58
- ## Release Boundary
59
-
60
- - The package version is 0.2.1 for this npm patch release candidate.
61
- - The regenerated Seed Dataset v0.2 records the local package/compiler metadata used for its generation and later metadata restamp.
62
- - No model weights are included.
63
- - No official ALE benchmark score is claimed for the mirrored transfer tasks.
@@ -1,20 +0,0 @@
1
- from __future__ import annotations
2
- import json, hashlib
3
- from pathlib import Path
4
- from pydantic import BaseModel
5
-
6
- def write_json(path: Path, obj):
7
- path.parent.mkdir(parents=True, exist_ok=True)
8
- data = obj.model_dump(mode='json') if isinstance(obj, BaseModel) else obj
9
- path.write_text(json.dumps(data, indent=2, sort_keys=True)+"\n")
10
-
11
- def read_json(path: Path): return json.loads(path.read_text())
12
- def append_jsonl(path: Path, obj):
13
- path.parent.mkdir(parents=True, exist_ok=True)
14
- data = obj.model_dump(mode='json') if isinstance(obj, BaseModel) else obj
15
- with path.open('a') as f: f.write(json.dumps(data, sort_keys=True)+"\n")
16
- def read_jsonl(path: Path):
17
- if not path.exists(): return []
18
- return [json.loads(l) for l in path.read_text().splitlines() if l.strip()]
19
- def sha256_file(path: Path):
20
- h=hashlib.sha256(); h.update(path.read_bytes()); return h.hexdigest()