agent-apprenticeship 0.2.0 → 0.3.0

package/schemas/trace_step.schema.json

@@ -0,0 +1,622 @@
+{
+  "$defs": {
+    "AudioSource": {
+      "additionalProperties": false,
+      "description": "Audio source specification for audio stored as files or at remote URLs.\n\nAdded in ATIF-v1.8. Mirrors ``ImageSource`` so that consumers can treat any\nreferenced media the same way, with one addition: ``duration_sec``. Duration\nis worth recording because several providers bill audio per second and\ndataset statistics need it, and unlike an image's dimensions it cannot be\nrecovered without decoding the file.",
+      "properties": {
+        "media_type": {
+          "description": "MIME type of the audio",
+          "enum": [
+            "audio/wav",
+            "audio/mpeg",
+            "audio/mp4",
+            "audio/aac",
+            "audio/ogg",
+            "audio/flac",
+            "audio/webm",
+            "audio/aiff"
+          ],
+          "title": "Media Type",
+          "type": "string"
+        },
+        "path": {
+          "description": "Location of the audio. Can be a relative or absolute file path, or a URL.",
+          "title": "Path",
+          "type": "string"
+        },
+        "duration_sec": {
+          "anyOf": [
+            {
+              "minimum": 0,
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Optional duration of the audio in seconds. Useful for cost accounting and dataset statistics; omit when unknown.",
+          "title": "Duration Sec"
+        }
+      },
+      "required": [
+        "media_type",
+        "path"
+      ],
+      "title": "AudioSource",
+      "type": "object"
+    },
+    "ContentPart": {
+      "additionalProperties": false,
+      "description": "A single content part within a multimodal message.\n\nUsed when a message or observation contains mixed content types (text,\nimages, or audio). For text-only content, a plain string can still be used\ninstead of a ContentPart array.",
+      "properties": {
+        "type": {
+          "description": "The type of content",
+          "enum": [
+            "text",
+            "image",
+            "audio"
+          ],
+          "title": "Type",
+          "type": "string"
+        },
+        "text": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Text content. Required when type='text'.",
+          "title": "Text"
+        },
+        "source": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/ImageSource"
+            },
+            {
+              "$ref": "#/$defs/AudioSource"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Media source (file reference). Required when type='image' or type='audio', and must match that type.",
+          "title": "Source"
+        }
+      },
+      "required": [
+        "type"
+      ],
+      "title": "ContentPart",
+      "type": "object"
+    },
+    "ImageSource": {
+      "additionalProperties": false,
+      "description": "Image source specification for images stored as files or at remote URLs.",
+      "properties": {
+        "media_type": {
+          "description": "MIME type of the image",
+          "enum": [
+            "image/jpeg",
+            "image/png",
+            "image/gif",
+            "image/webp"
+          ],
+          "title": "Media Type",
+          "type": "string"
+        },
+        "path": {
+          "description": "Location of the image. Can be a relative or absolute file path, or a URL.",
+          "title": "Path",
+          "type": "string"
+        }
+      },
+      "required": [
+        "media_type",
+        "path"
+      ],
+      "title": "ImageSource",
+      "type": "object"
+    },
+    "Metrics": {
+      "additionalProperties": false,
+      "description": "LLM operational and confidence data.",
+      "properties": {
+        "prompt_tokens": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Total input tokens including cached and non-cached",
+          "title": "Prompt Tokens"
+        },
+        "completion_tokens": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Total tokens generated by the LLM response",
+          "title": "Completion Tokens"
+        },
+        "cached_tokens": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Subset of prompt_tokens that were cache hits",
+          "title": "Cached Tokens"
+        },
+        "cost_usd": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Monetary cost of the API call in USD",
+          "title": "Cost Usd"
+        },
+        "prompt_token_ids": {
+          "anyOf": [
+            {
+              "items": {
+                "type": "integer"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Token IDs for prompt (input) tokens sent to the LLM, including chat history (if applicable)",
+          "title": "Prompt Token Ids"
+        },
+        "completion_token_ids": {
+          "anyOf": [
+            {
+              "items": {
+                "type": "integer"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Token IDs for completion (response) tokens, enabling RL training without retokenization drift",
+          "title": "Completion Token Ids"
+        },
+        "logprobs": {
+          "anyOf": [
+            {
+              "items": {
+                "type": "number"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Log probability assigned to each generated token",
+          "title": "Logprobs"
+        },
+        "extra": {
+          "anyOf": [
+            {
+              "additionalProperties": true,
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Other metrics",
+          "title": "Extra"
+        }
+      },
+      "title": "Metrics",
+      "type": "object"
+    },
+    "Observation": {
+      "additionalProperties": false,
+      "description": "Environment feedback/result after actions or system events.",
+      "properties": {
+        "results": {
+          "description": "Array of result objects from tool calls or actions",
+          "items": {
+            "$ref": "#/$defs/ObservationResult"
+          },
+          "title": "Results",
+          "type": "array"
+        }
+      },
+      "required": [
+        "results"
+      ],
+      "title": "Observation",
+      "type": "object"
+    },
+    "ObservationResult": {
+      "additionalProperties": false,
+      "description": "A single result within an observation.",
+      "properties": {
+        "source_call_id": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "The `tool_call_id` from the _tool_calls_ array in _StepObject_ that this result corresponds to. If null or omitted, the result comes from an action that doesn't use the standard tool calling format (e.g., agent actions without tool calls or system-initiated operations).",
+          "title": "Source Call Id"
+        },
+        "content": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "items": {
+                "$ref": "#/$defs/ContentPart"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "The output or result from the tool execution. String for text-only content, or array of ContentPart for multimodal content (added in ATIF-v1.6).",
+          "title": "Content"
+        },
+        "subagent_trajectory_ref": {
+          "anyOf": [
+            {
+              "items": {
+                "$ref": "#/$defs/SubagentTrajectoryRef"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Array of references to delegated subagent trajectories",
+          "title": "Subagent Trajectory Ref"
+        },
+        "extra": {
+          "anyOf": [
+            {
+              "additionalProperties": true,
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Custom observation-result-level metadata (e.g., confidence score, retrieval score, source document ID). Added in ATIF-v1.7.",
+          "title": "Extra"
+        }
+      },
+      "title": "ObservationResult",
+      "type": "object"
+    },
+    "SubagentTrajectoryRef": {
+      "additionalProperties": false,
+      "description": "Reference to a delegated subagent trajectory.\n\nA subagent reference is resolved by one of two mechanisms:\n\n1. **Embedded form** \u2014 set `trajectory_id` to match the\n   `Trajectory.trajectory_id` of an entry in the parent's\n   `subagent_trajectories` array.\n2. **File-ref form** \u2014 set `trajectory_path` to the location\n   (file path, S3 URL, etc.) of an external trajectory file.\n\nThese two mechanisms are the only resolution keys. `session_id`, when\npresent on the ref, is **informational only**: it records the run\nidentity of the delegated subagent for debug / correlation / cross-\ntrajectory search purposes, and MUST NOT be used as a matching key\n(it is run-scoped and MAY collide across siblings \u2014 see\n`Trajectory.session_id`). A ref therefore MUST set at least one of\n`trajectory_id` or `trajectory_path`; `session_id` alone is not a\nresolvable reference.",
+      "properties": {
+        "trajectory_id": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Canonical identifier of the delegated subagent trajectory. Matches `Trajectory.trajectory_id` of an entry in the parent's `subagent_trajectories` array and is the resolution key for embedded references. Added in ATIF-v1.7 to provide a document-unique matching key without overloading `session_id`.",
+          "title": "Trajectory Id"
+        },
+        "session_id": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Run identity of the delegated subagent trajectory. Informational only: recorded so consumers can correlate this ref back to the subagent's run for debug / search / display purposes. Run-scoped (see `Trajectory.session_id`) and therefore NOT a valid resolution key \u2014 consumers MUST resolve via `trajectory_id` (embedded) or `trajectory_path` (external file).",
+          "title": "Session Id"
+        },
+        "trajectory_path": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Location of the complete subagent trajectory as an external file (file path, S3 URL, database reference, etc.). Resolution key for file-ref references. When both `trajectory_id` and `trajectory_path` are set, consumers MAY choose either; typically `trajectory_id` is preferred when the embedded trajectory is available in-memory.",
+          "title": "Trajectory Path"
+        },
+        "extra": {
+          "anyOf": [
+            {
+              "additionalProperties": true,
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Custom metadata about the subagent execution",
+          "title": "Extra"
+        }
+      },
+      "title": "SubagentTrajectoryRef",
+      "type": "object"
+    },
+    "ToolCall": {
+      "additionalProperties": false,
+      "description": "A tool call within a step.",
+      "properties": {
+        "tool_call_id": {
+          "description": "Unique identifier for this specific tool call",
+          "title": "Tool Call Id",
+          "type": "string"
+        },
+        "function_name": {
+          "description": "The name of the function or tool being invoked",
+          "title": "Function Name",
+          "type": "string"
+        },
+        "arguments": {
+          "additionalProperties": true,
+          "description": "Arguments passed to the function (can be empty dict)",
+          "title": "Arguments",
+          "type": "object"
+        },
+        "extra": {
+          "anyOf": [
+            {
+              "additionalProperties": true,
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "description": "Custom tool-call-level metadata (e.g., timeout, retry count, tool version). Added in ATIF-v1.7.",
+          "title": "Extra"
+        }
+      },
+      "required": [
+        "tool_call_id",
+        "function_name",
+        "arguments"
+      ],
+      "title": "ToolCall",
+      "type": "object"
+    }
+  },
+  "additionalProperties": false,
+  "description": "A single step in the trajectory.",
+  "properties": {
+    "step_id": {
+      "description": "Ordinal index of the turn (starting from 1)",
+      "minimum": 1,
+      "title": "Step Id",
+      "type": "integer"
+    },
+    "timestamp": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "ISO 8601 timestamp indicating when this step occurred",
+      "title": "Timestamp"
+    },
+    "source": {
+      "description": "The originator of this step",
+      "enum": [
+        "system",
+        "user",
+        "agent"
+      ],
+      "title": "Source",
+      "type": "string"
+    },
+    "model_name": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "The specific LLM model used for this turn. Omission implies the model defined in the root-level agent config.",
+      "title": "Model Name"
+    },
+    "reasoning_effort": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "number"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Qualitative or quantitative measure of effort",
+      "title": "Reasoning Effort"
+    },
+    "message": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "items": {
+            "$ref": "#/$defs/ContentPart"
+          },
+          "type": "array"
+        }
+      ],
+      "description": "The dialogue message. String for text-only content, or array of ContentPart for multimodal content (added in ATIF-v1.6).",
+      "title": "Message"
+    },
+    "reasoning_content": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "The agent's explicit internal reasoning",
+      "title": "Reasoning Content"
+    },
+    "tool_calls": {
+      "anyOf": [
+        {
+          "items": {
+            "$ref": "#/$defs/ToolCall"
+          },
+          "type": "array"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Array of structured objects for the agent's actions",
+      "title": "Tool Calls"
+    },
+    "observation": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/Observation"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Environment feedback/result after actions or system events"
+    },
+    "metrics": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/Metrics"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "LLM operational and confidence data for this step"
+    },
+    "is_copied_context": {
+      "anyOf": [
+        {
+          "type": "boolean"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Indicates whether this step was copied from a previous trajectory for context (e.g., during continuation after summarization). Steps marked as copied context should not be included in training data as they represent previously-trained interactions. Added in ATIF-v1.5.",
+      "title": "Is Copied Context"
+    },
+    "llm_call_count": {
+      "anyOf": [
+        {
+          "minimum": 0,
+          "type": "integer"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Number of LLM inferences this step represents. When >1, metrics are aggregated across multiple LLM calls. When 1, the step represents exactly one inference. When 0 on a `source: 'agent'` step, the step represents a deterministic (non-LLM) dispatch; `metrics` and `reasoning_content` MUST be absent in that case. When null, the producer did not track this (backward-compatible default). Added in ATIF-v1.7.",
+      "title": "Llm Call Count"
+    },
+    "extra": {
+      "anyOf": [
+        {
+          "additionalProperties": true,
+          "type": "object"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Custom step-level metadata",
+      "title": "Extra"
+    }
+  },
+  "required": [
+    "step_id",
+    "source",
+    "message"
+  ],
+  "title": "Step",
+  "type": "object",
+  "$schema": "https://json-schema.org/draft/2020-12/schema"
+}

package/schemas/training_rows.schema.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://forsy.ai/schemas/agent-apprenticeship/training_rows.schema.json",
+  "title": "Training Rows",
+  "type": "object",
+  "required": ["row_id", "row_type", "task_id", "source_evidence_refs"],
+  "properties": {
+    "row_id": { "type": "string", "minLength": 1 },
+    "row_type": {
+      "type": "string",
+      "enum": ["sft", "process_supervision", "reward", "verifier", "preference_pair", "critique_revision", "failure_case", "task_variant", "rollout"]
+    },
+    "task_id": { "type": "string", "minLength": 1 },
+    "domain": { "type": "string" },
+    "subdomain": { "type": "string" },
+    "workflow_family": { "type": "string" },
+    "instruction": { "type": "string" },
+    "input_context": { "type": "string" },
+    "target_output": { "type": "string" },
+    "quality_score": { "type": "number", "minimum": 0, "maximum": 1 },
+    "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+    "source_evidence_refs": {
+      "type": "array",
+      "items": { "type": "string" },
+      "minItems": 1
+    }
+  },
+  "additionalProperties": true
+}

package/src/{agent_apprenticeship_trace → agent_apprenticeship}/__init__.py

@@ -1,2 +1,2 @@
 """Agent Apprenticeship framework."""
-__version__ = "0.2.0"
+__version__ = "0.3.0"