agentera 0.0.0 → 3.0.0-dev.0

package/bundle/references/analysis/startup-measurement-contract.yaml

@@ -0,0 +1,424 @@
+contract: startup_state_access_measurement
+version: startup-state-analysis-v1
+status: draft
+purpose: >-
+  Define the local-only, privacy-preserving contract for measuring Agentera
+  capability startup/state-gathering overhead after v2.3.0. The measured defect
+  is raw Agentera artifact access after Agentera CLI state calls before
+  implementation work begins.
+
+boundary:
+  source: git tag evidence
+  tag: v2.3.0
+  commit: b18e3dc7d768d4ad2726880916e7cfe6bd8617d3
+  committed_at: '2026-05-12T17:50:13+02:00'
+  rule: include records with timestamps strictly after committed_at
+
+section_22_compatibility:
+  consumes_adapter_version: agentera-v2-corpus-1
+  required_record_families:
+    - conversation_turn
+    - tool_call
+  optional_record_families:
+    - instruction_document
+    - history_prompt
+    - project_config_signal
+  base_envelope_changes: none
+  output_envelope: startup_state_analysis_v1
+  extension_rule: >-
+    Do not add required fields to the Section 22 corpus envelope. If startup
+    state analysis needs derived fields, write them under a separate versioned
+    startup_state_analysis_v1 output or an optional
+    analysis_extensions.startup_state_v1 object ignored by extract_corpus.py
+    and usage_stats.py.
+
+benchmark_execution:
+  rule: manual_only
+  command_surface: mage bench:startupState
+  normal_ci: forbidden
+  live_host_checks: forbidden_by_default
+  runtime_path_approval: >-
+    Running mage bench:startupState authorizes inspection of the documented
+    default runtime stores for codex, claude-code, opencode, and github-copilot.
+    Custom runtime history sources must be supplied as exact runtime labels and
+    concrete filesystem paths. A generic consent flag or runtime label without
+    its concrete path is insufficient for custom sources.
+  default_run: >-
+    The Mage benchmark target runs without environment variables by inspecting
+    documented runtime-store defaults, writing aggregate history under the default
+    Agentera benchmark directory, and reporting missing or unavailable stores as
+    bounded degradation evidence.
+  incremental_rule: >-
+    Runtime-store benchmark runs use retained aggregate history as the only
+    durable watermark source. The first run for a runtime scope analyzes all
+    post-boundary records; later runs for the same runtime_scope analyze only
+    records with timestamps strictly after the previous successful
+    benchmark_watermark_at.
+  metric_source_of_truth: scripts/startup_analysis_contract.py
+  startup_state_envelope_scope: excluded
+
+runtime_extraction_contract:
+  supported_runtime_order:
+    - codex
+    - claude-code
+    - opencode
+    - github-copilot
+  normalized_record_fields:
+    required:
+      - source_id
+      - timestamp
+      - project_id
+      - source_kind
+      - runtime
+      - adapter_version
+      - data
+    optional:
+      - project_path
+      - session_id
+    startup_required_semantics:
+      - runtime label
+      - timestamp
+      - conversation key from conversation_key, session_id, data.session_id, or source_id
+      - conversation_turn records with data.actor and data.content
+      - tool_call records with data.tool_name and data.arguments
+  runtime_status_outcomes:
+    schema_divergent:
+      status: degraded
+      reason: schema_divergent
+      rule: >-
+        One or more candidate runtime files or databases were found, but adapter
+        parsing encountered schema errors. This is extraction failure evidence,
+        not a successful zero-record benchmark window.
+      required_counts:
+        - candidate_count when discovery can count candidates
+        - record_count, which may be zero or partial
+        - error_count greater than zero
+      current_examples:
+        - runtime: claude-code
+          candidate_count: 4836
+          record_count: 0
+          error_count: 2
+        - runtime: github-copilot
+          candidate_count: 1
+          record_count: 0
+          error_count: 1
+    no_matching_records:
+      status: sparse
+      reason: no_matching_records
+      rule: >-
+        Candidate runtime storage was readable and matched an accepted store
+        shape, but no supported message or tool records were extracted. This is
+        sparse coverage evidence and remains distinct from schema divergence.
+      required_counts:
+        - candidate_count when discovery can count candidates
+        - "record_count: 0"
+        - no error_count greater than zero
+    successful_zero_record_window:
+      status: ok
+      reason: records_extracted
+      rule: >-
+        Adapter extraction succeeded for the runtime scope, but the benchmark
+        incremental window contains zero records after the previous successful
+        watermark. This is compatible successful behavior, not degradation.
+      required_counts:
+        - "record_count: 0"
+        - "error_count: 0"
+      current_examples:
+        - runtime: codex
+          record_count: 0
+          error_count: 0
+  supported_runtimes:
+    codex:
+      accepted_input_schema_classes:
+        - JSONL session files containing session_meta or turn_context records
+        - JSONL response_item or user_msg message records
+        - JSONL tool_call or function_call records with payload item, name, and arguments variants
+      normalized_record_fields:
+        conversation_turn: [runtime, timestamp, session_id, project_id, data.actor, data.content]
+        tool_call: [runtime, timestamp, session_id, project_id, data.tool_name, data.arguments]
+      status_mapping:
+        records extracted without adapter errors: ok / records_extracted
+        readable candidates but no supported records: sparse / no_matching_records
+        unreadable or locked store: degraded / store_unreadable or store_locked
+        parser/schema errors: degraded / schema_divergent
+      redaction_rules:
+        - Do not retain transcript content from content, text, message, prompt, input_text, or output_text.
+        - Do not retain cwd, project_path, store_path, file_path, raw session ids, or generated salted hashes.
+        - Retain only canonical runtime labels, canonical artifact labels, bounded counts, and aggregate status fields.
+    claude-code:
+      accepted_input_schema_classes:
+        - JSONL project session files with top-level role or type user/assistant events
+        - JSONL message objects with nested message.role plus content/text/message content variants
+        - JSONL tool_call, function_call, or tool_use records with argument/input variants
+      normalized_record_fields:
+        conversation_turn: [runtime, timestamp, session_id, project_id, data.actor, data.content]
+        tool_call: [runtime, timestamp, session_id, project_id, data.tool_name, data.arguments]
+      status_mapping:
+        records extracted without adapter errors: ok / records_extracted
+        readable candidates but no supported records: sparse / no_matching_records
+        unreadable or locked store: degraded / store_unreadable or store_locked
+        parser/schema errors: degraded / schema_divergent
+      redaction_rules:
+        - Do not retain raw project directories encoded in Claude project filenames or cwd fields.
+        - Do not retain transcript content, raw session ids, raw store paths, or generated salted hashes.
+        - Retain only canonical runtime labels, canonical artifact labels, bounded counts, and aggregate status fields.
+    opencode:
+      accepted_input_schema_classes:
+        - SQLite opencode.db with session, message, and part tables
+        - session id columns id, session_id, or sessionID
+        - message/part join, role, timestamp, text/content, and JSON data column variants
+        - tool parts encoded as part type tool or JSON data.tool with state.input arguments
+      normalized_record_fields:
+        conversation_turn: [runtime, timestamp, session_id, project_id, data.actor, data.content]
+        tool_call: [runtime, timestamp, session_id, project_id, data.tool_name, data.arguments]
+      status_mapping:
+        records extracted without adapter errors: ok / records_extracted
+        readable candidates but no supported records: sparse / no_matching_records
+        missing required tables or join columns: degraded / schema_divergent
+        unreadable or locked store: degraded / store_unreadable or store_locked
+      redaction_rules:
+        - Do not retain SQLite file paths, project paths, transcript text, raw session ids, or generated salted hashes.
+        - Retain only canonical runtime labels, canonical artifact labels, bounded counts, and aggregate status fields.
+    github-copilot:
+      accepted_input_schema_classes:
+        - SQLite session-store.db with sessions and turns tables
+        - session id columns id, session_id, or sessionID
+        - turns columns for session linkage, role, text/content/message/prompt/response, timestamp, and ordering variants
+        - turns JSON payload or tool columns containing tool_name/name/tool plus arguments/input/command variants
+      normalized_record_fields:
+        conversation_turn: [runtime, timestamp, session_id, project_id, data.actor, data.content]
+        tool_call: [runtime, timestamp, session_id, project_id, data.tool_name, data.arguments]
+      status_mapping:
+        records extracted without adapter errors: ok / records_extracted
+        readable candidates but no supported records: sparse / no_matching_records
+        missing required tables, join columns, role, or text columns: degraded / schema_divergent
+        unreadable or locked store: degraded / store_unreadable or store_locked
+      redaction_rules:
+        - Do not retain session-store paths, project paths, transcript text, raw session ids, or generated salted hashes.
+        - Retain remediation labels such as /chronicle reindex only when they are static product guidance, not local data.
+        - Retain only canonical runtime labels, canonical artifact labels, bounded counts, and aggregate status fields.
+
+token_impact_contract:
+  estimator_version: approx_bytes_div_4_v1
+  estimator_rule: >-
+    Estimate tokens as ceil(observed UTF-8 content bytes divided by 4). The
+    estimate is an aggregate benchmark signal, not model-tokenizer output.
+  transient_inputs:
+    - observed transcript or tool-argument content byte counts during analysis
+    - canonical artifact label associated with each raw artifact access event
+    - sequence classification for raw-after-CLI and redundant raw access
+  retained_output_rule: >-
+    Retained reports and history may store only aggregate estimated token counts
+    grouped by canonical artifact label and benchmark scope. They must not store
+    raw paths, transcript text, raw tool arguments, salts, or generated hashes.
+  aggregate_fields:
+    latest_report:
+      - token_estimator_version
+      - estimated_raw_after_cli_tokens
+      - estimated_redundant_raw_tokens
+      - estimated_raw_after_cli_tokens_by_artifact
+      - estimated_redundant_raw_tokens_by_artifact
+      - estimated_tokens_saved_vs_previous
+      - estimated_tokens_saved_vs_previous_null_reason
+    aggregate_history:
+      - token_estimator_version
+      - estimated_raw_after_cli_tokens
+      - estimated_redundant_raw_tokens
+      - estimated_raw_after_cli_tokens_by_artifact
+      - estimated_redundant_raw_tokens_by_artifact
+      - estimated_tokens_saved_vs_previous
+      - estimated_tokens_saved_vs_previous_null_reason
+  comparable_row_rules:
+    comparable_when_all_match:
+      - contract_version
+      - benchmark_mode
+      - runtime_scope
+      - token_estimator_version
+      - token aggregate fields present in both rows
+    saved_value: previous estimated_redundant_raw_tokens minus current estimated_redundant_raw_tokens
+    null_when_not_comparable: true
+  null_reasons:
+    - previous_row_missing
+    - previous_missing_token_estimates
+    - estimator_version_mismatch
+    - runtime_scope_mismatch
+    - benchmark_mode_mismatch
+    - contract_version_mismatch
+
+benchmark_storage:
+  default_directory: ${AGENTERA_HOME}/benchmarks/startup-state/
+  repository_outputs: forbidden
+  durable_outputs:
+    - runs.jsonl
+    - latest-report.json
+    - latest-report.md
+  retention_rule: >-
+    Retained benchmark history consists only of aggregate runs.jsonl plus the
+    fixed latest structured and human-readable report files. Runtime corpus files,
+    analyzer intermediates, per-run detailed reports, and any other generated
+    benchmark artifacts are temporary only and must not be retained in the
+    repository or benchmark history directory.
+  success_rule: >-
+    Successful benchmark runs append one aggregate row to runs.jsonl and replace
+    latest-report.json and latest-report.md after report generation succeeds.
+  failure_rule: >-
+    Failed benchmark runs do not append aggregate history and do not clobber the
+    previous latest report files.
+
+aggregate_history:
+  file: runs.jsonl
+  row_shape:
+    - contract_version
+    - generated_at
+    - agentera_version
+    - git_commit
+    - git_dirty
+    - runtime_scope
+    - benchmark_mode
+    - benchmark_previous_watermark_at
+    - benchmark_window_started_after
+    - benchmark_watermark_at
+    - total_records
+    - total_state_sequences
+    - state_sequences_with_raw_after_cli
+    - state_sequences_with_redundant_raw_access
+    - raw_after_cli_rate
+    - redundant_raw_access_rate
+    - cli_state_command_counts
+    - raw_artifact_access_after_cli_counts
+    - redundant_raw_artifact_access_counts
+    - per_capability_state_counts
+    - degradation_reason_counts
+    - bounded_degradation_counts
+    - startup_recommendation_action
+    - token_estimator_version
+    - estimated_raw_after_cli_tokens
+    - estimated_redundant_raw_tokens
+    - estimated_raw_after_cli_tokens_by_artifact
+    - estimated_redundant_raw_tokens_by_artifact
+    - estimated_tokens_saved_vs_previous
+    - estimated_tokens_saved_vs_previous_null_reason
+  forbidden_retained_fields:
+    - raw_transcripts
+    - raw_corpus_files
+    - raw_intermediates
+    - raw_store_paths
+    - raw_session_ids
+    - private_salts
+    - generated_salted_hashes
+    - raw_tool_arguments
+
+state_gathering_sequence:
+  start_anchor: first_agentera_state_call_after_capability_related_user_turn
+  capability_invocation_markers:
+    - slash route such as /agentera plan or /agentera build
+    - bare capability name such as planera, resonera, or orkestrera
+    - natural-language capability mention such as run orkestrera
+    - assistant capability intro marker
+  end_anchors:
+    - first_implementation_boundary
+    - next_user_turn
+  measured_defect: >-
+    Raw read, grep, or glob access to an Agentera artifact after a CLI state call
+    that already covered the same artifact family, before implementation work.
+
+counted_event_classes:
+  - class: cli_state_call
+    description: Agentera state CLI reads such as hej, todo, docs, plan, progress, decisions, health, and query.
+  - class: raw_artifact_access
+    description: Direct read, grep, or glob access to Agentera state artifacts after CLI state access.
+  - class: capability_prose_read
+    description: Reads of capability prose, schemas, protocol, or routing entry files.
+  - class: implementation_boundary
+    description: Code edits, write/edit/apply_patch, non-state bash commands, tests, or analysis work after startup state gathering.
+  - class: non_state_context
+    description: Events outside the measured state-gathering sequence and excluded from defect totals.
+
+degradation_reasons:
+  - pre_boundary_record
+  - missing_timestamp
+  - malformed_record
+  - missing_conversation_key
+  - no_agentera_state_sequence
+  - runtime_store_missing
+  - runtime_store_locked
+  - runtime_store_sparse
+  - runtime_schema_divergent
+  - privacy_redaction_required
+
+privacy_boundary:
+  forbidden_raw_outputs:
+    - transcript_text
+    - raw_corpus_file
+    - raw_intermediate
+    - full_local_path
+    - raw_store_path
+    - unredacted_session_id
+    - private_salt
+    - generated_salted_hash
+  allowed_replacements:
+    - canonical_artifact_label
+    - canonical_runtime_label
+    - canonical_event_class
+    - salted_non_reconstructable_hash
+  canonical_artifact_labels:
+    TODO.md: TODO.md
+    CHANGELOG.md: CHANGELOG.md
+    DESIGN.md: DESIGN.md
+    PLAN.md: PLAN.md
+    PROGRESS.md: PROGRESS.md
+    DOCS.md: DOCS.md
+    DECISIONS.md: DECISIONS.md
+    HEALTH.md: HEALTH.md
+    VISION.md: VISION.md
+    OBJECTIVE.md: OBJECTIVE.md
+    EXPERIMENTS.md: EXPERIMENTS.md
+    AGENTS.md: AGENTS.md
+    SKILL.md: SKILL.md
+    AGENTERA_ARTIFACTS: AGENTERA_ARTIFACTS
+  stdout_rule: >-
+    Startup state analysis stdout may include counts, canonical labels, runtime
+    names, boundary metadata, degradation labels, and salted hashes only.
+  retained_aggregate_history_rule: >-
+    Retained aggregate benchmark history must exclude raw transcripts, raw corpus
+    files, raw intermediates, raw store paths, raw session IDs, private salts, and
+    generated salted hashes. Store only bounded counts, rates, canonical labels,
+    runtime labels, version metadata, git metadata, degradation counts, token
+    estimate aggregates, and the recommendation action.
+  fixture_rule: >-
+    Committed fixtures and snapshots must use synthetic text and identifiers or
+    already-redacted labels; never paste raw transcript excerpts, full local
+    paths, raw store paths, or unredacted session ids.
+
+report_fields:
+  required:
+    - contract_version
+    - boundary_source
+    - boundary_commit
+    - boundary_committed_at
+    - generated_at
+    - corpus_adapter_version
+    - runtime_coverage
+    - total_state_sequences
+    - state_sequences_with_raw_after_cli
+    - state_sequences_with_redundant_raw_access
+    - cli_state_command_counts
+    - raw_artifact_access_after_cli_counts
+    - redundant_raw_artifact_access_counts
+    - per_capability_state_counts
+    - degradation_reason_counts
+    - threshold_derivation
+    - startup_recommendation
+    - privacy_redaction_summary
+    - compatibility_note
+    - token_estimator_version
+    - estimated_raw_after_cli_tokens
+    - estimated_redundant_raw_tokens
+    - estimated_raw_after_cli_tokens_by_artifact
+    - estimated_redundant_raw_tokens_by_artifact
+  optional:
+    - confidence_caveats
+    - insufficient_evidence_reason
+    - recommendation_gate_input
+    - estimated_tokens_saved_vs_previous
+    - estimated_tokens_saved_vs_previous_null_reason