dsh-compound-engineering 0.0.0-stage → 0.1.1

package/skills/ce-optimize/references/experiment-log-schema.yaml

@@ -0,0 +1,563 @@
+# Experiment Log Schema
+# This is the canonical schema for the experiment log file that accumulates
+# across an optimization run.
+#
+# Location: <state-root>/experiment-log.yaml, where <state-root> is the durable
+# state root resolved in Phase 0 (references/persistence.md): the harness's
+# durable store when it names one, otherwise
+# .context/compound-engineering/ce-optimize/<spec-name>/
+#
+# PERSISTENCE MODEL:
+# The experiment log on disk is the SINGLE SOURCE OF TRUTH. The agent's
+# in-memory context is expendable and will be compacted during long runs.
+#
+# Write discipline:
+# - Each experiment gets one log entry, appended on its first measurement
+#   (SKILL.md step 3.3), before batch evaluation
+# - Later ladder samples for that experiment update the same entry in place
+# - Outcome fields may also be updated in-place after batch evaluation (step 3.5)
+# - The `best` section is updated after each batch if a new best is found
+# - The `hypothesis_backlog` is updated after each batch
+# - `approval` is written once, the moment the Phase 1 gate is cleared
+# - `run_state` is written at CP-4 and whenever a pending wait is registered
+#   or cleared
+# - The agent re-reads this file from disk at every phase boundary
+#
+# The orchestrator does NOT read the full log each iteration -- it uses a
+# rolling window (last 10 experiments) + a strategy digest file for
+# hypothesis generation. But the full log exists on disk for resume,
+# crash recovery, and post-run analysis.
+
+# ============================================================================
+# TOP-LEVEL STRUCTURE
+# ============================================================================
+
+structure:
+
+  spec:
+    type: string
+    required: true
+    description: "Name of the optimization spec this log belongs to"
+
+  run_id:
+    type: string
+    required: true
+    description: "Unique identifier for this optimization run (timestamp-based). Distinguishes resumed runs from fresh starts."
+
+  started_at:
+    type: string
+    format: "ISO 8601 timestamp"
+    required: true
+
+  baseline:
+    type: object
+    required: true
+    description: "Metrics measured on the original code before any optimization"
+    children:
+      timestamp:
+        type: string
+        format: "ISO 8601 timestamp"
+      gates:
+        type: object
+        description: "Key-value pairs of gate metric names to their baseline values"
+      metrics:
+        type: object
+        description: "Required hard-objective snapshots (aggregate plus samples) that decide.mjs loads for later comparisons"
+      diagnostics:
+        type: object
+        description: "Key-value pairs of diagnostic metric names to their baseline values"
+      worst_cases:
+        type: array
+        description: "The few worst per-case results (items, tests, queries) with what went wrong, from the measurement output or judge reasons. Explains the score; never changes it"
+        items:
+          type: object
+          children:
+            case_id:
+              type: string
+            score:
+              type: string
+            reason:
+              type: string
+      judge:
+        type: object
+        description: "Judge scores on the baseline (only when primary type is 'judge')"
+        children:
+          # All fields from the scoring config appear here
+          # Plus:
+          sample_seed:
+            type: integer
+          judge_cost_usd:
+            type: number
+
+  harness_validation:
+    type: object
+    required: false
+    description: "Result of the Phase 1 metric validity gate, written at CP-1 before the baseline. Absent in legacy logs means the gate was not run, not that it passed."
+    children:
+      timestamp:
+        type: string
+        format: "ISO 8601 timestamp"
+      separates_known_good_from_bad:
+        type: boolean
+        description: "The harness scored every known-good exemplar above every known-bad one in the metric's direction; null when no exemplars existed and the probe was not run"
+      rejects_trivial_shortcut:
+        type: boolean
+        description: "An empty, constant, or copied output did not score at or above the known-good exemplars"
+      judge_agreement:
+        type: number
+        description: "Fraction of calibration labels the judge agreed with (judge primary only; null when waived or not applicable)"
+      calibration:
+        type: enum
+        values: [passed, waived, not_run, not_applicable]
+      waiver:
+        type: string
+        description: "Who waived calibration and when, verbatim from the user's message (only when calibration is waived)"
+      holdout_configured:
+        type: boolean
+        description: "Whether a held-out confirmation set was configured for this run"
+      probes:
+        type: array
+        description: "One record per probe run: name, inputs used, observed values, and pass/fail"
+
+  experiments:
+    type: array
+    required: true
+    description: "Ordered list of all experiments, including kept, reverted, errored, and deferred"
+    items:
+      type: object
+      # See EXPERIMENT ENTRY below
+
+  best:
+    type: object
+    required: true
+    description: "Summary of the current best result"
+    children:
+      iteration:
+        type: integer
+        description: "Iteration number of the best experiment (use 0 for the baseline snapshot before any experiment is kept)"
+      metrics:
+        type: object
+        description: "All metric values from the current best state (seed with baseline metrics during CP-1)"
+      judge:
+        type: object
+        description: "Judge scores from the best experiment (only when primary type is 'judge')"
+      total_judge_cost_usd:
+        type: number
+        description: "Running total of all judge costs across all experiments"
+      total_run_cost_usd:
+        type: number
+        description: "Running estimate of whole-run model spend: experiment workers, judges, and research subagents"
+      run_cost_basis:
+        type: string
+        description: "Which parts of total_run_cost_usd come from host-reported usage and which are estimates"
+
+  approval:
+    type: object
+    required: false
+    description: >
+      The Phase 1 user approval, recorded so a resume (including an unattended
+      wake) does not re-ask it. Valid while spec_sha256 matches the SHA-256 of
+      the spec.yaml bytes on disk and every cap below matches the spec. Absent,
+      or no longer matching, means the gate is presented again. Missing in
+      legacy logs means the gate is presented again on resume, as before.
+    children:
+      approved_at:
+        type: string
+        format: "ISO 8601 timestamp"
+      spec_sha256:
+        type: string
+        description: "SHA-256 hex digest of the saved spec.yaml bytes at approval time"
+      caps:
+        type: object
+        description: "The limits the user approved: stopping.* as in force, metric.judge.max_total_cost_usd (null only with the explicit uncapped-spend yes), execution.max_concurrent"
+
+  run_state:
+    type: object
+    required: false
+    description: >
+      Tick-level loop state for resume across turns. Written at CP-4 and
+      whenever a pending wait is registered or cleared. Absent in a log that
+      has not reached Phase 3 or in a legacy log; treat absent clocks as zero
+      and absent pending_waits as none.
+    children:
+      status:
+        type: string
+        enum: [running, waiting, blocked, final]
+        description: "running inside a tick; waiting on recorded pending waits or on the user's wrap-up choice; blocked on something outside the loop (report it, do not spin); final after CP-5"
+      tick:
+        type: integer
+        description: "Completed ticks (batches) so far"
+      phase3_started_at:
+        type: string
+        format: "ISO 8601 timestamp"
+        description: "Wall-clock anchor for stopping.max_wall_hours"
+      active_seconds:
+        type: number
+        description: "Time spent inside ticks since Phase 3 started, summed across ticks; the clock stopping.max_hours reads"
+      pending_waits:
+        type: array
+        description: "Work outstanding across the turn boundary. Empty when nothing is outstanding. Turn-end conditions, including the deadline timer and manual-resume handback, are defined in persistence.md (The Wait Record and Ticks)."
+        items:
+          type: object
+          children:
+            item:
+              type: string
+              description: "What is outstanding: an experiment iteration, a judge batch, a timer"
+            arrives_as:
+              type: string
+              description: "The observable the wake delivers: a store file, a pushed ref, a host message naming the launch, a timer fire"
+            wake:
+              type: string
+              description: "The registered wake identifier or description, or unavailable for a manual-resume checkpoint"
+            on_arrival:
+              type: string
+              description: "The action to take: collect through 3.3, evaluate, re-check stopping"
+
+  hypothesis_backlog:
+    type: array
+    description: "Remaining hypotheses not yet tested"
+    items:
+      type: object
+      children:
+        description:
+          type: string
+        category:
+          type: string
+        priority:
+          type: string
+          enum: [high, medium, low]
+        dep_status:
+          type: string
+          enum: [approved, needs_approval, not_applicable]
+        required_deps:
+          type: array
+          items:
+            type: string
+        opportunity:
+          type: object
+          description: "Pre-implementation opportunity record; see opportunity_record below. Optional for legacy logs."
+
+opportunity_record:
+  children:
+    workload:
+      type: string
+      description: "Representative workload and input identity"
+    baseline:
+      type: string
+      description: "Revision or recorded snapshot against which the benefit is estimated"
+    evidence:
+      type: string
+      description: "Source location and observed cost with units or workload share; rubric evidence for qualitative work"
+    expected_benefit:
+      type: string
+      description: "Target metric, units, expected reduction/increase range or upper bound, and assumptions; unknown with the missing evidence and cheapest resolving measurement when not estimable"
+    confidence:
+      type: string
+      description: "Confidence and the evidence or uncertainty that justifies it"
+    cost_and_risk:
+      type: string
+      description: "Estimated implementation and measurement effort, behavioral risk, and required correctness checks"
+
+# ============================================================================
+# EXPERIMENT ENTRY
+# ============================================================================
+
+experiment_entry:
+  required_children:
+
+    iteration:
+      type: integer
+      description: "Sequential experiment number (1-indexed, monotonically increasing)"
+
+    batch:
+      type: integer
+      description: "Batch number this experiment was part of. Multiple experiments in the same batch ran in parallel."
+
+    hypothesis:
+      type: string
+      description: "Human-readable description of what this experiment tried"
+
+    category:
+      type: string
+      description: "Category for grouping and diversity selection (e.g., signal-extraction, graph-signals, embedding, algorithm, preprocessing)"
+
+    outcome:
+      type: enum
+      values:
+        - measured                # measurement finished and metrics were persisted, awaiting batch evaluation / integration
+        - promising               # eligible on current samples but the ladder still needs confirmation before keep
+        - kept                    # eligible, confirmed, and integrated onto the optimization branch
+        - not_selected            # eligible after comparison but not integrated (not the winner, overlapping, or past the runner-up cap)
+        - reverted                # compared and not eligible (regressed a required objective, or none improved), or eligible but its gain did not come from doing the work the metric stands for (loop 3.4)
+        - inconclusive            # delta inside the comparison threshold; not a keep and not a demonstrated regression
+        - censored                # aborted as noncompetitive under the predeclared futility bound
+        - degenerate              # degenerate gate or smoke test failed -> immediately reverted, no judge evaluation
+        - error                   # measurement command crashed, timed out, or produced malformed output
+        - deferred_needs_approval # experiment needs an unapproved dependency -> set aside for batch approval
+        - timeout                 # measurement command exceeded timeout_seconds
+        - runner_up_kept          # file-disjoint runner-up that was cherry-picked and re-measured successfully
+        - runner_up_reverted      # file-disjoint runner-up that was cherry-picked but combined measurement was not better
+      description: >
+        The loop branches on this value.
+        'measured' and 'promising' are non-terminal: CP-3 persists raw metrics
+        before batch-level comparison, and 'promising' means the ladder still
+        needs confirmation samples. An eligible decide `keep` stays `measured`
+        until its diff is on the optimization branch. 'kept' and 'runner_up_kept'
+        mean that integration happened. 'not_selected' is terminal for an
+        eligible candidate that was not integrated. 'deferred_needs_approval'
+        items are re-presented at wrap-up. All other states are terminal for
+        that experiment.
+
+  optional_children:
+
+    opportunity:
+      type: object
+      description: "Copy of opportunity_record made before this experiment's implementation; never reconstructed from its result. Missing in legacy logs means unrecorded."
+
+    comparisons:
+      type: array
+      description: "One record per distinct reference/candidate/workload pairing used in a decision. Later in-place updates must not replace a previously persisted distinct pairing. Absent legacy evidence is unknown, not inferred from the current best."
+      items:
+        type: object
+        children:
+          kind:
+            type: string
+            enum: [standalone, integrated, holdout]
+            description: "holdout pairings are scored on the held-out set before a keep and at final confirmation; they never feed selection or hypothesis generation"
+          reference_revision:
+            type: string
+            description: "Identity that uniquely identifies the measured reference bytes. A revision is enough when it names those bytes; a HEAD shared by different uncommitted trees is not."
+          candidate_revision:
+            type: string
+            description: "Identity that uniquely identifies the measured candidate bytes. A revision is enough when it names those bytes; a HEAD shared by different uncommitted trees is not."
+          workload:
+            type: string
+          machine:
+            type: string
+            description: "Stable identifier of the machine or worker that produced both sides of this pairing, on every kind including holdout. Present for remote-backend results; absent means this checkout. Pairings from different machines are never compared to each other's absolute numbers."
+          measured_by:
+            type: string
+            enum: [orchestrator, worker, confirmation_worker]
+            description: "Who produced this pairing, on every kind including holdout. A remote winner needs one worker pairing and one pairing not produced by the candidate's author before keep; a remote holdout pairing is always orchestrator or confirmation_worker, never worker."
+          reference:
+            type: object
+            description: "Existing snapshot shape: metrics and judge, with aggregates and samples where available"
+          candidate:
+            type: object
+            description: "Existing snapshot shape: metrics and judge, with aggregates and samples where available"
+          uncertainty:
+            type: string
+            description: "Configured comparison method, observed variability, confirmation status, and any missing uncertainty evidence"
+          correctness:
+            type: string
+            description: "Checks performed and results or exact unverified constraints; passing a timing comparison alone is not correctness evidence"
+
+    changes:
+      type: array
+      description: "Files modified by this experiment"
+      items:
+        type: object
+        children:
+          file:
+            type: string
+          summary:
+            type: string
+
+    gates:
+      type: object
+      description: "Gate metric values from the measurement command"
+
+    gates_passed:
+      type: boolean
+      description: "Whether all degenerate gates passed"
+
+    diagnostics:
+      type: object
+      description: "Diagnostic metric values from the measurement command"
+
+    metrics:
+      type: object
+      description: "Required hard-objective snapshots (aggregate plus samples) that decide.mjs loads"
+
+    judge:
+      type: object
+      description: "Judge evaluation scores (only when primary type is 'judge' and gates passed)"
+      children:
+        # All fields from scoring.primary and scoring.secondary appear here
+        # Plus:
+        judge_cost_usd:
+          type: number
+          description: "Cost of judge calls for this experiment"
+        items:
+          type: array
+          description: "Per-item judge results. Feedback is what hypothesis generation reads; scores alone cannot say what to change."
+          items:
+            type: object
+            children:
+              item_id:
+                type: string
+              content_hash:
+                type: string
+                description: "Hash of the judged item bytes; the key for the run's judge cache, so identical output is not re-judged"
+              reason:
+                type: string
+                description: "The score explanation retained in worst_cases and cached with the judge result"
+              feedback:
+                type: string
+                description: "What is wrong with this item and what change would fix it, from the judge"
+              ambiguous:
+                type: boolean
+              # Plus every scoring field the rubric requests
+
+    cost:
+      type: object
+      description: "Per-experiment diagnostics recorded when the harness or judge reports them; absent otherwise. Never gated on."
+      children:
+        usd:
+          type: number
+        tokens:
+          type: integer
+        latency_seconds:
+          type: number
+
+    regressions:
+      type: array
+      description: "Case ids the reference passed and this candidate fails, from decide.mjs when measurement.per_case is true. Report only."
+      items:
+        type: string
+
+    worst_cases:
+      type: array
+      description: "Same shape as baseline.worst_cases, for this experiment's latest measurement"
+
+    primary_delta:
+      type: string
+      description: "Change in primary metric from current best (e.g., '+0.7', '-0.3')"
+
+    objective_results:
+      type: object
+      description: "Per-required-objective comparison from decide.mjs (verdict, delta, relative)"
+
+    sample_count:
+      type: integer
+      description: "How many harness samples were spent on this experiment"
+
+    next_measurement:
+      type: string
+      enum: [none, smoke, exploratory, add_sample, confirm, holdout]
+      description: "Ladder next step from decide.mjs; none when the decision is terminal. holdout means the selection comparison would keep, and the held-out comparison is still missing."
+
+    learnings:
+      type: string
+      description: "What was learned from this experiment. The agent reads these to avoid re-trying similar approaches and to inform new hypothesis generation. For a revert whose gain did not come from doing the work, this names the observation or constraint the harness lacks."
+
+    commit:
+      type: string
+      description: "Git commit SHA on the optimization branch (only for 'kept' and 'runner_up_kept' outcomes)"
+
+    deferred_reason:
+      type: string
+      description: "Why this experiment was deferred (only for 'deferred_needs_approval' outcome)"
+
+    error_message:
+      type: string
+      description: "Error details (only for 'error' and 'timeout' outcomes)"
+
+    merged_with:
+      type: integer
+      description: "Iteration number of the experiment this was merged with (only for 'runner_up_kept' and 'runner_up_reverted')"
+
+# ============================================================================
+# RESULT MARKER (result.yaml)
+# ============================================================================
+#
+# Written by the measuring party immediately after measurement, before CP-3.
+# Local backends write it at the experiment worktree root with metrics only.
+# A remote worker writes it at its checkout root, commits it on the result ref,
+# and the orchestrator copies it to <state-root>/exp-<NNN>-result.yaml.
+
+result_marker:
+  children:
+    experiment:
+      type: integer
+    base_sha:
+      type: string
+      description: "Commit the worker started from and measured as baseline; must equal the dispatched base_sha or the result is an error"
+    head_sha:
+      type: string
+      description: "The candidate commit (mutable-scope changes only); result.yaml is committed alone on top of it, so the result ref's tip has head_sha as its parent (remote only)"
+    machine:
+      type: string
+      description: "Stable identifier of the measuring machine or worker (remote only)"
+    measured_at:
+      type: string
+      format: "ISO 8601 timestamp"
+    baseline:
+      type: object
+      description: "Paired snapshot of base_sha on the same machine: gates, metrics { name: { aggregate, samples } }, diagnostics (remote only)"
+    candidate:
+      type: object
+      description: "Snapshot of head_sha; local backends write only this side's raw metrics"
+    correctness:
+      type: string
+      description: "Checks the measuring party ran and their results, or none"
+
+# ============================================================================
+# OUTCOME STATE TRANSITIONS
+# ============================================================================
+#
+# proposed (in hypothesis_backlog)
+#   -> selected for batch
+#     -> experiment dispatched
+#       -> measurement completed
+#         -> gates failed           -> outcome: degenerate
+#         -> measurement error      -> outcome: error
+#         -> measurement timeout    -> outcome: timeout
+#         -> smoke failed           -> outcome: degenerate
+#         -> futile / censored      -> outcome: censored
+#         -> gates passed
+#           -> persist raw metrics   -> outcome: measured or promising
+#           -> judge evaluated (if type: judge)
+#             -> decide.mjs eligible, next_measurement none -> stay measured until integration
+#               -> diff on optimization branch -> outcome: kept
+#               -> eligible leftover             -> outcome: not_selected
+#             -> runner-up, file-disjoint -> cherry-pick + re-measure
+#               -> combined eligible and integrated -> outcome: runner_up_kept
+#               -> combined not kept              -> outcome: runner_up_reverted
+#             -> inconclusive             -> outcome: inconclusive
+#             -> not eligible             -> outcome: reverted
+#       -> needs unapproved dep    -> outcome: deferred_needs_approval
+#
+# Only 'kept' and 'runner_up_kept' produce a commit on the optimization branch.
+# Only 'deferred_needs_approval' items are re-presented at wrap-up for approval.
+
+# ============================================================================
+# STRATEGY DIGEST (separate file)
+# ============================================================================
+#
+# Written after each batch to:
+#   <state-root>/strategy-digest.md
+#
+# Contains a compressed summary of:
+# - What hypothesis categories have been tried
+# - Which approaches succeeded (kept) and which failed (reverted)
+# - The exploration frontier: what hasn't been tried yet
+# - Key learnings that should inform next hypotheses
+# - Failure themes: recurring judge `feedback` (or failing `cases`) on the
+#   current best, grouped, with counts. Built from selection-sample items
+#   only; holdout items never appear here.
+#
+# The orchestrator reads the strategy digest (not the full experiment log)
+# when generating new hypotheses between batches.
+
+# ============================================================================
+# JUDGE CACHE (separate file)
+# ============================================================================
+#
+# Written as judge results land, to:
+#   <state-root>/judge-cache.yaml
+#
+# Maps content_hash -> the judge's item result (scores, reason, feedback, ambiguous).
+# Scoped to one run and one rubric: identical item bytes are scored once and
+# reused, so a long run does not pay to re-judge output that did not change.
+# A rubric change is a new run and a new cache.