harnex 0.8.0 → 0.9.0

data/lib/harnex/retention.rb

@@ -0,0 +1,311 @@
+require "fileutils"
+require "json"
+require "set"
+require "time"
+
+module Harnex
+  module Retention
+    DIRS = {
+      "events" => "events",
+      "output" => "output"
+    }.freeze
+    THROTTLE_SECONDS = 3600
+    MAX_REPORTED_PATHS = 100
+    METADATA_PATH = File.join(STATE_DIR, "retention.json").freeze
+    LOCK_PATH = File.join(STATE_DIR, "retention.lock").freeze
+
+    module_function
+
+    def status(repo_root:, env: ENV, now: Time.now)
+      limits = Config.retention_limits(repo_root, env: env)
+      {
+        ok: true,
+        checked_at: now.utc.iso8601,
+        config: limits,
+        directories: DIRS.to_h do |key, dirname|
+          files = inventory(File.join(STATE_DIR, dirname))
+          [key, {
+            path: File.join(STATE_DIR, dirname),
+            count: files.length,
+            bytes: files.sum { |file| file.fetch(:bytes) },
+            max_age_days: limits.fetch(key).fetch("max_age_days"),
+            max_bytes: limits.fetch(key).fetch("max_bytes")
+          }]
+        end,
+        last_prune: last_prune_metadata
+      }
+    end
+
+    def auto_prune(repo_root:, current_paths: [], env: ENV, now: Time.now)
+      prune(
+        repo_root: repo_root,
+        current_paths: current_paths,
+        env: env,
+        now: now,
+        dry_run: false,
+        force: false
+      )
+    rescue Config::ConfigError
+      raise
+    rescue StandardError => e
+      warn("harnex: retention prune failed: #{e.message}; run `harnex doctor --prune --dry-run` to inspect candidates")
+      { ok: false, error: e.message }
+    end
+
+    def prune(repo_root:, current_paths: [], env: ENV, now: Time.now, dry_run: false, force: false)
+      limits = Config.retention_limits(repo_root, env: env)
+      with_lock do
+        last = last_prune_metadata
+        if !force && !dry_run && throttled?(last, now)
+          return {
+            ok: true,
+            skipped: true,
+            reason: "throttled",
+            throttle_seconds: THROTTLE_SECONDS,
+            last_prune: last
+          }
+        end
+
+        protected = protected_paths(repo_root, current_paths)
+        report = {
+          ok: true,
+          skipped: false,
+          dry_run: dry_run,
+          timestamp: now.utc.iso8601,
+          directories: DIRS.to_h do |key, dirname|
+            [key.to_sym, prune_directory(
+              File.join(STATE_DIR, dirname),
+              limits.fetch(key),
+              protected,
+              now,
+              dry_run: dry_run
+            )]
+          end
+        }
+        persist_last_prune(report)
+        report
+      end
+    end
+
+    def inventory(dir)
+      FileUtils.mkdir_p(dir)
+      base = File.expand_path(dir)
+      Dir.children(base).sort.filter_map do |name|
+        path = File.expand_path(File.join(base, name))
+        next unless path.start_with?("#{base}#{File::SEPARATOR}")
+
+        stat = File.lstat(path)
+        next unless stat.file?
+
+        { path: path, bytes: stat.size, mtime: stat.mtime }
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR
+        nil
+      end
+    end
+
+    def prune_directory(dir, limits, protected, now, dry_run:)
+      files = inventory(dir)
+      before_bytes = files.sum { |file| file.fetch(:bytes) }
+      protected_files = files.select { |file| protected.include?(file.fetch(:path)) }
+      deleted = []
+      deleted_paths = Set.new
+      projected_bytes = before_bytes
+
+      cutoff = now - limits.fetch("max_age_days") * 86_400
+      age_candidates = files.reject { |file| protected.include?(file.fetch(:path)) }
+        .select { |file| file.fetch(:mtime) < cutoff }
+        .sort_by { |file| [file.fetch(:mtime), file.fetch(:path)] }
+      age_candidates.each do |file|
+        next if deleted_paths.include?(file.fetch(:path))
+
+        if delete_candidate(file, dry_run: dry_run)
+          deleted << file
+          deleted_paths.add(file.fetch(:path))
+          projected_bytes -= file.fetch(:bytes)
+        end
+      end
+
+      size_candidates = files.reject { |file| protected.include?(file.fetch(:path)) || deleted_paths.include?(file.fetch(:path)) }
+        .sort_by { |file| [file.fetch(:mtime), file.fetch(:path)] }
+      size_candidates.each do |file|
+        break if projected_bytes <= limits.fetch("max_bytes")
+
+        if delete_candidate(file, dry_run: dry_run)
+          deleted << file
+          deleted_paths.add(file.fetch(:path))
+          projected_bytes -= file.fetch(:bytes)
+        end
+      end
+
+      after_bytes = dry_run ? projected_bytes : inventory(dir).sum { |file| file.fetch(:bytes) }
+      {
+        path: dir,
+        before_count: files.length,
+        before_bytes: before_bytes,
+        after_count: dry_run ? files.length - deleted.length : inventory(dir).length,
+        after_bytes: after_bytes,
+        deleted_count: deleted.length,
+        deleted_bytes: deleted.sum { |file| file.fetch(:bytes) },
+        deleted_paths: deleted.first(MAX_REPORTED_PATHS).map { |file| file.fetch(:path) },
+        deleted_paths_truncated: deleted.length > MAX_REPORTED_PATHS,
+        protected_count: protected_files.length,
+        protected_bytes: protected_files.sum { |file| file.fetch(:bytes) },
+        max_age_days: limits.fetch("max_age_days"),
+        max_bytes: limits.fetch("max_bytes"),
+        over_cap: after_bytes > limits.fetch("max_bytes")
+      }
+    end
+
+    def delete_candidate(file, dry_run:)
+      return true if dry_run
+
+      File.delete(file.fetch(:path))
+      true
+    rescue Errno::ENOENT
+      true
+    rescue Errno::EACCES, Errno::EPERM, Errno::ENOTDIR
+      false
+    end
+
+    def protected_paths(repo_root, current_paths)
+      paths = Set.new
+      current_paths.each { |path| protect(paths, path) }
+      protect_live_registry_paths(paths)
+      protect_live_start_row_paths(paths, repo_root)
+      paths
+    end
+
+    def protect_live_registry_paths(paths)
+      return unless Dir.exist?(SESSIONS_DIR)
+
+      Dir.glob(File.join(SESSIONS_DIR, "*.json")).sort.each do |path|
+        data = JSON.parse(File.read(path))
+        next unless data["pid"] && Harnex.alive_pid?(data["pid"])
+
+        protect(paths, data["events_log_path"])
+        protect(paths, data["output_log_path"])
+        protect_derived_session_paths(paths, data)
+      rescue JSON::ParserError, ArgumentError, TypeError, Errno::ENOENT, Errno::EACCES
+        next
+      end
+    end
+
+    def protect_live_start_row_paths(paths, repo_root)
+      history_paths = [
+        DispatchHistory.path_for(repo_root),
+        DispatchHistory.global_path
+      ].uniq
+
+      history_paths.each do |path|
+        live_start_records(path).each do |record|
+          protect(paths, record["events_log_path"])
+          protect(paths, record["output_log_path"])
+          protect_derived_session_paths(paths, record)
+        end
+      end
+    end
+
+    def live_start_records(path)
+      return [] unless File.file?(path)
+
+      open = {}
+      File.foreach(path) do |line|
+        record = JSON.parse(line)
+        next unless record.is_a?(Hash)
+
+        if DispatchHistory.start_record?(record)
+          next unless DispatchHistory.same_host?(record)
+          next unless record["pid"] && Harnex.alive_pid?(record["pid"])
+
+          open[start_key(record)] = record
+        elsif DispatchHistory.end_record?(record)
+          open.delete_if { |_key, start| DispatchHistory.end_matches_start?(record, start) }
+        end
+      rescue JSON::ParserError, ArgumentError, TypeError
+        next
+      end
+      open.values
+    rescue Errno::ENOENT, Errno::EACCES
+      []
+    end
+
+    def start_key(record)
+      session_id = record["session_id"].to_s
+      return "session:#{session_id}" unless session_id.empty?
+
+      "id:#{record['id']}\0#{record['started_at']}"
+    end
+
+    def protect_derived_session_paths(paths, data)
+      repo = data["repo_root"].to_s
+      id = data["id"].to_s
+      return if repo.empty? || id.empty?
+
+      protect(paths, Harnex.events_log_path(repo, id))
+      protect(paths, Harnex.output_log_path(repo, id))
+    rescue StandardError
+      nil
+    end
+
+    def protect(paths, path)
+      text = path.to_s
+      return if text.empty?
+
+      paths.add(File.expand_path(text))
+    end
+
+    def with_lock
+      FileUtils.mkdir_p(STATE_DIR)
+      File.open(LOCK_PATH, File::RDWR | File::CREAT, 0o644) do |file|
+        file.flock(File::LOCK_EX)
+        yield
+      ensure
+        file.flock(File::LOCK_UN) unless file.closed?
+      end
+    end
+
+    def throttled?(last, now)
+      timestamp = last.is_a?(Hash) ? last["timestamp"].to_s : ""
+      return false if timestamp.empty?
+
+      now - Time.parse(timestamp) < THROTTLE_SECONDS
+    rescue ArgumentError
+      false
+    end
+
+    def last_prune_metadata
+      data = JSON.parse(File.read(METADATA_PATH))
+      data["last_prune"]
+    rescue Errno::ENOENT, JSON::ParserError
+      nil
+    end
+
+    def persist_last_prune(report)
+      payload = {
+        "last_prune" => {
+          "timestamp" => report.fetch(:timestamp),
+          "dry_run" => report.fetch(:dry_run),
+          "applied" => !report.fetch(:dry_run),
+          "directories" => report.fetch(:directories).transform_keys(&:to_s).transform_values do |stats|
+            {
+              "before_count" => stats.fetch(:before_count),
+              "before_bytes" => stats.fetch(:before_bytes),
+              "after_count" => stats.fetch(:after_count),
+              "after_bytes" => stats.fetch(:after_bytes),
+              "deleted_count" => stats.fetch(:deleted_count),
+              "deleted_bytes" => stats.fetch(:deleted_bytes),
+              "protected_count" => stats.fetch(:protected_count),
+              "protected_bytes" => stats.fetch(:protected_bytes),
+              "over_cap" => stats.fetch(:over_cap)
+            }
+          end
+        }
+      }
+      tmp = "#{METADATA_PATH}.tmp.#{Process.pid}"
+      File.write(tmp, JSON.pretty_generate(payload))
+      File.rename(tmp, METADATA_PATH)
+    rescue StandardError => e
+      warn("harnex: failed to persist retention metadata: #{e.message}")
+    end
+  end
+end

data/lib/harnex/runtime/session.rb

@@ -23,7 +23,7 @@ module Harnex
       terminal_tokens terminal_percent peak_tokens peak_percent
     ].freeze
     CONTEXT_SAMPLE_STATUSES = %w[observed estimated missing].freeze
-    ATTEMPT_KINDS = %w[initial retry fix review superseding].freeze
+    ATTEMPT_KINDS = %w[initial retry fix review fallback superseding].freeze
     SESSION_SUMMARY_SIGNAL_FIELDS = %i[
       input_tokens output_tokens reasoning_tokens cached_tokens total_tokens
       agent_session_id cost_usd
@@ -183,6 +183,7 @@ module Harnex
 
     def run(validate_binary: true)
       validate_binary! if validate_binary
+      prune_retained_logs
       prepare_output_log
       prepare_events_log
 
@@ -1154,6 +1155,13 @@ module Harnex
       @output_log_failed = false
     end
 
+    def prune_retained_logs
+      Harnex::Retention.auto_prune(
+        repo_root: repo_root,
+        current_paths: [output_log_path, events_log_path]
+      )
+    end
+
     def prepare_events_log
       @events_log&.close unless @events_log&.closed?
       @events_log = File.open(events_log_path, "ab")
@@ -1247,7 +1255,9 @@ module Harnex
     end
 
     def emit_summary_event
-      emit_event("summary", path: summary_out, exit: @exit_reason)
+      payload = { path: dispatch_history_path, exit: @exit_reason }
+      payload[:mirror_path] = summary_out if summary_out
+      emit_event("summary", **payload)
     end
 
     def emit_attempt_finished(attempt)
@@ -1284,9 +1294,9 @@ module Harnex
         warn("harnex: failed to collect session-end telemetry: #{e.message}")
       end
       @exit_reason ||= classify_exit
-      record = build_summary_record
+      record = DispatchHistory.build_record(self)
+      append_dispatch_history_record(record)
       append_summary_record(record)
-      append_dispatch_history_record
       emit_summary_event
       emit_attempt_finished(record.fetch(:attempt))
       emit_exit_event
@@ -1581,10 +1591,30 @@ module Harnex
       cost_source ||= "provider_reported" if !@usage_summary[:cost_usd].nil?
       cost_source ||= summary_string(declared["cost_source"]) || "caller_estimate" if status == "estimated"
 
+      cost_price_as_of = nil
+      if values[:cost_usd].nil? && %w[observed zero].include?(status)
+        priced = Pricing.compute(
+          provider: summary_agent_provider,
+          model: summary_model,
+          input_tokens: values[:input_tokens],
+          output_tokens: values[:output_tokens],
+          cached_tokens: values[:cached_tokens],
+          service_tier: summary_service_tier,
+          context_tokens: @context_summary[:peak_tokens],
+          input_includes_cached: adapter.usage_input_includes_cached?
+        )
+        if priced
+          values[:cost_usd] = priced[:cost_usd]
+          cost_source = "price_table"
+          cost_price_as_of = priced[:as_of]
+        end
+      end
+
       {
         "status" => status,
         "cost_usd" => values[:cost_usd],
         "cost_source" => cost_source,
+        "cost_price_as_of" => cost_price_as_of,
         "input_tokens" => values[:input_tokens],
         "output_tokens" => values[:output_tokens],
         "cached_input_tokens" => values[:cached_tokens],
@@ -1762,6 +1792,101 @@ module Harnex
       @exit_reason == "success"
     end
 
+    def attempt_chain_summary
+      @attempt_chain_summary ||= begin
+        current = {
+          id: id,
+          parent_dispatch_id: summary_string(meta_hash["parent_dispatch_id"]) || @parent_harnex_id,
+          kind: summary_attempt_kind,
+          outcome: summary_attempt_succeeded? ? "succeeded" : "failed"
+        }
+        index = historical_attempt_index
+        nodes = [current]
+        seen = {}
+        seen[current[:id]] = true if current[:id]
+        parent = current[:parent_dispatch_id]
+        immediate_parent = nil
+
+        while parent && !seen[parent]
+          record = index[parent]
+          break unless record
+
+          node = historical_attempt_node(record)
+          break unless node
+
+          immediate_parent ||= node
+          nodes << node
+          seen[node[:id]] = true if node[:id]
+          parent = node[:parent_dispatch_id]
+        end
+
+        {
+          attempts_total: nodes.length,
+          attempts_succeeded: nodes.count { |node| node[:outcome] == "succeeded" },
+          attempts_failed: nodes.count { |node| node[:outcome] == "failed" },
+          fallback_triggered: nodes.any? { |node| node[:kind] == "fallback" },
+          recovered: current[:outcome] == "succeeded" && immediate_parent&.dig(:outcome) == "failed"
+        }
+      end
+    end
+
+    def historical_attempt_index
+      path = dispatch_history_path
+      return {} unless File.file?(path)
+
+      records = {}
+      duplicates = {}
+      File.foreach(path) do |line|
+        record = JSON.parse(line)
+        next unless DispatchHistory.end_record?(record)
+
+        dispatch_id = summary_string(record["id"])
+        next unless dispatch_id
+
+        if records.key?(dispatch_id)
+          records.delete(dispatch_id)
+          duplicates[dispatch_id] = true
+        elsif !duplicates.key?(dispatch_id)
+          records[dispatch_id] = record
+        end
+      rescue JSON::ParserError
+        next
+      end
+      records
+    rescue StandardError
+      {}
+    end
+
+    def historical_attempt_node(record)
+      attempt = record["attempt"].is_a?(Hash) ? record["attempt"] : {}
+      meta = record["meta"].is_a?(Hash) ? record["meta"] : {}
+      dispatch_id = summary_string(record["id"])
+      return nil unless dispatch_id
+
+      {
+        id: dispatch_id,
+        parent_dispatch_id: summary_string(attempt["parent_dispatch_id"]) || summary_string(meta["parent_dispatch_id"]),
+        kind: summary_string(attempt["kind"]),
+        outcome: historical_attempt_outcome(record, attempt)
+      }
+    end
+
+    def historical_attempt_outcome(record, attempt)
+      case summary_string(attempt["status"])
+      when "succeeded"
+        "succeeded"
+      when "failed"
+        "failed"
+      else
+        case summary_string(record["status"])
+        when "completed"
+          "succeeded"
+        when "failed", "timeout", "killed"
+          "failed"
+        end
+      end
+    end
+
     def accepted_throughput_tokens_per_s(total_tokens, duration_s, accepted)
       return nil unless accepted && total_tokens.is_a?(Numeric) && duration_s.to_f.positive?
 
@@ -1785,6 +1910,7 @@ module Harnex
     def build_summary_reliability
       counters = reliability_event_counters
       real_disconnections = counters[:disconnections].to_i
+      chain = attempt_chain_summary
       {
         "adapter_close" => summary_adapter_close(real_disconnections),
         "real_disconnections" => real_disconnections,
@@ -1792,7 +1918,7 @@ module Harnex
         "stalls" => counters[:stalls].to_i,
         "force_resumes" => counters[:force_resumes].to_i,
         "compactions" => counters[:compactions].to_i,
-        "recovered" => false
+        "recovered" => chain[:recovered]
       }
     end
 
@@ -1801,7 +1927,7 @@ module Harnex
       output_measurements = summary_output_measurements
       accepted = outcome["status"] == "accepted"
       duration_s = @ended_at ? (@ended_at - @started_at).to_i : nil
-      attempt_succeeded = summary_attempt_succeeded?
+      chain = attempt_chain_summary
       total_tokens = @usage_summary[:total_tokens]
 
       actual = {
@@ -1839,9 +1965,9 @@ module Harnex
         event_records: @events_log_seq,
         output_log_path: output_log_path,
         events_log_path: events_log_path,
-        attempts_total: 1,
-        attempts_succeeded: attempt_succeeded ? 1 : 0,
-        attempts_failed: attempt_succeeded ? 0 : 1,
+        attempts_total: chain[:attempts_total],
+        attempts_succeeded: chain[:attempts_succeeded],
+        attempts_failed: chain[:attempts_failed],
         retry_count: counters[:retries],
         throttle_429_count: counters[:throttle_429],
         disconnect_count: counters[:disconnections],
@@ -1849,7 +1975,7 @@ module Harnex
         throughput_successes_per_h: accepted_throughput_successes_per_h(duration_s, accepted),
         retry_tax_pct: counters[:retries].to_i.zero? ? 0.0 : nil,
         unattributed: attribution["status"] != "complete",
-        fallback_triggered: false
+        fallback_triggered: chain[:fallback_triggered]
       }
       actual
     end
@@ -1983,6 +2109,8 @@ module Harnex
       meta.is_a?(Hash) ? meta : {}
     end
 
+    # Explicit-only mirror: the identical end record lands here in addition
+    # to the tracked stream when --summary-out is configured.
     def append_summary_record(record)
       return unless summary_out
 
@@ -2001,8 +2129,8 @@ module Harnex
       warn("harnex: failed to write dispatch start record: #{e.message}")
     end
 
-    def append_dispatch_history_record
-      DispatchHistory.append(dispatch_history_path, DispatchHistory.build_record(self))
+    def append_dispatch_history_record(record)
+      DispatchHistory.append(dispatch_history_path, record)
     rescue StandardError => e
       warn("harnex: failed to write dispatch history: #{e.message}")
     end

data/lib/harnex/terminal_status.rb

@@ -80,7 +80,16 @@ module Harnex
         next unless record.is_a?(Hash)
         next if DispatchHistory.start_record?(record)
 
-        if summary_record?(record) && record.dig("meta", "id").to_s == id
+        # Branch on record_type first: a v2 end row carries both the thin
+        # envelope and the rich sections, so it resolves as summary and
+        # history in one shot — never double-counted via the legacy
+        # duck-types below, which stay for pre-v2 files.
+        if record["record_type"] == "dispatch_end"
+          next unless record["id"].to_s == id
+
+          history_record = record
+          summary_record = record if summary_record?(record)
+        elsif summary_record?(record) && record.dig("meta", "id").to_s == id
           summary_record = record
         elsif history_record?(record) && record["id"].to_s == id
           history_record = record
@@ -98,6 +107,7 @@ module Harnex
       record["meta"].is_a?(Hash) && record["actual"].is_a?(Hash)
     end
 
+    # Legacy thin end rows predate record_type.
     def history_record?(record)
       record["schema_version"] == 1 && record.key?("status")
     end

data/lib/harnex/version.rb

@@ -1,4 +1,4 @@
 module Harnex
-  VERSION = "0.8.0"
-  RELEASE_DATE = "2026-08-02"
+  VERSION = "0.9.0"
+  RELEASE_DATE = "2026-08-03"
 end

data/lib/harnex.rb

@@ -5,6 +5,9 @@ require "open3"
 require_relative "harnex/version"
 require_relative "harnex/core"
 require_relative "harnex/dispatch_history"
+require_relative "harnex/config"
+require_relative "harnex/retention"
+require_relative "harnex/pricing"
 require_relative "harnex/terminal_status"
 require_relative "harnex/artifact_report"
 require_relative "harnex/context_telemetry"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: harnex
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Jikku Jose
@@ -26,6 +26,10 @@ files:
 - TECHNICAL.md
 - bin/gem-push
 - bin/harnex
+- docs/codex-appserver.md
+- docs/configuration.md
+- docs/dispatch-telemetry.md
+- docs/events.md
 - guides/01_dispatch.md
 - guides/02_chain.md
 - guides/03_buddy.md
@@ -60,10 +64,13 @@ files:
 - lib/harnex/commands/wait.rb
 - lib/harnex/commands/watch.rb
 - lib/harnex/commands/watch_presets.rb
+- lib/harnex/config.rb
 - lib/harnex/context_telemetry.rb
 - lib/harnex/core.rb
 - lib/harnex/dispatch_history.rb
 - lib/harnex/orchestration.rb
+- lib/harnex/pricing.rb
+- lib/harnex/retention.rb
 - lib/harnex/runtime/api_server.rb
 - lib/harnex/runtime/file_change_hook.rb
 - lib/harnex/runtime/inbox.rb