harnex 0.7.14 → 0.9.0

data/lib/harnex/pricing.rb

@@ -0,0 +1,135 @@
+module Harnex
+  # Static price table for computing usage.cost_usd when an adapter reports
+  # tokens but no cost (plan 33 Phase 2, locked decision 6). Cost is computed,
+  # never guessed: an unknown provider/model/service tier or a missing token
+  # component leaves cost_usd null.
+  #
+  # ## Update procedure
+  #
+  # Rates are USD per 1M tokens, copied by hand from the provider pricing
+  # pages at the URLs below. To update:
+  #
+  # 1. Fetch the provider's pricing page and read the per-1M-token rates for
+  #    input, cached input (cache read), and output.
+  # 2. Replace the entry's rates and set `as_of` to the date you read them.
+  #    Add new models as new entries; never delete an entry solely because a
+  #    model left the pricing page (old rows in dispatch.jsonl were priced
+  #    against it — the `as_of` on each row records which table vintage
+  #    applied).
+  # 3. Update the expected costs in test/harnex/pricing_test.rb — rate
+  #    changes are meant to be conscious, test-visible edits.
+  # 4. Never backfill: rows written before a table change keep the cost they
+  #    were written with.
+  #
+  # Models whose pricing varies by service tier use a nested `service_tiers`
+  # table. For those models a nil or unknown service tier is unpriceable. A
+  # `max_context_tokens_exclusive` entry also requires an observed context
+  # high-water below that boundary; Harnex leaves cost null when context is
+  # missing or entered a differently-priced long-context tier. Do not infer
+  # standard/flex/fast or context tier outside the recorded measurements.
+  # OpenAI documents `priority` as the fast alias for gpt-5.5 short-context
+  # pricing; Harnex maps that alias only where the source supports it.
+  #
+  # Known scheduled change: Anthropic's claude-sonnet-5 entry below carries
+  # introductory pricing ($2/$10) that ends 2026-08-31; standard pricing is
+  # $3/$15 (cache read $0.30) from 2026-09-01. Refresh the entry then.
+  #
+  # ## Token semantics (verified 2026-08-02, plan 33 Phase 2)
+  #
+  # What input_tokens contains depends on the CAPTURE PATH, not just the
+  # provider, so the caller passes `input_includes_cached:` (sourced from
+  # `Adapters::Base#usage_input_includes_cached?`):
+  #
+  # - codex app-server (input_includes_cached: true): the JSON
+  #   TokenUsageBreakdown reports cachedInputTokens as a subset of
+  #   inputTokens and reasoningOutputTokens as a subset of outputTokens
+  #   (verified against test/fixtures/codex_schema/v2 and a live captured
+  #   row: input 3,084,697 + output 17,705 == total 3,102,402 exactly).
+  #   Billable input = input - cached.
+  # - codex PTY (false): the scraped TUI line `input=X (+ Y cached)` shows
+  #   NON-cached input with cached as a separate additive count (fixture:
+  #   total 106,867 == input 104,158 + output 2,709, cached 250,880 > input).
+  #   Billable input = input; cached prices additively at the cached rate.
+  # - anthropic (false): the Messages API reports input_tokens EXCLUSIVE of
+  #   cache reads (cache_read_input_tokens is a separate field). Cache
+  #   WRITES (cache_creation_input_tokens, billed at 1.25x input) have no
+  #   harnex usage field yet — the Phase 3 Claude usage producer must either
+  #   fold them into input_tokens or extend this formula before pricing rows
+  #   that include cache writes.
+  #
+  # Reasoning tokens are never priced separately on any path — they are
+  # already inside output_tokens.
+  module Pricing
+    PRICES = {
+      "openai" => {
+        # Source: https://developers.openai.com/api/docs/pricing
+        "gpt-5.3-codex" => { input: 1.75, cached_input: 0.175, output: 14.00, as_of: "2026-08-02" },
+        # Short-context service-tier rates read 2026-08-03. Long-context
+        # rates are deliberately absent until exact source pricing is known.
+        "gpt-5.5" => {
+          max_context_tokens_exclusive: 272_000,
+          service_tier_aliases: { "priority" => "fast" }.freeze,
+          service_tiers: {
+            "standard" => { input: 5.00, cached_input: 0.50, output: 30.00, as_of: "2026-08-03" },
+            "flex" => { input: 2.50, cached_input: 0.25, output: 15.00, as_of: "2026-08-03" },
+            "fast" => { input: 12.50, cached_input: 1.25, output: 75.00, as_of: "2026-08-03" }
+          }.freeze
+        }.freeze,
+        "gpt-5.2" => { input: 1.75, cached_input: 0.175, output: 14.00, as_of: "2026-08-02" },
+        "gpt-5.1" => { input: 1.25, cached_input: 0.125, output: 10.00, as_of: "2026-08-02" },
+        "gpt-5" => { input: 1.25, cached_input: 0.125, output: 10.00, as_of: "2026-08-02" },
+        "gpt-5-mini" => { input: 0.25, cached_input: 0.025, output: 2.00, as_of: "2026-08-02" }
+      }.freeze,
+      "anthropic" => {
+        # Source: https://claude.com/platform/api (cache read = 0.1x input)
+        "claude-fable-5" => { input: 10.00, cached_input: 1.00, output: 50.00, as_of: "2026-08-02" },
+        "claude-opus-5" => { input: 5.00, cached_input: 0.50, output: 25.00, as_of: "2026-08-02" },
+        "claude-sonnet-5" => { input: 2.00, cached_input: 0.20, output: 10.00, as_of: "2026-08-02" },
+        "claude-haiku-4-5" => { input: 1.00, cached_input: 0.10, output: 5.00, as_of: "2026-08-02" }
+      }.freeze
+    }.freeze
+
+    module_function
+
+    # Returns { cost_usd:, as_of: } or nil when the cost cannot be computed
+    # (unknown provider/model/service tier, or input/output token counts missing).
+    def compute(provider:, model:, input_tokens:, output_tokens:, cached_tokens: nil,
+                service_tier: nil, context_tokens: nil,
+                input_includes_cached: false)
+      model_entry = PRICES.dig(provider.to_s, model.to_s)
+      return nil unless model_entry
+      return nil unless context_priceable?(model_entry, context_tokens)
+
+      entry = rates_for_service_tier(model_entry, service_tier)
+      return nil unless entry
+      return nil unless input_tokens.is_a?(Numeric) && output_tokens.is_a?(Numeric)
+
+      cached = cached_tokens.is_a?(Numeric) ? cached_tokens : 0
+      billable_input = input_includes_cached ? input_tokens - cached : input_tokens
+      billable_input = 0 if billable_input.negative?
+
+      cost = (billable_input * entry[:input] +
+              cached * entry[:cached_input] +
+              output_tokens * entry[:output]) / 1_000_000.0
+      { cost_usd: cost.round(6), as_of: entry[:as_of] }
+    end
+
+    def context_priceable?(entry, context_tokens)
+      limit = entry[:max_context_tokens_exclusive]
+      return true unless limit
+
+      context_tokens.is_a?(Numeric) && context_tokens >= 0 && context_tokens < limit
+    end
+
+    def rates_for_service_tier(entry, service_tier)
+      tiers = entry[:service_tiers]
+      return entry unless tiers
+
+      tier = service_tier.to_s
+      return nil if tier.empty?
+
+      aliases = entry[:service_tier_aliases] || {}
+      tiers[aliases.fetch(tier, tier)]
+    end
+  end
+end

data/lib/harnex/retention.rb

@@ -0,0 +1,311 @@
+require "fileutils"
+require "json"
+require "set"
+require "time"
+
+module Harnex
+  module Retention
+    DIRS = {
+      "events" => "events",
+      "output" => "output"
+    }.freeze
+    THROTTLE_SECONDS = 3600
+    MAX_REPORTED_PATHS = 100
+    METADATA_PATH = File.join(STATE_DIR, "retention.json").freeze
+    LOCK_PATH = File.join(STATE_DIR, "retention.lock").freeze
+
+    module_function
+
+    def status(repo_root:, env: ENV, now: Time.now)
+      limits = Config.retention_limits(repo_root, env: env)
+      {
+        ok: true,
+        checked_at: now.utc.iso8601,
+        config: limits,
+        directories: DIRS.to_h do |key, dirname|
+          files = inventory(File.join(STATE_DIR, dirname))
+          [key, {
+            path: File.join(STATE_DIR, dirname),
+            count: files.length,
+            bytes: files.sum { |file| file.fetch(:bytes) },
+            max_age_days: limits.fetch(key).fetch("max_age_days"),
+            max_bytes: limits.fetch(key).fetch("max_bytes")
+          }]
+        end,
+        last_prune: last_prune_metadata
+      }
+    end
+
+    def auto_prune(repo_root:, current_paths: [], env: ENV, now: Time.now)
+      prune(
+        repo_root: repo_root,
+        current_paths: current_paths,
+        env: env,
+        now: now,
+        dry_run: false,
+        force: false
+      )
+    rescue Config::ConfigError
+      raise
+    rescue StandardError => e
+      warn("harnex: retention prune failed: #{e.message}; run `harnex doctor --prune --dry-run` to inspect candidates")
+      { ok: false, error: e.message }
+    end
+
+    def prune(repo_root:, current_paths: [], env: ENV, now: Time.now, dry_run: false, force: false)
+      limits = Config.retention_limits(repo_root, env: env)
+      with_lock do
+        last = last_prune_metadata
+        if !force && !dry_run && throttled?(last, now)
+          return {
+            ok: true,
+            skipped: true,
+            reason: "throttled",
+            throttle_seconds: THROTTLE_SECONDS,
+            last_prune: last
+          }
+        end
+
+        protected = protected_paths(repo_root, current_paths)
+        report = {
+          ok: true,
+          skipped: false,
+          dry_run: dry_run,
+          timestamp: now.utc.iso8601,
+          directories: DIRS.to_h do |key, dirname|
+            [key.to_sym, prune_directory(
+              File.join(STATE_DIR, dirname),
+              limits.fetch(key),
+              protected,
+              now,
+              dry_run: dry_run
+            )]
+          end
+        }
+        persist_last_prune(report)
+        report
+      end
+    end
+
+    def inventory(dir)
+      FileUtils.mkdir_p(dir)
+      base = File.expand_path(dir)
+      Dir.children(base).sort.filter_map do |name|
+        path = File.expand_path(File.join(base, name))
+        next unless path.start_with?("#{base}#{File::SEPARATOR}")
+
+        stat = File.lstat(path)
+        next unless stat.file?
+
+        { path: path, bytes: stat.size, mtime: stat.mtime }
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR
+        nil
+      end
+    end
+
+    def prune_directory(dir, limits, protected, now, dry_run:)
+      files = inventory(dir)
+      before_bytes = files.sum { |file| file.fetch(:bytes) }
+      protected_files = files.select { |file| protected.include?(file.fetch(:path)) }
+      deleted = []
+      deleted_paths = Set.new
+      projected_bytes = before_bytes
+
+      cutoff = now - limits.fetch("max_age_days") * 86_400
+      age_candidates = files.reject { |file| protected.include?(file.fetch(:path)) }
+        .select { |file| file.fetch(:mtime) < cutoff }
+        .sort_by { |file| [file.fetch(:mtime), file.fetch(:path)] }
+      age_candidates.each do |file|
+        next if deleted_paths.include?(file.fetch(:path))
+
+        if delete_candidate(file, dry_run: dry_run)
+          deleted << file
+          deleted_paths.add(file.fetch(:path))
+          projected_bytes -= file.fetch(:bytes)
+        end
+      end
+
+      size_candidates = files.reject { |file| protected.include?(file.fetch(:path)) || deleted_paths.include?(file.fetch(:path)) }
+        .sort_by { |file| [file.fetch(:mtime), file.fetch(:path)] }
+      size_candidates.each do |file|
+        break if projected_bytes <= limits.fetch("max_bytes")
+
+        if delete_candidate(file, dry_run: dry_run)
+          deleted << file
+          deleted_paths.add(file.fetch(:path))
+          projected_bytes -= file.fetch(:bytes)
+        end
+      end
+
+      after_bytes = dry_run ? projected_bytes : inventory(dir).sum { |file| file.fetch(:bytes) }
+      {
+        path: dir,
+        before_count: files.length,
+        before_bytes: before_bytes,
+        after_count: dry_run ? files.length - deleted.length : inventory(dir).length,
+        after_bytes: after_bytes,
+        deleted_count: deleted.length,
+        deleted_bytes: deleted.sum { |file| file.fetch(:bytes) },
+        deleted_paths: deleted.first(MAX_REPORTED_PATHS).map { |file| file.fetch(:path) },
+        deleted_paths_truncated: deleted.length > MAX_REPORTED_PATHS,
+        protected_count: protected_files.length,
+        protected_bytes: protected_files.sum { |file| file.fetch(:bytes) },
+        max_age_days: limits.fetch("max_age_days"),
+        max_bytes: limits.fetch("max_bytes"),
+        over_cap: after_bytes > limits.fetch("max_bytes")
+      }
+    end
+
+    def delete_candidate(file, dry_run:)
+      return true if dry_run
+
+      File.delete(file.fetch(:path))
+      true
+    rescue Errno::ENOENT
+      true
+    rescue Errno::EACCES, Errno::EPERM, Errno::ENOTDIR
+      false
+    end
+
+    def protected_paths(repo_root, current_paths)
+      paths = Set.new
+      current_paths.each { |path| protect(paths, path) }
+      protect_live_registry_paths(paths)
+      protect_live_start_row_paths(paths, repo_root)
+      paths
+    end
+
+    def protect_live_registry_paths(paths)
+      return unless Dir.exist?(SESSIONS_DIR)
+
+      Dir.glob(File.join(SESSIONS_DIR, "*.json")).sort.each do |path|
+        data = JSON.parse(File.read(path))
+        next unless data["pid"] && Harnex.alive_pid?(data["pid"])
+
+        protect(paths, data["events_log_path"])
+        protect(paths, data["output_log_path"])
+        protect_derived_session_paths(paths, data)
+      rescue JSON::ParserError, ArgumentError, TypeError, Errno::ENOENT, Errno::EACCES
+        next
+      end
+    end
+
+    def protect_live_start_row_paths(paths, repo_root)
+      history_paths = [
+        DispatchHistory.path_for(repo_root),
+        DispatchHistory.global_path
+      ].uniq
+
+      history_paths.each do |path|
+        live_start_records(path).each do |record|
+          protect(paths, record["events_log_path"])
+          protect(paths, record["output_log_path"])
+          protect_derived_session_paths(paths, record)
+        end
+      end
+    end
+
+    def live_start_records(path)
+      return [] unless File.file?(path)
+
+      open = {}
+      File.foreach(path) do |line|
+        record = JSON.parse(line)
+        next unless record.is_a?(Hash)
+
+        if DispatchHistory.start_record?(record)
+          next unless DispatchHistory.same_host?(record)
+          next unless record["pid"] && Harnex.alive_pid?(record["pid"])
+
+          open[start_key(record)] = record
+        elsif DispatchHistory.end_record?(record)
+          open.delete_if { |_key, start| DispatchHistory.end_matches_start?(record, start) }
+        end
+      rescue JSON::ParserError, ArgumentError, TypeError
+        next
+      end
+      open.values
+    rescue Errno::ENOENT, Errno::EACCES
+      []
+    end
+
+    def start_key(record)
+      session_id = record["session_id"].to_s
+      return "session:#{session_id}" unless session_id.empty?
+
+      "id:#{record['id']}\0#{record['started_at']}"
+    end
+
+    def protect_derived_session_paths(paths, data)
+      repo = data["repo_root"].to_s
+      id = data["id"].to_s
+      return if repo.empty? || id.empty?
+
+      protect(paths, Harnex.events_log_path(repo, id))
+      protect(paths, Harnex.output_log_path(repo, id))
+    rescue StandardError
+      nil
+    end
+
+    def protect(paths, path)
+      text = path.to_s
+      return if text.empty?
+
+      paths.add(File.expand_path(text))
+    end
+
+    def with_lock
+      FileUtils.mkdir_p(STATE_DIR)
+      File.open(LOCK_PATH, File::RDWR | File::CREAT, 0o644) do |file|
+        file.flock(File::LOCK_EX)
+        yield
+      ensure
+        file.flock(File::LOCK_UN) unless file.closed?
+      end
+    end
+
+    def throttled?(last, now)
+      timestamp = last.is_a?(Hash) ? last["timestamp"].to_s : ""
+      return false if timestamp.empty?
+
+      now - Time.parse(timestamp) < THROTTLE_SECONDS
+    rescue ArgumentError
+      false
+    end
+
+    def last_prune_metadata
+      data = JSON.parse(File.read(METADATA_PATH))
+      data["last_prune"]
+    rescue Errno::ENOENT, JSON::ParserError
+      nil
+    end
+
+    def persist_last_prune(report)
+      payload = {
+        "last_prune" => {
+          "timestamp" => report.fetch(:timestamp),
+          "dry_run" => report.fetch(:dry_run),
+          "applied" => !report.fetch(:dry_run),
+          "directories" => report.fetch(:directories).transform_keys(&:to_s).transform_values do |stats|
+            {
+              "before_count" => stats.fetch(:before_count),
+              "before_bytes" => stats.fetch(:before_bytes),
+              "after_count" => stats.fetch(:after_count),
+              "after_bytes" => stats.fetch(:after_bytes),
+              "deleted_count" => stats.fetch(:deleted_count),
+              "deleted_bytes" => stats.fetch(:deleted_bytes),
+              "protected_count" => stats.fetch(:protected_count),
+              "protected_bytes" => stats.fetch(:protected_bytes),
+              "over_cap" => stats.fetch(:over_cap)
+            }
+          end
+        }
+      }
+      tmp = "#{METADATA_PATH}.tmp.#{Process.pid}"
+      File.write(tmp, JSON.pretty_generate(payload))
+      File.rename(tmp, METADATA_PATH)
+    rescue StandardError => e
+      warn("harnex: failed to persist retention metadata: #{e.message}")
+    end
+  end
+end