ace-compressor 0.24.1

data/lib/ace/compressor/atoms/retention_reporter.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Atoms
+      class RetentionReporter
+        LITERAL_PROTECTED_TYPES = %w[RULE CONSTRAINT CMD U].freeze
+        COUNT_PROTECTED_TYPES = %w[CODE TABLE].freeze
+        STRUCTURED_TYPES = %w[LIST PROBLEMS FILES TREE EXAMPLE].freeze
+        LOSS_MARKER_TYPES = %w[LOSS EXAMPLE_REF REFUSAL FALLBACK].freeze
+
+        def compare(reference_content:, candidate_content:)
+          reference = parse(reference_content)
+          candidate = parse(candidate_content)
+
+          {
+            "sections" => coverage_for_sections(reference, candidate),
+            "protected" => coverage_for_protected(reference, candidate),
+            "structured" => coverage_for_structured(reference, candidate),
+            "loss_markers" => LOSS_MARKER_TYPES.to_h { |type| [type.downcase, candidate.fetch("counts").fetch(type, 0)] }
+          }
+        end
+
+        private
+
+        def parse(content)
+          lines = content.to_s.lines.map(&:strip).reject(&:empty?)
+          counts = Hash.new(0)
+          literal = Hash.new { |hash, key| hash[key] = Set.new }
+          sections = Set.new
+          structured_keys = Hash.new { |hash, key| hash[key] = Set.new }
+
+          lines.each do |line|
+            type = line.split("|", 2).first
+            next if type.to_s.empty?
+
+            counts[type] += 1
+            sections << line.delete_prefix("SEC|") if type == "SEC"
+            literal[type] << line if LITERAL_PROTECTED_TYPES.include?(type)
+            structured_keys[type] << structured_key(type, line) if STRUCTURED_TYPES.include?(type)
+          end
+
+          {
+            "counts" => counts,
+            "sections" => sections,
+            "literal" => literal,
+            "structured_keys" => structured_keys
+          }
+        end
+
+        def coverage_for_sections(reference, candidate)
+          total = reference.fetch("sections").size
+          retained = reference.fetch("sections").intersection(candidate.fetch("sections")).size
+          coverage_hash(retained, total)
+        end
+
+        def coverage_for_protected(reference, candidate)
+          literal_total = 0
+          literal_retained = 0
+
+          LITERAL_PROTECTED_TYPES.each do |type|
+            ref_lines = reference.fetch("literal").fetch(type, Set.new)
+            cand_lines = candidate.fetch("literal").fetch(type, Set.new)
+            literal_total += ref_lines.size
+            literal_retained += ref_lines.intersection(cand_lines).size
+          end
+
+          count_total = COUNT_PROTECTED_TYPES.sum { |type| reference.fetch("counts").fetch(type, 0) }
+          count_retained = COUNT_PROTECTED_TYPES.sum do |type|
+            [reference.fetch("counts").fetch(type, 0), candidate.fetch("counts").fetch(type, 0)].min
+          end
+
+          coverage_hash(literal_retained + count_retained, literal_total + count_total)
+        end
+
+        def coverage_for_structured(reference, candidate)
+          total = 0
+          retained = 0
+
+          STRUCTURED_TYPES.each do |type|
+            ref_keys = reference.fetch("structured_keys").fetch(type, Set.new)
+            cand_keys = candidate.fetch("structured_keys").fetch(type, Set.new)
+            total += ref_keys.size
+            retained += ref_keys.intersection(cand_keys).size
+          end
+
+          coverage_hash(retained, total)
+        end
+
+        def structured_key(type, line)
+          case type
+          when "LIST", "FILES", "TREE"
+            line.split("|", 3)[1].to_s
+          when "EXAMPLE"
+            line.split("|", 2)[1].to_s
+          else
+            line
+          end
+        end
+
+        def coverage_hash(retained, total)
+          {
+            "retained" => retained,
+            "total" => total,
+            "percent" => total.zero? ? 100.0 : ((retained.to_f / total.to_f) * 100.0).round(1)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/cli/commands/benchmark.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "ace/support/cli"
+require "ace/core"
+
+module Ace
+  module Compressor
+    module CLI
+      module Commands
+        class Benchmark < Ace::Support::Cli::Command
+          include Ace::Support::Cli::Base
+
+          desc "Compare exact, compact, and agent output on live sources"
+
+          argument :sources, required: false, type: :array, desc: "File or directory paths"
+          option :modes, type: :string, desc: "Comma-delimited modes: exact,compact,agent"
+          option :format, type: :string, aliases: ["-f"], desc: "Benchmark output format: table|json"
+          option :verbose, type: :boolean, aliases: ["-v"], default: false, desc: "Show verbose output"
+
+          def call(**options)
+            sources = normalize_sources(options[:sources] || [])
+            if sources.empty?
+              raise Ace::Support::Cli::Error,
+                "Missing input path. Usage: ace-compressor benchmark <file-or-dir> [more-paths...]"
+            end
+
+            runner = Ace::Compressor::Organisms::BenchmarkRunner.new(
+              sources,
+              modes: options[:modes],
+              format: options[:format],
+              verbose: !!options[:verbose]
+            )
+            report = runner.call
+            puts runner.render(report)
+            0
+          rescue Ace::Compressor::Error => e
+            raise Ace::Support::Cli::Error, e.message
+          end
+
+          private
+
+          def normalize_sources(sources)
+            values = Array(sources).dup
+            values.shift if values.first == "benchmark"
+            values
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/cli/commands/compress.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "ace/support/cli"
+require "ace/core"
+
+module Ace
+  module Compressor
+    module CLI
+      module Commands
+        class Compress < Ace::Support::Cli::Command
+          include Ace::Support::Cli::Base
+
+          SUPPORTED_MODES = %w[exact compact agent].freeze
+          SUPPORTED_SOURCE_SCOPES = %w[merged per-source].freeze
+
+          desc "Compress markdown/text files into ContextPack/3 records"
+
+          argument :sources, required: false, type: :array, desc: "File or directory paths"
+          option :mode, type: :string, default: "exact", desc: "Compression mode (exact|compact|agent)"
+          option :source_scope, type: :string, default: "merged", desc: "Source handling mode (merged|per-source)"
+          option :output, type: :string, aliases: ["-o"], desc: "Save output to file or directory path"
+          option :format, type: :string, aliases: ["-f"], desc: "Console output format: path|stdio|stats"
+          option :version, type: :boolean, desc: "Show version information"
+          option :quiet, type: :boolean, aliases: ["-q"], default: false, desc: "Suppress non-essential output"
+          option :verbose, type: :boolean, aliases: ["-v"], default: false, desc: "Show verbose output"
+          option :debug, type: :boolean, aliases: ["-d"], default: false, desc: "Show debug output"
+
+          def call(**options)
+            if options[:version]
+              puts "ace-compressor #{Ace::Compressor::VERSION}"
+              return 0
+            end
+
+            sources = normalize_sources(options[:sources] || [])
+            if sources.empty?
+              raise Ace::Support::Cli::Error,
+                "Missing input path. Usage: ace-compressor <file-or-dir> [more-paths...] --mode <exact|compact|agent>"
+            end
+
+            mode = (options[:mode] || "exact").to_s
+            unless SUPPORTED_MODES.include?(mode)
+              raise Ace::Support::Cli::Error, "Unsupported mode '#{mode}'. Use --mode exact, --mode compact, or --mode agent"
+            end
+            source_scope = (options[:source_scope] || "merged").to_s
+            unless SUPPORTED_SOURCE_SCOPES.include?(source_scope)
+              raise Ace::Support::Cli::Error,
+                "Unsupported source scope '#{source_scope}'. Use --source-scope merged or --source-scope per-source"
+            end
+
+            runner = Ace::Compressor::Organisms::CompressionRunner.new(
+              sources,
+              mode: mode,
+              source_scope: source_scope,
+              output: options[:output],
+              format: options[:format],
+              verbose: !!options[:verbose]
+            )
+            result = runner.call
+            if options[:verbose]
+              result[:ignored_paths].each { |path| warn "Ignoring unsupported file: #{path}" }
+            end
+            puts result[:console_output]
+            if result[:exit_code].to_i.nonzero?
+              raise Ace::Support::Cli::Error, refusal_message_for(mode)
+            end
+          rescue Ace::Compressor::Error => e
+            raise Ace::Support::Cli::Error, e.message
+          end
+
+          private
+
+          def normalize_sources(sources)
+            values = Array(sources).dup
+            values.shift if values.first == "compress"
+            values
+          end
+
+          def refusal_message_for(mode)
+            if mode == "compact"
+              "One or more sources were refused in compact mode. Retry refused sources with --mode exact"
+            else
+              "One or more sources were refused in #{mode} mode. Retry refused sources with --mode exact"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/cli.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "ace/support/cli"
+require "ace/core"
+
+require_relative "cli/commands/compress"
+require_relative "cli/commands/benchmark"
+require_relative "version"
+
+module Ace
+  module Compressor
+    module CLI
+      def self.start(args)
+        args = ["--help"] if args.empty?
+        if args.first == "benchmark"
+          Ace::Support::Cli::Runner.new(Commands::Benchmark).call(args: args.drop(1))
+        else
+          Ace::Support::Cli::Runner.new(Commands::Compress).call(args: args)
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/models/context_pack.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Models
+      class ContextPack
+        SCHEMA = "ContextPack/3"
+
+        def self.escape(value)
+          value.to_s.gsub("|", "\\|").tr("\n", " ").strip
+        end
+
+        def self.header(mode)
+          "H|#{SCHEMA}|#{escape(mode)}"
+        end
+
+        def self.file_line(source)
+          "FILE|#{escape(source)}"
+        end
+
+        def self.policy_line(doc_class:, action:)
+          "POLICY|class=#{escape(doc_class)}|action=#{escape(action)}"
+        end
+
+        def self.fidelity_line(source:, status:, check:, details: nil)
+          line = "FIDELITY|source=#{escape(source)}|status=#{escape(status)}|check=#{escape(check)}"
+          details_text = details.to_s.strip
+          return line if details_text.empty?
+
+          "#{line}|details=#{escape(details_text)}"
+        end
+
+        def self.refusal_line(source:, reason:, failed_check:)
+          "REFUSAL|source=#{escape(source)}|reason=#{escape(reason)}|failed_check=#{escape(failed_check)}"
+        end
+
+        def self.guidance_line(source:, retry_with:)
+          "GUIDANCE|source=#{escape(source)}|retry_with=#{escape(retry_with)}"
+        end
+
+        def self.fallback_line(source:, from:, to:, reason:, check: nil, details: nil)
+          line = "FALLBACK|source=#{escape(source)}|from=#{escape(from)}|to=#{escape(to)}|reason=#{escape(reason)}"
+          line += "|check=#{escape(check)}" unless check.to_s.strip.empty?
+          line += "|details=#{escape(details)}" unless details.to_s.strip.empty?
+          line
+        end
+
+        def self.section_line(title)
+          "SEC|#{escape(title)}"
+        end
+
+        def self.summary_line(text)
+          "SUMMARY|#{escape(text)}"
+        end
+
+        def self.fact_line(text)
+          "FACT|#{escape(text)}"
+        end
+
+        def self.rule_line(text)
+          "RULE|#{escape(text)}"
+        end
+
+        def self.constraint_line(text)
+          "CONSTRAINT|#{escape(text)}"
+        end
+
+        def self.problems_line(items)
+          values = Array(items).map { |item| escape(item) }.join(",")
+          "PROBLEMS|[#{values}]"
+        end
+
+        def self.list_line(list_key, items)
+          values = Array(items).map { |item| escape(item) }.join(",")
+          key = list_key.to_s.strip
+          key = "items" if key.empty?
+          "LIST|#{escape(key)}|[#{values}]"
+        end
+
+        def self.example_line(tool)
+          "EXAMPLE|#{escape("tool=#{tool}")}"
+        end
+
+        def self.cmd_line(command)
+          "CMD|#{escape(command)}"
+        end
+
+        def self.files_line(label, files)
+          "FILES|#{escape(label)}|[#{Array(files).map { |value| escape(value) }.join(",")}]"
+        end
+
+        def self.tree_line(label, tree)
+          "TREE|#{escape(label)}|#{escape(tree)}"
+        end
+
+        def self.code_line(language, code)
+          language_value = language.to_s.strip.empty? ? "code" : language.to_s.strip
+          "CODE|#{escape(language_value)}|#{escape(code)}"
+        end
+
+        def self.table_line(rows, table_id: nil, strategy: nil)
+          fields = []
+          columns, data_rows = normalize_table_rows(rows)
+          fields << "id=#{escape(table_id)}" unless table_id.to_s.strip.empty?
+          fields << "strategy=#{escape(strategy)}" unless strategy.to_s.strip.empty?
+          fields << "cols=#{escape(columns.join(","))}" unless columns.empty?
+          fields << "rows=#{escape(encode_table_data_rows(data_rows))}"
+          "TABLE|#{fields.join("|")}"
+        end
+
+        def self.loss_line(kind:, target:, strategy:, original:, retained:, unit:, source: nil, details: nil)
+          unit_key = unit.to_s.strip
+          unit_key = "items" if unit_key.empty?
+          unit_key = unit_key.gsub(/[^a-z0-9_]/i, "_")
+
+          original_count = original.to_i
+          retained_count = retained.to_i
+          dropped_count = [original_count - retained_count, 0].max
+
+          line = [
+            "LOSS|kind=#{escape(kind)}",
+            "target=#{escape(target)}",
+            "strategy=#{escape(strategy)}",
+            "original_#{unit_key}=#{escape(original_count)}",
+            "retained_#{unit_key}=#{escape(retained_count)}",
+            "dropped_#{unit_key}=#{escape(dropped_count)}"
+          ].join("|")
+
+          line += "|source=#{escape(source)}" unless source.to_s.strip.empty?
+          line += "|details=#{escape(details)}" unless details.to_s.strip.empty?
+          line
+        end
+
+        def self.example_ref_line(tool:, source:, original_source:, reason: "duplicate")
+          "EXAMPLE_REF|tool=#{escape(tool)}|source=#{escape(source)}|original_source=#{escape(original_source)}|reason=#{escape(reason)}"
+        end
+
+        def self.unresolved_line(kind, raw)
+          "U|#{escape(kind)}|#{escape(raw)}"
+        end
+
+        def self.normalize_table_rows(rows)
+          row_values = Array(rows)
+          return [[], []] if row_values.empty?
+
+          return [[], []] if row_values.all? { |row| row.to_s.strip.empty? }
+
+          if row_values.length == 1 && !row_values.first.to_s.include?("|")
+            return [[], [row_values.first.to_s]]
+          end
+
+          header_cells = parse_table_cells(row_values[0])
+          data_rows = row_values[1..].to_a.reject { |row| table_separator_row?(row) }.map { |row| parse_table_cells(row) }
+          [header_cells, data_rows]
+        end
+
+        def self.encode_table_data_rows(rows)
+          Array(rows).map { |cells| Array(cells).map { |cell| escape_table_cell(cell) }.join(">") }.join(";")
+        end
+
+        def self.escape_table_cell(value)
+          value.to_s.gsub("\\", "\\\\").gsub(">", "\\>").gsub(";", "\\;").strip
+        end
+
+        def self.parse_table_cells(row)
+          row.to_s.split("|").map(&:strip).reject(&:empty?)
+        end
+
+        def self.table_separator_row?(row)
+          row.to_s.strip.match?(/\A\|?[-\s:|]+\|?\z/)
+        end
+      end
+    end
+  end
+end