ace-compressor 0.24.1

data/lib/ace/compressor/organisms/compression_runner.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Organisms
+      class CompressionRunner
+        SUPPORTED_FORMATS = %w[path stdio stats].freeze
+        SUPPORTED_MODES = %w[exact compact agent].freeze
+        SUPPORTED_SOURCE_SCOPES = %w[merged per-source].freeze
+
+        def initialize(paths, mode:, source_scope: "merged", output: nil, format: nil, verbose: false)
+          @paths = Array(paths)
+          @mode = mode
+          @source_scope = source_scope.to_s
+          @output = output
+          @format = (format || Ace::Compressor.config["default_format"] || "path").to_s
+          @verbose = verbose
+          @cache_store = Ace::Compressor::Molecules::CacheStore.new
+        end
+
+        def call
+          raise Ace::Compressor::Error, "Unsupported format '#{@format}'. Use --format path, stdio, or stats" unless SUPPORTED_FORMATS.include?(@format)
+          unless SUPPORTED_MODES.include?(@mode)
+            raise Ace::Compressor::Error,
+              "Unsupported mode '#{@mode}'. Use --mode exact, --mode compact, or --mode agent"
+          end
+          unless SUPPORTED_SOURCE_SCOPES.include?(@source_scope)
+            raise Ace::Compressor::Error,
+              "Unsupported source scope '#{@source_scope}'. Use --source-scope merged or --source-scope per-source"
+          end
+
+          resolver = Ace::Compressor::Molecules::InputResolver.new(@paths)
+          begin
+            resolved_inputs = resolver.call
+            return run_per_source(resolved_inputs) if @source_scope == "per-source"
+
+            run_for_sources(resolved_inputs)
+          ensure
+            resolver.cleanup if resolver.respond_to?(:cleanup)
+          end
+        end
+
+        private
+
+        def run_for_sources(resolved_inputs)
+          resolved_paths = Array(resolved_inputs).map { |entry| entry.fetch(:content_path) }
+          compressor = compressor_for_mode(resolved_paths)
+          sources = compressor.resolve_sources
+          source_metadata = build_source_metadata(sources, resolved_inputs)
+          manifest = @cache_store.manifest(mode: @mode, sources: source_metadata)
+
+          # If ace-bundle already compressed this content, override original sizes with
+          # the true uncompressed values and pass through without re-compression.
+          bundle_stats = extract_bundle_compression_stats(resolved_inputs)
+          if bundle_stats
+            manifest["original_bytes"] = bundle_stats["original_bytes"]
+            manifest["original_lines"] = bundle_stats["original_lines"]
+          end
+
+          canonical = @cache_store.canonical_paths(mode: @mode, sources: source_metadata, manifest_key: manifest["key"])
+          shared_manifest = @cache_store.shared_manifest(mode: @mode, sources: source_metadata)
+          shared_canonical = if shared_manifest
+            @cache_store.shared_canonical_paths(mode: @mode, sources: source_metadata, manifest_key: shared_manifest["key"])
+          end
+
+          cache_hit = @cache_store.cache_hit?(pack_path: canonical[:pack_path], metadata_path: canonical[:metadata_path])
+          content, metadata = if cache_hit
+            content = @cache_store.read_pack(canonical[:pack_path])
+            metadata = hydrate_metadata(
+              @cache_store.read_metadata(canonical[:metadata_path]),
+              manifest,
+              canonical,
+              content
+            )
+            [content, metadata]
+          elsif shared_cache_hit?(shared_canonical)
+            hydrate_from_shared_cache(manifest, canonical, shared_canonical)
+          elsif bundle_stats
+            build_passthrough_entry(sources, manifest, canonical, shared_canonical)
+          else
+            build_cache_entry(compressor, sources, source_metadata, manifest, canonical, shared_canonical)
+          end
+          cache_hit ||= shared_cache_hit?(shared_canonical)
+
+          output_path = @cache_store.output_path_for(
+            output: @output,
+            mode: @mode,
+            sources: source_metadata,
+            manifest_key: manifest["key"]
+          )
+
+          @cache_store.write_output(output_path, content) unless output_path == canonical[:pack_path]
+          refusal_lines = refusal_lines(content)
+          fallback_lines = fallback_lines(content)
+
+          {
+            console_output: format_console_output(
+              content: content,
+              cache_hit: cache_hit,
+              output_path: output_path,
+              metadata: metadata
+            ),
+            ignored_paths: compressor.ignored_paths,
+            output_path: output_path,
+            cache_hit: cache_hit,
+            metadata: metadata,
+            refusal_lines: refusal_lines,
+            fallback_lines: fallback_lines,
+            exit_code: refusal_lines.empty? ? 0 : 1
+          }
+        end
+
+        def run_per_source(resolved_inputs)
+          if resolved_inputs.size > 1 && output_file_target?
+            raise Ace::Compressor::Error,
+              "Per-source mode with multiple inputs requires --output to be a directory path"
+          end
+
+          results = resolved_inputs.map { |entry| run_for_sources([entry]) }
+          {
+            console_output: join_console_outputs(results.map { |result| result[:console_output] }),
+            ignored_paths: results.flat_map { |result| result[:ignored_paths] }.uniq,
+            output_path: results.first&.dig(:output_path),
+            output_paths: results.map { |result| result[:output_path] },
+            cache_hit: results.all? { |result| result[:cache_hit] },
+            metadata: results.map { |result| result[:metadata] },
+            refusal_lines: results.flat_map { |result| result[:refusal_lines] },
+            fallback_lines: results.flat_map { |result| result[:fallback_lines] },
+            exit_code: (results.any? { |result| result[:exit_code].to_i.nonzero? }) ? 1 : 0
+          }
+        end
+
+        def join_console_outputs(outputs)
+          values = Array(outputs).map(&:to_s).reject(&:empty?)
+          return "" if values.empty?
+
+          separator = (@format == "stats") ? "\n\n" : "\n"
+          values.join(separator)
+        end
+
+        def output_file_target?
+          return false if @output.nil? || @output.to_s.strip.empty?
+          return false if @output.end_with?(File::SEPARATOR)
+
+          !Dir.exist?(File.expand_path(@output))
+        end
+
+        def build_passthrough_entry(sources, manifest, canonical, shared_canonical = nil)
+          raw = sources.map { |s| File.read(s) }.join("\n")
+          header = Ace::Compressor::Models::ContextPack.header(@mode)
+          content = "#{header}\n#{raw}"
+          metadata = cache_metadata(manifest, canonical[:short_key], content)
+          @cache_store.write_cache(
+            pack_path: canonical[:pack_path],
+            metadata_path: canonical[:metadata_path],
+            content: content,
+            metadata: metadata
+          )
+          if shared_canonical
+            shared_metadata = cache_metadata(manifest, shared_canonical[:short_key], content, metadata.merge("cache_scope" => "shared"))
+            @cache_store.write_cache(
+              pack_path: shared_canonical[:pack_path],
+              metadata_path: shared_canonical[:metadata_path],
+              content: content,
+              metadata: shared_metadata
+            )
+          end
+          [content, metadata]
+        end
+
+        def extract_bundle_compression_stats(resolved_inputs)
+          Array(resolved_inputs).each do |entry|
+            stats = entry[:bundle_compression_stats]
+            return stats if stats
+          end
+          nil
+        end
+
+        def build_cache_entry(compressor, sources, source_metadata, manifest, canonical, shared_canonical = nil)
+          content = compress_content(compressor, sources, source_metadata)
+          metadata = cache_metadata(manifest, canonical[:short_key], content)
+          @cache_store.write_cache(
+            pack_path: canonical[:pack_path],
+            metadata_path: canonical[:metadata_path],
+            content: content,
+            metadata: metadata
+          )
+          if shared_canonical
+            shared_metadata = cache_metadata(manifest, shared_canonical[:short_key], content, metadata.merge("cache_scope" => "shared"))
+            @cache_store.write_cache(
+              pack_path: shared_canonical[:pack_path],
+              metadata_path: shared_canonical[:metadata_path],
+              content: content,
+              metadata: shared_metadata
+            )
+          end
+          [content, metadata]
+        end
+
+        def compress_content(compressor, sources, source_metadata)
+          source_paths = source_metadata.each_with_object({}) do |entry, hash|
+            hash[entry.fetch(:content_path)] = entry.fetch(:source_path)
+          end
+
+          parameters = compressor.method(:compress_sources).parameters
+          supports_source_paths = parameters.any? do |type, name|
+            [:key, :keyreq].include?(type) && name == :source_paths || type == :keyrest
+          end
+
+          return compressor.compress_sources(sources, source_paths: source_paths) if supports_source_paths
+
+          compressor.compress_sources(sources)
+        end
+
+        def build_source_metadata(sources, resolved_inputs)
+          identities = Array(resolved_inputs).each_with_object({}) do |entry, hash|
+            hash[File.expand_path(entry.fetch(:content_path))] = entry
+          end
+
+          Array(sources).map do |source|
+            expanded = File.expand_path(source)
+            resolved = identities[expanded]
+            if resolved
+              {
+                content_path: expanded,
+                source_path: resolved.fetch(:source_path),
+                source_kind: resolved.fetch(:source_kind)
+              }
+            else
+              {
+                content_path: expanded,
+                source_path: expanded,
+                source_kind: "file"
+              }
+            end
+          end
+        end
+
+        def format_console_output(content:, cache_hit:, output_path:, metadata:)
+          case @format
+          when "stdio"
+            content
+          when "stats"
+            @cache_store.stats_block(mode: @mode, cache_hit: cache_hit, output_path: output_path, metadata: metadata)
+          else
+            output_path
+          end
+        end
+
+        def cache_metadata(manifest, short_key, content, existing_metadata = nil)
+          base = existing_metadata || {}
+          base.merge(
+            "schema" => Ace::Compressor::Models::ContextPack::SCHEMA,
+            "mode" => @mode,
+            "key" => manifest["key"],
+            "short_key" => short_key,
+            "sources" => manifest["sources"],
+            "file_count" => manifest["sources"].size,
+            "original_bytes" => manifest["original_bytes"],
+            "original_lines" => manifest["original_lines"],
+            "packed_bytes" => content.bytesize,
+            "packed_lines" => content.lines.count
+          )
+        end
+
+        def hydrate_metadata(existing_metadata, manifest, canonical, content)
+          metadata = cache_metadata(manifest, canonical[:short_key], content, existing_metadata)
+          return metadata if metadata == existing_metadata
+
+          @cache_store.write_cache(
+            pack_path: canonical[:pack_path],
+            metadata_path: canonical[:metadata_path],
+            content: content,
+            metadata: metadata
+          )
+          metadata
+        end
+
+        def compressor_for_mode(paths)
+          return CompactCompressor.new(paths, verbose: @verbose) if @mode == "compact"
+          return AgentCompressor.new(paths, verbose: @verbose) if @mode == "agent"
+
+          ExactCompressor.new(paths, verbose: @verbose, mode_label: @mode)
+        end
+
+        def refusal_lines(content)
+          content.to_s.lines.map(&:strip).select { |line| line.start_with?("REFUSAL|") }
+        end
+
+        def fallback_lines(content)
+          content.to_s.lines.map(&:strip).select { |line| line.start_with?("FALLBACK|") }
+        end
+
+        def shared_cache_hit?(shared_canonical)
+          return false unless shared_canonical
+
+          @cache_store.cache_hit?(pack_path: shared_canonical[:pack_path], metadata_path: shared_canonical[:metadata_path])
+        end
+
+        def hydrate_from_shared_cache(manifest, canonical, shared_canonical)
+          content = @cache_store.read_pack(shared_canonical[:pack_path])
+          shared_metadata = @cache_store.read_metadata(shared_canonical[:metadata_path])
+          metadata = cache_metadata(manifest, canonical[:short_key], content, shared_metadata.merge("cache_scope" => "shared"))
+          @cache_store.write_cache(
+            pack_path: canonical[:pack_path],
+            metadata_path: canonical[:metadata_path],
+            content: content,
+            metadata: metadata
+          )
+          [content, metadata]
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/organisms/exact_compressor.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "find"
+require "pathname"
+
+module Ace
+  module Compressor
+    module Organisms
+      class ExactCompressor
+        SUPPORTED_EXTENSIONS = %w[.md .markdown .mdown .mkd .txt .text].freeze
+        BINARY_SNIFF_BYTES = 4096
+
+        attr_reader :ignored_paths
+
+        def initialize(paths, verbose: false, mode_label: "exact")
+          @paths = Array(paths)
+          @verbose = verbose
+          @mode_label = mode_label
+          @parser = Ace::Compressor::Atoms::MarkdownParser.new
+          @transformer = Ace::Compressor::Atoms::CanonicalBlockTransformer
+          @ignored_paths = []
+        end
+
+        def call
+          compress_sources(resolve_sources)
+        end
+
+        def resolve_sources
+          explicit_set = Set.new
+          resolved = []
+
+          @paths.each do |raw_path|
+            path = raw_path.to_s
+            expanded = File.expand_path(path)
+
+            if File.directory?(expanded)
+              directory_files = collect_supported_directory_files(expanded)
+              if directory_files.empty?
+                raise Ace::Compressor::Error,
+                  "Directory has no supported markdown/text sources: #{path}. Supported extensions: #{SUPPORTED_EXTENSIONS.join(", ")}"
+              end
+              directory_files.each do |file|
+                next if explicit_set.include?(file)
+
+                explicit_set << file
+                resolved << file
+              end
+            elsif File.file?(expanded)
+              validate_explicit_file!(expanded, path)
+              next if explicit_set.include?(expanded)
+
+              explicit_set << expanded
+              resolved << expanded
+            else
+              raise Ace::Compressor::Error, "Input source not found: #{path}"
+            end
+          end
+
+          sorted = resolved.sort
+          if sorted.empty?
+            raise Ace::Compressor::Error,
+              "No supported markdown/text sources found. Supported extensions: #{SUPPORTED_EXTENSIONS.join(", ")}"
+          end
+
+          sorted
+        end
+
+        def compress_sources(sources, source_paths: nil)
+          lines = [Ace::Compressor::Models::ContextPack.header("exact")]
+
+          sources.each do |source|
+            source_label = source_label(display_source(source, source_paths))
+            lines << Ace::Compressor::Models::ContextPack.file_line(source_label)
+            text = File.read(source)
+            if text.strip.empty?
+              raise Ace::Compressor::Error, "Input file is empty. #{mode_title} mode requires content: #{source}"
+            end
+            blocks = @parser.call(text)
+            if blocks.empty?
+              lines << text
+              next
+            end
+            lines.concat transformed_lines(source_label, blocks)
+          end
+
+          lines.join("\n")
+        end
+
+        # Compress a content string directly without filesystem access.
+        # Returns the compressed ContextPack text (without header line).
+        # @param text [String] markdown/text content to compress
+        # @param label [String] display label for the source (e.g. original file path)
+        # @return [String] compressed ContextPack records (no header)
+        def compress_text(text, label:)
+          return text if text.to_s.strip.empty?
+
+          blocks = @parser.call(text)
+          return text if blocks.empty?
+
+          lines = []
+          lines << Ace::Compressor::Models::ContextPack.file_line(label)
+          lines.concat @transformer.new(label).call(blocks)
+          lines.join("\n")
+        end
+
+        private
+
+        def transformed_lines(source, blocks)
+          transformer = @transformer.new(source)
+          transformer.call(blocks)
+        end
+
+        def display_source(source, source_paths)
+          return source unless source_paths
+
+          source_paths[File.expand_path(source)] || source_paths[source] || source
+        end
+
+        def collect_supported_directory_files(directory)
+          supported = []
+
+          Find.find(directory) do |entry|
+            next unless File.file?(entry)
+
+            expanded = File.expand_path(entry)
+            if supported_extension?(entry)
+              if binary_file?(expanded)
+                @ignored_paths << expanded if @verbose
+              else
+                supported << expanded
+              end
+            elsif @verbose
+              @ignored_paths << expanded
+            end
+          end
+
+          supported.sort
+        end
+
+        def validate_explicit_file!(expanded, original)
+          unless supported_extension?(expanded)
+            if binary_file?(expanded)
+              raise Ace::Compressor::Error, "Binary input is not supported in #{mode_label} mode: #{original}"
+            end
+
+            raise Ace::Compressor::Error,
+              "Unsupported explicit file: #{original}. Supported extensions: #{SUPPORTED_EXTENSIONS.join(", ")}"
+          end
+
+          return unless binary_file?(expanded)
+
+          raise Ace::Compressor::Error, "Binary input is not supported in #{mode_label} mode: #{original}"
+        end
+
+        def supported_extension?(path)
+          SUPPORTED_EXTENSIONS.include?(File.extname(path).downcase)
+        end
+
+        def binary_file?(path)
+          sample = File.binread(path, BINARY_SNIFF_BYTES) || ""
+          sample.include?("\x00")
+        rescue
+          false
+        end
+
+        def source_label(source)
+          pathname = Pathname.new(source)
+          project_root = Pathname.new(Dir.pwd)
+          relative = pathname.relative_path_from(project_root).to_s
+          return relative unless relative.start_with?("..")
+
+          source
+        rescue ArgumentError
+          source
+        end
+
+        def mode_label
+          @mode_label.to_s.strip.empty? ? "exact" : @mode_label
+        end
+
+        def mode_title
+          mode_label.capitalize
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    VERSION = "0.24.1"
+  end
+end

data/lib/ace/compressor.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require "yaml"
+require "ace/support/config"
+
+require_relative "compressor/version"
+require_relative "compressor/models/context_pack"
+require_relative "compressor/atoms/markdown_parser"
+require_relative "compressor/atoms/canonical_block_transformer"
+require_relative "compressor/atoms/compact_policy_classifier"
+require_relative "compressor/atoms/retention_reporter"
+require_relative "compressor/molecules/cache_store"
+require_relative "compressor/molecules/input_resolver"
+require_relative "compressor/organisms/exact_compressor"
+require_relative "compressor/organisms/compact_compressor"
+require_relative "compressor/organisms/agent_compressor"
+require_relative "compressor/organisms/compression_runner"
+require_relative "compressor/organisms/benchmark_runner"
+require_relative "compressor/cli"
+
+module Ace
+  module Compressor
+    class Error < StandardError; end
+
+    # Compress a content string directly (Ruby API for other gems).
+    # @param text [String] markdown/text content
+    # @param label [String] display label (original file path)
+    # @param mode [String] compression mode ("exact" or "agent")
+    # @return [String] compressed ContextPack records
+    def self.compress_text(text, label:, mode: "exact")
+      case mode
+      when "exact"
+        compressor = Organisms::ExactCompressor.new([], mode_label: mode)
+        compressor.compress_text(text, label: label)
+      when "agent"
+        compress_text_via_file(text, label: label, mode: mode)
+      else
+        raise Error, "compress_text only supports exact and agent modes, got: #{mode}"
+      end
+    end
+
+    def self.config
+      @config ||= begin
+        gem_root = Gem.loaded_specs["ace-compressor"]&.gem_dir ||
+          File.expand_path("../..", __dir__)
+
+        resolver = Ace::Support::Config.create(
+          config_dir: ".ace",
+          defaults_dir: ".ace-defaults",
+          gem_path: gem_root
+        )
+
+        config = resolver.resolve_namespace("compressor").to_h
+        config.empty? ? load_gem_defaults_fallback(gem_root) : config
+      end
+    end
+
+    def self.reset_config!
+      @config = nil
+    end
+
+    def self.load_gem_defaults_fallback(gem_root = nil)
+      root = gem_root || Gem.loaded_specs["ace-compressor"]&.gem_dir ||
+        File.expand_path("../..", __dir__)
+      defaults_path = File.join(root, ".ace-defaults", "compressor", "config.yml")
+      return {} unless File.exist?(defaults_path)
+
+      YAML.safe_load_file(defaults_path, aliases: true) || {}
+    rescue
+      {}
+    end
+
+    def self.compress_text_via_file(text, label:, mode:)
+      return text if text.to_s.strip.empty?
+
+      Dir.mktmpdir("ace_compressor_text") do |tmpdir|
+        source = File.join(tmpdir, label.to_s)
+        FileUtils.mkdir_p(File.dirname(source))
+        File.write(source, text)
+
+        compressor = build_text_compressor(mode, source)
+        output = compressor.compress_sources([source])
+        strip_context_pack_header(output)
+      end
+    end
+    private_class_method :compress_text_via_file
+
+    def self.build_text_compressor(mode, source)
+      case mode
+      when "agent"
+        Organisms::AgentCompressor.new([source])
+      else
+        raise Error, "Unsupported text compressor mode: #{mode}"
+      end
+    end
+    private_class_method :build_text_compressor
+
+    def self.strip_context_pack_header(output)
+      lines = output.to_s.lines
+      lines.shift if lines.first&.start_with?("H|ContextPack/")
+      lines.join.strip
+    end
+    private_class_method :strip_context_pack_header
+
+    private_class_method :load_gem_defaults_fallback
+  end
+end