ace-compressor 0.24.1

data/lib/ace/compressor/atoms/canonical_block_transformer.rb

@@ -0,0 +1,341 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Atoms
+      class CanonicalBlockTransformer
+        RULE_RE = /\b(?:must|must not|never|required|required to|should|shall|shall not|cannot|can't|do not)\b/i
+        CONSTRAINT_RE = /\b(?:constraint|no more than|at most|must not|never|cannot)\b/i
+        RULE_SECTION_RE = /(rules?|guidelines?|policy|requirements?|constraints?|musts?)/i
+        SUMMARY_SECTION_RE = /(overview|summary|vision|purpose|goal|why|motivation|introduction|intro)/i
+        EXAMPLE_HEADING_RE = /\Aexample\s*:\s*(.+)\z/i
+        PROBLEM_SECTION_RE = /(problems?|issues?|risks?|pitfalls?|drawbacks?)/i
+        PROBLEM_CONTEXT_RE = /\b(?:suffer from|struggle with|problems?|issues?|risks?|pitfalls?|drawbacks?|pain points?)\b/i
+        TREE_LINE_RE = /[│├└╰]--/
+        FILE_PATH_RE = /\A(?:\.{1,2}\/)?[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\z/
+        SHELL_LANGS = %w[bash sh shell zsh fish cmd powershell ps1].freeze
+        SHELL_CONTROL_RE = /\A(?:#|if\b|then\b|else\b|elif\b|fi\b|for\b|while\b|until\b|do\b|done\b|case\b|esac\b|function\b|\{|\})/
+        CONTEXTPACK_PREFIXES = %w[
+          H FILE POLICY FIDELITY REFUSAL GUIDANCE FALLBACK SEC SUMMARY FACT RULE CONSTRAINT
+          PROBLEMS LIST EXAMPLE CMD FILES TREE CODE TABLE LOSS EXAMPLE_REF U
+        ].freeze
+        LIST_NARRATIVE_MIN_CHARS = 48
+        LIST_NARRATIVE_MIN_WORDS = 6
+        LIST_STOPWORDS = %w[a an and as at by for from in into is of on or that the this to via with within].freeze
+        LIST_TOKEN_MAP = {
+          "architecture" => "arch",
+          "architectural" => "arch",
+          "configuration" => "config",
+          "documentation" => "docs",
+          "generation" => "gen",
+          "management" => "mgmt",
+          "repository" => "repo",
+          "repositories" => "repos",
+          "development" => "dev",
+          "integration" => "integr",
+          "execution" => "exec",
+          "reporting" => "reports",
+          "organization" => "org",
+          "organizations" => "orgs",
+          "capabilities" => "caps",
+          "capability" => "cap",
+          "foundation" => "base",
+          "tracking" => "track",
+          "powered" => "pwr",
+          "detected" => "detect",
+          "matching" => "match"
+        }.freeze
+
+        def initialize(source)
+          @source = source
+          @current_section = nil
+          @example_tool = nil
+          @last_text = nil
+        end
+
+        def call(blocks)
+          lines = []
+
+          Array(blocks).each do |block|
+            next unless block.is_a?(Hash) && block[:type]
+
+            case block[:type]
+            when :heading
+              lines << section_line(block)
+            when :text
+              lines << text_record(block[:text])
+            when :list
+              lines.concat list_lines(block)
+            when :fenced_code
+              lines.concat fenced_code_lines(block)
+            when :table
+              table = table_line(block)
+              lines << table if table
+            when :unresolved
+              lines << unresolved_line(block)
+            end
+          end
+
+          lines.compact
+        end
+
+        private
+
+        def section_line(block)
+          heading = normalize_heading_text(normalize_inline(block[:text].to_s))
+          return nil if heading.empty?
+
+          match = heading.match(EXAMPLE_HEADING_RE)
+          if match
+            tool = heading_tool_slug(match[1].to_s)
+            @example_tool = tool
+            @last_text = nil
+            "EXAMPLE|tool=#{tool}"
+          else
+            @example_tool = nil
+            @current_section = heading_slug(heading)
+            @last_text = nil
+            Ace::Compressor::Models::ContextPack.section_line(@current_section)
+          end
+        end
+
+        def text_record(raw_text)
+          text = normalize_inline(raw_text.to_s)
+          return nil if text.empty?
+
+          example_match = text.match(EXAMPLE_HEADING_RE)
+          if example_match
+            @example_tool = heading_tool_slug(example_match[1].to_s)
+            @last_text = text
+            return Ace::Compressor::Models::ContextPack.example_line(@example_tool)
+          end
+
+          kind =
+            if text_summary?
+              :summary
+            elsif RULE_RE.match?(text)
+              :rule
+            elsif CONSTRAINT_RE.match?(text)
+              :constraint
+            elsif RULE_SECTION_RE.match?(@current_section.to_s)
+              :rule
+            else
+              :fact
+            end
+
+          @last_text = text
+
+          case kind
+          when :summary
+            Ace::Compressor::Models::ContextPack.summary_line(text)
+          when :constraint
+            Ace::Compressor::Models::ContextPack.constraint_line(text)
+          when :rule
+            Ace::Compressor::Models::ContextPack.rule_line(text)
+          else
+            Ace::Compressor::Models::ContextPack.fact_line(text)
+          end
+        end
+
+        def list_lines(block)
+          items = Array(block[:items]).map { |item| list_item_slug(item.to_s) }.reject(&:empty?)
+          return [] if items.empty?
+
+          if problem_list_context?
+            [Ace::Compressor::Models::ContextPack.problems_line(items)]
+          else
+            list_key = @current_section.to_s.empty? ? "items" : @current_section
+            [Ace::Compressor::Models::ContextPack.list_line(list_key, items)]
+          end
+        end
+
+        def fenced_code_lines(block)
+          code_lines = Array(block[:content].to_s.lines).map(&:strip).reject(&:empty?)
+          return [] if code_lines.empty?
+
+          nested_lines = nested_contextpack_lines(code_lines)
+          return nested_lines if nested_lines
+
+          language = block[:language].to_s.strip.downcase
+          if file_list_block?(code_lines)
+            [Ace::Compressor::Models::ContextPack.files_line(file_label, code_lines)]
+          elsif tree_block?(code_lines)
+            [Ace::Compressor::Models::ContextPack.tree_line(tree_label, code_lines.join(" "))]
+          elsif shell_script_block?(language, code_lines)
+            [Ace::Compressor::Models::ContextPack.code_line(language.empty? ? "bash" : language, code_lines.join(" "))]
+          elsif shell_command_block?(language, code_lines)
+            code_lines.map { |line| Ace::Compressor::Models::ContextPack.cmd_line(line) }
+          else
+            [Ace::Compressor::Models::ContextPack.code_line(language.empty? ? "code" : language, code_lines.join(" "))]
+          end
+        end
+
+        def table_line(block)
+          rows = Array(block[:rows])
+          return nil if rows.empty?
+
+          Ace::Compressor::Models::ContextPack.table_line(rows)
+        end
+
+        def unresolved_line(block)
+          kind = block[:kind].to_s
+          raw = block[:text].to_s
+          Ace::Compressor::Models::ContextPack.unresolved_line(kind, raw)
+        end
+
+        def text_summary?
+          return false if @current_section.to_s.empty?
+
+          SUMMARY_SECTION_RE.match?(@current_section.to_s)
+        end
+
+        def shell_command_block?(language, lines)
+          return false if shell_script_block?(language, lines)
+          return true if SHELL_LANGS.include?(language)
+          return false unless language.empty?
+
+          lines.all? { |line| line.match?(/\A[a-zA-Z0-9_.\/-]+(?:\s+.+)?\z/) }
+        end
+
+        def shell_script_block?(language, lines)
+          return false if lines.empty?
+
+          shell_language = SHELL_LANGS.include?(language)
+          inferred_shell = language.empty? && lines.all? { |line| shellish_line?(line) }
+          return false unless shell_language || inferred_shell
+
+          lines.size > 3 || lines.any? { |line| shell_script_line?(line) }
+        end
+
+        def tree_block?(lines)
+          lines.any? { |line| TREE_LINE_RE.match?(line) || line.match?(/\A[|` ]*[├└]──/) }
+        end
+
+        def file_list_block?(lines)
+          return false if lines.empty?
+
+          lines.all? { |line| FILE_PATH_RE.match?(line) }
+        end
+
+        def nested_contextpack_lines(lines)
+          payload = Array(lines).reject { |line| contextpack_header_line?(line) }
+          return nil if payload.empty?
+          return nil unless payload.all? { |line| contextpack_line?(line) }
+
+          payload
+        end
+
+        def contextpack_line?(line)
+          prefix = line.to_s.split("|", 2).first
+          CONTEXTPACK_PREFIXES.include?(prefix)
+        end
+
+        def contextpack_header_line?(line)
+          line.to_s.start_with?("H|ContextPack/")
+        end
+
+        def tree_label
+          return @example_tool unless @example_tool.to_s.empty?
+          return @current_section unless @current_section.to_s.empty?
+
+          File.basename(@source).sub(/\.[^.]+\z/, "")
+        end
+
+        def file_label
+          return @example_tool unless @example_tool.to_s.empty?
+          return @current_section unless @current_section.to_s.empty?
+
+          "files"
+        end
+
+        def section_contains_problems?(section)
+          PROBLEM_SECTION_RE.match?(section.to_s.tr("_", " "))
+        end
+
+        def problem_list_context?
+          section_contains_problems?(@current_section) || PROBLEM_CONTEXT_RE.match?(@last_text.to_s)
+        end
+
+        def heading_slug(text)
+          normalize_heading_text(text)
+            .downcase
+            .gsub(/["']/, "")
+            .gsub(/\P{Alnum}+/, "_").squeeze("_")
+            .sub(/\A_+/, "")
+            .sub(/_+\z/, "")
+            .then { |value| value.empty? ? "section" : value }
+        end
+
+        def heading_tool_slug(text)
+          normalize_heading_text(text)
+            .downcase
+            .gsub(/["']/, "")
+            .gsub(/\P{Alnum}+/, "-").squeeze("-")
+            .sub(/\A-+/, "")
+            .sub(/-+\z/, "")
+            .then { |value| value.empty? ? "tool" : value }
+        end
+
+        def list_item_slug(text)
+          return heading_slug(text) unless narrative_list_item?(text)
+
+          compact_phrase_slug(text)
+        end
+
+        def narrative_list_item?(text)
+          raw = normalize_inline(text.to_s)
+          raw.length > LIST_NARRATIVE_MIN_CHARS || raw.split(/\s+/).length >= LIST_NARRATIVE_MIN_WORDS
+        end
+
+        def compact_phrase_slug(text)
+          tokens = normalize_heading_text(normalize_inline(text.to_s))
+            .downcase
+            .gsub(/["']/, "")
+            .scan(/[a-z0-9]+/)
+            .filter_map do |token|
+              next if LIST_STOPWORDS.include?(token)
+
+              LIST_TOKEN_MAP.fetch(token, token)
+            end
+
+          tokens = tokens.each_with_object([]) do |token, result|
+            result << token unless result.last == token
+          end
+
+          value = tokens.join("_")
+          value.empty? ? heading_slug(text) : value
+        end
+
+        def shellish_line?(line)
+          line.match?(/\A[a-zA-Z0-9_.\/\-$"'`#\[\]()=:;]+(?:\s+.+)?\z/)
+        end
+
+        def shell_script_line?(line)
+          line.match?(SHELL_CONTROL_RE)
+        end
+
+        def normalize_heading_text(text)
+          text
+            .to_s
+            .strip
+            .sub(/\A[#\s]+/, "")
+            .sub(/\A\d+(?:[.)])?\s*/, "")
+            .gsub(/^\p{Extended_Pictographic}+\s*/, "")
+        end
+
+        def normalize_inline(text)
+          without_links = text.gsub(/\[([^\]]+)\]\([^)]+\)/, "\\1")
+          without_emoji = without_links.gsub(emoji_re, "")
+          without_bold = without_emoji.gsub(/\*{1,3}([^*]+)\*{1,3}/, "\\1")
+          without_backticks = without_bold.gsub(/`([^`]+)`/, "\\1")
+          without_blockquote = without_backticks.gsub(/^(?:\s*>+\s*)+/, "")
+          without_blockquote.gsub(/\s+/, " ").strip
+        end
+
+        def emoji_re
+          /[\u{1F300}-\u{1FAFF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]+/u
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/atoms/compact_policy_classifier.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Atoms
+      # Classifies a source into compact-mode policy categories so the caller
+      # can choose the safest compression strategy for that document.
+      class CompactPolicyClassifier
+        # Narrative file-name hints commonly used for explanatory docs.
+        NARRATIVE_FILE_HINT_RE = /(?:^|\/)(?:readme|vision|guide|guides|architecture)(?:\.|\/|$)/i
+        # Headings that signal descriptive/explanatory prose.
+        NARRATIVE_HEADING_RE = /\b(?:overview|vision|introduction|core principles|why|how it works|purpose|motivation|summary|guide)\b/i
+        # Headings that indicate normative policy or constraints.
+        RULE_HEADING_RE = /\b(?:decision|impact|policy|rule|rules|requirement|requirements|constraint|constraints)\b/i
+        # Modal language that usually indicates must-follow rules.
+        RULE_TEXT_RE = /\b(?:must|must not|never|required|requires|should|shall|cannot|can't|do not|only)\b/i
+
+        def call(source:, blocks:)
+          stats = signal_stats(source, blocks)
+          doc_class = classify(stats)
+          action = action_for(doc_class)
+          {
+            "class" => doc_class,
+            "action" => action
+          }
+        end
+
+        private
+
+        def classify(stats)
+          return "rule-heavy" if rule_heavy?(stats)
+          return "mixed" if mixed?(stats)
+          return "narrative-heavy" if narrative_heavy?(stats)
+
+          "unknown"
+        end
+
+        def action_for(doc_class)
+          case doc_class
+          when "narrative-heavy"
+            "aggressive_compact"
+          when "mixed"
+            "compact_with_exact_rule_sections"
+          when "rule-heavy"
+            "refuse_compact"
+          else
+            "conservative_compact"
+          end
+        end
+
+        def signal_stats(source, blocks)
+          source_text = source.to_s
+          block_list = Array(blocks)
+          heading_hits = block_list.count { |block| narrative_heading?(block) }
+          text_blocks = block_list.count { |block| block[:type] == :text }
+          list_blocks = block_list.count { |block| block[:type] == :list }
+          file_hint = source_text.match?(NARRATIVE_FILE_HINT_RE)
+          rule_heading_hits = block_list.count { |block| rule_heading?(block) }
+          rule_text_hits = block_list.count { |block| rule_text?(block) }
+          rule_list_hits = block_list.sum { |block| rule_list_hits(block) }
+
+          {
+            heading_hits: heading_hits,
+            text_blocks: text_blocks,
+            list_blocks: list_blocks,
+            file_hint: file_hint,
+            rule_heading_hits: rule_heading_hits,
+            rule_signal_count: rule_heading_hits + rule_text_hits + rule_list_hits
+          }
+        end
+
+        def narrative_heavy?(stats)
+          heading_hits = stats.fetch(:heading_hits)
+          text_blocks = stats.fetch(:text_blocks)
+          list_blocks = stats.fetch(:list_blocks)
+          file_hint = stats.fetch(:file_hint)
+
+          return true if file_hint && text_blocks >= 2 && text_blocks >= list_blocks
+          return true if heading_hits >= 2 && text_blocks >= 2 && text_blocks >= list_blocks
+
+          false
+        end
+
+        def mixed?(stats)
+          rule_signal_count = stats.fetch(:rule_signal_count)
+          return false if rule_signal_count < 2
+          return false if rule_heavy?(stats)
+          return false if stats.fetch(:file_hint) && stats.fetch(:rule_heading_hits).zero?
+
+          narrative_heavy?(stats) || stats.fetch(:heading_hits) >= 1 || stats.fetch(:text_blocks) >= 2
+        end
+
+        def rule_heavy?(stats)
+          rule_signal_count = stats.fetch(:rule_signal_count)
+          rule_heading_hits = stats.fetch(:rule_heading_hits)
+          narrative_signals = stats.fetch(:heading_hits) + (stats.fetch(:file_hint) ? 1 : 0)
+
+          return true if rule_signal_count >= 6 && narrative_signals <= 1
+          return true if rule_signal_count >= 5 && rule_heading_hits >= 2
+
+          false
+        end
+
+        def narrative_heading?(block)
+          return false unless block[:type] == :heading
+
+          NARRATIVE_HEADING_RE.match?(block[:text].to_s)
+        end
+
+        def rule_heading?(block)
+          return false unless block[:type] == :heading
+
+          RULE_HEADING_RE.match?(block[:text].to_s)
+        end
+
+        def rule_text?(block)
+          return false unless block[:type] == :text
+
+          RULE_TEXT_RE.match?(block[:text].to_s)
+        end
+
+        def rule_list_hits(block)
+          return 0 unless block[:type] == :list
+
+          Array(block[:items]).count { |item| RULE_TEXT_RE.match?(item.to_s) }
+        end
+      end
+    end
+  end
+end

data/lib/ace/compressor/atoms/markdown_parser.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Ace
+  module Compressor
+    module Atoms
+      class MarkdownParser
+        HEADING_RE = /\A(#+)\s+(.+)\z/
+        BULLET_LIST_RE = /\A(?:\s*)[-*+]\s+(.+)\z/
+        NUMBERED_LIST_RE = /\A(?:\s*)\d+\.\s+(.+)\z/
+        IMAGE_ONLY_RE = /\A!\[[^\]]*\]\([^)]+\)\z/
+        TABLE_SEPARATOR_RE = /\A\|?[-\s:|]+\|?\z/
+        FENCE_START_RE = /\A```/
+
+        def call(text)
+          body = strip_frontmatter(text.to_s)
+          return [] if body.strip.empty?
+
+          blocks = []
+          paragraph_lines = []
+          lines = body.lines
+          index = 0
+          while index < lines.length
+            stripped = lines[index].strip
+
+            if stripped.empty?
+              flush_paragraph(blocks, paragraph_lines)
+              index += 1
+              next
+            end
+
+            if layout_separator?(stripped)
+              flush_paragraph(blocks, paragraph_lines)
+              index += 1
+              next
+            end
+
+            if blockquote_marker?(stripped)
+              flush_paragraph(blocks, paragraph_lines)
+              index += 1
+              next
+            end
+
+            if stripped.match?(FENCE_START_RE)
+              flush_paragraph(blocks, paragraph_lines)
+              language = stripped.sub(FENCE_START_RE, "").strip
+              fence_lines = []
+              index += 1
+              while index < lines.length
+                candidate = lines[index]
+                break if candidate.strip.match?(FENCE_START_RE)
+
+                fence_lines << candidate
+                index += 1
+              end
+              index += 1 if index < lines.length && lines[index]&.strip&.match?(FENCE_START_RE)
+              blocks << {
+                type: :fenced_code,
+                language: language,
+                content: fence_lines.join
+              }
+              next
+            end
+
+            if image_only_line?(stripped)
+              flush_paragraph(blocks, paragraph_lines)
+              blocks << {
+                type: :unresolved,
+                text: stripped,
+                kind: "image-only"
+              }
+              index += 1
+              next
+            end
+
+            if table_start?(lines, index)
+              flush_paragraph(blocks, paragraph_lines)
+              table_rows = []
+              while index < lines.length
+                candidate = lines[index]
+                break if candidate.strip.empty?
+                break unless candidate.include?("|")
+
+                table_rows << candidate.strip
+                index += 1
+              end
+              blocks << {type: :table, rows: table_rows}
+              next
+            end
+
+            if list_start?(stripped)
+              flush_paragraph(blocks, paragraph_lines)
+              list_items = []
+              ordered = false
+              while index < lines.length
+                item_line = lines[index].strip
+                break unless list_start?(item_line)
+
+                ordered = true if ordered_list_line?(item_line)
+                list_items << strip_list_marker(item_line)
+                index += 1
+              end
+              blocks << {
+                type: :list,
+                ordered: ordered,
+                items: list_items
+              }
+              next
+            end
+
+            heading_match = stripped.match(HEADING_RE)
+            if heading_match && heading_match[1].length <= 6
+              flush_paragraph(blocks, paragraph_lines)
+              blocks << {
+                type: :heading,
+                level: heading_match[1].length,
+                text: heading_match[2].strip
+              }
+            else
+              paragraph_lines << stripped
+            end
+
+            index += 1
+          end
+
+          flush_paragraph(blocks, paragraph_lines)
+          blocks
+        end
+
+        private
+
+        def strip_frontmatter(text)
+          return text unless text.start_with?("---\n", "---\r\n")
+
+          lines = text.lines
+          end_index = nil
+
+          lines[1..].each_with_index do |line, idx|
+            if line.strip == "---"
+              end_index = idx + 1
+              break
+            end
+          end
+
+          return text if end_index.nil?
+
+          lines[(end_index + 1)..]&.join.to_s
+        end
+
+        def flush_paragraph(blocks, lines)
+          return if lines.empty?
+
+          blocks << {type: :text, text: lines.join(" ")}
+          lines.clear
+        end
+
+        def layout_separator?(line)
+          line.match?(/\A(?:[-*_]){3,}\z/)
+        end
+
+        def image_only_line?(line)
+          line.match?(IMAGE_ONLY_RE)
+        end
+
+        def blockquote_marker?(line)
+          line.match?(/^>+$/)
+        end
+
+        def list_start?(line)
+          line.match?(BULLET_LIST_RE) || line.match?(NUMBERED_LIST_RE)
+        end
+
+        def ordered_list_line?(line)
+          line.match?(NUMBERED_LIST_RE)
+        end
+
+        def strip_list_marker(line)
+          line.sub(BULLET_LIST_RE, "\\1").sub(NUMBERED_LIST_RE, "\\1").strip
+        end
+
+        def table_start?(lines, index)
+          current = lines[index]&.strip.to_s
+          next_line = lines[index + 1]&.strip.to_s
+          return false unless current.include?("|")
+
+          next_line.match?(TABLE_SEPARATOR_RE)
+        end
+      end
+    end
+  end
+end