exhale 0.1.0

data/lib/exhale/contract/resolver.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require_relative "../unit"
+require_relative "../errors"
+require_relative "reference"
+
+module Exhale
+  class Contract
+    # Maps Contract references onto code units. Units are indexed once (by
+    # identity and by every namespace prefix) so each reference costs a hash
+    # lookup instead of a scan.
+    class Resolver
+      def initialize(contract, units)
+        @contract = contract
+        @units = units
+        @order = {}.compare_by_identity
+        units.each_with_index { |u, i| @order[u] = i }
+        @matches = {}
+        build_indexes
+      end
+
+      def errors
+        @errors ||= all_references.sort_by { |r| [r.path, r.line] }.filter_map do |ref|
+          next unless units_for(ref).empty?
+
+          ContractError.new(ref.path, ref.line, "reference names no unit: #{ref.text}")
+        end
+      end
+
+      def units_for(reference)
+        @units_for ||= {}
+        @units_for[reference.text] ||= matches(reference).keys.sort_by { |u| @order[u] }
+      end
+
+      def keeping_clause(unit_a, unit_b)
+        return nil if unit_a.equal?(unit_b)
+
+        @contract.clauses.find { |clause| keeps?(clause, unit_a, unit_b) }
+      end
+
+      def primitive_for(unit)
+        pairs = covering(unit)
+        top = pairs.map { |_, ref| ref.specificity }.max
+        pairs.select { |_, ref| ref.specificity == top }.map(&:first).min_by(&:name)
+      end
+
+      def settings_for(unit, defaults)
+        prim = primitive_for(unit)
+        defaults.merge(prim ? @contract.settings.fetch(prim.name, {}) : {})
+      end
+
+      def settings_for_pair(unit_a, unit_b, defaults)
+        a = settings_for(unit_a, defaults)
+        b = settings_for(unit_b, defaults)
+        a.merge(b) { |_key, x, y| [x, y].min }
+      end
+
+      private
+
+      def build_indexes
+        @by_identity = Hash.new { |h, k| h[k] = [] }
+        @by_prefix = Hash.new { |h, k| h[k] = [] }
+        @templates = []
+        @units.each { |unit| index(unit) }
+      end
+
+      def index(unit)
+        @templates << unit if unit.kind == :template
+        @by_identity[unit.identity] << unit if %i[method dsl].include?(unit.kind)
+        return unless unit.namespace
+
+        segments = unit.namespace.split("::")
+        (1..segments.size).each { |n| @by_prefix[segments.first(n).join("::")] << unit }
+      end
+
+      def all_references
+        @contract.primitives.flat_map(&:covers) + @contract.clauses.flat_map(&:references)
+      end
+
+      # {unit => match key}, cached per reference text.
+      def matches(ref)
+        @matches[ref.text] ||= case ref.kind
+                               when :method then method_matches(ref)
+                               when :constant then constant_matches(ref)
+                               when :const_glob then const_glob_matches(ref)
+                               else template_matches(ref)
+                               end
+      end
+
+      def method_matches(ref)
+        @by_identity.fetch(ref.text, []).to_h { |u| [u, ref.text] }.compare_by_identity
+      end
+
+      def constant_matches(ref)
+        @by_prefix.fetch(ref.text, []).to_h { |u| [u, ref.text] }.compare_by_identity
+      end
+
+      # A unit under several matching prefixes keys on the longest one.
+      def const_glob_matches(ref)
+        result = {}.compare_by_identity
+        @by_prefix.each do |prefix, units|
+          next unless ref.const_glob_match?(prefix.split("::"))
+
+          units.each { |u| result[u] = prefix if !result.key?(u) || result[u].length < prefix.length }
+        end
+        result
+      end
+
+      def template_matches(ref)
+        regex = ref.template_regex
+        @templates.select { |u| regex.match?(u.identity) }.to_h { |u| [u, u.identity] }.compare_by_identity
+      end
+
+      def keeps?(clause, unit_a, unit_b)
+        a = assignment(clause, unit_a)
+        b = assignment(clause, unit_b)
+        a && b && a != b
+      end
+
+      # The one reference (and match key) a unit belongs to within a clause:
+      # the most specific of those that match it.
+      def assignment(clause, unit)
+        hits = clause.references.uniq(&:text).filter_map { |r| (k = matches(r)[unit]) && [r, k] }
+        ref, key = hits.max_by { |r, _| r.specificity }
+        ref && [ref.text, key]
+      end
+
+      def covering(unit)
+        @covered_by ||= build_covered_by
+        @covered_by.fetch(unit, [])
+      end
+
+      def build_covered_by
+        map = {}.compare_by_identity
+        @contract.primitives.each do |prim|
+          prim.covers.each { |ref| matches(ref).each_key { |u| (map[u] ||= []) << [prim, ref] } }
+        end
+        map
+      end
+    end
+  end
+end

data/lib/exhale/contract.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "digest"
+require "find"
+require "pathname"
+require_relative "unit"
+require_relative "errors"
+require_relative "contract/markdown"
+require_relative "contract/reference"
+require_relative "contract/resolver"
+
+module Exhale
+  # The Contract: Markdown under <root>/contract/<primitive>/ that declares which
+  # code belongs to a primitive and which units are deliberately parallel.
+  class Contract
+    Primitive = Struct.new(:name, :path, :covers)
+    Clause = Struct.new(:primitive, :kind, :path, :line, :heading, :reason, :references, :key)
+
+    CONTENT = %w[covers parallel settings].freeze
+    SETTING_KEYS = { "threshold" => :threshold, "min-lines" => :min_lines, "min-nodes" => :min_nodes }.freeze
+
+    attr_reader :primitives, :clauses, :settings, :errors
+
+    def self.load(root, dir: "contract")
+      new(root, dir).tap(&:parse)
+    end
+
+    def initialize(root, dir)
+      @root = Pathname.new(root.to_s)
+      @dir = dir
+      @primitives = []
+      @clauses = []
+      @settings = {}
+      @errors = []
+    end
+
+    def parse
+      base = @root.join(@dir)
+      return self unless base.directory? || base.symlink?
+
+      flag_symlinks(base)
+      return self if base.symlink?
+
+      check_root_files(base)
+      base.children.select { |d| real_dir?(d) }.reject { |d| d.basename.to_s.start_with?(".") }
+          .sort_by { |d| d.basename.to_s }.each { |d| parse_primitive(d) }
+      @clauses.sort_by! { |c| [c.path, c.line] }
+      self
+    end
+
+    private
+
+    # Symlinks could point outside the repo and change the verdict without
+    # changing the commit, so every one is an error and none is followed.
+    def flag_symlinks(base)
+      Find.find(base.to_s) do |path|
+        next unless File.symlink?(path)
+
+        @errors << ContractError.new(rel(path), 1, "symlinks are not allowed in the Contract")
+        Find.prune
+      end
+    end
+
+    def real_dir?(path)
+      path.directory? && !path.symlink?
+    end
+
+    def real_file?(path)
+      path.file? && !path.symlink?
+    end
+
+    def parse_primitive(dir)
+      name = dir.basename.to_s
+      covers = []
+      readme = dir.join("README.md")
+      each_block(readme) { |block, path| readme_block(covers, block, path) } if real_file?(readme)
+      @primitives << Primitive.new(name, rel(dir), covers)
+      duplication_files(dir).each do |file|
+        each_block(file) { |block, path| duplication_block(name, block, path) }
+      end
+      stray_files(dir).each do |file|
+        each_block(file) { |block, path| misplaced(block, path, "found in a stray file; use README.md or duplication.md") if CONTENT.include?(block.info) }
+      end
+    end
+
+    def stray_files(dir)
+      known = [dir.join("README.md"), dir.join("duplication.md")].map(&:to_s)
+      markdown_files(dir).reject { |f| known.include?(f.to_s) || f.to_s.start_with?(dir.join("duplication/").to_s) }
+    end
+
+    def markdown_files(dir)
+      Dir.glob("**/*.md", base: dir.to_s).sort.map { |f| dir.join(f) }.reject { |f| f.symlink? || !f.file? }
+    end
+
+    def readme_block(covers, block, path)
+      case block.info
+      when "covers" then covers.concat(references(block, path))
+      when "parallel", "settings" then misplaced(block, path, "belongs in duplication.md")
+      end
+    end
+
+    def misplaced(block, path, message)
+      @errors << ContractError.new(path, block.line, "#{block.info} block #{message}")
+    end
+
+    def check_root_files(base)
+      base.children.select { |f| real_file?(f) }.select { |f| f.extname == ".md" }.sort_by(&:to_s).each do |file|
+        each_block(file) do |block, path|
+          misplaced(block, path, "found at the contract root; Contract content must live in a primitive directory") if CONTENT.include?(block.info)
+        end
+      end
+    end
+
+    def duplication_files(dir)
+      files = []
+      single = dir.join("duplication.md")
+      files << single if real_file?(single)
+      nested = dir.join("duplication")
+      files.concat(markdown_files(nested)) if real_dir?(nested)
+      files.sort_by(&:to_s)
+    end
+
+    def each_block(file)
+      path = rel(file)
+      text = File.read(file, encoding: "UTF-8")
+      return @errors << ContractError.new(path, 1, "file is not valid UTF-8") unless text.valid_encoding?
+
+      Markdown.blocks(text).each { |block| yield block, path }
+    end
+
+    def duplication_block(primitive, block, path)
+      case block.info
+      when "parallel" then add_clause(primitive, block, path)
+      when "settings" then add_settings(primitive, block, path)
+      when "covers" then misplaced(block, path, "belongs in README.md")
+      end
+    end
+
+    def references(block, path)
+      block.body.filter_map do |text, no|
+        ref = Markdown.strip_comment(text)
+        Reference.new(ref, path, no) unless ref.empty?
+      end
+    end
+
+    def add_clause(primitive, block, path)
+      refs = references(block, path)
+      return @errors << ContractError.new(path, block.line, "empty parallel block") if refs.empty?
+
+      key = Digest::SHA256.hexdigest("#{primitive}|parallel|#{refs.map(&:text).sort.join("\n")}")
+      @clauses << Clause.new(primitive, :parallel, path, block.line, block.heading, block.reason, refs, key)
+    end
+
+    def add_settings(primitive, block, path)
+      if @settings.key?(primitive)
+        return @errors << ContractError.new(path, block.line, "second settings block for #{primitive}")
+      end
+
+      @settings[primitive] = block.body.each_with_object({}) do |(text, no), acc|
+        line = Markdown.strip_comment(text)
+        next if line.empty?
+
+        parse_setting(line, path, no, acc)
+      end
+    end
+
+    def parse_setting(line, path, no, acc)
+      key, value = line.split(":", 2).map(&:strip)
+      sym = SETTING_KEYS[key]
+      return @errors << ContractError.new(path, no, "unknown setting: #{key}") unless sym
+
+      parsed = setting_value(sym, value)
+      return @errors << ContractError.new(path, no, "bad value for #{key}: #{value.inspect}") if parsed.nil?
+
+      return @errors << ContractError.new(path, no, "repeated setting: #{key}") if acc.key?(sym)
+
+      acc[sym] = parsed
+    end
+
+    def setting_value(sym, value)
+      if sym == :threshold
+        f = Float(value, exception: false)
+        f if f && f > 0 && f <= 1
+      else
+        i = Integer(value.to_s, 10, exception: false)
+        i if i && i >= 1
+      end
+    end
+
+    def rel(path)
+      Pathname.new(path.to_s).relative_path_from(@root).to_s
+    end
+  end
+end

data/lib/exhale/dry/check.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "securerandom"
+require "tmpdir"
+require "prism"
+require "herb"
+require_relative "../version"
+require_relative "../errors"
+require_relative "../git"
+require_relative "../source_files"
+require_relative "../units/ruby"
+require_relative "../units/erb"
+require_relative "../contract"
+require_relative "normalizer"
+require_relative "fingerprints"
+require_relative "index"
+require_relative "matcher"
+require_relative "gate"
+
+module Exhale
+  module Dry
+    DEFAULTS = { threshold: Rational(80, 100), min_lines: 4, min_nodes: 20 }.freeze
+
+    # Everything one tree yields: its units, their index, the Contract read
+    # from the same tree, and every match over the threshold.
+    class Sweep
+      attr_reader :root, :units, :parse_errors, :contract, :resolver, :index, :matches
+
+      def initialize(root, files: nil, include_tests: false, contract_dir: "contract", overrides: {})
+        @root = root
+        @files = files
+        @include_tests = include_tests
+        @contract_dir = contract_dir
+        @overrides = overrides
+      end
+
+      def run
+        @parse_errors = []
+        @units = read_units
+        @contract = Contract.load(@root, dir: @contract_dir)
+        @resolver = Contract::Resolver.new(@contract, @units)
+        @index = Index.new(entries)
+        @matches = Matcher.new(@index, floors: floors, settings_for_pair: method(:settings_for_pair)).matches
+        self
+      end
+
+      def settings_for_pair(a, b)
+        return DEFAULTS.merge(@overrides) unless @overrides.empty?
+
+        exact(@resolver.settings_for_pair(a, b, DEFAULTS))
+      end
+
+      private
+
+      # A file that can't be read, or isn't UTF-8, is a parse error like any
+      # other: the gate never passes code it didn't read.
+      def read_units
+        SourceFiles.list(@root, files: @files, include_tests: @include_tests).flat_map do |path, language|
+          source = read_source(path)
+          language == :ruby ? Units::Ruby.extract(source, path) : Units::Erb.extract(source, path)
+        rescue ParseError => e
+          @parse_errors << e
+          []
+        end
+      end
+
+      def read_source(path)
+        source = File.read(File.join(@root, path), encoding: "UTF-8")
+        return source if source.valid_encoding?
+
+        line = source.each_line.find_index { |text| !text.valid_encoding? }.to_i + 1
+        raise ParseError.new(path, line, "isn't valid UTF-8")
+      rescue SystemCallError, IOError => e
+        raise ParseError.new(path, 1, "can't be read: #{e.message}")
+      end
+
+      def entries
+        @units.each_with_index.map do |unit, id|
+          tree = Fingerprints.build(Normalizer.normalize(unit))
+          Entry.new(id: id, unit: unit, tree: tree, set: tree.digests)
+        end
+      end
+
+      # Candidates are generated at the loosest floors any primitive asks
+      # for; each pair is then judged by its own settings.
+      def floors
+        all = [DEFAULTS.merge(@overrides)]
+        all.concat(@contract.settings.values.map { |settings| DEFAULTS.merge(settings) }) if @overrides.empty?
+        { min_lines: all.map { |s| s[:min_lines] }.min, min_nodes: all.map { |s| s[:min_nodes] }.min,
+          threshold: all.map { |s| Rational(s[:threshold].to_s) }.min }
+      end
+
+      # Contract thresholds arrive as Floats; scores are Rationals. Reading
+      # the Float's decimal text keeps 0.75 exactly 3/4.
+      def exact(settings)
+        settings.merge(threshold: Rational(settings[:threshold].to_s))
+      end
+    end
+
+    # The duplication check: sweeps the tree on disk, sweeps the merge base
+    # for labels, and hands both to the Gate for the verdict.
+    class Check
+      def initialize(root:, base: nil, include_tests: false, contract_dir: "contract", overrides: {},
+                     introduced_only: false, paths: [], cache_dir: nil)
+        @root = File.expand_path(root)
+        @base_ref = base
+        @include_tests = include_tests
+        @contract_dir = contract_dir
+        @overrides = overrides
+        @introduced_only = introduced_only
+        @paths = paths
+        @cache_dir = cache_dir || File.join(@root, "tmp", "exhale")
+        @git = Git.new(@root)
+      end
+
+      def run
+        base_sha = validate!
+        head = Sweep.new(@root, files: (@git.files if @git.repo?), include_tests: @include_tests,
+                                contract_dir: @contract_dir, overrides: @overrides).run
+        Gate.new(head: head, base: base_sha && base_summary(base_sha), base_sha: base_sha, git: (@git if @git.repo?),
+                 changed_lines: base_sha ? @git.changed_lines(base_sha) : {},
+                 introduced_only: @introduced_only, paths: @paths, overrides: @overrides).verdict
+      end
+
+      # Checks the root and the base before anything is read, and returns
+      # the base's SHA (nil when there's no base to label against). The CLI
+      # calls it for explain too, so a typo there fails the same way.
+      def validate!
+        raise Error, "--root #{@root} isn't a directory" unless File.directory?(@root)
+
+        find_base
+      end
+
+      private
+
+      # A base the caller named and git can't find is an error: running on
+      # unlabeled would hide the mistake. A value that reads as an option
+      # never reaches git, where `--octopus` would resolve to HEAD.
+      def find_base
+        raise Error, "--base #{@base_ref} looks like an option, not a commit" if @base_ref&.start_with?("-")
+        raise Error, "--base #{@base_ref} needs a git repository, and #{@root} isn't in one" if @base_ref && !@git.repo?
+        return unless @git.repo?
+
+        ref = @base_ref || @git.default_branch_ref
+        sha = ref && @git.merge_base(ref)
+        raise Error, "--base #{@base_ref} names no commit with a merge base with HEAD" if @base_ref && sha.nil?
+
+        sha
+      end
+
+      # What the Gate needs from the base, cached on disk. The key covers
+      # everything that changes a base sweep, so a warm cache can never
+      # answer differently from a cold run.
+      def base_summary(sha)
+        path = File.join(@cache_dir, "base-#{sha}-#{VERSION}-n#{Normalizer::VERSION}-#{cache_key(sha)}.json")
+        cached = read_cache(path)
+        return cached if cached
+
+        summary = Dir.mktmpdir("exhale-base") do |dir|
+          files = base_files(sha)
+          @git.export_files(sha, files, dir)
+          # The same overrides as the head, so a pair that only the flags
+          # surface is labeled against a base swept the same way.
+          Gate.summarize(Sweep.new(dir, files: files, include_tests: @include_tests, contract_dir: @contract_dir,
+                                        overrides: @overrides).run)
+        end
+        write_cache(path, summary)
+        summary
+      end
+
+      def cache_key(sha)
+        parts = [sha, VERSION, Normalizer::VERSION, Prism::VERSION, Herb::VERSION, @include_tests, @git.prefix,
+                 @contract_dir, @overrides.sort.inspect]
+        Digest::SHA256.hexdigest(parts.join("\u0000"))[0, 16]
+      end
+
+      # Only the files a sweep can read: source and the Contract.
+      def base_files(sha)
+        contract = "#{@contract_dir.chomp('/')}/"
+        @git.files_at(sha).select { |path| SourceFiles.language(path) || path.start_with?(contract) }
+      end
+
+      # Plain JSON, never Marshal: loading a Marshal dump runs whatever it
+      # names. Anything at the path that isn't a regular file, or doesn't
+      # hold exactly the shape below, is rebuilt.
+      def read_cache(path)
+        return unless File.lstat(path).file?
+
+        BaseCache.load(File.read(path, encoding: "UTF-8"))
+      rescue SystemCallError, IOError
+        nil
+      end
+
+      # Written beside the target and renamed into place, so a reader never
+      # sees half a file. A symlink at the path is left alone: writing
+      # through it would overwrite whatever it points at.
+      def write_cache(path, summary)
+        FileUtils.mkdir_p(File.dirname(path))
+        return if File.symlink?(path)
+
+        temp = "#{path}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp"
+        File.open(temp, File::WRONLY | File::CREAT | File::EXCL, 0o644) { |f| f.write(BaseCache.dump(summary)) }
+        File.rename(temp, path)
+      rescue SystemCallError, IOError
+        FileUtils.rm_f(temp) if temp
+        nil
+      end
+    end
+
+    # The base summary as JSON, checked field by field on the way back in.
+    module BaseCache
+      FORMAT = 1
+      RATIONAL = %r{\A(-?\d+)/(\d+)\z}
+      KEYS = %w[format clusters structures pairs clause_keys].freeze
+
+      module_function
+
+      def dump(summary)
+        JSON.generate(
+          "format" => FORMAT,
+          "clusters" => summary.clusters.sort.to_h,
+          "structures" => summary.structures.sort.to_h { |key, ids| [key, ids.to_a.sort] },
+          "pairs" => summary.pairs.map { |row| [*row[0, 2], "#{row[2].numerator}/#{row[2].denominator}", *row[3..]] },
+          "clause_keys" => summary.clause_keys.to_a.sort
+        )
+      end
+
+      # nil for anything that isn't exactly what dump writes.
+      def load(text)
+        data = JSON.parse(text)
+        return unless data.is_a?(Hash) && data.keys.sort == KEYS.sort && data["format"] == FORMAT
+
+        clusters = data["clusters"]
+        structures = data["structures"]
+        pairs = data["pairs"]
+        clause_keys = data["clause_keys"]
+        return unless string_hash?(clusters) { |id| id.is_a?(Integer) }
+        return unless string_hash?(structures) { |ids| ids.is_a?(Array) && ids.all?(Integer) }
+        return unless clause_keys.is_a?(Array) && clause_keys.all?(String)
+        return unless pairs.is_a?(Array) && pairs.all? { |row| pair?(row) }
+
+        BaseSummary.new(clusters: clusters, structures: structures, clause_keys: Set.new(clause_keys),
+                        pairs: pairs.map { |row| [*row[0, 2], rational(row[2]), *row[3..]] })
+      rescue JSON::ParserError, EncodingError
+        nil
+      end
+
+      def string_hash?(value, &valid)
+        value.is_a?(Hash) && value.all? { |key, item| key.is_a?(String) && valid.call(item) }
+      end
+
+      def pair?(row)
+        row.is_a?(Array) && row.size == 7 && row.all?(String) && RATIONAL.match?(row[2]) &&
+          !RATIONAL.match(row[2])[2].to_i.zero?
+      end
+
+      def rational(text)
+        numerator, denominator = RATIONAL.match(text).captures.map(&:to_i)
+        Rational(numerator, denominator)
+      end
+    end
+  end
+end