exhale 0.1.0

data/lib/exhale/git.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+require "set"
+require_relative "errors"
+
+module Exhale
+  # Thin, deterministic wrapper over the git CLI. Every call uses argument
+  # arrays and ignores the user's quoting config.
+  class Git
+    # The user's git config must not change what exhale reads, or the same
+    # commit could get different verdicts on different machines. These pin
+    # every setting that changes diff or blame output.
+    BASE_ARGS = ["-c", "core.quotepath=off", "-c", "diff.renames=true", "-c", "diff.noprefix=false",
+                 "-c", "diff.mnemonicPrefix=false", "-c", "diff.relative=false",
+                 "-c", "diff.indentHeuristic=false", "-c", "diff.interHunkContext=0", "-c", "blame.ignoreRevsFile=",
+                 "-c", "blame.markIgnoredLines=false", "-c", "blame.markUnblamableLines=false"].freeze
+    DEFAULT_CANDIDATES = %w[origin/main origin/master main master].freeze
+    # A blame header line: the commit, SHA-1 or SHA-256, then the original
+    # and final line numbers.
+    BLAME_HEADER = /\A(\h{40}(?:\h{24})?) \d+ (\d+)/
+    HUNK = /\A@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/
+
+    attr_reader :root
+
+    def initialize(root)
+      @root = File.expand_path(root)
+    end
+
+    def repo?
+      out, _err, status = run("rev-parse", "--is-inside-work-tree")
+      status.success? && out.strip == "true"
+    end
+
+    def head_sha
+      out, _err, status = run("rev-parse", "--verify", "--quiet", "HEAD^{commit}")
+      status.success? ? out.strip : nil
+    end
+
+    def default_branch_ref
+      out, _err, status = run("symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD")
+      if status.success? && ref_exists?(out.strip)
+        return out.strip
+      end
+
+      DEFAULT_CANDIDATES.find { |ref| ref_exists?(ref) }
+    end
+
+    def merge_base(ref)
+      return nil if ref.nil? || ref.empty?
+
+      out, _err, status = run("merge-base", ref, "HEAD")
+      status.success? && !out.strip.empty? ? out.strip : nil
+    end
+
+    # Where root sits inside the repository, like "agora/" for an app in a
+    # monorepo. Every path exhale reports is relative to root, so exports and
+    # diffs are scoped to it too.
+    def prefix
+      @prefix ||= run!("rev-parse", "--show-prefix").strip
+    end
+
+    def toplevel
+      @toplevel ||= run!("rev-parse", "--show-toplevel").strip
+    end
+
+    # A sparse checkout leaves tracked files out of the work tree, so a sweep of
+    # the head would read less than the commit.
+    def sparse?
+      out, _err, status = run("config", "--bool", "core.sparseCheckout")
+      return true if status.success? && out.strip == "true"
+
+      run!("ls-files", "-t", "-z").split("\0").any? { |entry| entry.start_with?("S ") }
+    end
+
+    def files
+      raise GitError, "exhale needs the full tree; this checkout is sparse" if sparse?
+
+      out = run!("ls-files", "--cached", "--others", "--exclude-standard", "-z")
+      out.split("\0").reject(&:empty?).uniq.select do |path|
+        full = File.join(root, path)
+        File.file?(full) && !File.symlink?(full)
+      end.sort
+    end
+
+    # Symlinks (mode 120000) and submodules (type commit) aren't source.
+    def files_at(sha)
+      blob_oids(sha).keys.sort
+    end
+
+    # Writes the named files at sha into dir as raw blobs. git archive would
+    # apply export-ignore and export-subst, and the base must be the commit
+    # exactly as it is.
+    def export_files(sha, paths, dir)
+      FileUtils.mkdir_p(dir)
+      return dir if paths.empty?
+
+      # Blobs are requested by object id: a path with a newline would split a
+      # "sha:path" request in two and shift every response after it.
+      oids = blob_oids(sha)
+      paths = paths.select { |path| oids.key?(path) }
+      return dir if paths.empty?
+
+      out, err, status = Open3.capture3("git", *BASE_ARGS, "-C", root, "cat-file", "--batch",
+                                        stdin_data: paths.map { |path| "#{oids[path]}\n" }.join, binmode: true)
+      raise GitError, "git cat-file failed: #{utf8(err).strip}" unless status.success?
+
+      write_blobs(out, paths, dir)
+      dir
+    end
+
+    def changed_lines(base_sha)
+      diff = run!("diff", "--relative", "--unified=0", "--no-color", "--no-ext-diff", "--no-textconv",
+                  "--find-renames", "--diff-algorithm=myers", "--src-prefix=a/", "--dst-prefix=b/",
+                  base_sha, "--")
+      result = parse_diff(diff)
+      untracked_files.each { |path| result[path] = all_lines(path) }
+      result.reject { |_path, lines| lines.empty? }
+    end
+
+    # Committer time for every line of a file, index 0 being line 1, from one
+    # blame call. Uncommitted lines, and every line of an untracked file, are
+    # Float::INFINITY: newer than anything committed.
+    def blame_times(path)
+      full = File.join(root, path)
+      return Array.new(File.foreach(full).count, Float::INFINITY) unless tracked?(path)
+
+      # The flag, unlike an empty -c value, really does reset a repository's
+      # own blame.ignoreRevsFile.
+      out, err, status = run("blame", "--porcelain", "--no-ignore-revs-file", "--", path)
+      raise GitError, "git blame failed: #{err.strip}" unless status.success?
+
+      line_times(out)
+    end
+
+    def blame_time(path, start_line, end_line)
+      return Float::INFINITY unless tracked?(path)
+
+      out, err, status = run("blame", "--porcelain", "--no-ignore-revs-file", "-L", "#{start_line},#{end_line}",
+                             "--", path)
+      raise GitError, "git blame failed: #{err.strip}" unless status.success?
+
+      newest_time(out)
+    end
+
+    private
+
+    # Output is UTF-8 whatever the locale, so no verdict depends on it.
+    def run(*args)
+      out, err, status = Open3.capture3("git", *BASE_ARGS, "-C", root, *args)
+      [utf8(out), utf8(err), status]
+    end
+
+    def utf8(text)
+      text.dup.force_encoding(Encoding::UTF_8).scrub
+    end
+
+    # path => object id for each blob under root at sha. Symlinks (mode
+    # 120000) and submodules (type commit) aren't source.
+    def blob_oids(sha)
+      run!("ls-tree", "-r", "-z", sha).split("\0").each_with_object({}) do |entry, oids|
+        meta, path = entry.split("\t", 2)
+        mode, type, oid = meta.split(" ")
+        oids[path] = oid if type == "blob" && mode != "120000"
+      end
+    end
+
+    def run!(*args)
+      out, err, status = run(*args)
+      raise GitError, "git #{args.first} failed: #{err.strip}" unless status.success?
+
+      out
+    end
+
+    def ref_exists?(ref)
+      _out, _err, status = run("rev-parse", "--verify", "--quiet", "#{ref}^{commit}")
+      status.success?
+    end
+
+    def tracked?(path)
+      _out, _err, status = run("ls-files", "--error-unmatch", "--", path)
+      status.success?
+    end
+
+    def untracked_files
+      run!("ls-files", "--others", "--exclude-standard", "-z").split("\0").reject(&:empty?).sort
+    end
+
+    def all_lines(path)
+      full = File.join(root, path)
+      return Set.new unless File.file?(full)
+
+      count = File.foreach(full).count
+      Set.new(1..count)
+    end
+
+    # File headers only count between a "diff --git" line and the first hunk,
+    # so an added line that happens to start with "++ " can't pass for one.
+    def parse_diff(diff)
+      result = Hash.new { |h, k| h[k] = Set.new }
+      current = nil
+      in_header = false
+      diff.each_line(chomp: true) do |line|
+        if line.start_with?("diff --git ")
+          in_header = true
+          current = nil
+        elsif in_header && line.start_with?("+++ ")
+          current = new_path(line)
+        elsif (m = HUNK.match(line))
+          in_header = false
+          add_hunk(result[current], m[1].to_i, (m[2] || "1").to_i) if current
+        end
+      end
+      result.to_h
+    end
+
+    # git appends a tab to a header path that contains a space, and quotes a
+    # path with characters it won't print raw.
+    def new_path(line)
+      target = line.delete_prefix("+++ ").delete_suffix("\t")
+      return nil if target == "/dev/null"
+
+      target = target.undump if target.start_with?('"')
+      target.delete_prefix("b/")
+    end
+
+    def write_blobs(out, paths, dir)
+      pos = 0
+      paths.each do |path|
+        header_end = out.index("\n", pos)
+        header = out[pos...header_end]
+        pos = header_end + 1
+        next if header.end_with?(" missing")
+
+        size = header.split(" ").last.to_i
+        full = File.join(dir, path)
+        FileUtils.mkdir_p(File.dirname(full))
+        File.binwrite(full, out.byteslice(pos, size))
+        pos += size + 1
+      end
+    end
+
+    def add_hunk(set, start, count)
+      count.times { |i| set << start + i }
+    end
+
+    # Porcelain names a commit's metadata only the first time the commit
+    # appears, so times are remembered per SHA and handed to each line.
+    def line_times(porcelain)
+      times = {}
+      result = []
+      sha = nil
+      final = nil
+      porcelain.each_line(chomp: true) do |line|
+        if line.start_with?("\t")
+          result[final - 1] = uncommitted?(sha) ? Float::INFINITY : times.fetch(sha, 0)
+        elsif (m = BLAME_HEADER.match(line))
+          sha = m[1]
+          final = m[2].to_i
+        elsif line.start_with?("committer-time ")
+          times[sha] = line.split(" ", 2).last.to_i
+        end
+      end
+      result
+    end
+
+    def uncommitted?(sha)
+      sha.match?(/\A0+\z/)
+    end
+
+    def newest_time(porcelain)
+      return Float::INFINITY if porcelain.each_line.any? { |l| (m = BLAME_HEADER.match(l)) && uncommitted?(m[1]) }
+
+      times = porcelain.each_line.filter_map { |l| l.split(" ", 2).last.to_i if l.start_with?("committer-time ") }
+      times.max or raise GitError, "no blame data"
+    end
+  end
+end

data/lib/exhale/report.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "version"
+require_relative "dry/normalizer"
+
+module Exhale
+  # Renders a duplication check's result as text for the agent doing the
+  # exhale, JSON for tools, or EDN in the shape dryer writes to
+  # .metrics/dry.edn.
+  module Report
+    FORMATS = %w[text json edn].freeze
+    LABELS = { introduced: "INTRODUCED", shifted: "SHIFTED", already_there: "ALREADY THERE", found: "FOUND" }.freeze
+
+    module_function
+
+    def render(result, format)
+      case format
+      when "text" then Text.new(result).render
+      when "json" then JSON.pretty_generate(Json.new(result).to_h)
+      when "edn" then Edn.new(result).render
+      else raise ArgumentError, "unknown format #{format.inspect}"
+      end
+    end
+
+    def counts(result)
+      counts = Hash.new(0)
+      result.findings.each { |f| counts[f.klass] += 1 }
+      counts[:kept] = result.kept.size
+      counts[:contracted] = result.contracted.size
+      counts
+    end
+
+    def kind_name(kind, location)
+      kind == :fragment ? "fragment, #{location.lines} lines" : kind.to_s
+    end
+
+    def score(value)
+      value.to_f.round(2)
+    end
+
+    class Text
+      def initialize(result)
+        @result = result
+        @out = []
+      end
+
+      def render
+        @out << header
+        @result.notes.each { |note| @out << "note: #{note}" }
+        @result.findings.each { |f| finding(f) }
+        @result.kept.each { |k| kept(k) }
+        contracted
+        errors
+        "#{@out.join("\n")}\n"
+      end
+
+      private
+
+      def header
+        counts = Report.counts(@result)
+        parts = %i[introduced shifted already_there found kept contracted].filter_map do |key|
+          "#{counts[key]} #{key.to_s.tr('_', ' ')}" if counts[key].positive?
+        end
+        parts << plural(@result.clause_errors.size, "contract error") unless @result.clause_errors.empty?
+        parts << plural(@result.parse_errors.size, "parse error") unless @result.parse_errors.empty?
+        parts << (@result.exit_code.zero? ? "clean" : "failing") if parts.empty?
+        base = @result.base_sha ? "   base #{@result.base_sha[0, 7]}" : ""
+        "exhale dry: #{parts.join(', ')}#{base}"
+      end
+
+      def plural(count, noun)
+        "#{count} #{noun}#{'s' unless count == 1}"
+      end
+
+      def finding(f)
+        @out << ""
+        @out << "#{LABELS.fetch(f.klass)}  #{Report.score(f.score)}  #{Report.kind_name(f.kind, f.copy)}"
+        rows = [[f.touched.include?(f.copy) ? "new" : "copy", f.copy, nil]]
+        f.others.each { |l, s| rows << ["existing", l, f.others.size > 1 ? Report.score(s) : nil] }
+        table(rows)
+        @out << "  hint      #{f.hint}"
+      end
+
+      def kept(k)
+        @out << ""
+        flag = k.new_clause ? "  NEW CLAUSE, review its reason" : ""
+        @out << "KEPT  #{Report.score(k.score)}  #{k.clause.path}:#{k.clause.line} \"#{k.clause.heading}\"#{flag}"
+        table([["side", k.a, nil], ["side", k.b, nil]])
+      end
+
+      def table(rows)
+        width = rows.map { |_, l, _| place(l).size }.max
+        ident = rows.map { |_, l, _| l.unit.identity.size }.max
+        rows.each do |label, l, s|
+          line = "  #{label.ljust(9)} #{place(l).ljust(width)}  #{l.unit.identity.ljust(ident)}"
+          line += "  #{s}" if s
+          @out << line.rstrip
+        end
+      end
+
+      def place(location)
+        "#{location.path}:#{location.start_line}-#{location.end_line}"
+      end
+
+      def contracted
+        return if @result.contracted.empty?
+
+        @out << ""
+        @out << "CONTRACTED"
+        @result.contracted.first(20).each do |c|
+          @out << "  #{c.a} no longer matches #{c.b} (#{Report.score(c.score)} at base)"
+        end
+        more = @result.contracted.size - 20
+        @out << "  and #{more} more" if more.positive?
+      end
+
+      def errors
+        return if @result.clause_errors.empty? && @result.parse_errors.empty?
+
+        @out << ""
+        @result.clause_errors.each { |e| @out << "CONTRACT  #{e.message}" }
+        @result.parse_errors.each { |e| @out << "PARSE     #{e.message}" }
+      end
+    end
+
+    class Json
+      def initialize(result)
+        @result = result
+      end
+
+      def to_h
+        {
+          "exhale" => VERSION,
+          "check" => "dry",
+          "normalizer" => Dry::Normalizer::VERSION,
+          "base" => @result.base_sha,
+          "exit_code" => @result.exit_code,
+          "counts" => Report.counts(@result).transform_keys(&:to_s),
+          "notes" => @result.notes,
+          "findings" => @result.findings.map { |f| finding(f) },
+          "kept" => @result.kept.map { |k| kept(k) },
+          "contracted" => @result.contracted.map { |c| { "a" => c.a, "b" => c.b, "score" => Report.score(c.score) } },
+          "contract_errors" => @result.clause_errors.map { |e| error(e) },
+          "parse_errors" => @result.parse_errors.map { |e| error(e) }
+        }
+      end
+
+      private
+
+      def finding(f)
+        {
+          "class" => f.klass.to_s, "score" => Report.score(f.score), "kind" => f.kind.to_s,
+          "copy" => location(f.copy, f.touched),
+          "existing" => f.others.map { |l, s| location(l, f.touched).merge("score" => Report.score(s)) },
+          "hint" => f.hint, "payoff" => f.payoff, "primitive" => f.primitive
+        }
+      end
+
+      def kept(k)
+        {
+          "score" => Report.score(k.score), "a" => location(k.a, []), "b" => location(k.b, []),
+          "clause" => { "path" => k.clause.path, "line" => k.clause.line, "heading" => k.clause.heading },
+          "new_clause" => k.new_clause
+        }
+      end
+
+      def location(l, touched)
+        { "path" => l.path, "start_line" => l.start_line, "end_line" => l.end_line, "identity" => l.unit.identity,
+          "nodes" => l.size, "touched" => touched.include?(l) }
+      end
+
+      def error(e)
+        { "path" => e.path, "line" => e.line, "message" => e.message }
+      end
+    end
+
+    # The candidates list dryer writes, one entry per copy and existing pair.
+    class Edn
+      def initialize(result)
+        @result = result
+      end
+
+      def render
+        rows = @result.findings.flat_map do |f|
+          f.others.map { |l, s| candidate(f.copy, l, s) }
+        end
+        "{:candidates\n [#{rows.join("\n  ")}]}\n"
+      end
+
+      private
+
+      def candidate(left, right, score)
+        "{:score #{score.to_f.round(12)}\n   :language #{left.unit.language.to_s.inspect}\n   " \
+          ":left #{side(left)}\n   :right #{side(right)}\n   :left-nodes #{left.size}\n   :right-nodes #{right.size}}"
+      end
+
+      def side(l)
+        "{:file #{l.path.inspect}, :start-line #{l.start_line}, :end-line #{l.end_line}}"
+      end
+    end
+  end
+end

data/lib/exhale/shape.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Exhale
+  # One node of a normalized tree. Normalization keeps the names of
+  # operations and the shape of the tree, and replaces the names of things
+  # with markers, so two units that do the same thing to different locals
+  # produce equal trees.
+  #
+  # kind       - String. A Prism or Herb node type after normalization
+  #              ("call_node", "if_node", "html_element_node"), or a marker
+  #              for a dropped name: ":local", ":ivar", ":cvar", ":gvar",
+  #              ":const", ":literal".
+  # label      - String or nil. A name normalization keeps: the method name at
+  #              a call site (operators are calls too), an HTML tag name, an
+  #              attribute name, or a kept attribute value (data-controller,
+  #              data-action).
+  # children   - Array of Shape, in source order.
+  # start_line - 1-based first line in the unit's file.
+  # end_line   - 1-based last line.
+  # sequence   - true when children are a run of statements or sibling
+  #              markup (a method body, a block body, an element's children).
+  #              Statement-run fragments come only from sequences.
+  Shape = Struct.new(:kind, :label, :children, :start_line, :end_line, :sequence, keyword_init: true) do
+    def leaf?
+      children.empty?
+    end
+
+    def lines
+      end_line - start_line + 1
+    end
+  end
+end

data/lib/exhale/source_files.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module Exhale
+  # Lists the Ruby and ERB files exhale analyses.
+  module SourceFiles
+    EXCLUDED_ROOTS = %w[vendor node_modules tmp log coverage public storage bin].freeze
+    EXCLUDED_PREFIXES = ["app/assets/builds/"].freeze
+    # Schema, migrations and settings, at the root and in each engine or pack
+    # of a monorepo.
+    EXCLUDED_DIRS = %r{\A(?:(?:engines|packs|components|gems)/[^/]+/)?(?:db|config)/}
+    # A generator's header: a comment line that opens with the marker. A
+    # comment that only mentions generated code somewhere doesn't count.
+    GENERATED = %r{
+      \A\s*(?:<%\#|\#|//)\s*
+      (?:this\ file\ is\ auto-?generated|this\ file\ is\ automatically\ generated|auto-?generated\ by
+        |generated\ by\b.*\bdo\ not\ edit|do\ not\ edit|code\ generated\b.*\bdo\ not\ edit)
+    }xi
+    # Enough of a file to hold its first five lines.
+    HEADER_BYTES = 64 * 1024
+
+    module_function
+
+    def list(root, files: nil, include_tests: false)
+      candidates = files ? files.map(&:to_s) : walk(root)
+      candidates
+        .select { |path| language(path) && path_allowed?(path, include_tests) }
+        .sort
+        .uniq
+        .reject { |path| linked?(root, path) }
+        .select { |path| File.exist?(File.join(root, path)) }
+        .reject { |path| generated?(File.join(root, path)) }
+        .map { |path| [path, language(path)] }
+    end
+
+    def language(path)
+      case path
+      when /\.rb\z/ then :ruby
+      # Herb reads HTML with ERB in it. Text, JSON and prompt templates that
+      # happen to use ERB aren't HTML, so they aren't compared.
+      when /\.html(\+[\w-]+)?\.erb\z/, /\.turbo_stream\.erb\z/ then :erb
+      end
+    end
+
+    def walk(root)
+      Dir.glob("**/*", base: root).select do |path|
+        full = File.join(root, path)
+        File.file?(full) && !File.symlink?(full)
+      end
+    end
+
+    # A symlink, or a path under a symlinked directory, points outside the
+    # tree exhale was asked to read, or at a file it already reads.
+    def linked?(root, path)
+      segments = path.split("/")
+      segments.each_index.any? { |i| File.symlink?(File.join(root, *segments[0..i])) }
+    end
+
+    def path_allowed?(path, include_tests)
+      segments = path.split("/")
+      return false if segments.any? { |s| s.start_with?(".") }
+      return false if EXCLUDED_ROOTS.include?(segments.first) || path.match?(EXCLUDED_DIRS)
+      return false if EXCLUDED_PREFIXES.any? { |p| path.start_with?(p) }
+
+      include_tests || !test_path?(path, segments)
+    end
+
+    def test_path?(path, segments)
+      segments[0..-2].any? { |s| %w[spec test].include?(s) } || path.end_with?("_spec.rb", "_test.rb")
+    end
+
+    # Only a generator's header takes a file off the list. A file exhale
+    # can't read, or that isn't valid UTF-8, stays on it so the sweep reports
+    # it as a parse error; dropping it here would let it through the gate
+    # unread.
+    def generated?(full)
+      head = File.open(full, "rb") { |file| file.read(HEADER_BYTES) }.to_s
+      head.force_encoding(Encoding::UTF_8).scrub.each_line.first(5).any? { |line| line.match?(GENERATED) }
+    rescue SystemCallError
+      false
+    end
+  end
+end

data/lib/exhale/unit.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Exhale
+  # Anything that can appear in a finding: a method, the body of a Rails DSL
+  # call, or an ERB template. Fragments are found inside units later; they
+  # aren't units of their own.
+  #
+  # kind      - :method, :dsl or :template.
+  # identity  - Survives a file move. "Billing::Invoice#total" for an instance
+  #             method, "Billing::Invoice.build" for a singleton method,
+  #             "Order.scope(:settled)" or "Order.before_save[2]" for a DSL
+  #             body, "views/orders/_form.html.erb" for a template (its path
+  #             under app/).
+  # namespace - "Billing::Invoice" for a Ruby unit ("Object" at top level),
+  #             nil for a template.
+  # name      - "total", "build", "scope(:settled)"; nil for a template.
+  # path      - Path relative to the repository root.
+  # language  - :ruby or :erb.
+  # node      - The parser node the unit covers (a Prism::Node, or a Herb
+  #             node). Held in memory only; never cached.
+  Unit = Struct.new(:kind, :identity, :namespace, :name, :path, :start_line, :end_line, :language, :node,
+                    keyword_init: true) do
+    def lines
+      end_line - start_line + 1
+    end
+  end
+end

data/lib/exhale/units/erb.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "herb"
+require_relative "../errors"
+require_relative "../unit"
+
+module Exhale
+  module Units
+    # A template is one unit. Duplication inside it is found later, as
+    # fragments of its tree.
+    module Erb
+      module_function
+
+      def extract(source, path)
+        # strict: false lets valid HTML through (an omitted `</li>`); broken
+        # markup and broken Ruby are still errors.
+        result = Herb.parse(source, strict: false)
+        raise_first_error(result.errors, path) unless result.errors.empty?
+
+        [Unit.new(kind: :template, identity: path.delete_prefix("app/"), namespace: nil, name: nil, path: path,
+                  start_line: 1, end_line: [source.lines.size, 1].max, language: :erb, node: result.value)]
+      end
+
+      def raise_first_error(errors, path)
+        error = errors.min_by { |e| [e.location.start.line, e.location.start.column] }
+        raise ParseError.new(path, error.location.start.line, error.message)
+      end
+    end
+  end
+end