ruby-duplicates 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 03bc2b060ce66e3f69e705b7574ce6f3b30520fdb2da8e0e94f838c41cc4176d
+  data.tar.gz: 6f711d2390e3608f68bf079ab01a278cef417ee408439f3cf765ec8b25caeb77
+SHA512:
+  metadata.gz: a039bbfa6462b0955a10b0390b1007b414defe9e5536dc224846e06c6e40d71d05dad6fae0b7b2223a5a56db31e875168e1fd3680c36dbf05e803da7376d4ede
+  data.tar.gz: 73c1dfdea3cefeb83c8fc9b9a3d7b58c632689218db5f7b9570cf35f36d5e0baa7b110118beb955cebbd9e2aa20f96060f5271e5c76b4c073fc3402a305e5f94

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bartas Urba
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# ruby-duplicates
+
+A small duplicate-code metric for Ruby.
+
+`ruby-duplicates` parses Ruby with the standard library `Ripper`, normalizes syntax trees so names and literal values do not dominate the comparison, fingerprints method subtrees, and reports methods with high Jaccard similarity.
+
+It is inspired by Uncle Bob's [`dry4clj`](https://github.com/unclebob/dry4clj), which applies the same broad idea to Clojure code: compare normalized structure instead of doing plain text clone detection.
+
+This is a metric tool, not a refactoring engine. It points at suspiciously similar methods so a human or coding agent can decide whether the duplication is accidental, intentional symmetry, or data-shaped boilerplate.
+
+## Install
+
+From this repo:
+
+```bash
+bundle install
+exe/ruby-duplicates app lib test
+```
+
+As a gem from a local checkout:
+
+```bash
+gem build ruby-duplicates.gemspec
+gem install ruby-duplicates-*.gem
+ruby-duplicates app lib test
+```
+
+From another project before a RubyGems release, point at the GitHub repo:
+
+```ruby
+gem "ruby-duplicates", git: "https://github.com/barturba/ruby-duplicates.git"
+```
+
+## Usage
+
+```bash
+ruby-duplicates [options] [file-or-directory ...]
+```
+
+Examples:
+
+```bash
+ruby-duplicates app lib test
+ruby-duplicates --threshold 0.9 --min-lines 5 --min-nodes 30 app
+ruby-duplicates --json app/models app/controllers
+```
+
+Options:
+
+```bash
+--threshold N    Minimum similarity score, default 0.82
+--min-lines N    Minimum method source lines, default 4
+--min-nodes N    Minimum normalized syntax nodes, default 20
+--max-results N  Maximum matches to print, default 50
+--format F       text or json, default text
+--json           Same as --format json
+--ignore-dir N   Directory basename or path to skip; may be repeated
+```
+
+Example output:
+
+```text
+ruby_duplicates candidates=3 matches=1 threshold=0.82
+
+DUPLICATE score=1.00 shared=21
+  examples/duplicate_sample.rb:1-4 alpha nodes=64
+  examples/duplicate_sample.rb:7-10 beta nodes=64
+```
+
+## How It Works
+
+For each Ruby method, the scanner:
+
+1. Parses the file with `Ripper.sexp`.
+2. Extracts `def` and `defs` method nodes.
+3. Normalizes identifiers, constants, instance variables, globals, labels, strings, and numbers into token classes.
+4. Normalizes most non-head symbols so tiny operator/name differences do not hide repeated shape.
+5. Fingerprints every normalized subtree with SHA1.
+6. Compares method fingerprint sets with Jaccard similarity.
+
+The defaults intentionally favor high-signal matches. Lower `--threshold`, `--min-lines`, or `--min-nodes` when exploring.
+
+## Limits
+
+- It only scans Ruby methods, not arbitrary repeated blocks.
+- It is structural, not semantic.
+- Metaprogrammed code can look sparse because the useful behavior is hidden in data.
+- Rails controllers and tests can produce intentional symmetry. Treat those as review candidates, not automatic refactors.
+
+## Development
+
+```bash
+ruby -Ilib test/ruby_duplicates_test.rb
+gem build ruby-duplicates.gemspec
+```
+
+## Inspiration
+
+- Uncle Bob's `dry4clj`: https://github.com/unclebob/dry4clj

data/exe/ruby-duplicates

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+begin
+  require "ruby_duplicates"
+rescue LoadError
+  $LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+  require "ruby_duplicates"
+end
+
+RubyDuplicates::CLI.run(ARGV)

data/lib/ruby_duplicates/cli.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+require "find"
+require "json"
+require "optparse"
+require "ripper"
+require "set"
+
+Candidate = Data.define(:file, :name, :start_line, :end_line, :node_count, :fingerprints) do
+  def line_count
+    end_line - start_line + 1
+  end
+
+  def location
+    "#{file}:#{start_line}-#{end_line}"
+  end
+end
+
+Match = Data.define(:score, :shared, :left, :right)
+
+class RubyDuplicates
+  DEFAULT_IGNORES = %w[
+    .git
+    .bundle
+    .burndown-swarm
+    coverage
+    log
+    node_modules
+    tmp
+    vendor/bundle
+  ].freeze
+
+  TOKEN_GROUPS = {
+    :@ident => :identifier,
+    :@const => :constant,
+    :@ivar => :ivar,
+    :@cvar => :cvar,
+    :@gvar => :gvar,
+    :@label => :label,
+    :@int => :number,
+    :@float => :number,
+    :@rational => :number,
+    :@imaginary => :number,
+    :@CHAR => :string,
+    :@tstring_content => :string,
+    :@regexp_beg => :regexp,
+    :@regexp_end => :regexp,
+    :@period => :dot,
+    :@op => :operator
+  }.freeze
+
+  def initialize(options)
+    @threshold = options.fetch(:threshold)
+    @min_lines = options.fetch(:min_lines)
+    @min_nodes = options.fetch(:min_nodes)
+    @max_results = options.fetch(:max_results)
+    @format = options.fetch(:format)
+    @paths = options.fetch(:paths)
+    @ignore_dirs = options.fetch(:ignore_dirs)
+  end
+
+  def run
+    candidates = collect_candidates
+    matches = find_matches(candidates)
+
+    if @format == "json"
+      puts JSON.pretty_generate(json_payload(matches, candidates.length))
+    else
+      print_text(matches, candidates.length)
+    end
+  end
+
+  private
+
+  def collect_candidates
+    ruby_files.flat_map { |file| candidates_for_file(file) }
+      .select { |candidate| candidate.line_count >= @min_lines && candidate.node_count >= @min_nodes }
+  end
+
+  def ruby_files
+    @paths.flat_map do |path|
+      if File.directory?(path)
+        files_under(path)
+      elsif File.file?(path) && path.end_with?(".rb")
+        [path]
+      else
+        []
+      end
+    end.uniq.sort
+  end
+
+  def files_under(root)
+    files = []
+    Find.find(root) do |path|
+      if File.directory?(path) && ignored_directory?(path)
+        Find.prune
+      elsif File.file?(path) && path.end_with?(".rb")
+        files << path
+      end
+    end
+    files
+  end
+
+  def ignored_directory?(path)
+    base = File.basename(path)
+    relative = clean_path(path)
+    @ignore_dirs.include?(base) || @ignore_dirs.include?(relative)
+  end
+
+  def candidates_for_file(file)
+    source = File.read(file)
+    sexp = Ripper.sexp(source)
+    return warn_parse_failure(file) unless sexp
+
+    lines = source.lines
+    method_nodes(sexp).map do |node|
+      name, start_line = method_name_and_start(node)
+      end_line = [max_line(node), start_line].compact.max
+      end_line = [end_line, lines.length].min
+      fingerprints = Set.new
+      normalized = normalize(node)
+      collect_fingerprints(normalized, fingerprints)
+
+      Candidate.new(
+        file: clean_path(file),
+        name: name,
+        start_line: start_line,
+        end_line: end_line,
+        node_count: count_nodes(normalized),
+        fingerprints: fingerprints
+      )
+    end
+  rescue StandardError => error
+    warn "#{file}: #{error.class}: #{error.message}"
+    []
+  end
+
+  def warn_parse_failure(file)
+    warn "#{file}: could not parse Ruby source"
+    []
+  end
+
+  def method_nodes(node, found = [])
+    return found unless node.is_a?(Array)
+
+    found << node if %i[def defs].include?(node.first)
+    node.each { |child| method_nodes(child, found) if child.is_a?(Array) }
+    found
+  end
+
+  def method_name_and_start(node)
+    token = node.first == :def ? node[1] : node[3]
+    [token_value(token), token_line(token)]
+  end
+
+  def token_value(token)
+    token.is_a?(Array) ? token[1].to_s : "unknown"
+  end
+
+  def token_line(token)
+    token.is_a?(Array) && token[2].is_a?(Array) ? token[2][0].to_i : 1
+  end
+
+  def max_line(node)
+    lines = []
+    visit_tokens(node) do |token|
+      position = token[2]
+      lines << position[0].to_i if position.is_a?(Array)
+    end
+    lines.max || 1
+  end
+
+  def visit_tokens(node, &block)
+    return unless node.is_a?(Array)
+
+    if token_node?(node)
+      yield node
+      return
+    end
+
+    node.each { |child| visit_tokens(child, &block) if child.is_a?(Array) }
+  end
+
+  def normalize(node)
+    if token_node?(node)
+      normalize_token(node)
+    elsif node.is_a?(Array)
+      normalize_array(node)
+    elsif node.is_a?(Symbol)
+      :symbol
+    elsif node.nil? || node == false || node == true
+      node
+    else
+      :value
+    end
+  end
+
+  def normalize_array(node)
+    node.each_with_index.map do |child, index|
+      if index.zero? && child.is_a?(Symbol)
+        child
+      else
+        normalize(child)
+      end
+    end
+  end
+
+  def token_node?(node)
+    node.is_a?(Array) && node[0].is_a?(Symbol) && node[0].to_s.start_with?("@")
+  end
+
+  def normalize_token(token)
+    type = token[0]
+    group = TOKEN_GROUPS.fetch(type, type)
+
+    if type == :@op
+      [type, token[1].to_s]
+    else
+      [type, group]
+    end
+  end
+
+  def collect_fingerprints(node, fingerprints)
+    return unless node.is_a?(Array)
+
+    fingerprints << Digest::SHA1.hexdigest(JSON.generate(node))
+    node.each { |child| collect_fingerprints(child, fingerprints) if child.is_a?(Array) }
+  end
+
+  def count_nodes(node)
+    return 0 unless node.is_a?(Array)
+
+    1 + node.sum { |child| count_nodes(child) }
+  end
+
+  def find_matches(candidates)
+    matches = []
+    candidates.each_with_index do |left, index|
+      candidates[(index + 1)..].to_a.each do |right|
+        score, shared = jaccard(left.fingerprints, right.fingerprints)
+        next if score < @threshold
+
+        matches << Match.new(score: score, shared: shared, left: left, right: right)
+      end
+    end
+    matches.sort_by { |match| [-match.score, -match.shared, match.left.file, match.left.start_line] }.first(@max_results)
+  end
+
+  def jaccard(left, right)
+    shared = (left & right).length
+    total = (left | right).length
+    return [0.0, 0] if total.zero?
+
+    [shared.to_f / total, shared]
+  end
+
+  def print_text(matches, candidate_count)
+    puts "ruby_duplicates candidates=#{candidate_count} matches=#{matches.length} threshold=#{format_score(@threshold)}"
+    matches.each do |match|
+      puts
+      puts "DUPLICATE score=#{format_score(match.score)} shared=#{match.shared}"
+      puts "  #{match.left.location} #{match.left.name} nodes=#{match.left.node_count}"
+      puts "  #{match.right.location} #{match.right.name} nodes=#{match.right.node_count}"
+    end
+  end
+
+  def json_payload(matches, candidate_count)
+    {
+      candidates: candidate_count,
+      threshold: @threshold,
+      matches: matches.map do |match|
+        {
+          score: match.score,
+          shared: match.shared,
+          left: candidate_payload(match.left),
+          right: candidate_payload(match.right)
+        }
+      end
+    }
+  end
+
+  def candidate_payload(candidate)
+    {
+      file: candidate.file,
+      name: candidate.name,
+      start_line: candidate.start_line,
+      end_line: candidate.end_line,
+      nodes: candidate.node_count
+    }
+  end
+
+  def clean_path(path)
+    path = File.expand_path(path)
+    cwd = "#{Dir.pwd}/"
+    path.start_with?(cwd) ? path.delete_prefix(cwd) : path
+  end
+
+  def format_score(score)
+    format("%.2f", score)
+  end
+end
+
+class RubyDuplicates::CLI
+  def self.run(argv = ARGV)
+    options = {
+      threshold: 0.82,
+      min_lines: 4,
+      min_nodes: 20,
+      max_results: 50,
+      format: "text",
+      ignore_dirs: RubyDuplicates::DEFAULT_IGNORES.dup
+    }
+
+    parser = OptionParser.new do |opts|
+      opts.banner = "Usage: ruby-duplicates [options] [file-or-directory ...]"
+
+      opts.on("--threshold N", Float, "Minimum structural similarity score, default 0.82") { |value| options[:threshold] = value }
+      opts.on("--min-lines N", Integer, "Minimum method source lines, default 4") { |value| options[:min_lines] = value }
+      opts.on("--min-nodes N", Integer, "Minimum normalized syntax nodes, default 20") { |value| options[:min_nodes] = value }
+      opts.on("--max-results N", Integer, "Maximum matches to print, default 50") { |value| options[:max_results] = value }
+      opts.on("--format FORMAT", "text or json, default text") { |value| options[:format] = value }
+      opts.on("--json", "Same as --format json") { options[:format] = "json" }
+      opts.on("--ignore-dir NAME", "Directory basename or path to skip; may be repeated") { |value| options[:ignore_dirs] << value }
+    end
+
+    parser.parse!(argv)
+    options[:paths] = argv.empty? ? ["."] : argv
+
+    abort "format must be text or json" unless %w[text json].include?(options[:format])
+
+    RubyDuplicates.new(options).run
+  end
+end

data/lib/ruby_duplicates/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class RubyDuplicates
+  VERSION = "0.1.0"
+end

data/lib/ruby_duplicates.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative "ruby_duplicates/version"
+require_relative "ruby_duplicates/cli"

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: ruby-duplicates
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Bartas Urba
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: A small duplicate-code metric for Ruby that compares normalized Ripper
+  syntax fingerprints with Jaccard similarity. Inspired by Uncle Bob's dry4clj.
+email:
+- b@bartas.co
+executables:
+- ruby-duplicates
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- exe/ruby-duplicates
+- lib/ruby_duplicates.rb
+- lib/ruby_duplicates/cli.rb
+- lib/ruby_duplicates/version.rb
+homepage: https://github.com/barturba/ruby-duplicates
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/barturba/ruby-duplicates
+  inspiration_uri: https://github.com/unclebob/dry4clj
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Find structurally similar Ruby methods.
+test_files: []