semdiff 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7bb06af96fb6fc68ab1d7af274dee105769b871eed5fcae927a5525a22ec5007
+  data.tar.gz: c32ed6ac3e2ceb10d2be082cec86d0313cc3cf4d0ed11214174f92e855227d0f
+SHA512:
+  metadata.gz: c26a50d430f514e80682b9b8b848010f56775d19cf81f47a0c661d344ec670b59781a092244683f8b9e0ba0d779d0869c4705d977108a452a504a91c188d5f47
+  data.tar.gz: a7ac9c34728f1805a2b5284ca20def2f85b8a531386ca96c766bc4d5483a6ec077250c8310a074bd2b6cce342eb04fc5625a5a813cb7cec73fcab74ec8425959

data/.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = space
+indent_size = 2

data/.rubocop.yml

@@ -0,0 +1,28 @@
+inherit_from: .rubocop_todo.yml
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+AllCops:
+  NewCops: enable
+  Exclude:
+    - 'test/assets/yard/*'
+    - 'vendor/**/*'

data/.rubocop_todo.yml

@@ -0,0 +1,41 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config --exclude-limit 30`
+# on 2025-06-12 08:42:10 UTC using RuboCop version 1.76.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: Severity, Include.
+# Include: **/*.gemspec
+Gemspec/RequireMFA:
+  Exclude:
+    - 'semdiff.gemspec'
+
+# Offense count: 2
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AutoCorrect, AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods, NotImplementedExceptions.
+# NotImplementedExceptions: NotImplementedError
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/semdiff/cli/cli.rb'
+
+# Offense count: 4
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/semdiff.rb'
+    - 'lib/semdiff/cli/cli.rb'
+    - 'lib/semdiff/utils/compiler_utils.rb'
+    - 'lib/semdiff/utils/io_utils.rb'
+
+# Offense count: 3
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 136

data/.ruby-version

@@ -0,0 +1 @@
+3.1.2

data/Gemfile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in semdiff.gemspec
+gemspec
+
+group :developement, :test do
+  gem 'rake', '~> 13.0'
+
+  gem 'minitest', '~> 5.0'
+
+  gem 'rubocop'
+
+  gem 'bundler-audit'
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Tesorion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# Semdiff
+
+Semantic differences for Ruby code changes.
+
+Transforms Prism AST nodes to canonical forms whenever possible, for example: constant folding and reordering operands in expressions based on type signatures (YARD or RBS).
+
+Uses GumTree/difftastic to generate the diff on canonical ASTs.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+```
+
+## Usage
+Invoke with 2 target files and the `semdiff` executable, for example: 
+```
+semdiff --yard --gumtree --diff-original test/assets/yard/before.rb test/assets/yard/after.rb
+```
+All options:
+```
+Usage: semdiff [options] BEFORE.rb AFTER.rb
+
+Specific options:
+    -h, --help                       Prints this help
+    -o, --output-directory TARGET    Directory to output normalized unparsed files (default temporary)
+        --ignore-comments            Don't process and preserve comments in the unparsed files
+        --check-only                 Report whether there are any changes, but don't calculate them (much faster).
+        --diff-original              Show a diff of the original files above the normalized files
+        --yard [TARGET]              Use YARD (optional existing directory, default reparses input files)
+        --yard-files FILE1,FILE2,... Reparse specific YARD files (choose --yard or this, not both)
+        --rbs [TARGET]               Use RBS signatures (default ./sig/)
+        --gumtree                    Use gumtree webdiff after default difftastic
+        --skip-difftastic            Skip difftastic text diff
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/semdiff.

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/test_*.rb']
+end
+
+task default: :test

data/bin/semdiff

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'semdiff'
+
+Semdiff::CLI.new.parse(ARGV)

data/lib/semdiff/algebra_compiler.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Semdiff
+  # AlgebraCompiler is a compiler that simplifies expressions with algebraic
+  # relations based on type information.
+  #
+  # @example
+  #   ```ruby
+  #  # Expression              Canonical Form
+  #  b + a                     a + b # commutativity
+  #  c * a                     a * c # commutativity
+  #  ( a + b ) + c             a + b + c # associativity
+  #  ( x * b ) + ( x * c )     x * ( b + c ) # distributivity
+  #  x - ( - y )               x + y # negation
+  #  ( x ** m ) * ( x ** n )   x **( m + n ) # exponentials
+  #  ( a - b ) - b             a - 2* b # optimization
+  #  ```
+  class AlgebraCompiler < ::Prism::MutationCompiler
+    include Prism::DSL
+    include CompilerUtils
+
+    COMMUTATIVE_OPS = %i[+ *].freeze
+    ASSOCIATIVE_OPS = %i[+ *].freeze
+
+    def initialize(node_types)
+      @node_types = node_types
+      super()
+    end
+
+    def visit_call_node(node)
+      receiver  = visit(node.receiver)
+      arguments = visit(node.arguments)
+      block     = visit(node.block)
+      result    = nil
+
+      if COMMUTATIVE_OPS.include?(node.name) &&
+         receiver &&
+         arguments&.arguments&.size == 1 &&
+         block.nil?
+        if ASSOCIATIVE_OPS.include?(node.name)
+          operands = flatten_associative_operation(node.name, receiver, arguments.arguments.first)
+          if operands.all? { |op| numeric?(op) }
+            sorted_operands = operands.sort_by { |op| sort_key(op) }
+            result = rebuild_associative_operation(node, node.name, sorted_operands)
+          end
+        else
+          lhs = receiver
+          rhs = arguments.arguments.first
+          if numeric?(lhs) && numeric?(rhs) && should_swap?(lhs, rhs)
+            result = node.copy(
+              receiver: rhs,
+              arguments: arguments_node(
+                node_id: arguments.node_id,
+                source: arguments.send(:source),
+                location: arguments.location,
+                flags: arguments.send(:flags),
+                arguments: [lhs]
+              )
+            )
+          end
+        end
+      end
+
+      if result
+        CompilerUtils.inherit_newline(node, result)
+      else
+        node.copy(receiver: receiver, arguments: arguments, block: block)
+      end
+    end
+
+    def visit_parentheses_node(node)
+      if node.multiple_statements?
+        statements = visit_all(node.body.body)
+        body = node.body.copy(body: statements)
+        node.copy(body: body)
+      else
+        inner = visit(node.body.body.first)
+        if inner.type == :call_node &&
+           ASSOCIATIVE_OPS.include?(inner.name) &&
+           inner.arguments&.arguments&.size == 1 &&
+           numeric?(inner.receiver) &&
+           numeric?(inner.arguments.arguments.first)
+          inner
+        else
+          body = node.body.copy(body: [inner])
+          node.copy(body: body)
+        end
+      end
+    end
+
+    private
+
+    def numeric?(node)
+      case node.type
+      when :integer_node, :float_node
+        true
+      else
+        types = @node_types[node.node_id]
+        types&.anybits?(NodeTypeFlags::NUMERIC)
+      end
+    end
+
+    def should_swap?(lhs, rhs)
+      sort_key(lhs) > sort_key(rhs)
+    end
+
+    def sort_key(node)
+      case node.type
+      when :local_variable_read_node, :instance_variable_read_node,
+           :class_variable_read_node, :global_variable_read_node,
+           :constant_read_node, :call_node
+        node.name.to_s
+      when :integer_node, :float_node
+        "_#{node.value}"
+      else
+        "~#{node.type}"
+      end
+    end
+
+    def flatten_associative_operation(operation, left, right)
+      operands = []
+
+      left = unwrap_parentheses(left)
+      if left.type == :call_node &&
+         left.name == operation &&
+         left.arguments&.arguments&.size == 1 &&
+         left.block.nil?
+        operands.concat(flatten_associative_operation(operation, left.receiver, left.arguments.arguments.first))
+      else
+        operands << left
+      end
+
+      right = unwrap_parentheses(right)
+      if right.type == :call_node &&
+         right.name == operation &&
+         right.arguments&.arguments&.size == 1 &&
+         right.block.nil?
+        operands.concat(flatten_associative_operation(operation, right.receiver, right.arguments.arguments.first))
+      else
+        operands << right
+      end
+
+      operands
+    end
+
+    def unwrap_parentheses(node)
+      node = node.body.body.first while node.type == :parentheses_node && !node.multiple_statements?
+      node
+    end
+
+    def rebuild_associative_operation(original_node, operation, operands)
+      result = operands.first
+
+      operands[1..].each do |operand|
+        result = call_node(
+          node_id: original_node.node_id,
+          source: original_node.send(:source),
+          location: original_node.location,
+          flags: 0,
+          receiver: result,
+          call_operator_loc: nil,
+          name: operation,
+          message_loc: original_node.message_loc,
+          opening_loc: nil,
+          arguments: arguments_node(
+            node_id: original_node.arguments.node_id,
+            source: original_node.arguments.send(:source),
+            location: original_node.arguments.location,
+            flags: original_node.arguments.send(:flags),
+            arguments: [operand]
+          ),
+          closing_loc: nil,
+          block: nil
+        )
+      end
+
+      result
+    end
+  end
+end

data/lib/semdiff/aliasing_compiler.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Semdiff
+  # AliasingCompiler is a compiler that unifies method aliases
+  # according to the Ruby style guide (https://rubystyle.guide/)
+  # based on type information.
+  #
+  # @example
+  #   ```ruby
+  #   # Original        Canonical Form
+  #   [].collect        [].map
+  #   {}.detect         {}.find
+  #   (1..5).find_all   (1..5).select
+  #   [1, 2].inject(:+) [1, 2].reduce(:+)
+  #   [1].member?(1)    [1].include?(1)
+  #   "abc".length      "abc".size
+  #   [1,2,3].length    [1,2,3].size
+  #   ```
+  class AliasingCompiler < ::Prism::MutationCompiler
+    include Prism::DSL
+    include CompilerUtils
+
+    # NOTE: `Enumerables` (and others for `length`/`size`) typically
+    # respond to both the key and value method names in this map.
+    # We therefore assume that any Ruby object does (and should)
+    # have both methods for every pair, if they have at least one.
+    # This greatly simplifies the process of accounting for every
+    # (standard library) object and also allows us to map custom
+    # subclass (e.g., `Foo < Array`) methods automatically.
+    UNTYPED_ALIASES = {
+      collect: :map,
+      detect: :find,
+      find_all: :select,
+      inject: :reduce,
+      member?: :include?,
+      length: :size
+    }.freeze
+
+    def visit_call_node(node)
+      receiver  = visit(node.receiver)
+      arguments = visit(node.arguments)
+      block     = visit(node.block)
+      result    = nil
+
+      if UNTYPED_ALIASES.key?(node.name)
+        canonical_name = UNTYPED_ALIASES[node.name]
+        # It might be worth updating the message_loc to reflect
+        # the new size. However, the underlying message buffer
+        # would still point to the original name.
+        result = node.copy(
+          name: canonical_name,
+          receiver: receiver,
+          arguments: arguments,
+          block: block
+        )
+      end
+
+      if result
+        CompilerUtils.inherit_newline(node, result)
+      else
+        node.copy(receiver: receiver, arguments: arguments, block: block)
+      end
+    end
+  end
+end

data/lib/semdiff/cli/cli.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Semdiff
+  class CLI
+    include IOUtils
+
+    def initialize
+      @options = {}
+    end
+
+    def parse(args)
+      require 'optionparser'
+
+      parser = OptionParser.new do |opts|
+        opts.banner = 'Usage: semdiff [options] BEFORE.rb AFTER.rb'
+        opts.separator ''
+        opts.separator 'Specific options:'
+
+        opts.on('-h', '--help', 'Prints this help') do
+          puts opts
+          exit 0
+        end
+
+        opts.on('-o TARGET', '--output-directory', 'Directory to output normalized unparsed files (default temporary)', String) do |dir|
+          @options[:output_directory] = dir
+        end
+
+        opts.on('--ignore-comments', "Don't process and preserve comments in the unparsed files") do |p|
+          @options[:ignore_comments] = p
+        end
+
+        opts.on('--check-only', "Report whether there are any changes, but don't calculate them (much faster).") do |c|
+          @options[:check_only] = c
+        end
+
+        opts.on('--diff-original', 'Show a diff of the original files above the normalized files') do |d|
+          @options[:diff_original] = d
+        end
+
+        opts.on('--yard [TARGET]', 'Use YARD (optional existing directory, default reparses input files)') do |target|
+          @options[:annotation_type] = :yard
+          @options[:annotation_target] = target
+          @options[:override] = true if target.nil?
+        end
+
+        opts.on('--yard-files FILE1,FILE2,...', 'Reparse specific YARD files (choose --yard or this, not both)', Array) do |files|
+          @options[:annotation_type] = :yard
+          @options[:annotation_target] = files
+        end
+
+        opts.on('--rbs [TARGET]', 'Use RBS signatures (default ./sig/)') do |target|
+          @options[:annotation_type] = :rbs
+          @options[:annotation_target] = target || 'sig'
+        end
+
+        opts.on('--gumtree', 'Use gumtree webdiff after default difftastic') do |g|
+          @options[:gumtree] = g
+        end
+
+        opts.on('--skip-difftastic', 'Skip difftastic text diff') do |d|
+          @options[:skip_difftastic] = d
+        end
+      end
+
+      begin
+        files = parser.parse!(args)
+        raise ArgumentError, "Expected 2 files but received #{files.size}" if files.size != 2
+
+        types = case @options[:annotation_type]
+                when :yard
+                  ::Typeguard::TypeModel::Builder.yard
+                  @options[:annotation_target] ||= files
+                  ::Typeguard::TypeModel::Builder::IMPLEMENTATION.new(
+                    @options[:annotation_target],
+                    @options[:annotation_target].is_a?(Array)
+                  ).build
+                when :rbs
+                  ::Typeguard::TypeModel::Builder.rbs
+                  ::Typeguard::TypeModel::Builder::IMPLEMENTATION.new(
+                    @options[:annotation_target],
+                    false
+                  ).build
+                end
+
+        processed_asts = files.map do |file|
+          contents = File.read(file)
+          original_result = Prism.parse(contents)
+          ast = original_result.value
+          ast = ast.accept(Prism::DesugarCompiler.new)
+          ast = ast.accept(AliasingCompiler.new)
+          ast = ast.accept(StructuresCompiler.new)
+          node_types = TypeVisitor.new(types).visit(ast) unless types.nil?
+          ast = ast.accept(ConstantsCompiler.new)
+          ast = ast.accept(AlgebraCompiler.new(node_types)) unless types.nil?
+          ast = ast.accept(IdentityCompiler.new(node_types)) unless types.nil?
+          ast = ast.accept(ConstantsCompiler.new)
+
+          # NOTE: Uses the translation parser builder to
+          # translate the existing AST to whitequark/parser,
+          # so that we can rewrite with unparser and feed the
+          # files to difftastic/GumTree.
+          translator = Prism::Translation::Parser.new(
+            parser: Struct.new(:prism_ast, :original_result) do
+              def parse(source, **options)
+                Struct.new(:value, :comments, :magic_comments, :data_loc, :errors, :warnings, :source).new(
+                  prism_ast,
+                  original_result.comments,
+                  original_result.magic_comments,
+                  original_result.data_loc,
+                  original_result.errors,
+                  original_result.warnings,
+                  original_result.source
+                )
+              end
+            end.new(ast, original_result)
+          )
+          source_buffer = Parser::Source::Buffer.new(file)
+          source_buffer.source = contents
+          translator.send(@options[:ignore_comments] ? 'parse' : 'parse_with_comments', source_buffer)
+        end
+
+        unparsed = processed_asts.map do |r|
+          r.is_a?(Array) ? Unparser.unparse(r.first, comments: r.last) : Unparser.unparse(r)
+        end
+        file_names = files.map { |f| File.basename(f) }
+        if !@options[:skip_difftastic] && @options[:diff_original]
+          system <<~CMD
+            difft #{files.first} #{files.last} \
+            #{'--check-only --exit-code' if @options[:check_only]} \
+            #{'--ignore-comments' if @options[:ignore_comments]}
+          CMD
+        end
+        return if @options[:skip_difftastic] && !@options[:gumtree]
+
+        with_diff_files(unparsed, file_names, output_directory: @options[:output_directory]) do |b, a|
+          unless @options[:skip_difftastic]
+            system <<~CMD
+              difft #{b.path} #{a.path} \
+              #{'--check-only --exit-code' if @options[:check_only]} \
+              #{'--ignore-comments' if @options[:ignore_comments]}
+            CMD
+          end
+          if @options[:gumtree]
+            system <<~CMD
+              gumtree webdiff #{b.path} #{a.path} \
+              -g ruby-treesitter-ng
+            CMD
+          end
+        end
+      rescue OptionParser::InvalidOption => e
+        puts e.message
+        puts parser
+        puts "Invalid argument (use semdiff --help): #{e}"
+        exit 1
+      end
+
+      @options
+    end
+  end
+end