syntax_finder 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f396dd84ec243335733edbe3abc5c22c0f32ea6ade48dfc16336ad385f0a2bd8
+  data.tar.gz: 3e6c73de9f48e65ed5f645b7c0b8c25c139826202d739ca884545c1ff48a7b40
+SHA512:
+  metadata.gz: 0ce254cd70a3952f0a1d976cbd5ba9fe776ebb81137d8387d19b917548e4e3a7c5521ecdece26c70ebd712a67491327d1379855ad4dc058780f845b4bbaef0d0
+  data.tar.gz: 78a82aa09159d3cf9e764ea342c71e60a21574f0b55acfb76510ab802bdb8a9e7b5a042c200fdf62704958dce4304aa864f3d9a66211c19f707eb55ab46c58a8

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Koichi Sasada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# SyntaxFinder
+
+Find Ruby syntax patterns with Prism.
+
+## How to use
+
+Write your pattern in Ruby.
+
+```ruby
+# samples/if_then_finder.rb
+
+require 'syntax_finder'
+
+# Count up all `if` statements and `then` keywords.
+
+class IfThenFinder < SyntaxFinder
+  def look node
+    if node.type == :if_node
+      inc :if
+      inc :then if node.then_keyword_loc
+    end
+  end
+end
+```
+
+and run the script with Ruby script file names listed in STDIN like this.
+
+```sh
+$ find ruby/ruby -name '*.rb' | ruby samples/if_then_finder.rb
+[[:if, 32153], [:FILES, 9558], [:then, 3627], [:FAILED_PARSE, 5]]
+```
+
+In this case, with *.rb files in ruby/ruby directory, there are
+
+* 9,558 files
+* 32,153 if statements
+* 3,627 then keywords
+* 5 files are failed because of parsing.
+
+You can specify files with arguments like that:
+
+```sh
+$ ruby samples/if_then_finder.rb samples/*.rb
+[[[:FILES, 16]], [[:if, 26], [:then, 1]]]
+```
+
+You can specify `-j` or `-jN` for parallel processing like that:
+
+```sh
+$ time find ruby/ruby -name '*.rb' | ruby samples/if_then_finder.rb
+[[:if, 32153], [:FILES, 9558], [:then, 3627], [:FAILED_PARSE, 5]]
+
+real    0m3.627s
+user    0m3.451s
+sys     0m0.219s
+
+$ time find ruby/ruby -name '*.rb' | ruby samples/if_then_finder.rb -j
+[[:if, 32153], [:FILES, 9558], [:then, 3627], [:FAILED_PARSE, 5]]
+
+real    0m0.962s
+user    0m7.944s
+sys     0m0.854s
+```
+
+## How to write a finder
+
+1. Define a class derived with `SyntaxFinder` class.
+2. Define a `look(node)` method with your expected pattern. `node` is Prism's node.
+3. Use `inc(key)` if you want to aggregate the statistics in `look()`.
+
+See `examples/` for more examples.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test-unit` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ko1/syntax_finder.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test

data/lib/syntax_finder/version.rb

@@ -0,0 +1,3 @@
+class SyntaxFinder
+  VERSION = '0.1.0'
+end

data/lib/syntax_finder.rb

@@ -0,0 +1,272 @@
+require 'prism'
+require 'optparse'
+require_relative 'syntax_finder/version'
+
+class SyntaxFinder
+  def initialize file
+    @file = file
+  end
+
+  ### Traversing and looking methods
+
+  # Please write your pattern here to investigate the node.
+  # This method is called for each node.
+  def look node
+    # override here
+  end
+
+  # If you need to pass some context while traversing,
+  # redefine this method with context like `def traverse node, context = nil`.
+  def traverse node
+    look node
+    traverse_rest node
+  end
+
+  def traverse_rest node
+    node.child_nodes.compact.each do |child|
+      traverse child
+    end
+  end 
+
+  ### Utility methods for look
+
+  # node location information
+  def nloc node
+    "#{@file}:#{node.location.start_line}"
+  end
+
+  # node line (1st line of node lines)
+  def nline node
+    node.slice_lines.lines.first.chomp
+  end
+
+  # node lines
+  def nlines node
+    node.slice_lines
+  end
+
+  def success? r
+    return true if r.success?
+
+    if r.errors.size == 1 &&
+       r.errors.first.type == :script_not_found
+      true
+    else
+      false
+    end
+  end
+
+  # any object can be a key.
+  # if key is [A-Z_]+, it's system key.
+  # it's returns true if the key is already exists.
+  def inc key
+    has_key = @@result.has_key? key
+    @@result[key] += 1
+    has_key
+  end
+
+  def note key, note
+    @@result_notes[key] << note
+  end
+
+  def self.inc key
+    @@result[key] += 1
+  end
+
+  def self.note key, note
+    @@result_notes[key] << note
+  end
+
+  ### Execution management
+
+  def start_traverse
+    puts "Start #{@file}" if $DEBUG
+
+    @root = Prism.parse_file(@file)
+    if success?(@root)
+      traverse @ast = @root.value
+    else
+      # pp [@file, @root.errors]
+      # inc :FAILED, @file
+      inc :FAILED_PARSE
+    end
+  rescue Interrupt
+    exit
+  rescue Exception => e
+    pp [e, @file, e.backtrace] if $DEBUG
+    inc [:FAILED, e.class]
+  end
+
+  @@finders = [self]
+
+  def self.inherited klass
+    @@finders << klass
+  end
+
+  def self.last_finder
+    @@finders.last
+  end
+
+  @@result = Hash.new(0)
+  @@result_notes = Hash.new{|h, k| h[k] = []}
+  @@opt = {}
+
+  def self.setup_parallel finder, pn
+    taskq = @@opt[:taskq] = Queue.new
+    resq  = @@opt[:resq] = Queue.new
+    term_cmd = '-- terminate'
+
+    @@opt[:threads] = pn.times.map{
+      Thread.new _1 do |ti|
+        task_r, task_w = IO.pipe
+        res_r, res_w = IO.pipe
+
+        pid = fork do
+          trap(:INT, 'IGNORE')
+
+          loop do
+            file = task_r.gets.chomp
+
+            case file
+            when  term_cmd
+              @@result_notes.default_proc = nil
+              last_result = [@@result, @@result_notes]
+              break
+            else
+              kick finder, file
+              last_result = nil
+            end
+          rescue Interrupt
+            STDERR.puts "Worker is interrupted (pid: #{Process.pid})"
+            @@result_notes.default_proc = nil
+            last_result = [@@result, @@result_notes]
+            exit
+          ensure
+            res_w.puts Marshal.dump(last_result).dump
+          end
+        end
+
+        result = nil
+
+        while true
+          file = taskq.pop
+          task_w.puts file || term_cmd
+          result = Marshal.load(res_r.gets.chomp.undump) # wait for the result
+          
+          if result
+            resq << result
+            break
+          end
+        end
+      ensure
+        p [Thread.current, $!] if $DEBUG
+        Process.kill :KILL, pid
+        Process.waitpid pid
+      end
+    }
+  end
+
+  def self.cleanup_parallel
+    return unless taskq = @@opt[:taskq]
+    trap(:INT){
+      trap(:INT, 'DEFAULT')
+    }
+    taskq.close
+
+    @@opt[:parallel].times do |i|
+      results, notes = @@opt[:resq].pop
+      @@result.merge!(results){|k, a, b| a + b}
+      @@result_notes.merge!(notes)
+    end
+  end
+
+  def self.parse_opt finder, argv
+    o = OptionParser.new
+    o.on '-v', '--verbose', 'VERBOSE mode' do
+      $VERBOSE = true
+    end
+
+    o.on '-d', '--debug' do
+      $DEBUG = true
+    end
+
+    o.on '-q', '--quiet' do
+      @@opt[:quiet] = true
+    end
+
+    o.on '-j', '--jobs[=N]', 'Parallel mode' do
+      require 'etc'
+
+      pn = @@opt[:parallel] = _1&.to_i || Etc.nprocessors
+      setup_parallel finder, pn
+    end
+
+    o.parse! argv
+    pp options: @@opt if $DEBUG
+  end
+
+  def self.kick finder, file
+    finder.new(file).start_traverse
+  end
+
+  def self.aggregate_results
+    if @@result_notes.size > 0
+      results =  @@result.sort_by{|k, v| -v}.map{|k, v|
+        if !(note = @@result_notes[k]).empty?
+          [k, v, note]
+        else
+          [k, v]
+        end
+      }
+    else
+      results = @@result.sort_by{|k, v| -v}
+    end
+
+    results.partition{|key,|
+      Symbol === key && key.length > 1 && /^[A-Z_]+$/ =~ key
+    }
+  end
+
+  def self.print_result
+    system_results, user_results = aggregate_results
+    pp [system_results, user_results]
+  end
+
+  def self.run finder, argv = ARGV
+    @already_run = true
+    parse_opt finder, argv
+
+    files = argv.empty? ? ARGF : argv
+    files.each do |f|
+      f = f.scrub.strip
+      next unless FileTest.file?(f)
+  
+      inc :FILES
+  
+      if q = @@opt[:taskq]
+        q << f
+      else
+        kick finder, f
+      end
+    end
+  rescue Interrupt
+    STDERR.puts "-- Interrupted"
+    exit
+  ensure
+    cleanup_parallel
+    finder.print_result if finder
+  end
+
+  END{
+    if !@already_run && @@finders.size == 2
+      self.run last_finder
+    end
+  }
+end
+
+# for pretty integer results
+class Integer
+  def inspect
+    self.to_s.gsub(/(\d)(?=\d{3}+$)/, '\\1_')
+  end
+end

data/samples/all_lvar_finder.rb

@@ -0,0 +1,27 @@
+require 'syntax_finder'
+
+# Cound up all of local variable names.
+
+class LVarFinder < SyntaxFinder
+  def look node
+    if node.respond_to? :locals
+      node.locals.each{|lv|
+        case lv
+        when Symbol
+          inc lv
+        when Prism::BlockLocalVariableNode
+          inc lv.name
+        else
+          pp [node.type, lv.type, node.locals].inspect
+          exit
+        end
+      }
+    end
+  end
+
+  def self.print_result
+    @@result.sort_by{|lv, v| [-v, lv]}.each{|lv, v|
+      puts "#{lv}\t#{v}" 
+    }
+  end
+end

data/samples/all_method_name_finder.rb

@@ -0,0 +1,22 @@
+require 'syntax_finder'
+
+# Cound up all of method names.
+
+class MethodNameFinder < SyntaxFinder
+  def look node
+    if node.type == :call_node
+      inc [:call, node.name]
+      # pp [nloc(node),nlines(node).lines.first.chomp] unless @@opt[:quiet]
+    elsif node.type == :def_node
+      inc [:def, node.name]
+      # pp [nloc(node),nlines(node).lines.first.chomp] unless @@opt[:quiet]
+    end
+  end
+
+  def self.print_result
+    @@result.sort_by{|(t, m), v| [t, -v]}.each{|(t, m), v|
+      puts "#{v}\t#{t}\t#{m}" 
+    }
+  end
+end
+

data/samples/all_regexp_finder.rb

@@ -0,0 +1,13 @@
+relative 'syntax_finder'
+
+# Cound up all of regexps.
+
+class RegexpFinder < SyntaxFinder
+  def look node
+    if node.type == :regular_expression_node
+      # pp [[nloc(node), nlines(node).lines]] unless @@opt[:quiet]
+      inc :regexp
+      inc node.unescaped
+    end
+  end
+end

data/samples/assert_equal_paren.rb

@@ -0,0 +1,17 @@
+require 'syntax_finder'
+
+# cound `assert*(params)` and `assert params` patterns.`
+
+class AssertEqualParenFinder < SyntaxFinder
+  def look node
+    if node.type == :call_node && 
+       /^assert/ =~ node.name && node.arguments
+    then
+      if node.opening_loc
+        inc :paren
+      else
+        inc :no_paren
+      end
+    end
+  end
+end

data/samples/call_paren_finder.rb

@@ -0,0 +1,13 @@
+require 'syntax_finder'
+
+# Check call with paren or no paren
+
+class IfThenFinder < SyntaxFinder
+  def look node
+    if node.type == :call_node
+      has_params = !node.arguments.nil?
+      has_paren = !node.opening_loc.nil?
+      inc paren: has_paren, params: has_params
+    end
+  end
+end

data/samples/def_params_finder.rb

@@ -0,0 +1,27 @@
+require 'syntax_finder'
+
+# Check def with paren or no paren
+
+class DefParamsFinder < SyntaxFinder
+  def look node
+    if node.type == :def_node
+      inc :def
+      if node.parameters
+        params = {
+          required: node.parameters.requireds.size,
+          optional: node.parameters.optionals.size,
+          rest: !node.parameters.rest.nil?,
+          posts: node.parameters.posts.size,
+          keywords: node.parameters.keywords.size,
+          keyword_rest: !node.parameters.keyword_rest.nil?
+        }.reject { |_, v| v == 0 || v == false }
+      else
+        params = {required: 0}
+      end
+
+      unless inc params
+        note params, nloc(node)
+      end
+    end
+  end
+end

data/samples/def_paren_finder.rb

@@ -0,0 +1,13 @@
+require 'syntax_finder'
+
+# Check def with paren or no paren
+
+class IfThenFinder < SyntaxFinder
+  def look node
+    if node.type == :def_node
+      has_params = !node.parameters.nil?
+      has_paren = !node.lparen_loc.nil?
+      inc paren: has_paren, params: has_params
+    end
+  end
+end

data/samples/if_cond_cont_indent_finder.rb

@@ -0,0 +1,29 @@
+require 'syntax_finder'
+
+# Check indentation with `if` and `and`/`or`
+
+class IfCondContIndnetFinder < SyntaxFinder
+  def look node
+    if node.type == :if_node
+      inc :if
+      inc :then if node.then_keyword_loc
+      cond = node.predicate
+
+      case cond.type
+      when :and_node, :or_node
+        inc op: cond.operator_loc.slice
+        d = cond.location.end_line - cond.location.start_line
+        if d > 0
+          base = node.location.start_column
+          rest_lines = nlines(cond).lines[1..]
+          inc indent: rest_lines.map{|line|
+            /^(\s*)/ =~ line
+            $1.size - base
+          }.min
+          # puts nlines(cond)
+        end
+      end
+    end
+  end
+end
+

data/samples/if_then_finder.rb

@@ -0,0 +1,12 @@
+require 'syntax_finder'
+
+# Count up all `if` statements and `then` keywords.
+
+class IfThenFinder < SyntaxFinder
+  def look node
+    if node.type == :if_node
+      inc :if
+      inc :then if node.then_keyword_loc
+    end
+  end
+end

data/samples/integer_range_finder.rb

@@ -0,0 +1,18 @@
+require 'syntax_finder'
+
+class IntegerRangeFinder < SYntaxFinder
+  def look node
+    if node.type == :range_node
+       left = node.left 
+       right = node.right
+
+       if((left.nil? || left.type == :integer_node) &&
+          (right.nil? || right.type == :integer_node))
+
+        inc :integer_range
+        # pp [[nloc(node), nlines(node).lines]] unless @@opt[:quiet]
+        inc [left&.value, right&.value].inspect
+       end
+    end
+  end
+end

data/samples/integer_size_finder.rb

@@ -0,0 +1,10 @@
+require 'syntax_finder'
+
+class IntegerSizeFinder < SyntaxFinder
+  def look node
+    if node.type == :call_node && node.name == :'size' && 
+       node.receiver&.type == :integer_node
+      pp [nloc(node), nlines(node).lines.first.chomp]
+    end
+  end
+end

data/samples/method_name_finder.rb

@@ -0,0 +1,18 @@
+require 'syntax_finder'
+
+# count all of given method names via comma separated environment variables,
+# like `NAMES=map,collect`
+
+NAMES = ENV['NAMES']&.yield_self{|m| m.split(',').map{|m| m.strip.to_sym}}.to_h{|e| [e, true]} || raise
+
+class MethodNameFinder < SyntaxFinder
+  def look node
+    case t = node.type
+    when :call_node, :def_node
+      if NAMES[name = node.name]
+        inc [t, name]
+        pp [nloc(node),nlines(node).lines.first.chomp] if $VERBOSE
+      end
+    end
+  end
+end

data/samples/pragma_finder.rb

@@ -0,0 +1,17 @@
+require 'syntax_finder'
+
+class PragmaFinder < SyntaxFinder
+  def look node
+    @root.magic_comments.each{|c|
+      inc key = [c.key, c.value]
+      note key, @file if $VERBOSE
+    }
+
+    # No need for recursive travaersal.
+  end
+
+  def traverse node
+    look node
+    # check only root node
+  end
+end

data/samples/required_kwcall_wo_paren_finder.rb

@@ -0,0 +1,18 @@
+require 'syntax_finder'
+
+class ReqKWwoParenFinder < SyntaxFinder
+  def look node
+    if node.type == :call_node &&
+       node.opening_loc.nil? &&
+       node.arguments &&
+       node.arguments.arguments.any?{|n| n.type == :keyword_hash_node && n.elements.any?{|e|
+         e.type == :assoc_node &&
+         e.operator_loc.nil? &&
+         e.key.location.start_line != e.value.location.start_line}}
+
+      inc :found
+      puts "# #{nloc(node)}"
+      puts '> ' + nlines(node).lines.join('> ')
+    end
+  end
+end

data/samples/required_kwdef_wo_paren_finder.rb

@@ -0,0 +1,14 @@
+require 'syntax_finder'
+
+class ReqKWwoParenFinder < SyntaxFinder
+  def look node
+    if node.type == :def_node &&
+       node.lparen_loc.nil? &&
+       node.parameters&.keywords&.last&.type == :required_keyword_parameter_node &&
+       node.parameters&.block.nil?
+
+      inc :found
+      pp [nloc(node), nlines(node).lines.first.chomp]
+    end
+  end
+end

data/samples/singleton_class_in_def_finder.rb

@@ -0,0 +1,18 @@
+require 'syntax_finder'
+
+class SingletonClassInDefFinder < SyntaxFinder
+  # use traverse instead of look
+  def traverse node, in_def = nil, in_sclass = nil
+    if node.type == :class_node && in_sclass && node.constant_path.type != :constant_read_node
+      pp [nloc(node), nlines(node).lines.first.chomp]
+    end
+
+    if node.type == :def_node
+      in_def = true; in_sclass = false
+    elsif node.type == :singleton_class_node && in_def
+      in_sclass = true
+    end
+
+    node.child_nodes.compact.each{|n| traverse n, in_def, in_sclass}
+  end
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: syntax_finder
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Koichi Sasada
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-01-14 00:00:00.000000000 Z
+dependencies: []
+description: Analyze your Ruby scripts with prism.
+email:
+- ko1@atdot.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/syntax_finder.rb
+- lib/syntax_finder/version.rb
+- samples/all_lvar_finder.rb
+- samples/all_method_name_finder.rb
+- samples/all_regexp_finder.rb
+- samples/assert_equal_paren.rb
+- samples/call_paren_finder.rb
+- samples/def_params_finder.rb
+- samples/def_paren_finder.rb
+- samples/if_cond_cont_indent_finder.rb
+- samples/if_then_finder.rb
+- samples/integer_range_finder.rb
+- samples/integer_size_finder.rb
+- samples/method_name_finder.rb
+- samples/pragma_finder.rb
+- samples/required_kwcall_wo_paren_finder.rb
+- samples/required_kwdef_wo_paren_finder.rb
+- samples/singleton_class_in_def_finder.rb
+homepage: https://github.com/ko1/syntax_finder
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/ko1/syntax_finder
+  source_code_uri: https://github.com/ko1/syntax_finder
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key:
+specification_version: 4
+summary: Analyze your Ruby scripts with prism.
+test_files: []