ast_utils 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e1090f235aaa912ae5a85844b1e10463faaf584d
+  data.tar.gz: 14426ab5c982587d9e8a2dcc910addf95af061df
+SHA512:
+  metadata.gz: d699fb6c01dd21bdb32a4c95d4786167aa4a8581ef49c1facab6ee2e1fbd885f2e36dcdb6e71e8d8e20e686d75fa9bb7265c96c4896541a43e3e28a770c7fb7b
+  data.tar.gz: f8b6a0131c03bb38afcaa03cca9724afba47f6af3e0127aa8b8dac838d884eb6a9d0172b65fe98908cf916c8e0f384f3a6d676edc4df4a98d04b4228c785c0b9

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.13.7

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ast_utils.gemspec
+gemspec

data/README.md

@@ -0,0 +1,80 @@
+# ASTUtils
+
+ASTUtils provides some utility over parser gem AST, which aims to help analyzing Ruby programs.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ast_utils'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ast_utils
+
+## Usage
+
+### ASTUtils::Labeling
+
+`ASTUTils::Labeling` implements local variables labeling.
+
+In Ruby, local variable scope can be nested, and identification on local variables is not trivial.
+This utility give labels to identify local variables.
+
+```rb
+require "ast_utils"
+
+node = Parser::CurrentRuby.parse(source)
+labeled = ASTUtils::Labeling.translate(node: node)
+```
+
+### ASTUTils::Navigation
+
+`AST` has `children` but no pointer to its parent.
+
+```rb
+require "ast_utils"
+
+node = Parser::CurrentRuby.parse(source)
+navigation = ASTUtils::Navigation.from(node: node)
+
+parent = navigation.parent(child_node)
+```
+
+### ASTUtils::Scope
+
+`Scope` is about *scope* in Ruby.
+
+It associates outer scope and its inner scope.
+
+```rb
+require "ast_utils"
+
+node = Parser::CurrentRuby.parse(source)
+labeled = ASTUtils::Labeling.translate(node: node)
+scope = ASTUtils::Scope.from(node: labeled)
+
+scope.root
+scope.children(root)
+scope.subs(root)
+```
+
+`children` includes `def` and iterator block, but `subs` only includes blocks.
+
+It also defines `assignments` and `references` to refere assignments and references for local variables in the scope.
+
+`assignments` returns pair of:
+
+* Node which implements assignment
+* Labeled local variable name which is assigned
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/soutaro/ast_utils.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/ast_utils.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ast_utils/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ast_utils"
+  spec.version       = ASTUtils::VERSION
+  spec.authors       = ["Soutaro Matsumoto"]
+  spec.email         = ["matsumoto@soutaro.com"]
+
+  spec.summary       = %q{Ruby AST Utility}
+  spec.description   = %q{Ruby AST Utility}
+  spec.homepage      = "https://github.com/soutaro/ast_utils"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+
+  spec.add_runtime_dependency "parser", "~> 2.4"
+  spec.add_runtime_dependency "thor", "~> 0.19.4"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ast_utils"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/ast_utils

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(__dir__, "../lib")
+require "ast_utils"
+require "ast_utils/cli"
+
+ASTUtils::CLI.start(ARGV)

data/lib/ast_utils/cli.rb

@@ -0,0 +1,16 @@
+require "thor"
+
+module ASTUtils
+  class CLI < Thor
+    desc "label SCRIPTS...", "labeling Ruby scripts given as SCRIPTS..."
+    def label(*scripts)
+      scripts.map {|script| Pathname(script) }.each do |path|
+        puts "Parsing #{path}..."
+        node = Parser::CurrentRuby.parse(path.read, path.to_s)
+        puts "Translating node..."
+        labeled = Labeling.translate(node: node)
+        puts "#{labeled.inspect}"
+      end
+    end
+  end
+end

data/lib/ast_utils/labeling.rb

@@ -0,0 +1,178 @@
+module ASTUtils
+  class Labeling
+    class LabeledName
+      attr_reader :name
+      attr_reader :label
+
+      def initialize(name:, label:)
+        @name = name
+        @label = label
+      end
+
+      def inspect
+        "#{name}@#{label}"
+      end
+
+      def hash
+        name.hash ^ label.hash
+      end
+
+      def ==(other)
+        other.is_a?(LabeledName) && other.name == name && other.label == label
+      end
+
+      def equal?(other)
+        self == other
+      end
+
+      def eql?(other)
+        self == other
+      end
+    end
+
+    include NodeHelper
+
+    attr_accessor :counter
+
+    def initialize
+      self.counter = 0
+    end
+
+    def next_label!
+      self.counter += 1
+
+      new_label = self.counter
+
+      if block_given?
+        yield new_label
+      else
+        new_label
+      end
+    end
+
+    def translate(node, env)
+      case node.type
+      when :lvasgn
+        children = PartialMap.apply(node.children) do |map|
+          map.on!(0) {|name| lookup_env(name: name, env: env) }
+          map.on?(1) {|child| translate(child, env) }
+        end
+
+        node.updated(nil, children, nil)
+
+      when :lvar
+        children = replace(node.children, 0) {|name| lookup_env(name: name, env: env) }
+        node.updated(nil, children, nil)
+
+      when :arg, :restarg, :kwarg, :kwrestarg, :blockarg
+        name = node.children[0]
+
+        labeled_name = LabeledName.new(name: name, label: next_label!)
+        env[name] = labeled_name
+
+        children = replace(node.children, 0) {|_| labeled_name }
+        node.updated(nil, children, nil)
+
+      when :procarg0
+        case node.children[0]
+        when AST::Node
+          children = map_child_node(node) {|child| translate(child, env) }
+          node.updated(nil, children, nil)
+        when Symbol
+          name = node.children[0]
+
+          labeled_name = LabeledName.new(name: name, label: next_label!)
+          env[name] = labeled_name
+
+          children = replace(node.children, 0) {|_| labeled_name }
+          node.updated(nil, children, nil)
+        else
+          raise "Unexpected node structure: #{node}"
+        end
+      when :optarg, :kwoptarg
+        children = PartialMap.apply(node.children) do |map|
+          map.on!(0) {|name| lookup_env(name: name, env: env) }
+          map.on?(1) {|child| translate(child, env) }
+        end
+
+        node.updated(nil, children, nil)
+
+      when :def
+        env_ = {}
+        children = map_child_node(node) {|child| translate(child, env_) }
+        node.updated(nil, children, nil)
+
+      when :block
+        children = node.children.dup
+        translate_child!(children, 0, env)
+
+        block_env = env.dup
+        translate_child!(children, 1, block_env)
+        translate_child!(children, 2, block_env)
+
+        node.updated(nil, children, nil)
+
+      when :class
+        children = PartialMap.apply(node.children) do |map|
+          map.on!(0) {|child| translate(child, env) }
+          map.on?(1) {|child| translate(child, env) }
+          map.on?(2) {|child| translate(child, {}) }
+        end
+
+        node.updated(nil, children, nil)
+
+      when :module
+        children = PartialMap.apply(node.children) do |map|
+          map.on!(0) {|child| translate(child, env) }
+          map.on?(1) {|child| translate(child, {}) }
+        end
+
+        node.updated(nil, children, nil)
+
+      when :match_with_lvasgn
+        names = self.class.extract_variables(node.children[0].children[0].children[0])
+        vars = names.map {|name| lookup_env(name: name, env: env) }
+
+        children = map_child_node(node) {|child| translate(child, env) }
+        node.updated(nil, children + [vars], nil)
+
+      else
+        children = map_child_node(node) {|child| translate(child, env) }
+        node.updated(nil, children, nil)
+      end
+    end
+
+    def lookup_env(name:, env:)
+      labeled_name = env[name]
+
+      unless labeled_name
+        labeled_name = LabeledName.new(name: name, label: next_label!)
+        env[name] = labeled_name
+      end
+
+      labeled_name
+    end
+
+    def translate_child!(children, index, env)
+      if children[index]
+        children[index] = translate(children[index], env)
+      end
+    end
+
+    def replace(array, index)
+      array = array.dup
+      array[index] = yield(array[index])
+      array
+    end
+
+    def self.extract_variables(string)
+      string.scan(/\(\?(\<([\w_]+)\>)|(\'([\w_]+)\')/).map do |_, b, _, d|
+        (b || d).to_sym
+      end
+    end
+
+    def self.translate(node:)
+      self.new.translate(node, {})
+    end
+  end
+end

data/lib/ast_utils/navigation.rb

@@ -0,0 +1,36 @@
+module ASTUtils
+  class Navigation
+    include NodeHelper
+
+    attr_reader :nodes
+    attr_reader :parents
+    attr_reader :root
+
+    def initialize(node:)
+      @root = node
+      @nodes = NodeSet.new
+      @parents = {}
+    end
+
+    def construct
+      set_parent(root)
+    end
+
+    def set_parent(node)
+      nodes << node
+
+      each_child_node(node) do |child|
+        parents[child.__id__] = node
+        set_parent(child)
+      end
+    end
+
+    def parent(node)
+      parents[node.__id__]
+    end
+
+    def self.from(node:)
+      new(node: node).tap {|nav| nav.construct }
+    end
+  end
+end

data/lib/ast_utils/node_helper.rb

@@ -0,0 +1,30 @@
+module ASTUtils
+  module NodeHelper
+    def each_child_node(node)
+      if block_given?
+        node.children.each do |child|
+          if child.is_a?(AST::Node)
+            yield child
+          end
+        end
+      else
+        enum_for :each_child_node, node
+      end
+    end
+
+    # order preserved
+    def map_child_node(node)
+      if block_given?
+        node.children.each.with_object([]) do |child, array|
+          if child.is_a?(AST::Node)
+            array << yield(child)
+          else
+            array << child
+          end
+        end
+      else
+        enum_for :map_child_node, node
+      end
+    end
+  end
+end

data/lib/ast_utils/node_set.rb

@@ -0,0 +1,55 @@
+module ASTUtils
+  class NodeSet
+    class Item
+      attr_reader :object
+
+      def initialize(object)
+        @object = object
+      end
+
+      def hash
+        object.__id__
+      end
+
+      def eql?(other)
+        other.is_a?(Item) && other.object.__id__ == object.__id__
+      end
+
+      def ==(other)
+        eql?(other)
+      end
+    end
+
+    attr_reader :set
+
+    def initialize(objects = [])
+      @set = Set.new(objects.map {|object| Item.new(object) })
+    end
+
+    def <<(node)
+      set << Item.new(node)
+    end
+
+    def delete(node)
+      set.delete Item.new(node)
+    end
+
+    def each(&block)
+      set.map(&:object).each(&block)
+    end
+
+    def empty?
+      size == 0
+    end
+
+    def size
+      set.size
+    end
+
+    include Enumerable
+
+    def +(other)
+      self.class.new(self.set + other.set)
+    end
+  end
+end

data/lib/ast_utils/partial_map.rb

@@ -0,0 +1,53 @@
+module ASTUtils
+  class PartialMap
+    attr_reader :enumerable
+    attr_reader :updaters
+
+    def initialize(enumerable)
+      @enumerable = enumerable
+      @updaters = Array.new(enumerable.count)
+    end
+
+    def self.apply(enumerable)
+      map = new(enumerable)
+      yield map
+      map.apply
+    end
+
+    def on(index, &block)
+      updaters[index] = [:on, block]
+    end
+
+    def on!(index, &block)
+      updaters[index] = [:on!, block]
+    end
+
+    def on?(index, &block)
+      updaters[index] = [:on?, block]
+    end
+
+    def apply
+      enumerable.map.with_index do |value, index|
+        updater = updaters[index]
+
+        if updater
+          case updater&.first
+          when :on
+            updater.last[value]
+          when :on!
+            raise if value.nil?
+            updater.last[value]
+          when :on?
+            unless value.nil?
+              updater.last[value]
+            else
+              value
+            end
+          end
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/ast_utils/scope.rb

@@ -0,0 +1,187 @@
+module ASTUtils
+  class Scope
+    class Assignment
+      attr_reader :node, :variable
+
+      def initialize(node:, variable:)
+        @node = node
+        @variable = variable
+      end
+
+      def hash
+        node.__id__ ^ variable.hash
+      end
+
+      def eql?(other)
+        other.is_a?(Assignment) && other.node.__id__ == node.__id__ && other.variable == variable
+      end
+
+      def ==(other)
+        eql?(other)
+      end
+    end
+
+    include NodeHelper
+
+    attr_reader :root
+    attr_reader :all_scopes
+    attr_reader :child_scopes
+    attr_reader :parent_scopes
+    attr_reader :sub_scopes
+    attr_reader :super_scopes
+    attr_reader :assignment_nodes
+    attr_reader :reference_nodes
+
+    def initialize(root:)
+      @root = root
+      @all_scopes = NodeSet.new
+      @child_scopes = {}
+      @parent_scopes = {}
+      @sub_scopes = {}
+      @super_scopes = {}
+      @assignment_nodes = {}
+      @reference_nodes = {}
+    end
+
+    def children(scope)
+      child_scopes[scope.__id__]
+    end
+
+    def parent(scope)
+      parent_scopes[scope.__id__]
+    end
+
+    def subs(scope)
+      sub_scopes[scope.__id__]
+    end
+
+    def sup(scope)
+      super_scopes[scope.__id__]
+    end
+
+    def assignments(scope, include_subs: false)
+      if include_subs
+        subs(scope).inject(assignment_nodes[scope.__id__]) {|assignments, scope_|
+          assignments + assignments(scope_, include_subst: true)
+        }
+      else
+        assignment_nodes[scope.__id__]
+      end
+    end
+
+    def references(scope, include_subs: false)
+      if include_subs
+        subs(scope).inject(reference_nodes[scope.__id__]) {|references, scope_|
+          references + references(scope_, include_subs: true)
+        }
+      else
+        reference_nodes[scope.__id__]
+      end
+
+    end
+
+    def each(&block)
+      all_scopes.each(&block)
+    end
+
+    def construct
+      if Scope.scope_node?(root)
+        child_scope!(root, nil)
+      else
+        add_scope(root)
+        construct_node(root, root)
+      end
+    end
+
+    def add_scope(scope)
+      all_scopes << scope
+      child_scopes[scope.__id__] = NodeSet.new
+      sub_scopes[scope.__id__] = NodeSet.new
+      assignment_nodes[scope.__id__] = Set.new
+      reference_nodes[scope.__id__] = Set.new
+    end
+
+    def child_scope!(scope, parent_scope)
+      add_scope(scope)
+
+      if parent_scope
+        parent_scopes[scope.__id__] = parent_scope
+        child_scopes[parent_scope.__id__] << scope
+      end
+
+      each_child_node(scope) do |child|
+        construct_node(child, scope)
+      end
+    end
+
+    def nested_scope!(scope, super_scope)
+      add_scope(scope)
+
+      if super_scope
+        parent_scopes[scope.__id__] = super_scope
+        child_scopes[super_scope.__id__] << scope
+        super_scopes[scope.__id__] = super_scope
+        sub_scopes[super_scope.__id__] << scope
+      end
+
+      each_child_node(scope) do |child|
+        construct_node(child, scope)
+      end
+    end
+
+    def construct_node(node, current_scope)
+      case node.type
+      when :class, :module, :def
+        child_scope!(node, current_scope)
+      when :block
+        nested_scope!(node, current_scope)
+      when :lvar
+        reference_nodes[current_scope.__id__] << node
+      when :lvasgn, :arg, :optarg, :restarg, :kwarg, :kwoptarg, :kwrestarg, :blockarg
+        assignment_nodes[current_scope.__id__] << Assignment.new(node: node, variable: node.children[0])
+        construct_children(node, current_scope)
+      when :procarg0
+        case node.children[0]
+        when AST::Node
+          construct_children(node, current_scope)
+        else
+          assignment_nodes[current_scope.__id__] << Assignment.new(node: node, variable: node.children[0])
+          construct_children(node, current_scope)
+        end
+      when :match_with_lvasgn
+        node.children[2].each do |var|
+          assignment_nodes[current_scope.__id__] << Assignment.new(node: node, variable: var)
+        end
+
+        construct_children(node, current_scope)
+      else
+        construct_children(node, current_scope)
+      end
+    end
+
+    def construct_children(node, current_scope)
+      each_child_node(node) do |child|
+        construct_node(child, current_scope)
+      end
+    end
+
+    def valid_scope!(node)
+      unless root.equal?(node) || Scope.scope_node?(node)
+        raise "Invalid scope node given: #{node}"
+      end
+    end
+
+    def self.from(node:)
+      new(root: node).tap(&:construct)
+    end
+
+    def self.scope_node?(node)
+      case node.type
+      when :class, :module, :def, :block
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/ast_utils/version.rb

@@ -0,0 +1,3 @@
+module ASTUtils
+  VERSION = "0.1.0"
+end

data/lib/ast_utils.rb

@@ -0,0 +1,15 @@
+require "ast_utils/version"
+
+require "parser/current"
+require "pathname"
+require "set"
+
+require "ast_utils/node_set"
+require "ast_utils/node_helper"
+require "ast_utils/partial_map"
+require "ast_utils/labeling"
+require "ast_utils/navigation"
+require "ast_utils/scope"
+
+Parser::Builders::Default.emit_lambda = true
+Parser::Builders::Default.emit_procarg0 = true

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: ast_utils
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Soutaro Matsumoto
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.4
+description: Ruby AST Utility
+email:
+- matsumoto@soutaro.com
+executables:
+- ast_utils
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- ast_utils.gemspec
+- bin/console
+- bin/setup
+- exe/ast_utils
+- lib/ast_utils.rb
+- lib/ast_utils/cli.rb
+- lib/ast_utils/labeling.rb
+- lib/ast_utils/navigation.rb
+- lib/ast_utils/node_helper.rb
+- lib/ast_utils/node_set.rb
+- lib/ast_utils/partial_map.rb
+- lib/ast_utils/scope.rb
+- lib/ast_utils/version.rb
+homepage: https://github.com/soutaro/ast_utils
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Ruby AST Utility
+test_files: []