ikra 0.0.1

data/lib/sourcify/lib/sourcify/proc/parser/raw_scanner.rl

@@ -0,0 +1,149 @@
+Sourcify.require_rb('proc', 'parser', 'raw_scanner_extensions')
+
+module Sourcify
+  module Proc
+    class Parser
+      module RawScanner #:nodoc:all
+
+%%{
+
+  machine scanner;
+  include common 'common.rl';
+
+
+  ## MACHINE >> New statement
+  new_statement := |*
+
+    (kw_for | kw_while | kw_until) . ^vchar => {
+      push(:kw_do_alias2, ts, te)
+      increment_counter(:do_end, 0..1)
+      fgoto main;
+    };
+
+    (
+      ((kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless) . ^vchar) |
+      (kw_class . ospaces . '<<' . ospaces . ^newline+)
+    ) => {
+      push(:kw_do_alias1, ts, te)
+      increment_counter(:do_end, 1)
+      fgoto main;
+    };
+
+    ospaces => { push(:space, ts, te) };
+    any => { fhold; fgoto main; };
+
+  *|;
+
+
+  ## MACHINE >> Main
+  main := |*
+
+    ## == Start/end of do..end block
+
+    kw_do => {
+      push(:kw_do, ts, te)
+      increment_counter(:do_end)
+      fgoto new_statement;
+    };
+
+    kw_end => {
+      push(:kw_end, ts, te)
+      decrement_counter(:do_end)
+    };
+
+    ## == Start/end of {...} block
+
+    lbrace => {
+      push(:lbrace, ts, te)
+      increment_counter(:brace)
+    };
+
+    rbrace => {
+      push(:rbrace, ts, te)
+      decrement_counter(:brace)
+    };
+
+    assoc => {
+      push(:assoc, ts, te)
+      fix_counter_false_start(:brace)
+    };
+
+    label => {
+      push_label(ts, te)
+      fix_counter_false_start(:brace)
+    };
+
+    ## == New statement
+
+    newline => {
+      push(:newline, ts, te)
+      increment_lineno
+      fgoto new_statement;
+    };
+
+    smcolon | lparen | assgn | kw_then | comma => {
+      push(:newline_alias, ts, te)
+      fgoto new_statement;
+    };
+
+    ## == Comment
+
+    '#' => {
+      fgoto per_line_comment;
+    };
+
+    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
+      push_comment(ts, te)
+      increment_lineno
+      fgoto block_comment;
+    };
+
+    ## == Strings
+
+    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
+      push_heredoc(ts, te)
+      increment_lineno
+      fgoto heredoc;
+    };
+
+    ('"' | "'" | '`' | '%') => {
+      fhold; fgoto string;
+    };
+
+    '/' => {
+      if preceded_with?(:char, :digit, :var, :const, :symbol, :dstring, :sstring, ')', ']', '}')
+        push(:op, ts, te)
+      else
+        fhold; fgoto string;
+      end
+    };
+
+    ## == Misc
+
+    var         => { push(:var, ts, te) };
+    bslash      => { push(:bslash, ts, te) };
+    const       => { push(:const, ts, te) };
+    symbol      => { push(:symbol, ts, te) };
+    mspaces     => { push(:space, ts, te) };
+    [0-9]       => { push(:digit, ts, te) };
+    lower       => { push(:char, ts, te) };
+    any         => { push(:any, ts, te) };
+
+  *|;
+
+}%%
+%% write data;
+
+        extend Extensions
+
+        def self.execute!
+          data = @data
+          eof = data.length
+          %% write init;
+          %% write exec;
+        end
+
+      end
+    end
+  end
+end

data/lib/sourcify/lib/sourcify/proc/parser/raw_scanner_extensions.rb

@@ -0,0 +1,74 @@
+Sourcify.require_rb('common', 'parser', 'raw_scanner', 'extensions')
+
+module Sourcify
+  module Proc
+    class Parser
+      module RawScanner #:nodoc:all
+        module Extensions
+
+          include Common::Parser::RawScanner::Extensions
+          class DoEndBlockCounter < Common::Parser::RawScanner::Counter; end
+          class BraceBlockCounter < Common::Parser::RawScanner::Counter; end
+
+          def fix_counter_false_start(key)
+            return unless this_counter(key).just_started?
+            return unless really_false_started?
+            reset_attributes
+            @tokens, @false_start_backup = @false_start_backup.dup, nil if @false_start_backup
+          end
+
+          def increment_counter(key, count = 1)
+            return if other_counter(key).started?
+            unless (counter = this_counter(key)).started?
+              return if (@rejecting_block = codified_tokens !~ @start_pattern)
+              @false_start_backup = @tokens.dup if key == :brace
+              offset_attributes
+            end
+            counter.increment(count)
+          end
+
+          def decrement_counter(key)
+            return unless (counter = this_counter(key)).started?
+            counter.decrement
+            construct_result_code if counter.balanced?
+          end
+
+          def other_counter(type)
+            {:do_end => @brace_counter, :brace => @do_end_counter}[type]
+          end
+
+          def this_counter(type)
+            {:brace => @brace_counter, :do_end => @do_end_counter}[type]
+          end
+
+          def construct_result_code
+            codes = [false, true].map do |fix_heredoc|
+              %Q(proc #{codified_tokens(fix_heredoc)}).force_encoding(@encoding)
+            end
+
+            begin
+              if valid?(codes[1]) && @body_matcher.call(codes[0])
+                @results << codes
+                raise Escape if @stop_on_newline or @lineno != 1
+                reset_attributes
+              end
+            rescue Exception
+              raise if $!.is_a?(Escape)
+            end
+          end
+
+          def really_false_started?
+            valid?(%Q(#{codified_tokens(true)} 1}), :hash)
+          end
+
+          def reset_attributes
+            @do_end_counter = DoEndBlockCounter.new
+            @brace_counter = BraceBlockCounter.new
+            super
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/sourcify/lib/sourcify/proc/parser/scanner.rb

@@ -0,0 +1,49 @@
+Sourcify.require_rb('proc', 'parser', 'raw_scanner')
+
+module Sourcify
+  module Proc
+    class Parser #:nodoc:all
+      class Scanner
+        class << self
+
+          def process(source_code, opts, &matcher)
+            results = rscan(source_code.to_s, {
+              :start_pattern => scan_pattern_hint(opts[:attached_to]),
+              :body_matcher => opts[:body_matcher],
+              :ignore_nested => opts[:ignore_nested],
+              :stop_on_newline => false,
+            }).select{|(raw, normalized)| matcher.call(raw) }
+
+            case results.size
+            when 0 then raise NoMatchingProcError
+            when 1 then results[0]
+            else raise MultipleMatchingProcsPerLineError
+            end
+          end
+
+          def scan_pattern_hint(val)
+            case val
+            when Regexp then val
+            when String, Symbol then /^(?:.*?\W|)#{val}(?:\W)/
+            when nil then /.*/
+            else raise TypeError
+            end
+          end
+
+          def rscan(str, opts)
+            results = RawScanner.process(str, opts) || []
+            inner_opts = opts.merge(:stop_on_newline => true)
+            return results if opts[:ignore_nested]
+            results.map do |outer|
+              [
+                outer,
+                *rscan(outer[1].sub(/^proc\s*(do|\{)/,''), inner_opts)
+              ]
+            end.flatten(1)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/sourcify/lib/sourcify/proc/parser/source_code.rb

@@ -0,0 +1,8 @@
+module Sourcify
+  module Proc
+    class Parser #:nodoc:all
+      Sourcify.require_rb('common', 'parser', 'source_code')
+      SourceCode = Common::Parser::SourceCode
+    end
+  end
+end

data/lib/sourcify/lib/sourcify/version.rb

@@ -0,0 +1,3 @@
+module Sourcify
+  VERSION = "0.6.0.rc4"
+end

data/lib/sourcify/sourcify.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "sourcify/version"
+
+Gem::Specification.new do |s|
+  s.name        = "sourcify"
+  s.version     = Sourcify::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["NgTzeYang"]
+  s.email       = ["ngty77@gmail.com"]
+  s.homepage    = "http://github.com/ngty/sourcify"
+  s.summary     = %q{Workarounds before ruby-core officially supports Proc#to_source (& friends)}
+  s.description = %q{}
+
+  s.add_dependency 'ruby2ruby', '~> 1.3.1'
+  s.add_dependency 'sexp_processor', '~> 3.2.0'
+  s.add_dependency 'ruby_parser', '~> 2.3.1'
+  s.add_dependency 'file-tail', '~> 1.0.10'
+
+  s.add_development_dependency 'bacon'
+  s.add_development_dependency 'pry'
+
+  # ParseTree (better performance + dynamic goodness, but not supported on java & 1.9.*),
+  # optional for any 1.8.*.
+  #s.add_dependency "ParseTree", ">= 3.0.6"
+
+  s.extra_rdoc_files = ["README.rdoc"]
+  s.files            = `git ls-files`.split("\n")
+  s.test_files       = `git ls-files -- spec/*`.split("\n")
+  s.require_paths    = ["lib"]
+end

data/lib/sourcify/spec/dump_object_space_procs.rb

@@ -0,0 +1,84 @@
+require File.join(File.dirname(File.expand_path(__FILE__)), '..', 'lib', 'sourcify')
+require 'benchmark'
+require 'pp'
+
+if RUBY_PLATFORM =~ /java/i
+  require 'jruby'
+  JRuby.objectspace = true
+  reload! rescue nil
+end
+
+DUMPED_OBJECT_SPACE_PROCS = {:max => nil, :done => 0}
+DUMP_OBJECT_SPACE_PROCS_ERRORS = []
+class STOP_DUMPING_OBJECT_SPACE_PROCS < Exception; end
+
+def dump_object_space_procs(debug = false)
+  # Determine working dir
+  name = [
+    RUBY_DESCRIPTION =~ /enterprise/i ? 'ree' : (RUBY_PLATFORM =~ /java/i ? 'jruby' : 'mri'),
+    RUBY_VERSION,
+    Object.const_defined?(:ParseTree) ? 'parsetree' : nil
+  ].compact.join('~')
+  dump_dir = File.join(File.dirname(File.expand_path(__FILE__)), '..', 'tmp', name)
+  Dir.mkdir(dump_dir) unless File.exists?(dump_dir)
+
+  puts '',
+    '== NOTE: dump files can be found at %s' % dump_dir
+
+  # Core processing
+  ObjectSpace.each_object(Proc).to_a.select{|o| o.source_location }.
+    group_by{|o| o.source_location[0] }.each do |ofile, objs|
+      nfile = File.join(dump_dir, ofile.gsub('/','~'))
+      File.open(nfile,'w') do |f|
+        objs.sort_by{|o| o.source_location[1] }.map do |o|
+          DUMPED_OBJECT_SPACE_PROCS[:done] += 1
+          begin
+            data = {
+              :location => o.source_location,
+              :sexp => o.to_sexp,
+              :source => o.to_source
+            }
+            f.puts(data.pretty_inspect)
+            print '.'
+          rescue Exception
+            data = {
+              :location => o.source_location,
+              :error => $!.inspect
+            }
+            DUMP_OBJECT_SPACE_PROCS_ERRORS << data
+            f.puts(data.pretty_inspect)
+            pp(data) if debug
+            print 'x'
+          ensure
+            raise STOP_DUMPING_OBJECT_SPACE_PROCS \
+              if DUMPED_OBJECT_SPACE_PROCS.values.uniq.size == 1
+          end
+        end
+      end
+    end
+end
+
+# Dumping in action
+figures = Benchmark.measure do
+  begin
+    dump_object_space_procs
+  rescue STOP_DUMPING_OBJECT_SPACE_PROCS
+  end
+end
+
+# Printout the failures/passes
+unless DUMP_OBJECT_SPACE_PROCS_ERRORS.empty?
+  puts '', '== OOPS, we have some erorrs :('
+  DUMP_OBJECT_SPACE_PROCS_ERRORS.each_with_index do |e, i|
+    print "#{i}). %s" % e.pretty_inspect
+  end
+else
+  puts '', '== YEAH, no errors :)'
+end
+
+# Printout the benchmark results
+puts '',
+  '== Benchmark results for processing %s procs:' % DUMPED_OBJECT_SPACE_PROCS[:done],
+  figures.to_s, ''
+
+# __END__

data/lib/sourcify/spec/method/encoding_from_def_end_block_spec.rb

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
+
+describe "Encoding (from def..end block)" do
+  next unless Sourcify::IS_19x
+
+  should "handle body as UTF-8 string" do
+    def m1; "こんにちは"; end
+    method(:m1).should.be having_sexp(
+      s(:defn, :m1, s(:args), s(:scope, s(:block, s(:str, "こんにちは"))))
+    )
+  end
+
+  should "handle body with unicode regexp" do
+    # NOTE: This specifically addresses https://github.com/ngty/sourcify/issues/15
+    def m2; /\p{Lu}/ ; end
+    method(:m2).should.be having_sexp(
+      s(:defn, :m2, s(:args), s(:scope, s(:block, s(:lit, /\p{Lu}/))))
+    )
+  end
+
+  should "handle body with UTF-8 heredoc" do
+    def m3
+      <<-EOL
+        こんにちは
+      EOL
+    end
+    method(:m3).should.be having_sexp(
+      s(:defn, :m3, s(:args), s(:scope, s(:block, s(:str, "        こんにちは\n"))))
+    )
+  end
+
+end

data/lib/sourcify/spec/method/encoding_from_define_method_spec.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
+
+describe "Encoding (from define_method)" do
+  next unless Sourcify::IS_19x
+
+  before { @thing = Object.new }
+
+  should "handle body as UTF-8 string" do
+    b = lambda { "こんにちは" }
+    @thing.class.send(:define_method, :m1, &b)
+    @thing.method(:m1).should.be having_sexp(
+      s(:defn, :m1, s(:args), s(:scope, s(:block, s(:str, "こんにちは"))))
+    )
+  end
+
+  should "handle body with unicode regexp" do
+    # NOTE: This specifically addresses https://github.com/ngty/sourcify/issues/15
+    b = lambda { /\p{Lu}/ }
+    @thing.class.send(:define_method, :m2, &b)
+    @thing.method(:m2).should.be having_sexp(
+      s(:defn, :m2, s(:args), s(:scope, s(:block, s(:lit, /\p{Lu}/))))
+    )
+  end
+
+  should "handle body with UTF-8 heredoc" do
+    b = lambda do
+      <<-EOL
+        こんにちは
+      EOL
+    end
+    @thing.class.send(:define_method, :m3, &b)
+    @thing.method(:m3).should.be having_sexp(
+      s(:defn, :m3, s(:args), s(:scope, s(:block, s(:str, "        こんにちは\n"))))
+    )
+  end
+end