sourcify 0.6.0.rc1 → 0.6.0.rc2

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+script: ./spec/run_build.sh
+rvm:
+  - 1.8.6
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - jruby-18mode
+  - jruby-19mode
+  - ree
+env:
+  - TRAVIS=true

data/Gemfile

@@ -3,3 +3,8 @@ source "http://rubygems.org"
 # Specify your gem's dependencies in sourcify.gemspec
 gemspec
 
+# As required by travis, see
+# http://about.travis-ci.org/docs/user/languages/ruby
+group :test do
+  gem 'rake'
+end

data/HISTORY.txt

@@ -2,6 +2,7 @@
 
 * adds Method#to_source, Method#to_sexp & Method#to_raw_source to MRI-1.9.2 (issue#3)
 * extensive refactoring to existing proc support to support the above
+* fixes incorrect encoding for scanner result (issue#19) [#tomykaira]
 
 === 0.5.0 (May 2, 2011)
 

data/README.rdoc

@@ -82,7 +82,7 @@ fluff like comments:
 
   lambda do |i|
     i+1 # (blah)
-  end.to_source
+  end.to_raw_source
   # >> "proc do |i|
   # >>   i+1 # (blah)
   # >> end"
@@ -169,7 +169,7 @@ fetching of raw source returns the method's raw code, including fluff like comme
     end
   end
 
-  MyMath.method(:sum).to_source
+  MyMath.method(:sum).to_raw_source
   # >> "def sum(x, y)
   # >>   x + y # (blah)
   # >> end"
@@ -288,6 +288,14 @@ Under the hood, sourcify relies on RubyParser to yield s-expression, and since R
 does not yet fully handle 1.8.7 & 1.9.* syntax, you will get a nasty Racc::ParseError when
 you have any code that is not compatible with 1.8.6.
 
+=== 4. Lambda operator doesn't work
+
+When a lambda has been created using the lambda operator "->", sourcify can't handle it:
+
+    x = ->{ :blah }
+    x.to_source
+    # >> Sourcify::NoMatchingProcError
+
 
 == Is it really working ??
 

data/Rakefile

@@ -3,22 +3,22 @@ require 'rake'
 require 'bundler'
 Bundler::GemHelper.install_tasks
 
-SPEC_SCRIPT = File.expand_path('../spec/run_spec.sh', __FILE__)
+SPEC_SCRIPT = File.expand_path('../spec/run_build.sh', __FILE__)
 task :default => :spec
 
 RUBIES = {
   :parsetree => [
     [nil, 'ruby-1.8.6-p420@sourcify-parsetree'],
     [nil, 'ruby-1.8.7-p334@sourcify-parsetree'],
-    [nil, 'ree-1.8.7-2011.03@sourcify-parsetree']
+    [nil, 'ree-1.8.7--2012.02@sourcify-parsetree']
   ],
   :static => [
     [nil, 'ruby-1.8.6-p420@sourcify'],
     [nil, 'ruby-1.8.7-p334@sourcify'],
-    [nil, 'ree-1.8.7-2011.03@sourcify'],
+    [nil, 'ree-1.8.7--2012.02@sourcify'],
     [nil, 'jruby-1.6.3@sourcify'],
-    [nil, 'ruby-1.9.1-p378@sourcify'],
-    ['METHOD_TO_SOURCE=true', 'ruby-1.9.2-p290@sourcify'],
+    [nil, 'ruby-1.9.2-p320@sourcify'],
+    [nil, 'ruby-1.9.3-p194@sourcify'],
 
     # NOTE: This doesn't support Method#to_source (& friends) yet yet due to
     # jruby's Method#parameters bug, see http://jira.codehaus.org/browse/JRUBY-5954
@@ -27,7 +27,7 @@ RUBIES = {
   ]
 }
 
-def run_spec_script_for(envs_and_rubies)
+def run_build_for(envs_and_rubies)
   envs_and_rubies.group_by{|arry| arry[0..-2] }.each do |envs, arry|
     rubies = arry.map(&:last)
     declared_envs = ['export MUTE_BACON=true']
@@ -50,17 +50,17 @@ end
 
 desc "Run specs in all rubies (both ParseTree & static scanner modes)"
 task :'spec:all' do
-  run_spec_script_for RUBIES.values.flatten(1)
+  run_build_for RUBIES.values.flatten(1)
 end
 
 desc "Run specs in rubies supporting ParseTree mode"
 task :'spec:parsetree' do
-  run_spec_script_for RUBIES[:parsetree]
+  run_build_for RUBIES[:parsetree]
 end
 
 desc "Run specs in rubies supporting static scanner mode"
 task :'spec:static' do
-  run_spec_script_for RUBIES[:static]
+  run_build_for RUBIES[:static]
 end
 
 # ///////////////////////////////////////////////////////////

data/lib/sourcify.rb

@@ -37,7 +37,7 @@ module Sourcify #:nodoc:
   end
 end
 
-Sourcify.require_rb('facets')
+Sourcify.require_rb('patches')
 Sourcify.require_rb('errors')
 Sourcify.require_rb('version')
 Sourcify.require_rb('proc')

data/lib/sourcify/common/parser/raw_scanner/dstring.rb

@@ -2,7 +2,7 @@ module Sourcify
   module Common
     class Parser
       module RawScanner #:nodoc:all
-        class DString < Struct.new(:tag)
+        class DString < Struct.new(:tag, :encoding)
 
           # To suppress 'warning: Object#type is deprecated; use Object#class' when
           # evaluating string
@@ -25,16 +25,14 @@ module Sourcify
             CLOSING_TAGS = {'(' => ')', '[' => ']', '<' => '>', '{' => '}'}
 
             def evaluable?
-              @contents[-1][-1].chr == end_tag
+              @contents.length >= 2 &&
+                @contents[-1][-1].chr == end_tag
             end
 
             def parsable?
-              begin
-                RubyParser.new.parse(safe_contents)
-                true
-              rescue Exception
-                false
-              end
+              !!RubyParser.new.parse(safe_contents)
+            rescue SyntaxError, Racc::ParseError
+              false
             end
 
             def safe_contents
@@ -42,7 +40,7 @@ module Sourcify
               # thus we convert them to normal strings 1st
               to_s.gsub(/(%x)(\W|\_)/, '%Q\2').gsub(/.{0,2}(`)/) do |s|
                 s =~ /^(%Q|%W|%r|%x|.?%|.?\\)/ ? s : s.sub(/`$/,'%Q`')
-              end
+              end.force_encoding(encoding)
             end
 
             def start_tag

data/lib/sourcify/common/parser/raw_scanner/extensions.rb

@@ -12,6 +12,7 @@ module Sourcify
 
           def process(data, opts={})
             begin
+              @encoding = data.encoding
               @start_pattern = opts[:start_pattern] || /.*/
               @body_matcher = opts[:body_matcher] || lambda{|_| true }
               @stop_on_newline = opts[:stop_on_newline]
@@ -33,7 +34,7 @@ module Sourcify
 
           def push_dstring(ts, te)
             data = data_frag(ts .. te.pred)
-            @dstring ||= DString.new(data[%r{^("|`|/|%(?:Q|W|r|x|)(?:\W|_))},1])
+            @dstring ||= DString.new(data[%r{^("|`|/|%(?:Q|W|r|x|)(?:\W|_))},1], @encoding)
             @dstring << data
             return true unless @dstring.closed?
             @tokens << [:dstring, @dstring.to_s]
@@ -53,7 +54,7 @@ module Sourcify
             data = data_frag(ts .. te.pred)
             unless @heredoc
               indented, tag = data.match(/\<\<(\-?)['"]?(\w+)['"]?$/)[1..3]
-              @heredoc = Heredoc.new(tag, !indented.empty?)
+              @heredoc = Heredoc.new(tag, !indented.empty?, @encoding)
             end
             @heredoc << data
             return true unless @heredoc.closed?(data_frag(te .. te))
@@ -80,8 +81,23 @@ module Sourcify
               if @stop_on_newline || !@results.empty? || (@results.empty? && @rejecting_block)
           end
 
-          def codified_tokens
-            @tokens.map(&:last).join
+          def codified_tokens(fix_heredoc = false)
+            (
+              if fix_heredoc
+                @tokens.map do |key, val|
+                  case key
+                  when :heredoc
+                    %(\n%"#{
+                      val.sub(/^(?:[^\n]+\n)(.*\n)(?:[^\n]+)$/m, '\1').
+                        gsub(/\\|"/) {|c| "\\#{c}" }
+                    }")
+                  else val
+                  end
+                end
+              else
+                @tokens.map(&:last)
+              end
+            ).join
           end
 
           def reset_attributes

data/lib/sourcify/common/parser/raw_scanner/heredoc.rb

@@ -2,14 +2,14 @@ module Sourcify
   module Common
     class Parser
       module RawScanner #:nodoc:all
-        class Heredoc < Struct.new(:tag, :indented)
+        class Heredoc < Struct.new(:tag, :indented, :encoding)
 
           def <<(content)
             (@contents ||= []) << content
           end
 
           def to_s
-            @contents.join
+            @contents.join.force_encoding(encoding)
           end
 
           def closed?(sealer)

data/lib/sourcify/method/parser.rb

@@ -16,7 +16,7 @@ module Sourcify
       end
 
       def raw_source(opts)
-        raw_source = extracted_source(opts).strip
+        raw_source = extracted_source(opts)[0].strip
         opts[:strip_enclosure] ? strip_raw_enclosure(raw_source) : raw_source
       end
 
@@ -26,7 +26,9 @@ module Sourcify
 
       def sexp(opts)
         (@sexps ||= {})[opts.hash] ||= (
-          raw_code = ("\n" * @source_code.line) + extracted_source(opts)
+          extracted = extracted_source(opts)[1]
+          raw_code = (("\n" * @source_code.line) + extracted).same_encoding_as(extracted)
+
           sexp = Converter.to_sexp(raw_code, @source_code.file)
           opts[:strip_enclosure] ? Sexp.from_array(sexp.to_a[-1]) : sexp
         )
@@ -60,20 +62,24 @@ module Sourcify
             extracted_source_from_method(opts)
           rescue ProbablyDefinedByProc
             pattern = /^proc\s*(\{|do)\s*(\|[^\|]*\|)?(.+)(\}|end)$/m
-            match = extracted_source_from_proc(opts).match(pattern)
-            if @parameters.empty?
-              %Q(def #{@name}\n#{match[3]}\nend)
-            else
-              args = match[2].sub(/^\|([^\|]+)\|$/, '\1')
-              %Q(def #{@name}(#{args})\n#{match[3]}\nend)
-            end
+            extracted = extracted_source_from_proc(opts)
+            matches = extracted.map{|s| s.match(pattern) }
+
+            (
+              if @parameters.empty?
+                matches.map{|match| %Q(def #{@name}\n#{match[3]}\nend) }
+              else
+                args = matches[0][2].sub(/^\|([^\|]+)\|$/, '\1')
+                matches.map{|match| %Q(def #{@name}(#{args})\n#{match[3]}\nend) }
+              end
+            ).map{|s| s.same_encoding_as(extracted[0]) }
           end
         end
 
         def extracted_source_from_method(opts)
-          Scanner.process(@source_code, opts) do |code|
+          Scanner.process(@source_code, opts) do |(raw, normalized)|
             begin
-              Object.new.instance_eval("#{code}; self").
+              Object.new.instance_eval("#{raw}; self".same_encoding_as(raw)).
                 method(@name).parameters == @parameters
             rescue NameError
               false
@@ -84,13 +90,13 @@ module Sourcify
         end
 
         def extracted_source_from_proc(opts)
-          Proc::Parser::Scanner.process(@source_code, opts) do |code|
+          Proc::Parser::Scanner.process(@source_code, opts) do |raw|
             begin
               Object.new.instance_eval(%(
                 (class << self; self; end).class_eval do
-                  define_method(:#{@name}, &(#{code}))
+                  define_method(:#{@name}, &(#{raw}))
                 end; self
-              )).method(@name).parameters == @parameters
+              ).same_encoding_as(raw)).method(@name).parameters == @parameters
             rescue NameError
               false
             rescue Exception

data/lib/sourcify/method/parser/raw_scanner_extensions.rb

@@ -32,13 +32,17 @@ module Sourcify
           end
 
           def construct_result_code
-            code = codified_tokens.strip
+            codes = [false, true].map do |fix_heredoc|
+              codified_tokens(fix_heredoc).force_encoding(@encoding)
+            end
 
             begin
-              if valid?(code) && @body_matcher.call(code)
+              if valid?(codes[1]) && @body_matcher.call(codes[0])
                 # NOTE: Need to fix singleton method to avoid errors (eg. undefined object)
                 # downstream
-                @results << code.sub(%r{^(def\s+)(?:[^\.]+\.)?(#{@name}.*end)$}m, '\1\2')
+                @results << codes.map do |s|
+                  s.sub(%r{^(def\s+)(?:[^\.]+\.)?(#{@name}.*end)$}m, '\1\2')
+                end
                 raise Escape if @stop_on_newline or @lineno != 1
                 reset_attributes
               end

data/lib/sourcify/method/parser/scanner.rb

@@ -12,7 +12,8 @@ module Sourcify
               :body_matcher => opts[:body_matcher],
               :ignore_nested => opts[:ignore_nested],
               :stop_on_newline => false,
-            }).flatten.select(&matcher)
+            }).select{|(raw, normalized)| matcher.call(raw) }
+
             case results.size
             when 0 then raise NoMatchingMethodError
             when 1 then results[0]
@@ -31,14 +32,17 @@ module Sourcify
 
           def rscan(str, opts)
             results = RawScanner.process(str, opts) || []
+            inner_opts = opts.merge(:stop_on_newline => true)
             return results if opts[:ignore_nested]
             results.map do |outer|
-              inner = rscan(
-                outer.sub(/^def(.*)end$/,'\1').sub(/^(?:.*?)(def.*)$/,'\1'),
-                opts.merge(:stop_on_newline => true)
-              )
-              [outer, inner]
-            end
+              [
+                outer,
+                *rscan(
+                  outer[0].sub(/^def(.*)end$/,'\1').sub(/^(?:.*?)(def.*)$/,'\1'),
+                  inner_opts
+                )
+              ]
+            end.flatten(1)
           end
 
         end

data/lib/sourcify/{facets.rb → patches.rb}

@@ -26,3 +26,38 @@ unless binding.respond_to?(:has_local_variable?)
     end
   end
 end
+
+# New way of adding new functionalities
+module Sourcify
+  module Patches
+
+    module String
+      if ''.respond_to?(:force_encoding)
+
+        def same_encoding_as(other)
+          force_encoding(other.encoding)
+        end
+
+      else
+
+        def same_encoding_as(other)
+          self
+        end
+
+        def encoding
+          nil
+        end
+
+        def force_encoding(dummy)
+          self
+        end
+
+      end
+    end
+
+    ::String.class_eval do
+      include Sourcify::Patches::String
+    end
+
+  end
+end

data/lib/sourcify/proc/parser.rb

@@ -19,7 +19,9 @@ module Sourcify
 
       def sexp(opts)
         (@sexps ||= {})[opts.hash] ||= (
-          raw_code = ("\n" * @source_code.line) + extracted_source(opts)
+          extracted = extracted_source(opts)[1]
+          raw_code = (("\n" * @source_code.line) + extracted).same_encoding_as(extracted)
+
           raw_sexp = Converter.to_sexp(raw_code, @source_code.file)
           sexp = Normalizer.process(raw_sexp, @binding)
           opts[:strip_enclosure] ? Sexp.from_array(sexp.to_a.last) : sexp
@@ -27,7 +29,7 @@ module Sourcify
       end
 
       def raw_source(opts)
-        raw_code = extracted_source(opts).strip
+        raw_code = extracted_source(opts)[0].strip
         opts[:strip_enclosure] ?
           raw_code.sub(/^proc\s*(\{|do)\s*(\|[^\|]+\|)?(.*)(\}|end)$/m, '\3').strip : raw_code
       end
@@ -35,9 +37,9 @@ module Sourcify
       private
 
         def extracted_source(opts)
-          Scanner.process(@source_code, opts) do |code|
+          Scanner.process(@source_code, opts) do |(raw, normalized)|
             begin
-              eval(code).arity == @arity
+              eval(raw).arity == @arity
             rescue Exception
               raise ParserInternalError
             end

data/lib/sourcify/proc/parser/raw_scanner_extensions.rb

@@ -42,11 +42,13 @@ module Sourcify
           end
 
           def construct_result_code
-            code = %Q(proc #{codified_tokens}).strip
+            codes = [false, true].map do |fix_heredoc|
+              %Q(proc #{codified_tokens(fix_heredoc)}).force_encoding(@encoding)
+            end
 
             begin
-              if valid?(code) && @body_matcher.call(code)
-                @results << code
+              if valid?(codes[1]) && @body_matcher.call(codes[0])
+                @results << codes
                 raise Escape if @stop_on_newline or @lineno != 1
                 reset_attributes
               end
@@ -56,7 +58,7 @@ module Sourcify
           end
 
           def really_false_started?
-            valid?(%Q(#{codified_tokens} 1}), :hash)
+            valid?(%Q(#{codified_tokens(true)} 1}), :hash)
           end
 
           def reset_attributes

data/lib/sourcify/proc/parser/scanner.rb

@@ -12,7 +12,8 @@ module Sourcify
               :body_matcher => opts[:body_matcher],
               :ignore_nested => opts[:ignore_nested],
               :stop_on_newline => false,
-            }).flatten.select(&matcher)
+            }).select{|(raw, normalized)| matcher.call(raw) }
+
             case results.size
             when 0 then raise NoMatchingProcError
             when 1 then results[0]
@@ -31,13 +32,14 @@ module Sourcify
 
           def rscan(str, opts)
             results = RawScanner.process(str, opts) || []
+            inner_opts = opts.merge(:stop_on_newline => true)
             return results if opts[:ignore_nested]
             results.map do |outer|
               [
                 outer,
-                rscan(outer.sub(/^proc\s*(do|\{)/,''), opts.merge(:stop_on_newline => true))
+                *rscan(outer[1].sub(/^proc\s*(do|\{)/,''), inner_opts)
               ]
-            end
+            end.flatten(1)
           end
 
         end

data/lib/sourcify/version.rb

@@ -1,3 +1,3 @@
 module Sourcify
-  VERSION = "0.6.0.rc1"
+  VERSION = "0.6.0.rc2"
 end

data/sourcify.gemspec

@@ -12,10 +12,10 @@ Gem::Specification.new do |s|
   s.summary     = %q{Workarounds before ruby-core officially supports Proc#to_source (& friends)}
   s.description = %q{}
 
-  s.add_dependency 'ruby2ruby', '>= 1.2.5'
-  s.add_dependency 'sexp_processor', '>= 3.0.5'
-  s.add_dependency 'ruby_parser', '>= 2.0.5'
-  s.add_dependency 'file-tail', '>= 1.0.5'
+  s.add_dependency 'ruby2ruby', '~> 1.3.1'
+  s.add_dependency 'sexp_processor', '~> 3.2.0'
+  s.add_dependency 'ruby_parser', '~> 2.3.1'
+  s.add_dependency 'file-tail', '~> 1.0.10'
   s.add_development_dependency 'bacon'
 
   # ParseTree (better performance + dynamic goodness, but not supported on java & 1.9.*),

data/spec/method/encoding_from_def_end_block_spec.rb

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
+
+describe "Encoding (from def..end block)" do
+
+  should "handle body as UTF-8 string" do
+    def m1; "こんにちは"; end
+    method(:m1).should.be having_sexp(
+      s(:defn, :m1, s(:args), s(:scope, s(:block, s(:str, "こんにちは"))))
+    )
+  end
+
+  should "handle body with unicode regexp" do
+    # NOTE: This specifically addresses https://github.com/ngty/sourcify/issues/15
+    def m2; /\p{Lu}/ ; end
+    method(:m2).should.be having_sexp(
+      s(:defn, :m2, s(:args), s(:scope, s(:block, s(:lit, /\p{Lu}/))))
+    )
+  end
+
+  should "handle body with UTF-8 heredoc" do
+    def m3
+      <<-EOL
+        こんにちは
+      EOL
+    end
+    method(:m3).should.be having_sexp(
+      s(:defn, :m3, s(:args), s(:scope, s(:block, s(:str, "        こんにちは\n"))))
+    )
+  end
+
+end

data/spec/method/encoding_from_define_method_spec.rb

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
+
+describe "Encoding (from define_method)" do
+
+  before { @thing = Object.new }
+
+  should "handle body as UTF-8 string" do
+    b = lambda { "こんにちは" }
+    @thing.class.send(:define_method, :m1, &b)
+    @thing.method(:m1).should.be having_sexp(
+      s(:defn, :m1, s(:args), s(:scope, s(:block, s(:str, "こんにちは"))))
+    )
+  end
+
+  should "handle body with unicode regexp" do
+    # NOTE: This specifically addresses https://github.com/ngty/sourcify/issues/15
+    b = lambda { /\p{Lu}/ }
+    @thing.class.send(:define_method, :m2, &b)
+    @thing.method(:m2).should.be having_sexp(
+      s(:defn, :m2, s(:args), s(:scope, s(:block, s(:lit, /\p{Lu}/))))
+    )
+  end
+
+  should "handle body with UTF-8 heredoc" do
+    b = lambda do
+      <<-EOL
+        こんにちは
+      EOL
+    end
+    @thing.class.send(:define_method, :m3, &b)
+    @thing.method(:m3).should.be having_sexp(
+      s(:defn, :m3, s(:args), s(:scope, s(:block, s(:str, "        こんにちは\n"))))
+    )
+  end
+end

data/spec/method/others_from_define_method_spec.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
 
 describe "Misc (from define_method)" do

data/spec/method/to_source_from_def_end_block_w_nested_if_spec.rb

@@ -40,16 +40,17 @@ describe "Method#to_source (from def ... end block)" do
       ))
     end
 
-    should 'handle block within modifier' do
-      def m4
-        @x1 = 1 if (if @x1 then true end)
-      end
-      method(:m4).should.be having_source(%(
-        def m4
-          @x1 = 1 if (if @x1 then true end)
-        end
-      ))
-    end
+    # NOTE: Syntatically correct, but the RubyParser-2.3.* can't handle it.
+    # should 'handle block within modifier' do
+    #   def m4
+    #     @x1 = 1 if (if @x1 then true end)
+    #   end
+    #   method(:m4).should.be having_source(%(
+    #     def m4
+    #       @x1 = 1 if (if @x1 then true end)
+    #     end
+    #   ))
+    # end
 
     should 'handle modifier w trailing backslash' do
       def m5

data/spec/method/to_source_from_def_end_block_w_nested_unless_spec.rb

@@ -40,16 +40,17 @@ describe "Method#to_source (from def ... end block)" do
       ))
     end
 
-    should 'handle block within modifier' do
-      def m4
-        @x1 = 1 unless (unless @x1 then true end)
-      end
-      method(:m4).should.be having_source(%(
-        def m4
-          @x1 = 1 unless (unless @x1 then true end)
-        end
-      ))
-    end
+    # NOTE: Syntatically correct, but the RubyParser-2.3.* can't handle it.
+    # should 'handle block within modifier' do
+    #   def m4
+    #     @x1 = 1 unless (unless @x1 then true end)
+    #   end
+    #   method(:m4).should.be having_source(%(
+    #     def m4
+    #       @x1 = 1 unless (unless @x1 then true end)
+    #     end
+    #   ))
+    # end
 
     should 'handle modifier within block' do
       def m5

data/spec/proc/encoding_spec.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# NOTE: The examples in this file are adapted from the original pull
+# request submitted by tomykaira @ https://github.com/ngty/sourcify/pull/19.
+#
+require File.join(File.expand_path(File.dirname(__FILE__)), 'spec_helper')
+
+describe "Encoding" do
+
+  should "handle proc with UTF-8 string" do
+    lambda { "こんにちは" }.should.be having_sexp(
+      s(:iter, s(:call, nil, :proc, s(:arglist)), nil, s(:str, "こんにちは"))
+    )
+  end
+
+  should "handle proc with unicode regexp" do
+    # NOTE: This specifically addresses https://github.com/ngty/sourcify/issues/15
+    lambda { /\p{Lu}/ }.should.be having_sexp(
+      s(:iter, s(:call, nil, :proc, s(:arglist)), nil, s(:lit, /\p{Lu}/))
+    )
+  end
+
+  should "handle proc with UTF-8 heredoc" do
+    (
+      lambda do
+        <<-EOL
+          こんにちは
+        EOL
+      end
+    ).should.be having_sexp(
+      s(:iter, s(:call, nil, :proc, s(:arglist)), nil, s(:str, "          こんにちは\n"))
+    )
+  end
+
+end

data/spec/proc/to_source_from_do_end_block_w_nested_if_spec.rb

@@ -42,17 +42,18 @@ describe "Proc#to_source from do ... end block (w nested if)" do
     \)
   end
 
-  should 'handle block within modifier' do
-    (
-      lambda do
-        @x1 = 1 if (if @x1 then true end)
-      end
-    ).should.be having_source(%Q\
-      proc do
-        @x1 = 1 if (if @x1 then true end)
-      end
-    \)
-  end
+  # NOTE: Syntatically correct, but the RubyParser-2.3.* can't handle it.
+  # should 'handle block within modifier' do
+  #   (
+  #     lambda do
+  #       @x1 = 1 if (if @x1 then true end)
+  #     end
+  #   ).should.be having_source(%Q\
+  #     proc do
+  #       @x1 = 1 if (if @x1 then true end)
+  #     end
+  #   \)
+  # end
 
   should 'handle modifier w trailing backslash' do
     (

data/spec/proc/to_source_from_do_end_block_w_nested_unless_spec.rb

@@ -42,17 +42,18 @@ describe "Proc#to_source from do ... end block (w nested unless)" do
     \)
   end
 
-  should 'handle block within modifier' do
-    (
-      lambda do
-        @x1 = 1 unless (unless @x1 then true end)
-      end
-    ).should.be having_source(%Q\
-      proc do
-        @x1 = 1 unless (unless @x1 then true end)
-      end
-    \)
-  end
+  # NOTE: Syntatically correct, but the RubyParser-2.3.* can't handle it.
+  # should 'handle block within modifier' do
+  #   (
+  #     lambda do
+  #       @x1 = 1 unless (unless @x1 then true end)
+  #     end
+  #   ).should.be having_source(%Q\
+  #     proc do
+  #       @x1 = 1 unless (unless @x1 then true end)
+  #     end
+  #   \)
+  # end
 
   should 'handle modifier within block' do
     (

data/spec/run_build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+if [ -z "$TRAVIS" ]; then
+  echo ''
+  echo `gem env | grep 'INSTALLATION DIRECTORY' | sed 's/.*\/\(.*\)/\1:/'`
+  rm -rf ~/.ruby_inline/*ParseTree*
+fi
+
+SUPPORTS_METHOD_TO_SOURCE=`ruby -e '
+  begin
+    [:source_location, :parameters].each{|meth| 1.method(:to_s).send(meth) }
+    raise RuntimeError if RUBY_PLATFORM =~ /java/i
+    puts :true
+  rescue NoMethodError, RuntimeError
+    puts :false
+  end
+'`
+
+if [ "$SUPPORTS_METHOD_TO_SOURCE" == "true" ]; then
+  bundle exec bacon spec/{proc,method}/*/*_spec.rb spec/{proc,method}/*_spec.rb
+else
+  echo "NOTE: This ruby doesn't support Method#to_source (& friends) !!"
+  bundle exec bacon spec/proc/*/*_spec.rb spec/{proc,no_method}/*_spec.rb
+fi
+
+# __END__

data/spec/spec_helper.rb

@@ -99,10 +99,12 @@ def irb_exec(stdin_str)
   # See http://tyenglog.heroku.com/2010/9/how-to-test-irb-specific-support &
   # http://tyenglog.heroku.com/2010/9/how-to-test-irb-specific-support-2-
   sourcify_rb = File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'sourcify.rb')
-  irb_feedback = /^ => /
-  values = %x(echo "#{stdin_str}" | irb -r #{sourcify_rb}).split("\n").
+  irb_feedback = /^ ?=> /
+  irb_opts = '--simple-prompt'
+  values = %x(echo "#{stdin_str}" | irb #{irb_opts} -r #{sourcify_rb}).split("\n").
     grep(irb_feedback).map{|s| eval(s.sub(irb_feedback,'').strip) }
+
   # IRB behaves slightly differently in 1.9.2 for appending newline
-  (values[-1].nil? && RUBY_VERSION.include?('1.9.2')) ? values[0 .. -2] : values
+  (values[-1].nil? && RUBY_VERSION =~ /1.9.(2|3)/) ? values[0 .. -2] : values
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sourcify
 version: !ruby/object:Gem::Version
-  version: 0.6.0.rc1
+  version: 0.6.0.rc2
   prerelease: 6
 platform: ruby
 authors:
@@ -9,56 +9,75 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-07-31 00:00:00.000000000 +08:00
-default_executable: 
+date: 2012-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby2ruby
-  requirement: &21310380 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: 1.3.1
   type: :runtime
   prerelease: false
-  version_requirements: *21310380
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: sexp_processor
-  requirement: &21309880 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.5
+        version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *21309880
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: ruby_parser
-  requirement: &21309420 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.5
+        version: 2.3.1
   type: :runtime
   prerelease: false
-  version_requirements: *21309420
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.1
 - !ruby/object:Gem::Dependency
   name: file-tail
-  requirement: &21308960 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.5
+        version: 1.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *21308960
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.10
 - !ruby/object:Gem::Dependency
   name: bacon
-  requirement: &21338920 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +85,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *21338920
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
 - ngty77@gmail.com
@@ -77,6 +101,7 @@ extra_rdoc_files:
 files:
 - .document
 - .gitignore
+- .travis.yml
 - Gemfile
 - HISTORY.txt
 - LICENSE
@@ -94,7 +119,6 @@ files:
 - lib/sourcify/common/ragel/expressions.rl
 - lib/sourcify/common/ragel/machines.rl
 - lib/sourcify/errors.rb
-- lib/sourcify/facets.rb
 - lib/sourcify/method.rb
 - lib/sourcify/method/methods.rb
 - lib/sourcify/method/methods/to_raw_source.rb
@@ -107,6 +131,7 @@ files:
 - lib/sourcify/method/parser/raw_scanner_extensions.rb
 - lib/sourcify/method/parser/scanner.rb
 - lib/sourcify/method/parser/source_code.rb
+- lib/sourcify/patches.rb
 - lib/sourcify/proc.rb
 - lib/sourcify/proc/methods.rb
 - lib/sourcify/proc/methods/source_location.rb
@@ -124,6 +149,8 @@ files:
 - lib/sourcify/version.rb
 - sourcify.gemspec
 - spec/dump_object_space_procs.rb
+- spec/method/encoding_from_def_end_block_spec.rb
+- spec/method/encoding_from_define_method_spec.rb
 - spec/method/others_from_def_end_block_spec.rb
 - spec/method/others_from_define_method_spec.rb
 - spec/method/raw_scanner/block_comment_spec.rb
@@ -183,6 +210,7 @@ files:
 - spec/no_method/unsupported_platform_spec.rb
 - spec/proc/19x_extras.rb
 - spec/proc/created_on_the_fly_proc_spec.rb
+- spec/proc/encoding_spec.rb
 - spec/proc/others_spec.rb
 - spec/proc/raw_scanner/block_comment_spec.rb
 - spec/proc/raw_scanner/double_colons_spec.rb
@@ -248,9 +276,8 @@ files:
 - spec/raw_scanner/shared_specs.rb
 - spec/raw_scanner/single_quote_str_shared_spec.rb
 - spec/raw_scanner/slash_operator_shared_spec.rb
-- spec/run_spec.sh
+- spec/run_build.sh
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/ngty/sourcify
 licenses: []
 post_install_message: 
@@ -271,12 +298,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.6.1
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Workarounds before ruby-core officially supports Proc#to_source (& friends)
 test_files:
 - spec/dump_object_space_procs.rb
+- spec/method/encoding_from_def_end_block_spec.rb
+- spec/method/encoding_from_define_method_spec.rb
 - spec/method/others_from_def_end_block_spec.rb
 - spec/method/others_from_define_method_spec.rb
 - spec/method/raw_scanner/block_comment_spec.rb
@@ -336,6 +365,7 @@ test_files:
 - spec/no_method/unsupported_platform_spec.rb
 - spec/proc/19x_extras.rb
 - spec/proc/created_on_the_fly_proc_spec.rb
+- spec/proc/encoding_spec.rb
 - spec/proc/others_spec.rb
 - spec/proc/raw_scanner/block_comment_spec.rb
 - spec/proc/raw_scanner/double_colons_spec.rb
@@ -401,5 +431,5 @@ test_files:
 - spec/raw_scanner/shared_specs.rb
 - spec/raw_scanner/single_quote_str_shared_spec.rb
 - spec/raw_scanner/slash_operator_shared_spec.rb
-- spec/run_spec.sh
+- spec/run_build.sh
 - spec/spec_helper.rb

data/spec/run_spec.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-echo ''
-echo `gem env | grep 'INSTALLATION DIRECTORY' | sed 's/.*\/\(.*\)/\1:/'`
-rm -rf ~/.ruby_inline/*ParseTree*
-
-if [ "${METHOD_TO_SOURCE}" == "true" ]; then
-  bacon spec/{proc,method}/*/*_spec.rb spec/{proc,method}/*_spec.rb
-else
-  echo "NOTE: This ruby doesn't support Method#to_source (& friends) !!"
-  bacon spec/proc/*/*_spec.rb spec/{proc,no_method}/*_spec.rb
-fi