sourcify 0.1.1 → 0.1.2

data/HISTORY.txt

@@ -1,3 +1,9 @@
+=== 0.1.2 (Aug 30, 2010)
+
+* introduced throwing of Sourcify::LexerInternalError when parser cannot handle
+  incorrectly lexed code fragments [#ngty]
+* fixed bug in lexing keyword literals (represented as Symbol) [#ngty]
+
 === 0.1.1 (Aug 30, 2010)
 
 * fixed empty return for Proc#to_sexp after calling of Proc#to_source [#ngty]

data/README.rdoc

@@ -139,6 +139,22 @@ following bug under 1.8.* may trip u over:
 This is another reason to install ParseTree when u are on 1.8.*. On JRuby, where u don't
 have the choice, just avoid multiple procs here line.
 
+=== 3. Imperfect lexer
+
+Nothing is perfect, bugs are always lurking somewhere. When the parser encounters
+unexpected lexing error (which may be due to Ripper, RubyLex, or Sourcify's wrapper lexer),
+Sourcify::LexerInternalError is thrown.
+
+== Additional Resources
+
+Sourcify is heavily inspired by many ideas gathered from the ruby community:
+* http://www.justskins.com/forums/breaking-ruby-code-into-117453.html
+* http://rubyquiz.com/quiz38.html (Florian Groß's solution)
+* http://svenfuchs.com/2009/07/05/using-ruby-1-9-ripper.html
+
+The sad fact that Proc#to_source wouldn't be available in the near future:
+* http://redmine.ruby-lang.org/issues/show/2080
+
 == Note on Patches/Pull Requests
  
 * Fork the project.

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/sourcify/proc.rb

@@ -1,6 +1,7 @@
 module Sourcify
 
   class MultipleMatchingProcsPerLineError < Exception ; end
+  class LexerInternalError < Exception ; end
 
   module Proc
 
@@ -19,13 +20,12 @@ module Sourcify
           end
         end
 
-        # When ParseTree is available, we just make use of all the convenience it offers :)
         if Object.const_defined?(:ParseTree)
-
+          # When ParseTree is available, we just make use of all the convenience it offers :)
           alias_method :to_source, :to_ruby
 
-        # Otherwise, we are going to do abit of static text parsing :(
         else
+          # Otherwise, we are going to do abit of static text parsing :(
 
           unless meths.include?('to_source')
             def to_source

data/lib/sourcify/proc/counter.rb

@@ -1,6 +1,6 @@
 module Sourcify
   module Proc
-    class Counter
+    class Counter #:nodoc:all
 
       attr_accessor :marker
 

data/lib/sourcify/proc/lexer.rb

@@ -1,6 +1,6 @@
 module Sourcify
   module Proc
-    module Lexer
+    module Lexer #:nodoc:all
 
       class << self
         def new(*args)

data/lib/sourcify/proc/lexer18.rb

@@ -3,11 +3,12 @@ require 'irb/ruby-token'
 
 module Sourcify
   module Proc
+    class Lexer18 #:nodoc:all
 
-    class Lexer18
-
-      # Implementation of this class has been inspired by the discussion at
-      # http://www.justskins.com/forums/breaking-ruby-code-into-117453.html
+      # NOTE: Implementation of this class has been heavily inspired by:
+      # * the discussion @ http://www.justskins.com/forums/breaking-ruby-code-into-117453.html
+      # * Florian Groß's solution for ruby quiz 'SerializableProc (#38)'
+      #   @ http://rubyquiz.com/quiz38.html
 
       include Lexer::Commons
 
@@ -26,8 +27,9 @@ module Sourcify
           tkc = @tk.class.to_s.sub(/\ARubyToken::/, '').downcase.to_sym
           @tokens << [@tk.line_no, @tk.char_no, tkc]
           post_qstring if @qstring_data
-          send(:"on_#{tkc}") rescue NoMethodError
           @lex.get_readed if tkc == :tknl
+          next if @do_end_counter.started? && @tokens.curr.symbolized_keyword?
+          send(:"on_#{tkc}") rescue NoMethodError
         end
       end
 
@@ -180,6 +182,17 @@ module Sourcify
         end
 
         def curr
+          (self[-1]).respond_to?(:symbolized_keyword?) ? self[-1] : (
+            preceding, current = self[-2 .. -1]
+            (class << current ; self ; end).class_eval do
+              define_method(:symbolized_keyword?) do
+                preceding[TYP] == :tksymbeg && %w{
+                  class module def while until if unless for do begin case end
+                }.include?(current[TYP].to_s.sub('tk',''))
+              end
+            end
+            current
+          )
           self[-1]
         end
 

data/lib/sourcify/proc/lexer19.rb

@@ -2,7 +2,7 @@ require 'ripper'
 
 module Sourcify
   module Proc
-    class Lexer19 < Ripper::Lexer
+    class Lexer19 < Ripper::Lexer #:nodoc:all
 
       include Lexer::Commons
 
@@ -16,6 +16,7 @@ module Sourcify
 
       def on_kw(token)
         super.tap do |rs|
+          next if @do_end_counter.started? && rs.curr.symbolized_keyword?
           send(:"on_kw_#{token}", rs) rescue NoMethodError
         end
       end
@@ -125,7 +126,15 @@ module Sourcify
         end
 
         def curr
-          self[-1]
+          (self[-1]).respond_to?(:symbolized_keyword?) ? self[-1] : (
+            preceding, current = self[-2 .. -1]
+            (class << current ; self ; end).class_eval do
+              define_method(:symbolized_keyword?) do
+                current[TYP] == :on_kw && preceding[TYP] == :on_symbeg
+              end
+            end
+            current
+          )
         end
 
         def same_line(line)

data/lib/sourcify/proc/parser.rb

@@ -3,7 +3,7 @@ require 'sourcify/proc/counter'
 
 module Sourcify
   module Proc
-    class Parser
+    class Parser #:nodoc:all
 
       RUBY_PARSER = RubyParser.new
       RUBY_2_RUBY = Ruby2Ruby.new
@@ -27,12 +27,22 @@ module Sourcify
       private
 
         def raw_source
-          @raw_source ||= (
-            frags = Sourcify::Proc::Lexer.new(raw_source_io, @file, @line).work.
-              select{|frag| eval('proc ' + frag).arity == @arity }
-            raise MultipleMatchingProcsPerLineError if frags.size > 1
-            'proc %s' % frags[0]
-          )
+          @raw_source ||=
+            if (frags = raw_source_frags).size > 1
+              raise MultipleMatchingProcsPerLineError
+            else
+              'proc %s' % frags[0]
+            end
+        end
+
+        def raw_source_frags
+          Sourcify::Proc::Lexer.new(raw_source_io, @file, @line).work.select do |frag|
+            begin
+              eval('proc ' + frag).arity == @arity
+            rescue SyntaxError, TypeError
+              raise LexerInternalError
+            end
+          end
         end
 
         def raw_source_io

data/sourcify.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{sourcify}
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["NgTzeYang"]
@@ -45,6 +45,7 @@ Gem::Specification.new do |s|
      "spec/proc/to_source_from_do_end_block_w_nested_do_end_block_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_for_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_if_spec.rb",
+     "spec/proc/to_source_from_do_end_block_w_nested_literal_keyword_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_method_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_module_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_unless_spec.rb",
@@ -79,6 +80,7 @@ Gem::Specification.new do |s|
      "spec/proc/to_source_from_do_end_block_w_nested_class_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_case_spec.rb",
      "spec/proc/to_source_magic_line_var_spec.rb",
+     "spec/proc/to_source_from_do_end_block_w_nested_literal_keyword_spec.rb",
      "spec/proc/to_source_from_do_end_block_w_nested_unless_spec.rb",
      "spec/proc/19x_extras.rb",
      "spec/proc/to_source_from_braced_block_w_nested_braced_block_spec.rb",

data/spec/proc/to_source_from_do_end_block_w_nested_literal_keyword_spec.rb

@@ -0,0 +1,101 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper')
+
+describe "Proc#to_source from do ... end block (w nested literal keyword)" do
+
+  should 'handle :class' do
+    (
+      lambda do
+        x = :class
+      end
+    ).should.be having_source('proc { x = :class }')
+  end
+
+  should 'handle :module' do
+    (
+      lambda do
+        x = :module
+      end
+    ).should.be having_source('proc { x = :module }')
+  end
+
+  should 'handle :def' do
+    (
+      lambda do
+        x = :def
+      end
+    ).should.be having_source('proc { x = :def }')
+  end
+
+  should 'handle :if' do
+    (
+      lambda do
+        x = :if
+      end
+    ).should.be having_source('proc { x = :if }')
+  end
+
+  should 'handle :unless' do
+    (
+      lambda do
+        x = :unless
+      end
+    ).should.be having_source('proc { x = :unless }')
+  end
+
+  should 'handle :for' do
+    (
+      lambda do
+        x = :for
+      end
+    ).should.be having_source('proc { x = :for }')
+  end
+
+  should 'handle :while' do
+    (
+      lambda do
+        x = :while
+      end
+    ).should.be having_source('proc { x = :while }')
+  end
+
+  should 'handle :until' do
+    (
+      lambda do
+        x = :until
+      end
+    ).should.be having_source('proc { x = :until }')
+  end
+
+  should 'handle :begin' do
+    (
+      lambda do
+        x = :begin
+      end
+    ).should.be having_source('proc { x = :begin }')
+  end
+
+  should 'handle :case' do
+    (
+      lambda do
+        x = :case
+      end
+    ).should.be having_source('proc { x = :case }')
+  end
+
+  should 'handle :do' do
+    (
+      lambda do
+        x = :do
+      end
+    ).should.be having_source('proc { x = :do }')
+  end
+
+  should 'handle :end' do
+    (
+      lambda do
+        x = :end
+      end
+    ).should.be having_source('proc { x = :end }')
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sourcify
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: false
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - NgTzeYang
@@ -86,6 +86,7 @@ files:
 - spec/proc/to_source_from_do_end_block_w_nested_do_end_block_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_for_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_if_spec.rb
+- spec/proc/to_source_from_do_end_block_w_nested_literal_keyword_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_method_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_module_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_unless_spec.rb
@@ -148,6 +149,7 @@ test_files:
 - spec/proc/to_source_from_do_end_block_w_nested_class_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_case_spec.rb
 - spec/proc/to_source_magic_line_var_spec.rb
+- spec/proc/to_source_from_do_end_block_w_nested_literal_keyword_spec.rb
 - spec/proc/to_source_from_do_end_block_w_nested_unless_spec.rb
 - spec/proc/19x_extras.rb
 - spec/proc/to_source_from_braced_block_w_nested_braced_block_spec.rb