rkelly 1.0.1 → 1.0.3

data/CHANGELOG.rdoc

@@ -1,5 +1,13 @@
 = RKelly CHANGELOG
 
+=== 1.0.3
+
+* Bugfixes
+
+  * Fixed a typo in visitor.rb
+  * Fixed syntax error class
+  * Regex tonenization (thanks Josh Dzielak)
+
 === 1.0.1
 
 * Bugfixes

data/Manifest.txt

@@ -51,6 +51,7 @@ lib/rkelly/parser.rb
 lib/rkelly/runtime.rb
 lib/rkelly/runtime/ruby_function.rb
 lib/rkelly/runtime/scope_chain.rb
+lib/rkelly/syntax_error.rb
 lib/rkelly/token.rb
 lib/rkelly/tokenizer.rb
 lib/rkelly/visitable.rb

data/Rakefile

@@ -1,18 +1,17 @@
 require 'rubygems'
 require 'hoe'
 
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
-
-require 'rkelly/constants'
+Hoe.plugin :gemspec # `gem install hoe-gemspec`
+Hoe.plugin :git     # `gem install hoe-git`
 
 GENERATED_PARSER = "lib/rkelly/generated_parser.rb"
 
-HOE = Hoe.new('rkelly', RKelly::VERSION) do |p|
-  p.developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  p.readme_file   = 'README.rdoc'
-  p.history_file  = 'CHANGELOG.rdoc'
-  p.extra_rdoc_files  = FileList['*.rdoc']
-  p.clean_globs   = [GENERATED_PARSER]
+HOE = Hoe.spec('rkelly') do |p|
+  developer('Aaron Patterson', 'aaron.patterson@gmail.com')
+  self.readme_file   = 'README.rdoc'
+  self.history_file  = 'CHANGELOG.rdoc'
+  self.extra_rdoc_files  = FileList['*.rdoc']
+  self.clean_globs   = [GENERATED_PARSER]
 end
 
 file GENERATED_PARSER => "lib/parser.y" do |t|

data/lib/parser.y

@@ -590,7 +590,7 @@ rule
   | ExprNoBF error {
       result = ExpressionStatementNode.new(val.first)
       debug(result)
-      yyabort unless allow_auto_semi?(val.last)
+      raise RKelly::SyntaxError unless allow_auto_semi?(val.last)
     }
   ;
 

data/lib/rkelly.rb

@@ -3,6 +3,7 @@ require 'rkelly/visitable'
 require 'rkelly/visitors'
 require 'rkelly/parser'
 require 'rkelly/runtime'
+require 'rkelly/syntax_error'
 
 module RKelly
   class << self

data/lib/rkelly/constants.rb

@@ -1,3 +1,3 @@
 module RKelly
-  VERSION = '1.0.1'
+  VERSION = '1.0.3'
 end

data/lib/rkelly/generated_parser.rb

@@ -2777,7 +2777,7 @@ module_eval(<<'.,.,', 'parser.y', 590)
   def _reduce_240(val, _values, result)
           result = ExpressionStatementNode.new(val.first)
       debug(result)
-      yyabort unless allow_auto_semi?(val.last)
+      raise RKelly::SyntaxError unless allow_auto_semi?(val.last)
     
     result
   end

data/lib/rkelly/parser.rb

@@ -40,6 +40,10 @@ module RKelly
       apply_comments(ast)
     end
 
+    def yyabort
+      raise "something bad happened, please report a bug with sample JavaScript"
+    end
+
     private
     def apply_comments(ast)
       ast_hash = Hash.new { |h,k| h[k] = [] }

data/lib/rkelly/syntax_error.rb

@@ -0,0 +1,4 @@
+module RKelly
+  class SyntaxError < ::SyntaxError
+  end
+end

data/lib/rkelly/tokenizer.rb

@@ -45,6 +45,8 @@ module RKelly
       '/='  => :DIVEQUAL,
     }
 
+    TOKENS_THAT_IMPLY_DIVISION = [:IDENT, :NUMBER, ')', ']', '}']
+
     def initialize(&block)
       @lexemes = []
 
@@ -70,7 +72,7 @@ module RKelly
         [LITERALS[value], value]
       end
 
-      token(:IDENT, /\A(\w|\$)+/) do |type,value|
+      token(:IDENT, /\A([_\$A-Za-z][_\$0-9A-Za-z]*)/) do |type,value|
         if KEYWORDS.include?(value)
           [value.upcase.to_sym, value]
         elsif RESERVED.include?(value)
@@ -95,10 +97,13 @@ module RKelly
     def raw_tokens(string)
       tokens = []
       line_number = 1
+      accepting_regexp = true
       while string.length > 0
         longest_token = nil
 
         @lexemes.each { |lexeme|
+          next if lexeme.name == :REGEXP && !accepting_regexp
+
           match = lexeme.match(string)
           next if match.nil?
           longest_token = match if longest_token.nil?
@@ -106,6 +111,8 @@ module RKelly
           longest_token = match
         }
 
+        accepting_regexp = followable_by_regex(longest_token)
+
         longest_token.line = line_number
         line_number += longest_token.value.scan(/\n/).length
         string = string.slice(Range.new(longest_token.value.length, -1))
@@ -118,5 +125,12 @@ module RKelly
     def token(name, pattern = nil, &block)
       @lexemes << Lexeme.new(name, pattern, &block)
     end
+
+    def followable_by_regex(current_token)
+     name = current_token.name
+     name = current_token.value if name == :SINGLE_CHAR
+     #the tokens that imply division vs. start of regex form a disjoint set
+     !TOKENS_THAT_IMPLY_DIVISION.include?(name)
+    end
   end
 end

data/lib/rkelly/visitors/visitor.rb

@@ -29,7 +29,7 @@ module RKelly
       CONDITIONAL_NODES     = %w{ If Conditional }
       FUNC_CALL_NODES       = %w{ NewExpr FunctionCall }
       FUNC_DECL_NODES       = %w{ FunctionExpr FunctionDecl }
-      ALL_NODES = %w{ For ForIn Try BracketAccessor DotAcessor } +
+      ALL_NODES = %w{ For ForIn Try BracketAccessor DotAccessor } +
         TERMINAL_NODES + SINGLE_VALUE_NODES + BINARY_NODES + ARRAY_VALUE_NODES +
         NAME_VALUE_NODES + PREFIX_POSTFIX_NODES + CONDITIONAL_NODES +
         FUNC_CALL_NODES + FUNC_DECL_NODES

data/test/test_parser.rb

@@ -5,6 +5,12 @@ class ParserTest < Test::Unit::TestCase
     @parser = RKelly::Parser.new
   end
 
+  def test_birthday!
+    assert_raises(RKelly::SyntaxError) do
+      RKelly::Parser.new.parse "Happy birthday, tenderlove!"
+    end
+  end
+
   def test_array_access
     assert_sexp(
       [

data/test/test_tokenizer.rb

@@ -49,6 +49,11 @@ class TokenizerTest < Test::Unit::TestCase
     assert_tokens([[:IDENT, 'foo']], tokens)
   end
 
+  def test_ignore_identifier
+    tokens = @tokenizer.tokenize("0foo")
+    assert_tokens([[:NUMBER, 0], [:IDENT, 'foo']], tokens)
+  end
+
   def test_increment
     tokens = @tokenizer.tokenize("foo += 1;")
     assert_tokens([
@@ -79,6 +84,23 @@ class TokenizerTest < Test::Unit::TestCase
     ], tokens)
   end
 
+  def test_regular_expression_is_not_found_if_prev_token_implies_division
+    {:IDENT => 'foo',
+     :NUMBER => 1,
+     ')' => ')',
+     ']' => ']',
+     '}' => '}'}.each do |name, value|
+      tokens = @tokenizer.tokenize("#{value}/2/3")
+      assert_tokens([
+                  [name, value],
+                   ["/", "/"],
+                   [:NUMBER, 2],
+                   ["/", "/"],
+                   [:NUMBER, 3],
+      ], tokens)
+    end
+  end
+
   def test_comment_assign
     tokens = @tokenizer.tokenize("foo = /**/;")
     assert_tokens([

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rkelly
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  hash: 17
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 3
+  version: 1.0.3
 platform: ruby
 authors: 
 - Aaron Patterson
@@ -9,22 +15,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-08 00:00:00 -07:00
+date: 2011-02-28 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.12.2
-    version: 
+        hash: 41
+        segments: 
+        - 2
+        - 9
+        - 1
+        version: 2.9.1
+  type: :development
+  version_requirements: *id001
 description: The RKelly library will parse JavaScript and return a parse tree.
 email: 
-- aaronp@rubyforge.org
+- aaron.patterson@gmail.com
 executables: []
 
 extensions: []
@@ -87,6 +99,7 @@ files:
 - lib/rkelly/runtime.rb
 - lib/rkelly/runtime/ruby_function.rb
 - lib/rkelly/runtime/scope_chain.rb
+- lib/rkelly/syntax_error.rb
 - lib/rkelly/token.rb
 - lib/rkelly/tokenizer.rb
 - lib/rkelly/visitable.rb
@@ -232,6 +245,7 @@ files:
 - test/test_void_node.rb
 - test/test_while_node.rb
 - test/test_with_node.rb
+- .gemtest
 has_rdoc: true
 homepage: http://rkelly.rubyforge.org/
 licenses: []
@@ -243,21 +257,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: rkelly
-rubygems_version: 1.3.3
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: The RKelly library will parse JavaScript and return a parse tree.