rkelly-remix 0.0.1 → 0.0.2

data/README.rdoc

@@ -1,38 +1,12 @@
-= RKelly
-
-* http://rkelly.rubyforge.org/
+= RKelly Remix
 
 == DESCRIPTION
 
-The RKelly library will parse JavaScript and return a parse tree.
-
-== Example
-
-  ##
-  # Iterate over and modify a JavaScript AST.  Then print the modified
-  # AST as JavaScript.
-  require 'rkelly'
+Fork of the RKelly library to make it suitable as the JavaScript parser
+in JSDuck.
 
-  parser = RKelly::Parser.new
-  ast    = parser.parse(
-    "for(var i = 0; i < 10; i++) { var x = 5 + 5; }"
-  )
-  
-  ast.each do |node|
-    node.value  = 'hello' if node.value == 'i'
-    node.name   = 'hello' if node.respond_to?(:name) && node.name == 'i'
-  end
-  puts ast.to_ecma # => awesome javascript
-
-== Acknowledgments
-
-The original javascript parser was was taken from rbnarcissus written by
-Paul Sowden.  Thanks Paul!
-
-  http://idontsmoke.co.uk/2005/rbnarcissus/
+* http://rkelly.rubyforge.org/
 
-The current parser was ported from WebKit.  Thank you WebKit team!
-  
 == License
 
 The MIT License

data/lib/rkelly/constants.rb

@@ -1,3 +1,3 @@
 module RKelly
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

data/lib/rkelly/tokenizer.rb

@@ -64,12 +64,23 @@ module RKelly
       '}' => true,
     }
 
+    # Determine the method to use to measure String length in bytes,
+    # because StringScanner#pos can obly be set in bytes.
+    #
+    # - In Ruby 1.8 String#length returns always the string length
+    #   in bytes.
+    #
+    # - In Ruby 1.9+ String#length returns string length in
+    #   characters and we need to use String#bytesize instead.
+    #
+    BYTESIZE_METHOD = "".respond_to?(:bytesize) ? :bytesize : :length
+
     def initialize(&block)
       @lexemes = Hash.new {|hash, key| hash[key] = [] }
 
       token(:COMMENT, /\/(?:\*(?:.)*?\*\/|\/[^\n]*)/m, ['/'])
       token(:STRING, /"(?:[^"\\]*(?:\\.[^"\\]*)*)"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'/m, ["'", '"'])
-      token(:S, /[\s\r\n]*/m, [" ", "\t", "\r", "\n", "\f"])
+      token(:S, /\s*/m, [" ", "\t", "\r", "\n", "\f"])
 
       # A regexp to match floating point literals (but not integer literals).
       digits = ('0'..'9').to_a
@@ -148,7 +159,7 @@ module RKelly
           accepting_regexp = followable_by_regex(token)
         end
 
-        scanner.pos += token.value.length
+        scanner.pos += token.value.send(BYTESIZE_METHOD)
         token.range = range = range.next(token.value)
         tokens << token
       end

data/test/test_tokenizer.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 require File.dirname(__FILE__) + "/helper"
 
 class TokenizerTest < Test::Unit::TestCase
@@ -162,6 +163,26 @@ class TokenizerTest < Test::Unit::TestCase
     ], tokens)
   end
 
+  def test_unicode_string
+    tokens = @tokenizer.tokenize("foo = 'öäüõ';")
+    assert_tokens([
+                 [:IDENT, 'foo'],
+                 ['=', '='],
+                 [:STRING, "'öäüõ'"],
+                 [';', ';'],
+    ], tokens)
+  end
+
+  def test_unicode_regex
+    tokens = @tokenizer.tokenize("foo = /öäüõ/;")
+    assert_tokens([
+                 [:IDENT, 'foo'],
+                 ['=', '='],
+                 [:REGEXP, "/öäüõ/"],
+                 [';', ';'],
+    ], tokens)
+  end
+
   def assert_tokens(expected, actual)
     assert_equal(expected, actual.select { |x| x[0] != :S })
   end

metadata

@@ -1,66 +1,65 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rkelly-remix
-version: !ruby/object:Gem::Version 
-  hash: 29
+version: !ruby/object:Gem::Version
+  version: 0.0.2
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 1
-  version: 0.0.1
 platform: ruby
-authors: 
+authors:
 - Aaron Patterson
 - Rene Saarsoo
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-07-03 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-07-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rdoc
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 4
-        - 0
-        version: "4.0"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: hoe
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 3
-        - 6
-        version: "3.6"
+      - !ruby/object:Gem::Version
+        version: '3.6'
   type: :development
-  version_requirements: *id002
-description: The RKelly library will parse JavaScript and return a parse tree.
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.6'
+description: ! 'Fork of the RKelly library to make it suitable as the JavaScript parser
+
+  in JSDuck.
+
+
+  * http://rkelly.rubyforge.org/'
+email:
 - aaron.patterson@gmail.com
 - rene.saarsoo@sencha.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - CHANGELOG.rdoc
 - Manifest.txt
 - README.rdoc
-files: 
+files:
 - CHANGELOG.rdoc
 - Manifest.txt
 - README.rdoc
@@ -265,41 +264,34 @@ files:
 - test/test_while_node.rb
 - test/test_with_node.rb
 - .gemtest
-homepage: http://rkelly.rubyforge.org/
+homepage: 
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --main
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: rkelly-remix
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: The RKelly library will parse JavaScript and return a parse tree.
-test_files: 
+summary: Fork of the RKelly library to make it suitable as the JavaScript parser in
+  JSDuck
+test_files:
 - test/test_parser.rb
 - test/test_ecma_visitor.rb
 - test/test_arguments_node.rb