coderay 0.9.7.pre → 0.9.7

data/lib/README

@@ -18,7 +18,7 @@ And with line numbers.
 * is what everybody should have on their website
 * solves all your problems and makes the girls run after you
 
-Version: 0.9.6
+Version: 0.9.7
 Author:: murphy (Kornelius Kalnbach)
 Contact:: murphy rubychan de
 Website:: coderay.rubychan.de[http://coderay.rubychan.de]
@@ -94,6 +94,7 @@ Please report errors in this documentation to <murphy rubychan de>.
 * Rob Aldred for the terminal encoder
 * Trans for pointing out $DEBUG dependencies
 * Flameeyes for finding that Term::ANSIColor was obsolete
+* Etienne Massip for reporting a serious bug in JavaScript scanner
 * matz and all Ruby gods and gurus
 * The inventors of: the computer, the internet, the true color display, HTML &
   CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime, 

data/lib/coderay/scanners/java_script-0.9.6.rb

@@ -0,0 +1,224 @@
+module CodeRay
+module Scanners
+
+  class JavaScript < Scanner
+
+    include Streamable
+
+    register_for :java_script
+    file_extension 'js'
+
+    # The actual JavaScript keywords.
+    KEYWORDS = %w[
+      break case catch continue default delete do else
+      finally for function if in instanceof new
+      return switch throw try typeof var void while with
+    ]
+    PREDEFINED_CONSTANTS = %w[
+      false null true undefined
+    ]
+    
+    MAGIC_VARIABLES = %w[ this arguments ]  # arguments was introduced in JavaScript 1.4
+    
+    KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+      case delete in instanceof new return throw typeof with
+    ]
+    
+    # Reserved for future use.
+    RESERVED_WORDS = %w[
+      abstract boolean byte char class debugger double enum export extends
+      final float goto implements import int interface long native package
+      private protected public short static super synchronized throws transient
+      volatile
+    ]
+    
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant).
+      add(MAGIC_VARIABLES, :local_variable).
+      add(KEYWORDS, :keyword)
+
+    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+    REGEXP_ESCAPE =  / [bBdDsSwW] /x
+    STRING_CONTENT_PATTERN = {
+      "'" => /[^\\']+/,
+      '"' => /[^\\"]+/,
+      '/' => /[^\\\/]+/,
+    }
+    KEY_CHECK_PATTERN = {
+      "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
+      '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
+    }
+
+    def scan_tokens tokens, options
+
+      state = :initial
+      string_delimiter = nil
+      value_expected = true
+      key_expected = false
+      function_expected = false
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            value_expected = true if !value_expected && match.index(?\n)
+            tokens << [match, :space]
+            next
+
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            value_expected = true
+            kind = :comment
+
+          elsif check(/\.?\d/)
+            key_expected = value_expected = false
+            if scan(/0[xX][0-9A-Fa-f]+/)
+              kind = :hex
+            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              kind = :oct
+            elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+              kind = :float
+            elsif scan(/\d+/)
+              kind = :integer
+            end
+          
+          elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
+            # FIXME: scan over nested tags
+            xml_scanner.tokenize match
+            value_expected = false
+            next
+            
+          elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
+            value_expected = true
+            last_operator = match[-1]
+            key_expected = (last_operator == ?{) || (last_operator == ?,)
+            function_expected = false
+            kind = :operator
+
+          elsif scan(/ [)\]}]+ /x)
+            function_expected = key_expected = value_expected = false
+            kind = :operator
+
+          elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
+            kind = IDENT_KIND[match]
+            value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+            # TODO: labels
+            if kind == :ident
+              if match.index(?$)  # $ allowed inside an identifier
+                kind = :predefined
+              elsif function_expected
+                kind = :function
+              elsif check(/\s*[=:]\s*function\b/)
+                kind = :function
+              elsif key_expected && check(/\s*:/)
+                kind = :key
+              end
+            end
+            function_expected = (kind == :keyword) && (match == 'function')
+            key_expected = false
+          
+          elsif match = scan(/["']/)
+            if key_expected && check(KEY_CHECK_PATTERN[match])
+              state = :key
+            else
+              state = :string
+            end
+            tokens << [:open, state]
+            string_delimiter = match
+            kind = :delimiter
+
+          elsif value_expected && (match = scan(/\/(?=\S)/))
+            tokens << [:open, :regexp]
+            state = :regexp
+            string_delimiter = '/'
+            kind = :delimiter
+
+          elsif scan(/ \/ /x)
+            value_expected = true
+            key_expected = false
+            kind = :operator
+
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string, :regexp, :key
+          if scan(STRING_CONTENT_PATTERN[string_delimiter])
+            kind = :content
+          elsif match = scan(/["'\/]/)
+            tokens << [match, :delimiter]
+            if state == :regexp
+              modifiers = scan(/[gim]+/)
+              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+            end
+            tokens << [:close, state]
+            string_delimiter = nil
+            key_expected = value_expected = false
+            state = :initial
+            next
+          elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+            if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
+              kind = :content
+            else
+              kind = :char
+            end
+          elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/\\./m)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, state]
+            kind = :error
+            key_expected = value_expected = false
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $CODERAY_DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+
+      end
+
+      if [:string, :regexp].include? state
+        tokens << [:close, state]
+      end
+
+      tokens
+    end
+
+  protected
+
+    def reset_instance
+      super
+      @xml_scanner.reset if defined? @xml_scanner
+    end
+
+    def xml_scanner
+      @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
+    end
+
+  end
+  
+end
+end

data/lib/coderay/scanners/java_script.rb

@@ -47,8 +47,8 @@ module Scanners
       '/' => /[^\\\/]+/,
     }
     KEY_CHECK_PATTERN = {
-      "'" => / (?> [^\\']* (?: \\.? [^\\']* )* ) '? \s* : /x,
-      '"' => / (?> [^\\"]* (?: \\.? [^\\"]* )* ) "? \s* : /x,
+      "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx,
+      '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx,
     }
 
     def scan_tokens tokens, options

data/test/functional/basic.rbc

@@ -110,7 +110,7 @@ x
 9
 BasicTest
 i
-249
+263
 5
 66
 99
@@ -359,6 +359,20 @@ i
 49
 3
 4
+15
+99
+7
+49
+7
+50
+65
+67
+49
+2
+0
+49
+3
+4
 11
 I
 a
@@ -370,7 +384,7 @@ I
 0
 n
 p
-49
+51
 x
 12
 test_version
@@ -1807,8 +1821,74 @@ x
 /Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
 p
 0
-p
+x
+25
+test_scan_a_frozen_string
+M
+1
+n
+n
+x
+25
+test_scan_a_frozen_string
+i
+12
+45
+0
+1
+45
+2
+3
+7
+4
 49
+5
+2
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+6
+x
+7
+CodeRay
+n
+x
+12
+RUBY_VERSION
+n
+x
+4
+ruby
+x
+4
+scan
+p
+5
+I
+0
+I
+76
+I
+0
+I
+77
+I
+c
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+p
+51
 I
 2
 I
@@ -1907,6 +1987,10 @@ I
 71
 I
 f9
+I
+76
+I
+107
 x
 55
 /Users/murphy/ruby/coderay-0.9/test/functional/basic.rb

data/test/functional/for_redcloth.rb

@@ -1,5 +1,5 @@
 require 'test/unit'
-$: << 'lib'
+$:.unshift 'lib'
 require 'coderay'
 
 begin

data/test/functional/suite.rb

@@ -2,7 +2,7 @@ require 'test/unit'
 
 MYDIR = File.dirname(__FILE__)
 
-$: << 'lib'
+$:.unshift 'lib'
 require 'coderay'
 puts "Running basic CodeRay #{CodeRay::VERSION} tests..."
 

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: coderay
 version: !ruby/object:Gem::Version 
-  hash: 961915920
-  prerelease: true
+  hash: 53
+  prerelease: false
   segments: 
   - 0
   - 9
   - 7
-  - pre
-  version: 0.9.7.pre
+  version: 0.9.7
 platform: ruby
 authors: 
 - murphy
@@ -16,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-14 00:00:00 +01:00
+date: 2011-01-15 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -74,6 +73,7 @@ files:
 - ./lib/coderay/scanners/html.rb
 - ./lib/coderay/scanners/java/builtin_types.rb
 - ./lib/coderay/scanners/java.rb
+- ./lib/coderay/scanners/java_script-0.9.6.rb
 - ./lib/coderay/scanners/java_script.rb
 - ./lib/coderay/scanners/json.rb
 - ./lib/coderay/scanners/nitro_xhtml.rb
@@ -138,14 +138,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version 
-      hash: 25
+      hash: 3
       segments: 
-      - 1
-      - 3
-      - 1
-      version: 1.3.1
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: coderay