visionmedia-lightr 0.0.4 → 0.0.5

data/lib/lightr/mixins/grammar.rb

@@ -1,53 +1,29 @@
 
 module Lightr
-  
-  ##
-  # = Grammar
-  #
-  # Primary Lightr syntax highlighting mixin, providing
-  # the #match method to create your own Lightr DSL.
-  #
-  # === Examples
-  #
-  #   module Lightr
-  #     class Ruby
-  #       include Grammar 
-  #       id  = /[\w\d]*/
-  #       int = /\d[\d_]*/
-  #       match :comment,  /#.*?(\n|\z)/
-  #       match :constant, /[A-Z]#{id}/
-  #       match :symbol,   /:#{id}/
-  #       match :number,   /-?#{int}(?:\.#{int})?/
-  #       match nil,       /\n|\s|.+?/
-  #    ...
-  #
-  
   module Grammar
     def self.included base
       base.extend ClassMethods
       base.send :include, InstanceMethods
     end
-    
+  
     module ClassMethods
-      def match name, regexp
-        (@matchers ||= []) << [name, regexp]
-      end
-      
       def parse input
         new(input).parse!
       end
     end
-    
+  
     module InstanceMethods
       attr_reader :input, :output
       alias :to_s :output
-      
+
       def initialize input
         @input, @output = input, ''
-        @matchers = self.class.instance_variable_get :"@matchers"
-        @regexp = Regexp.union *@matchers.transpose[1]
       end
-      
+
+      def parse!
+        parse @input
+      end
+    
       def escape string
         string.
           gsub(/&/, '&amp;').
@@ -55,21 +31,6 @@ module Lightr
           gsub(/>/, '&gt;').
           gsub(/"/, '&quot;')
       end
-      
-      def wrap string
-        @matchers.each do |name, regexp|
-          return escape(string) if name.nil? and string =~ regexp
-          return %(<span class="#{name}">#{escape(string)}</span>) if string =~ regexp
-        end
-      end
-      
-      def parse!
-        @input.dup.gsub @regexp do |match|
-          @output << wrap(match)
-        end
-        self
-      end
     end
-    
   end
-end
+end

data/lib/lightr/version.rb

@@ -1,4 +1,4 @@
 
 module Lightr
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lightr.gemspec

@@ -2,22 +2,22 @@
 
 Gem::Specification.new do |s|
   s.name = %q{lightr}
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["TJ Holowaychuk"]
-  s.date = %q{2009-03-07}
-  s.description = %q{Extremely light-weight syntax highlighting DSL.}
+  s.date = %q{2009-04-06}
+  s.description = %q{High performance syntax highlighting.}
   s.email = %q{tj@vision-media.ca}
-  s.extra_rdoc_files = ["lib/lightr/grammars/c.rb", "lib/lightr/grammars/javascript.rb", "lib/lightr/grammars/ruby.rb", "lib/lightr/grammars.rb", "lib/lightr/mixins/grammar.rb", "lib/lightr/mixins.rb", "lib/lightr/version.rb", "lib/lightr.rb", "README.rdoc", "tasks/docs.rake", "tasks/gemspec.rake", "tasks/spec.rake"]
-  s.files = ["examples/c.rb", "examples/javascript.rb", "examples/ruby.rb", "History.rdoc", "lib/lightr/grammars/c.rb", "lib/lightr/grammars/javascript.rb", "lib/lightr/grammars/ruby.rb", "lib/lightr/grammars.rb", "lib/lightr/mixins/grammar.rb", "lib/lightr/mixins.rb", "lib/lightr/version.rb", "lib/lightr.rb", "lightr.gemspec", "Manifest", "Rakefile", "README.rdoc", "spec/grammars/c_spec.rb", "spec/grammars/javascript_spec.rb", "spec/grammars/ruby_spec.rb", "spec/spec_helper.rb", "tasks/docs.rake", "tasks/gemspec.rake", "tasks/spec.rake", "Todo.rdoc"]
+  s.extra_rdoc_files = ["ext/javascript.rl", "ext/ruby.rl", "lib/lightr/grammars/javascript.rb", "lib/lightr/grammars/ruby.rb", "lib/lightr/grammars.rb", "lib/lightr/mixins/grammar.rb", "lib/lightr/mixins.rb", "lib/lightr/version.rb", "lib/lightr.rb", "README.rdoc", "tasks/benchmark.rake", "tasks/docs.rake", "tasks/ext.rake", "tasks/gemspec.rake", "tasks/spec.rake"]
+  s.files = ["benchmarks/_huge.rb", "benchmarks/_large.rb", "benchmarks/helper.rb", "benchmarks/huge.rb", "benchmarks/large.rb", "examples/javascript.rb", "examples/ruby.rb", "ext/javascript.rl", "ext/ruby.rl", "History.rdoc", "lib/lightr/grammars/javascript.rb", "lib/lightr/grammars/ruby.rb", "lib/lightr/grammars.rb", "lib/lightr/mixins/grammar.rb", "lib/lightr/mixins.rb", "lib/lightr/version.rb", "lib/lightr.rb", "lightr.gemspec", "Manifest", "Rakefile", "README.rdoc", "spec/grammars/javascript_spec.rb", "spec/grammars/ruby_spec.rb", "spec/spec_helper.rb", "tasks/benchmark.rake", "tasks/docs.rake", "tasks/ext.rake", "tasks/gemspec.rake", "tasks/spec.rake", "Todo.rdoc"]
   s.has_rdoc = true
   s.homepage = %q{http://github.com/visionmedia/lightr}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Lightr", "--main", "README.rdoc"]
-  s.require_paths = ["lib"]
+  s.require_paths = ["lib", "ext"]
   s.rubyforge_project = %q{lightr}
   s.rubygems_version = %q{1.3.1}
-  s.summary = %q{Extremely light-weight syntax highlighting DSL.}
+  s.summary = %q{High performance syntax highlighting.}
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/spec/grammars/javascript_spec.rb

@@ -8,30 +8,57 @@ describe Lightr::JavaScript do
   describe "should highlight" do
 
     it 'strings' do
-      parse('"test"').should include('class="string">"test"')
-      parse("'test'").should include(%(class="string">'test'))
+      parse('"test"').should highlight(:string, '"test"')
+      parse("'test'").should highlight(:string, "'test'")
+    end
+    
+    it "strings with escaped quotes" do
+      parse("'\''").should highlight(:string, "'\\''")
+      parse('"\""').should highlight(:string, '"\""')
+    end
+
+    it 'this' do
+      parse('this.test').should highlight(:self, 'this')
     end
     
     it 'comments' do
-      parse('test // comment').should include('class="comment">// comment')
-      # TODO: fixme
-      # parse('test // comment "Test"').should include('class="comment">// comment')
+      parse("//foo bar\n").should highlight(:comment, "//foo bar\n")
+      parse("test /// foo 'bar' \n").should highlight(:comment, "/// foo 'bar' \n")
+    end
+    
+    it "regular expressions" do
+      parse('/test/').should highlight(:regexp, '/test/')
+    end
+
+    it "regular expressions with mode flags" do
+      parse('/test/sex').should highlight(:regexp, '/test/sex')
+      parse('/test/sex test').should highlight(:regexp, '/test/sex')
+    end
+    
+    it "integers" do
+      parse('100').should highlight(:number, '100')
+      parse('100').should highlight(:int, '100')
+    end
+
+    it "floats" do
+      parse('1.3').should highlight(:number, '1.3')
+      parse('1.3').should highlight(:float, '1.3')
     end
         
-    %w( if else throw break case catch continue default delete do double true 
-      false final finally float for function int instanceof in long new null return short
-      switch throw try typeof var void while with ).
+    it "constructors as constants" do
+      parse('FooBar').should highlight(:constant, 'FooBar')
+      parse('FOOBAR').should highlight(:constant, 'FOOBAR')
+    end
+        
+    %w( if else break default case continue do delete true false class enum const final 
+    for function instanceof new null return throw try typeof var void with while finally in ).
     each do |keyword|
       it "#{keyword.strip} as a keyword" do
-        parse("#{keyword.strip} foo").should include(%(class="keyword">#{keyword.strip} <))
+        parse("\n#{keyword.strip} foo").should highlight(:keyword, keyword.strip)
+        parse("(#{keyword.strip})").should highlight(:keyword, keyword.strip)
       end
     end
-        
-  end
-
-  it 'should not highlight // within strings as comment' do
-    parse('"//test"').should include(%(class="string">&quot;//test&quot;))
-    parse("'//test'").should include(%(class="string">'//test'))
+    
   end
-  
-end
+    
+end

data/spec/grammars/ruby_spec.rb

@@ -8,65 +8,169 @@ describe Lightr::Ruby do
   describe "should highlight" do
 
     it 'symbols' do
-      parse('a :symbol').should include('class="symbol">:symbol')
-      parse(':"foo bar" test').should include('class="symbol">:&quot;foo bar&quot;<')
+      parse('a :symbol is cool').should highlight(:symbol, ':symbol')
+      parse(':"foo bar" test').should highlight(:symbol, ':&quot;foo bar&quot;')
     end
     
     it 'strings' do
-      parse('"test"').should include('class="string">&quot;test&quot;')
-      parse("'test'").should include(%(class="string">'test'))
+      parse('"test"').should highlight(:string, '&quot;test&quot;')
+      parse("'test'").should highlight(:string, "'test'")
+    end
+    
+    it "strings with escaped quotes" do
+      parse("'\''").should highlight(:string, "'\\''")
+      parse('"\""').should highlight(:string, '&quot;\&quot;&quot;')
+    end
+    
+    it "string %( alternate syntax" do
+      parse('%(something)').should highlight(:string, '%(something)')
+    end
+    
+    it "strings with interpolation" do
+      output = parse("begin#\{ (foo ? 'foo' : 'bar' ) \}end")
+      output.should include('class="string">begin<')
+      output.should include('class="string">foo<')
+      output.should include('class="string">bar<')
+      output.should include('class="string">end<')
+    end
+    
+    it "array %w( alternative syntax" do
+      parse('%w( foo bar )').should highlight(:string, '%w( foo bar )')
+      parse('%w( foo bar )').should highlight(:array, '%w( foo bar )')
     end
     
     it 'self' do
-      parse('self.test').should include('class="self">self<')
+      parse('self.test').should highlight(:self, 'self')
+    end
+    
+    it "block params" do
+      parse('|foo, bar|').should highlight(:params, '|foo, bar|')
     end
     
     it 'comments' do
-      parse('test # comment').should include('class="comment"># comment')
+      parse("# foo bar\n").should highlight(:comment, "# foo bar\n")
+      parse("test # foo 'bar' \n").should highlight(:comment, "# foo 'bar' \n")
+    end
+    
+    it "shell" do
+      parse("`foo bar`").should highlight(:shell, '`foo bar`')
+      parse("`foo bar`").should highlight(:string, '`foo bar`')
+    end
+    
+    it "shell %x{ alternative syntax" do
+      parse("%x{foo}").should highlight(:shell, '%x{foo}')
+      parse("%x{foo}").should highlight(:string, '%x{foo}')
     end
     
     it "regular expressions" do
-      parse('/test/').should include('class="regexp">/test/')
+      parse('/test/').should highlight(:regexp, '/test/')
+    end
+    
+    it "regular expressions when using %r{ alternative syntax" do
+      parse('%r{/foo/}').should highlight(:regexp, '%r{/foo/}')
+      parse('%r{/foo/}').should highlight(:string, '%r{/foo/}')
     end
     
     it "regular expressions with mode flags" do
-      parse('/test/sex').should include('class="regexp">/test/sex')
-      parse('/test/sex test').should include('class="regexp">/test/sex<')
+      parse('/test/sex').should highlight(:regexp, '/test/sex')
+      parse('/test/sex test').should highlight(:regexp, '/test/sex')
+      parse('%r{test}sex test').should highlight(:regexp, '%r{test}sex')
     end
     
     it "instance variables" do
-      parse('@foo_1').should include('class="variable">@foo_1<')
+      parse('@foo_1').should highlight(:instance, '@foo_1')
+      parse('@foo_1').should highlight(:variable, '@foo_1')
     end
     
     it "class variables" do
-      parse('@@__bar').should include('class="variable">@@__bar<')
+      parse('@@__bar').should highlight(:class, '@@__bar')
+      parse('@@__bar').should highlight(:variable, '@@__bar')
+    end
+
+    it "global variables" do
+      parse('$foo').should highlight(:variable, '$foo')
+      parse('$foo').should highlight(:global, '$foo')
+      parse('$:').should highlight(:variable, '$:')
+      parse('$:').should highlight(:global, '$:')
+      parse('($~)').should highlight(:variable, '$~')
+      parse('($~)').should highlight(:global, '$~')
     end
     
-    %w( 11 1_000 1.1 1.100_00 ).each do |number|
-      it "#{number} as number" do
-        parse(number).should include('class="number"')
-      end
+    it "integers" do
+      parse('100_000').should highlight(:number, '100_000')
+      parse('100_000').should highlight(:int, '100_000')
+    end
+
+    it "floats" do
+      parse('1.3').should highlight(:number, '1.3')
+      parse('1.3').should highlight(:float, '1.3')
     end
     
+    it "ranges" do
+      parse('1..10').should highlight(:number, '1..10')
+      parse('1..10').should highlight(:range, '1..10')
+      parse('1...10').should highlight(:number, '1...10')
+      parse('1...10').should highlight(:range, '1...10')
+    end
+        
     it "constants" do
-      parse('FooBar').should include('class="constant"')
-      parse('FOOBAR').should include('class="constant"')
+      parse('FooBar').should highlight(:constant, 'FooBar')
+      parse('FOOBAR').should highlight(:constant, 'FOOBAR')
+      parse('__FILE__').should highlight(:constant, '__FILE__')
+    end
+    
+    it "constants with namespace resolution" do
+      parse('Foo::Bar').should highlight(:constant, 'Foo::Bar')
+      parse('Foo::Bar.test()').should highlight(:constant, 'Foo::Bar')
+    end
+    
+    it "heredoc syntax" do
+      parse('
+        <<-EOF
+          stuff
+        EOF
+      ').should highlight(:string, "<<-EOF\n          stuff\n        EOF")
+      parse('
+        <<EOF
+          stuff
+        EOF
+      ').should highlight(:string, "<<EOF\n          stuff\n        EOF")
+      parse("
+        <<-'...'
+          stuff
+        ...
+      ").should highlight(:string, "<<-'...'\n          stuff\n        ...")
+    end
+    
+    it "contents after __END__ as a comment" do
+      ruby = parse(%(
+        :foo
+        __END__
+        :bar
+        should not be literal ruby
+        "test"
+      ))
+      ruby.should include('class="string symbol">:foo')
+      ruby.should include('class="keyword">__END__</span><span class="comment"')
     end
         
     %w( if else elsif unless retry super then true while until class module def end in alias 
     BEGIN END begin break defined do nil not or and redo rescue return yield ).
     each do |keyword|
       it "#{keyword.strip} as a keyword" do
-        parse("#{keyword.strip} foo").should include(%(class="keyword">#{keyword.strip} <))
+        parse("\n#{keyword.strip} foo").should highlight(:keyword, keyword.strip)
+        parse("(#{keyword.strip})").should highlight(:keyword, keyword.strip)
       end
     end
-        
-  end
     
-  it 'should not highlight # within regexp or strings as comment' do
-    parse('/#test/').should include('class="regexp">/#test/')
-    parse('"#test"').should include(%(class="string">&quot;#test&quot;))
-    parse("'#test'").should include(%(class="string">'#test'))
   end
   
+  describe "should not highlight" do
+    
+    it "keywords when part of other words" do
+      parse('define').should_not highlight(:keyword, 'def')
+    end
+    
+  end
+    
 end

data/spec/spec_helper.rb

@@ -1,2 +1,31 @@
 
 require 'lightr'
+
+class HighlightMatcher
+  def initialize klass, contents
+    @klass, @contents = klass, contents
+  end
+
+  def matches? got
+    @input = got
+    @classes = $1.split if got.match /class="(.*?)"/
+    @got = $1 if got.match /">(.*?)<\//
+    @classes.include? @klass.to_s and @got == @contents rescue false
+  end
+
+  def failure_message
+    "expected #{@input} to have class '#{@klass}' and contents of '#{@contents}'"
+  end
+
+  def negative_failure_message
+    "expected #{@input} to not have class '#{@klass}' and contents of '#{@contents}'"
+  end
+
+  def description
+    "highlight #{@klass}s"
+  end 
+end
+
+def highlight klass, contents
+  HighlightMatcher.new klass, contents
+end

data/tasks/benchmark.rake

@@ -0,0 +1,6 @@
+
+desc 'Run all benchmark suites for SIZE or large'
+task :benchmark do
+  size = ENV['SIZE'] || 'large'
+  ruby "benchmarks/#{size}.rb"
+end

data/tasks/ext.rake

@@ -0,0 +1,20 @@
+
+FSMS = Dir['ext/*.rl']
+
+namespace :ext do
+  
+  desc 'Build ragel scanners'
+  task :build do
+    FSMS.each do |fsm| 
+      `ragel #{fsm} -R -F1 -o lib/lightr/grammars/#{ File.basename(fsm).sub('.rl', '.rb') }`
+      puts "Built #{fsm}"
+    end
+  end
+  
+  desc 'Build scanners and run specs'
+  task :spec do
+    task(:'ext:build').invoke
+    task(:'spec').invoke
+  end
+  
+end