textpow1x 1.1.0 → 1.2.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    textpow1x (1.1.0)
+    textpow1x (1.2.0)
       plist (>= 3.0.1)
 
 GEM

data/README.rdoc

@@ -1,7 +1,10 @@
 Read {TextMate}[http://macromates.com/] syntax files and parse text with them.
 
-For Ruby 1.8 and 1.9.
-
+Improvements over old textpow:
+ - Ruby 1.8 and 1.9 support
+ - Tested, documented, refactored and made +faster+
+ - Added tons of syntax files (li/textpow/syntax)
+ - Most included syntaxes work out of the box
 
 == INSTALL:
 
@@ -13,7 +16,7 @@ Install oniguruma
   # Ubuntu
   sudo apt-get -y install libonig-dev
   gem install oniguruma
-  
+
   # OsX (no idea if this works...)
   port install oniguruma4
 
@@ -23,7 +26,7 @@ Install oniguruma
 1. Load a Syntax File {list of included syntax}[https://github.com/grosser/textpow/tree/master/lib/textpow/syntax]
 
   require 'textpow'
-  syntax = Textpow::SyntaxNode.load("#{Textpow.syntax_path}/ruby.syntax")
+  syntax = Textpow.syntax('ruby') # or Textpow::Syntax.load(syntax_file)
 
 2. Initialize a processor:
 

data/examples/benchmark_js.rb

@@ -1,11 +1,13 @@
+# test changes for performance
+# ruby examples/benchmark_js.rb 100; git co lib/textpow/syntax.rb ; ruby examples/benchmark_js.rb 100
 $LOAD_PATH.unshift 'lib'
 require 'textpow'
 
-syntax = Textpow::SyntaxNode.load("#{Textpow.syntax_path}/javascript.syntax")
+syntax = Textpow.syntax('javascript')
 text = File.read('examples/jquery.js')
 processor = Textpow::RecordingProcessor.new
 
 start = Time.now.to_f
-5.times{ syntax.parse(text,  processor) }
+(ARGV[0] || 1).to_i.times{ syntax.parse(text,  processor) }
 puts Time.now.to_f - start
 

data/lib/textpow/syntax/{property_list.syntax → text.plist.syntax}

@@ -14,7 +14,7 @@ fileTypes:
 - scriptSuite
 - scriptTerminology
 - savedSearch
-scopeName: ""
+scopeName: "text.plist"
 repository: 
   openstep_comment: 
     patterns: 

data/lib/textpow/syntax.rb

@@ -4,37 +4,32 @@ end
 require 'oniguruma' unless Textpow::RUBY_19
 
 module Textpow
+  # at load time we do not know all patterns / all syntaxes
+  # so we store a proxy, that tries to find the correct syntax at runtime
   class SyntaxProxy
-    def initialize hash, syntax
+    def initialize(included_name, syntax)
       @syntax = syntax
-      @proxy = hash["include"]
+      @included_name = included_name
     end
 
     def method_missing method, *args, &block
-      if @proxy
-        @proxy_value = proxy unless @proxy_value
-        if @proxy_value
-          @proxy_value.send(method, *args, &block)
-        else
-          STDERR.puts "Failed proxying #{@proxy}.#{method}(#{args.join(', ')})"
-        end
+      if @proxy ||= proxy
+        @proxy.send(method, *args, &block)
+      else
+        STDERR.puts "Failed proxying #{@proxy_name}.#{method}(#{args.join(', ')})"
       end
     end
 
+  private
+
     def proxy
-      case @proxy
+      case @included_name
       when /^#/
-        if @syntax.repository && @syntax.repository[@proxy[1..-1]]
-          #puts "Repository"
-          #@table["syntax"].repository.each_key{|k| puts k}
-          return @syntax.repository[@proxy[1..-1]]
-        end
-      when "$self"
-        return @syntax
-      when "$base"
-        return @syntax
+        @syntax.repository and @syntax.repository[@included_name[1..-1]]
+      when "$self", "$base"
+        @syntax
       else
-        return @syntax.syntaxes[@proxy]
+        @syntax.syntaxes[@included_name] || Textpow.syntax(@included_name)
       end
     end
   end
@@ -103,7 +98,7 @@ module Textpow
               Oniguruma::ORegexp.new(value, :options => Oniguruma::OPTION_CAPTURE_GROUP)
             end
             instance_variable_set("@#{key}", regex)
-          rescue ArgumentError => e
+          rescue RegexpError, ArgumentError => e
             raise ParsingError, "Parsing error in #{value}: #{e.to_s}"
           end
         when "content", "fileTypes", "name", "contentName", "end", "scopeName", "keyEquivalent"
@@ -143,20 +138,19 @@ module Textpow
       @repository = {}
       repository.each do |key, value|
         if value["include"]
-          @repository[key] = SyntaxProxy.new( value, self.syntax )
+          @repository[key] = SyntaxProxy.new(value["include"], syntax)
         else
-          @repository[key] = SyntaxNode.new( value, self.syntax, @name_space )
+          @repository[key] = SyntaxNode.new(value, syntax, @name_space)
         end
       end
     end
 
     def create_children patterns
-      @patterns = []
-      patterns.each do |p|
-        if p["include"]
-          @patterns << SyntaxProxy.new( p, self.syntax )
+      @patterns = patterns.map do |pattern|
+        if pattern["include"]
+          SyntaxProxy.new(pattern["include"], syntax)
         else
-          @patterns << SyntaxNode.new( p, self.syntax, @name_space )
+          SyntaxNode.new(pattern, syntax, @name_space)
         end
       end
     end
@@ -239,20 +233,25 @@ module Textpow
       end
     end
 
+    # find earliest matching pattern
     def match_first_son(string, position)
       return if not patterns
 
-      match = nil
-      patterns.each do |p|
-        tmatch = p.match_first string, position
-        if tmatch
-          if not match or match_offset(match[1]).first > match_offset(tmatch[1]).first
-            match = tmatch
-          end
-          #break if tmatch[1].offset.first == position
+      earliest_match = nil
+      earliest_match_offset = nil
+      patterns.each do |pattern|
+        next unless match = pattern.match_first(string, position)
+
+        match_offset = match_offset(match[1]).first
+        return match if match_offset == 0 # no need to look any further
+
+        if not earliest_match or earliest_match_offset > match_offset
+          earliest_match = match
+          earliest_match_offset = match_offset
         end
       end
-      match
+
+      earliest_match
     end
 
     def parse_line(stack, line, processor)

data/lib/textpow/version.rb

@@ -1,3 +1,3 @@
 module Textpow
-  VERSION = Version = "1.1.0"
+  VERSION = Version = "1.2.0"
 end

data/lib/textpow.rb

@@ -11,5 +11,40 @@ module Textpow
   def self.syntax_path
     File.join(File.dirname(__FILE__), 'textpow', 'syntax')
   end
+
+  @@syntax = {}
+  def self.syntax(syntax_name)
+    syntax_name = syntax_name.downcase
+    if @@syntax.has_key?(syntax_name)
+      @@syntax[syntax_name]
+    else
+      @@syntax[syntax_name] = uncached_syntax(syntax_name)
+    end
+  end
+
+private
+
+  def self.uncached_syntax(name)
+    path = (
+      find_syntax_by_path(name) ||
+      find_syntax_by_scope_name(name) ||
+      find_syntax_by_fuzzy_name(name)
+    )
+    SyntaxNode.load(path) if path
+  end
+
+  def self.find_syntax_by_scope_name(name)
+    path = File.join(syntax_path, "#{name}.syntax")
+    path if File.exist?(path)
+  end
+
+  def self.find_syntax_by_fuzzy_name(name)
+    path = Dir.glob(File.join(syntax_path, "*.#{name}.*")).sort_by(&:size).first
+    path if path and File.exist?(path)
+  end
+
+  def self.find_syntax_by_path(path)
+    path if File.exist?(path)
+  end
 end
 

data/spec/textpow/syntax_spec.rb

@@ -11,7 +11,7 @@ describe Textpow::SyntaxNode do
     end
 
     it "can load from yaml .syntax" do
-      Textpow::SyntaxNode.load('lib/textpow/syntax/ruby.syntax').should_not == nil
+      Textpow::SyntaxNode.load('lib/textpow/syntax/source.ruby.syntax').should_not == nil
     end
 
     it "cannot load missing file" do
@@ -71,7 +71,7 @@ describe Textpow::SyntaxNode do
   end
 
   describe "#parse" do
-    let(:node){ Textpow::SyntaxNode.load('lib/textpow/syntax/ruby.syntax') }
+    let(:node){ Textpow.syntax('ruby') }
 
     it "uses a RecordingProcessor by default" do
       node.parse("111").stack.should == [
@@ -95,5 +95,131 @@ describe Textpow::SyntaxNode do
         [:end_parsing, "source.ruby"]
       ]
     end
+
+    it "can parse multiline syntax via stack" do
+      node.parse("=begin\n111\n=end").stack.should == [
+        [:start_parsing, "source.ruby"],
+        [:new_line, "=begin\n"],
+        [:open_tag, "comment.block.documentation.ruby", 0],
+        [:open_tag, "punctuation.definition.comment.ruby", 0],
+        [:close_tag, "punctuation.definition.comment.ruby", 6],
+        [:new_line, "111\n"],
+        [:new_line, "=end"],
+        [:open_tag, "punctuation.definition.comment.ruby", 0],
+        [:close_tag, "punctuation.definition.comment.ruby", 4],
+        [:close_tag, "comment.block.documentation.ruby", 4],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+
+    it "loads included syntax files" do
+      pending
+      node = Textpow.syntax('html_rails')
+      node.parse("<br/>")
+    end
+  end
+
+  describe "#match_first_son" do
+    it "returns nil when patterns are empty" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx'})
+      node.send(:match_first_son, "xxx", 0).should == nil
+    end
+
+    it "matches a pattern" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "xxx", "name" => "foo"}
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyy", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'foo'
+    end
+
+    it "matches the first pattern" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "yyy", "name" => "bar"},
+        {"match" => "xxx", "name" => "foo"},
+        {"match" => "zzz", "name" => "baz"},
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyyyzzz", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'foo'
+    end
+
+    it "matches the first pattern for equal positions" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "xxx", "name" => "bar"},
+        {"match" => "xxxyyy", "name" => "foo"},
+        {"match" => "xxxyy", "name" => "baz"},
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyyyzzz", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'bar'
+    end
+  end
+
+  describe "proxying" do
+    it "cannot proxy by scopeName if syntax is missing" do
+      STDERR.should_receive(:puts)
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'foo'}]})
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy by scopeName if syntax is available in repo" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'source.ruby'}]})
+      node.parse('1').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "1"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 1],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy by scopeName if syntax is loaded (even later)" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'foo'}]})
+      Textpow::SyntaxNode.new({"scopeName" => 'foo', 'patterns' => [{'name' => 'foo.1', 'match' => 'bar'}]})
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy to a pattern defined in an repository" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx',
+        'repository' => {'foo' => {'name' => 'foo.1', 'match' => 'bar'}},
+        'patterns' => [{'include' => '#foo'}]
+      })
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy to a pattern nested included in an repository" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx',
+        'repository' => {
+          'baz' => {'patterns' => [{'include' => '#foo'}]},
+          'foo' => {'name' => 'foo.1', 'match' => 'bar'}
+        },
+        'patterns' => [{'include' => '#baz'}]
+      })
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
   end
 end

data/spec/textpow_spec.rb

@@ -1,7 +1,53 @@
 require 'spec_helper'
 
 describe Textpow do
+  before do
+    Textpow.send(:class_variable_set, "@@syntax", {})
+  end
+
   it "has a version" do
     Textpow::Version =~ /^\d\.\d\.\d$/
   end
+
+  it "has syntax files named after their scopeName" do
+    Dir["#{Textpow.syntax_path}/*"].each do |file|
+      next if File.directory?(file)
+      (YAML.load_file(file)['scopeName'] + '.syntax').should == File.basename(file)
+    end
+  end
+
+  describe "syntax" do
+    it "finds syntax by path" do
+      Textpow.syntax('lib/textpow/syntax/source.ruby.syntax').name.should == 'Ruby'
+    end
+
+    it "finds a syntax by scopeName" do
+      Textpow.syntax('source.ruby').name.should == 'Ruby'
+    end
+
+    it "caches found syntax" do
+      Textpow.syntax('ruby').name.should == 'Ruby'
+      Dir.should_not_receive(:glob)
+      Textpow.syntax('ruby').name.should == 'Ruby'
+    end
+
+    it "finds a syntax by name parts" do
+      Textpow.syntax('ruby').name.should == 'Ruby'
+    end
+
+    it "returns the found syntax for mixed case" do
+      Textpow.syntax('RuBy').name.should == 'Ruby'
+    end
+
+    it "returns nil for unfound syntax" do
+      Textpow.syntax('buby').should == nil
+    end
+
+    it "caches nil for unfound syntax" do
+      Dir.should_receive(:glob).and_return []
+      Textpow.syntax('buby').should == nil
+      Dir.should_not_receive(:glob)
+      Textpow.syntax('buby').should == nil
+    end
+  end
 end

data/textpow1x.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "textpow/version"
+
+Gem::Specification.new do |s|
+  s.name                  = "textpow1x"
+  s.version               = Textpow::Version
+  s.authors               = ["Dizan Vasquez", "Spox", "Chris Hoffman", "Michael Grosser"]
+  s.email                 = ["michael@grosser.it"]
+  s.homepage              = "http://github.com/grosser/textpow"
+  s.summary               = "A library for parsing TextMate bundles on ruby 1.x"
+  s.description           = s.summary
+  s.license               = "MIT"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+  s.executables = ["plist2yaml", "plist2syntax"]
+  s.rdoc_options = ["--main", "README.rdoc"]
+
+  s.add_dependency "plist", '>=3.0.1'
+end