textpow1x 1.0.0 → 1.1.0

data/lib/textpow/syntax.rb

@@ -3,8 +3,6 @@ module Textpow
 end
 require 'oniguruma' unless Textpow::RUBY_19
 
-require 'plist'
-
 module Textpow
   class SyntaxProxy
     def initialize hash, syntax
@@ -42,10 +40,6 @@ module Textpow
   end
 
   class SyntaxNode
-    unless Textpow::RUBY_19
-      OPTIONS = {:options => Oniguruma::OPTION_CAPTURE_GROUP}
-    end
-
     @@syntaxes = {}
 
     attr_accessor :syntax
@@ -67,72 +61,84 @@ module Textpow
     attr_accessor :repository
     attr_accessor :patterns
 
-    def self.load filename, name_space = :default
-      table = nil
-      case filename
-      when /(\.tmSyntax|\.plist)$/
-        table = Plist::parse_xml( filename )
-      else
-        File.open( filename ) do |f|
-          table = YAML.load( f )
-        end
-      end
-      if table
-        SyntaxNode.new( table, nil, name_space )
-      else
-        nil
-      end
+    def self.load(file, name_space = :default)
+      table = convert_file_to_table(file)
+      SyntaxNode.new(table, nil, name_space)
     end
 
-    def initialize hash, syntax = nil, name_space = :default
-      @name_space = name_space
-      @@syntaxes[@name_space] ||= {}
-      @@syntaxes[@name_space][hash["scopeName"]] = self if hash["scopeName"]
+    def initialize(table, syntax = nil, name_space = :default)
       @syntax = syntax || self
-      hash.each do |key, value|
+      @name_space = name_space
+
+      register_in_syntaxes(table["scopeName"])
+      parse_and_store_syntax_info(table)
+    end
+
+    def syntaxes
+      @@syntaxes[@name_space]
+    end
+
+    def parse(string, processor = RecordingProcessor.new)
+      processor.start_parsing scopeName
+      stack = [[self, nil]]
+      string.each_line do |line|
+        parse_line stack, line, processor
+      end
+      processor.end_parsing scopeName
+
+      processor
+    end
+
+  protected
+
+    def parse_and_store_syntax_info(table)
+      table.each do |key, value|
         case key
         when "firstLineMatch", "foldingStartMarker", "foldingStopMarker", "match", "begin"
           begin
-            if Textpow::RUBY_19
+            regex = if Textpow::RUBY_19
               value.force_encoding("ASCII-8BIT")
-              instance_variable_set( "@#{key}", Regexp.new( value ) )
+              Regexp.new(value)
             else
-              instance_variable_set( "@#{key}", Oniguruma::ORegexp.new( value, OPTIONS ) )
+              Oniguruma::ORegexp.new(value, :options => Oniguruma::OPTION_CAPTURE_GROUP)
             end
+            instance_variable_set("@#{key}", regex)
           rescue ArgumentError => e
             raise ParsingError, "Parsing error in #{value}: #{e.to_s}"
           end
         when "content", "fileTypes", "name", "contentName", "end", "scopeName", "keyEquivalent"
-          instance_variable_set( "@#{key}", value )
+          instance_variable_set("@#{key}", value)
         when "captures", "beginCaptures", "endCaptures"
-          instance_variable_set( "@#{key}", value.sort )
+          instance_variable_set("@#{key}", value.sort)
         when "repository"
           parse_repository value
         when "patterns"
           create_children value
+        when "comment"
         else
           STDERR.puts "Ignoring: #{key} => #{value.gsub("\n", "\n>>")}" if $DEBUG
         end
       end
     end
 
-
-    def syntaxes
-      @@syntaxes[@name_space]
+    # register in global syntax list -> can be found by include
+    def register_in_syntaxes(scope)
+      @@syntaxes[@name_space] ||= {}
+      @@syntaxes[@name_space][scope] = self if scope
     end
 
-    def parse( string, processor = nil )
-      processor.start_parsing self.scopeName if processor
-      stack = [[self, nil]]
-      string.each_line do |line|
-        parse_line stack, line, processor
+    def self.convert_file_to_table(file)
+      raise "File not found: #{file}" unless File.exist?(file)
+
+      case file
+      when /(\.tmSyntax|\.plist)$/
+        require 'plist'
+        Plist::parse_xml(file)
+      else
+        YAML.load_file(file)
       end
-      processor.end_parsing self.scopeName if processor
-      processor
     end
 
-    protected
-
     def parse_repository repository
       @repository = {}
       repository.each do |key, value|
@@ -233,91 +239,80 @@ module Textpow
       end
     end
 
-    def match_first_son string, position
+    def match_first_son(string, position)
+      return if not patterns
+
       match = nil
-      if self.patterns
-        self.patterns.each do |p|
-          tmatch = p.match_first string, position
-          if tmatch
-            ok = if Textpow::RUBY_19
-              ! match || match[1].offset(0).first > tmatch[1].offset(0).first
-            else
-              ! match || match[1].offset.first > tmatch[1].offset.first
-            end
-            match = tmatch if ok
-            #break if tmatch[1].offset.first == position
+      patterns.each do |p|
+        tmatch = p.match_first string, position
+        if tmatch
+          if not match or match_offset(match[1]).first > match_offset(tmatch[1]).first
+            match = tmatch
           end
+          #break if tmatch[1].offset.first == position
         end
       end
       match
     end
 
-    def parse_line stack, line, processor
-      processor.new_line line if processor
+    def parse_line(stack, line, processor)
+      processor.new_line line
       top, match = stack.last
       position = 0
       #@ln ||= 0
       #@ln += 1
       #STDERR.puts @ln
-      while true
+      loop do
         if top.patterns
-          pattern, pattern_match = top.match_first_son line, position
-        else
-          pattern, pattern_match = nil
+          pattern, pattern_match = top.match_first_son(line, position)
         end
 
-        end_match = nil
         if top.end
           end_match = top.match_end( line, match, position )
         end
 
-        ok = if Textpow::RUBY_19
-          end_match && ( ! pattern_match || pattern_match.offset(0).first >= end_match.offset(0).first )
-        else
-          end_match && ( ! pattern_match || pattern_match.offset.first >= end_match.offset.first )
-        end
-
-        if ok
+        if end_match and (not pattern_match or match_offset(pattern_match).first >= match_offset(end_match).first)
           pattern_match = end_match
-          if Textpow::RUBY_19
-            start_pos = pattern_match.offset(0).first
-            end_pos = pattern_match.offset(0).last
-          else
-            start_pos = pattern_match.offset.first
-            end_pos = pattern_match.offset.last
-          end
-          processor.close_tag top.contentName, start_pos if top.contentName && processor
-          parse_captures "captures", top, pattern_match, processor if processor
-          parse_captures "endCaptures", top, pattern_match, processor if processor
-          processor.close_tag top.name, end_pos if top.name && processor
+          start_pos = match_offset(pattern_match).first
+          end_pos = match_offset(pattern_match).last
+
+          processor.close_tag top.contentName, start_pos if top.contentName
+          parse_captures "captures", top, pattern_match, processor
+          parse_captures "endCaptures", top, pattern_match, processor
+          processor.close_tag top.name, end_pos if top.name
           stack.pop
           top, match = stack.last
         else
           break unless pattern
-          if Textpow::RUBY_19
-            start_pos = pattern_match.offset(0).first
-            end_pos = pattern_match.offset(0).last
-          else
-            start_pos = pattern_match.offset.first
-            end_pos = pattern_match.offset.last
-          end
+
+          start_pos = match_offset(pattern_match).first
+          end_pos = match_offset(pattern_match).last
 
           if pattern.begin
-            processor.open_tag pattern.name, start_pos if pattern.name && processor
-            parse_captures "captures", pattern, pattern_match, processor if processor
-            parse_captures "beginCaptures", pattern, pattern_match, processor if processor
-            processor.open_tag pattern.contentName, end_pos if pattern.contentName && processor
+            processor.open_tag pattern.name, start_pos if pattern.name
+            parse_captures "captures", pattern, pattern_match, processor
+            parse_captures "beginCaptures", pattern, pattern_match, processor
+            processor.open_tag pattern.contentName, end_pos if pattern.contentName
             top = pattern
             match = pattern_match
             stack << [top, match]
           elsif pattern.match
-            processor.open_tag pattern.name, start_pos if pattern.name && processor
-            parse_captures "captures", pattern, pattern_match, processor if processor
-            processor.close_tag pattern.name, end_pos if pattern.name && processor
+            processor.open_tag pattern.name, start_pos if pattern.name
+            parse_captures "captures", pattern, pattern_match, processor
+            processor.close_tag pattern.name, end_pos if pattern.name
           end
         end
+
         position = end_pos
       end
     end
+
+    def match_offset(match)
+      if Textpow::RUBY_19
+        match.offset(0)
+      else
+        match.offset
+      end
+    end
   end
 end

data/lib/textpow/version.rb

@@ -1,3 +1,3 @@
 module Textpow
-  Version = "1.0.0"
+  VERSION = Version = "1.1.0"
 end

data/lib/textpow.rb

@@ -1,6 +1,7 @@
 require 'yaml'
 require 'textpow/syntax'
 require 'textpow/debug_processor'
+require 'textpow/recording_processor'
 require 'textpow/score_manager'
 require 'textpow/version'
 

data/spec/fixtures/objeck.plist

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>BBEditDocumentType</key>
+	<string>CodelessLanguageModule</string>
+	<key>BBLMColorsSyntax</key>
+	<true/>
+	<key>BBLMIsCaseSensitive</key>
+	<true/>
+	<key>BBLMKeywordList</key>
+	<array>
+		<string>and</string>
+		<string>or</string>
+		<string>xor</string>
+		<string>virtual</string>
+		<string>if</string>
+		<string>else</string>
+		<string>do</string>
+		<string>while</string>
+		<string>use</string>
+		<string>bundle</string>
+		<string>native</string>
+		<string>static</string>
+		<string>public</string>
+		<string>private</string>
+		<string>class</string>
+		<string>interface</string>
+		<string>select</string>
+		<string>other</string>
+		<string>enum</string>
+		<string>for</string>
+		<string>each</string>
+		<string>label</string>
+		<string>return</string>
+		<string>Byte</string>
+		<string>Int</string>
+		<string>Parent</string>
+		<string>from</string>
+		<string>Float</string>
+		<string>Char</string>
+		<string>Bool</string>
+		<string>String</string>
+		<string>Nil</string>
+		<string>true</string>
+		<string>false</string>
+		<string>function</string>
+		<string>method</string>
+	</array>
+	<key>BBLMLanguageCode</key>
+	<string>LSL</string>
+	<key>BBLMLanguageDisplayName</key>
+	<string>Objeck</string>
+	<key>BBLMScansFunctions</key>
+	<true/>
+	<key>BBLMSuffixMap</key>
+	<array>
+		<dict>
+			<key>BBLMLanguageSuffix</key>
+			<string>.obs</string>
+		</dict>
+	</array>
+	<key>Language Features</key>
+	<dict>
+		<key>Close Block Comments</key>
+		<string>~#</string>
+		<key>Close Parameter Lists</key>
+		<string>)</string>
+		<key>Close Statement Blocks</key>
+		<string>}</string>
+		<key>Close Strings 1</key>
+		<string>&quot;</string>
+		<key>Close Strings 2</key>
+		<string>&apos;</string>
+		<key>End-of-line Ends Strings 1</key>
+		<false/>
+		<key>End-of-line Ends Strings 2</key>
+		<false/>
+		<key>Escape Char in Strings 1</key>
+		<string>\</string>
+		<key>Escape Char in Strings 2</key>
+		<string>\</string>
+		<key>Identifier and Keyword Characters</key>
+		<string>@0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz</string>
+		<key>Open Block Comments</key>
+		<string>#~</string>
+		<key>Open Line Comments</key>
+		<string>#</string>
+		<key>Open Parameter Lists</key>
+		<string>(</string>
+		<key>Open Statement Blocks</key>
+		<string>{</string>
+		<key>Open Strings 1</key>
+		<string>&quot;</string>
+		<key>Open Strings 2</key>
+		<string>&apos;</string>
+		<key>Prefix for Functions</key>
+		<string></string>
+		<key>Prefix for Procedures</key>
+		<string></string>
+		<key>Terminator for Prototypes 1</key>
+		<string></string>
+		<key>Terminator for Prototypes 2</key>
+		<string></string>
+	</dict>
+</dict>
+</plist>

data/spec/textpow/score_manager_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Textpow::ScoreManager do
+  it "calculates scores" do
+    sp = Textpow::ScoreManager.new
+    reference_scope = 'text.html.basic source.php.embedded.html string.quoted.double.php'
+
+    sp.score('source.php string', reference_scope).should_not == 0
+    sp.score('text.html source.php', reference_scope).should_not == 0
+    sp.score('string source.php', reference_scope).should == 0
+    sp.score('source.php text.html', reference_scope).should == 0
+
+    sp.score('text.html source.php - string', reference_scope).should == 0
+    sp.score('text.html source.php - ruby', reference_scope ).should_not == 0
+
+    sp.score('string', reference_scope).should > sp.score('source.php', reference_scope)
+    sp.score('string.quoted', reference_scope).should > sp.score('source.php', reference_scope)
+    sp.score('text source string', reference_scope).should > sp.score( 'source string', reference_scope)
+  end
+end

data/spec/textpow/syntax_files_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe "syntax files" do
+  before do
+    STDERR.stub!(:puts)
+  end
+
+  let(:processor){ Textpow::DebugProcessor.new }
+
+  it "has syntax files" do
+    Dir["#{Textpow.syntax_path}/*.syntax"].should_not == []
+  end
+
+  Dir["#{Textpow.syntax_path}/*.syntax"].each do |syntax|
+    it "#{syntax} can parse" do
+      node = Textpow::SyntaxNode.load(syntax)
+      node.parse("xxx\n1 + 1\n### xxx", processor)
+    end
+  end
+
+  # syntax broken in 1.9
+  xit "parses markdown" do
+    node = Textpow::SyntaxNode.load("#{Textpow.syntax_path}/broken/markdown.syntax")
+    node.parse("### xxx\nabc\n    xxx\n    yyy\n - abc\n - ac", processor)
+  end
+end

data/spec/textpow/syntax_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+describe Textpow::SyntaxNode do
+  before do
+    Textpow::SyntaxNode.send(:class_variable_set, "@@syntaxes", {})
+  end
+
+  describe "#load" do
+    it "can load from xml .plist" do
+      Textpow::SyntaxNode.load('spec/fixtures/objeck.plist').should_not == nil
+    end
+
+    it "can load from yaml .syntax" do
+      Textpow::SyntaxNode.load('lib/textpow/syntax/ruby.syntax').should_not == nil
+    end
+
+    it "cannot load missing file" do
+      lambda{
+        Textpow::SyntaxNode.load('xxx.syntax')
+      }.should raise_error
+    end
+
+    it "cannot load missing plist file" do
+      lambda{
+        Textpow::SyntaxNode.load('xxx.plist')
+      }.should raise_error
+    end
+  end
+
+  describe "#new" do
+    it "loads strings from given hash" do
+      syntax = Textpow::SyntaxNode.new('content' => 'CONTENT', 'name' => 'NAME')
+      syntax.content.should == 'CONTENT'
+      syntax.name.should == 'NAME'
+    end
+
+    it "loads regex from given hash" do
+      syntax = Textpow::SyntaxNode.new('firstLineMatch' => 'aaa', 'foldingStartMarker' => 'bbb')
+      syntax.firstLineMatch.inspect.should == "/aaa/"
+      syntax.foldingStartMarker.inspect.should == "/bbb/"
+    end
+
+    it "raises ParsingError on invalid regex" do
+      lambda{
+        Textpow::SyntaxNode.new('firstLineMatch' => '$?)(:[/\]')
+      }.should raise_error(Textpow::ParsingError)
+    end
+
+    it "stores itself as parent" do
+      node = Textpow::SyntaxNode.new({})
+      node.syntax.should == node
+    end
+
+    it "stores given parent" do
+      node = Textpow::SyntaxNode.new({}, 1)
+      node.syntax.should == 1
+    end
+
+    it "stores itself in global namespace under scopeName" do
+      node = Textpow::SyntaxNode.new("scopeName" => 'xxx')
+      node.syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({}).syntaxes['xxx'].should == node
+    end
+
+    it "stores itself in scoped global namespace under scopeName" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx'}, nil, 'foo')
+      node.syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({},nil,'foo').syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({}).syntaxes['xxx'].should == nil
+    end
+  end
+
+  describe "#parse" do
+    let(:node){ Textpow::SyntaxNode.load('lib/textpow/syntax/ruby.syntax') }
+
+    it "uses a RecordingProcessor by default" do
+      node.parse("111").stack.should == [
+        [:start_parsing, "source.ruby"],
+        [:new_line, "111"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 3],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+
+    it "can parse with a processor" do
+      processor = Textpow::RecordingProcessor.new
+      processor.stack << 'xxx'
+      node.parse("111", processor).stack.should == [
+        "xxx",
+        [:start_parsing, "source.ruby"],
+        [:new_line, "111"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 3],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+  end
+end

data/spec/textpow_spec.rb

@@ -1,34 +1,7 @@
 require 'spec_helper'
 
-describe Textpow::ScoreManager do
-  # TODO legacy test
-  it "calculates scores" do
-    sp = Textpow::ScoreManager.new
-    reference_scope = 'text.html.basic source.php.embedded.html string.quoted.double.php'
-
-    sp.score('source.php string', reference_scope).should_not == 0
-    sp.score('text.html source.php', reference_scope).should_not == 0
-    sp.score('string source.php', reference_scope).should == 0
-    sp.score('source.php text.html', reference_scope).should == 0
-
-    sp.score('text.html source.php - string', reference_scope).should == 0
-    sp.score('text.html source.php - ruby', reference_scope ).should_not == 0
-
-    sp.score('string', reference_scope).should > sp.score('source.php', reference_scope)
-    sp.score('string.quoted', reference_scope).should > sp.score('source.php', reference_scope)
-    sp.score('text source string', reference_scope).should > sp.score( 'source string', reference_scope)
-  end
-
-  describe "syntax" do
-    it "has syntax files" do
-      Dir["#{Textpow.syntax_path}/*.syntax"].should_not == []
-    end
-
-    Dir["#{Textpow.syntax_path}/*.syntax"].each do |syntax|
-      it "#{syntax} can parse" do
-        node = Textpow::SyntaxNode.load(syntax)
-        node.parse("xxx\n1 + 1\n### xxx")
-      end
-    end
+describe Textpow do
+  it "has a version" do
+    Textpow::Version =~ /^\d\.\d\.\d$/
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: textpow1x
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: 
   segments: 
   - 1
+  - 1
   - 0
-  - 0
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Dizan Vasquez
@@ -18,7 +18,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-02 00:00:00 +02:00
+date: 2011-10-06 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -49,16 +49,21 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - History.rdoc
+- MIT-LICENSE.txt
 - Manifest.txt
 - README.rdoc
 - Rakefile
 - bin/plist2syntax
 - bin/plist2yaml
+- examples/benchmark_js.rb
+- examples/jquery.js
 - lib/textpow.rb
 - lib/textpow/debug_processor.rb
+- lib/textpow/recording_processor.rb
 - lib/textpow/score_manager.rb
 - lib/textpow/syntax.rb
 - lib/textpow/syntax/actionscript.syntax
@@ -225,7 +230,11 @@ files:
 - lib/textpow/syntax/yaml.syntax
 - lib/textpow/syntax/yui_javascript.syntax
 - lib/textpow/version.rb
+- spec/fixtures/objeck.plist
 - spec/spec_helper.rb
+- spec/textpow/score_manager_spec.rb
+- spec/textpow/syntax_files_spec.rb
+- spec/textpow/syntax_spec.rb
 - spec/textpow_spec.rb
 has_rdoc: true
 homepage: http://github.com/grosser/textpow