textpow 0.10.1 → 1.3.0

data/lib/textpow/syntax/text.xml.xsl.syntax

@@ -0,0 +1,60 @@
+--- 
+name: XSL
+fileTypes: 
+- xsl
+- xslt
+scopeName: text.xml.xsl
+repository: 
+  singlequotedString: 
+    name: string.quoted.single.xml
+    endCaptures: 
+      "0": 
+        name: punctuation.definition.string.end.xml
+    begin: "'"
+    beginCaptures: 
+      "0": 
+        name: punctuation.definition.string.begin.xml
+    end: "'"
+  doublequotedString: 
+    name: string.quoted.double.xml
+    endCaptures: 
+      "0": 
+        name: punctuation.definition.string.end.xml
+    begin: "\""
+    beginCaptures: 
+      "0": 
+        name: punctuation.definition.string.begin.xml
+    end: "\""
+uuid: DB8033A1-6D8E-4D80-B8A2-8768AAC6125D
+foldingStartMarker: ^\s*(<[^!?%/](?!.+?(/>|</.+?>))|<[!%]--(?!.+?--%?>)|<%[!]?(?!.+?%>))
+patterns: 
+- name: meta.tag.xml.template
+  captures: 
+    "1": 
+      name: punctuation.definition.tag.xml
+    "2": 
+      name: entity.name.tag.namespace.xml
+    "3": 
+      name: entity.name.tag.xml
+    "4": 
+      name: punctuation.separator.namespace.xml
+    "5": 
+      name: entity.name.tag.localname.xml
+  begin: (<)(xsl)((:))(template)
+  end: (>)
+  patterns: 
+  - captures: 
+      "1": 
+        name: entity.other.attribute-name.namespace.xml
+      "2": 
+        name: entity.other.attribute-name.xml
+      "3": 
+        name: punctuation.separator.namespace.xml
+      "4": 
+        name: entity.other.attribute-name.localname.xml
+    match: " (?:([-_a-zA-Z0-9]+)((:)))?([a-zA-Z-]+)"
+  - include: "#doublequotedString"
+  - include: "#singlequotedString"
+- include: text.xml
+foldingStopMarker: ^\s*(</[^>]+>|[/%]>|-->)\s*$
+keyEquivalent: ^~X

data/lib/textpow/version.rb

@@ -0,0 +1,3 @@
+module Textpow
+  VERSION = Version = "1.3.0"
+end

data/spec/fixtures/objeck.plist

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>BBEditDocumentType</key>
+	<string>CodelessLanguageModule</string>
+	<key>BBLMColorsSyntax</key>
+	<true/>
+	<key>BBLMIsCaseSensitive</key>
+	<true/>
+	<key>BBLMKeywordList</key>
+	<array>
+		<string>and</string>
+		<string>or</string>
+		<string>xor</string>
+		<string>virtual</string>
+		<string>if</string>
+		<string>else</string>
+		<string>do</string>
+		<string>while</string>
+		<string>use</string>
+		<string>bundle</string>
+		<string>native</string>
+		<string>static</string>
+		<string>public</string>
+		<string>private</string>
+		<string>class</string>
+		<string>interface</string>
+		<string>select</string>
+		<string>other</string>
+		<string>enum</string>
+		<string>for</string>
+		<string>each</string>
+		<string>label</string>
+		<string>return</string>
+		<string>Byte</string>
+		<string>Int</string>
+		<string>Parent</string>
+		<string>from</string>
+		<string>Float</string>
+		<string>Char</string>
+		<string>Bool</string>
+		<string>String</string>
+		<string>Nil</string>
+		<string>true</string>
+		<string>false</string>
+		<string>function</string>
+		<string>method</string>
+	</array>
+	<key>BBLMLanguageCode</key>
+	<string>LSL</string>
+	<key>BBLMLanguageDisplayName</key>
+	<string>Objeck</string>
+	<key>BBLMScansFunctions</key>
+	<true/>
+	<key>BBLMSuffixMap</key>
+	<array>
+		<dict>
+			<key>BBLMLanguageSuffix</key>
+			<string>.obs</string>
+		</dict>
+	</array>
+	<key>Language Features</key>
+	<dict>
+		<key>Close Block Comments</key>
+		<string>~#</string>
+		<key>Close Parameter Lists</key>
+		<string>)</string>
+		<key>Close Statement Blocks</key>
+		<string>}</string>
+		<key>Close Strings 1</key>
+		<string>&quot;</string>
+		<key>Close Strings 2</key>
+		<string>&apos;</string>
+		<key>End-of-line Ends Strings 1</key>
+		<false/>
+		<key>End-of-line Ends Strings 2</key>
+		<false/>
+		<key>Escape Char in Strings 1</key>
+		<string>\</string>
+		<key>Escape Char in Strings 2</key>
+		<string>\</string>
+		<key>Identifier and Keyword Characters</key>
+		<string>@0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz</string>
+		<key>Open Block Comments</key>
+		<string>#~</string>
+		<key>Open Line Comments</key>
+		<string>#</string>
+		<key>Open Parameter Lists</key>
+		<string>(</string>
+		<key>Open Statement Blocks</key>
+		<string>{</string>
+		<key>Open Strings 1</key>
+		<string>&quot;</string>
+		<key>Open Strings 2</key>
+		<string>&apos;</string>
+		<key>Prefix for Functions</key>
+		<string></string>
+		<key>Prefix for Procedures</key>
+		<string></string>
+		<key>Terminator for Prototypes 1</key>
+		<string></string>
+		<key>Terminator for Prototypes 2</key>
+		<string></string>
+	</dict>
+</dict>
+</plist>

data/spec/fixtures/utf8.txt

@@ -0,0 +1 @@
+ü

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift 'lib'
+require 'textpow'

data/spec/textpow/score_manager_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Textpow::ScoreManager do
+  it "calculates scores" do
+    sp = Textpow::ScoreManager.new
+    reference_scope = 'text.html.basic source.php.embedded.html string.quoted.double.php'
+
+    sp.score('source.php string', reference_scope).should_not == 0
+    sp.score('text.html source.php', reference_scope).should_not == 0
+    sp.score('string source.php', reference_scope).should == 0
+    sp.score('source.php text.html', reference_scope).should == 0
+
+    sp.score('text.html source.php - string', reference_scope).should == 0
+    sp.score('text.html source.php - ruby', reference_scope ).should_not == 0
+
+    sp.score('string', reference_scope).should > sp.score('source.php', reference_scope)
+    sp.score('string.quoted', reference_scope).should > sp.score('source.php', reference_scope)
+    sp.score('text source string', reference_scope).should > sp.score( 'source string', reference_scope)
+  end
+end

data/spec/textpow/syntax_files_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe "syntax files" do
+  before do
+    STDERR.stub!(:puts)
+  end
+
+  it "has syntax files" do
+    Dir["#{Textpow.syntax_path}/*.syntax"].should_not == []
+  end
+
+  Dir["#{Textpow.syntax_path}/*.syntax"].each do |syntax|
+    it "can parse with #{syntax}" do
+      Textpow.syntax(syntax).parse("xxx\n1 + 1\n### xxx")
+    end
+
+    it "can parse UTF-8 with #{syntax}" do
+      Textpow.syntax(syntax).parse(File.read("spec/fixtures/utf8.txt"))
+    end
+  end
+
+  xit "parses markdown" do
+    node = Textpow.syntax("lib/textpow/syntax/broken/markdown.syntax")
+    node.parse("### xxx\nabc\n    xxx\nyyy\n - abc\n - ac").stack.should_not == []
+  end
+end

data/spec/textpow/syntax_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+
+describe Textpow::SyntaxNode do
+  before do
+    Textpow::SyntaxNode.send(:class_variable_set, "@@syntaxes", {})
+  end
+
+  describe "#load" do
+    it "can load from xml .plist" do
+      Textpow::SyntaxNode.load('spec/fixtures/objeck.plist').should_not == nil
+    end
+
+    it "can load from yaml .syntax" do
+      Textpow::SyntaxNode.load('lib/textpow/syntax/source.ruby.syntax').should_not == nil
+    end
+
+    it "cannot load missing file" do
+      lambda{
+        Textpow::SyntaxNode.load('xxx.syntax')
+      }.should raise_error
+    end
+
+    it "cannot load missing plist file" do
+      lambda{
+        Textpow::SyntaxNode.load('xxx.plist')
+      }.should raise_error
+    end
+  end
+
+  describe "#new" do
+    it "loads strings from given hash" do
+      syntax = Textpow::SyntaxNode.new('content' => 'CONTENT', 'name' => 'NAME')
+      syntax.content.should == 'CONTENT'
+      syntax.name.should == 'NAME'
+    end
+
+    it "loads regex from given hash" do
+      syntax = Textpow::SyntaxNode.new('firstLineMatch' => 'aaa', 'foldingStartMarker' => 'bbb')
+      syntax.firstLineMatch.inspect.should == "/aaa/"
+      syntax.foldingStartMarker.inspect.should == "/bbb/"
+    end
+
+    it "raises ParsingError on invalid regex" do
+      lambda{
+        Textpow::SyntaxNode.new('firstLineMatch' => '$?)(:[/\]')
+      }.should raise_error(Textpow::ParsingError)
+    end
+
+    it "stores itself as parent" do
+      node = Textpow::SyntaxNode.new({})
+      node.syntax.should == node
+    end
+
+    it "stores given parent" do
+      node = Textpow::SyntaxNode.new({}, :syntax => 1)
+      node.syntax.should == 1
+    end
+
+    it "stores itself in global namespace under scopeName" do
+      node = Textpow::SyntaxNode.new("scopeName" => 'xxx')
+      node.syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({}).syntaxes['xxx'].should == node
+    end
+
+    it "stores itself in scoped global namespace under scopeName" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx'}, :name_space => 'foo')
+      node.syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({},:name_space => 'foo').syntaxes['xxx'].should == node
+      Textpow::SyntaxNode.new({}).syntaxes['xxx'].should == nil
+    end
+  end
+
+  describe "#parse" do
+    let(:node){ Textpow.syntax('ruby') }
+
+    it "uses a RecordingProcessor by default" do
+      node.parse("111").stack.should == [
+        [:start_parsing, "source.ruby"],
+        [:new_line, "111"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 3],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+
+    it "can parse with a processor" do
+      processor = Textpow::RecordingProcessor.new
+      processor.stack << 'xxx'
+      node.parse("111", processor).stack.should == [
+        "xxx",
+        [:start_parsing, "source.ruby"],
+        [:new_line, "111"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 3],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+
+    it "can parse multiline syntax via stack" do
+      node.parse("=begin\n111\n=end").stack.should == [
+        [:start_parsing, "source.ruby"],
+        [:new_line, "=begin\n"],
+        [:open_tag, "comment.block.documentation.ruby", 0],
+        [:open_tag, "punctuation.definition.comment.ruby", 0],
+        [:close_tag, "punctuation.definition.comment.ruby", 6],
+        [:new_line, "111\n"],
+        [:new_line, "=end"],
+        [:open_tag, "punctuation.definition.comment.ruby", 0],
+        [:close_tag, "punctuation.definition.comment.ruby", 4],
+        [:close_tag, "comment.block.documentation.ruby", 4],
+        [:end_parsing, "source.ruby"]
+      ]
+    end
+
+    it "loads included syntax files" do
+      pending
+      node = Textpow.syntax('html_rails')
+      node.parse("<br/>")
+    end
+  end
+
+  describe "#match_first_son" do
+    it "returns nil when patterns are empty" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx'})
+      node.send(:match_first_son, "xxx", 0).should == nil
+    end
+
+    it "matches a pattern" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "xxx", "name" => "foo"}
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyy", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'foo'
+    end
+
+    it "matches the first pattern" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "yyy", "name" => "bar"},
+        {"match" => "xxx", "name" => "foo"},
+        {"match" => "zzz", "name" => "baz"},
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyyyzzz", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'foo'
+    end
+
+    it "matches the first pattern for equal positions" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [
+        {"match" => "xxx", "name" => "bar"},
+        {"match" => "xxxyyy", "name" => "foo"},
+        {"match" => "xxxyy", "name" => "baz"},
+      ]})
+      pattern, match = node.send(:match_first_son, "xxxyyyzzz", 0)
+      match.to_s.should == 'xxx'
+      pattern.name.should == 'bar'
+    end
+  end
+
+  describe "proxying" do
+    it "cannot proxy by scopeName if syntax is missing" do
+      STDERR.should_not_receive(:puts)
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'foo'}]})
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy by scopeName if syntax is available in repo" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'source.ruby'}]})
+      node.parse('1').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "1"],
+        [:open_tag, "constant.numeric.ruby", 0],
+        [:close_tag, "constant.numeric.ruby", 1],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy by scopeName if syntax is loaded (even later)" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx', 'patterns' => [{'include' => 'foo'}]})
+      Textpow::SyntaxNode.new({"scopeName" => 'foo', 'patterns' => [{'name' => 'foo.1', 'match' => 'bar'}]})
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy to a pattern defined in an repository" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx',
+        'repository' => {'foo' => {'name' => 'foo.1', 'match' => 'bar'}},
+        'patterns' => [{'include' => '#foo'}]
+      })
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
+
+    it "can proxy to a pattern nested included in an repository" do
+      node = Textpow::SyntaxNode.new({"scopeName" => 'xxx',
+        'repository' => {
+          'baz' => {'patterns' => [{'include' => '#foo'}]},
+          'foo' => {'name' => 'foo.1', 'match' => 'bar'}
+        },
+        'patterns' => [{'include' => '#baz'}]
+      })
+      node.parse('bar').stack.should == [
+        [:start_parsing, "xxx"],
+        [:new_line, "bar"],
+        [:open_tag, "foo.1", 0],
+        [:close_tag, "foo.1", 3],
+        [:end_parsing, "xxx"]
+      ]
+    end
+  end
+end