textpow19 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.rdoc +17 -0
- data/Manifest.txt +13 -0
- data/README.rdoc +210 -0
- data/Rakefile +30 -0
- data/bin/plist2syntax +11 -0
- data/bin/plist2yaml +8 -0
- data/lib/textpow.rb +10 -0
- data/lib/textpow/debug_processor.rb +36 -0
- data/lib/textpow/score_manager.rb +65 -0
- data/lib/textpow/syntax.rb +287 -0
- data/lib/textpow/version.rb +3 -0
- data/test/test_textpow.rb +25 -0
- metadata +128 -0
data/History.rdoc
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
== 0.11.0 / 2009-06-12
|
2
|
+
* Updated to 1.9 compatibility. Does not use onigumura gem anymore
|
3
|
+
|
4
|
+
== 0.10.1 / 2008-02-20
|
5
|
+
* FIX: Endless loop in method_missing when @proxy_variable is not present.
|
6
|
+
|
7
|
+
== 0.10.0 / 2007-06-15
|
8
|
+
* CHANGE: Now the scopeName for the entire syntax is passed to start_parsing and end_parsing. This means that programs using older version should modify their processors accordingly.
|
9
|
+
|
10
|
+
== 0.9.1 / 2007-06-14
|
11
|
+
* FIX: Score manager now handles multiple subtractions in scope definitions.
|
12
|
+
|
13
|
+
== 0.9.0 / 2007-03-19
|
14
|
+
|
15
|
+
* 1 major enhancement
|
16
|
+
* Birthday!
|
17
|
+
|
data/Manifest.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
bin/plist2yaml
|
2
|
+
bin/plist2syntax
|
3
|
+
lib/textpow.rb
|
4
|
+
lib/textpow
|
5
|
+
lib/textpow/syntax.rb
|
6
|
+
lib/textpow/version.rb
|
7
|
+
lib/textpow/score_manager.rb
|
8
|
+
lib/textpow/debug_processor.rb
|
9
|
+
test/test_textpow.rb
|
10
|
+
History.rdoc
|
11
|
+
Rakefile
|
12
|
+
Manifest.txt
|
13
|
+
README.rdoc
|
data/README.rdoc
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
= Textpow
|
2
|
+
|
3
|
+
* http://github.com/cehoffman/textpow
|
4
|
+
* http://github.com/spox/textpow
|
5
|
+
* http://textpow.rubyforge.org
|
6
|
+
|
7
|
+
== DESCRIPTION:
|
8
|
+
|
9
|
+
A library for parsing {TextMate}[http://macromates.com/] bundles.
|
10
|
+
|
11
|
+
== SYNOPSIS:
|
12
|
+
|
13
|
+
|
14
|
+
Parsing a file using Textpow is as easy as 1-2-3!
|
15
|
+
|
16
|
+
1. Load the Syntax File:
|
17
|
+
|
18
|
+
require 'textpow'
|
19
|
+
syntax = Textpow::SyntaxNode.load("ruby.tmSyntax")
|
20
|
+
|
21
|
+
2. Initialize a processor:
|
22
|
+
|
23
|
+
processor = Textpow::DebugProcessor.new
|
24
|
+
|
25
|
+
3. Parse some text:
|
26
|
+
|
27
|
+
syntax.parse( text, processor )
|
28
|
+
|
29
|
+
|
30
|
+
=== INDEPTH:
|
31
|
+
|
32
|
+
At the heart of syntax parsing are ..., well, syntax files. Lets see for instance
|
33
|
+
the example syntax that appears in textmate's
|
34
|
+
{documentation}[http://macromates.com/textmate/manual/language_grammars#language_grammars]:
|
35
|
+
|
36
|
+
|
37
|
+
{ scopeName = 'source.untitled';
|
38
|
+
fileTypes = ( txt );
|
39
|
+
foldingStartMarker = '\{\s*$';
|
40
|
+
foldingStopMarker = '^\s*\}';
|
41
|
+
patterns = (
|
42
|
+
{ name = 'keyword.control.untitled';
|
43
|
+
match = '\b(if|while|for|return)\b';
|
44
|
+
},
|
45
|
+
{ name = 'string.quoted.double.untitled';
|
46
|
+
begin = '"';
|
47
|
+
end = '"';
|
48
|
+
patterns = (
|
49
|
+
{ name = 'constant.character.escape.untitled';
|
50
|
+
match = '\\.';
|
51
|
+
}
|
52
|
+
);
|
53
|
+
},
|
54
|
+
);
|
55
|
+
}
|
56
|
+
|
57
|
+
But Textpow is not able to parse text pfiles. However, in practice this is not a problem,
|
58
|
+
since it is possible to convert both text and binary pfiles to an XML format. Indeed, all
|
59
|
+
the syntaxes in the Textmate syntax {repository}[http://macromates.com/svn/Bundles/trunk/Bundles/]
|
60
|
+
are in XML format:
|
61
|
+
|
62
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
63
|
+
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
64
|
+
<plist version="1.0">
|
65
|
+
<dict>
|
66
|
+
<key>scopeName</key>
|
67
|
+
<string>source.untitled</string>
|
68
|
+
<key>fileTypes</key>
|
69
|
+
<array>
|
70
|
+
<string>txt</string>
|
71
|
+
</array>
|
72
|
+
<key>foldingStartMarker</key>
|
73
|
+
<string>\{\s*$</string>
|
74
|
+
<key>foldingStopMarker</key>
|
75
|
+
<string>^\s*\}</string>
|
76
|
+
<key>patterns</key>
|
77
|
+
<array>
|
78
|
+
<dict>
|
79
|
+
<key>name</key>
|
80
|
+
<string>keyword.control.untitled</string>
|
81
|
+
<key>match</key>
|
82
|
+
<string>\b(if|while|for|return)\b</string>
|
83
|
+
</dict>
|
84
|
+
<dict>
|
85
|
+
<key>name</key>
|
86
|
+
<string>string.quoted.double.untitled</string>
|
87
|
+
<key>begin</key>
|
88
|
+
<string>"</string>
|
89
|
+
<key>end</key>
|
90
|
+
<string>"</string>
|
91
|
+
<key>patterns</key>
|
92
|
+
<array>
|
93
|
+
<dict>
|
94
|
+
<key>name</key>
|
95
|
+
<string>constant.character.escape.untitled</string>
|
96
|
+
<key>match</key>
|
97
|
+
<string>\\.</string>
|
98
|
+
</dict>
|
99
|
+
</array>
|
100
|
+
</dict>
|
101
|
+
</array>
|
102
|
+
</dict>
|
103
|
+
|
104
|
+
Of course, most people find XML both ugly and cumbersome. Fortunately, it is
|
105
|
+
also possible to store syntax files in YAML format, which is much easier to
|
106
|
+
read:
|
107
|
+
|
108
|
+
---
|
109
|
+
fileTypes:
|
110
|
+
- txt
|
111
|
+
scopeName: source.untitled
|
112
|
+
foldingStartMarker: \{\s*$
|
113
|
+
foldingStopMarker: ^\s*\}
|
114
|
+
patterns:
|
115
|
+
- name: keyword.control.untitled
|
116
|
+
match: \b(if|while|for|return)\b
|
117
|
+
- name: string.quoted.double.untitled
|
118
|
+
begin: '"'
|
119
|
+
end: '"'
|
120
|
+
patterns:
|
121
|
+
- name: constant.character.escape.untitled
|
122
|
+
match: \\.
|
123
|
+
|
124
|
+
==== Processors
|
125
|
+
|
126
|
+
Until now we have talked about the parsing process without explaining what
|
127
|
+
it is exactly. Basically, parsing consists in reading text from a string or
|
128
|
+
file and applying tags to parts of the text according to what has been
|
129
|
+
specified in the syntax file.
|
130
|
+
|
131
|
+
In textpow, the process takes place line by line, from the beginning to the
|
132
|
+
end and from left to right for every line. As the text is parsed, events are
|
133
|
+
sent to a processor object when a tag is open or closed and so on.
|
134
|
+
A processor is any object which implements one or more of the following
|
135
|
+
methods:
|
136
|
+
|
137
|
+
class Processor
|
138
|
+
def open_tag name, position
|
139
|
+
end
|
140
|
+
|
141
|
+
def close_tag name, position
|
142
|
+
end
|
143
|
+
|
144
|
+
def new_line line
|
145
|
+
end
|
146
|
+
|
147
|
+
def start_parsing name
|
148
|
+
end
|
149
|
+
|
150
|
+
def end_parsing name
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
* <tt>open_tag</tt>. Is called when a new tag is opened, it receives the tag's name and
|
155
|
+
its position (relative to the current line).
|
156
|
+
* <tt>close_tag</tt>. The same that <tt>open_tag</tt>, but it is called when a tag is closed.
|
157
|
+
* <tt>new_line</tt>. Is called every time that a new line is processed, it receives the
|
158
|
+
line's contents.
|
159
|
+
* <tt>start_parsing</tt>. Is called once at the beginning of the parsing process. It
|
160
|
+
receives the scope name for the syntax being used.
|
161
|
+
* <tt>end_parsing</tt>. Is called once after all the input text has been parsed. It
|
162
|
+
receives the scope name for the syntax being used.
|
163
|
+
|
164
|
+
Textpow ensures that the methods are called in parsing order, thus,
|
165
|
+
for example, if there are two subsequent calls to <tt>open_tag</tt>, the first
|
166
|
+
having <tt>name="text.string", position=10</tt> and the second having
|
167
|
+
<tt>name="markup.string", position=10</tt>, it should be understood that the
|
168
|
+
<tt>"markup.string"</tt> tag is inside the <tt>"text.string"</tt> tag.
|
169
|
+
|
170
|
+
== REQUIREMENTS:
|
171
|
+
|
172
|
+
* Ruby 1.9
|
173
|
+
* plist > 3.0
|
174
|
+
|
175
|
+
== INSTALL:
|
176
|
+
|
177
|
+
gem install textpow19
|
178
|
+
|
179
|
+
== CREDITS:
|
180
|
+
|
181
|
+
* {Chris Hoffman}[http://github.com/cehoffman]
|
182
|
+
* {Spox}[http://github.com/spox]
|
183
|
+
* Dizan Vasque
|
184
|
+
|
185
|
+
== LICENSE:
|
186
|
+
|
187
|
+
(The MIT License)
|
188
|
+
|
189
|
+
* Copyright (c) 2010 Chris Hoffman
|
190
|
+
* Copyright (c) 2009 Spox
|
191
|
+
* Copyright (c) 2007-2008 Dizan Vasquez
|
192
|
+
|
193
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
194
|
+
a copy of this software and associated documentation files (the
|
195
|
+
'Software'), to deal in the Software without restriction, including
|
196
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
197
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
198
|
+
permit persons to whom the Software is furnished to do so, subject to
|
199
|
+
the following conditions:
|
200
|
+
|
201
|
+
The above copyright notice and this permission notice shall be
|
202
|
+
included in all copies or substantial portions of the Software.
|
203
|
+
|
204
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
205
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
206
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
207
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
208
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
209
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
210
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'hoe'
|
5
|
+
require File.expand_path("../lib/textpow/version", __FILE__)
|
6
|
+
|
7
|
+
Hoe.plugin :gemcutter
|
8
|
+
|
9
|
+
Hoe.spec 'textpow19' do
|
10
|
+
developer("Chris Hoffman", "cehoffman@gmail.com")
|
11
|
+
developer("Spox", "spox@modspox.com")
|
12
|
+
developer("Dizan Vasquez", "dichodaemon@gmail.com")
|
13
|
+
|
14
|
+
self.version = Textpow::Version
|
15
|
+
self.extra_deps << ['plist', '>= 3.0.1']
|
16
|
+
self.readme_file = "README.rdoc"
|
17
|
+
self.history_file = "History.rdoc"
|
18
|
+
spec_extras[:required_ruby_version] = ">= 1.9.0"
|
19
|
+
end
|
20
|
+
|
21
|
+
task :gemspec do
|
22
|
+
sh %{rake debug_gem | grep -v "(in " > `basename \\\`pwd\\\``.gemspec}
|
23
|
+
end
|
24
|
+
|
25
|
+
rescue LoadError => e
|
26
|
+
desc 'Run the test suite.'
|
27
|
+
task :test do
|
28
|
+
system "ruby -Ibin:lib:test test_#{rubyforge_name}.rb"
|
29
|
+
end
|
30
|
+
end
|
data/bin/plist2syntax
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'plist'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
result = Plist::parse_xml( ARGV[0] )
|
8
|
+
standard_name = File.basename( ARGV[0] ).downcase.gsub(/\s+/, '_').gsub(/\.(plist|tm[Ll]anguage)/, '').gsub(/\(|\)|:/, '').gsub(/_+/, '_')
|
9
|
+
puts standard_name
|
10
|
+
|
11
|
+
File.open( "#{standard_name}.yaml", "w" ) {|f| YAML.dump( result, f ) }
|
data/bin/plist2yaml
ADDED
data/lib/textpow.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
module Textpow
|
2
|
+
class DebugProcessor
|
3
|
+
def initialize
|
4
|
+
@line_number = 0
|
5
|
+
@printable_line = ""
|
6
|
+
end
|
7
|
+
|
8
|
+
def pprint line, string, position = 0
|
9
|
+
line.replace line.ljust( position + string.size, " ")
|
10
|
+
line[position,string.size] = string
|
11
|
+
line
|
12
|
+
end
|
13
|
+
|
14
|
+
def open_tag name, position
|
15
|
+
STDERR.puts pprint( "", "{#{name}", position + @line_marks.size)
|
16
|
+
end
|
17
|
+
|
18
|
+
def close_tag name, position
|
19
|
+
STDERR.puts pprint( "", "}#{name}", position + @line_marks.size)
|
20
|
+
end
|
21
|
+
|
22
|
+
def new_line line
|
23
|
+
@line_number += 1
|
24
|
+
@line_marks = "[#{@line_number.to_s.rjust( 4, '0' )}] "
|
25
|
+
STDERR.puts "#{@line_marks}#{line}"
|
26
|
+
end
|
27
|
+
|
28
|
+
def start_parsing name
|
29
|
+
STDERR.puts "{#{name}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def end_parsing name
|
33
|
+
STDERR.puts "}#{name}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Textpow
|
2
|
+
class ScoreManager
|
3
|
+
POINT_DEPTH = 4
|
4
|
+
NESTING_DEPTH = 40
|
5
|
+
START_VALUE = 2 ** ( POINT_DEPTH * NESTING_DEPTH )
|
6
|
+
BASE = 2 ** POINT_DEPTH
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@scores = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def score search_scope, reference_scope
|
13
|
+
max = 0
|
14
|
+
search_scope.split( ',' ).each do |scope|
|
15
|
+
arrays = scope.split(/\B-/)
|
16
|
+
if arrays.size == 1
|
17
|
+
max = [max, score_term( arrays[0], reference_scope )].max
|
18
|
+
elsif arrays.size > 1
|
19
|
+
excluded = false
|
20
|
+
arrays[1..-1].each do |a|
|
21
|
+
if score_term( arrays[1], reference_scope ) > 0
|
22
|
+
excluded = true
|
23
|
+
break
|
24
|
+
end
|
25
|
+
end
|
26
|
+
max = [max, score_term( arrays[0], reference_scope )].max unless excluded
|
27
|
+
else
|
28
|
+
raise ParsingError, "Error in scope string: '#{search_scope}' #{arrays.size} is not a valid number of operands" if arrays.size < 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
max
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def score_term search_scope, reference_scope
|
37
|
+
unless @scores[reference_scope] && @scores[reference_scope][search_scope]
|
38
|
+
@scores[reference_scope] ||= {}
|
39
|
+
@scores[reference_scope][search_scope] = score_array( search_scope.split(' '), reference_scope.split( ' ' ) )
|
40
|
+
end
|
41
|
+
@scores[reference_scope][search_scope]
|
42
|
+
end
|
43
|
+
|
44
|
+
def score_array search_array, reference_array
|
45
|
+
pending = search_array
|
46
|
+
current = reference_array.last
|
47
|
+
reg = Regexp.new( "^#{Regexp.escape( pending.last )}" )
|
48
|
+
multiplier = START_VALUE
|
49
|
+
result = 0
|
50
|
+
while pending.size > 0 && current
|
51
|
+
if reg =~ current
|
52
|
+
point_score = (2**POINT_DEPTH) - current.count( '.' ) + Regexp.last_match[0].count( '.' )
|
53
|
+
result += point_score * multiplier
|
54
|
+
pending.pop
|
55
|
+
reg = Regexp.new( "^#{Regexp.escape( pending.last )}" ) if pending.size > 0
|
56
|
+
end
|
57
|
+
multiplier = multiplier / BASE
|
58
|
+
reference_array.pop
|
59
|
+
current = reference_array.last
|
60
|
+
end
|
61
|
+
result = 0 if pending.size > 0
|
62
|
+
result
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,287 @@
|
|
1
|
+
require 'plist'
|
2
|
+
|
3
|
+
module Textpow
|
4
|
+
|
5
|
+
class SyntaxProxy
|
6
|
+
def initialize hash, syntax
|
7
|
+
@syntax = syntax
|
8
|
+
@proxy = hash["include"]
|
9
|
+
end
|
10
|
+
|
11
|
+
def method_missing method, *args, &block
|
12
|
+
if @proxy
|
13
|
+
@proxy_value = proxy unless @proxy_value
|
14
|
+
if @proxy_value
|
15
|
+
@proxy_value.send(method, *args, &block)
|
16
|
+
else
|
17
|
+
STDERR.puts "Failed proxying #{@proxy}.#{method}(#{args.join(', ')})"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def proxy
|
23
|
+
case @proxy
|
24
|
+
when /^#/
|
25
|
+
if @syntax.repository && @syntax.repository[@proxy[1..-1]]
|
26
|
+
#puts "Repository"
|
27
|
+
#@table["syntax"].repository.each_key{|k| puts k}
|
28
|
+
return @syntax.repository[@proxy[1..-1]]
|
29
|
+
end
|
30
|
+
when "$self"
|
31
|
+
return @syntax
|
32
|
+
when "$base"
|
33
|
+
return @syntax
|
34
|
+
else
|
35
|
+
return @syntax.syntaxes[@proxy]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class SyntaxNode
|
41
|
+
#OPTIONS = {:options => Oniguruma::OPTION_CAPTURE_GROUP}
|
42
|
+
|
43
|
+
@@syntaxes = {}
|
44
|
+
|
45
|
+
attr_accessor :syntax
|
46
|
+
attr_accessor :firstLineMatch
|
47
|
+
attr_accessor :foldingStartMarker
|
48
|
+
attr_accessor :foldingStopMarker
|
49
|
+
attr_accessor :match
|
50
|
+
attr_accessor :begin
|
51
|
+
attr_accessor :content
|
52
|
+
attr_accessor :fileTypes
|
53
|
+
attr_accessor :name
|
54
|
+
attr_accessor :contentName
|
55
|
+
attr_accessor :end
|
56
|
+
attr_accessor :scopeName
|
57
|
+
attr_accessor :keyEquivalent
|
58
|
+
attr_accessor :captures
|
59
|
+
attr_accessor :beginCaptures
|
60
|
+
attr_accessor :endCaptures
|
61
|
+
attr_accessor :repository
|
62
|
+
attr_accessor :patterns
|
63
|
+
|
64
|
+
def self.load filename, name_space = :default
|
65
|
+
table = nil
|
66
|
+
case filename
|
67
|
+
when /(\.tmSyntax|\.plist)$/
|
68
|
+
table = Plist::parse_xml( filename )
|
69
|
+
else
|
70
|
+
File.open( filename ) do |f|
|
71
|
+
table = YAML.load( f )
|
72
|
+
end
|
73
|
+
end
|
74
|
+
if table
|
75
|
+
SyntaxNode.new( table, nil, name_space )
|
76
|
+
else
|
77
|
+
nil
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def initialize hash, syntax = nil, name_space = :default
|
82
|
+
@name_space = name_space
|
83
|
+
@@syntaxes[@name_space] ||= {}
|
84
|
+
@@syntaxes[@name_space][hash["scopeName"]] = self if hash["scopeName"]
|
85
|
+
@syntax = syntax || self
|
86
|
+
hash.each do |key, value|
|
87
|
+
case key
|
88
|
+
when "firstLineMatch", "foldingStartMarker", "foldingStopMarker", "match", "begin"
|
89
|
+
begin
|
90
|
+
value.force_encoding("ASCII-8BIT")
|
91
|
+
instance_variable_set( "@#{key}", Regexp.new( value ) )
|
92
|
+
rescue ArgumentError => e
|
93
|
+
raise ParsingError, "Parsing error in #{value}: #{e.to_s}"
|
94
|
+
end
|
95
|
+
when "content", "fileTypes", "name", "contentName", "end", "scopeName", "keyEquivalent"
|
96
|
+
instance_variable_set( "@#{key}", value )
|
97
|
+
when "captures", "beginCaptures", "endCaptures"
|
98
|
+
instance_variable_set( "@#{key}", value.sort )
|
99
|
+
when "repository"
|
100
|
+
parse_repository value
|
101
|
+
when "patterns"
|
102
|
+
create_children value
|
103
|
+
else
|
104
|
+
STDERR.puts "Ignoring: #{key} => #{value.gsub("\n", "\n>>")}" if $DEBUG
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
def syntaxes
|
111
|
+
@@syntaxes[@name_space]
|
112
|
+
end
|
113
|
+
|
114
|
+
def parse( string, processor = nil )
|
115
|
+
processor.start_parsing self.scopeName if processor
|
116
|
+
stack = [[self, nil]]
|
117
|
+
string.each_line do |line|
|
118
|
+
parse_line stack, line, processor
|
119
|
+
end
|
120
|
+
processor.end_parsing self.scopeName if processor
|
121
|
+
processor
|
122
|
+
end
|
123
|
+
|
124
|
+
protected
|
125
|
+
|
126
|
+
def parse_repository repository
|
127
|
+
@repository = {}
|
128
|
+
repository.each do |key, value|
|
129
|
+
if value["include"]
|
130
|
+
@repository[key] = SyntaxProxy.new( value, self.syntax )
|
131
|
+
else
|
132
|
+
@repository[key] = SyntaxNode.new( value, self.syntax, @name_space )
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def create_children patterns
|
138
|
+
@patterns = []
|
139
|
+
patterns.each do |p|
|
140
|
+
if p["include"]
|
141
|
+
@patterns << SyntaxProxy.new( p, self.syntax )
|
142
|
+
else
|
143
|
+
@patterns << SyntaxNode.new( p, self.syntax, @name_space )
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def parse_captures name, pattern, match, processor
|
149
|
+
captures = pattern.match_captures( name, match )
|
150
|
+
captures.reject! { |group, range, name| ! range.first || range.first == range.last }
|
151
|
+
starts = []
|
152
|
+
ends = []
|
153
|
+
captures.each do |group, range, name|
|
154
|
+
starts << [range.first, group, name]
|
155
|
+
ends << [range.last, -group, name]
|
156
|
+
end
|
157
|
+
|
158
|
+
# STDERR.puts '-' * 100
|
159
|
+
# starts.sort!.reverse!.each{|c| STDERR.puts c.join(', ')}
|
160
|
+
# STDERR.puts
|
161
|
+
# ends.sort!.reverse!.each{|c| STDERR.puts c.join(', ')}
|
162
|
+
starts.sort!.reverse!
|
163
|
+
ends.sort!.reverse!
|
164
|
+
|
165
|
+
while ! starts.empty? || ! ends.empty?
|
166
|
+
if starts.empty?
|
167
|
+
pos, key, name = ends.pop
|
168
|
+
processor.close_tag name, pos
|
169
|
+
elsif ends.empty?
|
170
|
+
pos, key, name = starts.pop
|
171
|
+
processor.open_tag name, pos
|
172
|
+
elsif ends.last[1].abs < starts.last[1]
|
173
|
+
pos, key, name = ends.pop
|
174
|
+
processor.close_tag name, pos
|
175
|
+
else
|
176
|
+
pos, key, name = starts.pop
|
177
|
+
processor.open_tag name, pos
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def match_captures name, match
|
183
|
+
matches = []
|
184
|
+
captures = instance_variable_get "@#{name}"
|
185
|
+
if captures
|
186
|
+
captures.each do |key, value|
|
187
|
+
if key =~ /^\d*$/
|
188
|
+
matches << [key.to_i, match.offset( key.to_i ), value["name"]] if key.to_i < match.size
|
189
|
+
else
|
190
|
+
matches << [match.to_index( key.to_sym ), match.offset( key.to_sym), value["name"]] if match.to_index( key.to_sym )
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
matches
|
195
|
+
end
|
196
|
+
|
197
|
+
def match_first string, position
|
198
|
+
if self.match
|
199
|
+
if match = self.match.match( string, position )
|
200
|
+
return [self, match]
|
201
|
+
end
|
202
|
+
elsif self.begin
|
203
|
+
if match = self.begin.match( string, position )
|
204
|
+
return [self, match]
|
205
|
+
end
|
206
|
+
elsif self.end
|
207
|
+
else
|
208
|
+
return match_first_son( string, position )
|
209
|
+
end
|
210
|
+
nil
|
211
|
+
end
|
212
|
+
|
213
|
+
def match_end string, match, position
|
214
|
+
regstring = self.end.clone
|
215
|
+
regstring.gsub!( /\\([1-9])/ ) { |s| match[$1.to_i] }
|
216
|
+
regstring.gsub!( /\\k<(.*?)>/ ) { |s| match[$1.to_sym] }
|
217
|
+
Regexp.new( regstring ).match( string, position )
|
218
|
+
end
|
219
|
+
|
220
|
+
def match_first_son string, position
|
221
|
+
match = nil
|
222
|
+
if self.patterns
|
223
|
+
self.patterns.each do |p|
|
224
|
+
tmatch = p.match_first string, position
|
225
|
+
if tmatch
|
226
|
+
if ! match || match[1].offset(0).first > tmatch[1].offset(0).first
|
227
|
+
match = tmatch
|
228
|
+
end
|
229
|
+
#break if tmatch[1].offset.first == position
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
233
|
+
match
|
234
|
+
end
|
235
|
+
|
236
|
+
def parse_line stack, line, processor
|
237
|
+
processor.new_line line if processor
|
238
|
+
top, match = stack.last
|
239
|
+
position = 0
|
240
|
+
#@ln ||= 0
|
241
|
+
#@ln += 1
|
242
|
+
#STDERR.puts @ln
|
243
|
+
while true
|
244
|
+
if top.patterns
|
245
|
+
pattern, pattern_match = top.match_first_son line, position
|
246
|
+
else
|
247
|
+
pattern, pattern_match = nil
|
248
|
+
end
|
249
|
+
|
250
|
+
end_match = nil
|
251
|
+
if top.end
|
252
|
+
end_match = top.match_end( line, match, position )
|
253
|
+
end
|
254
|
+
|
255
|
+
if end_match && ( ! pattern_match || pattern_match.offset(0).first >= end_match.offset(0).first )
|
256
|
+
pattern_match = end_match
|
257
|
+
start_pos = pattern_match.offset(0).first
|
258
|
+
end_pos = pattern_match.offset(0).last
|
259
|
+
processor.close_tag top.contentName, start_pos if top.contentName && processor
|
260
|
+
parse_captures "captures", top, pattern_match, processor if processor
|
261
|
+
parse_captures "endCaptures", top, pattern_match, processor if processor
|
262
|
+
processor.close_tag top.name, end_pos if top.name && processor
|
263
|
+
stack.pop
|
264
|
+
top, match = stack.last
|
265
|
+
else
|
266
|
+
break unless pattern
|
267
|
+
start_pos = pattern_match.offset(0).first
|
268
|
+
end_pos = pattern_match.offset(0).last
|
269
|
+
if pattern.begin
|
270
|
+
processor.open_tag pattern.name, start_pos if pattern.name && processor
|
271
|
+
parse_captures "captures", pattern, pattern_match, processor if processor
|
272
|
+
parse_captures "beginCaptures", pattern, pattern_match, processor if processor
|
273
|
+
processor.open_tag pattern.contentName, end_pos if pattern.contentName && processor
|
274
|
+
top = pattern
|
275
|
+
match = pattern_match
|
276
|
+
stack << [top, match]
|
277
|
+
elsif pattern.match
|
278
|
+
processor.open_tag pattern.name, start_pos if pattern.name && processor
|
279
|
+
parse_captures "captures", pattern, pattern_match, processor if processor
|
280
|
+
processor.close_tag pattern.name, end_pos if pattern.name && processor
|
281
|
+
end
|
282
|
+
end
|
283
|
+
position = end_pos
|
284
|
+
end
|
285
|
+
end
|
286
|
+
end
|
287
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'textpow'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class ScoreManagerTest < Test::Unit::TestCase
|
6
|
+
include Textpow
|
7
|
+
|
8
|
+
def test_score
|
9
|
+
sp = ScoreManager.new
|
10
|
+
reference_scope = 'text.html.basic source.php.embedded.html string.quoted.double.php'
|
11
|
+
|
12
|
+
assert_not_equal( 0, sp.score( 'source.php string', reference_scope ) )
|
13
|
+
assert_not_equal( 0, sp.score( 'text.html source.php', reference_scope ) )
|
14
|
+
assert_equal( 0, sp.score( 'string source.php', reference_scope ) )
|
15
|
+
assert_equal( 0, sp.score( 'source.php text.html', reference_scope ) )
|
16
|
+
|
17
|
+
assert_equal( 0, sp.score( 'text.html source.php - string', reference_scope ) )
|
18
|
+
assert_not_equal( 0, sp.score( 'text.html source.php - ruby', reference_scope ) )
|
19
|
+
|
20
|
+
assert( sp.score( 'string', reference_scope ) > sp.score( 'source.php', reference_scope ) )
|
21
|
+
assert( sp.score( 'string.quoted', reference_scope ) > sp.score( 'source.php', reference_scope ) )
|
22
|
+
assert( sp.score( 'text source string', reference_scope ) > sp.score( 'source string', reference_scope ) )
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: textpow19
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 11
|
8
|
+
- 0
|
9
|
+
version: 0.11.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Chris Hoffman
|
13
|
+
- Spox
|
14
|
+
- Dizan Vasquez
|
15
|
+
autorequire:
|
16
|
+
bindir: bin
|
17
|
+
cert_chain: []
|
18
|
+
|
19
|
+
date: 2010-05-21 00:00:00 -05:00
|
20
|
+
default_executable:
|
21
|
+
dependencies:
|
22
|
+
- !ruby/object:Gem::Dependency
|
23
|
+
name: plist
|
24
|
+
prerelease: false
|
25
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ">="
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
segments:
|
31
|
+
- 3
|
32
|
+
- 0
|
33
|
+
- 1
|
34
|
+
version: 3.0.1
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rubyforge
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
segments:
|
46
|
+
- 2
|
47
|
+
- 0
|
48
|
+
- 4
|
49
|
+
version: 2.0.4
|
50
|
+
type: :development
|
51
|
+
version_requirements: *id002
|
52
|
+
- !ruby/object:Gem::Dependency
|
53
|
+
name: hoe
|
54
|
+
prerelease: false
|
55
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
segments:
|
61
|
+
- 2
|
62
|
+
- 6
|
63
|
+
- 0
|
64
|
+
version: 2.6.0
|
65
|
+
type: :development
|
66
|
+
version_requirements: *id003
|
67
|
+
description: A library for parsing {TextMate}[http://macromates.com/] bundles.
|
68
|
+
email:
|
69
|
+
- cehoffman@gmail.com
|
70
|
+
- spox@modspox.com
|
71
|
+
- dichodaemon@gmail.com
|
72
|
+
executables:
|
73
|
+
- plist2yaml
|
74
|
+
- plist2syntax
|
75
|
+
extensions: []
|
76
|
+
|
77
|
+
extra_rdoc_files:
|
78
|
+
- Manifest.txt
|
79
|
+
files:
|
80
|
+
- bin/plist2yaml
|
81
|
+
- bin/plist2syntax
|
82
|
+
- lib/textpow.rb
|
83
|
+
- lib/textpow/syntax.rb
|
84
|
+
- lib/textpow/version.rb
|
85
|
+
- lib/textpow/score_manager.rb
|
86
|
+
- lib/textpow/debug_processor.rb
|
87
|
+
- test/test_textpow.rb
|
88
|
+
- History.rdoc
|
89
|
+
- Rakefile
|
90
|
+
- Manifest.txt
|
91
|
+
- README.rdoc
|
92
|
+
has_rdoc: true
|
93
|
+
homepage: http://github.com/cehoffman/textpow
|
94
|
+
licenses: []
|
95
|
+
|
96
|
+
post_install_message:
|
97
|
+
rdoc_options:
|
98
|
+
- --main
|
99
|
+
- README.rdoc
|
100
|
+
require_paths:
|
101
|
+
- lib
|
102
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
103
|
+
none: false
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
segments:
|
108
|
+
- 1
|
109
|
+
- 9
|
110
|
+
- 0
|
111
|
+
version: 1.9.0
|
112
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
segments:
|
118
|
+
- 0
|
119
|
+
version: "0"
|
120
|
+
requirements: []
|
121
|
+
|
122
|
+
rubyforge_project: textpow19
|
123
|
+
rubygems_version: 1.3.7
|
124
|
+
signing_key:
|
125
|
+
specification_version: 3
|
126
|
+
summary: A library for parsing {TextMate}[http://macromates.com/] bundles.
|
127
|
+
test_files:
|
128
|
+
- test/test_textpow.rb
|