citrus 1.8.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +53 -46
- data/benchmark/after.dat +192 -0
- data/benchmark/before.dat +192 -0
- data/benchmark/master.dat +192 -0
- data/doc/background.markdown +9 -10
- data/doc/example.markdown +24 -15
- data/doc/syntax.markdown +20 -21
- data/lib/citrus.rb +208 -178
- data/lib/citrus/debug.rb +34 -4
- data/test/file_test.rb +12 -12
- data/test/helper.rb +27 -5
- data/test/match_test.rb +18 -34
- data/test/parse_error_test.rb +56 -0
- data/test/terminal_test.rb +56 -0
- metadata +10 -7
- data/test/expression_test.rb +0 -29
- data/test/fixed_width_test.rb +0 -37
data/lib/citrus/debug.rb
CHANGED
@@ -3,6 +3,16 @@ require 'builder'
|
|
3
3
|
|
4
4
|
module Citrus
|
5
5
|
class Match
|
6
|
+
# The offset at which this match was found in the input.
|
7
|
+
attr_accessor :offset
|
8
|
+
|
9
|
+
def debug_attrs
|
10
|
+
{ "names" => names.join(','),
|
11
|
+
"text" => to_s,
|
12
|
+
"offset" => offset
|
13
|
+
}
|
14
|
+
end
|
15
|
+
|
6
16
|
# Creates a Builder::XmlMarkup object from this match. Useful when
|
7
17
|
# inspecting a nested match. The +xml+ argument may be a Hash of
|
8
18
|
# Builder::XmlMarkup options.
|
@@ -13,12 +23,10 @@ module Citrus
|
|
13
23
|
xml.instruct!
|
14
24
|
end
|
15
25
|
|
16
|
-
attrs = { "names" => names.join(','), "text" => to_s, "offset" => offset }
|
17
|
-
|
18
26
|
if matches.empty?
|
19
|
-
xml.match(
|
27
|
+
xml.match(debug_attrs)
|
20
28
|
else
|
21
|
-
xml.match(
|
29
|
+
xml.match(debug_attrs) do
|
22
30
|
matches.each {|m| m.to_markup(xml) }
|
23
31
|
end
|
24
32
|
end
|
@@ -36,4 +44,26 @@ module Citrus
|
|
36
44
|
to_xml
|
37
45
|
end
|
38
46
|
end
|
47
|
+
|
48
|
+
# Hijack all classes that use Rule#create_match to create matches. Now, when
|
49
|
+
# matches are created they will also record their offset to help debugging.
|
50
|
+
# This functionality is included in this file because calculating the offset
|
51
|
+
# of every match as it is created can slow things down quite a bit.
|
52
|
+
[ Terminal,
|
53
|
+
AndPredicate,
|
54
|
+
NotPredicate,
|
55
|
+
ButPredicate,
|
56
|
+
Repeat,
|
57
|
+
Sequence
|
58
|
+
].each do |rule_class|
|
59
|
+
rule_class.class_eval do
|
60
|
+
alias original_match match
|
61
|
+
|
62
|
+
def match(input)
|
63
|
+
m = original_match(input)
|
64
|
+
m.offset = input.pos - m.length if m
|
65
|
+
m
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
39
69
|
end
|
data/test/file_test.rb
CHANGED
@@ -49,12 +49,12 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
49
49
|
match = grammar.parse('""')
|
50
50
|
assert(match)
|
51
51
|
assert_kind_of(Rule, match.value)
|
52
|
-
assert_instance_of(
|
52
|
+
assert_instance_of(Terminal, match.value)
|
53
53
|
|
54
54
|
match = grammar.parse('"a"')
|
55
55
|
assert(match)
|
56
56
|
assert_kind_of(Rule, match.value)
|
57
|
-
assert_instance_of(
|
57
|
+
assert_instance_of(Terminal, match.value)
|
58
58
|
|
59
59
|
match = grammar.parse('"a" "b"')
|
60
60
|
assert(match)
|
@@ -69,17 +69,17 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
69
69
|
match = grammar.parse('.')
|
70
70
|
assert(match)
|
71
71
|
assert_kind_of(Rule, match.value)
|
72
|
-
assert_instance_of(
|
72
|
+
assert_instance_of(Terminal, match.value)
|
73
73
|
|
74
74
|
match = grammar.parse('[a-z]')
|
75
75
|
assert(match)
|
76
76
|
assert_kind_of(Rule, match.value)
|
77
|
-
assert_instance_of(
|
77
|
+
assert_instance_of(Terminal, match.value)
|
78
78
|
|
79
79
|
match = grammar.parse('/./')
|
80
80
|
assert(match)
|
81
81
|
assert_kind_of(Rule, match.value)
|
82
|
-
assert_instance_of(
|
82
|
+
assert_instance_of(Terminal, match.value)
|
83
83
|
|
84
84
|
match = grammar.parse('/./ /./')
|
85
85
|
assert(match)
|
@@ -98,7 +98,7 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
98
98
|
match = grammar.parse('"" {}')
|
99
99
|
assert(match)
|
100
100
|
assert_kind_of(Rule, match.value)
|
101
|
-
assert_instance_of(
|
101
|
+
assert_instance_of(Terminal, match.value)
|
102
102
|
|
103
103
|
match = grammar.parse('""* {}')
|
104
104
|
assert(match)
|
@@ -180,7 +180,7 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
180
180
|
match = grammar.parse("[0-9] {\n def value\n text.to_i\n end\n}\n")
|
181
181
|
assert(match)
|
182
182
|
assert_kind_of(Rule, match.value)
|
183
|
-
assert_instance_of(
|
183
|
+
assert_instance_of(Terminal, match.value)
|
184
184
|
|
185
185
|
match = grammar.parse("[0-9]+ {\n def value\n text.to_i\n end\n}\n")
|
186
186
|
assert(match)
|
@@ -297,7 +297,7 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
297
297
|
match = grammar.parse('"a"')
|
298
298
|
assert(match)
|
299
299
|
assert_kind_of(Rule, match.value)
|
300
|
-
assert_instance_of(
|
300
|
+
assert_instance_of(Terminal, match.value)
|
301
301
|
end
|
302
302
|
|
303
303
|
|
@@ -362,25 +362,25 @@ class CitrusFileTest < Test::Unit::TestCase
|
|
362
362
|
match = grammar.parse('"a"')
|
363
363
|
assert(match)
|
364
364
|
assert_kind_of(Rule, match.value)
|
365
|
-
assert_instance_of(
|
365
|
+
assert_instance_of(Terminal, match.value)
|
366
366
|
assert(match.value.terminal?)
|
367
367
|
|
368
368
|
match = grammar.parse('[a-z]')
|
369
369
|
assert(match)
|
370
370
|
assert_kind_of(Rule, match.value)
|
371
|
-
assert_instance_of(
|
371
|
+
assert_instance_of(Terminal, match.value)
|
372
372
|
assert(match.value.terminal?)
|
373
373
|
|
374
374
|
match = grammar.parse('.')
|
375
375
|
assert(match)
|
376
376
|
assert_kind_of(Rule, match.value)
|
377
|
-
assert_instance_of(
|
377
|
+
assert_instance_of(Terminal, match.value)
|
378
378
|
assert(match.value.terminal?)
|
379
379
|
|
380
380
|
match = grammar.parse('/./')
|
381
381
|
assert(match)
|
382
382
|
assert_kind_of(Rule, match.value)
|
383
|
-
assert_instance_of(
|
383
|
+
assert_instance_of(Terminal, match.value)
|
384
384
|
assert(match.value.terminal?)
|
385
385
|
end
|
386
386
|
|
data/test/helper.rb
CHANGED
@@ -11,9 +11,7 @@ class Test::Unit::TestCase
|
|
11
11
|
Input.new(str)
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
include Citrus::Grammar
|
16
|
-
|
14
|
+
TestGrammar = Grammar.new do
|
17
15
|
rule :alpha do
|
18
16
|
/[a-zA-Z]/
|
19
17
|
end
|
@@ -27,6 +25,30 @@ class Test::Unit::TestCase
|
|
27
25
|
end
|
28
26
|
end
|
29
27
|
|
28
|
+
Double = Grammar.new do
|
29
|
+
include TestGrammar
|
30
|
+
|
31
|
+
root :double
|
32
|
+
|
33
|
+
rule :double do
|
34
|
+
one_or_more(:num)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
Words = Grammar.new do
|
39
|
+
include TestGrammar
|
40
|
+
|
41
|
+
root :words
|
42
|
+
|
43
|
+
rule :word do
|
44
|
+
one_or_more(:alpha)
|
45
|
+
end
|
46
|
+
|
47
|
+
rule :words do
|
48
|
+
[ :word, zero_or_more([ ' ', :word ]) ]
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
30
52
|
class EqualRule
|
31
53
|
include Citrus::Rule
|
32
54
|
|
@@ -34,8 +56,8 @@ class Test::Unit::TestCase
|
|
34
56
|
@value = value
|
35
57
|
end
|
36
58
|
|
37
|
-
def match(input
|
38
|
-
create_match(@value.to_s.dup
|
59
|
+
def match(input)
|
60
|
+
create_match(@value.to_s.dup) if @value.to_s == input.string
|
39
61
|
end
|
40
62
|
end
|
41
63
|
|
data/test/match_test.rb
CHANGED
@@ -2,25 +2,6 @@ require File.expand_path('../helper', __FILE__)
|
|
2
2
|
|
3
3
|
class MatchTest < Test::Unit::TestCase
|
4
4
|
|
5
|
-
Double = Grammar.new {
|
6
|
-
include MatchTest::TestGrammar
|
7
|
-
root :double
|
8
|
-
rule :double do
|
9
|
-
one_or_more(:num)
|
10
|
-
end
|
11
|
-
}
|
12
|
-
|
13
|
-
Sentence = Grammar.new {
|
14
|
-
include MatchTest::TestGrammar
|
15
|
-
root :sentence
|
16
|
-
rule :word do
|
17
|
-
one_or_more(:alpha)
|
18
|
-
end
|
19
|
-
rule :sentence do
|
20
|
-
[ :word, zero_or_more([ ' ', :word ]) ]
|
21
|
-
end
|
22
|
-
}
|
23
|
-
|
24
5
|
def test_string
|
25
6
|
match = Match.new('hello')
|
26
7
|
assert_equal('hello', match)
|
@@ -33,20 +14,7 @@ class MatchTest < Test::Unit::TestCase
|
|
33
14
|
match = Match.new([match1, match2])
|
34
15
|
assert_equal('ab', match)
|
35
16
|
assert_equal(2, match.length)
|
36
|
-
|
37
|
-
|
38
|
-
def test_match_data
|
39
|
-
match = Match.new('hello world'.match(/^(\w+) /))
|
40
|
-
assert_equal('hello ', match)
|
41
|
-
assert_equal(6, match.length)
|
42
|
-
end
|
43
|
-
|
44
|
-
def test_array_match_data
|
45
|
-
match1 = Match.new('hello')
|
46
|
-
match2 = Match.new(' world'.match(/.+/))
|
47
|
-
match = Match.new([match1, match2])
|
48
|
-
assert_equal('hello world', match)
|
49
|
-
assert_equal(11, match.length)
|
17
|
+
assert_equal(2, match.matches.length)
|
50
18
|
end
|
51
19
|
|
52
20
|
def test_equality
|
@@ -80,9 +48,25 @@ class MatchTest < Test::Unit::TestCase
|
|
80
48
|
end
|
81
49
|
|
82
50
|
def test_matches_deep
|
83
|
-
match =
|
51
|
+
match = Words.parse('one two three four')
|
84
52
|
assert(match)
|
85
53
|
assert_equal(15, match.find(:alpha).length)
|
86
54
|
end
|
87
55
|
|
56
|
+
def test_offset
|
57
|
+
match = Words.parse('one two')
|
58
|
+
assert(match)
|
59
|
+
assert_equal(0, match.offset)
|
60
|
+
|
61
|
+
words = match.find(:word)
|
62
|
+
assert(match)
|
63
|
+
assert_equal(2, words.length)
|
64
|
+
|
65
|
+
assert_equal('one', words[0])
|
66
|
+
assert_equal(0, words[0].offset)
|
67
|
+
|
68
|
+
assert_equal('two', words[1])
|
69
|
+
assert_equal(4, words[1].offset)
|
70
|
+
end
|
71
|
+
|
88
72
|
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require File.expand_path('../helper', __FILE__)
|
2
|
+
|
3
|
+
class ParseErrorTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
Sentence = Grammar.new do
|
6
|
+
include Words
|
7
|
+
|
8
|
+
rule :sentence do
|
9
|
+
all(:capital_word, one_or_more([ :space, :word ]), :period)
|
10
|
+
end
|
11
|
+
|
12
|
+
rule :capital_word do
|
13
|
+
all(/[A-Z]/, zero_or_more(:alpha))
|
14
|
+
end
|
15
|
+
|
16
|
+
rule :space do
|
17
|
+
one_or_more(any(" ", "\n", "\r\n"))
|
18
|
+
end
|
19
|
+
|
20
|
+
rule :period, '.'
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_basic
|
24
|
+
begin
|
25
|
+
TestGrammar.parse('#')
|
26
|
+
rescue ParseError => e
|
27
|
+
assert_equal(0, e.offset)
|
28
|
+
assert_equal('#', e.line)
|
29
|
+
assert_equal(1, e.line_number)
|
30
|
+
assert_equal(0, e.line_offset)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_single_line
|
35
|
+
begin
|
36
|
+
Sentence.parse('Once upon 4 time.')
|
37
|
+
rescue ParseError => e
|
38
|
+
assert_equal(10, e.offset)
|
39
|
+
assert_equal('Once upon 4 time.', e.line)
|
40
|
+
assert_equal(1, e.line_number)
|
41
|
+
assert_equal(10, e.line_offset)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_multi_line
|
46
|
+
begin
|
47
|
+
Sentence.parse("Once\nupon a\r\ntim3.")
|
48
|
+
rescue ParseError => e
|
49
|
+
assert_equal(16, e.offset)
|
50
|
+
assert_equal('tim3.', e.line)
|
51
|
+
assert_equal(3, e.line_number)
|
52
|
+
assert_equal(3, e.line_offset)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require File.expand_path('../helper', __FILE__)
|
2
|
+
|
3
|
+
class TerminalTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def test_terminal?
|
6
|
+
rule = Terminal.new
|
7
|
+
assert(rule.terminal?)
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_regexp_match
|
11
|
+
rule = Terminal.new(/\d+/)
|
12
|
+
match = rule.match(input('123 456'))
|
13
|
+
assert(match)
|
14
|
+
assert_equal('123', match)
|
15
|
+
assert_equal(3, match.length)
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_regexp_match_failure
|
19
|
+
rule = Terminal.new(/\d+/)
|
20
|
+
match = rule.match(input(' 456'))
|
21
|
+
assert_equal(nil, match)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_regexp_to_s
|
25
|
+
rule = Terminal.new(/\d+/)
|
26
|
+
assert_equal('/\\d+/', rule.to_s)
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_string_match
|
30
|
+
rule = Terminal.new('abc')
|
31
|
+
match = rule.match(input('abc'))
|
32
|
+
assert(match)
|
33
|
+
assert_equal('abc', match)
|
34
|
+
assert_equal(3, match.length)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_string_match_short
|
38
|
+
rule = Terminal.new('abc')
|
39
|
+
match = rule.match(input('ab'))
|
40
|
+
assert_equal(nil, match)
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_string_match_long
|
44
|
+
rule = Terminal.new('abc')
|
45
|
+
match = rule.match(input('abcd'))
|
46
|
+
assert(match)
|
47
|
+
assert_equal('abc', match)
|
48
|
+
assert_equal(3, match.length)
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_string_to_s
|
52
|
+
rule = Terminal.new('abc')
|
53
|
+
assert_equal('"abc"', rule.to_s)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
metadata
CHANGED
@@ -3,10 +3,10 @@ name: citrus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
|
-
-
|
7
|
-
- 8
|
6
|
+
- 2
|
8
7
|
- 0
|
9
|
-
|
8
|
+
- 0
|
9
|
+
version: 2.0.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Michael Jackson
|
@@ -52,6 +52,9 @@ extensions: []
|
|
52
52
|
extra_rdoc_files:
|
53
53
|
- README
|
54
54
|
files:
|
55
|
+
- benchmark/after.dat
|
56
|
+
- benchmark/before.dat
|
57
|
+
- benchmark/master.dat
|
55
58
|
- benchmark/seqpar.citrus
|
56
59
|
- benchmark/seqpar.gnuplot
|
57
60
|
- benchmark/seqpar.rb
|
@@ -83,19 +86,19 @@ files:
|
|
83
86
|
- test/calc_file_test.rb
|
84
87
|
- test/calc_test.rb
|
85
88
|
- test/choice_test.rb
|
86
|
-
- test/expression_test.rb
|
87
89
|
- test/file_test.rb
|
88
|
-
- test/fixed_width_test.rb
|
89
90
|
- test/grammar_test.rb
|
90
91
|
- test/helper.rb
|
91
92
|
- test/label_test.rb
|
92
93
|
- test/match_test.rb
|
93
94
|
- test/multibyte_test.rb
|
94
95
|
- test/not_predicate_test.rb
|
96
|
+
- test/parse_error_test.rb
|
95
97
|
- test/repeat_test.rb
|
96
98
|
- test/rule_test.rb
|
97
99
|
- test/sequence_test.rb
|
98
100
|
- test/super_test.rb
|
101
|
+
- test/terminal_test.rb
|
99
102
|
- citrus.gemspec
|
100
103
|
- Rakefile
|
101
104
|
- README
|
@@ -143,15 +146,15 @@ test_files:
|
|
143
146
|
- test/calc_file_test.rb
|
144
147
|
- test/calc_test.rb
|
145
148
|
- test/choice_test.rb
|
146
|
-
- test/expression_test.rb
|
147
149
|
- test/file_test.rb
|
148
|
-
- test/fixed_width_test.rb
|
149
150
|
- test/grammar_test.rb
|
150
151
|
- test/label_test.rb
|
151
152
|
- test/match_test.rb
|
152
153
|
- test/multibyte_test.rb
|
153
154
|
- test/not_predicate_test.rb
|
155
|
+
- test/parse_error_test.rb
|
154
156
|
- test/repeat_test.rb
|
155
157
|
- test/rule_test.rb
|
156
158
|
- test/sequence_test.rb
|
157
159
|
- test/super_test.rb
|
160
|
+
- test/terminal_test.rb
|