regexp_parser 1.8.2 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (27) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +34 -0
  3. data/lib/regexp_parser/expression.rb +4 -17
  4. data/lib/regexp_parser/expression/classes/group.rb +17 -2
  5. data/lib/regexp_parser/expression/classes/root.rb +4 -16
  6. data/lib/regexp_parser/expression/quantifier.rb +9 -0
  7. data/lib/regexp_parser/expression/sequence.rb +0 -10
  8. data/lib/regexp_parser/lexer.rb +2 -2
  9. data/lib/regexp_parser/parser.rb +27 -0
  10. data/lib/regexp_parser/scanner.rb +901 -820
  11. data/lib/regexp_parser/scanner/char_type.rl +11 -11
  12. data/lib/regexp_parser/scanner/property.rl +2 -2
  13. data/lib/regexp_parser/scanner/scanner.rl +152 -153
  14. data/lib/regexp_parser/version.rb +1 -1
  15. data/spec/expression/base_spec.rb +10 -0
  16. data/spec/expression/to_s_spec.rb +16 -0
  17. data/spec/lexer/literals_spec.rb +24 -49
  18. data/spec/parser/escapes_spec.rb +1 -1
  19. data/spec/parser/quantifiers_spec.rb +15 -0
  20. data/spec/parser/set/ranges_spec.rb +3 -3
  21. data/spec/scanner/escapes_spec.rb +6 -0
  22. data/spec/scanner/literals_spec.rb +28 -38
  23. data/spec/scanner/quantifiers_spec.rb +18 -13
  24. data/spec/scanner/sets_spec.rb +8 -2
  25. metadata +2 -6
  26. data/spec/expression/root_spec.rb +0 -9
  27. data/spec/expression/sequence_spec.rb +0 -9
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2d1c795eb67eee01aa7976a672349b9a0c7f6364262ffb8f53488f6d9949db07
4
- data.tar.gz: a38fddc769f16841e54f5dde90535e31ef7145e5bd41560ccf70443968495493
3
+ metadata.gz: dcf56dd42e703e7f1f846762c418e83792d46f2c7d9efffc1fb1612b4325e076
4
+ data.tar.gz: a6197d98af2325a93ed60a102f09bc98e2163dec0d2b57fbede82dbf0479ea8b
5
5
  SHA512:
6
- metadata.gz: 8dfcd3ba9f2a09370142e892184ccb9a1a973e24dea691582ca43cc670589dd7050eb30dbd924ae9e0584d124183dc2f62adca475628a726ecffaa22fcfb8903
7
- data.tar.gz: cd23a308e4d62af9a07bbec164c96ca3213de5a4f686c450fc90bca56387faa88febb8a3387e810d0fd8b78e06b191bd76777fe8b3f3cda9da5582b9009d1144
6
+ metadata.gz: 8b9db6543c87b63c49e24666e06bc012ea9a6e330711c9fbd35961c70d1222988bf2403927fdbe2f8797176d674b83c7c3f2d1215c5f92b0d6f00c6ab7fe37af
7
+ data.tar.gz: 6cf07796d7c6ab1520a63b0f3b65d2e513caccf0b81306321376473fc438c7573bcf867e74ea8465122e12a47fe2e7e09652473f83e26f1038e112c2d66b3d2c
data/CHANGELOG.md CHANGED
@@ -1,5 +1,39 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [2.0.0] - 2020-11-25 - [Janosch Müller](mailto:janosch84@gmail.com)
4
+
5
+ ### Changed
6
+
7
+ - some methods that used to return byte-based indices now return char-based indices
8
+ * the returned values have only changed for Regexps that contain multibyte chars
9
+ * this is only a breaking change if you used such methods directly AND relied on them pointing to bytes
10
+ * affected methods:
11
+ * `Regexp::Token` `#length`, `#offset`, `#te`, `#ts`
12
+ * `Regexp::Expression::Base` `#full_length`, `#offset`, `#starts_at`, `#te`, `#ts`
13
+ * thanks to [Akinori MUSHA](https://github.com/knu) for the report
14
+ - removed some deprecated methods/signatures
15
+ * these are rarely used and have been showing deprecation warnings for a long time
16
+ * `Regexp::Expression::Subexpression.new` with 3 arguments
17
+ * `Regexp::Expression::Root.new` without a token argument
18
+ * `Regexp::Expression.parsed`
19
+
20
+ ### Added
21
+
22
+ - `Regexp::Expression::Base#base_length`
23
+ * returns the character count of an expression body, ignoring any quantifier
24
+ - pragmatic, experimental support for chained quantifiers
25
+ * e.g.: `/^a{10}{4,6}$/` matches exactly 40, 50 or 60 `a`s
26
+ * successive quantifiers used to be silently dropped by the parser
27
+ * they are now wrapped with passive groups as if they were written `(?:a{10}){4,6}`
28
+ * thanks to [calfeld](https://github.com/calfeld) for reporting this a while back
29
+
30
+ ### Fixed
31
+
32
+ - incorrect encoding output for non-ascii comments
33
+ * this led to a crash when calling `#to_s` on parse results containing such comments
34
+ * thanks to [Michael Glass](https://github.com/michaelglass) for the report
35
+ - some crashes when scanning contrived patterns such as `'\😋'`
36
+
3
37
  ### [1.8.2] - 2020-10-11 - [Janosch Müller](mailto:janosch84@gmail.com)
4
38
 
5
39
  ### Fixed
data/lib/regexp_parser/expression.rb CHANGED
@@ -34,6 +34,10 @@ module Regexp::Expression
34
34
 
35
35
  alias :starts_at :ts
36
36
 
37
+ def base_length
38
+ to_s(:base).length
39
+ end
40
+
37
41
  def full_length
38
42
  to_s.length
39
43
  end
@@ -118,23 +122,6 @@ module Regexp::Expression
118
122
  alias :to_h :attributes
119
123
  end
120
124
 
121
- def self.parsed(exp)
122
- warn('WARNING: Regexp::Expression::Base.parsed is buggy and '\
123
- 'will be removed in 2.0.0. Use Regexp::Parser.parse instead.')
124
- case exp
125
- when String
126
- Regexp::Parser.parse(exp)
127
- when Regexp
128
- Regexp::Parser.parse(exp.source) # <- causes loss of root options
129
- when Regexp::Expression # <- never triggers
130
- exp
131
- else
132
- raise ArgumentError, 'Expression.parsed accepts a String, Regexp, or '\
133
- 'a Regexp::Expression as a value for exp, but it '\
134
- "was given #{exp.class.name}."
135
- end
136
- end
137
-
138
125
  end # module Regexp::Expression
139
126
 
140
127
  require 'regexp_parser/expression/quantifier'
data/lib/regexp_parser/expression/classes/group.rb CHANGED
@@ -10,9 +10,24 @@ module Regexp::Expression
10
10
  def comment?; false end
11
11
  end
12
12
 
13
- class Atomic < Group::Base; end
14
- class Passive < Group::Base; end
13
+ class Passive < Group::Base
14
+ attr_writer :implicit
15
+
16
+ def to_s(format = :full)
17
+ if implicit?
18
+ "#{expressions.join}#{quantifier_affix(format)}"
19
+ else
20
+ super
21
+ end
22
+ end
23
+
24
+ def implicit?
25
+ @implicit ||= false
26
+ end
27
+ end
28
+
15
29
  class Absence < Group::Base; end
30
+ class Atomic < Group::Base; end
16
31
  class Options < Group::Base
17
32
  attr_accessor :option_changes
18
33
  end
data/lib/regexp_parser/expression/classes/root.rb CHANGED
@@ -1,24 +1,12 @@
1
1
  module Regexp::Expression
2
2
 
3
3
  class Root < Regexp::Expression::Subexpression
4
- # TODO: this override is here for backwards compatibility, remove in 2.0.0
5
- def initialize(*args)
6
- unless args.first.is_a?(Regexp::Token)
7
- warn('WARNING: Root.new without a Token argument is deprecated and '\
8
- 'will be removed in 2.0.0. Use Root.build for the old behavior.')
9
- return super(self.class.build_token, *args)
10
- end
11
- super
4
+ def self.build(options = {})
5
+ new(build_token, options)
12
6
  end
13
7
 
14
- class << self
15
- def build(options = {})
16
- new(build_token, options)
17
- end
18
-
19
- def build_token
20
- Regexp::Token.new(:expression, :root, '', 0)
21
- end
8
+ def self.build_token
9
+ Regexp::Token.new(:expression, :root, '', 0)
22
10
  end
23
11
  end
24
12
  end
data/lib/regexp_parser/expression/quantifier.rb CHANGED
@@ -40,5 +40,14 @@ module Regexp::Expression
40
40
  RUBY
41
41
  end
42
42
  alias :lazy? :reluctant?
43
+
44
+ def ==(other)
45
+ other.class == self.class &&
46
+ other.token == token &&
47
+ other.mode == mode &&
48
+ other.min == min &&
49
+ other.max == max
50
+ end
51
+ alias :eq :==
43
52
  end
44
53
  end
data/lib/regexp_parser/expression/sequence.rb CHANGED
@@ -7,16 +7,6 @@ module Regexp::Expression
7
7
  # Used as the base class for the Alternation alternatives, Conditional
8
8
  # branches, and CharacterSet::Intersection intersected sequences.
9
9
  class Sequence < Regexp::Expression::Subexpression
10
- # TODO: this override is here for backwards compatibility, remove in 2.0.0
11
- def initialize(*args)
12
- if args.count == 3
13
- warn('WARNING: Sequence.new without a Regexp::Token argument is '\
14
- 'deprecated and will be removed in 2.0.0.')
15
- return self.class.at_levels(*args)
16
- end
17
- super
18
- end
19
-
20
10
  class << self
21
11
  def add_to(subexpression, params = {}, active_opts = {})
22
12
  sequence = at_levels(
data/lib/regexp_parser/lexer.rb CHANGED
@@ -96,10 +96,10 @@ class Regexp::Lexer
96
96
 
97
97
  tokens.pop
98
98
  tokens << Regexp::Token.new(:literal, :literal, lead,
99
- token.ts, (token.te - last.bytesize),
99
+ token.ts, (token.te - last.length),
100
100
  nesting, set_nesting, conditional_nesting)
101
101
  tokens << Regexp::Token.new(:literal, :literal, last,
102
- (token.ts + lead.bytesize), token.te,
102
+ (token.ts + lead.length), token.te,
103
103
  nesting, set_nesting, conditional_nesting)
104
104
  end
105
105
 
data/lib/regexp_parser/parser.rb CHANGED
@@ -438,6 +438,28 @@ class Regexp::Parser
438
438
  target_node || raise(ArgumentError, 'No valid target found for '\
439
439
  "'#{token.text}' ")
440
440
 
441
+ # in case of chained quantifiers, wrap target in an implicit passive group
442
+ # description of the problem: https://github.com/ammar/regexp_parser/issues/3
443
+ # rationale for this solution: https://github.com/ammar/regexp_parser/pull/69
444
+ if target_node.quantified?
445
+ new_token = Regexp::Token.new(
446
+ :group,
447
+ :passive,
448
+ '', # text
449
+ target_node.ts,
450
+ nil, # te (unused)
451
+ target_node.level,
452
+ target_node.set_level,
453
+ target_node.conditional_level
454
+ )
455
+ new_group = Group::Passive.new(new_token, active_opts)
456
+ new_group.implicit = true
457
+ new_group << target_node
458
+ increase_level(target_node)
459
+ node.expressions[offset] = new_group
460
+ target_node = new_group
461
+ end
462
+
441
463
  case token.token
442
464
  when :zero_or_one
443
465
  target_node.quantify(:zero_or_one, token.text, 0, 1, :greedy)
@@ -468,6 +490,11 @@ class Regexp::Parser
468
490
  end
469
491
  end
470
492
 
493
+ def increase_level(exp)
494
+ exp.level += 1
495
+ exp.respond_to?(:each) && exp.each { |subexp| increase_level(subexp) }
496
+ end
497
+
471
498
  def interval(target_node, token)
472
499
  text = token.text
473
500
  mchr = text[text.length-1].chr =~ /[?+]/ ? text[text.length-1].chr : nil
data/lib/regexp_parser/scanner.rb CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  # line 1 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
4
4
 
5
- # line 678 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
5
+ # line 680 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
6
6
 
7
7
 
8
8
  # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
@@ -88,9 +88,10 @@ class Regexp::Scanner
88
88
  self.set_depth = 0
89
89
  self.group_depth = 0
90
90
  self.conditional_stack = []
91
+ self.char_pos = 0
91
92
 
92
93
 
93
- # line 93 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
94
+ # line 94 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
94
95
  class << self
95
96
  attr_accessor :_re_scanner_trans_keys
96
97
  private :_re_scanner_trans_keys, :_re_scanner_trans_keys=
@@ -129,31 +130,33 @@ self._re_scanner_trans_keys = [
129
130
  112, 112, 111, 111, 114, 114,
130
131
  100, 100, 100, 100, 65,
131
132
  122, 61, 61, 93, 93,
132
- 45, 45, 92, 92, 92, 92,
133
- 45, 45, 92, 92, 92,
134
- 92, 48, 123, 48, 102,
135
- 48, 102, 48, 102, 48, 102,
136
- 9, 125, 9, 125, 9,
133
+ -128, -65, -128, -65, 45, 45,
134
+ 92, 92, 92, 92, 45,
135
+ 45, 92, 92, 92, 92,
136
+ 48, 123, 48, 102, 48, 102,
137
+ 48, 102, 48, 102, 9,
137
138
  125, 9, 125, 9, 125,
138
- 9, 125, 48, 123, 41, 41,
139
- 39, 122, 41, 57, 48,
140
- 122, -128, 127, -62, -33,
141
- -32, -17, -16, -12, 1, 127,
142
- 1, 127, 9, 32, 33,
143
- 126, 10, 10, 63, 63,
144
- 33, 126, 33, 126, 43, 63,
145
- 43, 63, 43, 63, 65,
146
- 122, 44, 57, 43, 63,
147
- 68, 119, 80, 112, -62, 125,
148
- -128, -65, -128, -65, -128,
149
- -65, 38, 38, 38, 93,
150
- 46, 61, 48, 122, 36, 125,
151
- 48, 55, 48, 55, 77,
152
- 77, 45, 45, 0, 0,
153
- 67, 99, 45, 45, 0, 0,
154
- 92, 92, 48, 102, 39,
155
- 60, 39, 122, 49, 57,
156
- 41, 57, 48, 122, 0
139
+ 9, 125, 9, 125, 9, 125,
140
+ 48, 123, 41, 41, 39,
141
+ 122, 41, 57, 48, 122,
142
+ -128, 127, -62, -33, -32, -17,
143
+ -16, -12, 1, 127, 1,
144
+ 127, 9, 32, 33, 126,
145
+ 10, 10, 63, 63, 33, 126,
146
+ 33, 126, 43, 63, 43,
147
+ 63, 43, 63, 65, 122,
148
+ 44, 57, 43, 63, 68, 119,
149
+ 80, 112, -62, 125, -128,
150
+ -65, -128, -65, -128, -65,
151
+ 38, 38, 38, 93, 46, 61,
152
+ 48, 122, -62, 125, -128,
153
+ -65, -128, -65, -128, -65,
154
+ 48, 55, 48, 55, 77, 77,
155
+ 45, 45, 0, 0, 67,
156
+ 99, 45, 45, 0, 0,
157
+ 92, 92, 48, 102, 39, 60,
158
+ 39, 122, 49, 57, 41,
159
+ 57, 48, 122, 0
157
160
  ]
158
161
 
159
162
  class << self
@@ -172,15 +175,15 @@ self._re_scanner_key_spans = [
172
175
  1, 1, 1, 1, 1, 1, 4, 1,
173
176
  1, 1, 1, 1, 1, 1, 1, 1,
174
177
  1, 1, 1, 1, 1, 58, 1, 1,
175
- 1, 1, 1, 1, 1, 1, 76, 55,
176
- 55, 55, 55, 117, 117, 117, 117, 117,
177
- 117, 76, 1, 84, 17, 75, 256, 30,
178
- 16, 5, 127, 127, 24, 94, 1, 1,
179
- 94, 94, 21, 21, 21, 58, 14, 21,
180
- 52, 33, 188, 64, 64, 64, 1, 56,
181
- 16, 75, 90, 8, 8, 1, 1, 0,
182
- 33, 1, 0, 1, 55, 22, 84, 9,
183
- 17, 75
178
+ 64, 64, 1, 1, 1, 1, 1, 1,
179
+ 76, 55, 55, 55, 55, 117, 117, 117,
180
+ 117, 117, 117, 76, 1, 84, 17, 75,
181
+ 256, 30, 16, 5, 127, 127, 24, 94,
182
+ 1, 1, 94, 94, 21, 21, 21, 58,
183
+ 14, 21, 52, 33, 188, 64, 64, 64,
184
+ 1, 56, 16, 75, 188, 64, 64, 64,
185
+ 8, 8, 1, 1, 0, 33, 1, 0,
186
+ 1, 55, 22, 84, 9, 17, 75
184
187
  ]
185
188
 
186
189
  class << self
@@ -199,15 +202,15 @@ self._re_scanner_index_offsets = [
199
202
  2047, 2049, 2051, 2053, 2055, 2057, 2059, 2064,
200
203
  2066, 2068, 2070, 2072, 2074, 2076, 2078, 2080,
201
204
  2082, 2084, 2086, 2088, 2090, 2092, 2151, 2153,
202
- 2155, 2157, 2159, 2161, 2163, 2165, 2167, 2244,
203
- 2300, 2356, 2412, 2468, 2586, 2704, 2822, 2940,
204
- 3058, 3176, 3253, 3255, 3340, 3358, 3434, 3691,
205
- 3722, 3739, 3745, 3873, 4001, 4026, 4121, 4123,
206
- 4125, 4220, 4315, 4337, 4359, 4381, 4440, 4455,
207
- 4477, 4530, 4564, 4753, 4818, 4883, 4948, 4950,
208
- 5007, 5024, 5100, 5191, 5200, 5209, 5211, 5213,
209
- 5214, 5248, 5250, 5251, 5253, 5309, 5332, 5417,
210
- 5427, 5445
205
+ 2155, 2220, 2285, 2287, 2289, 2291, 2293, 2295,
206
+ 2297, 2374, 2430, 2486, 2542, 2598, 2716, 2834,
207
+ 2952, 3070, 3188, 3306, 3383, 3385, 3470, 3488,
208
+ 3564, 3821, 3852, 3869, 3875, 4003, 4131, 4156,
209
+ 4251, 4253, 4255, 4350, 4445, 4467, 4489, 4511,
210
+ 4570, 4585, 4607, 4660, 4694, 4883, 4948, 5013,
211
+ 5078, 5080, 5137, 5154, 5230, 5419, 5484, 5549,
212
+ 5614, 5623, 5632, 5634, 5636, 5637, 5671, 5673,
213
+ 5674, 5676, 5732, 5755, 5840, 5850, 5868
211
214
  ]
212
215
 
213
216
  class << self
@@ -484,123 +487,140 @@ self._re_scanner_indicies = [
484
487
  95, 95, 95, 95, 95, 95, 95, 95,
485
488
  95, 95, 95, 95, 95, 95, 95, 95,
486
489
  95, 95, 95, 95, 95, 95, 43, 96,
487
- 43, 97, 43, 98, 33, 100, 99, 102,
488
- 99, 103, 33, 105, 104, 107, 104, 108,
489
- 108, 108, 108, 108, 108, 108, 108, 108,
490
- 108, 33, 33, 33, 33, 33, 33, 33,
491
- 108, 108, 108, 108, 108, 108, 33, 33,
490
+ 43, 97, 43, 99, 99, 99, 99, 99,
491
+ 99, 99, 99, 99, 99, 99, 99, 99,
492
+ 99, 99, 99, 99, 99, 99, 99, 99,
493
+ 99, 99, 99, 99, 99, 99, 99, 99,
494
+ 99, 99, 99, 99, 99, 99, 99, 99,
495
+ 99, 99, 99, 99, 99, 99, 99, 99,
496
+ 99, 99, 99, 99, 99, 99, 99, 99,
497
+ 99, 99, 99, 99, 99, 99, 99, 99,
498
+ 99, 99, 99, 98, 100, 100, 100, 100,
499
+ 100, 100, 100, 100, 100, 100, 100, 100,
500
+ 100, 100, 100, 100, 100, 100, 100, 100,
501
+ 100, 100, 100, 100, 100, 100, 100, 100,
502
+ 100, 100, 100, 100, 100, 100, 100, 100,
503
+ 100, 100, 100, 100, 100, 100, 100, 100,
504
+ 100, 100, 100, 100, 100, 100, 100, 100,
505
+ 100, 100, 100, 100, 100, 100, 100, 100,
506
+ 100, 100, 100, 100, 98, 101, 33, 103,
507
+ 102, 105, 102, 106, 33, 108, 107, 110,
508
+ 107, 111, 111, 111, 111, 111, 111, 111,
509
+ 111, 111, 111, 33, 33, 33, 33, 33,
510
+ 33, 33, 111, 111, 111, 111, 111, 111,
492
511
  33, 33, 33, 33, 33, 33, 33, 33,
493
512
  33, 33, 33, 33, 33, 33, 33, 33,
494
513
  33, 33, 33, 33, 33, 33, 33, 33,
495
- 108, 108, 108, 108, 108, 108, 33, 33,
514
+ 33, 33, 111, 111, 111, 111, 111, 111,
496
515
  33, 33, 33, 33, 33, 33, 33, 33,
497
516
  33, 33, 33, 33, 33, 33, 33, 33,
498
- 33, 33, 109, 33, 110, 110, 110, 110,
499
- 110, 110, 110, 110, 110, 110, 33, 33,
500
- 33, 33, 33, 33, 33, 110, 110, 110,
501
- 110, 110, 110, 33, 33, 33, 33, 33,
517
+ 33, 33, 33, 33, 112, 33, 113, 113,
518
+ 113, 113, 113, 113, 113, 113, 113, 113,
519
+ 33, 33, 33, 33, 33, 33, 33, 113,
520
+ 113, 113, 113, 113, 113, 33, 33, 33,
502
521
  33, 33, 33, 33, 33, 33, 33, 33,
503
522
  33, 33, 33, 33, 33, 33, 33, 33,
504
- 33, 33, 33, 33, 33, 110, 110, 110,
505
- 110, 110, 110, 33, 111, 111, 111, 111,
506
- 111, 111, 111, 111, 111, 111, 33, 33,
507
- 33, 33, 33, 33, 33, 111, 111, 111,
508
- 111, 111, 111, 33, 33, 33, 33, 33,
523
+ 33, 33, 33, 33, 33, 33, 33, 113,
524
+ 113, 113, 113, 113, 113, 33, 114, 114,
525
+ 114, 114, 114, 114, 114, 114, 114, 114,
526
+ 33, 33, 33, 33, 33, 33, 33, 114,
527
+ 114, 114, 114, 114, 114, 33, 33, 33,
509
528
  33, 33, 33, 33, 33, 33, 33, 33,
510
529
  33, 33, 33, 33, 33, 33, 33, 33,
511
- 33, 33, 33, 33, 33, 111, 111, 111,
512
- 111, 111, 111, 33, 112, 112, 112, 112,
513
- 112, 112, 112, 112, 112, 112, 33, 33,
514
- 33, 33, 33, 33, 33, 112, 112, 112,
515
- 112, 112, 112, 33, 33, 33, 33, 33,
530
+ 33, 33, 33, 33, 33, 33, 33, 114,
531
+ 114, 114, 114, 114, 114, 33, 115, 115,
532
+ 115, 115, 115, 115, 115, 115, 115, 115,
533
+ 33, 33, 33, 33, 33, 33, 33, 115,
534
+ 115, 115, 115, 115, 115, 33, 33, 33,
516
535
  33, 33, 33, 33, 33, 33, 33, 33,
517
536
  33, 33, 33, 33, 33, 33, 33, 33,
518
- 33, 33, 33, 33, 33, 112, 112, 112,
519
- 112, 112, 112, 33, 113, 113, 113, 113,
520
- 113, 113, 113, 113, 113, 113, 33, 33,
521
- 33, 33, 33, 33, 33, 113, 113, 113,
522
- 113, 113, 113, 33, 33, 33, 33, 33,
537
+ 33, 33, 33, 33, 33, 33, 33, 115,
538
+ 115, 115, 115, 115, 115, 33, 116, 116,
539
+ 116, 116, 116, 116, 116, 116, 116, 116,
540
+ 33, 33, 33, 33, 33, 33, 33, 116,
541
+ 116, 116, 116, 116, 116, 33, 33, 33,
523
542
  33, 33, 33, 33, 33, 33, 33, 33,
524
543
  33, 33, 33, 33, 33, 33, 33, 33,
525
- 33, 33, 33, 33, 33, 113, 113, 113,
526
- 113, 113, 113, 33, 109, 109, 109, 109,
527
- 109, 33, 33, 33, 33, 33, 33, 33,
544
+ 33, 33, 33, 33, 33, 33, 33, 116,
545
+ 116, 116, 116, 116, 116, 33, 112, 112,
546
+ 112, 112, 112, 33, 33, 33, 33, 33,
528
547
  33, 33, 33, 33, 33, 33, 33, 33,
529
- 33, 33, 33, 109, 33, 33, 33, 33,
548
+ 33, 33, 33, 33, 33, 112, 33, 33,
530
549
  33, 33, 33, 33, 33, 33, 33, 33,
531
- 33, 33, 33, 114, 114, 114, 114, 114,
532
- 114, 114, 114, 114, 114, 33, 33, 33,
533
- 33, 33, 33, 33, 114, 114, 114, 114,
534
- 114, 114, 33, 33, 33, 33, 33, 33,
550
+ 33, 33, 33, 33, 33, 117, 117, 117,
551
+ 117, 117, 117, 117, 117, 117, 117, 33,
552
+ 33, 33, 33, 33, 33, 33, 117, 117,
553
+ 117, 117, 117, 117, 33, 33, 33, 33,
535
554
  33, 33, 33, 33, 33, 33, 33, 33,
536
555
  33, 33, 33, 33, 33, 33, 33, 33,
537
- 33, 33, 33, 33, 114, 114, 114, 114,
538
- 114, 114, 33, 33, 33, 33, 33, 33,
556
+ 33, 33, 33, 33, 33, 33, 117, 117,
557
+ 117, 117, 117, 117, 33, 33, 33, 33,
539
558
  33, 33, 33, 33, 33, 33, 33, 33,
540
559
  33, 33, 33, 33, 33, 33, 33, 33,
541
- 112, 33, 109, 109, 109, 109, 109, 33,
560
+ 33, 33, 115, 33, 112, 112, 112, 112,
561
+ 112, 33, 33, 33, 33, 33, 33, 33,
542
562
  33, 33, 33, 33, 33, 33, 33, 33,
563
+ 33, 33, 33, 112, 33, 33, 33, 33,
543
564
  33, 33, 33, 33, 33, 33, 33, 33,
544
- 33, 109, 33, 33, 33, 33, 33, 33,
565
+ 33, 33, 33, 118, 118, 118, 118, 118,
566
+ 118, 118, 118, 118, 118, 33, 33, 33,
567
+ 33, 33, 33, 33, 118, 118, 118, 118,
568
+ 118, 118, 33, 33, 33, 33, 33, 33,
545
569
  33, 33, 33, 33, 33, 33, 33, 33,
546
- 33, 115, 115, 115, 115, 115, 115, 115,
547
- 115, 115, 115, 33, 33, 33, 33, 33,
548
- 33, 33, 115, 115, 115, 115, 115, 115,
549
570
  33, 33, 33, 33, 33, 33, 33, 33,
571
+ 33, 33, 33, 33, 118, 118, 118, 118,
572
+ 118, 118, 33, 33, 33, 33, 33, 33,
550
573
  33, 33, 33, 33, 33, 33, 33, 33,
551
574
  33, 33, 33, 33, 33, 33, 33, 33,
552
- 33, 33, 115, 115, 115, 115, 115, 115,
575
+ 115, 33, 112, 112, 112, 112, 112, 33,
553
576
  33, 33, 33, 33, 33, 33, 33, 33,
554
577
  33, 33, 33, 33, 33, 33, 33, 33,
555
- 33, 33, 33, 33, 33, 33, 112, 33,
556
- 109, 109, 109, 109, 109, 33, 33, 33,
578
+ 33, 112, 33, 33, 33, 33, 33, 33,
557
579
  33, 33, 33, 33, 33, 33, 33, 33,
558
- 33, 33, 33, 33, 33, 33, 33, 109,
580
+ 33, 119, 119, 119, 119, 119, 119, 119,
581
+ 119, 119, 119, 33, 33, 33, 33, 33,
582
+ 33, 33, 119, 119, 119, 119, 119, 119,
559
583
  33, 33, 33, 33, 33, 33, 33, 33,
560
- 33, 33, 33, 33, 33, 33, 33, 116,
561
- 116, 116, 116, 116, 116, 116, 116, 116,
562
- 116, 33, 33, 33, 33, 33, 33, 33,
563
- 116, 116, 116, 116, 116, 116, 33, 33,
564
584
  33, 33, 33, 33, 33, 33, 33, 33,
565
585
  33, 33, 33, 33, 33, 33, 33, 33,
586
+ 33, 33, 119, 119, 119, 119, 119, 119,
566
587
  33, 33, 33, 33, 33, 33, 33, 33,
567
- 116, 116, 116, 116, 116, 116, 33, 33,
568
588
  33, 33, 33, 33, 33, 33, 33, 33,
589
+ 33, 33, 33, 33, 33, 33, 115, 33,
590
+ 112, 112, 112, 112, 112, 33, 33, 33,
569
591
  33, 33, 33, 33, 33, 33, 33, 33,
570
- 33, 33, 33, 33, 112, 33, 109, 109,
571
- 109, 109, 109, 33, 33, 33, 33, 33,
592
+ 33, 33, 33, 33, 33, 33, 33, 112,
572
593
  33, 33, 33, 33, 33, 33, 33, 33,
573
- 33, 33, 33, 33, 33, 109, 33, 33,
594
+ 33, 33, 33, 33, 33, 33, 33, 120,
595
+ 120, 120, 120, 120, 120, 120, 120, 120,
596
+ 120, 33, 33, 33, 33, 33, 33, 33,
597
+ 120, 120, 120, 120, 120, 120, 33, 33,
574
598
  33, 33, 33, 33, 33, 33, 33, 33,
575
- 33, 33, 33, 33, 33, 117, 117, 117,
576
- 117, 117, 117, 117, 117, 117, 117, 33,
577
- 33, 33, 33, 33, 33, 33, 117, 117,
578
- 117, 117, 117, 117, 33, 33, 33, 33,
579
599
  33, 33, 33, 33, 33, 33, 33, 33,
580
600
  33, 33, 33, 33, 33, 33, 33, 33,
581
- 33, 33, 33, 33, 33, 33, 117, 117,
582
- 117, 117, 117, 117, 33, 33, 33, 33,
601
+ 120, 120, 120, 120, 120, 120, 33, 33,
583
602
  33, 33, 33, 33, 33, 33, 33, 33,
584
603
  33, 33, 33, 33, 33, 33, 33, 33,
585
- 33, 33, 112, 33, 109, 109, 109, 109,
586
- 109, 33, 33, 33, 33, 33, 33, 33,
604
+ 33, 33, 33, 33, 115, 33, 112, 112,
605
+ 112, 112, 112, 33, 33, 33, 33, 33,
587
606
  33, 33, 33, 33, 33, 33, 33, 33,
588
- 33, 33, 33, 109, 33, 33, 33, 33,
607
+ 33, 33, 33, 33, 33, 112, 33, 33,
589
608
  33, 33, 33, 33, 33, 33, 33, 33,
590
- 33, 33, 33, 118, 118, 118, 118, 118,
591
- 118, 118, 118, 118, 118, 33, 33, 33,
592
- 33, 33, 33, 33, 118, 118, 118, 118,
593
- 118, 118, 33, 33, 33, 33, 33, 33,
609
+ 33, 33, 33, 33, 33, 121, 121, 121,
610
+ 121, 121, 121, 121, 121, 121, 121, 33,
611
+ 33, 33, 33, 33, 33, 33, 121, 121,
612
+ 121, 121, 121, 121, 33, 33, 33, 33,
594
613
  33, 33, 33, 33, 33, 33, 33, 33,
595
614
  33, 33, 33, 33, 33, 33, 33, 33,
596
- 33, 33, 33, 33, 118, 118, 118, 118,
597
- 118, 118, 33, 33, 33, 33, 33, 33,
615
+ 33, 33, 33, 33, 33, 33, 121, 121,
616
+ 121, 121, 121, 121, 33, 33, 33, 33,
598
617
  33, 33, 33, 33, 33, 33, 33, 33,
599
618
  33, 33, 33, 33, 33, 33, 33, 33,
600
- 112, 33, 109, 109, 109, 109, 109, 33,
619
+ 33, 33, 115, 33, 112, 112, 112, 112,
620
+ 112, 33, 33, 33, 33, 33, 33, 33,
601
621
  33, 33, 33, 33, 33, 33, 33, 33,
622
+ 33, 33, 33, 112, 33, 33, 33, 33,
602
623
  33, 33, 33, 33, 33, 33, 33, 33,
603
- 33, 109, 33, 33, 33, 33, 33, 33,
604
624
  33, 33, 33, 33, 33, 33, 33, 33,
605
625
  33, 33, 33, 33, 33, 33, 33, 33,
606
626
  33, 33, 33, 33, 33, 33, 33, 33,
@@ -611,205 +631,204 @@ self._re_scanner_indicies = [
611
631
  33, 33, 33, 33, 33, 33, 33, 33,
612
632
  33, 33, 33, 33, 33, 33, 33, 33,
613
633
  33, 33, 33, 33, 33, 33, 33, 33,
614
- 33, 33, 33, 33, 33, 33, 112, 33,
615
- 120, 120, 120, 120, 120, 120, 120, 120,
616
- 120, 120, 119, 119, 119, 119, 119, 119,
617
- 119, 120, 120, 120, 120, 120, 120, 119,
618
- 119, 119, 119, 119, 119, 119, 119, 119,
619
- 119, 119, 119, 119, 119, 119, 119, 119,
620
- 119, 119, 119, 119, 119, 119, 119, 119,
621
- 119, 120, 120, 120, 120, 120, 120, 119,
622
- 119, 119, 119, 119, 119, 119, 119, 119,
623
- 119, 119, 119, 119, 119, 119, 119, 119,
624
- 119, 119, 119, 33, 119, 122, 121, 123,
625
- 121, 121, 121, 121, 121, 121, 121, 121,
626
- 124, 124, 124, 124, 124, 124, 124, 124,
627
- 124, 124, 121, 121, 121, 121, 121, 121,
628
- 121, 124, 124, 124, 124, 124, 124, 124,
629
- 124, 124, 124, 124, 124, 124, 124, 124,
630
- 124, 124, 124, 124, 124, 124, 124, 124,
631
- 124, 124, 124, 121, 121, 121, 121, 124,
632
- 121, 124, 124, 124, 124, 124, 124, 124,
633
- 124, 124, 124, 124, 124, 124, 124, 124,
634
- 124, 124, 124, 124, 124, 124, 124, 124,
635
- 124, 124, 124, 121, 122, 121, 121, 121,
636
- 121, 121, 121, 125, 125, 125, 125, 125,
637
- 125, 125, 125, 125, 125, 121, 126, 126,
638
- 126, 126, 126, 126, 126, 126, 126, 126,
639
- 121, 121, 121, 121, 123, 121, 121, 126,
640
- 126, 126, 126, 126, 126, 126, 126, 126,
641
- 126, 126, 126, 126, 126, 126, 126, 126,
642
- 126, 126, 126, 126, 126, 126, 126, 126,
643
- 126, 121, 121, 121, 121, 126, 121, 126,
644
- 126, 126, 126, 126, 126, 126, 126, 126,
645
- 126, 126, 126, 126, 126, 126, 126, 126,
646
- 126, 126, 126, 126, 126, 126, 126, 126,
647
- 126, 121, 33, 33, 33, 33, 33, 33,
648
- 33, 33, 33, 33, 33, 33, 33, 33,
649
- 33, 33, 33, 33, 33, 33, 33, 33,
650
- 33, 33, 33, 33, 33, 33, 33, 33,
651
- 33, 33, 33, 33, 33, 33, 33, 33,
652
- 33, 33, 33, 33, 33, 33, 33, 33,
653
- 33, 33, 33, 33, 33, 33, 33, 33,
654
- 33, 33, 33, 33, 33, 33, 33, 33,
655
- 33, 33, 33, 33, 127, 127, 127, 127,
634
+ 115, 33, 123, 123, 123, 123, 123, 123,
635
+ 123, 123, 123, 123, 122, 122, 122, 122,
636
+ 122, 122, 122, 123, 123, 123, 123, 123,
637
+ 123, 122, 122, 122, 122, 122, 122, 122,
638
+ 122, 122, 122, 122, 122, 122, 122, 122,
639
+ 122, 122, 122, 122, 122, 122, 122, 122,
640
+ 122, 122, 122, 123, 123, 123, 123, 123,
641
+ 123, 122, 122, 122, 122, 122, 122, 122,
642
+ 122, 122, 122, 122, 122, 122, 122, 122,
643
+ 122, 122, 122, 122, 122, 33, 122, 125,
644
+ 124, 126, 124, 124, 124, 124, 124, 124,
645
+ 124, 124, 127, 127, 127, 127, 127, 127,
646
+ 127, 127, 127, 127, 124, 124, 124, 124,
647
+ 124, 124, 124, 127, 127, 127, 127, 127,
656
648
  127, 127, 127, 127, 127, 127, 127, 127,
657
649
  127, 127, 127, 127, 127, 127, 127, 127,
650
+ 127, 127, 127, 127, 127, 124, 124, 124,
651
+ 124, 127, 124, 127, 127, 127, 127, 127,
658
652
  127, 127, 127, 127, 127, 127, 127, 127,
659
- 127, 127, 128, 128, 128, 128, 128, 128,
660
- 128, 128, 128, 128, 128, 128, 128, 128,
661
- 128, 128, 129, 129, 129, 129, 129, 33,
653
+ 127, 127, 127, 127, 127, 127, 127, 127,
654
+ 127, 127, 127, 127, 127, 124, 125, 124,
655
+ 124, 124, 124, 124, 124, 128, 128, 128,
656
+ 128, 128, 128, 128, 128, 128, 128, 124,
657
+ 129, 129, 129, 129, 129, 129, 129, 129,
658
+ 129, 129, 124, 124, 124, 124, 126, 124,
659
+ 124, 129, 129, 129, 129, 129, 129, 129,
660
+ 129, 129, 129, 129, 129, 129, 129, 129,
661
+ 129, 129, 129, 129, 129, 129, 129, 129,
662
+ 129, 129, 129, 124, 124, 124, 124, 129,
663
+ 124, 129, 129, 129, 129, 129, 129, 129,
664
+ 129, 129, 129, 129, 129, 129, 129, 129,
665
+ 129, 129, 129, 129, 129, 129, 129, 129,
666
+ 129, 129, 129, 124, 33, 33, 33, 33,
662
667
  33, 33, 33, 33, 33, 33, 33, 33,
663
- 33, 33, 33, 130, 130, 130, 130, 130,
664
- 130, 130, 130, 131, 131, 131, 131, 131,
665
- 130, 130, 130, 130, 130, 130, 130, 130,
668
+ 33, 33, 33, 33, 33, 33, 33, 33,
669
+ 33, 33, 33, 33, 33, 33, 33, 33,
670
+ 33, 33, 33, 33, 33, 33, 33, 33,
671
+ 33, 33, 33, 33, 33, 33, 33, 33,
672
+ 33, 33, 33, 33, 33, 33, 33, 33,
673
+ 33, 33, 33, 33, 33, 33, 33, 33,
674
+ 33, 33, 33, 33, 33, 33, 130, 130,
666
675
  130, 130, 130, 130, 130, 130, 130, 130,
667
- 130, 130, 132, 133, 133, 134, 135, 133,
668
- 133, 133, 136, 137, 138, 139, 133, 133,
669
- 140, 133, 133, 133, 133, 133, 133, 133,
670
- 133, 133, 133, 133, 133, 133, 133, 133,
671
- 133, 141, 133, 133, 133, 133, 133, 133,
672
- 133, 133, 133, 133, 133, 133, 133, 133,
673
- 133, 133, 133, 133, 133, 133, 133, 133,
674
- 133, 133, 133, 133, 133, 142, 143, 144,
675
- 145, 133, 133, 133, 133, 133, 133, 133,
676
- 133, 133, 133, 133, 133, 133, 133, 133,
677
- 133, 133, 133, 133, 133, 133, 133, 133,
678
- 133, 133, 133, 133, 133, 146, 147, 144,
679
- 133, 130, 133, 127, 127, 127, 127, 127,
680
- 127, 127, 127, 127, 127, 127, 127, 127,
681
- 127, 127, 127, 127, 127, 127, 127, 127,
682
- 127, 127, 127, 127, 127, 127, 127, 127,
683
- 127, 148, 128, 128, 128, 128, 128, 128,
684
- 128, 128, 128, 128, 128, 128, 128, 128,
685
- 128, 128, 148, 129, 129, 129, 129, 129,
686
- 148, 130, 130, 130, 130, 130, 130, 130,
687
676
  130, 130, 130, 130, 130, 130, 130, 130,
688
677
  130, 130, 130, 130, 130, 130, 130, 130,
678
+ 130, 130, 130, 130, 131, 131, 131, 131,
679
+ 131, 131, 131, 131, 131, 131, 131, 131,
680
+ 131, 131, 131, 131, 132, 132, 132, 132,
681
+ 132, 33, 33, 33, 33, 33, 33, 33,
682
+ 33, 33, 33, 33, 33, 133, 133, 133,
683
+ 133, 133, 133, 133, 133, 134, 134, 134,
684
+ 134, 134, 133, 133, 133, 133, 133, 133,
685
+ 133, 133, 133, 133, 133, 133, 133, 133,
686
+ 133, 133, 133, 133, 135, 136, 136, 137,
687
+ 138, 136, 136, 136, 139, 140, 141, 142,
688
+ 136, 136, 143, 136, 136, 136, 136, 136,
689
+ 136, 136, 136, 136, 136, 136, 136, 136,
690
+ 136, 136, 136, 144, 136, 136, 136, 136,
691
+ 136, 136, 136, 136, 136, 136, 136, 136,
692
+ 136, 136, 136, 136, 136, 136, 136, 136,
693
+ 136, 136, 136, 136, 136, 136, 136, 145,
694
+ 146, 147, 148, 136, 136, 136, 136, 136,
695
+ 136, 136, 136, 136, 136, 136, 136, 136,
696
+ 136, 136, 136, 136, 136, 136, 136, 136,
697
+ 136, 136, 136, 136, 136, 136, 136, 149,
698
+ 150, 147, 136, 133, 136, 130, 130, 130,
689
699
  130, 130, 130, 130, 130, 130, 130, 130,
690
- 148, 148, 148, 148, 148, 148, 148, 148,
691
- 148, 148, 148, 148, 148, 148, 148, 148,
692
- 148, 148, 148, 148, 148, 148, 148, 148,
693
- 148, 148, 148, 148, 148, 148, 148, 148,
694
- 148, 148, 148, 148, 148, 148, 148, 148,
695
- 148, 148, 148, 148, 148, 148, 148, 148,
696
- 148, 148, 148, 148, 148, 148, 148, 148,
697
- 148, 148, 148, 148, 148, 148, 148, 148,
698
- 148, 148, 148, 148, 148, 148, 148, 148,
699
- 148, 148, 148, 148, 148, 148, 148, 148,
700
- 148, 148, 148, 148, 148, 148, 148, 148,
701
- 148, 148, 148, 148, 148, 148, 148, 130,
702
- 148, 130, 130, 130, 130, 130, 130, 130,
703
- 130, 131, 131, 131, 131, 131, 130, 130,
704
700
  130, 130, 130, 130, 130, 130, 130, 130,
705
701
  130, 130, 130, 130, 130, 130, 130, 130,
706
- 132, 149, 149, 149, 149, 149, 149, 149,
707
- 149, 149, 149, 149, 149, 149, 149, 149,
708
- 149, 149, 149, 149, 149, 149, 149, 149,
709
- 149, 149, 149, 149, 149, 149, 149, 149,
710
- 149, 149, 149, 149, 149, 149, 149, 149,
711
- 149, 149, 149, 149, 149, 149, 149, 149,
712
- 149, 149, 149, 149, 149, 149, 149, 149,
713
- 149, 149, 149, 149, 149, 149, 149, 149,
714
- 149, 149, 149, 149, 149, 149, 149, 149,
715
- 149, 149, 149, 149, 149, 149, 149, 149,
716
- 149, 149, 149, 149, 149, 149, 149, 149,
717
- 149, 149, 149, 149, 149, 149, 149, 130,
718
- 149, 132, 132, 132, 132, 132, 149, 149,
719
- 149, 149, 149, 149, 149, 149, 149, 149,
720
- 149, 149, 149, 149, 149, 149, 149, 149,
721
- 132, 149, 133, 133, 148, 148, 133, 133,
722
- 133, 148, 148, 148, 148, 133, 133, 148,
723
- 133, 133, 133, 133, 133, 133, 133, 133,
702
+ 130, 130, 130, 151, 131, 131, 131, 131,
703
+ 131, 131, 131, 131, 131, 131, 131, 131,
704
+ 131, 131, 131, 131, 151, 132, 132, 132,
705
+ 132, 132, 151, 133, 133, 133, 133, 133,
724
706
  133, 133, 133, 133, 133, 133, 133, 133,
725
- 148, 133, 133, 133, 133, 133, 133, 133,
726
707
  133, 133, 133, 133, 133, 133, 133, 133,
727
708
  133, 133, 133, 133, 133, 133, 133, 133,
728
- 133, 133, 133, 133, 148, 148, 148, 148,
709
+ 133, 133, 151, 151, 151, 151, 151, 151,
710
+ 151, 151, 151, 151, 151, 151, 151, 151,
711
+ 151, 151, 151, 151, 151, 151, 151, 151,
712
+ 151, 151, 151, 151, 151, 151, 151, 151,
713
+ 151, 151, 151, 151, 151, 151, 151, 151,
714
+ 151, 151, 151, 151, 151, 151, 151, 151,
715
+ 151, 151, 151, 151, 151, 151, 151, 151,
716
+ 151, 151, 151, 151, 151, 151, 151, 151,
717
+ 151, 151, 151, 151, 151, 151, 151, 151,
718
+ 151, 151, 151, 151, 151, 151, 151, 151,
719
+ 151, 151, 151, 151, 151, 151, 151, 151,
720
+ 151, 151, 151, 151, 151, 151, 151, 151,
721
+ 151, 133, 151, 133, 133, 133, 133, 133,
722
+ 133, 133, 133, 134, 134, 134, 134, 134,
729
723
  133, 133, 133, 133, 133, 133, 133, 133,
730
724
  133, 133, 133, 133, 133, 133, 133, 133,
731
- 133, 133, 133, 133, 133, 133, 133, 133,
732
- 133, 133, 133, 133, 148, 148, 148, 133,
733
- 148, 151, 134, 153, 152, 12, 155, 8,
734
- 155, 155, 155, 11, 156, 154, 155, 155,
735
- 155, 155, 155, 155, 155, 155, 155, 155,
736
- 155, 155, 155, 155, 155, 155, 10, 155,
737
- 157, 12, 10, 155, 155, 155, 155, 155,
738
- 155, 155, 155, 155, 155, 155, 155, 155,
739
- 155, 155, 155, 155, 155, 155, 155, 155,
740
- 155, 155, 155, 155, 155, 155, 155, 155,
741
- 155, 155, 155, 155, 155, 155, 155, 155,
742
- 155, 155, 155, 155, 155, 155, 155, 155,
743
- 155, 155, 155, 155, 155, 155, 155, 155,
744
- 155, 155, 155, 155, 155, 155, 155, 155,
745
- 155, 155, 10, 155, 154, 155, 154, 155,
746
- 155, 155, 154, 154, 154, 155, 155, 155,
747
- 155, 155, 155, 155, 155, 155, 155, 155,
748
- 155, 155, 155, 155, 155, 158, 155, 154,
749
- 154, 154, 155, 155, 155, 155, 155, 155,
750
- 155, 155, 155, 155, 155, 155, 155, 155,
751
- 155, 155, 155, 155, 155, 155, 155, 155,
752
- 155, 155, 155, 155, 155, 155, 155, 155,
753
- 155, 155, 155, 155, 155, 155, 155, 155,
754
- 155, 155, 155, 155, 155, 155, 155, 155,
755
- 155, 155, 155, 155, 155, 155, 155, 155,
756
- 155, 155, 155, 155, 155, 155, 155, 155,
757
- 155, 154, 155, 160, 159, 159, 159, 159,
758
- 159, 159, 159, 159, 159, 159, 159, 159,
759
- 159, 159, 159, 159, 159, 159, 159, 160,
760
- 159, 162, 161, 161, 161, 161, 161, 161,
761
- 161, 161, 161, 161, 161, 161, 161, 161,
762
- 161, 161, 161, 161, 161, 162, 161, 164,
763
- 163, 163, 163, 163, 163, 163, 163, 163,
764
- 163, 163, 163, 163, 163, 163, 163, 163,
765
- 163, 163, 163, 164, 163, 166, 166, 165,
766
- 165, 165, 165, 166, 165, 165, 165, 167,
767
- 165, 165, 165, 165, 165, 165, 165, 165,
768
- 165, 165, 165, 165, 165, 165, 166, 165,
769
- 165, 165, 165, 165, 165, 165, 166, 165,
770
- 165, 165, 165, 168, 165, 165, 165, 168,
771
- 165, 165, 165, 165, 165, 165, 165, 165,
772
- 165, 165, 165, 165, 165, 165, 166, 165,
773
- 170, 169, 169, 169, 31, 31, 31, 31,
774
- 31, 31, 31, 31, 31, 31, 169, 172,
775
- 171, 171, 171, 171, 171, 171, 171, 171,
776
- 171, 171, 171, 171, 171, 171, 171, 171,
777
- 171, 171, 171, 172, 171, 173, 33, 33,
778
- 33, 173, 33, 33, 33, 33, 33, 33,
779
- 33, 33, 33, 173, 173, 33, 33, 33,
780
- 173, 173, 33, 33, 33, 33, 33, 33,
781
- 33, 33, 33, 33, 33, 173, 33, 33,
782
- 33, 173, 33, 33, 33, 33, 33, 33,
783
- 33, 33, 33, 33, 173, 33, 33, 33,
784
- 173, 33, 174, 33, 33, 33, 33, 33,
785
- 33, 33, 33, 33, 33, 33, 33, 33,
786
- 33, 33, 33, 33, 33, 33, 33, 33,
787
- 33, 33, 33, 33, 33, 33, 33, 33,
788
- 33, 33, 174, 33, 175, 175, 175, 175,
789
- 175, 175, 175, 175, 175, 175, 175, 175,
790
- 175, 175, 175, 175, 175, 175, 175, 175,
791
- 175, 175, 175, 175, 175, 175, 175, 175,
792
- 175, 175, 176, 176, 176, 176, 176, 176,
793
- 176, 176, 176, 176, 176, 176, 176, 176,
794
- 176, 176, 177, 177, 177, 177, 177, 38,
795
- 38, 38, 38, 38, 38, 38, 38, 38,
725
+ 133, 133, 135, 152, 152, 152, 152, 152,
726
+ 152, 152, 152, 152, 152, 152, 152, 152,
727
+ 152, 152, 152, 152, 152, 152, 152, 152,
728
+ 152, 152, 152, 152, 152, 152, 152, 152,
729
+ 152, 152, 152, 152, 152, 152, 152, 152,
730
+ 152, 152, 152, 152, 152, 152, 152, 152,
731
+ 152, 152, 152, 152, 152, 152, 152, 152,
732
+ 152, 152, 152, 152, 152, 152, 152, 152,
733
+ 152, 152, 152, 152, 152, 152, 152, 152,
734
+ 152, 152, 152, 152, 152, 152, 152, 152,
735
+ 152, 152, 152, 152, 152, 152, 152, 152,
736
+ 152, 152, 152, 152, 152, 152, 152, 152,
737
+ 152, 133, 152, 135, 135, 135, 135, 135,
738
+ 152, 152, 152, 152, 152, 152, 152, 152,
739
+ 152, 152, 152, 152, 152, 152, 152, 152,
740
+ 152, 152, 135, 152, 136, 136, 151, 151,
741
+ 136, 136, 136, 151, 151, 151, 151, 136,
742
+ 136, 151, 136, 136, 136, 136, 136, 136,
743
+ 136, 136, 136, 136, 136, 136, 136, 136,
744
+ 136, 136, 151, 136, 136, 136, 136, 136,
745
+ 136, 136, 136, 136, 136, 136, 136, 136,
746
+ 136, 136, 136, 136, 136, 136, 136, 136,
747
+ 136, 136, 136, 136, 136, 136, 151, 151,
748
+ 151, 151, 136, 136, 136, 136, 136, 136,
749
+ 136, 136, 136, 136, 136, 136, 136, 136,
750
+ 136, 136, 136, 136, 136, 136, 136, 136,
751
+ 136, 136, 136, 136, 136, 136, 151, 151,
752
+ 151, 136, 151, 154, 137, 156, 155, 12,
753
+ 158, 8, 158, 158, 158, 11, 159, 157,
754
+ 158, 158, 158, 158, 158, 158, 158, 158,
755
+ 158, 158, 158, 158, 158, 158, 158, 158,
756
+ 10, 158, 160, 12, 10, 158, 158, 158,
757
+ 158, 158, 158, 158, 158, 158, 158, 158,
758
+ 158, 158, 158, 158, 158, 158, 158, 158,
759
+ 158, 158, 158, 158, 158, 158, 158, 158,
760
+ 158, 158, 158, 158, 158, 158, 158, 158,
761
+ 158, 158, 158, 158, 158, 158, 158, 158,
762
+ 158, 158, 158, 158, 158, 158, 158, 158,
763
+ 158, 158, 158, 158, 158, 158, 158, 158,
764
+ 158, 158, 158, 158, 10, 158, 157, 158,
765
+ 157, 158, 158, 158, 157, 157, 157, 158,
766
+ 158, 158, 158, 158, 158, 158, 158, 158,
767
+ 158, 158, 158, 158, 158, 158, 158, 161,
768
+ 158, 157, 157, 157, 158, 158, 158, 158,
769
+ 158, 158, 158, 158, 158, 158, 158, 158,
770
+ 158, 158, 158, 158, 158, 158, 158, 158,
771
+ 158, 158, 158, 158, 158, 158, 158, 158,
772
+ 158, 158, 158, 158, 158, 158, 158, 158,
773
+ 158, 158, 158, 158, 158, 158, 158, 158,
774
+ 158, 158, 158, 158, 158, 158, 158, 158,
775
+ 158, 158, 158, 158, 158, 158, 158, 158,
776
+ 158, 158, 158, 157, 158, 163, 162, 162,
777
+ 162, 162, 162, 162, 162, 162, 162, 162,
778
+ 162, 162, 162, 162, 162, 162, 162, 162,
779
+ 162, 163, 162, 165, 164, 164, 164, 164,
780
+ 164, 164, 164, 164, 164, 164, 164, 164,
781
+ 164, 164, 164, 164, 164, 164, 164, 165,
782
+ 164, 167, 166, 166, 166, 166, 166, 166,
783
+ 166, 166, 166, 166, 166, 166, 166, 166,
784
+ 166, 166, 166, 166, 166, 167, 166, 169,
785
+ 169, 168, 168, 168, 168, 169, 168, 168,
786
+ 168, 170, 168, 168, 168, 168, 168, 168,
787
+ 168, 168, 168, 168, 168, 168, 168, 168,
788
+ 169, 168, 168, 168, 168, 168, 168, 168,
789
+ 169, 168, 168, 168, 168, 171, 168, 168,
790
+ 168, 171, 168, 168, 168, 168, 168, 168,
791
+ 168, 168, 168, 168, 168, 168, 168, 168,
792
+ 169, 168, 173, 172, 172, 172, 31, 31,
793
+ 31, 31, 31, 31, 31, 31, 31, 31,
794
+ 172, 175, 174, 174, 174, 174, 174, 174,
795
+ 174, 174, 174, 174, 174, 174, 174, 174,
796
+ 174, 174, 174, 174, 174, 175, 174, 176,
797
+ 33, 33, 33, 176, 33, 33, 33, 33,
798
+ 33, 33, 33, 33, 33, 176, 176, 33,
799
+ 33, 33, 176, 176, 33, 33, 33, 33,
800
+ 33, 33, 33, 33, 33, 33, 33, 176,
801
+ 33, 33, 33, 176, 33, 33, 33, 33,
802
+ 33, 33, 33, 33, 33, 33, 176, 33,
803
+ 33, 33, 176, 33, 177, 33, 33, 33,
804
+ 33, 33, 33, 33, 33, 33, 33, 33,
805
+ 33, 33, 33, 33, 33, 33, 33, 33,
806
+ 33, 33, 33, 33, 33, 33, 33, 33,
807
+ 33, 33, 33, 33, 177, 33, 178, 178,
808
+ 178, 178, 178, 178, 178, 178, 178, 178,
809
+ 178, 178, 178, 178, 178, 178, 178, 178,
810
+ 178, 178, 178, 178, 178, 178, 178, 178,
811
+ 178, 178, 178, 178, 179, 179, 179, 179,
812
+ 179, 179, 179, 179, 179, 179, 179, 179,
813
+ 179, 179, 179, 179, 180, 180, 180, 180,
814
+ 180, 38, 38, 38, 38, 38, 38, 38,
796
815
  38, 38, 38, 38, 38, 38, 38, 38,
797
816
  38, 38, 38, 38, 38, 38, 38, 38,
798
817
  38, 38, 38, 38, 38, 38, 38, 38,
799
818
  38, 38, 38, 38, 38, 38, 38, 38,
800
- 38, 38, 38, 38, 38, 38, 178, 38,
801
- 179, 38, 178, 178, 178, 178, 38, 180,
802
- 178, 38, 38, 38, 38, 38, 38, 38,
803
819
  38, 38, 38, 38, 38, 38, 38, 38,
804
- 38, 178, 38, 38, 38, 38, 38, 38,
820
+ 181, 38, 182, 38, 181, 181, 181, 181,
821
+ 38, 183, 181, 38, 38, 38, 38, 38,
805
822
  38, 38, 38, 38, 38, 38, 38, 38,
823
+ 38, 38, 38, 181, 38, 38, 38, 38,
806
824
  38, 38, 38, 38, 38, 38, 38, 38,
807
- 38, 38, 38, 38, 38, 181, 182, 183,
808
- 184, 38, 38, 38, 38, 38, 38, 38,
809
825
  38, 38, 38, 38, 38, 38, 38, 38,
826
+ 38, 38, 38, 38, 38, 38, 38, 184,
827
+ 185, 186, 187, 38, 38, 38, 38, 38,
810
828
  38, 38, 38, 38, 38, 38, 38, 38,
811
- 38, 38, 38, 38, 38, 178, 178, 178,
812
829
  38, 38, 38, 38, 38, 38, 38, 38,
830
+ 38, 38, 38, 38, 38, 38, 38, 181,
831
+ 181, 181, 38, 38, 38, 38, 38, 38,
813
832
  38, 38, 38, 38, 38, 38, 38, 38,
814
833
  38, 38, 38, 38, 38, 38, 38, 38,
815
834
  38, 38, 38, 38, 38, 38, 38, 38,
@@ -817,7 +836,7 @@ self._re_scanner_indicies = [
817
836
  38, 38, 38, 38, 38, 38, 38, 38,
818
837
  38, 38, 38, 38, 38, 38, 38, 38,
819
838
  38, 38, 38, 38, 38, 38, 38, 38,
820
- 38, 185, 39, 39, 39, 39, 39, 39,
839
+ 38, 38, 38, 188, 39, 39, 39, 39,
821
840
  39, 39, 39, 39, 39, 39, 39, 39,
822
841
  39, 39, 39, 39, 39, 39, 39, 39,
823
842
  39, 39, 39, 39, 39, 39, 39, 39,
@@ -825,87 +844,124 @@ self._re_scanner_indicies = [
825
844
  39, 39, 39, 39, 39, 39, 39, 39,
826
845
  39, 39, 39, 39, 39, 39, 39, 39,
827
846
  39, 39, 39, 39, 39, 39, 39, 39,
828
- 39, 39, 185, 186, 186, 186, 186, 186,
829
- 186, 186, 186, 186, 186, 186, 186, 186,
830
- 186, 186, 186, 186, 186, 186, 186, 186,
831
- 186, 186, 186, 186, 186, 186, 186, 186,
832
- 186, 186, 186, 186, 186, 186, 186, 186,
833
- 186, 186, 186, 186, 186, 186, 186, 186,
834
- 186, 186, 186, 186, 186, 186, 186, 186,
835
- 186, 186, 186, 186, 186, 186, 186, 186,
836
- 186, 186, 186, 185, 187, 185, 189, 188,
837
- 188, 188, 188, 188, 188, 188, 188, 188,
838
- 188, 188, 188, 188, 188, 188, 188, 188,
839
- 188, 188, 188, 188, 188, 188, 188, 188,
840
- 188, 188, 188, 188, 188, 188, 188, 188,
841
- 188, 188, 188, 188, 188, 188, 188, 188,
842
- 188, 188, 188, 188, 188, 188, 188, 188,
843
- 188, 188, 188, 188, 188, 190, 188, 193,
844
- 192, 192, 192, 192, 192, 192, 192, 192,
845
- 192, 192, 192, 194, 192, 192, 195, 192,
846
- 197, 197, 197, 197, 197, 197, 197, 197,
847
- 197, 197, 196, 196, 196, 196, 196, 196,
848
- 196, 197, 197, 197, 196, 196, 196, 197,
849
- 196, 196, 196, 197, 196, 197, 196, 196,
850
- 196, 196, 197, 196, 196, 196, 196, 196,
851
- 197, 196, 197, 196, 196, 196, 196, 196,
852
- 196, 196, 196, 197, 196, 196, 196, 197,
853
- 196, 196, 196, 197, 196, 196, 196, 196,
854
- 196, 196, 196, 196, 196, 196, 196, 196,
855
- 196, 196, 197, 196, 199, 198, 198, 198,
856
- 199, 199, 199, 199, 198, 198, 199, 198,
857
- 200, 201, 201, 201, 201, 201, 201, 201,
858
- 202, 202, 198, 198, 198, 198, 198, 199,
859
- 198, 33, 33, 203, 204, 198, 198, 33,
860
- 204, 198, 198, 33, 198, 205, 198, 198,
861
- 206, 198, 204, 204, 198, 198, 198, 204,
862
- 204, 198, 33, 199, 199, 199, 199, 198,
863
- 198, 207, 207, 98, 204, 207, 207, 198,
864
- 204, 198, 198, 198, 198, 198, 207, 198,
865
- 206, 198, 207, 204, 207, 208, 207, 204,
866
- 209, 198, 33, 199, 199, 199, 198, 211,
867
- 211, 211, 211, 211, 211, 211, 211, 210,
868
- 213, 213, 213, 213, 213, 213, 213, 213,
869
- 212, 215, 99, 217, 216, 99, 219, 104,
870
- 104, 104, 104, 104, 104, 104, 104, 104,
871
- 104, 104, 104, 104, 104, 104, 104, 104,
872
- 104, 104, 104, 104, 104, 104, 104, 104,
873
- 104, 104, 104, 104, 104, 104, 220, 104,
874
- 222, 221, 104, 107, 104, 225, 225, 225,
875
- 225, 225, 225, 225, 225, 225, 225, 224,
876
- 224, 224, 224, 224, 224, 224, 225, 225,
877
- 225, 225, 225, 225, 224, 224, 224, 224,
878
- 224, 224, 224, 224, 224, 224, 224, 224,
879
- 224, 224, 224, 224, 224, 224, 224, 224,
880
- 224, 224, 224, 224, 224, 224, 225, 225,
881
- 225, 225, 225, 225, 224, 227, 226, 226,
882
- 226, 226, 226, 228, 226, 226, 226, 229,
883
- 229, 229, 229, 229, 229, 229, 229, 229,
884
- 226, 226, 230, 226, 123, 231, 231, 231,
885
- 231, 231, 231, 231, 231, 124, 124, 124,
886
- 124, 124, 124, 124, 124, 124, 124, 231,
887
- 231, 231, 231, 231, 231, 231, 124, 124,
888
- 124, 124, 124, 124, 124, 124, 124, 124,
889
- 124, 124, 124, 124, 124, 124, 124, 124,
890
- 124, 124, 124, 124, 124, 124, 124, 124,
891
- 231, 231, 231, 231, 124, 231, 124, 124,
892
- 124, 124, 124, 124, 124, 124, 124, 124,
893
- 124, 124, 124, 124, 124, 124, 124, 124,
894
- 124, 124, 124, 124, 124, 124, 124, 124,
895
- 231, 125, 125, 125, 125, 125, 125, 125,
896
- 125, 125, 231, 122, 231, 231, 231, 231,
897
- 231, 231, 125, 125, 125, 125, 125, 125,
898
- 125, 125, 125, 125, 231, 126, 126, 126,
899
- 126, 126, 126, 126, 126, 126, 126, 231,
900
- 231, 231, 231, 123, 231, 231, 126, 126,
901
- 126, 126, 126, 126, 126, 126, 126, 126,
902
- 126, 126, 126, 126, 126, 126, 126, 126,
903
- 126, 126, 126, 126, 126, 126, 126, 126,
904
- 231, 231, 231, 231, 126, 231, 126, 126,
905
- 126, 126, 126, 126, 126, 126, 126, 126,
906
- 126, 126, 126, 126, 126, 126, 126, 126,
907
- 126, 126, 126, 126, 126, 126, 126, 126,
908
- 231, 0
847
+ 39, 39, 39, 39, 188, 189, 189, 189,
848
+ 189, 189, 189, 189, 189, 189, 189, 189,
849
+ 189, 189, 189, 189, 189, 189, 189, 189,
850
+ 189, 189, 189, 189, 189, 189, 189, 189,
851
+ 189, 189, 189, 189, 189, 189, 189, 189,
852
+ 189, 189, 189, 189, 189, 189, 189, 189,
853
+ 189, 189, 189, 189, 189, 189, 189, 189,
854
+ 189, 189, 189, 189, 189, 189, 189, 189,
855
+ 189, 189, 189, 189, 189, 188, 190, 188,
856
+ 192, 191, 191, 191, 191, 191, 191, 191,
857
+ 191, 191, 191, 191, 191, 191, 191, 191,
858
+ 191, 191, 191, 191, 191, 191, 191, 191,
859
+ 191, 191, 191, 191, 191, 191, 191, 191,
860
+ 191, 191, 191, 191, 191, 191, 191, 191,
861
+ 191, 191, 191, 191, 191, 191, 191, 191,
862
+ 191, 191, 191, 191, 191, 191, 191, 193,
863
+ 191, 196, 195, 195, 195, 195, 195, 195,
864
+ 195, 195, 195, 195, 195, 197, 195, 195,
865
+ 198, 195, 200, 200, 200, 200, 200, 200,
866
+ 200, 200, 200, 200, 199, 199, 199, 199,
867
+ 199, 199, 199, 200, 200, 200, 199, 199,
868
+ 199, 200, 199, 199, 199, 200, 199, 200,
869
+ 199, 199, 199, 199, 200, 199, 199, 199,
870
+ 199, 199, 200, 199, 200, 199, 199, 199,
871
+ 199, 199, 199, 199, 199, 200, 199, 199,
872
+ 199, 200, 199, 199, 199, 200, 199, 199,
873
+ 199, 199, 199, 199, 199, 199, 199, 199,
874
+ 199, 199, 199, 199, 200, 199, 201, 201,
875
+ 201, 201, 201, 201, 201, 201, 201, 201,
876
+ 201, 201, 201, 201, 201, 201, 201, 201,
877
+ 201, 201, 201, 201, 201, 201, 201, 201,
878
+ 201, 201, 201, 201, 202, 202, 202, 202,
879
+ 202, 202, 202, 202, 202, 202, 202, 202,
880
+ 202, 202, 202, 202, 203, 203, 203, 203,
881
+ 203, 99, 99, 99, 99, 99, 99, 99,
882
+ 99, 99, 99, 99, 99, 99, 99, 99,
883
+ 99, 99, 99, 99, 99, 99, 99, 99,
884
+ 99, 99, 99, 99, 99, 99, 99, 99,
885
+ 99, 99, 99, 99, 99, 99, 99, 99,
886
+ 99, 99, 99, 99, 99, 99, 99, 99,
887
+ 204, 99, 99, 99, 204, 204, 204, 204,
888
+ 99, 99, 204, 99, 205, 206, 206, 206,
889
+ 206, 206, 206, 206, 207, 207, 99, 99,
890
+ 99, 99, 99, 204, 99, 33, 33, 208,
891
+ 209, 99, 99, 33, 209, 99, 99, 33,
892
+ 99, 210, 99, 99, 211, 99, 209, 209,
893
+ 99, 99, 99, 209, 209, 99, 33, 204,
894
+ 204, 204, 204, 99, 99, 212, 212, 101,
895
+ 209, 212, 212, 99, 209, 99, 99, 99,
896
+ 99, 99, 212, 99, 211, 99, 212, 209,
897
+ 212, 213, 212, 209, 214, 99, 33, 204,
898
+ 204, 204, 99, 99, 99, 99, 99, 99,
899
+ 99, 99, 99, 99, 99, 99, 99, 99,
900
+ 99, 99, 99, 99, 99, 99, 99, 99,
901
+ 99, 99, 99, 99, 99, 99, 99, 99,
902
+ 99, 99, 99, 99, 99, 99, 99, 99,
903
+ 99, 99, 99, 99, 99, 99, 99, 99,
904
+ 99, 99, 99, 99, 99, 99, 99, 99,
905
+ 99, 99, 99, 99, 99, 99, 99, 99,
906
+ 99, 99, 99, 215, 100, 100, 100, 100,
907
+ 100, 100, 100, 100, 100, 100, 100, 100,
908
+ 100, 100, 100, 100, 100, 100, 100, 100,
909
+ 100, 100, 100, 100, 100, 100, 100, 100,
910
+ 100, 100, 100, 100, 100, 100, 100, 100,
911
+ 100, 100, 100, 100, 100, 100, 100, 100,
912
+ 100, 100, 100, 100, 100, 100, 100, 100,
913
+ 100, 100, 100, 100, 100, 100, 100, 100,
914
+ 100, 100, 100, 100, 215, 216, 216, 216,
915
+ 216, 216, 216, 216, 216, 216, 216, 216,
916
+ 216, 216, 216, 216, 216, 216, 216, 216,
917
+ 216, 216, 216, 216, 216, 216, 216, 216,
918
+ 216, 216, 216, 216, 216, 216, 216, 216,
919
+ 216, 216, 216, 216, 216, 216, 216, 216,
920
+ 216, 216, 216, 216, 216, 216, 216, 216,
921
+ 216, 216, 216, 216, 216, 216, 216, 216,
922
+ 216, 216, 216, 216, 216, 215, 218, 218,
923
+ 218, 218, 218, 218, 218, 218, 217, 220,
924
+ 220, 220, 220, 220, 220, 220, 220, 219,
925
+ 222, 102, 224, 223, 102, 226, 107, 107,
926
+ 107, 107, 107, 107, 107, 107, 107, 107,
927
+ 107, 107, 107, 107, 107, 107, 107, 107,
928
+ 107, 107, 107, 107, 107, 107, 107, 107,
929
+ 107, 107, 107, 107, 107, 227, 107, 229,
930
+ 228, 107, 110, 107, 232, 232, 232, 232,
931
+ 232, 232, 232, 232, 232, 232, 231, 231,
932
+ 231, 231, 231, 231, 231, 232, 232, 232,
933
+ 232, 232, 232, 231, 231, 231, 231, 231,
934
+ 231, 231, 231, 231, 231, 231, 231, 231,
935
+ 231, 231, 231, 231, 231, 231, 231, 231,
936
+ 231, 231, 231, 231, 231, 232, 232, 232,
937
+ 232, 232, 232, 231, 234, 233, 233, 233,
938
+ 233, 233, 235, 233, 233, 233, 236, 236,
939
+ 236, 236, 236, 236, 236, 236, 236, 233,
940
+ 233, 237, 233, 126, 238, 238, 238, 238,
941
+ 238, 238, 238, 238, 127, 127, 127, 127,
942
+ 127, 127, 127, 127, 127, 127, 238, 238,
943
+ 238, 238, 238, 238, 238, 127, 127, 127,
944
+ 127, 127, 127, 127, 127, 127, 127, 127,
945
+ 127, 127, 127, 127, 127, 127, 127, 127,
946
+ 127, 127, 127, 127, 127, 127, 127, 238,
947
+ 238, 238, 238, 127, 238, 127, 127, 127,
948
+ 127, 127, 127, 127, 127, 127, 127, 127,
949
+ 127, 127, 127, 127, 127, 127, 127, 127,
950
+ 127, 127, 127, 127, 127, 127, 127, 238,
951
+ 128, 128, 128, 128, 128, 128, 128, 128,
952
+ 128, 238, 125, 238, 238, 238, 238, 238,
953
+ 238, 128, 128, 128, 128, 128, 128, 128,
954
+ 128, 128, 128, 238, 129, 129, 129, 129,
955
+ 129, 129, 129, 129, 129, 129, 238, 238,
956
+ 238, 238, 126, 238, 238, 129, 129, 129,
957
+ 129, 129, 129, 129, 129, 129, 129, 129,
958
+ 129, 129, 129, 129, 129, 129, 129, 129,
959
+ 129, 129, 129, 129, 129, 129, 129, 238,
960
+ 238, 238, 238, 129, 238, 129, 129, 129,
961
+ 129, 129, 129, 129, 129, 129, 129, 129,
962
+ 129, 129, 129, 129, 129, 129, 129, 129,
963
+ 129, 129, 129, 129, 129, 129, 129, 238,
964
+ 0
909
965
  ]
910
966
 
911
967
  class << self
@@ -913,35 +969,36 @@ class << self
913
969
  private :_re_scanner_trans_targs, :_re_scanner_trans_targs=
914
970
  end
915
971
  self._re_scanner_trans_targs = [
916
- 110, 111, 3, 112, 5, 6, 113, 110,
917
- 7, 110, 110, 8, 110, 10, 110, 12,
918
- 18, 110, 13, 15, 17, 14, 16, 19,
919
- 21, 23, 20, 22, 110, 25, 127, 26,
920
- 28, 0, 29, 30, 129, 130, 130, 31,
921
- 130, 130, 130, 130, 35, 36, 130, 38,
972
+ 112, 113, 3, 114, 5, 6, 115, 112,
973
+ 7, 112, 112, 8, 112, 10, 112, 12,
974
+ 18, 112, 13, 15, 17, 14, 16, 19,
975
+ 21, 23, 20, 22, 112, 25, 129, 26,
976
+ 28, 0, 29, 30, 131, 132, 132, 31,
977
+ 132, 132, 132, 132, 35, 36, 132, 38,
922
978
  39, 50, 54, 58, 62, 66, 70, 75,
923
979
  79, 81, 84, 40, 47, 41, 45, 42,
924
- 43, 44, 130, 46, 48, 49, 51, 52,
980
+ 43, 44, 132, 46, 48, 49, 51, 52,
925
981
  53, 55, 56, 57, 59, 60, 61, 63,
926
982
  64, 65, 67, 68, 69, 71, 73, 72,
927
983
  74, 76, 77, 78, 80, 82, 83, 86,
928
- 87, 130, 89, 138, 141, 138, 143, 92,
929
- 138, 144, 138, 146, 95, 98, 96, 97,
930
- 138, 99, 100, 101, 102, 103, 104, 138,
931
- 148, 149, 149, 106, 107, 108, 109, 1,
932
- 2, 4, 114, 115, 116, 117, 118, 110,
933
- 119, 110, 122, 123, 110, 124, 110, 125,
934
- 110, 110, 126, 110, 110, 110, 110, 110,
935
- 110, 120, 110, 121, 110, 9, 110, 110,
936
- 110, 110, 110, 110, 110, 110, 110, 110,
937
- 11, 110, 24, 110, 110, 128, 27, 131,
938
- 132, 133, 130, 134, 135, 136, 130, 130,
939
- 130, 130, 32, 130, 130, 33, 130, 130,
940
- 130, 34, 37, 85, 137, 137, 138, 138,
941
- 139, 139, 138, 88, 138, 91, 138, 138,
942
- 94, 105, 138, 140, 138, 138, 138, 142,
943
- 138, 90, 138, 145, 147, 138, 93, 138,
944
- 138, 138, 149, 150, 151, 152, 153, 149
984
+ 87, 132, 140, 140, 88, 91, 140, 146,
985
+ 140, 148, 94, 140, 149, 140, 151, 97,
986
+ 100, 98, 99, 140, 101, 102, 103, 104,
987
+ 105, 106, 140, 153, 154, 154, 108, 109,
988
+ 110, 111, 1, 2, 4, 116, 117, 118,
989
+ 119, 120, 112, 121, 112, 124, 125, 112,
990
+ 126, 112, 127, 112, 112, 128, 112, 112,
991
+ 112, 112, 112, 112, 122, 112, 123, 112,
992
+ 9, 112, 112, 112, 112, 112, 112, 112,
993
+ 112, 112, 112, 11, 112, 24, 112, 112,
994
+ 130, 27, 133, 134, 135, 132, 136, 137,
995
+ 138, 132, 132, 132, 132, 32, 132, 132,
996
+ 33, 132, 132, 132, 34, 37, 85, 139,
997
+ 139, 141, 142, 143, 140, 144, 144, 140,
998
+ 90, 140, 93, 140, 140, 96, 107, 140,
999
+ 89, 140, 145, 140, 140, 140, 147, 140,
1000
+ 92, 140, 150, 152, 140, 95, 140, 140,
1001
+ 140, 154, 155, 156, 157, 158, 154
945
1002
  ]
946
1003
 
947
1004
  class << self
@@ -961,23 +1018,24 @@ self._re_scanner_trans_actions = [
961
1018
  0, 0, 0, 0, 0, 0, 0, 0,
962
1019
  0, 0, 0, 0, 0, 0, 0, 0,
963
1020
  0, 0, 0, 0, 0, 0, 0, 0,
964
- 0, 21, 0, 23, 0, 24, 0, 0,
965
- 25, 0, 26, 0, 0, 0, 0, 0,
966
- 27, 0, 0, 0, 0, 0, 0, 28,
967
- 0, 29, 30, 0, 0, 0, 0, 0,
968
- 0, 0, 0, 0, 0, 0, 0, 33,
969
- 34, 35, 0, 0, 36, 0, 37, 38,
970
- 39, 40, 38, 41, 42, 43, 44, 45,
971
- 46, 47, 48, 0, 49, 0, 50, 51,
972
- 52, 53, 54, 55, 56, 57, 58, 59,
973
- 0, 60, 0, 61, 62, 64, 0, 0,
974
- 38, 38, 65, 0, 38, 66, 67, 68,
975
- 69, 70, 0, 71, 72, 0, 73, 74,
976
- 75, 0, 0, 0, 76, 77, 78, 79,
977
- 80, 81, 82, 0, 83, 0, 84, 85,
978
- 0, 0, 86, 0, 87, 88, 89, 38,
979
- 90, 0, 91, 38, 0, 92, 0, 93,
980
- 94, 95, 96, 38, 38, 38, 38, 97
1021
+ 0, 21, 22, 23, 0, 0, 25, 0,
1022
+ 26, 0, 0, 27, 0, 28, 0, 0,
1023
+ 0, 0, 0, 29, 0, 0, 0, 0,
1024
+ 0, 0, 30, 0, 31, 32, 0, 0,
1025
+ 0, 0, 0, 0, 0, 0, 0, 0,
1026
+ 0, 0, 35, 36, 37, 0, 0, 38,
1027
+ 0, 39, 40, 41, 42, 40, 43, 44,
1028
+ 45, 46, 47, 48, 49, 50, 0, 51,
1029
+ 0, 52, 53, 54, 55, 56, 57, 58,
1030
+ 59, 60, 61, 0, 62, 0, 63, 64,
1031
+ 66, 0, 0, 40, 40, 67, 0, 40,
1032
+ 68, 69, 70, 71, 72, 0, 73, 74,
1033
+ 0, 75, 76, 77, 0, 0, 0, 78,
1034
+ 79, 0, 40, 40, 80, 81, 82, 83,
1035
+ 0, 84, 0, 85, 86, 0, 0, 87,
1036
+ 0, 88, 0, 89, 90, 91, 40, 92,
1037
+ 0, 93, 40, 0, 94, 0, 95, 96,
1038
+ 97, 98, 40, 40, 40, 40, 99
981
1039
  ]
982
1040
 
983
1041
  class << self
@@ -998,13 +1056,13 @@ self._re_scanner_to_state_actions = [
998
1056
  0, 0, 0, 0, 0, 0, 0, 0,
999
1057
  0, 0, 0, 0, 0, 0, 0, 0,
1000
1058
  0, 0, 0, 0, 0, 0, 0, 0,
1001
- 0, 0, 0, 0, 0, 0, 31, 0,
1002
1059
  0, 0, 0, 0, 0, 0, 0, 0,
1060
+ 33, 0, 0, 0, 0, 0, 0, 0,
1003
1061
  0, 0, 0, 0, 0, 0, 0, 0,
1004
- 63, 63, 63, 0, 0, 0, 0, 0,
1005
- 0, 63, 63, 0, 0, 0, 0, 0,
1006
- 0, 0, 0, 0, 0, 63, 0, 0,
1007
- 0, 0
1062
+ 0, 0, 65, 65, 65, 0, 0, 0,
1063
+ 0, 0, 0, 65, 65, 0, 0, 0,
1064
+ 0, 0, 0, 0, 0, 0, 0, 0,
1065
+ 0, 0, 65, 0, 0, 0, 0
1008
1066
  ]
1009
1067
 
1010
1068
  class << self
@@ -1025,13 +1083,13 @@ self._re_scanner_from_state_actions = [
1025
1083
  0, 0, 0, 0, 0, 0, 0, 0,
1026
1084
  0, 0, 0, 0, 0, 0, 0, 0,
1027
1085
  0, 0, 0, 0, 0, 0, 0, 0,
1028
- 0, 0, 0, 0, 0, 0, 32, 0,
1029
1086
  0, 0, 0, 0, 0, 0, 0, 0,
1087
+ 34, 0, 0, 0, 0, 0, 0, 0,
1088
+ 0, 0, 0, 0, 0, 0, 0, 0,
1089
+ 0, 0, 34, 34, 34, 0, 0, 0,
1090
+ 0, 0, 0, 34, 34, 0, 0, 0,
1030
1091
  0, 0, 0, 0, 0, 0, 0, 0,
1031
- 32, 32, 32, 0, 0, 0, 0, 0,
1032
- 0, 32, 32, 0, 0, 0, 0, 0,
1033
- 0, 0, 0, 0, 0, 32, 0, 0,
1034
- 0, 0
1092
+ 0, 0, 34, 0, 0, 0, 0
1035
1093
  ]
1036
1094
 
1037
1095
  class << self
@@ -1050,15 +1108,15 @@ self._re_scanner_eof_actions = [
1050
1108
  0, 0, 0, 0, 0, 0, 0, 0,
1051
1109
  0, 0, 0, 0, 0, 0, 0, 0,
1052
1110
  0, 0, 0, 0, 0, 0, 0, 0,
1053
- 22, 22, 0, 22, 22, 0, 22, 22,
1054
- 22, 22, 22, 22, 22, 22, 22, 22,
1055
- 22, 22, 0, 0, 0, 0, 0, 0,
1111
+ 0, 0, 24, 24, 0, 24, 24, 0,
1112
+ 24, 24, 24, 24, 24, 24, 24, 24,
1113
+ 24, 24, 24, 24, 0, 0, 0, 0,
1056
1114
  0, 0, 0, 0, 0, 0, 0, 0,
1057
1115
  0, 0, 0, 0, 0, 0, 0, 0,
1058
- 0, 0, 22, 0, 0, 0, 0, 0,
1059
- 0, 0, 22, 0, 0, 0, 0, 0,
1116
+ 0, 0, 0, 0, 24, 0, 0, 0,
1117
+ 0, 0, 0, 0, 24, 0, 0, 0,
1060
1118
  0, 0, 0, 0, 0, 0, 0, 0,
1061
- 0, 0
1119
+ 0, 0, 0, 0, 0, 0, 0
1062
1120
  ]
1063
1121
 
1064
1122
  class << self
@@ -1077,25 +1135,25 @@ self._re_scanner_eof_trans = [
1077
1135
  43, 43, 43, 43, 43, 43, 43, 43,
1078
1136
  43, 43, 43, 43, 43, 43, 43, 43,
1079
1137
  43, 43, 43, 43, 43, 43, 43, 43,
1080
- 0, 0, 102, 0, 0, 107, 0, 0,
1138
+ 99, 99, 0, 0, 105, 0, 0, 110,
1081
1139
  0, 0, 0, 0, 0, 0, 0, 0,
1082
- 0, 0, 122, 122, 122, 122, 0, 149,
1083
- 149, 149, 149, 150, 150, 149, 151, 153,
1084
- 155, 155, 160, 162, 164, 166, 170, 172,
1085
- 0, 0, 0, 186, 186, 186, 186, 189,
1086
- 192, 0, 0, 211, 213, 215, 215, 215,
1087
- 219, 219, 219, 219, 224, 0, 232, 232,
1088
- 232, 232
1140
+ 0, 0, 0, 0, 125, 125, 125, 125,
1141
+ 0, 152, 152, 152, 152, 153, 153, 152,
1142
+ 154, 156, 158, 158, 163, 165, 167, 169,
1143
+ 173, 175, 0, 0, 0, 189, 189, 189,
1144
+ 189, 192, 195, 0, 0, 216, 216, 216,
1145
+ 218, 220, 222, 222, 222, 226, 226, 226,
1146
+ 226, 231, 0, 239, 239, 239, 239
1089
1147
  ]
1090
1148
 
1091
1149
  class << self
1092
1150
  attr_accessor :re_scanner_start
1093
1151
  end
1094
- self.re_scanner_start = 110;
1152
+ self.re_scanner_start = 112;
1095
1153
  class << self
1096
1154
  attr_accessor :re_scanner_first_final
1097
1155
  end
1098
- self.re_scanner_first_final = 110;
1156
+ self.re_scanner_first_final = 112;
1099
1157
  class << self
1100
1158
  attr_accessor :re_scanner_error
1101
1159
  end
@@ -1104,36 +1162,36 @@ self.re_scanner_error = 0;
1104
1162
  class << self
1105
1163
  attr_accessor :re_scanner_en_char_type
1106
1164
  end
1107
- self.re_scanner_en_char_type = 128;
1165
+ self.re_scanner_en_char_type = 130;
1108
1166
  class << self
1109
1167
  attr_accessor :re_scanner_en_unicode_property
1110
1168
  end
1111
- self.re_scanner_en_unicode_property = 129;
1169
+ self.re_scanner_en_unicode_property = 131;
1112
1170
  class << self
1113
1171
  attr_accessor :re_scanner_en_character_set
1114
1172
  end
1115
- self.re_scanner_en_character_set = 130;
1173
+ self.re_scanner_en_character_set = 132;
1116
1174
  class << self
1117
1175
  attr_accessor :re_scanner_en_set_escape_sequence
1118
1176
  end
1119
- self.re_scanner_en_set_escape_sequence = 137;
1177
+ self.re_scanner_en_set_escape_sequence = 139;
1120
1178
  class << self
1121
1179
  attr_accessor :re_scanner_en_escape_sequence
1122
1180
  end
1123
- self.re_scanner_en_escape_sequence = 138;
1181
+ self.re_scanner_en_escape_sequence = 140;
1124
1182
  class << self
1125
1183
  attr_accessor :re_scanner_en_conditional_expression
1126
1184
  end
1127
- self.re_scanner_en_conditional_expression = 149;
1185
+ self.re_scanner_en_conditional_expression = 154;
1128
1186
  class << self
1129
1187
  attr_accessor :re_scanner_en_main
1130
1188
  end
1131
- self.re_scanner_en_main = 110;
1189
+ self.re_scanner_en_main = 112;
1132
1190
 
1133
1191
 
1134
- # line 765 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1192
+ # line 768 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1135
1193
 
1136
- # line 1136 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1194
+ # line 1194 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1137
1195
  begin
1138
1196
  p ||= 0
1139
1197
  pe ||= data.length
@@ -1144,9 +1202,9 @@ begin
1144
1202
  act = 0
1145
1203
  end
1146
1204
 
1147
- # line 766 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1205
+ # line 769 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1148
1206
 
1149
- # line 1149 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1207
+ # line 1207 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1150
1208
  begin
1151
1209
  testEof = false
1152
1210
  _slen, _trans, _keys, _inds, _acts, _nacts = nil
@@ -1169,12 +1227,12 @@ begin
1169
1227
  end
1170
1228
  if _goto_level <= _resume
1171
1229
  case _re_scanner_from_state_actions[cs]
1172
- when 32 then
1230
+ when 34 then
1173
1231
  # line 1 "NONE"
1174
1232
  begin
1175
1233
  ts = p
1176
1234
  end
1177
- # line 1177 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1235
+ # line 1235 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
1178
1236
  end
1179
1237
  _keys = cs << 1
1180
1238
  _inds = _re_scanner_index_offsets[cs]
@@ -1193,7 +1251,7 @@ ts = p
1193
1251
  cs = _re_scanner_trans_targs[_trans]
1194
1252
  if _re_scanner_trans_actions[_trans] != 0
1195
1253
  case _re_scanner_trans_actions[_trans]
1196
- when 34 then
1254
+ when 36 then
1197
1255
  # line 149 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1198
1256
  begin
1199
1257
  self.group_depth = group_depth + 1 end
@@ -1201,27 +1259,27 @@ ts = p
1201
1259
  # line 150 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1202
1260
  begin
1203
1261
  self.group_depth = group_depth - 1 end
1204
- when 38 then
1262
+ when 40 then
1205
1263
  # line 1 "NONE"
1206
1264
  begin
1207
1265
  te = p+1
1208
1266
  end
1209
- when 64 then
1267
+ when 66 then
1210
1268
  # line 12 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/char_type.rl"
1211
1269
  begin
1212
1270
  te = p+1
1213
1271
  begin
1214
- case text = text(data, ts, te, 1).first
1215
- when '\d'; emit(:type, :digit, text, ts - 1, te)
1216
- when '\D'; emit(:type, :nondigit, text, ts - 1, te)
1217
- when '\h'; emit(:type, :hex, text, ts - 1, te)
1218
- when '\H'; emit(:type, :nonhex, text, ts - 1, te)
1219
- when '\s'; emit(:type, :space, text, ts - 1, te)
1220
- when '\S'; emit(:type, :nonspace, text, ts - 1, te)
1221
- when '\w'; emit(:type, :word, text, ts - 1, te)
1222
- when '\W'; emit(:type, :nonword, text, ts - 1, te)
1223
- when '\R'; emit(:type, :linebreak, text, ts - 1, te)
1224
- when '\X'; emit(:type, :xgrapheme, text, ts - 1, te)
1272
+ case text = copy(data, ts-1, te)
1273
+ when '\d'; emit(:type, :digit, text)
1274
+ when '\D'; emit(:type, :nondigit, text)
1275
+ when '\h'; emit(:type, :hex, text)
1276
+ when '\H'; emit(:type, :nonhex, text)
1277
+ when '\s'; emit(:type, :space, text)
1278
+ when '\S'; emit(:type, :nonspace, text)
1279
+ when '\w'; emit(:type, :word, text)
1280
+ when '\W'; emit(:type, :nonword, text)
1281
+ when '\R'; emit(:type, :linebreak, text)
1282
+ when '\X'; emit(:type, :xgrapheme, text)
1225
1283
  end
1226
1284
  begin
1227
1285
  top -= 1
@@ -1237,7 +1295,7 @@ te = p+1
1237
1295
  begin
1238
1296
  te = p+1
1239
1297
  begin
1240
- text = text(data, ts, te, 1).first
1298
+ text = copy(data, ts-1, te)
1241
1299
  type = (text[1] == 'P') ^ (text[3] == '^') ? :nonproperty : :property
1242
1300
 
1243
1301
  name = data[ts+2..te-2].pack('c*').gsub(/[\^\s_\-]/, '').downcase
@@ -1245,7 +1303,7 @@ te = p+1
1245
1303
  token = self.class.short_prop_map[name] || self.class.long_prop_map[name]
1246
1304
  raise UnknownUnicodePropertyError.new(name) unless token
1247
1305
 
1248
- self.emit(type, token.to_sym, text, ts-1, te)
1306
+ self.emit(type, token.to_sym, text)
1249
1307
 
1250
1308
  begin
1251
1309
  top -= 1
@@ -1261,32 +1319,32 @@ te = p+1
1261
1319
  begin
1262
1320
  te = p+1
1263
1321
  begin # special case, emits two tokens
1264
- emit(:literal, :literal, '-', ts, te)
1265
- emit(:set, :intersection, '&&', ts, te)
1322
+ emit(:literal, :literal, '-')
1323
+ emit(:set, :intersection, '&&')
1266
1324
  end
1267
1325
  end
1268
- when 69 then
1326
+ when 71 then
1269
1327
  # line 182 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1270
1328
  begin
1271
1329
  te = p+1
1272
1330
  begin
1273
- text = text(data, ts, te).first
1331
+ text = copy(data, ts, te)
1274
1332
  if tokens.last[1] == :open
1275
- emit(:set, :negate, text, ts, te)
1333
+ emit(:set, :negate, text)
1276
1334
  else
1277
- emit(:literal, :literal, text, ts, te)
1335
+ emit(:literal, :literal, text)
1278
1336
  end
1279
1337
  end
1280
1338
  end
1281
- when 71 then
1339
+ when 73 then
1282
1340
  # line 203 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1283
1341
  begin
1284
1342
  te = p+1
1285
1343
  begin
1286
- emit(:set, :intersection, *text(data, ts, te))
1344
+ emit(:set, :intersection, copy(data, ts, te))
1287
1345
  end
1288
1346
  end
1289
- when 67 then
1347
+ when 69 then
1290
1348
  # line 207 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1291
1349
  begin
1292
1350
  te = p+1
@@ -1294,19 +1352,19 @@ te = p+1
1294
1352
  begin
1295
1353
  stack[top] = cs
1296
1354
  top+= 1
1297
- cs = 137
1355
+ cs = 139
1298
1356
  _goto_level = _again
1299
1357
  next
1300
1358
  end
1301
1359
 
1302
1360
  end
1303
1361
  end
1304
- when 65 then
1362
+ when 67 then
1305
1363
  # line 237 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1306
1364
  begin
1307
1365
  te = p+1
1308
1366
  begin
1309
- emit(:literal, :literal, *text(data, ts, te))
1367
+ emit(:literal, :literal, copy(data, ts, te))
1310
1368
  end
1311
1369
  end
1312
1370
  when 14 then
@@ -1314,49 +1372,47 @@ te = p+1
1314
1372
  begin
1315
1373
  te = p+1
1316
1374
  begin
1317
- char, *rest = *text(data, ts, te)
1318
- char.force_encoding('utf-8') if char.respond_to?(:force_encoding)
1319
- emit(:literal, :literal, char, *rest)
1375
+ text = copy(data, ts, te)
1376
+ emit(:literal, :literal, text)
1320
1377
  end
1321
1378
  end
1322
- when 72 then
1379
+ when 74 then
1323
1380
  # line 191 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1324
1381
  begin
1325
1382
  te = p
1326
1383
  p = p - 1; begin
1327
- text = text(data, ts, te).first
1384
+ text = copy(data, ts, te)
1328
1385
  # ranges cant start with a subset or intersection/negation/range operator
1329
1386
  if tokens.last[0] == :set
1330
- emit(:literal, :literal, text, ts, te)
1387
+ emit(:literal, :literal, text)
1331
1388
  else
1332
- emit(:set, :range, text, ts, te)
1389
+ emit(:set, :range, text)
1333
1390
  end
1334
1391
  end
1335
1392
  end
1336
- when 75 then
1393
+ when 77 then
1337
1394
  # line 211 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1338
1395
  begin
1339
1396
  te = p
1340
1397
  p = p - 1; begin
1341
- emit(:set, :open, *text(data, ts, te))
1398
+ emit(:set, :open, copy(data, ts, te))
1342
1399
  begin
1343
1400
  stack[top] = cs
1344
1401
  top+= 1
1345
- cs = 130
1402
+ cs = 132
1346
1403
  _goto_level = _again
1347
1404
  next
1348
1405
  end
1349
1406
 
1350
1407
  end
1351
1408
  end
1352
- when 70 then
1409
+ when 72 then
1353
1410
  # line 245 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1354
1411
  begin
1355
1412
  te = p
1356
1413
  p = p - 1; begin
1357
- char, *rest = *text(data, ts, te)
1358
- char.force_encoding('utf-8') if char.respond_to?(:force_encoding)
1359
- emit(:literal, :literal, char, *rest)
1414
+ text = copy(data, ts, te)
1415
+ emit(:literal, :literal, text)
1360
1416
  end
1361
1417
  end
1362
1418
  when 15 then
@@ -1364,12 +1420,12 @@ p = p - 1; begin
1364
1420
  begin
1365
1421
  begin p = ((te))-1; end
1366
1422
  begin
1367
- text = text(data, ts, te).first
1423
+ text = copy(data, ts, te)
1368
1424
  # ranges cant start with a subset or intersection/negation/range operator
1369
1425
  if tokens.last[0] == :set
1370
- emit(:literal, :literal, text, ts, te)
1426
+ emit(:literal, :literal, text)
1371
1427
  else
1372
- emit(:set, :range, text, ts, te)
1428
+ emit(:set, :range, text)
1373
1429
  end
1374
1430
  end
1375
1431
  end
@@ -1378,11 +1434,11 @@ p = p - 1; begin
1378
1434
  begin
1379
1435
  begin p = ((te))-1; end
1380
1436
  begin
1381
- emit(:set, :open, *text(data, ts, te))
1437
+ emit(:set, :open, copy(data, ts, te))
1382
1438
  begin
1383
1439
  stack[top] = cs
1384
1440
  top+= 1
1385
- cs = 130
1441
+ cs = 132
1386
1442
  _goto_level = _again
1387
1443
  next
1388
1444
  end
@@ -1394,17 +1450,16 @@ p = p - 1; begin
1394
1450
  begin
1395
1451
  begin p = ((te))-1; end
1396
1452
  begin
1397
- char, *rest = *text(data, ts, te)
1398
- char.force_encoding('utf-8') if char.respond_to?(:force_encoding)
1399
- emit(:literal, :literal, char, *rest)
1453
+ text = copy(data, ts, te)
1454
+ emit(:literal, :literal, text)
1400
1455
  end
1401
1456
  end
1402
- when 77 then
1403
- # line 255 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1457
+ when 79 then
1458
+ # line 254 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1404
1459
  begin
1405
1460
  te = p+1
1406
1461
  begin
1407
- emit(:escape, :literal, *text(data, ts, te, 1))
1462
+ emit(:escape, :literal, copy(data, ts-1, te))
1408
1463
  begin
1409
1464
  top -= 1
1410
1465
  cs = stack[top]
@@ -1414,30 +1469,30 @@ te = p+1
1414
1469
 
1415
1470
  end
1416
1471
  end
1417
- when 76 then
1418
- # line 260 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1472
+ when 78 then
1473
+ # line 259 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1419
1474
  begin
1420
1475
  te = p+1
1421
1476
  begin
1422
1477
  p = p - 1;
1423
- cs = 130;
1478
+ cs = 132;
1424
1479
  begin
1425
1480
  stack[top] = cs
1426
1481
  top+= 1
1427
- cs = 138
1482
+ cs = 140
1428
1483
  _goto_level = _again
1429
1484
  next
1430
1485
  end
1431
1486
 
1432
1487
  end
1433
1488
  end
1434
- when 82 then
1435
- # line 271 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1489
+ when 83 then
1490
+ # line 270 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1436
1491
  begin
1437
1492
  te = p+1
1438
1493
  begin
1439
- text = text(data, ts, te, 1).first
1440
- emit(:backref, :number, text, ts-1, te)
1494
+ text = copy(data, ts-1, te)
1495
+ emit(:backref, :number, text)
1441
1496
  begin
1442
1497
  top -= 1
1443
1498
  cs = stack[top]
@@ -1447,12 +1502,12 @@ te = p+1
1447
1502
 
1448
1503
  end
1449
1504
  end
1450
- when 88 then
1451
- # line 277 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1505
+ when 90 then
1506
+ # line 276 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1452
1507
  begin
1453
1508
  te = p+1
1454
1509
  begin
1455
- emit(:escape, :octal, *text(data, ts, te, 1))
1510
+ emit(:escape, :octal, copy(data, ts-1, te))
1456
1511
  begin
1457
1512
  top -= 1
1458
1513
  cs = stack[top]
@@ -1462,27 +1517,27 @@ te = p+1
1462
1517
 
1463
1518
  end
1464
1519
  end
1465
- when 79 then
1466
- # line 282 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1520
+ when 80 then
1521
+ # line 281 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1467
1522
  begin
1468
1523
  te = p+1
1469
1524
  begin
1470
- case text = text(data, ts, te, 1).first
1471
- when '\.'; emit(:escape, :dot, text, ts-1, te)
1472
- when '\|'; emit(:escape, :alternation, text, ts-1, te)
1473
- when '\^'; emit(:escape, :bol, text, ts-1, te)
1474
- when '\$'; emit(:escape, :eol, text, ts-1, te)
1475
- when '\?'; emit(:escape, :zero_or_one, text, ts-1, te)
1476
- when '\*'; emit(:escape, :zero_or_more, text, ts-1, te)
1477
- when '\+'; emit(:escape, :one_or_more, text, ts-1, te)
1478
- when '\('; emit(:escape, :group_open, text, ts-1, te)
1479
- when '\)'; emit(:escape, :group_close, text, ts-1, te)
1480
- when '\{'; emit(:escape, :interval_open, text, ts-1, te)
1481
- when '\}'; emit(:escape, :interval_close, text, ts-1, te)
1482
- when '\['; emit(:escape, :set_open, text, ts-1, te)
1483
- when '\]'; emit(:escape, :set_close, text, ts-1, te)
1525
+ case text = copy(data, ts-1, te)
1526
+ when '\.'; emit(:escape, :dot, text)
1527
+ when '\|'; emit(:escape, :alternation, text)
1528
+ when '\^'; emit(:escape, :bol, text)
1529
+ when '\$'; emit(:escape, :eol, text)
1530
+ when '\?'; emit(:escape, :zero_or_one, text)
1531
+ when '\*'; emit(:escape, :zero_or_more, text)
1532
+ when '\+'; emit(:escape, :one_or_more, text)
1533
+ when '\('; emit(:escape, :group_open, text)
1534
+ when '\)'; emit(:escape, :group_close, text)
1535
+ when '\{'; emit(:escape, :interval_open, text)
1536
+ when '\}'; emit(:escape, :interval_close, text)
1537
+ when '\['; emit(:escape, :set_open, text)
1538
+ when '\]'; emit(:escape, :set_close, text)
1484
1539
  when "\\\\";
1485
- emit(:escape, :backslash, text, ts-1, te)
1540
+ emit(:escape, :backslash, text)
1486
1541
  end
1487
1542
  begin
1488
1543
  top -= 1
@@ -1493,22 +1548,22 @@ te = p+1
1493
1548
 
1494
1549
  end
1495
1550
  end
1496
- when 85 then
1497
- # line 303 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1551
+ when 86 then
1552
+ # line 302 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1498
1553
  begin
1499
1554
  te = p+1
1500
1555
  begin
1501
1556
  # \b is emitted as backspace only when inside a character set, otherwise
1502
1557
  # it is a word boundary anchor. A syntax might "normalize" it if needed.
1503
- case text = text(data, ts, te, 1).first
1504
- when '\a'; emit(:escape, :bell, text, ts-1, te)
1505
- when '\b'; emit(:escape, :backspace, text, ts-1, te)
1506
- when '\e'; emit(:escape, :escape, text, ts-1, te)
1507
- when '\f'; emit(:escape, :form_feed, text, ts-1, te)
1508
- when '\n'; emit(:escape, :newline, text, ts-1, te)
1509
- when '\r'; emit(:escape, :carriage, text, ts-1, te)
1510
- when '\t'; emit(:escape, :tab, text, ts-1, te)
1511
- when '\v'; emit(:escape, :vertical_tab, text, ts-1, te)
1558
+ case text = copy(data, ts-1, te)
1559
+ when '\a'; emit(:escape, :bell, text)
1560
+ when '\b'; emit(:escape, :backspace, text)
1561
+ when '\e'; emit(:escape, :escape, text)
1562
+ when '\f'; emit(:escape, :form_feed, text)
1563
+ when '\n'; emit(:escape, :newline, text)
1564
+ when '\r'; emit(:escape, :carriage, text)
1565
+ when '\t'; emit(:escape, :tab, text)
1566
+ when '\v'; emit(:escape, :vertical_tab, text)
1512
1567
  end
1513
1568
  begin
1514
1569
  top -= 1
@@ -1519,16 +1574,16 @@ te = p+1
1519
1574
 
1520
1575
  end
1521
1576
  end
1522
- when 27 then
1523
- # line 319 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1577
+ when 29 then
1578
+ # line 318 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1524
1579
  begin
1525
1580
  te = p+1
1526
1581
  begin
1527
- text = text(data, ts, te, 1).first
1582
+ text = copy(data, ts-1, te)
1528
1583
  if text[2].chr == '{'
1529
- emit(:escape, :codepoint_list, text, ts-1, te)
1584
+ emit(:escape, :codepoint_list, text)
1530
1585
  else
1531
- emit(:escape, :codepoint, text, ts-1, te)
1586
+ emit(:escape, :codepoint, text)
1532
1587
  end
1533
1588
  begin
1534
1589
  top -= 1
@@ -1539,12 +1594,12 @@ te = p+1
1539
1594
 
1540
1595
  end
1541
1596
  end
1542
- when 95 then
1543
- # line 329 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1597
+ when 97 then
1598
+ # line 328 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1544
1599
  begin
1545
1600
  te = p+1
1546
1601
  begin
1547
- emit(:escape, :hex, *text(data, ts, te, 1))
1602
+ emit(:escape, :hex, copy(data, ts-1, te))
1548
1603
  begin
1549
1604
  top -= 1
1550
1605
  cs = stack[top]
@@ -1554,8 +1609,8 @@ te = p+1
1554
1609
 
1555
1610
  end
1556
1611
  end
1557
- when 23 then
1558
- # line 338 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1612
+ when 25 then
1613
+ # line 337 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1559
1614
  begin
1560
1615
  te = p+1
1561
1616
  begin
@@ -1569,8 +1624,8 @@ te = p+1
1569
1624
 
1570
1625
  end
1571
1626
  end
1572
- when 25 then
1573
- # line 343 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1627
+ when 27 then
1628
+ # line 342 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1574
1629
  begin
1575
1630
  te = p+1
1576
1631
  begin
@@ -1584,46 +1639,46 @@ te = p+1
1584
1639
 
1585
1640
  end
1586
1641
  end
1587
- when 83 then
1588
- # line 348 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1642
+ when 84 then
1643
+ # line 347 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1589
1644
  begin
1590
1645
  te = p+1
1591
1646
  begin
1592
1647
  p = p - 1;
1593
- cs = ((in_set? ? 130 : 110));
1648
+ cs = ((in_set? ? 132 : 112));
1594
1649
  begin
1595
1650
  stack[top] = cs
1596
1651
  top+= 1
1597
- cs = 128
1652
+ cs = 130
1598
1653
  _goto_level = _again
1599
1654
  next
1600
1655
  end
1601
1656
 
1602
1657
  end
1603
1658
  end
1604
- when 84 then
1605
- # line 354 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1659
+ when 85 then
1660
+ # line 353 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1606
1661
  begin
1607
1662
  te = p+1
1608
1663
  begin
1609
1664
  p = p - 1;
1610
- cs = ((in_set? ? 130 : 110));
1665
+ cs = ((in_set? ? 132 : 112));
1611
1666
  begin
1612
1667
  stack[top] = cs
1613
1668
  top+= 1
1614
- cs = 129
1669
+ cs = 131
1615
1670
  _goto_level = _again
1616
1671
  next
1617
1672
  end
1618
1673
 
1619
1674
  end
1620
1675
  end
1621
- when 78 then
1622
- # line 360 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1676
+ when 23 then
1677
+ # line 362 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1623
1678
  begin
1624
1679
  te = p+1
1625
1680
  begin
1626
- emit(:escape, :literal, *text(data, ts, te, 1))
1681
+ emit(:escape, :literal, copy(data, ts-1, te))
1627
1682
  begin
1628
1683
  top -= 1
1629
1684
  cs = stack[top]
@@ -1633,12 +1688,12 @@ te = p+1
1633
1688
 
1634
1689
  end
1635
1690
  end
1636
- when 87 then
1637
- # line 277 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1691
+ when 89 then
1692
+ # line 276 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1638
1693
  begin
1639
1694
  te = p
1640
1695
  p = p - 1; begin
1641
- emit(:escape, :octal, *text(data, ts, te, 1))
1696
+ emit(:escape, :octal, copy(data, ts-1, te))
1642
1697
  begin
1643
1698
  top -= 1
1644
1699
  cs = stack[top]
@@ -1648,12 +1703,12 @@ p = p - 1; begin
1648
1703
 
1649
1704
  end
1650
1705
  end
1651
- when 94 then
1652
- # line 329 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1706
+ when 96 then
1707
+ # line 328 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1653
1708
  begin
1654
1709
  te = p
1655
1710
  p = p - 1; begin
1656
- emit(:escape, :hex, *text(data, ts, te, 1))
1711
+ emit(:escape, :hex, copy(data, ts-1, te))
1657
1712
  begin
1658
1713
  top -= 1
1659
1714
  cs = stack[top]
@@ -1663,8 +1718,8 @@ p = p - 1; begin
1663
1718
 
1664
1719
  end
1665
1720
  end
1666
- when 90 then
1667
- # line 338 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1721
+ when 92 then
1722
+ # line 337 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1668
1723
  begin
1669
1724
  te = p
1670
1725
  p = p - 1; begin
@@ -1678,8 +1733,8 @@ p = p - 1; begin
1678
1733
 
1679
1734
  end
1680
1735
  end
1681
- when 92 then
1682
- # line 343 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1736
+ when 94 then
1737
+ # line 342 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1683
1738
  begin
1684
1739
  te = p
1685
1740
  p = p - 1; begin
@@ -1693,15 +1748,45 @@ p = p - 1; begin
1693
1748
 
1694
1749
  end
1695
1750
  end
1696
- when 86 then
1751
+ when 87 then
1752
+ # line 362 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1753
+ begin
1754
+ te = p
1755
+ p = p - 1; begin
1756
+ emit(:escape, :literal, copy(data, ts-1, te))
1757
+ begin
1758
+ top -= 1
1759
+ cs = stack[top]
1760
+ _goto_level = _again
1761
+ next
1762
+ end
1763
+
1764
+ end
1765
+ end
1766
+ when 22 then
1767
+ # line 362 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1768
+ begin
1769
+ begin p = ((te))-1; end
1770
+ begin
1771
+ emit(:escape, :literal, copy(data, ts-1, te))
1772
+ begin
1773
+ top -= 1
1774
+ cs = stack[top]
1775
+ _goto_level = _again
1776
+ next
1777
+ end
1778
+
1779
+ end
1780
+ end
1781
+ when 88 then
1697
1782
  # line 1 "NONE"
1698
1783
  begin
1699
1784
  case act
1700
1785
  when 18 then
1701
1786
  begin begin p = ((te))-1; end
1702
1787
 
1703
- text = text(data, ts, te, 1).first
1704
- emit(:backref, :number, text, ts-1, te)
1788
+ text = copy(data, ts-1, te)
1789
+ emit(:backref, :number, text)
1705
1790
  begin
1706
1791
  top -= 1
1707
1792
  cs = stack[top]
@@ -1713,7 +1798,7 @@ p = p - 1; begin
1713
1798
  when 19 then
1714
1799
  begin begin p = ((te))-1; end
1715
1800
 
1716
- emit(:escape, :octal, *text(data, ts, te, 1))
1801
+ emit(:escape, :octal, copy(data, ts-1, te))
1717
1802
  begin
1718
1803
  top -= 1
1719
1804
  cs = stack[top]
@@ -1724,18 +1809,18 @@ p = p - 1; begin
1724
1809
  end
1725
1810
  end
1726
1811
  end
1727
- when 30 then
1728
- # line 370 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1812
+ when 32 then
1813
+ # line 372 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1729
1814
  begin
1730
1815
  te = p+1
1731
1816
  begin
1732
- text = text(data, ts, te-1).first
1733
- emit(:conditional, :condition, text, ts, te-1)
1734
- emit(:conditional, :condition_close, ')', te-1, te)
1817
+ text = copy(data, ts, te-1)
1818
+ emit(:conditional, :condition, text)
1819
+ emit(:conditional, :condition_close, ')')
1735
1820
  end
1736
1821
  end
1737
- when 96 then
1738
- # line 376 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1822
+ when 98 then
1823
+ # line 378 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1739
1824
  begin
1740
1825
  te = p+1
1741
1826
  begin
@@ -1743,15 +1828,15 @@ te = p+1
1743
1828
  begin
1744
1829
  stack[top] = cs
1745
1830
  top+= 1
1746
- cs = 110
1831
+ cs = 112
1747
1832
  _goto_level = _again
1748
1833
  next
1749
1834
  end
1750
1835
 
1751
1836
  end
1752
1837
  end
1753
- when 97 then
1754
- # line 376 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1838
+ when 99 then
1839
+ # line 378 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1755
1840
  begin
1756
1841
  te = p
1757
1842
  p = p - 1; begin
@@ -1759,15 +1844,15 @@ p = p - 1; begin
1759
1844
  begin
1760
1845
  stack[top] = cs
1761
1846
  top+= 1
1762
- cs = 110
1847
+ cs = 112
1763
1848
  _goto_level = _again
1764
1849
  next
1765
1850
  end
1766
1851
 
1767
1852
  end
1768
1853
  end
1769
- when 29 then
1770
- # line 376 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1854
+ when 31 then
1855
+ # line 378 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1771
1856
  begin
1772
1857
  begin p = ((te))-1; end
1773
1858
  begin
@@ -1775,265 +1860,265 @@ p = p - 1; begin
1775
1860
  begin
1776
1861
  stack[top] = cs
1777
1862
  top+= 1
1778
- cs = 110
1863
+ cs = 112
1779
1864
  _goto_level = _again
1780
1865
  next
1781
1866
  end
1782
1867
 
1783
1868
  end
1784
1869
  end
1785
- when 36 then
1786
- # line 389 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1870
+ when 38 then
1871
+ # line 391 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1787
1872
  begin
1788
1873
  te = p+1
1789
1874
  begin
1790
- emit(:meta, :dot, *text(data, ts, te))
1875
+ emit(:meta, :dot, copy(data, ts, te))
1791
1876
  end
1792
1877
  end
1793
- when 41 then
1794
- # line 393 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1878
+ when 43 then
1879
+ # line 395 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1795
1880
  begin
1796
1881
  te = p+1
1797
1882
  begin
1798
1883
  if conditional_stack.last == group_depth
1799
- emit(:conditional, :separator, *text(data, ts, te))
1884
+ emit(:conditional, :separator, copy(data, ts, te))
1800
1885
  else
1801
- emit(:meta, :alternation, *text(data, ts, te))
1886
+ emit(:meta, :alternation, copy(data, ts, te))
1802
1887
  end
1803
1888
  end
1804
1889
  end
1805
- when 40 then
1806
- # line 403 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1890
+ when 42 then
1891
+ # line 405 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1807
1892
  begin
1808
1893
  te = p+1
1809
1894
  begin
1810
- emit(:anchor, :bol, *text(data, ts, te))
1895
+ emit(:anchor, :bol, copy(data, ts, te))
1811
1896
  end
1812
1897
  end
1813
- when 33 then
1814
- # line 407 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1898
+ when 35 then
1899
+ # line 409 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1815
1900
  begin
1816
1901
  te = p+1
1817
1902
  begin
1818
- emit(:anchor, :eol, *text(data, ts, te))
1903
+ emit(:anchor, :eol, copy(data, ts, te))
1819
1904
  end
1820
1905
  end
1821
- when 59 then
1822
- # line 411 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1906
+ when 61 then
1907
+ # line 413 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1823
1908
  begin
1824
1909
  te = p+1
1825
1910
  begin
1826
- emit(:keep, :mark, *text(data, ts, te))
1911
+ emit(:keep, :mark, copy(data, ts, te))
1827
1912
  end
1828
1913
  end
1829
- when 58 then
1830
- # line 415 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1914
+ when 60 then
1915
+ # line 417 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1831
1916
  begin
1832
1917
  te = p+1
1833
1918
  begin
1834
- case text = text(data, ts, te).first
1835
- when '\\A'; emit(:anchor, :bos, text, ts, te)
1836
- when '\\z'; emit(:anchor, :eos, text, ts, te)
1837
- when '\\Z'; emit(:anchor, :eos_ob_eol, text, ts, te)
1838
- when '\\b'; emit(:anchor, :word_boundary, text, ts, te)
1839
- when '\\B'; emit(:anchor, :nonword_boundary, text, ts, te)
1840
- when '\\G'; emit(:anchor, :match_start, text, ts, te)
1919
+ case text = copy(data, ts, te)
1920
+ when '\\A'; emit(:anchor, :bos, text)
1921
+ when '\\z'; emit(:anchor, :eos, text)
1922
+ when '\\Z'; emit(:anchor, :eos_ob_eol, text)
1923
+ when '\\b'; emit(:anchor, :word_boundary, text)
1924
+ when '\\B'; emit(:anchor, :nonword_boundary, text)
1925
+ when '\\G'; emit(:anchor, :match_start, text)
1841
1926
  end
1842
1927
  end
1843
1928
  end
1844
- when 39 then
1845
- # line 426 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1929
+ when 41 then
1930
+ # line 428 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1846
1931
  begin
1847
1932
  te = p+1
1848
1933
  begin
1849
1934
  append_literal(data, ts, te)
1850
1935
  end
1851
1936
  end
1852
- when 49 then
1853
- # line 441 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1937
+ when 51 then
1938
+ # line 443 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1854
1939
  begin
1855
1940
  te = p+1
1856
1941
  begin
1857
- text = text(data, ts, te).first
1942
+ text = copy(data, ts, te)
1858
1943
 
1859
1944
  conditional_stack << group_depth
1860
1945
 
1861
- emit(:conditional, :open, text[0..-2], ts, te-1)
1862
- emit(:conditional, :condition_open, '(', te-1, te)
1946
+ emit(:conditional, :open, text[0..-2])
1947
+ emit(:conditional, :condition_open, '(')
1863
1948
  begin
1864
1949
  stack[top] = cs
1865
1950
  top+= 1
1866
- cs = 149
1951
+ cs = 154
1867
1952
  _goto_level = _again
1868
1953
  next
1869
1954
  end
1870
1955
 
1871
1956
  end
1872
1957
  end
1873
- when 50 then
1874
- # line 472 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1958
+ when 52 then
1959
+ # line 474 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1875
1960
  begin
1876
1961
  te = p+1
1877
1962
  begin
1878
- text = text(data, ts, te).first
1963
+ text = copy(data, ts, te)
1879
1964
  if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
1880
1965
  raise InvalidGroupOption.new($1 || "-#{$2}", text)
1881
1966
  end
1882
- emit_options(text, ts, te)
1967
+ emit_options(text)
1883
1968
  end
1884
1969
  end
1885
1970
  when 7 then
1886
- # line 486 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1971
+ # line 488 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1887
1972
  begin
1888
1973
  te = p+1
1889
1974
  begin
1890
- case text = text(data, ts, te).first
1891
- when '(?='; emit(:assertion, :lookahead, text, ts, te)
1892
- when '(?!'; emit(:assertion, :nlookahead, text, ts, te)
1893
- when '(?<='; emit(:assertion, :lookbehind, text, ts, te)
1894
- when '(?<!'; emit(:assertion, :nlookbehind, text, ts, te)
1975
+ case text = copy(data, ts, te)
1976
+ when '(?='; emit(:assertion, :lookahead, text)
1977
+ when '(?!'; emit(:assertion, :nlookahead, text)
1978
+ when '(?<='; emit(:assertion, :lookbehind, text)
1979
+ when '(?<!'; emit(:assertion, :nlookbehind, text)
1895
1980
  end
1896
1981
  end
1897
1982
  end
1898
1983
  when 6 then
1899
- # line 503 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1984
+ # line 505 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1900
1985
  begin
1901
1986
  te = p+1
1902
1987
  begin
1903
- case text = text(data, ts, te).first
1904
- when '(?:'; emit(:group, :passive, text, ts, te)
1905
- when '(?>'; emit(:group, :atomic, text, ts, te)
1906
- when '(?~'; emit(:group, :absence, text, ts, te)
1988
+ case text = copy(data, ts, te)
1989
+ when '(?:'; emit(:group, :passive, text)
1990
+ when '(?>'; emit(:group, :atomic, text)
1991
+ when '(?~'; emit(:group, :absence, text)
1907
1992
 
1908
1993
  when /^\(\?(?:<>|'')/
1909
1994
  validation_error(:group, 'named group', 'name is empty')
1910
1995
 
1911
1996
  when /^\(\?<\w*>/
1912
- emit(:group, :named_ab, text, ts, te)
1997
+ emit(:group, :named_ab, text)
1913
1998
 
1914
1999
  when /^\(\?'\w*'/
1915
- emit(:group, :named_sq, text, ts, te)
2000
+ emit(:group, :named_sq, text)
1916
2001
 
1917
2002
  end
1918
2003
  end
1919
2004
  end
1920
2005
  when 9 then
1921
- # line 544 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2006
+ # line 546 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1922
2007
  begin
1923
2008
  te = p+1
1924
2009
  begin
1925
- case text = text(data, ts, te).first
2010
+ case text = copy(data, ts, te)
1926
2011
  when /^\\([gk])(<>|'')/ # angle brackets
1927
2012
  validation_error(:backref, 'ref/call', 'ref ID is empty')
1928
2013
 
1929
2014
  when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
1930
2015
  if $1 == 'k'
1931
- emit(:backref, :name_ref_ab, text, ts, te)
2016
+ emit(:backref, :name_ref_ab, text)
1932
2017
  else
1933
- emit(:backref, :name_call_ab, text, ts, te)
2018
+ emit(:backref, :name_call_ab, text)
1934
2019
  end
1935
2020
 
1936
2021
  when /^\\([gk])'[^\d+-]\w*'/ #single quotes
1937
2022
  if $1 == 'k'
1938
- emit(:backref, :name_ref_sq, text, ts, te)
2023
+ emit(:backref, :name_ref_sq, text)
1939
2024
  else
1940
- emit(:backref, :name_call_sq, text, ts, te)
2025
+ emit(:backref, :name_call_sq, text)
1941
2026
  end
1942
2027
 
1943
2028
  when /^\\([gk])<\d+>/ # angle-brackets
1944
2029
  if $1 == 'k'
1945
- emit(:backref, :number_ref_ab, text, ts, te)
2030
+ emit(:backref, :number_ref_ab, text)
1946
2031
  else
1947
- emit(:backref, :number_call_ab, text, ts, te)
2032
+ emit(:backref, :number_call_ab, text)
1948
2033
  end
1949
2034
 
1950
2035
  when /^\\([gk])'\d+'/ # single quotes
1951
2036
  if $1 == 'k'
1952
- emit(:backref, :number_ref_sq, text, ts, te)
2037
+ emit(:backref, :number_ref_sq, text)
1953
2038
  else
1954
- emit(:backref, :number_call_sq, text, ts, te)
2039
+ emit(:backref, :number_call_sq, text)
1955
2040
  end
1956
2041
 
1957
2042
  when /^\\(?:g<\+|g<-|(k)<-)\d+>/ # angle-brackets
1958
2043
  if $1 == 'k'
1959
- emit(:backref, :number_rel_ref_ab, text, ts, te)
2044
+ emit(:backref, :number_rel_ref_ab, text)
1960
2045
  else
1961
- emit(:backref, :number_rel_call_ab, text, ts, te)
2046
+ emit(:backref, :number_rel_call_ab, text)
1962
2047
  end
1963
2048
 
1964
2049
  when /^\\(?:g'\+|g'-|(k)'-)\d+'/ # single quotes
1965
2050
  if $1 == 'k'
1966
- emit(:backref, :number_rel_ref_sq, text, ts, te)
2051
+ emit(:backref, :number_rel_ref_sq, text)
1967
2052
  else
1968
- emit(:backref, :number_rel_call_sq, text, ts, te)
2053
+ emit(:backref, :number_rel_call_sq, text)
1969
2054
  end
1970
2055
 
1971
2056
  when /^\\k<[^\d+\-]\w*[+\-]\d+>/ # angle-brackets
1972
- emit(:backref, :name_recursion_ref_ab, text, ts, te)
2057
+ emit(:backref, :name_recursion_ref_ab, text)
1973
2058
 
1974
2059
  when /^\\k'[^\d+\-]\w*[+\-]\d+'/ # single-quotes
1975
- emit(:backref, :name_recursion_ref_sq, text, ts, te)
2060
+ emit(:backref, :name_recursion_ref_sq, text)
1976
2061
 
1977
2062
  when /^\\([gk])<[+\-]?\d+[+\-]\d+>/ # angle-brackets
1978
- emit(:backref, :number_recursion_ref_ab, text, ts, te)
2063
+ emit(:backref, :number_recursion_ref_ab, text)
1979
2064
 
1980
2065
  when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
1981
- emit(:backref, :number_recursion_ref_sq, text, ts, te)
2066
+ emit(:backref, :number_recursion_ref_sq, text)
1982
2067
 
1983
2068
  end
1984
2069
  end
1985
2070
  end
1986
- when 56 then
1987
- # line 609 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2071
+ when 58 then
2072
+ # line 611 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
1988
2073
  begin
1989
2074
  te = p+1
1990
2075
  begin
1991
- case text = text(data, ts, te).first
1992
- when '?' ; emit(:quantifier, :zero_or_one, text, ts, te)
1993
- when '??'; emit(:quantifier, :zero_or_one_reluctant, text, ts, te)
1994
- when '?+'; emit(:quantifier, :zero_or_one_possessive, text, ts, te)
2076
+ case text = copy(data, ts, te)
2077
+ when '?' ; emit(:quantifier, :zero_or_one, text)
2078
+ when '??'; emit(:quantifier, :zero_or_one_reluctant, text)
2079
+ when '?+'; emit(:quantifier, :zero_or_one_possessive, text)
1995
2080
  end
1996
2081
  end
1997
2082
  end
1998
- when 52 then
1999
- # line 617 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2083
+ when 54 then
2084
+ # line 619 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2000
2085
  begin
2001
2086
  te = p+1
2002
2087
  begin
2003
- case text = text(data, ts, te).first
2004
- when '*' ; emit(:quantifier, :zero_or_more, text, ts, te)
2005
- when '*?'; emit(:quantifier, :zero_or_more_reluctant, text, ts, te)
2006
- when '*+'; emit(:quantifier, :zero_or_more_possessive, text, ts, te)
2088
+ case text = copy(data, ts, te)
2089
+ when '*' ; emit(:quantifier, :zero_or_more, text)
2090
+ when '*?'; emit(:quantifier, :zero_or_more_reluctant, text)
2091
+ when '*+'; emit(:quantifier, :zero_or_more_possessive, text)
2007
2092
  end
2008
2093
  end
2009
2094
  end
2010
- when 54 then
2011
- # line 625 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2095
+ when 56 then
2096
+ # line 627 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2012
2097
  begin
2013
2098
  te = p+1
2014
2099
  begin
2015
- case text = text(data, ts, te).first
2016
- when '+' ; emit(:quantifier, :one_or_more, text, ts, te)
2017
- when '+?'; emit(:quantifier, :one_or_more_reluctant, text, ts, te)
2018
- when '++'; emit(:quantifier, :one_or_more_possessive, text, ts, te)
2100
+ case text = copy(data, ts, te)
2101
+ when '+' ; emit(:quantifier, :one_or_more, text)
2102
+ when '+?'; emit(:quantifier, :one_or_more_reluctant, text)
2103
+ when '++'; emit(:quantifier, :one_or_more_possessive, text)
2019
2104
  end
2020
2105
  end
2021
2106
  end
2022
- when 62 then
2023
- # line 633 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2107
+ when 64 then
2108
+ # line 635 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2024
2109
  begin
2025
2110
  te = p+1
2026
2111
  begin
2027
- emit(:quantifier, :interval, *text(data, ts, te))
2112
+ emit(:quantifier, :interval, copy(data, ts, te))
2028
2113
  end
2029
2114
  end
2030
- when 45 then
2031
- # line 648 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2115
+ when 47 then
2116
+ # line 650 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2032
2117
  begin
2033
2118
  te = p+1
2034
2119
  begin
2035
2120
  if free_spacing
2036
- emit(:free_space, :comment, *text(data, ts, te))
2121
+ emit(:free_space, :comment, copy(data, ts, te))
2037
2122
  else
2038
2123
  # consume only the pound sign (#) and backtrack to do regular scanning
2039
2124
  append_literal(data, ts, ts + 1)
@@ -2042,101 +2127,101 @@ te = p+1
2042
2127
  end
2043
2128
  end
2044
2129
  end
2045
- when 48 then
2046
- # line 472 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2130
+ when 50 then
2131
+ # line 474 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2047
2132
  begin
2048
2133
  te = p
2049
2134
  p = p - 1; begin
2050
- text = text(data, ts, te).first
2135
+ text = copy(data, ts, te)
2051
2136
  if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
2052
2137
  raise InvalidGroupOption.new($1 || "-#{$2}", text)
2053
2138
  end
2054
- emit_options(text, ts, te)
2139
+ emit_options(text)
2055
2140
  end
2056
2141
  end
2057
- when 46 then
2058
- # line 521 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2142
+ when 48 then
2143
+ # line 523 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2059
2144
  begin
2060
2145
  te = p
2061
2146
  p = p - 1; begin
2062
- text = text(data, ts, te).first
2063
- emit(:group, :capture, text, ts, te)
2147
+ text = copy(data, ts, te)
2148
+ emit(:group, :capture, text)
2064
2149
  end
2065
2150
  end
2066
- when 55 then
2067
- # line 609 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2151
+ when 57 then
2152
+ # line 611 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2068
2153
  begin
2069
2154
  te = p
2070
2155
  p = p - 1; begin
2071
- case text = text(data, ts, te).first
2072
- when '?' ; emit(:quantifier, :zero_or_one, text, ts, te)
2073
- when '??'; emit(:quantifier, :zero_or_one_reluctant, text, ts, te)
2074
- when '?+'; emit(:quantifier, :zero_or_one_possessive, text, ts, te)
2156
+ case text = copy(data, ts, te)
2157
+ when '?' ; emit(:quantifier, :zero_or_one, text)
2158
+ when '??'; emit(:quantifier, :zero_or_one_reluctant, text)
2159
+ when '?+'; emit(:quantifier, :zero_or_one_possessive, text)
2075
2160
  end
2076
2161
  end
2077
2162
  end
2078
- when 51 then
2079
- # line 617 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2163
+ when 53 then
2164
+ # line 619 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2080
2165
  begin
2081
2166
  te = p
2082
2167
  p = p - 1; begin
2083
- case text = text(data, ts, te).first
2084
- when '*' ; emit(:quantifier, :zero_or_more, text, ts, te)
2085
- when '*?'; emit(:quantifier, :zero_or_more_reluctant, text, ts, te)
2086
- when '*+'; emit(:quantifier, :zero_or_more_possessive, text, ts, te)
2168
+ case text = copy(data, ts, te)
2169
+ when '*' ; emit(:quantifier, :zero_or_more, text)
2170
+ when '*?'; emit(:quantifier, :zero_or_more_reluctant, text)
2171
+ when '*+'; emit(:quantifier, :zero_or_more_possessive, text)
2087
2172
  end
2088
2173
  end
2089
2174
  end
2090
- when 53 then
2091
- # line 625 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2175
+ when 55 then
2176
+ # line 627 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2092
2177
  begin
2093
2178
  te = p
2094
2179
  p = p - 1; begin
2095
- case text = text(data, ts, te).first
2096
- when '+' ; emit(:quantifier, :one_or_more, text, ts, te)
2097
- when '+?'; emit(:quantifier, :one_or_more_reluctant, text, ts, te)
2098
- when '++'; emit(:quantifier, :one_or_more_possessive, text, ts, te)
2180
+ case text = copy(data, ts, te)
2181
+ when '+' ; emit(:quantifier, :one_or_more, text)
2182
+ when '+?'; emit(:quantifier, :one_or_more_reluctant, text)
2183
+ when '++'; emit(:quantifier, :one_or_more_possessive, text)
2099
2184
  end
2100
2185
  end
2101
2186
  end
2102
- when 61 then
2103
- # line 633 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2187
+ when 63 then
2188
+ # line 635 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2104
2189
  begin
2105
2190
  te = p
2106
2191
  p = p - 1; begin
2107
- emit(:quantifier, :interval, *text(data, ts, te))
2192
+ emit(:quantifier, :interval, copy(data, ts, te))
2108
2193
  end
2109
2194
  end
2110
- when 60 then
2111
- # line 638 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2195
+ when 62 then
2196
+ # line 640 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2112
2197
  begin
2113
2198
  te = p
2114
2199
  p = p - 1; begin
2115
2200
  append_literal(data, ts, te)
2116
2201
  end
2117
2202
  end
2118
- when 57 then
2119
- # line 644 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2203
+ when 59 then
2204
+ # line 646 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2120
2205
  begin
2121
2206
  te = p
2122
2207
  p = p - 1; begin
2123
2208
  begin
2124
2209
  stack[top] = cs
2125
2210
  top+= 1
2126
- cs = 138
2211
+ cs = 140
2127
2212
  _goto_level = _again
2128
2213
  next
2129
2214
  end
2130
2215
 
2131
2216
  end
2132
2217
  end
2133
- when 44 then
2134
- # line 648 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2218
+ when 46 then
2219
+ # line 650 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2135
2220
  begin
2136
2221
  te = p
2137
2222
  p = p - 1; begin
2138
2223
  if free_spacing
2139
- emit(:free_space, :comment, *text(data, ts, te))
2224
+ emit(:free_space, :comment, copy(data, ts, te))
2140
2225
  else
2141
2226
  # consume only the pound sign (#) and backtrack to do regular scanning
2142
2227
  append_literal(data, ts, ts + 1)
@@ -2145,20 +2230,20 @@ p = p - 1; begin
2145
2230
  end
2146
2231
  end
2147
2232
  end
2148
- when 43 then
2149
- # line 658 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2233
+ when 45 then
2234
+ # line 660 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2150
2235
  begin
2151
2236
  te = p
2152
2237
  p = p - 1; begin
2153
2238
  if free_spacing
2154
- emit(:free_space, :whitespace, *text(data, ts, te))
2239
+ emit(:free_space, :whitespace, copy(data, ts, te))
2155
2240
  else
2156
2241
  append_literal(data, ts, te)
2157
2242
  end
2158
2243
  end
2159
2244
  end
2160
- when 42 then
2161
- # line 673 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2245
+ when 44 then
2246
+ # line 675 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2162
2247
  begin
2163
2248
  te = p
2164
2249
  p = p - 1; begin
@@ -2166,19 +2251,19 @@ p = p - 1; begin
2166
2251
  end
2167
2252
  end
2168
2253
  when 3 then
2169
- # line 472 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2254
+ # line 474 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2170
2255
  begin
2171
2256
  begin p = ((te))-1; end
2172
2257
  begin
2173
- text = text(data, ts, te).first
2258
+ text = copy(data, ts, te)
2174
2259
  if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
2175
2260
  raise InvalidGroupOption.new($1 || "-#{$2}", text)
2176
2261
  end
2177
- emit_options(text, ts, te)
2262
+ emit_options(text)
2178
2263
  end
2179
2264
  end
2180
2265
  when 10 then
2181
- # line 638 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2266
+ # line 640 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2182
2267
  begin
2183
2268
  begin p = ((te))-1; end
2184
2269
  begin
@@ -2186,14 +2271,14 @@ p = p - 1; begin
2186
2271
  end
2187
2272
  end
2188
2273
  when 8 then
2189
- # line 644 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2274
+ # line 646 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2190
2275
  begin
2191
2276
  begin p = ((te))-1; end
2192
2277
  begin
2193
2278
  begin
2194
2279
  stack[top] = cs
2195
2280
  top+= 1
2196
- cs = 138
2281
+ cs = 140
2197
2282
  _goto_level = _again
2198
2283
  next
2199
2284
  end
@@ -2218,22 +2303,22 @@ end
2218
2303
  end
2219
2304
  end
2220
2305
  end
2221
- when 74 then
2306
+ when 76 then
2222
2307
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2223
2308
  begin
2224
2309
 
2225
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2310
+ text = copy(data, ts ? ts-1 : 0, -1)
2226
2311
  raise PrematureEndError.new( text )
2227
2312
  end
2228
2313
  # line 211 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2229
2314
  begin
2230
2315
  te = p
2231
2316
  p = p - 1; begin
2232
- emit(:set, :open, *text(data, ts, te))
2317
+ emit(:set, :open, copy(data, ts, te))
2233
2318
  begin
2234
2319
  stack[top] = cs
2235
2320
  top+= 1
2236
- cs = 130
2321
+ cs = 132
2237
2322
  _goto_level = _again
2238
2323
  next
2239
2324
  end
@@ -2244,36 +2329,36 @@ p = p - 1; begin
2244
2329
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2245
2330
  begin
2246
2331
 
2247
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2332
+ text = copy(data, ts ? ts-1 : 0, -1)
2248
2333
  raise PrematureEndError.new( text )
2249
2334
  end
2250
2335
  # line 211 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2251
2336
  begin
2252
2337
  begin p = ((te))-1; end
2253
2338
  begin
2254
- emit(:set, :open, *text(data, ts, te))
2339
+ emit(:set, :open, copy(data, ts, te))
2255
2340
  begin
2256
2341
  stack[top] = cs
2257
2342
  top+= 1
2258
- cs = 130
2343
+ cs = 132
2259
2344
  _goto_level = _again
2260
2345
  next
2261
2346
  end
2262
2347
 
2263
2348
  end
2264
2349
  end
2265
- when 93 then
2350
+ when 95 then
2266
2351
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2267
2352
  begin
2268
2353
 
2269
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2354
+ text = copy(data, ts ? ts-1 : 0, -1)
2270
2355
  raise PrematureEndError.new( text )
2271
2356
  end
2272
- # line 329 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2357
+ # line 328 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2273
2358
  begin
2274
2359
  te = p
2275
2360
  p = p - 1; begin
2276
- emit(:escape, :hex, *text(data, ts, te, 1))
2361
+ emit(:escape, :hex, copy(data, ts-1, te))
2277
2362
  begin
2278
2363
  top -= 1
2279
2364
  cs = stack[top]
@@ -2283,14 +2368,14 @@ p = p - 1; begin
2283
2368
 
2284
2369
  end
2285
2370
  end
2286
- when 89 then
2371
+ when 91 then
2287
2372
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2288
2373
  begin
2289
2374
 
2290
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2375
+ text = copy(data, ts ? ts-1 : 0, -1)
2291
2376
  raise PrematureEndError.new( text )
2292
2377
  end
2293
- # line 338 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2378
+ # line 337 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2294
2379
  begin
2295
2380
  te = p
2296
2381
  p = p - 1; begin
@@ -2304,14 +2389,14 @@ p = p - 1; begin
2304
2389
 
2305
2390
  end
2306
2391
  end
2307
- when 91 then
2392
+ when 93 then
2308
2393
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2309
2394
  begin
2310
2395
 
2311
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2396
+ text = copy(data, ts ? ts-1 : 0, -1)
2312
2397
  raise PrematureEndError.new( text )
2313
2398
  end
2314
- # line 343 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2399
+ # line 342 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2315
2400
  begin
2316
2401
  te = p
2317
2402
  p = p - 1; begin
@@ -2325,14 +2410,14 @@ p = p - 1; begin
2325
2410
 
2326
2411
  end
2327
2412
  end
2328
- when 24 then
2413
+ when 26 then
2329
2414
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2330
2415
  begin
2331
2416
 
2332
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2417
+ text = copy(data, ts ? ts-1 : 0, -1)
2333
2418
  raise PrematureEndError.new( text )
2334
2419
  end
2335
- # line 338 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2420
+ # line 337 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2336
2421
  begin
2337
2422
  begin p = ((te))-1; end
2338
2423
  begin
@@ -2346,14 +2431,14 @@ p = p - 1; begin
2346
2431
 
2347
2432
  end
2348
2433
  end
2349
- when 26 then
2434
+ when 28 then
2350
2435
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2351
2436
  begin
2352
2437
 
2353
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2438
+ text = copy(data, ts ? ts-1 : 0, -1)
2354
2439
  raise PrematureEndError.new( text )
2355
2440
  end
2356
- # line 343 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2441
+ # line 342 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2357
2442
  begin
2358
2443
  begin p = ((te))-1; end
2359
2444
  begin
@@ -2367,14 +2452,14 @@ p = p - 1; begin
2367
2452
 
2368
2453
  end
2369
2454
  end
2370
- when 28 then
2455
+ when 30 then
2371
2456
  # line 143 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2372
2457
  begin
2373
2458
 
2374
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2459
+ text = copy(data, ts ? ts-1 : 0, -1)
2375
2460
  validation_error(:sequence, 'sequence', text)
2376
2461
  end
2377
- # line 334 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2462
+ # line 333 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2378
2463
  begin
2379
2464
  te = p+1
2380
2465
  begin
@@ -2391,24 +2476,24 @@ te = p+1
2391
2476
  # line 150 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2392
2477
  begin
2393
2478
  self.group_depth = group_depth - 1 end
2394
- # line 457 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2479
+ # line 459 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2395
2480
  begin
2396
2481
  te = p+1
2397
2482
  begin
2398
- emit(:group, :comment, *text(data, ts, te))
2483
+ emit(:group, :comment, copy(data, ts, te))
2399
2484
  end
2400
2485
  end
2401
- when 35 then
2486
+ when 37 then
2402
2487
  # line 150 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2403
2488
  begin
2404
2489
  self.group_depth = group_depth - 1 end
2405
- # line 526 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2490
+ # line 528 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2406
2491
  begin
2407
2492
  te = p+1
2408
2493
  begin
2409
2494
  if conditional_stack.last == group_depth + 1
2410
2495
  conditional_stack.pop
2411
- emit(:conditional, :close, *text(data, ts, te))
2496
+ emit(:conditional, :close, copy(data, ts, te))
2412
2497
  else
2413
2498
  if spacing_stack.length > 1 &&
2414
2499
  spacing_stack.last[:depth] == group_depth + 1
@@ -2416,30 +2501,30 @@ te = p+1
2416
2501
  self.free_spacing = spacing_stack.last[:free_spacing]
2417
2502
  end
2418
2503
 
2419
- emit(:group, :close, *text(data, ts, te))
2504
+ emit(:group, :close, copy(data, ts, te))
2420
2505
  end
2421
2506
  end
2422
2507
  end
2423
- when 37 then
2508
+ when 39 then
2424
2509
  # line 151 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2425
2510
  begin
2426
2511
  self.set_depth = set_depth + 1 end
2427
- # line 432 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2512
+ # line 434 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2428
2513
  begin
2429
2514
  te = p+1
2430
2515
  begin
2431
- emit(:set, :open, *text(data, ts, te))
2516
+ emit(:set, :open, copy(data, ts, te))
2432
2517
  begin
2433
2518
  stack[top] = cs
2434
2519
  top+= 1
2435
- cs = 130
2520
+ cs = 132
2436
2521
  _goto_level = _again
2437
2522
  next
2438
2523
  end
2439
2524
 
2440
2525
  end
2441
2526
  end
2442
- when 68 then
2527
+ when 70 then
2443
2528
  # line 152 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2444
2529
  begin
2445
2530
  self.set_depth = set_depth - 1 end
@@ -2447,7 +2532,7 @@ te = p+1
2447
2532
  begin
2448
2533
  te = p+1
2449
2534
  begin
2450
- emit(:set, :close, *text(data, ts, te))
2535
+ emit(:set, :close, copy(data, ts, te))
2451
2536
  if in_set?
2452
2537
  begin
2453
2538
  top -= 1
@@ -2458,7 +2543,7 @@ te = p+1
2458
2543
 
2459
2544
  else
2460
2545
  begin
2461
- cs = 110
2546
+ cs = 112
2462
2547
  _goto_level = _again
2463
2548
  next
2464
2549
  end
@@ -2466,7 +2551,7 @@ te = p+1
2466
2551
  end
2467
2552
  end
2468
2553
  end
2469
- when 73 then
2554
+ when 75 then
2470
2555
  # line 152 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2471
2556
  begin
2472
2557
  self.set_depth = set_depth - 1 end
@@ -2474,8 +2559,8 @@ te = p+1
2474
2559
  begin
2475
2560
  te = p+1
2476
2561
  begin # special case, emits two tokens
2477
- emit(:literal, :literal, copy(data, ts..te-2), ts, te - 1)
2478
- emit(:set, :close, copy(data, ts+1..te-1), ts + 1, te)
2562
+ emit(:literal, :literal, copy(data, ts, te-1))
2563
+ emit(:set, :close, copy(data, ts+1, te))
2479
2564
  if in_set?
2480
2565
  begin
2481
2566
  top -= 1
@@ -2486,7 +2571,7 @@ te = p+1
2486
2571
 
2487
2572
  else
2488
2573
  begin
2489
- cs = 110
2574
+ cs = 112
2490
2575
  _goto_level = _again
2491
2576
  next
2492
2577
  end
@@ -2502,7 +2587,7 @@ te = p+1
2502
2587
  begin
2503
2588
  te = p+1
2504
2589
  begin
2505
- text = text(data, ts, te).first
2590
+ text = copy(data, ts, te)
2506
2591
 
2507
2592
  type = :posixclass
2508
2593
  class_name = text[2..-3]
@@ -2511,7 +2596,7 @@ te = p+1
2511
2596
  type = :nonposixclass
2512
2597
  end
2513
2598
 
2514
- emit(type, class_name.to_sym, text, ts, te)
2599
+ emit(type, class_name.to_sym, text)
2515
2600
  end
2516
2601
  end
2517
2602
  when 19 then
@@ -2522,7 +2607,7 @@ te = p+1
2522
2607
  begin
2523
2608
  te = p+1
2524
2609
  begin
2525
- emit(:set, :collation, *text(data, ts, te))
2610
+ emit(:set, :collation, copy(data, ts, te))
2526
2611
  end
2527
2612
  end
2528
2613
  when 21 then
@@ -2533,10 +2618,10 @@ te = p+1
2533
2618
  begin
2534
2619
  te = p+1
2535
2620
  begin
2536
- emit(:set, :equivalent, *text(data, ts, te))
2621
+ emit(:set, :equivalent, copy(data, ts, te))
2537
2622
  end
2538
2623
  end
2539
- when 66 then
2624
+ when 68 then
2540
2625
  # line 1 "NONE"
2541
2626
  begin
2542
2627
  te = p+1
@@ -2544,20 +2629,20 @@ te = p+1
2544
2629
  # line 151 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2545
2630
  begin
2546
2631
  self.set_depth = set_depth + 1 end
2547
- when 81 then
2632
+ when 82 then
2548
2633
  # line 1 "NONE"
2549
2634
  begin
2550
2635
  te = p+1
2551
2636
  end
2552
- # line 271 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2637
+ # line 270 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2553
2638
  begin
2554
2639
  act = 18; end
2555
- when 80 then
2640
+ when 81 then
2556
2641
  # line 1 "NONE"
2557
2642
  begin
2558
2643
  te = p+1
2559
2644
  end
2560
- # line 277 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2645
+ # line 276 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2561
2646
  begin
2562
2647
  act = 19; end
2563
2648
  when 2 then
@@ -2565,10 +2650,10 @@ act = 19; end
2565
2650
  begin
2566
2651
  te = p+1
2567
2652
  end
2568
- # line 673 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2653
+ # line 675 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2569
2654
  begin
2570
2655
  act = 56; end
2571
- when 47 then
2656
+ when 49 then
2572
2657
  # line 1 "NONE"
2573
2658
  begin
2574
2659
  te = p+1
@@ -2579,17 +2664,17 @@ te = p+1
2579
2664
  # line 149 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2580
2665
  begin
2581
2666
  self.group_depth = group_depth + 1 end
2582
- # line 2582 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2667
+ # line 2667 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2583
2668
  end
2584
2669
  end
2585
2670
  end
2586
2671
  if _goto_level <= _again
2587
2672
  case _re_scanner_to_state_actions[cs]
2588
- when 63 then
2673
+ when 65 then
2589
2674
  # line 1 "NONE"
2590
2675
  begin
2591
2676
  ts = nil; end
2592
- when 31 then
2677
+ when 33 then
2593
2678
  # line 1 "NONE"
2594
2679
  begin
2595
2680
  ts = nil; end
@@ -2597,7 +2682,7 @@ ts = nil; end
2597
2682
  begin
2598
2683
  act = 0
2599
2684
  end
2600
- # line 2600 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2685
+ # line 2685 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2601
2686
  end
2602
2687
 
2603
2688
  if cs == 0
@@ -2624,14 +2709,14 @@ act = 0
2624
2709
 
2625
2710
  raise PrematureEndError.new('unicode property')
2626
2711
  end
2627
- when 22 then
2712
+ when 24 then
2628
2713
  # line 137 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2629
2714
  begin
2630
2715
 
2631
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2716
+ text = copy(data, ts ? ts-1 : 0, -1)
2632
2717
  raise PrematureEndError.new( text )
2633
2718
  end
2634
- # line 2634 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2719
+ # line 2719 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner.rb"
2635
2720
  end
2636
2721
  end
2637
2722
 
@@ -2642,13 +2727,13 @@ act = 0
2642
2727
  end
2643
2728
  end
2644
2729
 
2645
- # line 767 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2730
+ # line 770 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
2646
2731
 
2647
2732
  # to avoid "warning: assigned but unused variable - testEof"
2648
2733
  testEof = testEof
2649
2734
 
2650
2735
  if cs == re_scanner_error
2651
- text = ts ? copy(data, ts-1..-1) : data.pack('c*')
2736
+ text = copy(data, ts ? ts-1 : 0, -1)
2652
2737
  raise ScannerError.new("Scan error at '#{text}'")
2653
2738
  end
2654
2739
 
@@ -2676,22 +2761,29 @@ end
2676
2761
  end
2677
2762
 
2678
2763
  # Emits an array with the details of the scanned pattern
2679
- def emit(type, token, text, ts, te)
2764
+ def emit(type, token, text)
2680
2765
  #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"
2681
2766
 
2682
2767
  emit_literal if literal
2683
2768
 
2769
+ # Ragel runs with byte-based indices (ts, te). These are of little value to
2770
+ # end-users, so we keep track of char-based indices and emit those instead.
2771
+ ts_char_pos = char_pos
2772
+ te_char_pos = char_pos + text.length
2773
+
2684
2774
  if block
2685
- block.call type, token, text, ts, te
2775
+ block.call type, token, text, ts_char_pos, te_char_pos
2686
2776
  end
2687
2777
 
2688
- tokens << [type, token, text, ts, te]
2778
+ tokens << [type, token, text, ts_char_pos, te_char_pos]
2779
+
2780
+ self.char_pos = te_char_pos
2689
2781
  end
2690
2782
 
2691
2783
  private
2692
2784
 
2693
2785
  attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
2694
- :group_depth, :set_depth, :conditional_stack
2786
+ :group_depth, :set_depth, :conditional_stack, :char_pos
2695
2787
 
2696
2788
  def free_spacing?(input_object, options)
2697
2789
  if options && !input_object.is_a?(String)
@@ -2714,36 +2806,25 @@ end
2714
2806
  end
2715
2807
 
2716
2808
  # Copy from ts to te from data as text
2717
- def copy(data, range)
2718
- data[range].pack('c*')
2719
- end
2720
-
2721
- # Copy from ts to te from data as text, returning an array with the text
2722
- # and the offsets used to copy it.
2723
- def text(data, ts, te, soff = 0)
2724
- [copy(data, ts-soff..te-1), ts-soff, te]
2809
+ def copy(data, ts, te)
2810
+ data[ts...te].pack('c*').force_encoding('utf-8')
2725
2811
  end
2726
2812
 
2727
2813
  # Appends one or more characters to the literal buffer, to be emitted later
2728
- # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
2814
+ # by a call to emit_literal.
2729
2815
  def append_literal(data, ts, te)
2730
2816
  self.literal = literal || []
2731
- literal << text(data, ts, te)
2817
+ literal << copy(data, ts, te)
2732
2818
  end
2733
2819
 
2734
- # Emits the literal run collected by calls to the append_literal method,
2735
- # using the total start (ts) and end (te) offsets of the run.
2820
+ # Emits the literal run collected by calls to the append_literal method.
2736
2821
  def emit_literal
2737
- ts, te = literal.first[1], literal.last[2]
2738
- text = literal.map {|t| t[0]}.join
2739
-
2740
- text.force_encoding('utf-8') if text.respond_to?(:force_encoding)
2741
-
2822
+ text = literal.join
2742
2823
  self.literal = nil
2743
- emit(:literal, :literal, text, ts, te)
2824
+ emit(:literal, :literal, text)
2744
2825
  end
2745
2826
 
2746
- def emit_options(text, ts, te)
2827
+ def emit_options(text)
2747
2828
  token = nil
2748
2829
 
2749
2830
  # Ruby allows things like '(?-xxxx)' or '(?xx-xx--xx-:abc)'.
@@ -2769,14 +2850,14 @@ end
2769
2850
  token = :options_switch
2770
2851
  end
2771
2852
 
2772
- emit(:group, token, text, ts, te)
2853
+ emit(:group, token, text)
2773
2854
  end
2774
2855
 
2775
2856
  def emit_meta_control_sequence(data, ts, te, token)
2776
2857
  if data.last < 0x00 || data.last > 0x7F
2777
2858
  validation_error(:sequence, 'escape', token.to_s)
2778
2859
  end
2779
- emit(:escape, token, *text(data, ts, te, 1))
2860
+ emit(:escape, token, copy(data, ts-1, te))
2780
2861
  end
2781
2862
 
2782
2863
  # Centralizes and unifies the handling of validation related