gunsnprices 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,731 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "tables"
4
+
5
+ module GunsNPrices
6
+ # Python compatibility layer.
7
+ #
8
+ # The rules (the patterns in this gem and the ~3,000 patterns in the model dictionaries) are written for Python's
9
+ # `re` module, the engine of the reference implementation, and the code around them relies on Python's string
10
+ # functions. This module reproduces both on top of Ruby's Onigmo engine:
11
+ #
12
+ # * {Rx} translates a Python pattern to Onigmo syntax: `^` is `\A`, `$` also matches before a final newline,
13
+ # `\Z` is `\z`, `{` that is not a repetition is a literal, `[` and `&&` inside a class are literals, and `\s`
14
+ # also covers U+001C..U+001F.
15
+ # * On ASCII text, Onigmo's own `\d`, `\w`, `\b` and case-insensitive matching mean the same as Python's, so each
16
+ # pattern runs natively. On other text a second form is used, with Python's digit, space and word characters
17
+ # spelled out from Python's tables and `re.IGNORECASE` expanded by hand (Onigmo would also match "ß" for "ss" and
18
+ # would not match "ı" or "İ" for "i"). Onigmo's `\b` counts combining marks and a few other characters as word
19
+ # characters and Python does not; only a text with such a character gets a third form, with `\b` written as
20
+ # look-around.
21
+ # * `lower` handles the final sigma, numbers are read from any script's digits, `round4` rounds half to even on the
22
+ # exact binary value, and `unescape` is `html.unescape` with the HTML5 entity table.
23
+ #
24
+ # Both engines backtrack with leftmost-first alternation, so the match found is the one Python finds.
25
+ #
26
+ # @api private
27
+ module Py
28
+ # Sorted, disjoint inclusive code point ranges.
29
+ class RangeSet
30
+ attr_reader :ranges
31
+
32
+ # @param ranges [Array<Array(Integer, Integer)>] inclusive ranges, in any order, may overlap
33
+ def initialize(ranges)
34
+ merged = []
35
+ ranges.sort.each do |lo, hi|
36
+ if merged.any? && lo <= merged[-1][1] + 1
37
+ merged[-1][1] = hi if hi > merged[-1][1]
38
+ else
39
+ merged << [lo, hi]
40
+ end
41
+ end
42
+ @ranges = merged.map(&:freeze).freeze
43
+ @starts = @ranges.map(&:first).freeze
44
+ freeze
45
+ end
46
+
47
+ def self.from_flat(flat)
48
+ new(flat.each_slice(2).to_a)
49
+ end
50
+
51
+ def include?(cp)
52
+ i = @starts.bsearch_index { |x| x > cp }
53
+ i = (i || @starts.size) - 1
54
+ i >= 0 && cp <= @ranges[i][1]
55
+ end
56
+
57
+ def complement
58
+ out = []
59
+ nxt = 0
60
+ @ranges.each do |lo, hi|
61
+ out << [nxt, lo - 1] if lo > nxt
62
+ nxt = hi + 1
63
+ end
64
+ out << [nxt, 0x10FFFF] if nxt <= 0x10FFFF
65
+ RangeSet.new(out)
66
+ end
67
+
68
+ def |(other)
69
+ RangeSet.new(@ranges + other.ranges)
70
+ end
71
+
72
+ # Body of an Onigmo character class (without the brackets). Surrogates can't be written in a UTF-8 pattern
73
+ # and never occur in UTF-8 text, so they are left out.
74
+ def class_body
75
+ out = +""
76
+ @ranges.each do |lo, hi|
77
+ [[lo, [hi, 0xD7FF].min], [[lo, 0xE000].max, hi]].each do |a, b|
78
+ next if a > b
79
+
80
+ out << Py.class_char(a)
81
+ next if a == b
82
+
83
+ out << "-" if b > a + 1
84
+ out << Py.class_char(b)
85
+ end
86
+ end
87
+ out
88
+ end
89
+ end
90
+
91
+ DIGITS = RangeSet.new(Tables::DIGIT_ZEROS.map { |z| [z, z + 9] })
92
+ SPACES = RangeSet.from_flat(Tables::SPACE)
93
+ WORDS = RangeSet.from_flat(Tables::WORD)
94
+ CASED = RangeSet.from_flat(Tables::CASED)
95
+ CASE_IGNORABLE = RangeSet.from_flat(Tables::CASE_IGNORABLE)
96
+
97
+ # ASCII-only forms of the classes, enough for ASCII text
98
+ ASCII_DIGITS = RangeSet.new([[0x30, 0x39]])
99
+ ASCII_SPACES = RangeSet.new([[0x09, 0x0D], [0x1C, 0x20]])
100
+ ASCII_WORDS = RangeSet.new([[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]])
101
+
102
+ # re.IGNORECASE: code point -> the code points it matches (itself included)
103
+ CASE_GROUP = Tables::CASE_GROUPS.each_with_object({}) do |g, h|
104
+ cps = g.codepoints.freeze
105
+ cps.each { |cp| h[cp] = cps }
106
+ end.freeze
107
+ CASE_MEMBERS = CASE_GROUP.keys.sort.freeze
108
+
109
+ INVALID_CODEPOINTS = Tables::INVALID_CODEPOINTS.to_h { |cp| [cp, true] }.freeze
110
+ LOWER = Tables::LOWER.to_h { |cp, s| [cp.chr(Encoding::UTF_8), s] }.freeze
111
+
112
+ class << self
113
+ # ---------- text ----------
114
+
115
+ # A title or field as a UTF-8 string (nil -> ""). Other encodings are converted; invalid bytes become U+FFFD.
116
+ def str(value)
117
+ s = value.nil? ? "" : value.to_s
118
+ if s.encoding != Encoding::UTF_8
119
+ s = if s.encoding == Encoding::BINARY || s.encoding == Encoding::US_ASCII
120
+ s.dup.force_encoding(Encoding::UTF_8)
121
+ else
122
+ s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
123
+ end
124
+ end
125
+ s.valid_encoding? ? s : s.scrub("�")
126
+ end
127
+
128
+ # Python truthiness of an optional string, number or list.
129
+ def present?(value)
130
+ !(value.nil? || value == false || (value.respond_to?(:empty?) && value.empty?) || value == 0)
131
+ end
132
+
133
+ # Python `str.lower()`: full Unicode lowercase mapping, with the final sigma ("ΑΣ" -> "ας").
134
+ def lower(s)
135
+ return s.downcase(:ascii) if s.ascii_only?
136
+ return lower_sigma(s) if s.include?("Σ")
137
+
138
+ s.downcase(:ascii).gsub(/[^\x00-\x7F]/) { |c| LOWER[c] || c }
139
+ end
140
+
141
+ # Python `str.upper()`. Only used on short matches of ASCII-letter patterns, where Ruby's mapping is the same.
142
+ def upper(s)
143
+ s.upcase
144
+ end
145
+
146
+ # Python `str.strip()`.
147
+ def strip(s)
148
+ STRIP.sub(s, "")
149
+ end
150
+
151
+ # Python `re.sub(r"\s+", " ", s)`.
152
+ def collapse_ws(s)
153
+ WS_RUN.sub(s, " ")
154
+ end
155
+
156
+ # Python `re.split(r"\s+", s)`.
157
+ def split_ws(s)
158
+ out = []
159
+ last = 0
160
+ WS_RUN.each_match(s) do |m|
161
+ out << s[last...m.begin(0)]
162
+ last = m.end(0)
163
+ end
164
+ out << s[last..]
165
+ end
166
+
167
+ # Python `re.escape` (3.7+).
168
+ def escape(s)
169
+ s.gsub(/[()\[\]{}?*+\-|^$\\.&~# \t\n\r\v\f]/) { |c| "\\#{c}" }
170
+ end
171
+
172
+ # Value of a Unicode decimal digit (what Python's `\d` matches), or nil.
173
+ def digit_value(ch)
174
+ cp = ch.ord
175
+ return cp - 0x30 if cp >= 0x30 && cp <= 0x39
176
+ return nil if cp < 0x80
177
+
178
+ i = Tables::DIGIT_ZEROS.bsearch_index { |z| z > cp }
179
+ zero = Tables::DIGIT_ZEROS[(i || Tables::DIGIT_ZEROS.size) - 1]
180
+ zero && cp >= zero && cp - zero < 10 ? cp - zero : nil
181
+ end
182
+
183
+ # Python `int(s)` for a string of decimal digits of any script.
184
+ def int(s)
185
+ return s.to_i if s.ascii_only?
186
+
187
+ s.each_char.reduce(0) { |acc, c| (acc * 10) + digit_value(c) }
188
+ end
189
+
190
+ # Python `float(s)` for digits of any script with an optional decimal point.
191
+ def float(s)
192
+ return Float(s) if s.ascii_only?
193
+
194
+ Float(s.each_char.map { |c| c == "." ? "." : digit_value(c).to_s }.join)
195
+ end
196
+
197
+ # Python `round(x, 4)`: the exact binary value of x rounded half to even at 4 decimals, then the nearest float.
198
+ def round4(x)
199
+ return x unless x.finite?
200
+
201
+ q = (x.to_r * 10_000).round(half: :even)
202
+ q.zero? && x.negative? ? -0.0 : q.fdiv(10_000)
203
+ end
204
+
205
+ # Python `html.unescape`: HTML5 named references with and without ";", the longest known prefix of an unknown
206
+ # name ("&ampx" -> "&x"), numeric references remapped or dropped the way browsers do.
207
+ def unescape(s)
208
+ return s unless s.include?("&")
209
+
210
+ s.gsub(CHARREF) { replace_charref(Regexp.last_match(1)) }
211
+ end
212
+
213
+ # ---------- patterns ----------
214
+
215
+ # Ruby source of a pattern character as a class member.
216
+ def class_char(cp)
217
+ if cp < 0x20 || cp == 0x7F
218
+ format("\\x%02X", cp)
219
+ elsif cp >= 0x80
220
+ format("\\u{%X}", cp)
221
+ elsif (cp >= 0x30 && cp <= 0x39) || (cp >= 0x41 && cp <= 0x5A) || (cp >= 0x61 && cp <= 0x7A)
222
+ cp.chr
223
+ else
224
+ "\\#{cp.chr}"
225
+ end
226
+ end
227
+
228
+ # Ruby source of a pattern character outside a class.
229
+ def literal_char(cp)
230
+ if cp < 0x20 || cp == 0x7F
231
+ format("\\x%02X", cp)
232
+ elsif cp >= 0x80
233
+ format("\\u{%X}", cp)
234
+ elsif ".^$|?*+()[]{}\\-&~#/".include?(cp.chr)
235
+ "\\#{cp.chr}"
236
+ else
237
+ cp.chr
238
+ end
239
+ end
240
+
241
+ # The characters `re.IGNORECASE` lets a class match, for an explicit set of members.
242
+ def case_expand(set)
243
+ extra = []
244
+ set.ranges.each do |lo, hi|
245
+ i = CASE_MEMBERS.bsearch_index { |x| x >= lo }
246
+ next unless i
247
+
248
+ while i < CASE_MEMBERS.size && CASE_MEMBERS[i] <= hi
249
+ CASE_GROUP[CASE_MEMBERS[i]].each { |cp| extra << [cp, cp] }
250
+ i += 1
251
+ end
252
+ end
253
+ extra.empty? ? set : RangeSet.new(set.ranges + extra)
254
+ end
255
+
256
+ # The text has a character that Onigmo's \b counts as a word character and Python's \w does not, or the other
257
+ # way round (combining marks, connector punctuation other than "_", join controls, circled letters, ...).
258
+ def word_risk?(s)
259
+ !s.ascii_only? && WORD_RISK.match?(s)
260
+ end
261
+
262
+ private
263
+
264
+ def lower_sigma(s)
265
+ cps = s.codepoints
266
+ out = +""
267
+ cps.each_with_index do |cp, i|
268
+ out << if cp == 0x3A3
269
+ final_sigma?(cps, i) ? "ς" : "σ"
270
+ elsif cp < 0x80
271
+ cp >= 0x41 && cp <= 0x5A ? (cp + 32).chr : cp.chr
272
+ else
273
+ c = cp.chr(Encoding::UTF_8)
274
+ LOWER[c] || c
275
+ end
276
+ end
277
+ out
278
+ end
279
+
280
+ # Python: U+03A3 is a final sigma when, skipping case-ignorable characters, a cased character comes before it
281
+ # and none after it.
282
+ def final_sigma?(cps, i)
283
+ j = i - 1
284
+ j -= 1 while j >= 0 && CASE_IGNORABLE.include?(cps[j])
285
+ return false unless j >= 0 && CASED.include?(cps[j])
286
+
287
+ j = i + 1
288
+ j += 1 while j < cps.size && CASE_IGNORABLE.include?(cps[j])
289
+ j == cps.size || !CASED.include?(cps[j])
290
+ end
291
+
292
+ def replace_charref(ref)
293
+ if ref.start_with?("#")
294
+ num = ref[1] == "x" || ref[1] == "X" ? ref[2..].delete_suffix(";").to_i(16) : ref[1..].delete_suffix(";").to_i
295
+ return Tables::INVALID_CHARREFS[num] if Tables::INVALID_CHARREFS.key?(num)
296
+ return "�" if (num >= 0xD800 && num <= 0xDFFF) || num > 0x10FFFF
297
+ return "" if INVALID_CODEPOINTS.key?(num)
298
+
299
+ return num.chr(Encoding::UTF_8)
300
+ end
301
+ return Tables::HTML5[ref] if Tables::HTML5.key?(ref)
302
+
303
+ # the longest known name that the reference starts with
304
+ (ref.length - 1).downto(2) do |x|
305
+ v = Tables::HTML5[ref[0, x]]
306
+ return v + ref[x..] if v
307
+ end
308
+ "&#{ref}"
309
+ end
310
+ end
311
+
312
+ CHARREF = /&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)/.freeze
313
+
314
+ # A Python-syntax regular expression, translated when created and compiled on first use.
315
+ class Rx
316
+ attr_reader :pattern
317
+
318
+ # @param pattern [String] Python `re` syntax
319
+ # @param ignore_case [Boolean] Python `re.IGNORECASE`
320
+ # @param anchor [nil, :start, :full] nil: search; :start: match at the given position (`re.match`);
321
+ # :full: match the whole string (`re.fullmatch`)
322
+ def initialize(pattern, ignore_case: false, anchor: nil)
323
+ @pattern = pattern
324
+ @ignore_case = ignore_case
325
+ @anchor = anchor
326
+ @nodes = Parser.new(pattern).parse
327
+ @has_b = @nodes.any? { |n| n == :b || n == :B }
328
+ @mutex = Mutex.new
329
+ @fast = @exact = @exact_b = nil
330
+ Rx.register(self)
331
+ end
332
+
333
+ @registry = []
334
+ @registry_mutex = Mutex.new
335
+
336
+ class << self
337
+ # Every pattern created so far (the rules are created once, when their module or the data is loaded).
338
+ def registry
339
+ @registry_mutex.synchronize { @registry.dup }
340
+ end
341
+
342
+ def register(rx)
343
+ @registry_mutex.synchronize { @registry << rx }
344
+ end
345
+ end
346
+
347
+ # Compile the form used for ASCII text (most titles) now.
348
+ def compile_fast
349
+ fast
350
+ self
351
+ end
352
+
353
+ # Compile all forms now (tests) and return self.
354
+ def compile_all
355
+ fast
356
+ exact
357
+ exact_b
358
+ self
359
+ end
360
+
361
+ # Python `rx.search(s, pos)`: MatchData or nil (positions are character offsets, like Python's).
362
+ def search(s, pos = 0)
363
+ for_text(s).match(s, pos)
364
+ end
365
+
366
+ # Python `rx.search(s) is not None`.
367
+ def match?(s)
368
+ for_text(s).match?(s)
369
+ end
370
+
371
+ # Python `rx.finditer(s)`: every non-overlapping match, as MatchData.
372
+ def each_match(s, &block)
373
+ rx = for_text(s)
374
+ return enum_for(:each_match, s) unless block
375
+
376
+ s.scan(rx) { block.call(Regexp.last_match) }
377
+ end
378
+
379
+ def matches(s)
380
+ each_match(s).to_a
381
+ end
382
+
383
+ # Python `rx.sub(repl, s)`: repl is a String (a Python template such as "\\g<1> gen") or a block.
384
+ def sub(s, repl = nil, &block)
385
+ rx = for_text(s)
386
+ if block
387
+ s.gsub(rx) { block.call(Regexp.last_match) }
388
+ elsif repl.include?("\\")
389
+ parts = Py::Template.parse(repl)
390
+ s.gsub(rx) { Py::Template.expand(parts, Regexp.last_match) }
391
+ else
392
+ s.gsub(rx) { repl }
393
+ end
394
+ end
395
+
396
+ private
397
+
398
+ def for_text(s)
399
+ return fast if s.ascii_only?
400
+ return exact unless @has_b && Py.word_risk?(s)
401
+
402
+ exact_b
403
+ end
404
+
405
+ def fast
406
+ @fast || @mutex.synchronize { @fast ||= build(:fast) }
407
+ end
408
+
409
+ def exact
410
+ @exact || @mutex.synchronize { @exact ||= build(:exact) }
411
+ end
412
+
413
+ def exact_b
414
+ return exact unless @has_b
415
+
416
+ @exact_b || @mutex.synchronize { @exact_b ||= build(:exact_b) }
417
+ end
418
+
419
+ def build(mode)
420
+ native_ic = mode == :fast && @ignore_case
421
+ body = Emitter.new(mode, @ignore_case).emit(@nodes)
422
+ src = case @anchor
423
+ when :start then "\\G(?:#{body})"
424
+ when :full then "\\A(?:#{body})\\z"
425
+ else body
426
+ end
427
+ Regexp.new(src, native_ic ? Regexp::IGNORECASE : 0).freeze
428
+ rescue RegexpError => e
429
+ raise ArgumentError, "cannot translate #{@pattern.inspect} (#{mode}): #{e.message}"
430
+ end
431
+ end
432
+
433
+ # Python replacement templates ("\\g<1> gen", "\\1", "\\\\").
434
+ module Template
435
+ ESCAPES = { "n" => "\n", "t" => "\t", "r" => "\r", "f" => "\f", "v" => "\v", "a" => "\a", "b" => "\b",
436
+ "\\" => "\\" }.freeze
437
+
438
+ module_function
439
+
440
+ # Literal strings and group numbers.
441
+ def parse(repl)
442
+ parts = []
443
+ lit = +""
444
+ i = 0
445
+ while i < repl.length
446
+ c = repl[i]
447
+ if c != "\\"
448
+ lit << c
449
+ i += 1
450
+ next
451
+ end
452
+ nxt = repl[i + 1]
453
+ raise ArgumentError, "bad escape at the end of #{repl.inspect}" if nxt.nil?
454
+
455
+ if nxt == "g" && (m = /\A\\g<(\d+)>/.match(repl[i..]))
456
+ parts << lit << m[1].to_i
457
+ lit = +""
458
+ i += m[0].length
459
+ elsif nxt.match?(/[0-9]/)
460
+ m = /\A\\(\d\d?)/.match(repl[i..])
461
+ parts << lit << m[1].to_i
462
+ lit = +""
463
+ i += m[0].length
464
+ elsif ESCAPES.key?(nxt)
465
+ lit << ESCAPES[nxt]
466
+ i += 2
467
+ elsif nxt.match?(/[A-Za-z]/)
468
+ raise ArgumentError, "bad escape \\#{nxt} in #{repl.inspect}"
469
+ else
470
+ lit << "\\" << nxt
471
+ i += 2
472
+ end
473
+ end
474
+ parts << lit
475
+ parts.reject { |p| p == "" }.freeze
476
+ end
477
+
478
+ def expand(parts, m)
479
+ parts.map { |p| p.is_a?(Integer) ? (m[p] || "") : p }.join
480
+ end
481
+ end
482
+
483
+ # Python `re` syntax -> a list of nodes: Strings are Onigmo syntax passed through, Integers are literal code
484
+ # points, Arrays are [:class, negated, [members]] or [:cat, letter]; :b, :B, :bos, :eos, :eos_nl are anchors.
485
+ class Parser
486
+ def initialize(pattern)
487
+ @s = pattern
488
+ @i = 0
489
+ end
490
+
491
+ def parse
492
+ nodes = []
493
+ while @i < @s.length
494
+ c = @s[@i]
495
+ @i += 1
496
+ case c
497
+ when "\\" then nodes << escape
498
+ when "[" then nodes << char_class
499
+ when "(" then nodes << group
500
+ when "{" then nodes << repetition(nodes)
501
+ when "^" then nodes << :bos
502
+ when "$" then nodes << :eos_nl
503
+ when "." , "|", ")", "*", "+", "?" then nodes << c
504
+ else nodes << c.ord
505
+ end
506
+ end
507
+ nodes.freeze
508
+ end
509
+
510
+ private
511
+
512
+ def fail!(why)
513
+ raise ArgumentError, "unsupported pattern #{@s.inspect}: #{why}"
514
+ end
515
+
516
+ def escape
517
+ c = @s[@i] or fail!("trailing backslash")
518
+ @i += 1
519
+ case c
520
+ when "d", "D", "w", "W", "s", "S" then [:cat, c].freeze
521
+ when "b" then :b
522
+ when "B" then :B
523
+ when "A" then :bos
524
+ when "Z" then :eos
525
+ else char_escape(c)
526
+ end
527
+ end
528
+
529
+ # an escape that stands for one character (outside or inside a class)
530
+ def char_escape(c)
531
+ case c
532
+ when "n" then 0x0A
533
+ when "t" then 0x09
534
+ when "r" then 0x0D
535
+ when "f" then 0x0C
536
+ when "v" then 0x0B
537
+ when "a" then 0x07
538
+ when "x" then hex(2)
539
+ when "u" then hex(4)
540
+ when "U" then hex(8)
541
+ when "0" then 0
542
+ when /[0-9A-Za-z]/ then fail!("escape \\#{c}")
543
+ else c.ord
544
+ end
545
+ end
546
+
547
+ def hex(n)
548
+ digits = @s[@i, n]
549
+ fail!("bad hex escape") unless digits && digits.length == n && digits.match?(/\A\h+\z/)
550
+ @i += n
551
+ digits.to_i(16)
552
+ end
553
+
554
+ def group
555
+ return "(" unless @s[@i] == "?"
556
+
557
+ rest = @s[@i + 1..]
558
+ if rest.start_with?(":", "=", "!", "<=", "<!", ">")
559
+ tok = "(?#{rest.start_with?("<") ? rest[0, 2] : rest[0]}"
560
+ @i += tok.length - 1
561
+ tok
562
+ elsif rest.start_with?("#")
563
+ close = @s.index(")", @i) or fail!("unterminated comment")
564
+ @i = close + 1
565
+ ""
566
+ else
567
+ fail!("group (?#{rest[0]}")
568
+ end
569
+ end
570
+
571
+ # Python reads "{m}", "{m,}", "{,n}", "{m,n}" as repetitions and any other "{" as a literal.
572
+ def repetition(nodes)
573
+ m = /\A(\d*)(?:(,)(\d*))?\}/.match(@s[@i..])
574
+ return "{".ord if m.nil? || (m[1].empty? && m[2].nil?)
575
+
576
+ @i += m[0].length
577
+ lo = m[1].empty? ? "0" : m[1]
578
+ fail!("repetition of nothing") if nodes.empty?
579
+ q = if m[2].nil? then "{#{lo}}"
580
+ elsif m[3].empty? then "{#{lo},}"
581
+ else "{#{lo},#{m[3]}}"
582
+ end
583
+ # Onigmo reads "{n}?" as "({n})?" and "{n}+" as "({n})+"; Python's lazy "{n}?" is just "{n}"
584
+ if m[2].nil? && @s[@i] == "?"
585
+ @i += 1
586
+ elsif @s[@i] == "+"
587
+ fail!("possessive repetition")
588
+ end
589
+ q
590
+ end
591
+
592
+ def char_class
593
+ negated = false
594
+ if @s[@i] == "^"
595
+ negated = true
596
+ @i += 1
597
+ end
598
+ members = []
599
+ first = true
600
+ loop do
601
+ c = @s[@i] or fail!("unterminated character class")
602
+ @i += 1
603
+ break if c == "]" && !first
604
+
605
+ first = false
606
+ lo = if c == "\\"
607
+ e = @s[@i] or fail!("trailing backslash")
608
+ @i += 1
609
+ if "dDwWsS".include?(e)
610
+ members << [:cat, e].freeze
611
+ next
612
+ end
613
+ e == "b" ? 0x08 : char_escape(e)
614
+ else
615
+ c.ord
616
+ end
617
+ if @s[@i] == "-" && @s[@i + 1] && @s[@i + 1] != "]"
618
+ @i += 1
619
+ d = @s[@i]
620
+ @i += 1
621
+ if d == "\\"
622
+ e = @s[@i] or fail!("trailing backslash")
623
+ @i += 1
624
+ fail!("bad range") if "dDwWsS".include?(e)
625
+ d_cp = e == "b" ? 0x08 : char_escape(e)
626
+ else
627
+ d_cp = d.ord
628
+ end
629
+ fail!("bad character range") if d_cp < lo
630
+ members << [lo, d_cp].freeze
631
+ else
632
+ members << [lo, lo].freeze
633
+ end
634
+ end
635
+ [:class, negated, members.freeze].freeze
636
+ end
637
+ end
638
+
639
+ # Nodes -> Onigmo source for one form (:fast, :exact or :exact_b).
640
+ class Emitter
641
+ def initialize(mode, ignore_case)
642
+ @mode = mode
643
+ @ic = ignore_case
644
+ end
645
+
646
+ def emit(nodes)
647
+ nodes.map { |n| node(n) }.join
648
+ end
649
+
650
+ private
651
+
652
+ def exact?
653
+ @mode != :fast
654
+ end
655
+
656
+ def node(n)
657
+ case n
658
+ when String then n
659
+ when Integer then literal(n)
660
+ when :b then @mode == :exact_b ? Emitter.word_boundary : "\\b"
661
+ when :B then @mode == :exact_b ? Emitter.not_word_boundary : "\\B"
662
+ when :bos then "\\A"
663
+ when :eos then "\\z"
664
+ when :eos_nl then "(?=\\n?\\z)"
665
+ else
666
+ case n[0]
667
+ when :cat then "[#{category(n[1]).class_body}]"
668
+ when :class then char_class(n[1], n[2])
669
+ end
670
+ end
671
+ end
672
+
673
+ def literal(cp)
674
+ group = @ic && CASE_GROUP[cp]
675
+ return Py.literal_char(cp) unless group
676
+ # Onigmo's own case folding is right for ASCII text and ASCII pattern characters
677
+ return Py.literal_char(cp) if !exact? && cp < 0x80
678
+
679
+ cls = "[#{RangeSet.new(group.map { |x| [x, x] }).class_body}]"
680
+ exact? ? cls : "(?-i:#{cls})"
681
+ end
682
+
683
+ def category(letter)
684
+ set = case letter.downcase
685
+ when "d" then exact? ? DIGITS : ASCII_DIGITS
686
+ when "s" then exact? ? SPACES : ASCII_SPACES
687
+ else exact? ? WORDS : ASCII_WORDS
688
+ end
689
+ letter == letter.upcase ? set.complement : set
690
+ end
691
+
692
+ def char_class(negated, members)
693
+ explicit = RangeSet.new(members.reject { |m| m[0] == :cat })
694
+ cats = members.select { |m| m[0] == :cat }
695
+ hand_folded = false
696
+ if @ic
697
+ non_ascii = explicit.ranges.any? { |_lo, hi| hi >= 0x80 }
698
+ if exact? || non_ascii
699
+ explicit = Py.case_expand(explicit)
700
+ hand_folded = !exact?
701
+ end
702
+ end
703
+ set = cats.reduce(explicit) { |acc, (_, letter)| acc | category(letter) }
704
+ cls = "[#{"^" if negated}#{set.class_body}]"
705
+ hand_folded ? "(?-i:#{cls})" : cls
706
+ end
707
+
708
+ class << self
709
+ # Python's \b and \B with Python's word characters
710
+ def word_boundary
711
+ @word_boundary ||= begin
712
+ w = "[#{WORDS.class_body}]"
713
+ "(?:(?<=#{w})(?!#{w})|(?<!#{w})(?=#{w}))".freeze
714
+ end
715
+ end
716
+
717
+ def not_word_boundary
718
+ @not_word_boundary ||= begin
719
+ w = "[#{WORDS.class_body}]"
720
+ "(?:(?<=#{w})(?=#{w})|(?<!#{w})(?!#{w}))".freeze
721
+ end
722
+ end
723
+ end
724
+ end
725
+
726
+ STRIP = Rx.new("\\A\\s+|\\s+\\Z")
727
+ WS_RUN = Rx.new("\\s+")
728
+ # Onigmo's \b word characters ((?u)\w) that are not Python's, and Python's that are not Onigmo's
729
+ WORD_RISK = Regexp.new("(?u:[\\w&&[^#{WORDS.class_body}]]|[#{WORDS.class_body}&&\\W])").freeze
730
+ end
731
+ end