gunsnprices 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE +21 -0
- data/LICENSE-DATA +17 -0
- data/README.md +267 -0
- data/bin/gunsnprices +12 -0
- data/data/calibers.json +1 -0
- data/data/models.json +1 -0
- data/gunsnprices.gemspec +37 -0
- data/lib/gunsnprices/calibers.rb +280 -0
- data/lib/gunsnprices/cli.rb +53 -0
- data/lib/gunsnprices/data_files.rb +46 -0
- data/lib/gunsnprices/models.rb +348 -0
- data/lib/gunsnprices/py.rb +731 -0
- data/lib/gunsnprices/specs.rb +151 -0
- data/lib/gunsnprices/tables.rb +5436 -0
- data/lib/gunsnprices/types.rb +93 -0
- data/lib/gunsnprices/version.rb +6 -0
- data/lib/gunsnprices.rb +192 -0
- metadata +72 -0
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "tables"
|
|
4
|
+
|
|
5
|
+
module GunsNPrices
|
|
6
|
+
# Python compatibility layer.
|
|
7
|
+
#
|
|
8
|
+
# The rules (the patterns in this gem and the ~3,000 patterns in the model dictionaries) are written for Python's
|
|
9
|
+
# `re` module, the engine of the reference implementation, and the code around them relies on Python's string
|
|
10
|
+
# functions. This module reproduces both on top of Ruby's Onigmo engine:
|
|
11
|
+
#
|
|
12
|
+
# * {Rx} translates a Python pattern to Onigmo syntax: `^` is `\A`, `$` also matches before a final newline,
|
|
13
|
+
# `\Z` is `\z`, `{` that is not a repetition is a literal, `[` and `&&` inside a class are literals, and `\s`
|
|
14
|
+
# also covers U+001C..U+001F.
|
|
15
|
+
# * On ASCII text, Onigmo's own `\d`, `\w`, `\b` and case-insensitive matching mean the same as Python's, so each
|
|
16
|
+
# pattern runs natively. On other text a second form is used, with Python's digit, space and word characters
|
|
17
|
+
# spelled out from Python's tables and `re.IGNORECASE` expanded by hand (Onigmo would also match "ß" for "ss" and
|
|
18
|
+
# would not match "ı" or "İ" for "i"). Onigmo's `\b` counts combining marks and a few other characters as word
|
|
19
|
+
# characters and Python does not; only a text with such a character gets a third form, with `\b` written as
|
|
20
|
+
# look-around.
|
|
21
|
+
# * `lower` handles the final sigma, numbers are read from any script's digits, `round4` rounds half to even on the
|
|
22
|
+
# exact binary value, and `unescape` is `html.unescape` with the HTML5 entity table.
|
|
23
|
+
#
|
|
24
|
+
# Both engines backtrack with leftmost-first alternation, so the match found is the one Python finds.
|
|
25
|
+
#
|
|
26
|
+
# @api private
|
|
27
|
+
module Py
|
|
28
|
+
# Sorted, disjoint inclusive code point ranges.
|
|
29
|
+
class RangeSet
|
|
30
|
+
attr_reader :ranges
|
|
31
|
+
|
|
32
|
+
# @param ranges [Array<Array(Integer, Integer)>] inclusive ranges, in any order, may overlap
|
|
33
|
+
def initialize(ranges)
|
|
34
|
+
merged = []
|
|
35
|
+
ranges.sort.each do |lo, hi|
|
|
36
|
+
if merged.any? && lo <= merged[-1][1] + 1
|
|
37
|
+
merged[-1][1] = hi if hi > merged[-1][1]
|
|
38
|
+
else
|
|
39
|
+
merged << [lo, hi]
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
@ranges = merged.map(&:freeze).freeze
|
|
43
|
+
@starts = @ranges.map(&:first).freeze
|
|
44
|
+
freeze
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.from_flat(flat)
|
|
48
|
+
new(flat.each_slice(2).to_a)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def include?(cp)
|
|
52
|
+
i = @starts.bsearch_index { |x| x > cp }
|
|
53
|
+
i = (i || @starts.size) - 1
|
|
54
|
+
i >= 0 && cp <= @ranges[i][1]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def complement
|
|
58
|
+
out = []
|
|
59
|
+
nxt = 0
|
|
60
|
+
@ranges.each do |lo, hi|
|
|
61
|
+
out << [nxt, lo - 1] if lo > nxt
|
|
62
|
+
nxt = hi + 1
|
|
63
|
+
end
|
|
64
|
+
out << [nxt, 0x10FFFF] if nxt <= 0x10FFFF
|
|
65
|
+
RangeSet.new(out)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def |(other)
|
|
69
|
+
RangeSet.new(@ranges + other.ranges)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Body of an Onigmo character class (without the brackets). Surrogates can't be written in a UTF-8 pattern
|
|
73
|
+
# and never occur in UTF-8 text, so they are left out.
|
|
74
|
+
def class_body
|
|
75
|
+
out = +""
|
|
76
|
+
@ranges.each do |lo, hi|
|
|
77
|
+
[[lo, [hi, 0xD7FF].min], [[lo, 0xE000].max, hi]].each do |a, b|
|
|
78
|
+
next if a > b
|
|
79
|
+
|
|
80
|
+
out << Py.class_char(a)
|
|
81
|
+
next if a == b
|
|
82
|
+
|
|
83
|
+
out << "-" if b > a + 1
|
|
84
|
+
out << Py.class_char(b)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
out
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
DIGITS = RangeSet.new(Tables::DIGIT_ZEROS.map { |z| [z, z + 9] })
|
|
92
|
+
SPACES = RangeSet.from_flat(Tables::SPACE)
|
|
93
|
+
WORDS = RangeSet.from_flat(Tables::WORD)
|
|
94
|
+
CASED = RangeSet.from_flat(Tables::CASED)
|
|
95
|
+
CASE_IGNORABLE = RangeSet.from_flat(Tables::CASE_IGNORABLE)
|
|
96
|
+
|
|
97
|
+
# ASCII-only forms of the classes, enough for ASCII text
|
|
98
|
+
ASCII_DIGITS = RangeSet.new([[0x30, 0x39]])
|
|
99
|
+
ASCII_SPACES = RangeSet.new([[0x09, 0x0D], [0x1C, 0x20]])
|
|
100
|
+
ASCII_WORDS = RangeSet.new([[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]])
|
|
101
|
+
|
|
102
|
+
# re.IGNORECASE: code point -> the code points it matches (itself included)
|
|
103
|
+
CASE_GROUP = Tables::CASE_GROUPS.each_with_object({}) do |g, h|
|
|
104
|
+
cps = g.codepoints.freeze
|
|
105
|
+
cps.each { |cp| h[cp] = cps }
|
|
106
|
+
end.freeze
|
|
107
|
+
CASE_MEMBERS = CASE_GROUP.keys.sort.freeze
|
|
108
|
+
|
|
109
|
+
INVALID_CODEPOINTS = Tables::INVALID_CODEPOINTS.to_h { |cp| [cp, true] }.freeze
|
|
110
|
+
LOWER = Tables::LOWER.to_h { |cp, s| [cp.chr(Encoding::UTF_8), s] }.freeze
|
|
111
|
+
|
|
112
|
+
class << self
|
|
113
|
+
# ---------- text ----------
|
|
114
|
+
|
|
115
|
+
# A title or field as a UTF-8 string (nil -> ""). Other encodings are converted; invalid bytes become U+FFFD.
|
|
116
|
+
def str(value)
|
|
117
|
+
s = value.nil? ? "" : value.to_s
|
|
118
|
+
if s.encoding != Encoding::UTF_8
|
|
119
|
+
s = if s.encoding == Encoding::BINARY || s.encoding == Encoding::US_ASCII
|
|
120
|
+
s.dup.force_encoding(Encoding::UTF_8)
|
|
121
|
+
else
|
|
122
|
+
s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
s.valid_encoding? ? s : s.scrub("�")
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Python truthiness of an optional string, number or list.
|
|
129
|
+
def present?(value)
|
|
130
|
+
!(value.nil? || value == false || (value.respond_to?(:empty?) && value.empty?) || value == 0)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Python `str.lower()`: full Unicode lowercase mapping, with the final sigma ("ΑΣ" -> "ας").
|
|
134
|
+
def lower(s)
|
|
135
|
+
return s.downcase(:ascii) if s.ascii_only?
|
|
136
|
+
return lower_sigma(s) if s.include?("Σ")
|
|
137
|
+
|
|
138
|
+
s.downcase(:ascii).gsub(/[^\x00-\x7F]/) { |c| LOWER[c] || c }
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Python `str.upper()`. Only used on short matches of ASCII-letter patterns, where Ruby's mapping is the same.
|
|
142
|
+
def upper(s)
|
|
143
|
+
s.upcase
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Python `str.strip()`.
|
|
147
|
+
def strip(s)
|
|
148
|
+
STRIP.sub(s, "")
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Python `re.sub(r"\s+", " ", s)`.
|
|
152
|
+
def collapse_ws(s)
|
|
153
|
+
WS_RUN.sub(s, " ")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Python `re.split(r"\s+", s)`.
|
|
157
|
+
def split_ws(s)
|
|
158
|
+
out = []
|
|
159
|
+
last = 0
|
|
160
|
+
WS_RUN.each_match(s) do |m|
|
|
161
|
+
out << s[last...m.begin(0)]
|
|
162
|
+
last = m.end(0)
|
|
163
|
+
end
|
|
164
|
+
out << s[last..]
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Python `re.escape` (3.7+).
|
|
168
|
+
def escape(s)
|
|
169
|
+
s.gsub(/[()\[\]{}?*+\-|^$\\.&~# \t\n\r\v\f]/) { |c| "\\#{c}" }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Value of a Unicode decimal digit (what Python's `\d` matches), or nil.
|
|
173
|
+
def digit_value(ch)
|
|
174
|
+
cp = ch.ord
|
|
175
|
+
return cp - 0x30 if cp >= 0x30 && cp <= 0x39
|
|
176
|
+
return nil if cp < 0x80
|
|
177
|
+
|
|
178
|
+
i = Tables::DIGIT_ZEROS.bsearch_index { |z| z > cp }
|
|
179
|
+
zero = Tables::DIGIT_ZEROS[(i || Tables::DIGIT_ZEROS.size) - 1]
|
|
180
|
+
zero && cp >= zero && cp - zero < 10 ? cp - zero : nil
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Python `int(s)` for a string of decimal digits of any script.
|
|
184
|
+
def int(s)
|
|
185
|
+
return s.to_i if s.ascii_only?
|
|
186
|
+
|
|
187
|
+
s.each_char.reduce(0) { |acc, c| (acc * 10) + digit_value(c) }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Python `float(s)` for digits of any script with an optional decimal point.
|
|
191
|
+
def float(s)
|
|
192
|
+
return Float(s) if s.ascii_only?
|
|
193
|
+
|
|
194
|
+
Float(s.each_char.map { |c| c == "." ? "." : digit_value(c).to_s }.join)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Python `round(x, 4)`: the exact binary value of x rounded half to even at 4 decimals, then the nearest float.
|
|
198
|
+
def round4(x)
|
|
199
|
+
return x unless x.finite?
|
|
200
|
+
|
|
201
|
+
q = (x.to_r * 10_000).round(half: :even)
|
|
202
|
+
q.zero? && x.negative? ? -0.0 : q.fdiv(10_000)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Python `html.unescape`: HTML5 named references with and without ";", the longest known prefix of an unknown
|
|
206
|
+
# name ("&x" -> "&x"), numeric references remapped or dropped the way browsers do.
|
|
207
|
+
def unescape(s)
|
|
208
|
+
return s unless s.include?("&")
|
|
209
|
+
|
|
210
|
+
s.gsub(CHARREF) { replace_charref(Regexp.last_match(1)) }
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# ---------- patterns ----------
|
|
214
|
+
|
|
215
|
+
# Ruby source of a pattern character as a class member.
|
|
216
|
+
def class_char(cp)
|
|
217
|
+
if cp < 0x20 || cp == 0x7F
|
|
218
|
+
format("\\x%02X", cp)
|
|
219
|
+
elsif cp >= 0x80
|
|
220
|
+
format("\\u{%X}", cp)
|
|
221
|
+
elsif (cp >= 0x30 && cp <= 0x39) || (cp >= 0x41 && cp <= 0x5A) || (cp >= 0x61 && cp <= 0x7A)
|
|
222
|
+
cp.chr
|
|
223
|
+
else
|
|
224
|
+
"\\#{cp.chr}"
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Ruby source of a pattern character outside a class.
|
|
229
|
+
def literal_char(cp)
|
|
230
|
+
if cp < 0x20 || cp == 0x7F
|
|
231
|
+
format("\\x%02X", cp)
|
|
232
|
+
elsif cp >= 0x80
|
|
233
|
+
format("\\u{%X}", cp)
|
|
234
|
+
elsif ".^$|?*+()[]{}\\-&~#/".include?(cp.chr)
|
|
235
|
+
"\\#{cp.chr}"
|
|
236
|
+
else
|
|
237
|
+
cp.chr
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# The characters `re.IGNORECASE` lets a class match, for an explicit set of members.
|
|
242
|
+
def case_expand(set)
|
|
243
|
+
extra = []
|
|
244
|
+
set.ranges.each do |lo, hi|
|
|
245
|
+
i = CASE_MEMBERS.bsearch_index { |x| x >= lo }
|
|
246
|
+
next unless i
|
|
247
|
+
|
|
248
|
+
while i < CASE_MEMBERS.size && CASE_MEMBERS[i] <= hi
|
|
249
|
+
CASE_GROUP[CASE_MEMBERS[i]].each { |cp| extra << [cp, cp] }
|
|
250
|
+
i += 1
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
extra.empty? ? set : RangeSet.new(set.ranges + extra)
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# The text has a character that Onigmo's \b counts as a word character and Python's \w does not, or the other
|
|
257
|
+
# way round (combining marks, connector punctuation other than "_", join controls, circled letters, ...).
|
|
258
|
+
def word_risk?(s)
|
|
259
|
+
!s.ascii_only? && WORD_RISK.match?(s)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
private
|
|
263
|
+
|
|
264
|
+
def lower_sigma(s)
|
|
265
|
+
cps = s.codepoints
|
|
266
|
+
out = +""
|
|
267
|
+
cps.each_with_index do |cp, i|
|
|
268
|
+
out << if cp == 0x3A3
|
|
269
|
+
final_sigma?(cps, i) ? "ς" : "σ"
|
|
270
|
+
elsif cp < 0x80
|
|
271
|
+
cp >= 0x41 && cp <= 0x5A ? (cp + 32).chr : cp.chr
|
|
272
|
+
else
|
|
273
|
+
c = cp.chr(Encoding::UTF_8)
|
|
274
|
+
LOWER[c] || c
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
out
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Python: U+03A3 is a final sigma when, skipping case-ignorable characters, a cased character comes before it
|
|
281
|
+
# and none after it.
|
|
282
|
+
def final_sigma?(cps, i)
|
|
283
|
+
j = i - 1
|
|
284
|
+
j -= 1 while j >= 0 && CASE_IGNORABLE.include?(cps[j])
|
|
285
|
+
return false unless j >= 0 && CASED.include?(cps[j])
|
|
286
|
+
|
|
287
|
+
j = i + 1
|
|
288
|
+
j += 1 while j < cps.size && CASE_IGNORABLE.include?(cps[j])
|
|
289
|
+
j == cps.size || !CASED.include?(cps[j])
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def replace_charref(ref)
|
|
293
|
+
if ref.start_with?("#")
|
|
294
|
+
num = ref[1] == "x" || ref[1] == "X" ? ref[2..].delete_suffix(";").to_i(16) : ref[1..].delete_suffix(";").to_i
|
|
295
|
+
return Tables::INVALID_CHARREFS[num] if Tables::INVALID_CHARREFS.key?(num)
|
|
296
|
+
return "�" if (num >= 0xD800 && num <= 0xDFFF) || num > 0x10FFFF
|
|
297
|
+
return "" if INVALID_CODEPOINTS.key?(num)
|
|
298
|
+
|
|
299
|
+
return num.chr(Encoding::UTF_8)
|
|
300
|
+
end
|
|
301
|
+
return Tables::HTML5[ref] if Tables::HTML5.key?(ref)
|
|
302
|
+
|
|
303
|
+
# the longest known name that the reference starts with
|
|
304
|
+
(ref.length - 1).downto(2) do |x|
|
|
305
|
+
v = Tables::HTML5[ref[0, x]]
|
|
306
|
+
return v + ref[x..] if v
|
|
307
|
+
end
|
|
308
|
+
"&#{ref}"
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
CHARREF = /&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)/.freeze
|
|
313
|
+
|
|
314
|
+
# A Python-syntax regular expression, translated when created and compiled on first use.
|
|
315
|
+
class Rx
|
|
316
|
+
attr_reader :pattern
|
|
317
|
+
|
|
318
|
+
# @param pattern [String] Python `re` syntax
|
|
319
|
+
# @param ignore_case [Boolean] Python `re.IGNORECASE`
|
|
320
|
+
# @param anchor [nil, :start, :full] nil: search; :start: match at the given position (`re.match`);
|
|
321
|
+
# :full: match the whole string (`re.fullmatch`)
|
|
322
|
+
def initialize(pattern, ignore_case: false, anchor: nil)
|
|
323
|
+
@pattern = pattern
|
|
324
|
+
@ignore_case = ignore_case
|
|
325
|
+
@anchor = anchor
|
|
326
|
+
@nodes = Parser.new(pattern).parse
|
|
327
|
+
@has_b = @nodes.any? { |n| n == :b || n == :B }
|
|
328
|
+
@mutex = Mutex.new
|
|
329
|
+
@fast = @exact = @exact_b = nil
|
|
330
|
+
Rx.register(self)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
@registry = []
|
|
334
|
+
@registry_mutex = Mutex.new
|
|
335
|
+
|
|
336
|
+
class << self
|
|
337
|
+
# Every pattern created so far (the rules are created once, when their module or the data is loaded).
|
|
338
|
+
def registry
|
|
339
|
+
@registry_mutex.synchronize { @registry.dup }
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def register(rx)
|
|
343
|
+
@registry_mutex.synchronize { @registry << rx }
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Compile the form used for ASCII text (most titles) now.
|
|
348
|
+
def compile_fast
|
|
349
|
+
fast
|
|
350
|
+
self
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Compile all forms now (tests) and return self.
|
|
354
|
+
def compile_all
|
|
355
|
+
fast
|
|
356
|
+
exact
|
|
357
|
+
exact_b
|
|
358
|
+
self
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# Python `rx.search(s, pos)`: MatchData or nil (positions are character offsets, like Python's).
|
|
362
|
+
def search(s, pos = 0)
|
|
363
|
+
for_text(s).match(s, pos)
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# Python `rx.search(s) is not None`.
|
|
367
|
+
def match?(s)
|
|
368
|
+
for_text(s).match?(s)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# Python `rx.finditer(s)`: every non-overlapping match, as MatchData.
|
|
372
|
+
def each_match(s, &block)
|
|
373
|
+
rx = for_text(s)
|
|
374
|
+
return enum_for(:each_match, s) unless block
|
|
375
|
+
|
|
376
|
+
s.scan(rx) { block.call(Regexp.last_match) }
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
def matches(s)
|
|
380
|
+
each_match(s).to_a
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
# Python `rx.sub(repl, s)`: repl is a String (a Python template such as "\\g<1> gen") or a block.
|
|
384
|
+
def sub(s, repl = nil, &block)
|
|
385
|
+
rx = for_text(s)
|
|
386
|
+
if block
|
|
387
|
+
s.gsub(rx) { block.call(Regexp.last_match) }
|
|
388
|
+
elsif repl.include?("\\")
|
|
389
|
+
parts = Py::Template.parse(repl)
|
|
390
|
+
s.gsub(rx) { Py::Template.expand(parts, Regexp.last_match) }
|
|
391
|
+
else
|
|
392
|
+
s.gsub(rx) { repl }
|
|
393
|
+
end
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
private
|
|
397
|
+
|
|
398
|
+
def for_text(s)
|
|
399
|
+
return fast if s.ascii_only?
|
|
400
|
+
return exact unless @has_b && Py.word_risk?(s)
|
|
401
|
+
|
|
402
|
+
exact_b
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def fast
|
|
406
|
+
@fast || @mutex.synchronize { @fast ||= build(:fast) }
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
def exact
|
|
410
|
+
@exact || @mutex.synchronize { @exact ||= build(:exact) }
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def exact_b
|
|
414
|
+
return exact unless @has_b
|
|
415
|
+
|
|
416
|
+
@exact_b || @mutex.synchronize { @exact_b ||= build(:exact_b) }
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
def build(mode)
|
|
420
|
+
native_ic = mode == :fast && @ignore_case
|
|
421
|
+
body = Emitter.new(mode, @ignore_case).emit(@nodes)
|
|
422
|
+
src = case @anchor
|
|
423
|
+
when :start then "\\G(?:#{body})"
|
|
424
|
+
when :full then "\\A(?:#{body})\\z"
|
|
425
|
+
else body
|
|
426
|
+
end
|
|
427
|
+
Regexp.new(src, native_ic ? Regexp::IGNORECASE : 0).freeze
|
|
428
|
+
rescue RegexpError => e
|
|
429
|
+
raise ArgumentError, "cannot translate #{@pattern.inspect} (#{mode}): #{e.message}"
|
|
430
|
+
end
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
# Python replacement templates ("\\g<1> gen", "\\1", "\\\\").
|
|
434
|
+
module Template
|
|
435
|
+
ESCAPES = { "n" => "\n", "t" => "\t", "r" => "\r", "f" => "\f", "v" => "\v", "a" => "\a", "b" => "\b",
|
|
436
|
+
"\\" => "\\" }.freeze
|
|
437
|
+
|
|
438
|
+
module_function
|
|
439
|
+
|
|
440
|
+
# Literal strings and group numbers.
|
|
441
|
+
def parse(repl)
|
|
442
|
+
parts = []
|
|
443
|
+
lit = +""
|
|
444
|
+
i = 0
|
|
445
|
+
while i < repl.length
|
|
446
|
+
c = repl[i]
|
|
447
|
+
if c != "\\"
|
|
448
|
+
lit << c
|
|
449
|
+
i += 1
|
|
450
|
+
next
|
|
451
|
+
end
|
|
452
|
+
nxt = repl[i + 1]
|
|
453
|
+
raise ArgumentError, "bad escape at the end of #{repl.inspect}" if nxt.nil?
|
|
454
|
+
|
|
455
|
+
if nxt == "g" && (m = /\A\\g<(\d+)>/.match(repl[i..]))
|
|
456
|
+
parts << lit << m[1].to_i
|
|
457
|
+
lit = +""
|
|
458
|
+
i += m[0].length
|
|
459
|
+
elsif nxt.match?(/[0-9]/)
|
|
460
|
+
m = /\A\\(\d\d?)/.match(repl[i..])
|
|
461
|
+
parts << lit << m[1].to_i
|
|
462
|
+
lit = +""
|
|
463
|
+
i += m[0].length
|
|
464
|
+
elsif ESCAPES.key?(nxt)
|
|
465
|
+
lit << ESCAPES[nxt]
|
|
466
|
+
i += 2
|
|
467
|
+
elsif nxt.match?(/[A-Za-z]/)
|
|
468
|
+
raise ArgumentError, "bad escape \\#{nxt} in #{repl.inspect}"
|
|
469
|
+
else
|
|
470
|
+
lit << "\\" << nxt
|
|
471
|
+
i += 2
|
|
472
|
+
end
|
|
473
|
+
end
|
|
474
|
+
parts << lit
|
|
475
|
+
parts.reject { |p| p == "" }.freeze
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def expand(parts, m)
|
|
479
|
+
parts.map { |p| p.is_a?(Integer) ? (m[p] || "") : p }.join
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
# Python `re` syntax -> a list of nodes: Strings are Onigmo syntax passed through, Integers are literal code
|
|
484
|
+
# points, Arrays are [:class, negated, [members]] or [:cat, letter]; :b, :B, :bos, :eos, :eos_nl are anchors.
|
|
485
|
+
class Parser
|
|
486
|
+
def initialize(pattern)
|
|
487
|
+
@s = pattern
|
|
488
|
+
@i = 0
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
def parse
|
|
492
|
+
nodes = []
|
|
493
|
+
while @i < @s.length
|
|
494
|
+
c = @s[@i]
|
|
495
|
+
@i += 1
|
|
496
|
+
case c
|
|
497
|
+
when "\\" then nodes << escape
|
|
498
|
+
when "[" then nodes << char_class
|
|
499
|
+
when "(" then nodes << group
|
|
500
|
+
when "{" then nodes << repetition(nodes)
|
|
501
|
+
when "^" then nodes << :bos
|
|
502
|
+
when "$" then nodes << :eos_nl
|
|
503
|
+
when "." , "|", ")", "*", "+", "?" then nodes << c
|
|
504
|
+
else nodes << c.ord
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
nodes.freeze
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
private
|
|
511
|
+
|
|
512
|
+
def fail!(why)
|
|
513
|
+
raise ArgumentError, "unsupported pattern #{@s.inspect}: #{why}"
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
def escape
|
|
517
|
+
c = @s[@i] or fail!("trailing backslash")
|
|
518
|
+
@i += 1
|
|
519
|
+
case c
|
|
520
|
+
when "d", "D", "w", "W", "s", "S" then [:cat, c].freeze
|
|
521
|
+
when "b" then :b
|
|
522
|
+
when "B" then :B
|
|
523
|
+
when "A" then :bos
|
|
524
|
+
when "Z" then :eos
|
|
525
|
+
else char_escape(c)
|
|
526
|
+
end
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# an escape that stands for one character (outside or inside a class)
|
|
530
|
+
def char_escape(c)
|
|
531
|
+
case c
|
|
532
|
+
when "n" then 0x0A
|
|
533
|
+
when "t" then 0x09
|
|
534
|
+
when "r" then 0x0D
|
|
535
|
+
when "f" then 0x0C
|
|
536
|
+
when "v" then 0x0B
|
|
537
|
+
when "a" then 0x07
|
|
538
|
+
when "x" then hex(2)
|
|
539
|
+
when "u" then hex(4)
|
|
540
|
+
when "U" then hex(8)
|
|
541
|
+
when "0" then 0
|
|
542
|
+
when /[0-9A-Za-z]/ then fail!("escape \\#{c}")
|
|
543
|
+
else c.ord
|
|
544
|
+
end
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
def hex(n)
|
|
548
|
+
digits = @s[@i, n]
|
|
549
|
+
fail!("bad hex escape") unless digits && digits.length == n && digits.match?(/\A\h+\z/)
|
|
550
|
+
@i += n
|
|
551
|
+
digits.to_i(16)
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
def group
|
|
555
|
+
return "(" unless @s[@i] == "?"
|
|
556
|
+
|
|
557
|
+
rest = @s[@i + 1..]
|
|
558
|
+
if rest.start_with?(":", "=", "!", "<=", "<!", ">")
|
|
559
|
+
tok = "(?#{rest.start_with?("<") ? rest[0, 2] : rest[0]}"
|
|
560
|
+
@i += tok.length - 1
|
|
561
|
+
tok
|
|
562
|
+
elsif rest.start_with?("#")
|
|
563
|
+
close = @s.index(")", @i) or fail!("unterminated comment")
|
|
564
|
+
@i = close + 1
|
|
565
|
+
""
|
|
566
|
+
else
|
|
567
|
+
fail!("group (?#{rest[0]}")
|
|
568
|
+
end
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
# Python reads "{m}", "{m,}", "{,n}", "{m,n}" as repetitions and any other "{" as a literal.
|
|
572
|
+
def repetition(nodes)
|
|
573
|
+
m = /\A(\d*)(?:(,)(\d*))?\}/.match(@s[@i..])
|
|
574
|
+
return "{".ord if m.nil? || (m[1].empty? && m[2].nil?)
|
|
575
|
+
|
|
576
|
+
@i += m[0].length
|
|
577
|
+
lo = m[1].empty? ? "0" : m[1]
|
|
578
|
+
fail!("repetition of nothing") if nodes.empty?
|
|
579
|
+
q = if m[2].nil? then "{#{lo}}"
|
|
580
|
+
elsif m[3].empty? then "{#{lo},}"
|
|
581
|
+
else "{#{lo},#{m[3]}}"
|
|
582
|
+
end
|
|
583
|
+
# Onigmo reads "{n}?" as "({n})?" and "{n}+" as "({n})+"; Python's lazy "{n}?" is just "{n}"
|
|
584
|
+
if m[2].nil? && @s[@i] == "?"
|
|
585
|
+
@i += 1
|
|
586
|
+
elsif @s[@i] == "+"
|
|
587
|
+
fail!("possessive repetition")
|
|
588
|
+
end
|
|
589
|
+
q
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
def char_class
|
|
593
|
+
negated = false
|
|
594
|
+
if @s[@i] == "^"
|
|
595
|
+
negated = true
|
|
596
|
+
@i += 1
|
|
597
|
+
end
|
|
598
|
+
members = []
|
|
599
|
+
first = true
|
|
600
|
+
loop do
|
|
601
|
+
c = @s[@i] or fail!("unterminated character class")
|
|
602
|
+
@i += 1
|
|
603
|
+
break if c == "]" && !first
|
|
604
|
+
|
|
605
|
+
first = false
|
|
606
|
+
lo = if c == "\\"
|
|
607
|
+
e = @s[@i] or fail!("trailing backslash")
|
|
608
|
+
@i += 1
|
|
609
|
+
if "dDwWsS".include?(e)
|
|
610
|
+
members << [:cat, e].freeze
|
|
611
|
+
next
|
|
612
|
+
end
|
|
613
|
+
e == "b" ? 0x08 : char_escape(e)
|
|
614
|
+
else
|
|
615
|
+
c.ord
|
|
616
|
+
end
|
|
617
|
+
if @s[@i] == "-" && @s[@i + 1] && @s[@i + 1] != "]"
|
|
618
|
+
@i += 1
|
|
619
|
+
d = @s[@i]
|
|
620
|
+
@i += 1
|
|
621
|
+
if d == "\\"
|
|
622
|
+
e = @s[@i] or fail!("trailing backslash")
|
|
623
|
+
@i += 1
|
|
624
|
+
fail!("bad range") if "dDwWsS".include?(e)
|
|
625
|
+
d_cp = e == "b" ? 0x08 : char_escape(e)
|
|
626
|
+
else
|
|
627
|
+
d_cp = d.ord
|
|
628
|
+
end
|
|
629
|
+
fail!("bad character range") if d_cp < lo
|
|
630
|
+
members << [lo, d_cp].freeze
|
|
631
|
+
else
|
|
632
|
+
members << [lo, lo].freeze
|
|
633
|
+
end
|
|
634
|
+
end
|
|
635
|
+
[:class, negated, members.freeze].freeze
|
|
636
|
+
end
|
|
637
|
+
end
|
|
638
|
+
|
|
639
|
+
# Nodes -> Onigmo source for one form (:fast, :exact or :exact_b).
|
|
640
|
+
class Emitter
|
|
641
|
+
def initialize(mode, ignore_case)
|
|
642
|
+
@mode = mode
|
|
643
|
+
@ic = ignore_case
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
def emit(nodes)
|
|
647
|
+
nodes.map { |n| node(n) }.join
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
private
|
|
651
|
+
|
|
652
|
+
def exact?
|
|
653
|
+
@mode != :fast
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
def node(n)
|
|
657
|
+
case n
|
|
658
|
+
when String then n
|
|
659
|
+
when Integer then literal(n)
|
|
660
|
+
when :b then @mode == :exact_b ? Emitter.word_boundary : "\\b"
|
|
661
|
+
when :B then @mode == :exact_b ? Emitter.not_word_boundary : "\\B"
|
|
662
|
+
when :bos then "\\A"
|
|
663
|
+
when :eos then "\\z"
|
|
664
|
+
when :eos_nl then "(?=\\n?\\z)"
|
|
665
|
+
else
|
|
666
|
+
case n[0]
|
|
667
|
+
when :cat then "[#{category(n[1]).class_body}]"
|
|
668
|
+
when :class then char_class(n[1], n[2])
|
|
669
|
+
end
|
|
670
|
+
end
|
|
671
|
+
end
|
|
672
|
+
|
|
673
|
+
def literal(cp)
|
|
674
|
+
group = @ic && CASE_GROUP[cp]
|
|
675
|
+
return Py.literal_char(cp) unless group
|
|
676
|
+
# Onigmo's own case folding is right for ASCII text and ASCII pattern characters
|
|
677
|
+
return Py.literal_char(cp) if !exact? && cp < 0x80
|
|
678
|
+
|
|
679
|
+
cls = "[#{RangeSet.new(group.map { |x| [x, x] }).class_body}]"
|
|
680
|
+
exact? ? cls : "(?-i:#{cls})"
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
def category(letter)
|
|
684
|
+
set = case letter.downcase
|
|
685
|
+
when "d" then exact? ? DIGITS : ASCII_DIGITS
|
|
686
|
+
when "s" then exact? ? SPACES : ASCII_SPACES
|
|
687
|
+
else exact? ? WORDS : ASCII_WORDS
|
|
688
|
+
end
|
|
689
|
+
letter == letter.upcase ? set.complement : set
|
|
690
|
+
end
|
|
691
|
+
|
|
692
|
+
def char_class(negated, members)
|
|
693
|
+
explicit = RangeSet.new(members.reject { |m| m[0] == :cat })
|
|
694
|
+
cats = members.select { |m| m[0] == :cat }
|
|
695
|
+
hand_folded = false
|
|
696
|
+
if @ic
|
|
697
|
+
non_ascii = explicit.ranges.any? { |_lo, hi| hi >= 0x80 }
|
|
698
|
+
if exact? || non_ascii
|
|
699
|
+
explicit = Py.case_expand(explicit)
|
|
700
|
+
hand_folded = !exact?
|
|
701
|
+
end
|
|
702
|
+
end
|
|
703
|
+
set = cats.reduce(explicit) { |acc, (_, letter)| acc | category(letter) }
|
|
704
|
+
cls = "[#{"^" if negated}#{set.class_body}]"
|
|
705
|
+
hand_folded ? "(?-i:#{cls})" : cls
|
|
706
|
+
end
|
|
707
|
+
|
|
708
|
+
class << self
|
|
709
|
+
# Python's \b and \B with Python's word characters
|
|
710
|
+
def word_boundary
|
|
711
|
+
@word_boundary ||= begin
|
|
712
|
+
w = "[#{WORDS.class_body}]"
|
|
713
|
+
"(?:(?<=#{w})(?!#{w})|(?<!#{w})(?=#{w}))".freeze
|
|
714
|
+
end
|
|
715
|
+
end
|
|
716
|
+
|
|
717
|
+
def not_word_boundary
|
|
718
|
+
@not_word_boundary ||= begin
|
|
719
|
+
w = "[#{WORDS.class_body}]"
|
|
720
|
+
"(?:(?<=#{w})(?=#{w})|(?<!#{w})(?!#{w}))".freeze
|
|
721
|
+
end
|
|
722
|
+
end
|
|
723
|
+
end
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
STRIP = Rx.new("\\A\\s+|\\s+\\Z")
|
|
727
|
+
WS_RUN = Rx.new("\\s+")
|
|
728
|
+
# Onigmo's \b word characters ((?u)\w) that are not Python's, and Python's that are not Onigmo's
|
|
729
|
+
WORD_RISK = Regexp.new("(?u:[\\w&&[^#{WORDS.class_body}]]|[#{WORDS.class_body}&&\\W])").freeze
|
|
730
|
+
end
|
|
731
|
+
end
|