gunsnprices 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,348 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "py"
4
+ require_relative "types"
5
+ require_relative "data_files"
6
+
7
+ module GunsNPrices
8
+ # Identify the firearm model in a product title: brand -> family -> model.
9
+ #
10
+ # The rules are GunsNPrices' hand-built model dictionaries (50 brands), the same rules that assign models to
11
+ # listings on gunsnprices.com.
12
+ # @api private
13
+ module Models
14
+ Rx = Py::Rx
15
+
16
+ # airguns and airsoft are not firearms (the site filters them by category; here we go by the title)
17
+ AIRGUN = Rx.new('\bair\s*-?\s*(?:gun|rifle|pistol)s?\b|\bairgun|\bairsoft|\bbb\s*(?:gun|pistol|rifle)|\bpellets?\b|' \
18
+ '\bco2\b', ignore_case: true)
19
+
20
+ # accessories named after a gun ("Glock 19 holster", "Sig P365 XL 12rd Magazine", "Magpul PMAG for Glock 19"). A
21
+ # title that describes a gun (pistol/rifle/..., "15+1", a barrel length) or mentions the item as included
22
+ # ("w/ 3 magazines", "no magazine") is not an accessory.
23
+ ACC_NOUN = Rx.new(
24
+ '\b(?:holsters?|magazines?(?!\s*(?:pattern|compatible|well|release|disconnect|safety|catch|cut|tube|extension|' \
25
+ 'fed)\b)|pmags?|slings?|grip panels?|scope mounts?|recoil pads?|speed ?loaders?|mag(?:azine)? loaders?|' \
26
+ 'cleaning kits?|pouch(?:es)?|snap caps?|replacement barrels?|conversion kits?|parts kits?)\b'
27
+ )
28
+ ACC_INCLUDED = Rx.new(
29
+ '(?:(?:\bw/|\bwith\b|\bwithout\b|\bno\b|\band\b|&|\+|\bincl\w*|\bplus\b|\(\d\)|\b[1-9]\b|\btwo\b|\bthree\b|' \
30
+ '\bfour\b|,|\bor\b|\bextra\b|\bspare\b|\bfactory\b|\bfits\b|\btakes\b|\baccepts\b|\buses\b)[^,]{0,14}|/\s*)$'
31
+ )
32
+ GUN_WORDS = Rx.new(
33
+ '\b(?:pistol|rifle|shotgun|revolver|handgun|carbine|firearm|semi-?auto(?:matic)?|bolt[- ]?action|pump|' \
34
+ 'lever[- ]?action|single[- ]?shot|o/u|over/under|sxs|sbr|derringer)\b|\d+\+1\b|\d+(?:\.\d+)?\s*-?\s*(?:"|in\b|inch)'
35
+ )
36
+
37
+ DASHES = { "‐" => "-", "‑" => "-", "–" => "-", "—" => "-", "“" => '"', "”" => '"', "’" => "'",
38
+ "|" => " | ", "*" => " ", "®" => "", "™" => "" }.freeze
39
+ DASHES_RX = /[‐‑–—“”’|*®™]/.freeze
40
+
41
+ NAME_STRIP = Rx.new('[^a-z0-9]+')
42
+ TWO_DIGITS = Rx.new('^(\d{2})(x)?\b')
43
+ GEN = Rx.new('\bgen(?:eration)?\.?\s*-?\s*([1-6])\b|\b([1-6])(?:st|nd|rd|th)\s*gen\b|\bg([1-6])\b')
44
+ GEN_V = Rx.new('^\s*v\b')
45
+ MPN_TB = Rx.new('\d(?:mos)?tb\b', ignore_case: true)
46
+ MPN_IN_NAME = Rx.new('\b(?:gl)?[pue][a-z0-9]\d{3}[a-z0-9-]*|\bg\d{2}c?\d{3}us\b|\bg\d{4}[a-z]*aut\b', ignore_case: true)
47
+ MPN_US = Rx.new('^G(\d{2}C?)([345])(\d{2})US$', ignore_case: true)
48
+ MPN_AUT = Rx.new('^G(\d{2})\d{2}[A-Z]*AUT$', ignore_case: true)
49
+ MPN_CODE = Rx.new('^(?:GL)?([PUE])([A-Z0-9])(\d{2})(\d)', ignore_case: true)
50
+ WORDS = Rx.new('[a-z0-9.&+]+')
51
+
52
+ # One brand's dictionary with its patterns compiled.
53
+ class Dict
54
+ attr_reader :raw, :brand, :terms, :rewrite, :exclude, :exclude_soft, :block, :brand_any, :brand_start, :configs,
55
+ :features, :gens, :brand_names, :deny_names, :models, :find_models, :own_start, :bare_tokens,
56
+ :leaf_rx
57
+
58
+ def initialize(d)
59
+ @raw = d
60
+ @brand = d["brand"]
61
+ @terms = d["mode"] == "terms"
62
+ @rewrite = (d["title_rewrite"] || {}).map { |k, v| [Rx.new(k, ignore_case: true), v] }.freeze
63
+ @exclude = any(d["exclude"])
64
+ @exclude_soft = any(d["exclude_soft"])
65
+ @block = any(d["other_brand_block"])
66
+ @brand_any = if Py.present?(d["brand_rx"])
67
+ Rx.new(d["brand_rx"], ignore_case: true)
68
+ else
69
+ Rx.new("(?<![a-z])#{Py.escape(d["brand_word"])}(?![a-z])", ignore_case: true)
70
+ end
71
+ @brand_start = Py.present?(d["brand_rx"]) ? Rx.new("^\\W*(?:#{d["brand_rx"]})", ignore_case: true) : nil
72
+ @configs = (d["configs"] || {}).map { |k, v| [k, Rx.new(v, ignore_case: true)] }.freeze
73
+ @features = (d["features"] || {}).map { |k, v| [k, Rx.new(v, ignore_case: true)] }.freeze
74
+ @gens = (d["gens"] || {}).map { |k, v| [k, Rx.new(v, ignore_case: true)] }.freeze
75
+ @brand_names = (d["brand_names"] || []).to_h { |x| [Models.key(x), true] }.freeze
76
+ @deny_names = (d["deny_brand_names"] || []).to_h { |x| [Models.key(x), true] }.freeze
77
+ # skip a config already spelled out in the model name ("Plus Shield" on "S&W M&P Shield Plus")
78
+ @leaf_rx = @configs.to_h { |k, _| [k, Rx.new("(?<![a-z0-9])#{Py.escape(k)}(?![a-z0-9])", ignore_case: true)] }.freeze
79
+ if @terms
80
+ @models = d["models"].map do |m|
81
+ { name: m["name"], family: m["family"], match: Rx.new(m["match"], ignore_case: true),
82
+ not: Py.present?(m["not"]) ? Rx.new(m["not"], ignore_case: true) : nil,
83
+ configs: m["configs"] || [], implied_configs: m["implied_configs"] || [] }.freeze
84
+ end.freeze
85
+ else
86
+ @models = d["models"]
87
+ bw = Py.escape(d["brand_word"])
88
+ @find_models = Rx.new(
89
+ "(?<![a-z0-9.])(?:#{bw}\\s*-?\\s*g?\\s*-?\\s*|g)(\\d{2})(?:(sf|x|c|l|m|s)|\\s(sf|x)(?![a-z0-9]))?" \
90
+ '(?:g([1-6])(?:us)?|(v))?(fs|mos|rtf)?(?![a-z0-9&\"+/])(?!\s*-?\s*(?:round|rounds|rd|rds|rnd|mag|mags|' \
91
+ 'magazine)\b)', ignore_case: true
92
+ )
93
+ @own_start = Rx.new("^\\W*#{bw}\\b", ignore_case: true)
94
+ tokens = d["bare_model_tokens"]
95
+ @bare_tokens = Py.present?(tokens) ? Rx.new("\\b(#{tokens.join("|")})\\b") : nil
96
+ end
97
+ freeze
98
+ end
99
+
100
+ def [](key)
101
+ @raw[key]
102
+ end
103
+
104
+ # every compiled pattern (for warm_up and the tests)
105
+ def patterns
106
+ list = @rewrite.map(&:first) + [@exclude, @exclude_soft, @block, @brand_any, @brand_start] +
107
+ (@configs + @features + @gens).map(&:last) + @leaf_rx.values + [@find_models, @own_start, @bare_tokens]
108
+ list += @models.flat_map { |m| [m[:match], m[:not]] } if @terms
109
+ list.compact
110
+ end
111
+
112
+ private
113
+
114
+ def any(patterns)
115
+ Py.present?(patterns) ? Rx.new(patterns.join("|"), ignore_case: true) : nil
116
+ end
117
+ end
118
+
119
+ @dicts = nil
120
+ @mutex = Mutex.new
121
+
122
+ class << self
123
+ def dicts
124
+ @dicts || @mutex.synchronize { @dicts ||= DataFiles.models.map { |d| Dict.new(d) }.freeze }
125
+ end
126
+
127
+ def list_brands
128
+ dicts.map(&:brand).freeze
129
+ end
130
+
131
+ def key(s)
132
+ NAME_STRIP.sub(Py.lower(Py.unescape(Py.str(s))), "")
133
+ end
134
+
135
+ def identify_model(title, brand: nil, mpn: nil, sku: nil)
136
+ title = Py.str(title)
137
+ brand = Py.present?(brand) ? Py.str(brand) : nil
138
+ mpn = Py.str(mpn)
139
+ sku = Py.str(sku)
140
+ return nil if title.empty? || AIRGUN.match?(title) || accessory?(title)
141
+
142
+ brand_key = brand && key(brand)
143
+ base = base_norm(title)
144
+ hits = []
145
+ dicts.each_with_index do |d, i|
146
+ t = d.rewrite.reduce(base) { |s, (rx, to)| rx.sub(s, to) }
147
+ r = parse_one(t, title, d, brand_key, mpn, sku)
148
+ hits << [rank(t, d, brand_key), i, d, r] if r
149
+ end
150
+ return nil if hits.empty?
151
+
152
+ _, _, d, r = hits.min_by { |rank, i, _, _| [*rank, i] }
153
+ fam = d["family_display"]&.[](r[:family])
154
+ fam = d.terms ? r[:family] : "#{d.brand} #{r[:family]}" unless Py.present?(fam)
155
+ ModelMatch.new(brand: d.brand, family: -fam, model: -leaf(r, d), generation: -r[:gen],
156
+ configs: r[:configs].map { |c| -c }.freeze, features: r[:features].map { |f| -f }.freeze,
157
+ source: r[:src]).freeze
158
+ end
159
+
160
+ private
161
+
162
+ def accessory?(title)
163
+ t = Py.lower(Py.unescape(Py.unescape(title)))
164
+ return false if GUN_WORDS.match?(t)
165
+
166
+ ACC_NOUN.each_match(t).any? { |m| !ACC_INCLUDED.match?(t[[0, m.begin(0) - 16].max...m.begin(0)]) }
167
+ end
168
+
169
+ # Python _norm() up to the dictionary's own rewrite rules (the same for every dictionary)
170
+ def base_norm(title)
171
+ s = Py.unescape(title).gsub(DASHES_RX, DASHES)
172
+ Py.strip(Py.collapse_ws(Py.lower(s)))
173
+ end
174
+
175
+ # ---------- numeric mode (Glock): codes like 19, 19X, 43X, 20SF; generation from the title or the part number
176
+
177
+ def find_models(t, d)
178
+ found = []
179
+ d.find_models.each_match(t) do |m|
180
+ num = Py.lower(m[1])
181
+ suf = Py.upper(Py.present?(m[2]) ? m[2] : (m[3] || ""))
182
+ code = if d.models.key?(num + suf) then num + suf
183
+ elsif d.models.key?(num) then num
184
+ else ""
185
+ end
186
+ next if code.empty?
187
+
188
+ found << { code: code, num: num, pos: m.begin(0), end: m.end(0), mos: Py.lower(m[6] || "") == "mos",
189
+ gen: Py.present?(m[4]) ? m[4] : (m[5] ? "V" : "") }
190
+ end
191
+ found
192
+ end
193
+
194
+ def decode_mpn(name, mpn, sku, d)
195
+ cands = [mpn, sku] + MPN_IN_NAME.each_match(name).map { |m| m[0] }
196
+ cands.each do |c|
197
+ c = Py.strip(c)
198
+ m = MPN_US.search(c)
199
+ if m && d.models.key?(Py.upper(m[1]))
200
+ return { code: Py.upper(m[1]), letter: "", mpn: c, gen: m[2] }
201
+ end
202
+
203
+ m = MPN_AUT.search(c)
204
+ return { code: m[1], letter: "", mpn: c, gen: "" } if m && d.models.key?(m[1])
205
+
206
+ m = MPN_CODE.search(c)
207
+ next unless m
208
+
209
+ letter = Py.upper(m[2])
210
+ num = m[3]
211
+ num = (d["mpn_model_alias"] || {}).fetch(num, num)
212
+ code = (d["mpn_letter_model"] || {}).fetch(letter + num, num)
213
+ next unless d.models.key?(code)
214
+
215
+ return { code: code, letter: letter, mpn: c, gen: (d["mpn_letter_gen"] || {}).fetch(letter, "") }
216
+ end
217
+ nil
218
+ end
219
+
220
+ def parse_numeric(t, name, brand_key, mpn, sku, d)
221
+ pm = decode_mpn(name, mpn, sku, d)
222
+ return nil if !pm && d.exclude_soft&.match?(t)
223
+
224
+ own = if brand_key
225
+ d.brand_names.key?(brand_key) || !pm.nil?
226
+ else
227
+ !pm.nil? || d.own_start.match?(t)
228
+ end
229
+ models = find_models(t, d)
230
+ if models.empty? && own
231
+ m = TWO_DIGITS.search(t)
232
+ if m
233
+ code = m[1] + Py.upper(m[2] || "")
234
+ models = [{ code: code, num: m[1], pos: 0, end: m.end(0), mos: false, gen: "" }] if d.models.key?(code)
235
+ end
236
+ end
237
+ if models.empty? && own && d.bare_tokens
238
+ m = d.bare_tokens.search(t)
239
+ if m
240
+ models = [{ code: Py.upper(m[1]), num: m[1][0, 2], pos: m.begin(0), end: m.end(0), mos: false, gen: "" }]
241
+ end
242
+ end
243
+ unless own
244
+ return nil if models.empty?
245
+ return nil if d.block&.match?(t)
246
+ end
247
+ src = "title"
248
+ first = nil
249
+ if models.any?
250
+ first = models[0]
251
+ code = first[:code]
252
+ models.each { |x| code = x[:code] if x[:num] == first[:num] && x[:code].length > code.length }
253
+ elsif own && pm
254
+ code = pm[:code]
255
+ src = "mpn"
256
+ else
257
+ return nil
258
+ end
259
+ gen = ""
260
+ if first && Py.present?(first[:gen])
261
+ gen = first[:gen]
262
+ elsif (m = GEN.search(t))
263
+ gen = [m[1], m[2], m[3]].find { |g| Py.present?(g) }
264
+ elsif first && GEN_V.match?(t[first[:end]..])
265
+ gen = "V"
266
+ elsif pm && Py.present?(pm[:gen]) && pm[:code] == code
267
+ gen = pm[:gen]
268
+ end
269
+ gen = "" if (d["no_gen_models"] || []).include?(code) && gen != "V"
270
+ configs = d.configs.select do |n, rx|
271
+ rx.match?(t) || (n == "MOS" && ((first && first[:mos]) || (pm && Py.lower(pm[:mpn]).include?("mos"))))
272
+ end.map(&:first)
273
+ features = d.features.select { |_, rx| rx.match?(t) }.map(&:first)
274
+ features << "Threaded Barrel" if pm && MPN_TB.match?(pm[:mpn]) && !features.include?("Threaded Barrel")
275
+ { code: code, family: d.models[code], gen: gen, configs: configs, features: features, src: src }
276
+ end
277
+
278
+ # ---------- terms mode (all other brands): ordered list of model patterns, first match wins
279
+
280
+ def parse_terms(t, brand_key, d)
281
+ return nil if brand_key && d.deny_names.key?(brand_key)
282
+
283
+ own = (brand_key && d.brand_names.key?(brand_key)) || d.brand_start.match?(t)
284
+ unless own
285
+ return nil unless d.brand_any.match?(t)
286
+ return nil if d.block&.match?(t)
287
+ end
288
+ hit = d.models.find { |m| m[:match].match?(t) && !m[:not]&.match?(t) }
289
+ return nil unless hit
290
+
291
+ allowed = d["other_brand_models"]
292
+ return nil if !own && Py.present?(allowed) && !allowed.include?(hit[:name])
293
+
294
+ gen = d.gens.find { |_, rx| rx.match?(t) }&.first || ""
295
+ implied = d["config_implied_by_gen_family"] || {}
296
+ configs = d.configs.select do |name, rx|
297
+ next false unless hit[:configs].include?(name)
298
+
299
+ imp = (implied[name] || {})[hit[:family]] || []
300
+ rx.match?(t) || (Py.present?(gen) && imp.include?(gen)) || hit[:implied_configs].include?(name)
301
+ end.map(&:first)
302
+ features = d.features.select { |_, rx| rx.match?(t) }.map(&:first)
303
+ { code: hit[:name], family: hit[:family], gen: gen, configs: configs, features: features, src: "title" }
304
+ end
305
+
306
+ # Which matching dictionary wins: the product's own brand -> brand at the start of the title -> brand
307
+ # mentioned earlier.
308
+ def rank(t, d, brand_key)
309
+ return [0, 0] if brand_key && d.brand_names.key?(brand_key)
310
+
311
+ m = d.brand_any.search(t)
312
+ return [1, 0] if m && m.begin(0) <= 2
313
+
314
+ [2, m ? m.begin(0) : 10**6]
315
+ end
316
+
317
+ def leaf(r, d)
318
+ if d.terms
319
+ s = Py.present?(r[:gen]) ? "#{r[:code]} #{r[:gen]}" : r[:code].dup
320
+ else
321
+ s = "#{d.brand} #{r[:code]}"
322
+ s += r[:gen] == "V" ? " V" : " Gen #{r[:gen]}" if Py.present?(r[:gen])
323
+ end
324
+ r[:configs].each do |c|
325
+ next if d.leaf_rx[c].match?(s)
326
+ next if (WORDS.matches(Py.lower(c)).map { |m| m[0] } - WORDS.matches(Py.lower(s)).map { |m| m[0] }).empty?
327
+
328
+ s += " #{c}"
329
+ end
330
+ needed = (d["needed"] || {})[r[:family]] || []
331
+ (d["feature_order"] || []).each do |f|
332
+ s += " #{f}" if r[:features].include?(f) && needed.include?(f)
333
+ end
334
+ s
335
+ end
336
+
337
+ def parse_one(t, title, d, brand_key, mpn, sku)
338
+ return nil if d.exclude&.match?(t)
339
+ return parse_terms(t, brand_key, d) if d.terms
340
+ # the numeric dictionary (Glock) only runs when the brand is given or named in the title;
341
+ # otherwise part-number decoding picks up other brands' codes (CVA "PR3110S" -> "Glock 31")
342
+ return nil unless (brand_key && d.brand_names.key?(brand_key)) || d.brand_any.match?(t)
343
+
344
+ parse_numeric(t, title, brand_key, mpn, sku, d)
345
+ end
346
+ end
347
+ end
348
+ end