gunsnprices 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/gunsnprices/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "gunsnprices"
7
+ spec.version = GunsNPrices::VERSION
8
+ spec.authors = ["GunsNPrices"]
9
+ spec.email = ["app@gunsnprices.com"]
10
+
11
+ spec.summary = "Parse firearm and ammunition product titles: brand, model, caliber, magazine capacity, " \
12
+ "barrel length, rounds per box, cost per round."
13
+ spec.description = "The title parser behind GunsNPrices (gunsnprices.com), a US gun and ammo price comparison " \
14
+ "site. Reads store titles such as \"G19 GEN5 9MM 15RD FS\" or \"Federal 308 150gr 20rds\" " \
15
+ "and returns the brand and model (50 brands), calibers (293), magazine capacity, barrel " \
16
+ "length, rounds per box and cost per round. Ruby port of the Python package gunsnprices " \
17
+ "with identical results. Pure Ruby, no runtime dependencies; the model dictionaries and " \
18
+ "caliber list (CC BY 4.0) ship inside the gem."
19
+ spec.homepage = "https://gunsnprices.com"
20
+ spec.licenses = ["MIT", "CC-BY-4.0"]
21
+ spec.required_ruby_version = ">= 3.0"
22
+
23
+ spec.metadata = {
24
+ "homepage_uri" => "https://gunsnprices.com",
25
+ "source_code_uri" => "https://github.com/gunsnprices/gunsnprices-ruby",
26
+ "documentation_uri" => "https://github.com/gunsnprices/gunsnprices-ruby#readme",
27
+ "changelog_uri" => "https://github.com/gunsnprices/gunsnprices-ruby/blob/main/CHANGELOG.md",
28
+ "bug_tracker_uri" => "https://github.com/gunsnprices/gunsnprices-ruby/issues",
29
+ "rubygems_mfa_required" => "true"
30
+ }
31
+
32
+ spec.files = Dir["lib/**/*.rb", "data/*.json", "bin/gunsnprices"] +
33
+ %w[README.md CHANGELOG.md LICENSE LICENSE-DATA gunsnprices.gemspec]
34
+ spec.bindir = "bin"
35
+ spec.executables = ["gunsnprices"]
36
+ spec.require_paths = ["lib"]
37
+ end
@@ -0,0 +1,280 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "py"
4
+ require_relative "types"
5
+ require_relative "data_files"
6
+
7
+ module GunsNPrices
8
+ # Recognize firearm calibers in product titles and normalize caliber fields.
9
+ #
10
+ # The caliber list and the retailer spellings come from the GunsNPrices calibers dataset. On top of the dataset
11
+ # spellings, common title forms are generated: "300 Blackout", "12ga", "7.62 x 39", "45 Auto", "6.5 Creed",
12
+ # ".308 Winchester" and so on.
13
+ # @api private
14
+ module Calibers
15
+ Rx = Py::Rx
16
+
17
+ # word -> spellings used in titles
18
+ SYN = {
19
+ "win" => %w[win winchester], "winchester" => %w[winchester win],
20
+ "rem" => %w[rem remington], "remington" => %w[remington rem],
21
+ "mag" => %w[mag magnum], "magnum" => %w[magnum mag],
22
+ "wby" => %w[wby weatherby], "weatherby" => %w[weatherby wby],
23
+ "spl" => %w[spl special], "special" => %w[special spl],
24
+ "sprg" => %w[sprg springfield spfd], "springfield" => %w[springfield sprg spfd],
25
+ "government" => %w[government govt gov],
26
+ "acp" => %w[acp auto], "auto" => %w[auto acp],
27
+ "s&w" => ["s&w", "s & w", "sw", "smith & wesson", "smith and wesson", "smith&wesson"],
28
+ "blackout" => ["blackout", "blk", "black out"],
29
+ "creedmoor" => %w[creedmoor creed cm crd creedmore],
30
+ "parabellum" => %w[parabellum para luger],
31
+ "gauge" => %w[gauge ga gauges],
32
+ "bore" => %w[bore ga gauge],
33
+ "lr" => ["lr", "long rifle"],
34
+ "h&h" => ["h&h", "h & h", "holland & holland", "holland and holland"],
35
+ "h&r" => ["h&r", "h & r"],
36
+ "bm" => %w[bm bushmaster bush]
37
+ }.freeze
38
+ # a word that may be left out ("300 Blackout" for ".300 AAC Blackout")
39
+ OPTIONAL = %w[aac nato win winchester rem remington sprg springfield government].freeze
40
+ SEP = '[\s\-]*'
41
+
42
+ # common title spellings that the synonym table does not produce
43
+ EXTRA = {
44
+ ".300 AAC Blackout" => ["300 aac", "300 bo", "300 black out", "300 blackout", "300 blk"],
45
+ "6.5mm Creedmoor" => ["6.5 creedmoor", "6.5 crd", "6.5 cm", "6.5mm cm", "6.5 cr"],
46
+ "6mm Creedmoor" => ["6 creedmoor"],
47
+ ".22 LR" => ["22lr", "22 lr", "22 long rifle"],
48
+ ".22 WMR" => ["22 mag", "22 magnum", "22 win mag", "22 wmr"],
49
+ "5.56×45mm NATO" => ["5.56 nato", "5.56mm nato", "556 nato", "5.56x45", "5.56", "5.56mm", "5.56 mm"],
50
+ "7.62x51mm NATO" => ["7.62 nato", "7.62mm nato", "762 nato", "762x51"],
51
+ "7.62x39mm" => ["762x39"],
52
+ ".410 Bore" => ["410 ga", "410 gauge", "410 bore"],
53
+ ".45 Colt" => ["45 lc", "45 long colt"],
54
+ ".44 S&W Special" => ["44 spl", "44 special", "44 spcl"],
55
+ ".38 S&W Special" => ["38 spl", "38 special", "38 spcl", "38 spc", "38 sp", "38 spp", "38+p"],
56
+ ".357 Mag" => ["357 rm", "357 rem mag", "357 rem magnum", "357 remington magnum", "357 remington mag"],
57
+ "9×19mm Parabellum" => ["9mm luger", "9mm para", "9 mm luger", "9mm"],
58
+ "10mm Auto" => ["10mm", "10 mm"],
59
+ ".45 ACP" => ["45 acp", "45 auto", "45 automatic", "45 automatic colt pistol"],
60
+ ".40 S&W" => ["40 s&w", "40 sw", "40 cal", "40s", ".40"],
61
+ ".380 ACP" => ["380 acp", "380 auto", "380"],
62
+ ".243 Win" => ["243", ".243"],
63
+ ".308 Win" => ["308", ".308"],
64
+ ".350 Legend" => [".350", "350 leg", "350 lgnd", "350 lgd"],
65
+ ".450 BM" => ["450 bushmaster", "450 bush"],
66
+ ".204 Ruger" => [".204"],
67
+ ".177" => ["177", ".177"],
68
+ ".300 Win Mag" => ["300 wm"],
69
+ "7mm Remington Magnum" => ["7mm rm", "7 rm", "7mm rem mag"],
70
+ ".338 Lapua Mag" => ["338 lapua", "338 lap"],
71
+ ".30 Carbine" => ["30 m1", "30 m1 carbine"],
72
+ ".30-06 Springfield" => ["30-06 sp"],
73
+ "7.92×57mm Mauser" => ["8x57", "8x57mm", "8x57 js", "8x57 is"],
74
+ ".50 AE" => ["50 action express"],
75
+ "12 Gauge" => ["12 ga", "12ga", "12 gauge", "12-gauge", "12g"],
76
+ "20 Gauge" => ["20 ga", "20ga", "20 gauge", "20-gauge", "20g"],
77
+ "16 Gauge" => ["16 ga", "16ga", "16 gauge", "16g"],
78
+ "28 Gauge" => ["28 ga", "28ga", "28 gauge", "28g"],
79
+ "10 Gauge" => ["10g"]
80
+ }.freeze
81
+ # a bare number or "Nmm" ("20", "307", "6.5mm") in a title is usually something else, not a caliber;
82
+ # the exceptions below were checked against GunsNPrices listings (right in 97%+ of cases)
83
+ BARE = Rx.new('^\.?\d+(?:\.\d+)?(?:\s*mm)?$')
84
+ ALLOW_BARE = ["380", "410", "9mm", "10mm", "10 mm", "5.56", "5.56mm", "5.56 mm", "243", ".243", "308", ".308",
85
+ ".40", ".350", ".204", "177", ".177"].freeze
86
+ # "12g" next to CO2 is a 12-gram CO2 cartridge, not 12 gauge
87
+ GRAM = Rx.new('^\d+\s?g$')
88
+ # shotguns "O/U 12/28": gauge / barrel length; on ammo "20/25" is rounds per box / boxes per case;
89
+ # "20/28 gauge" is a two-gauge set
90
+ CONTEXT = [
91
+ [Rx.new('(?<![a-z0-9./$#-])12/(?:1[89]|2\d|3[0-4])(?:\.5)?(?![\d/.])(?!\s*(?:ga|gauge|rd|round))'), "12gauge"],
92
+ [Rx.new('(?<![a-z0-9./$#-])20/(?:18|18\.5|2[0-46-9]|3[0-2])(?![\d/.])(?!\s*(?:ga|gauge|rd|round))'), "20gauge"],
93
+ [Rx.new('(?<![a-z0-9./$#-])28/(?:2[6-9]|3[0-2])(?![\d/.])(?!\s*(?:ga|gauge|rd|round))'), "28gauge"],
94
+ [Rx.new('(?<![a-z0-9.$#-])\.?357(?=\s*/\s*\.?38)'), "357mag"] # ".357/.38 Spl"
95
+ ].freeze
96
+
97
+ KEY_DOT = Rx.new('(?<=\d)\.(?=\d)')
98
+ KEY_STRIP = Rx.new('[^a-z0-9#&]+')
99
+ LEAD_DOT = Rx.new('^\.')
100
+ MM_PAIR = Rx.new('\.?\d+(?:\.\d+)?x\d+mm[a-z]*', anchor: :full) # 9x19mm, 5.56x45mm, 7.62x54mmr
101
+ MM_ONE = Rx.new('\d+mm', anchor: :full)
102
+ X_DIGITS = Rx.new('(?<=\d)x(?=\d)')
103
+ HAS_DIGIT = Rx.new('\d')
104
+
105
+ # The spelling index, built from the data on first use.
106
+ Index = Struct.new(:by_key, :big, :by_id, :list, :spellings, keyword_init: true)
107
+
108
+ @index = nil
109
+ @mutex = Mutex.new
110
+
111
+ class << self
112
+ def find_calibers(text)
113
+ ix = index
114
+ s = prep(text)
115
+ co2 = s.include?("co2")
116
+ hits = []
117
+ ix.big.each_match(s) do |m|
118
+ next if co2 && GRAM.match?(m[0])
119
+
120
+ c = ix.by_key[key(m[0])]
121
+ hits << [m.begin(0), c] if c
122
+ end
123
+ CONTEXT.each do |rx, cid|
124
+ rx.each_match(s) { |m| hits << [m.begin(0), ix.by_id[cid]] }
125
+ end
126
+ seen = {}
127
+ out = []
128
+ stable_sort_by(hits) { |pos, _| pos }.each do |_, c|
129
+ next if seen[c.id]
130
+
131
+ seen[c.id] = true
132
+ out << c
133
+ end
134
+ out.freeze
135
+ end
136
+
137
+ def find_caliber(text)
138
+ find_calibers(text).first
139
+ end
140
+
141
+ def normalize_caliber(value)
142
+ index.by_key[key(value)] || find_caliber(value)
143
+ end
144
+
145
+ def list_calibers
146
+ index.list
147
+ end
148
+
149
+ def caliber_by_id(id)
150
+ index.by_id[id]
151
+ end
152
+
153
+ # [spelling key, caliber id] in the order the big pattern tries them (for the tests)
154
+ def spellings
155
+ index.spellings
156
+ end
157
+
158
+ def index
159
+ @index || @mutex.synchronize { @index ||= build_index }
160
+ end
161
+
162
+ # lookup key of a spelling: lower case, letters and digits only, "&" as "and", decimal points kept
163
+ def key(s)
164
+ s = Py.lower(Py.unescape(Py.str(s))).tr("×х", "xx")
165
+ s = KEY_DOT.sub(s, "#")
166
+ s = KEY_STRIP.sub(s, "")
167
+ s.tr("#", ".").gsub("&", "and")
168
+ end
169
+
170
+ # all title spellings generated from a caliber name (Python: at most 400 combinations)
171
+ def variants(name)
172
+ toks = tokens(name)
173
+ return [] if toks.empty? || name.include?("/")
174
+
175
+ out = {}
176
+ product(toks.map { |t| token_alts(t) }, 400).each do |combo|
177
+ v = combo.reject(&:empty?).join(" ")
178
+ out[v] = true if !v.empty? && (!BARE.match?(v) || ALLOW_BARE.include?(v))
179
+ end
180
+ out.keys.sort
181
+ end
182
+
183
+ private
184
+
185
+ def prep(text)
186
+ s = Py.lower(Py.unescape(Py.str(text))).gsub("×", "x").gsub("”", '"').gsub("″", '"')
187
+ s = s.delete("®").delete("™")
188
+ Py.collapse_ws(s)
189
+ end
190
+
191
+ def tokens(name)
192
+ s = Py.lower(Py.unescape(name)).gsub("×", "x")
193
+ s = LEAD_DOT.sub(s, "")
194
+ Py.split_ws(Py.strip(s)).reject(&:empty?)
195
+ end
196
+
197
+ def token_alts(tok)
198
+ return [tok, tok.gsub("mm", "")] if MM_PAIR.match?(tok)
199
+ return [tok, "#{tok[0...-2]} mm"] if MM_ONE.match?(tok)
200
+
201
+ alts = (SYN[tok] || [tok]).dup
202
+ alts << "" if OPTIONAL.include?(tok)
203
+ alts
204
+ end
205
+
206
+ # itertools.islice(itertools.product(*lists), limit): the last list changes fastest
207
+ def product(lists, limit)
208
+ out = []
209
+ walk = lambda do |i, acc|
210
+ if i == lists.size
211
+ out << acc.dup
212
+ return
213
+ end
214
+ lists[i].each do |x|
215
+ break if out.size >= limit
216
+
217
+ acc.push(x)
218
+ walk.call(i + 1, acc)
219
+ acc.pop
220
+ end
221
+ end
222
+ walk.call(0, [])
223
+ out
224
+ end
225
+
226
+ # Python-syntax pattern for one spelling: optional leading dot, flexible "x" and separators
227
+ def spelling_rx(variant)
228
+ parts = variant.empty? ? [""] : variant.split(/ /, -1) # Python str.split(" ")
229
+ body = parts.map do |tok|
230
+ t = Py.escape(tok)
231
+ t = X_DIGITS.sub(t) { '\s*x\s*' }
232
+ t.gsub("\\-") { SEP }
233
+ end.join(SEP)
234
+ first = variant[0]
235
+ first && Py.digit_value(first) ? "\\.?#{body}" : body
236
+ end
237
+
238
+ def build_index
239
+ raw = DataFiles.calibers
240
+ by_id = {}
241
+ conflict = {}
242
+ pats = {}
243
+ raw.each do |c|
244
+ cal = Caliber.new(id: c["id"], name: c["name"], type: c["type"], url: c["url"] || "").freeze
245
+ by_id[c["id"]] = cal
246
+ names = [c["name"], c["site_name"]] + c["aliases"]
247
+ (names + (EXTRA[c["name"]] || [])).each do |n|
248
+ own = Py.lower(Py.unescape(n))
249
+ vs = variants(n)
250
+ vs += [own] if !BARE.match?(own) || ALLOW_BARE.include?(own)
251
+ vs.each do |v|
252
+ k = key(v)
253
+ next if k.empty? || !HAS_DIGIT.match?(k)
254
+
255
+ cands = (conflict[k] ||= [])
256
+ cands << [c.fetch("n", 0), cal] if cands.none? { |_, x| x.id == cal.id }
257
+ pats[k] ||= v != own || !v.include?("/") ? spelling_rx(v) : Py.escape(v)
258
+ end
259
+ end
260
+ end
261
+ by_key = {}
262
+ conflict.each do |k, cands|
263
+ cands = stable_sort_by(cands) { |n, _| -n }
264
+ # one spelling, several calibers: keep the one with 10x+ more listings, otherwise the spelling is ambiguous
265
+ by_key[k] = cands[0][1] if cands.size == 1 || cands[0][0] >= 10 * [1, cands[1][0]].max
266
+ end
267
+ keys = stable_sort_by(by_key.keys) { |k| -pats[k].length }
268
+ big = Rx.new("(?<![a-z0-9.$])(?:#{keys.map { |k| pats[k] }.join("|")})(?![a-z0-9])", ignore_case: true)
269
+ list = stable_sort_by(by_id.values, &:name).freeze
270
+ Index.new(by_key: by_key.freeze, big: big, by_id: by_id.freeze, list: list,
271
+ spellings: keys.map { |k| [k, by_key[k].id].freeze }.freeze).freeze
272
+ end
273
+
274
+ # Python's sorted(): stable
275
+ def stable_sort_by(items)
276
+ items.each_with_index.sort_by { |x, i| [yield(x), i] }.map(&:first)
277
+ end
278
+ end
279
+ end
280
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "optparse"
5
+ require_relative "../gunsnprices"
6
+
7
+ module GunsNPrices
8
+ # The `gunsnprices` command: parse titles given as arguments (pretty JSON) or read from standard input, one per
9
+ # line (JSON Lines out).
10
+ # @api private
11
+ module CLI
12
+ module_function
13
+
14
+ # @return [Integer] exit status
15
+ def run(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr)
16
+ opts = {}
17
+ parser = OptionParser.new do |o|
18
+ o.banner = "Usage: gunsnprices [options] [TITLE...]\n" \
19
+ "Parse firearm and ammunition product titles. Without TITLE, reads titles from standard input, " \
20
+ "one per line (JSON Lines out)."
21
+ o.on("--brand BRAND", "manufacturer, if known") { |v| opts[:brand] = v }
22
+ o.on("--mpn MPN", "manufacturer part number, if known") { |v| opts[:mpn] = v }
23
+ o.on("--sku SKU", "store SKU, if known") { |v| opts[:sku] = v }
24
+ o.on("--price PRICE", Float, "listing price (gives cost per round for ammo)") { |v| opts[:price] = v }
25
+ o.on("--kind KIND", %w[firearm ammo], "firearm or ammo: skip the guess") { |v| opts[:kind] = v }
26
+ o.on("-v", "--version", "print the version") do
27
+ stdout.puts "gunsnprices #{VERSION}"
28
+ return 0
29
+ end
30
+ o.on("-h", "--help", "print this help") do
31
+ stdout.puts o
32
+ return 0
33
+ end
34
+ end
35
+ begin
36
+ words = parser.parse(argv)
37
+ rescue OptionParser::ParseError => e
38
+ stderr.puts "gunsnprices: #{e.message}"
39
+ stderr.puts parser.banner
40
+ return 2
41
+ end
42
+ if words.any?
43
+ stdout.puts JSON.pretty_generate(GunsNPrices.parse_title(words.join(" "), **opts).to_h)
44
+ else
45
+ stdin.each_line do |line|
46
+ title = line.strip
47
+ stdout.puts JSON.generate(GunsNPrices.parse_title(title, **opts).to_h) unless title.empty?
48
+ end
49
+ end
50
+ 0
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module GunsNPrices
6
+ # The model dictionaries and the caliber list shipped in the gem's data/ directory (CC BY 4.0, see LICENSE-DATA).
7
+ # Read on first use; JSON objects keep their order (the order of rewrite rules, configurations and features matters).
8
+ # @api private
9
+ module DataFiles
10
+ DIR = File.expand_path("../../data", __dir__)
11
+
12
+ @mutex = Mutex.new
13
+ @cache = {}
14
+
15
+ class << self
16
+ # @return [Array<Hash>] the model dictionaries, one per brand
17
+ def models
18
+ load("models.json")["dicts"]
19
+ end
20
+
21
+ # @return [Array<Hash>] the calibers
22
+ def calibers
23
+ load("calibers.json")["calibers"]
24
+ end
25
+
26
+ private
27
+
28
+ def load(name)
29
+ @cache[name] || @mutex.synchronize do
30
+ @cache[name] ||= deep_freeze(JSON.parse(File.read(File.join(DIR, name), encoding: "UTF-8"), freeze: true))
31
+ end
32
+ end
33
+
34
+ # (json before 2.3 ignores freeze: true)
35
+ def deep_freeze(value)
36
+ return value if value.frozen? && !value.is_a?(Hash) && !value.is_a?(Array)
37
+
38
+ case value
39
+ when Hash then value.each { |k, v| deep_freeze(k) && deep_freeze(v) }
40
+ when Array then value.each { |v| deep_freeze(v) }
41
+ end
42
+ value.freeze
43
+ end
44
+ end
45
+ end
46
+ end