cockatrice_feeder 0.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/gobble +4 -0
- data/lib/cockatrice_feeder.rb +369 -0
- metadata +89 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9bc7f2a80c7c57558a92463c5415eac22bfc4832
|
4
|
+
data.tar.gz: a646c73dfbc6946e1eba60ae9a4addfaf89d9733
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 280d96f1a6c97ce6f92b38feb6cb33618f991b57a07c8934051d01c230172c318feea3737b02e32d1d606b7bc7bd1c6c6bdb34855ee5f530da846772d85575d2
|
7
|
+
data.tar.gz: f394b4ab9cf428e80ba1f7df8a11dac7d5112cad210ec7c8a9618dd59b765bdffb5a0534bb27d5c362c9ac74ff336798c7d1d143f7647bc8ea8e6eabd310523c
|
data/bin/gobble
ADDED
@@ -0,0 +1,369 @@
|
|
1
|
+
module CockatriceFeeder
|
2
|
+
require 'httparty'
|
3
|
+
require 'awesome_print'
|
4
|
+
require 'nokogiri'
|
5
|
+
|
6
|
+
@@app_dir = Dir.pwd+"/"
|
7
|
+
@@deck_dir = @@app_dir+"decks/"
|
8
|
+
@@meta_dir = @@app_dir+"meta/"
|
9
|
+
|
10
|
+
def self.set_app_dir(dir)
|
11
|
+
@@app_dir = (dir + (dir[-1] != "/" ? "/" : ""))
|
12
|
+
@@deck_dir = @@app_dir+"decks/"
|
13
|
+
@@meta_dir = @@app_dir+"meta/"
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.app_dir
|
17
|
+
@@app_dir
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.set_deck_dir(dir)
|
21
|
+
@@deck_dir = (dir + (dir[-1] != "/" ? "/" : ""))
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.deck_dir
|
25
|
+
@@deck_dir
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.set_meta_dir(dir)
|
29
|
+
@@meta_dir = (dir + (dir[-1] != "/" ? "/" : ""))
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.meta_dir
|
33
|
+
@@meta_dir
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.setup
|
37
|
+
unless File.directory?(@@meta_dir)
|
38
|
+
Dir.mkdir(@@meta_dir)
|
39
|
+
puts "Creating a folder at '#{@@meta_dir}' for storing meta data."
|
40
|
+
puts "Fetching meta data."
|
41
|
+
update_commanders()
|
42
|
+
update_banned()
|
43
|
+
update_commander_tiers()
|
44
|
+
end
|
45
|
+
|
46
|
+
unless File.directory?(@@deck_dir)
|
47
|
+
Dir.mkdir(@@deck_dir)
|
48
|
+
puts "Creating a folder at '#{@@deck_dir}' for generated decks."
|
49
|
+
|
50
|
+
folders = %w(edhrecavg mtgdecks tappedout deckstats)
|
51
|
+
|
52
|
+
folders.each do |folder|
|
53
|
+
unless File.directory?(@@deck_dir+folder)
|
54
|
+
Dir.mkdir(@@deck_dir+folder)
|
55
|
+
puts "Creating a deck subfolder at '#{@@deck_dir+folder}'."
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
puts "Ready to scraw!"
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.update_commanders
|
64
|
+
puts "Downloading a list of all commanderse from EDHREC."
|
65
|
+
commander_cids = %w(
|
66
|
+
w g r u b
|
67
|
+
wu ub br rg gw wb ur bg rw gu
|
68
|
+
wub ubr brg rgw gwu wbg urw bgu rwb gur
|
69
|
+
wubr ubrg brgw rgwu gwub
|
70
|
+
wubrg
|
71
|
+
)
|
72
|
+
|
73
|
+
commanders = []
|
74
|
+
|
75
|
+
commander_cids.each do |cid|
|
76
|
+
url = "https://edhrec-json.s3.amazonaws.com/en/commanders/#{cid}.json"
|
77
|
+
commanders.concat(
|
78
|
+
JSON.parse(
|
79
|
+
HTTParty.get(url).body
|
80
|
+
)["container"]["json_dict"]["cardlists"].first["cardviews"].map{|c|
|
81
|
+
{
|
82
|
+
link: c["sanitized"],
|
83
|
+
name: c["names"].first,
|
84
|
+
color_identity: cid,
|
85
|
+
deckstats_uri: c["deckstats_uri"]
|
86
|
+
}
|
87
|
+
}
|
88
|
+
)
|
89
|
+
end
|
90
|
+
|
91
|
+
File.open(@@meta_dir+"commanders.json", "wb") do |file|
|
92
|
+
file.write(commanders.to_json)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.update_banned
|
97
|
+
puts "Downloading the current EDH banned card list."
|
98
|
+
banned_list = []
|
99
|
+
|
100
|
+
more_pages = true
|
101
|
+
page = 1
|
102
|
+
while(more_pages)
|
103
|
+
url = "https://api.magicthegathering.io/v1/cards?gameFormat=Commander&legality=Banned&page=#{page}"
|
104
|
+
cards = JSON.parse(HTTParty.get(url).body)["cards"]
|
105
|
+
banned_list.concat(cards)
|
106
|
+
page += 1
|
107
|
+
if cards.length == 0
|
108
|
+
more_pages = false
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
File.open(@@meta_dir+"banned.json", "wb") do |file|
|
113
|
+
file.write(banned_list.to_json)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.update_commander_tiers
|
118
|
+
puts "Downloading a tappedout deck that ranks EDG commanders into tiers."
|
119
|
+
doc = Nokogiri::HTML(HTTParty.get("https://tappedout.net/mtg-decks/list-multiplayer-edh-generals-by-tier/").body)
|
120
|
+
|
121
|
+
tiers = {}
|
122
|
+
doc.css(".board-container .board-col").each do |col|
|
123
|
+
col.css("h3").each_with_index do |h,i|
|
124
|
+
tier = h.content.split(")").first.gsub("(","")
|
125
|
+
|
126
|
+
tiers[tier] = []
|
127
|
+
col.css("ul")[i].css("a.card-hover").each do |a|
|
128
|
+
tiers[tier] << a.attribute("href").value.split("/").last.gsub("-foil","").downcase
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
File.open(@@meta_dir+"tiers.json", "wb") do |file|
|
135
|
+
file.write(tiers.to_json)
|
136
|
+
end
|
137
|
+
|
138
|
+
tiers
|
139
|
+
end
|
140
|
+
|
141
|
+
def self.commanders
|
142
|
+
JSON.parse(File.read(@@meta_dir+"commanders.json"))
|
143
|
+
end
|
144
|
+
|
145
|
+
def self.banned
|
146
|
+
JSON.parse(File.read(@@meta_dir+"banned.json"))
|
147
|
+
end
|
148
|
+
# names = banned.map{|c| c["name"]}.uniq.sort
|
149
|
+
|
150
|
+
def self.tiers
|
151
|
+
JSON.parse(File.read(@@meta_dir+"tiers.json"))
|
152
|
+
end
|
153
|
+
|
154
|
+
def self.output_cod(deck, subfolder)
|
155
|
+
comments = [
|
156
|
+
deck[:name],
|
157
|
+
deck[:link],
|
158
|
+
deck[:commanders].join("\n"),
|
159
|
+
deck[:price]
|
160
|
+
].join("\n")
|
161
|
+
|
162
|
+
|
163
|
+
filename = [
|
164
|
+
deck[:commanders].join("-and-"),
|
165
|
+
deck[:name],
|
166
|
+
deck[:price],
|
167
|
+
subfolder
|
168
|
+
].compact.reject(&:empty?).uniq.join('_')
|
169
|
+
|
170
|
+
builder = Nokogiri::XML::Builder.new do |xml|
|
171
|
+
xml.cockatrice_deck(:version => "1"){
|
172
|
+
xml.deckname {
|
173
|
+
xml.text(deck[:name])
|
174
|
+
}
|
175
|
+
xml.comments {
|
176
|
+
xml.text(comments)
|
177
|
+
}
|
178
|
+
xml.zone(:name => "main") {
|
179
|
+
deck[:cardlist].each do |crd|
|
180
|
+
xml.card(
|
181
|
+
:number => crd.split(" ").first,
|
182
|
+
:name => crd.split(" ")[1..-1].join(" ")
|
183
|
+
)
|
184
|
+
end
|
185
|
+
}
|
186
|
+
}
|
187
|
+
end
|
188
|
+
|
189
|
+
File.open(@@deck_dir+"#{subfolder}/#{filename}.cod", "wb") do |file|
|
190
|
+
file.write(builder.to_xml)
|
191
|
+
end
|
192
|
+
|
193
|
+
puts "created deck at #{@@deck_dir+"#{subfolder}/#{filename}.cod"}"
|
194
|
+
end
|
195
|
+
|
196
|
+
|
197
|
+
# ordering
|
198
|
+
# -rating
|
199
|
+
# -date_updated
|
200
|
+
# -competitive_score
|
201
|
+
def self.tappedout_decklist(pages = (1..10), order_by = "-date_updated", price_min = "", price_max = "")
|
202
|
+
decks = []
|
203
|
+
pages.each do |page|
|
204
|
+
doc = Nokogiri::HTML(
|
205
|
+
HTTParty.get(
|
206
|
+
"https://tappedout.net/mtg-decks/search/?q=&format=edh&is_top=on&price_min=#{price_min}&price_max=#{price_max}&o=#{order_by}&submit=Filter+results&p=#{page}&page=#{page}"
|
207
|
+
).body
|
208
|
+
)
|
209
|
+
|
210
|
+
doc.css(".deck-wide-header a").each do |a|
|
211
|
+
link = a.attribute("href").value
|
212
|
+
if link.include?("/mtg-decks/")
|
213
|
+
decks << {
|
214
|
+
name: link.split("/").last,
|
215
|
+
commanders: [],
|
216
|
+
link: "https://tappedout.net"+link,
|
217
|
+
date: nil,
|
218
|
+
price: nil,
|
219
|
+
cardlist: []
|
220
|
+
}
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
decks
|
226
|
+
end
|
227
|
+
|
228
|
+
def self.tappedout_deck(deck)
|
229
|
+
doc2 = Nokogiri::HTML(HTTParty.get(deck[:link]).body)
|
230
|
+
|
231
|
+
deck[:cardlist] = doc2.css("#mtga-textarea").first.content.
|
232
|
+
split("\n").select{|c| c != ""}.
|
233
|
+
map{|c| c.split("(").first.strip}
|
234
|
+
|
235
|
+
commanders = []
|
236
|
+
doc2.css(".board-container .board-col").each do |col|
|
237
|
+
col.css("h3").each_with_index do |h,i|
|
238
|
+
if h.content.include?("Commander")
|
239
|
+
col.css("ul")[i].css("a.card-hover").each do |a|
|
240
|
+
if a.css("img.commander-img").length > 0
|
241
|
+
commanders << a.attribute("href").value.split("/").last.gsub("-foil","")
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
deck[:commanders] = commanders
|
248
|
+
|
249
|
+
price = nil
|
250
|
+
doc2.css("form").each do |f|
|
251
|
+
fname = f.attribute("name")
|
252
|
+
if !fname.nil? && fname.value == "ck_checkout"
|
253
|
+
price = f.css("span.pull-right").first.content.split(" - ").first.
|
254
|
+
strip.gsub("$","").split(".").first
|
255
|
+
end
|
256
|
+
end
|
257
|
+
deck[:price] = price
|
258
|
+
|
259
|
+
subfolder = "tappedout"
|
260
|
+
output_cod(deck, subfolder)
|
261
|
+
end
|
262
|
+
|
263
|
+
def self.edhrecavg_decklist
|
264
|
+
commanders.map{|c| c["link"]}.map do |c|
|
265
|
+
{
|
266
|
+
name: c,
|
267
|
+
commanders: [c],
|
268
|
+
link: "https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json",
|
269
|
+
price: nil,
|
270
|
+
date: nil,
|
271
|
+
cardlist: []
|
272
|
+
}
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
def self.edhrecavg_deck(deck)
|
277
|
+
deck[:cardlist] = JSON.parse(HTTParty.get(deck[:link]).body)["description"].
|
278
|
+
split("</a>").last.split("\n").select{|s| s != ""}
|
279
|
+
|
280
|
+
output_cod(deck,"edhrecavg")
|
281
|
+
end
|
282
|
+
|
283
|
+
def self.deckstats_decklist
|
284
|
+
|
285
|
+
end
|
286
|
+
|
287
|
+
def self.deckstats_deck
|
288
|
+
|
289
|
+
end
|
290
|
+
|
291
|
+
def self.mtgdecks_decklist(pages = (1..1))
|
292
|
+
decks = []
|
293
|
+
pages.each do |page|
|
294
|
+
puts "https://mtgdecks.net/Commander/decklists/page:#{page}"
|
295
|
+
doc = Nokogiri::HTML(HTTParty.get("https://mtgdecks.net/Commander/decklists/page:#{page}").body)
|
296
|
+
doc.css(".decks tr.previewable").each do |r|
|
297
|
+
if r.css("td")[0].css(".label-danger").length == 0
|
298
|
+
decks << {
|
299
|
+
name: "", # r.css("td")[1].css("a")[0].content,
|
300
|
+
link: "https://mtgdecks.net"+r.css("td")[1].css("a")[0].attribute("href").value,
|
301
|
+
date: r.css("td")[6].css("strong")[0].content.
|
302
|
+
gsub("<span class=\"hidden-xs\">","").
|
303
|
+
gsub("</span>","").gsub(/\s+/, ""),
|
304
|
+
price: r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, ""),
|
305
|
+
commanders: [],
|
306
|
+
cardlist: []
|
307
|
+
}
|
308
|
+
end
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
312
|
+
decks
|
313
|
+
end
|
314
|
+
|
315
|
+
def self.mtgdecks_deck(deck)
|
316
|
+
puts deck[:link]
|
317
|
+
doc = Nokogiri::HTML(HTTParty.get(deck[:link]).body)
|
318
|
+
|
319
|
+
cardlist = []
|
320
|
+
doc.css(".cardItem").each do |card|
|
321
|
+
cardlist << "#{card.attribute("data-required").value} #{card.attribute("data-card-id").value}"
|
322
|
+
end
|
323
|
+
deck[:cardlist] = cardlist
|
324
|
+
|
325
|
+
commanders = []
|
326
|
+
doc.css(".breadcrumbs a").each do |a|
|
327
|
+
href = a.attribute("href").value
|
328
|
+
if href.include?("/Commander/")
|
329
|
+
commanders << href.split("/").last
|
330
|
+
end
|
331
|
+
end
|
332
|
+
deck[:commanders] = commanders
|
333
|
+
|
334
|
+
output_cod(deck,"mtgdecks")
|
335
|
+
end
|
336
|
+
|
337
|
+
def self.gobble
|
338
|
+
setup()
|
339
|
+
|
340
|
+
total_decks = 0
|
341
|
+
|
342
|
+
# puts "Fetching the most recently updated ranked decks from tappedout"
|
343
|
+
# decks = CockatriceFeeder.tappedout_decklist(1..10)
|
344
|
+
# puts "#{decks.length} decks found."
|
345
|
+
# decks.each {|d|
|
346
|
+
# CockatriceFeeder.tappedout_deck(d)
|
347
|
+
# total_decks += 1
|
348
|
+
# }
|
349
|
+
#
|
350
|
+
# puts "Fetching the first 10 pages of decks at https://mtgdecks.net/Commander/decklists/"
|
351
|
+
# decks = CockatriceFeeder.mtgdecks_decklist(1..10)
|
352
|
+
# puts "#{decks.length} decks found."
|
353
|
+
# decks.each {|d|
|
354
|
+
# CockatriceFeeder.mtgdecks_deck(d)
|
355
|
+
# total_decks += 1
|
356
|
+
# }
|
357
|
+
|
358
|
+
puts "Fetching the average deck for every commander on EDHREC"
|
359
|
+
decks = CockatriceFeeder.edhrecavg_decklist
|
360
|
+
puts "#{decks.length} decks found."
|
361
|
+
decks.each {|d|
|
362
|
+
CockatriceFeeder.edhrecavg_deck(d)
|
363
|
+
total_decks += 1
|
364
|
+
}
|
365
|
+
|
366
|
+
puts "#{total_decks} decks created."
|
367
|
+
puts "Scraw!"
|
368
|
+
end
|
369
|
+
end
|
metadata
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cockatrice_feeder
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matthew Steinwachs
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-11-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: nokogiri
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.10'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.10'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: httparty
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.18'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.18'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: awesome_print
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.8'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.8'
|
55
|
+
description: A tool to scrape MTG decks from the internet along with some meta information
|
56
|
+
and create Cockatrice compatible deck files.
|
57
|
+
email: matt.steinwachs@gmail.com
|
58
|
+
executables:
|
59
|
+
- gobble
|
60
|
+
extensions: []
|
61
|
+
extra_rdoc_files: []
|
62
|
+
files:
|
63
|
+
- bin/gobble
|
64
|
+
- lib/cockatrice_feeder.rb
|
65
|
+
homepage: https://github.com/matt-steinwachs/cockatrice_feeder
|
66
|
+
licenses:
|
67
|
+
- MIT
|
68
|
+
metadata: {}
|
69
|
+
post_install_message: Scraw!
|
70
|
+
rdoc_options: []
|
71
|
+
require_paths:
|
72
|
+
- lib
|
73
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
requirements: []
|
84
|
+
rubyforge_project:
|
85
|
+
rubygems_version: 2.6.14
|
86
|
+
signing_key:
|
87
|
+
specification_version: 4
|
88
|
+
summary: Scrape and generate decks for cockatrice
|
89
|
+
test_files: []
|