cockatrice_feeder 0.0.3 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cockatrice_feeder.rb +266 -41
- metadata +5 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35f49ec43fbddcb43336894609d1f110ed83b9c4
|
4
|
+
data.tar.gz: d43a4dc9a50c9f8e8c0dd0e8df2a1d08721ebe4d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f185893e9afa3c3cbed1c9217a55d58d7d456d6887c0078dac5b0d37f0c8ab54d5720983ce7e813ff95f2785c2caabf6e4b766f7cd90c6148213fbf230dd34d
|
7
|
+
data.tar.gz: 35bf4d00a14d7b2b7806d84ab8230915082da0f298dced2c0bd4acc3b4eef818b763292fa5b05305f9c04e5a055ca6b2a56badb051830f2c9cc5a752f2ad3c7c
|
data/lib/cockatrice_feeder.rb
CHANGED
@@ -2,7 +2,6 @@ module CockatriceFeeder
|
|
2
2
|
require 'httparty'
|
3
3
|
require 'nokogiri'
|
4
4
|
require 'fileutils'
|
5
|
-
require 'descriptive_statistics'
|
6
5
|
|
7
6
|
@@app_dir = Dir.pwd+"/"
|
8
7
|
@@deck_dir = @@app_dir+"decks/"
|
@@ -55,7 +54,7 @@ module CockatriceFeeder
|
|
55
54
|
Dir.mkdir(@@deck_dir)
|
56
55
|
puts "Creating a folder at '#{@@deck_dir}' for generated decks."
|
57
56
|
|
58
|
-
folders = %w(edhrecavg mtgdecks tappedout deckstats)
|
57
|
+
folders = %w(edhrecavg mtgdecks tappedout deckstats archidekt)
|
59
58
|
|
60
59
|
folders.each do |folder|
|
61
60
|
unless File.directory?(@@deck_dir+folder)
|
@@ -146,6 +145,15 @@ module CockatriceFeeder
|
|
146
145
|
tiers
|
147
146
|
end
|
148
147
|
|
148
|
+
def self.update_full_data
|
149
|
+
File.open(@@meta_dir+"AllPrintings.json", "w") do |file|
|
150
|
+
file.binmode
|
151
|
+
HTTParty.get("https://mtgjson.com/api/v5/AllPrintings.json", stream_body: true) do |fragment|
|
152
|
+
file.write(fragment)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
149
157
|
def self.commanders
|
150
158
|
unless File.exist?(@@meta_dir+"commanders.json")
|
151
159
|
update_commanders()
|
@@ -168,6 +176,24 @@ module CockatriceFeeder
|
|
168
176
|
JSON.parse(File.read(@@meta_dir+"tiers.json"))
|
169
177
|
end
|
170
178
|
|
179
|
+
def self.full_data
|
180
|
+
unless File.exist?(@@meta_dir+"AllPrintings.json")
|
181
|
+
update_full_data()
|
182
|
+
end
|
183
|
+
JSON.parse(File.read(@@meta_dir+"AllPrintings.json"))
|
184
|
+
end
|
185
|
+
|
186
|
+
def self.deck_obj(link = "", name = "", commanders = [], date = nil, price = nil)
|
187
|
+
{
|
188
|
+
link: link,
|
189
|
+
name: name,
|
190
|
+
commanders: commanders,
|
191
|
+
date: date,
|
192
|
+
price: price,
|
193
|
+
cardlist: []
|
194
|
+
}
|
195
|
+
end
|
196
|
+
|
171
197
|
def self.output_cod(deck, subfolder)
|
172
198
|
comments = [
|
173
199
|
deck[:name],
|
@@ -227,14 +253,7 @@ module CockatriceFeeder
|
|
227
253
|
doc.css(".deck-wide-header a").each do |a|
|
228
254
|
link = a.attribute("href").value
|
229
255
|
if link.include?("/mtg-decks/")
|
230
|
-
decks <<
|
231
|
-
name: link.split("/").last,
|
232
|
-
commanders: [],
|
233
|
-
link: "https://tappedout.net"+link,
|
234
|
-
date: nil,
|
235
|
-
price: nil,
|
236
|
-
cardlist: []
|
237
|
-
}
|
256
|
+
decks << deck_obj("https://tappedout.net"+link, link.split("/").last)
|
238
257
|
end
|
239
258
|
end
|
240
259
|
end
|
@@ -279,14 +298,7 @@ module CockatriceFeeder
|
|
279
298
|
|
280
299
|
def self.edhrecavg_decklist
|
281
300
|
commanders.map{|c| c["link"]}.map do |c|
|
282
|
-
{
|
283
|
-
name: c,
|
284
|
-
commanders: [c],
|
285
|
-
link: "https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json",
|
286
|
-
price: nil,
|
287
|
-
date: nil,
|
288
|
-
cardlist: []
|
289
|
-
}
|
301
|
+
deck_obj("https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json", c, [c])
|
290
302
|
end
|
291
303
|
end
|
292
304
|
|
@@ -300,7 +312,7 @@ module CockatriceFeeder
|
|
300
312
|
|
301
313
|
#order ["views,desc", "price,desc", "likes,desc", "updated,desc"]
|
302
314
|
#commander should be a name attribute from the commanders array of objects
|
303
|
-
def self.deckstats_decklist(commander = "
|
315
|
+
def self.deckstats_decklist(commander = "", pages = (1..1), order_by = "likes,desc", price_min = "", price_max = "")
|
304
316
|
decklist = []
|
305
317
|
pages.each do |page|
|
306
318
|
url = [
|
@@ -317,7 +329,7 @@ module CockatriceFeeder
|
|
317
329
|
"&search_number_cards_sideboard=",
|
318
330
|
"&search_cards%5B%5D=",
|
319
331
|
"&search_tags=",
|
320
|
-
"&search_order=#{URI.encode_www_form_component(
|
332
|
+
"&search_order=#{URI.encode_www_form_component(order_by)}",
|
321
333
|
"&utf8=%E2%9C%94",
|
322
334
|
"&page=#{page}"
|
323
335
|
].join("")
|
@@ -326,14 +338,7 @@ module CockatriceFeeder
|
|
326
338
|
|
327
339
|
doc.css(".deck_row").each do |dr|
|
328
340
|
link = dr.css("td")[1].css("a").first.attribute("href").value
|
329
|
-
decklist <<
|
330
|
-
link: link,
|
331
|
-
name: link.split("/")[-2],
|
332
|
-
commanders: [commander].reject(&:empty?),
|
333
|
-
price: nil,
|
334
|
-
date: nil,
|
335
|
-
cardlist: []
|
336
|
-
}
|
341
|
+
decklist << deck_obj(link,link.split("/")[-2],[commander].reject(&:empty?))
|
337
342
|
end
|
338
343
|
end
|
339
344
|
|
@@ -357,12 +362,91 @@ module CockatriceFeeder
|
|
357
362
|
sec["cards"].map{|c| "#{c["amount"]} #{c["name"]}"}
|
358
363
|
end.flatten
|
359
364
|
|
360
|
-
deck[:price] =
|
365
|
+
deck[:price] = (
|
366
|
+
!doc.css(".deck_overview_price").first.nil? ?
|
367
|
+
doc.css(".deck_overview_price").first.content.gsub("$","").strip.split(".").first
|
368
|
+
: nil
|
369
|
+
)
|
361
370
|
|
362
371
|
output_cod(deck,'deckstats')
|
363
372
|
end
|
364
373
|
end
|
365
374
|
|
375
|
+
#colors = "White,Blue,Black,Red,Green,Colorless"
|
376
|
+
#orderBy = "-updatedAt", "-createdAt", "-points", "-viewCount"
|
377
|
+
def self.archidekt_decklist(
|
378
|
+
andcolors = nil, colors = nil, commander = nil, owner = nil, formats = 3, orderBy = "-createdAt", size: 100, pageSize: 50
|
379
|
+
)
|
380
|
+
|
381
|
+
url = [
|
382
|
+
"https://www.archidekt.com/api/decks/cards/?",
|
383
|
+
[
|
384
|
+
(andcolors.nil? ? nil : "true"),
|
385
|
+
(colors.nil? ? nil : "colors=#{URI.encode_www_form_component(colors)}"),
|
386
|
+
(commander.nil? ? nil : "commanders=\"#{URI.encode_www_form_component(commander)}\""),
|
387
|
+
(owner.nil? ? nil : "owner=#{URI.encode_www_form_component(owner)}"),
|
388
|
+
"formats=#{formats}",
|
389
|
+
"orderBy=#{orderBy}",
|
390
|
+
"size=#{size}",
|
391
|
+
"pageSize=#{pageSize}"
|
392
|
+
].compact.join("&")
|
393
|
+
].join("")
|
394
|
+
|
395
|
+
puts url
|
396
|
+
|
397
|
+
decklist = []
|
398
|
+
data = JSON.parse(HTTParty.get(url).body)
|
399
|
+
|
400
|
+
data["results"].each do |r|
|
401
|
+
decklist << deck_obj("https://www.archidekt.com/decks/#{r["id"]}", r["name"])
|
402
|
+
end
|
403
|
+
|
404
|
+
decklist
|
405
|
+
end
|
406
|
+
|
407
|
+
# deck = CockatriceFeeder.deck_obj("https://www.archidekt.com/decks/992684#Rocking_that_equipment_Bro")
|
408
|
+
# CockatriceFeeder.archidekt_deck(deck)
|
409
|
+
def self.archidekt_deck(deck)
|
410
|
+
deck_id = deck[:link].split("/").last.split("#").first
|
411
|
+
|
412
|
+
api_url = "https://www.archidekt.com/api/decks/#{deck_id}/"
|
413
|
+
|
414
|
+
deck_data = JSON.parse(HTTParty.get(api_url).body)
|
415
|
+
|
416
|
+
included_categories = deck_data["categories"].select{|c| c["includedInDeck"]}.map{|c| c["name"] }
|
417
|
+
commander_categories = deck_data["categories"].select{|c| c["isPremier"]}.map{|c| c["name"] }
|
418
|
+
cardlist = []
|
419
|
+
tcg_price = 0.0
|
420
|
+
ck_price = 0.0
|
421
|
+
deck_data["cards"].each do |card|
|
422
|
+
cname = card["card"]["oracleCard"]["name"]
|
423
|
+
|
424
|
+
if card["card"]["oracleCard"]["layout"] != "split"
|
425
|
+
cname = cname.split(" // ").first
|
426
|
+
end
|
427
|
+
|
428
|
+
if (included_categories & card["categories"]).length > 0
|
429
|
+
cardlist << "#{card["quantity"]} #{cname}"
|
430
|
+
|
431
|
+
tcg_price += (card["card"]["prices"]["tcg"] * card["quantity"].to_f)
|
432
|
+
ck_price += (card["card"]["prices"]["ck"] * card["quantity"].to_f)
|
433
|
+
end
|
434
|
+
|
435
|
+
if (commander_categories & card["categories"]).length > 0
|
436
|
+
deck[:commanders] << cname
|
437
|
+
end
|
438
|
+
end
|
439
|
+
|
440
|
+
deck[:cardlist] = cardlist
|
441
|
+
deck[:price] = tcg_price.to_i.to_s
|
442
|
+
|
443
|
+
deck[:name] = deck_data["name"]
|
444
|
+
|
445
|
+
deck[:date] = deck_data["updatedAt"]
|
446
|
+
|
447
|
+
output_cod(deck,"archidekt")
|
448
|
+
end
|
449
|
+
|
366
450
|
def self.mtgdecks_decklist(pages = (1..1))
|
367
451
|
decks = []
|
368
452
|
pages.each do |page|
|
@@ -370,16 +454,14 @@ module CockatriceFeeder
|
|
370
454
|
doc = Nokogiri::HTML(HTTParty.get("https://mtgdecks.net/Commander/decklists/page:#{page}").body)
|
371
455
|
doc.css(".decks tr.previewable").each do |r|
|
372
456
|
if r.css("td")[0].css(".label-danger").length == 0
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
cardlist: []
|
382
|
-
}
|
457
|
+
link = "https://mtgdecks.net"+r.css("td")[1].css("a")[0].attribute("href").value
|
458
|
+
name = link.split("/").last
|
459
|
+
date = r.css("td")[6].css("strong")[0].content.
|
460
|
+
gsub("<span class=\"hidden-xs\">","").
|
461
|
+
gsub("</span>","").gsub(/\s+/, "")
|
462
|
+
price = r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, "")
|
463
|
+
|
464
|
+
decks << deck_obj(link,name,nil,date,price)
|
383
465
|
end
|
384
466
|
end
|
385
467
|
end
|
@@ -408,6 +490,139 @@ module CockatriceFeeder
|
|
408
490
|
output_cod(deck,"mtgdecks")
|
409
491
|
end
|
410
492
|
|
493
|
+
def self.mtggoldfish_pricer(deck)
|
494
|
+
doc = Nokogiri::HTML(HTTParty.get("https://www.mtggoldfish.com/tools/deck_pricer#paper"))
|
495
|
+
csrf_token = nil
|
496
|
+
doc.css("meta").each do |m|
|
497
|
+
if !m.attribute("name").nil? && m.attribute("name").value == "csrf-token"
|
498
|
+
csrf_token = m.attribute("content").value
|
499
|
+
end
|
500
|
+
end
|
501
|
+
|
502
|
+
doc2 = Nokogiri::HTML(HTTParty.post("https://www.mtggoldfish.com/tools/deck_pricer#paper", {
|
503
|
+
body: {
|
504
|
+
utf8: "✓",
|
505
|
+
authenticity_token: csrf_token,
|
506
|
+
deck: deck[:cardlist].join("\n")
|
507
|
+
}
|
508
|
+
}))
|
509
|
+
|
510
|
+
deck[:price] = doc2.css(".deck-price-v2.paper").first.
|
511
|
+
content.strip.split(" ").last.split(".").first.gsub(",","")
|
512
|
+
end
|
513
|
+
|
514
|
+
def self.mtgapi_card(params = {multiverse_id: nil, name: nil, set:nil})
|
515
|
+
defaults = {multiverse_id: nil, name: nil}
|
516
|
+
params = defaults.merge(params)
|
517
|
+
|
518
|
+
puts "looking up #{params}"
|
519
|
+
|
520
|
+
card = nil
|
521
|
+
if params[:multiverse_id] != nil
|
522
|
+
card = JSON.parse(
|
523
|
+
HTTParty.get("https://api.magicthegathering.io/v1/cards/#{params[:multiverse_id]}").body
|
524
|
+
)["card"]
|
525
|
+
elsif params[:name] != nil
|
526
|
+
query = [
|
527
|
+
"name=#{URI.encode_www_form_component(params[:name])}",
|
528
|
+
(params[:set] != nil ? "set=#{params[:set]}" : nil)
|
529
|
+
].compact.join("&")
|
530
|
+
|
531
|
+
card = JSON.parse(
|
532
|
+
HTTParty.get("https://api.magicthegathering.io/v1/cards?#{query}").body
|
533
|
+
)["cards"].first
|
534
|
+
end
|
535
|
+
return card
|
536
|
+
end
|
537
|
+
|
538
|
+
# CockatriceFeeder.build_deck_from_sets("Lazav, Dimir Mastermind", ["RAV", "GPT", "DIS", "RTR", "GTC", "DGM", "GRN", "RNA", "WAR"])
|
539
|
+
def self.build_deck_from_sets(commander, sets)
|
540
|
+
#lookup edhrecavg deck for commander
|
541
|
+
c = commanders.select{|c| c["name"] == commander}.first
|
542
|
+
d = deck_obj("https://edhrec-json.s3.amazonaws.com/en/decks/#{c["link"]}.json", c["name"]+" SCRAW", [c["name"]])
|
543
|
+
edhrecavg_deck(d)
|
544
|
+
|
545
|
+
#remember cards to avoid repeat lookups to mtgapi
|
546
|
+
mem = {}
|
547
|
+
|
548
|
+
#throw out any cards not in sets
|
549
|
+
d[:cardlist] = d[:cardlist].select do |cl|
|
550
|
+
name = cl.split(' ')[1..-1].join(" ")
|
551
|
+
unless mem.has_key?(name)
|
552
|
+
mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
|
553
|
+
end
|
554
|
+
card = mem[name]
|
555
|
+
|
556
|
+
(card["printings"] & sets).length > 0
|
557
|
+
end
|
558
|
+
|
559
|
+
#lookup archidekt decks for the commander to find other cards from the set
|
560
|
+
arch_decks = archidekt_decklist(nil, nil, c["name"])
|
561
|
+
arch_decks.each do |ad| archidekt_deck(ad) end
|
562
|
+
arch_decks.each do |ad|
|
563
|
+
in_set = ad[:cardlist].select do |cl|
|
564
|
+
name = cl.split(' ')[1..-1].join(" ")
|
565
|
+
unless mem.has_key?(name)
|
566
|
+
mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
|
567
|
+
end
|
568
|
+
card = mem[name]
|
569
|
+
|
570
|
+
#flip/dual cards don't always work
|
571
|
+
if card != nil
|
572
|
+
(card["printings"] & sets).length > 0
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
in_set.each do |cl|
|
577
|
+
name = cl.split(' ')[1..-1].join(" ")
|
578
|
+
cur = d[:cardlist].map{|x| x.split(' ')[1..-1].join(" ")}
|
579
|
+
curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
|
580
|
+
unless cur.include?(name) || curlen >= 100
|
581
|
+
d[:cardlist] << cl
|
582
|
+
end
|
583
|
+
end
|
584
|
+
|
585
|
+
curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
|
586
|
+
if curlen >= 100
|
587
|
+
break
|
588
|
+
end
|
589
|
+
end
|
590
|
+
|
591
|
+
#lookup deckstats decks for the commander
|
592
|
+
curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
|
593
|
+
if curlen < 100
|
594
|
+
dstat_decks = deckstats_decklist(c["name"],1..5)
|
595
|
+
dstat_decks.each do |ds| deckstats_deck(ds) end
|
596
|
+
dstat_decks.each do |ds|
|
597
|
+
in_set = ds[:cardlist].select do |cl|
|
598
|
+
name = cl.split(' ')[1..-1].join(" ")
|
599
|
+
unless mem.has_key?(name)
|
600
|
+
mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
|
601
|
+
end
|
602
|
+
card = mem[name]
|
603
|
+
|
604
|
+
(card["printings"] & sets).length > 0
|
605
|
+
end
|
606
|
+
|
607
|
+
in_set.each do |cl|
|
608
|
+
name = cl.split(' ')[1..-1].join(" ")
|
609
|
+
cur = d[:cardlist].map{|x| x.split(' ')[1..-1].join(" ")}
|
610
|
+
curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
|
611
|
+
unless cur.include?(name) || curlen >= 100
|
612
|
+
d[:cardlist] << cl
|
613
|
+
end
|
614
|
+
end
|
615
|
+
|
616
|
+
curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
|
617
|
+
if curlen >= 100
|
618
|
+
break
|
619
|
+
end
|
620
|
+
end
|
621
|
+
end
|
622
|
+
|
623
|
+
d
|
624
|
+
end
|
625
|
+
|
411
626
|
def self.gobble
|
412
627
|
setup(skip_meta = true)
|
413
628
|
update_commanders()
|
@@ -435,7 +650,7 @@ module CockatriceFeeder
|
|
435
650
|
puts "#{decks.length} decks found."
|
436
651
|
decks.each {|d|
|
437
652
|
CockatriceFeeder.mtgdecks_deck(d)
|
438
|
-
if d[:cardlist].length
|
653
|
+
if d[:cardlist].length > 0
|
439
654
|
total_decks += 1
|
440
655
|
end
|
441
656
|
}
|
@@ -445,7 +660,17 @@ module CockatriceFeeder
|
|
445
660
|
puts "#{decks.length} decks found."
|
446
661
|
decks.each {|d|
|
447
662
|
CockatriceFeeder.deckstats_deck(d)
|
448
|
-
if d[:cardlist].length
|
663
|
+
if d[:cardlist].length > 0
|
664
|
+
total_decks += 1
|
665
|
+
end
|
666
|
+
}
|
667
|
+
|
668
|
+
puts "Fetching the first page of edh decks from archidekt ordered by createdAt"
|
669
|
+
decks = CockatriceFeeder.archidekt_decklist()
|
670
|
+
puts "#{decks.length} decks found."
|
671
|
+
decks.each {|d|
|
672
|
+
CockatriceFeeder.archidekt_deck(d)
|
673
|
+
if d[:cardlist].length > 0
|
449
674
|
total_decks += 1
|
450
675
|
end
|
451
676
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cockatrice_feeder
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Steinwachs
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -38,22 +38,8 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0.18'
|
41
|
-
|
42
|
-
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - "~>"
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '2.5'
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - "~>"
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '2.5'
|
55
|
-
description: A tool to scrape MTG decks from the internet along with some meta information
|
56
|
-
and create Cockatrice compatible deck files.
|
41
|
+
description: A tool to scrape MTG EDH decks from the internet along with some meta
|
42
|
+
information and create Cockatrice compatible deck files.
|
57
43
|
email: matt.steinwachs@gmail.com
|
58
44
|
executables:
|
59
45
|
- gobble
|
@@ -85,5 +71,5 @@ rubyforge_project:
|
|
85
71
|
rubygems_version: 2.6.14
|
86
72
|
signing_key:
|
87
73
|
specification_version: 4
|
88
|
-
summary: Scrape and generate decks for cockatrice
|
74
|
+
summary: Scrape and generate MTG EDH decks for cockatrice
|
89
75
|
test_files: []
|