cockatrice_feeder 0.0.0 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9bc7f2a80c7c57558a92463c5415eac22bfc4832
-  data.tar.gz: a646c73dfbc6946e1eba60ae9a4addfaf89d9733
+  metadata.gz: 943a26f97319e634fd454cc60782500e15dcf6d6
+  data.tar.gz: e46ec8058c325032947d80c31b81179e18501a88
 SHA512:
-  metadata.gz: 280d96f1a6c97ce6f92b38feb6cb33618f991b57a07c8934051d01c230172c318feea3737b02e32d1d606b7bc7bd1c6c6bdb34855ee5f530da846772d85575d2
-  data.tar.gz: f394b4ab9cf428e80ba1f7df8a11dac7d5112cad210ec7c8a9618dd59b765bdffb5a0534bb27d5c362c9ac74ff336798c7d1d143f7647bc8ea8e6eabd310523c
+  metadata.gz: bfb439e966d8a1a2201b04dbad02209e88855ce688bcfc699f290d439b03d4c914520b4af1d09c8821f41c9f724be9f39ec614f87c4936870dc435a08e044a69
+  data.tar.gz: 43b50c5b107abb0dc447ea5f49bc9ab65f33b68597873fc238cd66302add8ec1800c9309e1dc1d80213338e6e3bc225d6b0e7e14e4453d440e65889a3277c415

data/bin/gobble

@@ -1,4 +1,3 @@
 #!/usr/bin/env ruby
-
 require 'cockatrice_feeder'
 CockatriceFeeder.gobble

data/lib/cockatrice_feeder.rb

@@ -1,7 +1,7 @@
 module CockatriceFeeder
   require 'httparty'
-  require 'awesome_print'
   require 'nokogiri'
+  require 'fileutils'
 
   @@app_dir = Dir.pwd+"/"
   @@deck_dir = @@app_dir+"decks/"
@@ -11,6 +11,9 @@ module CockatriceFeeder
     @@app_dir = (dir + (dir[-1] != "/" ? "/" : ""))
     @@deck_dir = @@app_dir+"decks/"
     @@meta_dir = @@app_dir+"meta/"
+
+    puts "decks will go here: #{@@deck_dir}"
+    puts "meta data will go here: #{@@meta_dir}"
   end
 
   def self.app_dir
@@ -19,6 +22,7 @@ module CockatriceFeeder
 
   def self.set_deck_dir(dir)
     @@deck_dir = (dir + (dir[-1] != "/" ? "/" : ""))
+    puts "decks will go here: #{@@deck_dir}"
   end
 
   def self.deck_dir
@@ -27,27 +31,30 @@ module CockatriceFeeder
 
   def self.set_meta_dir(dir)
     @@meta_dir = (dir + (dir[-1] != "/" ? "/" : ""))
+    puts "meta data will go here: #{@@meta_dir}"
   end
 
   def self.meta_dir
     @@meta_dir
   end
 
-  def self.setup
+  def self.setup(skip_meta = false)
     unless File.directory?(@@meta_dir)
       Dir.mkdir(@@meta_dir)
       puts "Creating a folder at '#{@@meta_dir}' for storing meta data."
-      puts "Fetching meta data."
-      update_commanders()
-      update_banned()
-      update_commander_tiers()
+      unless skip_meta
+        puts "Fetching meta data."
+        update_commanders()
+        update_banned()
+        update_commander_tiers()
+      end
     end
 
     unless File.directory?(@@deck_dir)
       Dir.mkdir(@@deck_dir)
       puts "Creating a folder at '#{@@deck_dir}' for generated decks."
 
-      folders = %w(edhrecavg mtgdecks tappedout deckstats)
+      folders = %w(edhrecavg mtgdecks tappedout deckstats archidekt)
 
       folders.each do |folder|
         unless File.directory?(@@deck_dir+folder)
@@ -61,7 +68,7 @@ module CockatriceFeeder
   end
 
   def self.update_commanders
-    puts "Downloading a list of all commanderse from EDHREC."
+    puts "Downloading a list of all commanders from EDHREC."
     commander_cids = %w(
       w g r u b
       wu ub br rg gw wb ur bg rw gu
@@ -139,18 +146,38 @@ module CockatriceFeeder
   end
 
   def self.commanders
+    unless File.exist?(@@meta_dir+"commanders.json")
+      update_commanders()
+    end
     JSON.parse(File.read(@@meta_dir+"commanders.json"))
   end
 
   def self.banned
+    unless File.exist?(@@meta_dir+"banned.json")
+      update_banned()
+    end
     JSON.parse(File.read(@@meta_dir+"banned.json"))
   end
   # names = banned.map{|c| c["name"]}.uniq.sort
 
-  def self.tiers
+  def self.commander_tiers
+    unless File.exist?(@@meta_dir+"tiers.json")
+      update_commander_tiers()
+    end
     JSON.parse(File.read(@@meta_dir+"tiers.json"))
   end
 
+  def self.deck_obj(link = "", name = "", commanders = [], date = nil, price = nil)
+    {
+      link: link,
+      name: name,
+      commanders: commanders,
+      date: nil,
+      price: nil,
+      cardlist: []
+    }
+  end
+
   def self.output_cod(deck, subfolder)
     comments = [
       deck[:name],
@@ -165,7 +192,7 @@ module CockatriceFeeder
       deck[:name],
       deck[:price],
       subfolder
-    ].compact.reject(&:empty?).uniq.join('_')
+    ].compact.reject(&:empty?).uniq.join('_').gsub("/","")
 
     builder = Nokogiri::XML::Builder.new do |xml|
       xml.cockatrice_deck(:version => "1"){
@@ -210,14 +237,7 @@ module CockatriceFeeder
       doc.css(".deck-wide-header a").each do |a|
         link = a.attribute("href").value
         if link.include?("/mtg-decks/")
-          decks << {
-            name: link.split("/").last,
-            commanders: [],
-            link: "https://tappedout.net"+link,
-            date: nil,
-            price: nil,
-            cardlist: []
-          }
+          decks << deck_obj("https://tappedout.net"+link, link.split("/").last)
         end
       end
     end
@@ -262,14 +282,7 @@ module CockatriceFeeder
 
   def self.edhrecavg_decklist
     commanders.map{|c| c["link"]}.map do |c|
-      {
-        name: c,
-        commanders: [c],
-        link: "https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json",
-        price: nil,
-        date: nil,
-        cardlist: []
-      }
+      deck_obj("https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json", c, [c])
     end
   end
 
@@ -280,12 +293,141 @@ module CockatriceFeeder
     output_cod(deck,"edhrecavg")
   end
 
-  def self.deckstats_decklist
 
+  #order ["views,desc", "price,desc", "likes,desc", "updated,desc"]
+  #commander should be a name attribute from the commanders array of objects
+  def self.deckstats_decklist(commander = "", pages = (1..1), order_by = "likes,desc", price_min = "", price_max = "")
+    decklist = []
+    pages.each do |page|
+      url = [
+        "https://deckstats.net/decks/search/?lng=en",
+        "&search_title=",
+        "&search_format=10",
+        "&search_season=0",
+        "&search_cards_commander%5B%5D=#{URI.encode_www_form_component(commander)}",
+        "&search_cards_commander%5B%5D=",
+        "&search_price_min=#{price_min}",
+        "&search_price_max=#{price_max}",
+        "&search_colors%5B%5D=",
+        "&search_number_cards_main=100",
+        "&search_number_cards_sideboard=",
+        "&search_cards%5B%5D=",
+        "&search_tags=",
+        "&search_order=#{URI.encode_www_form_component(order_by)}",
+        "&utf8=%E2%9C%94",
+        "&page=#{page}"
+      ].join("")
+
+      doc = Nokogiri::HTML(HTTParty.get(url).body)
+
+      doc.css(".deck_row").each do |dr|
+        link = dr.css("td")[1].css("a").first.attribute("href").value
+        decklist << deck_obj(link,link.split("/")[-2],[commander].reject(&:empty?))
+      end
+    end
+
+    decklist
+  end
+
+  def self.deckstats_deck(deck)
+    docstring = HTTParty.get(deck[:link]).body
+
+    doc = Nokogiri::HTML(docstring)
+
+    legal = (doc.css(".fa-exclamation-triangle").count == 0)
+
+    if legal
+      deck_data = JSON.parse(docstring.split("init_deck_data(").last.split(");deck_display();").first)
+      deck[:date] = DateTime.strptime(deck_data["updated"].to_s,'%s')
+      unless deck_data["highlight_cards"].nil?
+        deck[:commanders] = deck_data["highlight_cards"]
+      end
+      deck[:cardlist] = deck_data["sections"].map do |sec|
+        sec["cards"].map{|c| "#{c["amount"]} #{c["name"]}"}
+      end.flatten
+
+      deck[:price] = (
+        !doc.css(".deck_overview_price").first.nil? ?
+          doc.css(".deck_overview_price").first.content.gsub("$","").strip.split(".").first
+          : nil
+      )
+
+      output_cod(deck,'deckstats')
+    end
   end
 
-  def self.deckstats_deck
+  #colors = "White,Blue,Black,Red,Green,Colorless"
+  #orderBy = "-updatedAt", "-createdAt", "-points", 
+  def self.archidekt_decklist(
+    andcolors = nil, colors = nil, commander = nil, formats = 3, orderBy = "-createdAt", size: 100, pageSize: 50
+  )
+
+    url = [
+      "https://www.archidekt.com/api/decks/cards/?",
+      [
+        (andcolors.nil? ? nil : "true")
+        (colors.nil? ? nil : "colors=#{URI.encode_www_form_component(colors)}"),
+        (commander.nil? ? nil : "commanders=#{URI.encode_www_form_component(commander)}"),
+        "formats=#{formats}",
+        "orderBy=#{orderBy}",
+        "size=#{size}",
+        "pageSize=#{pageSize}"
+      ].compact.join("&")
+    ].join("")
+
+    puts url
+
+    decklist = []
+    data = JSON.parse(HTTParty.get(url).body)
+
+    data["results"].each do |r|
+      decklist << deck_obj("https://www.archidekt.com/decks/#{r["id"]}", r["name"])
+    end
 
+    decklist
+  end
+
+  # deck = CockatriceFeeder.deck_obj("https://www.archidekt.com/decks/992684#Rocking_that_equipment_Bro")
+  # CockatriceFeeder.archidekt_deck(deck)
+  def self.archidekt_deck(deck)
+    deck_id = deck[:link].split("/").last.split("#").first
+
+    api_url = "https://www.archidekt.com/api/decks/#{deck_id}/"
+
+    deck_data = JSON.parse(HTTParty.get(api_url).body)
+
+    included_categories = deck_data["categories"].select{|c| c["includedInDeck"]}.map{|c| c["name"] }
+    commander_categories = deck_data["categories"].select{|c| c["isPremier"]}.map{|c| c["name"] }
+    cardlist = []
+    tcg_price = 0.0
+    ck_price = 0.0
+    deck_data["cards"].each do |card|
+      cname = card["card"]["oracleCard"]["name"]
+
+      if card["card"]["oracleCard"]["layout"] != "split"
+        cname = cname.split(" // ").first
+      end
+
+      if (included_categories & card["categories"]).length > 0
+        cardlist << "#{card["quantity"]} #{cname}"
+
+        tcg_price += (card["card"]["prices"]["tcg"] * card["quantity"].to_f)
+        ck_price += (card["card"]["prices"]["ck"] * card["quantity"].to_f)
+      end
+
+      if (commander_categories & card["categories"]).length > 0
+        deck[:commanders] << cname
+      end
+    end
+
+    deck[:cardlist] = cardlist
+    deck[:price] = tcg_price.to_i.to_s
+
+    deck[:name] = deck_data["name"]
+
+    deck[:date] = deck_data["updatedAt"]
+
+    output_cod(deck,"archidekt")
   end
 
   def self.mtgdecks_decklist(pages = (1..1))
@@ -295,16 +437,16 @@ module CockatriceFeeder
       doc = Nokogiri::HTML(HTTParty.get("https://mtgdecks.net/Commander/decklists/page:#{page}").body)
       doc.css(".decks tr.previewable").each do |r|
         if r.css("td")[0].css(".label-danger").length == 0
-          decks << {
-            name: "", # r.css("td")[1].css("a")[0].content,
+          date = r.css("td")[6].css("strong")[0].content.
+            gsub("<span class=\"hidden-xs\">","").
+            gsub("</span>","").gsub(/\s+/, "")
+          price = r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, "")
+
+          decks << deck_obj(
             link: "https://mtgdecks.net"+r.css("td")[1].css("a")[0].attribute("href").value,
-            date: r.css("td")[6].css("strong")[0].content.
-              gsub("<span class=\"hidden-xs\">","").
-              gsub("</span>","").gsub(/\s+/, ""),
-            price: r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, ""),
-            commanders: [],
-            cardlist: []
-          }
+            date: date,
+            price: price
+          )
         end
       end
     end
@@ -313,7 +455,6 @@ module CockatriceFeeder
   end
 
   def self.mtgdecks_deck(deck)
-    puts deck[:link]
     doc = Nokogiri::HTML(HTTParty.get(deck[:link]).body)
 
     cardlist = []
@@ -334,27 +475,33 @@ module CockatriceFeeder
     output_cod(deck,"mtgdecks")
   end
 
+  def self.mtggoldfish_pricer(deck)
+    doc = Nokogiri::HTML(HTTParty.get("https://www.mtggoldfish.com/tools/deck_pricer#paper"))
+    csrf_token = nil
+    doc.css("meta").each do |m|
+      if !m.attribute("name").nil? && m.attribute("name").value == "csrf-token"
+        csrf_token = m.attribute("content").value
+      end
+    end
+
+    doc2 = Nokogiri::HTML(HTTParty.post("https://www.mtggoldfish.com/tools/deck_pricer#paper", {
+      body: {
+        utf8: "✓",
+        authenticity_token: csrf_token,
+        deck: deck[:cardlist].join("\n")
+      }
+    }))
+
+    deck[:price] = doc2.css(".deck-price-v2.paper").first.
+      content.strip.split(" ").last.split(".").first.gsub(",","")
+  end
+
   def self.gobble
-    setup()
+    setup(skip_meta = true)
+    update_commanders()
 
     total_decks = 0
 
-    # puts "Fetching the most recently updated ranked decks from tappedout"
-    # decks = CockatriceFeeder.tappedout_decklist(1..10)
-    # puts "#{decks.length} decks found."
-    # decks.each {|d|
-    #   CockatriceFeeder.tappedout_deck(d)
-    #   total_decks += 1
-    # }
-    #
-    # puts "Fetching the first 10 pages of decks at https://mtgdecks.net/Commander/decklists/"
-    # decks = CockatriceFeeder.mtgdecks_decklist(1..10)
-    # puts "#{decks.length} decks found."
-    # decks.each {|d|
-    #   CockatriceFeeder.mtgdecks_deck(d)
-    #   total_decks += 1
-    # }
-
     puts "Fetching the average deck for every commander on EDHREC"
     decks = CockatriceFeeder.edhrecavg_decklist
     puts "#{decks.length} decks found."
@@ -363,7 +510,38 @@ module CockatriceFeeder
       total_decks += 1
     }
 
-    puts "#{total_decks} decks created."
+    puts "Fetching the most recently updated ranked decks from tappedout"
+    decks = CockatriceFeeder.tappedout_decklist(1..10)
+    puts "#{decks.length} decks found."
+    decks.each {|d|
+      CockatriceFeeder.tappedout_deck(d)
+      total_decks += 1
+    }
+
+    puts "Fetching the first 10 pages of decks at https://mtgdecks.net/Commander/decklists/"
+    decks = CockatriceFeeder.mtgdecks_decklist(1..10)
+    puts "#{decks.length} decks found."
+    decks.each {|d|
+      CockatriceFeeder.mtgdecks_deck(d)
+      if d[:cardlist].length > 0
+        total_decks += 1
+      end
+    }
+
+    puts "Fetching the first 5 pages of edh decks from deckstats ordered by likes"
+    decks = CockatriceFeeder.deckstats_decklist("", (1..5))
+    puts "#{decks.length} decks found."
+    decks.each {|d|
+      CockatriceFeeder.deckstats_deck(d)
+      if d[:cardlist].length > 0
+        total_decks += 1
+      end
+    }
+
+    puts "#{total_decks} decks created at #{@@deck_dir}."
+
+    puts "cleaning up"
+    FileUtils.remove_dir(@@meta_dir)
     puts "Scraw!"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cockatrice_feeder
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.6
 platform: ruby
 authors:
 - Matthew Steinwachs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2020-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -38,22 +38,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
-- !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.8'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.8'
-description: A tool to scrape MTG decks from the internet along with some meta information
-  and create Cockatrice compatible deck files.
+description: A tool to scrape MTG EDH decks from the internet along with some meta
+  information and create Cockatrice compatible deck files.
 email: matt.steinwachs@gmail.com
 executables:
 - gobble
@@ -85,5 +71,5 @@ rubyforge_project:
 rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
-summary: Scrape and generate decks for cockatrice
+summary: Scrape and generate MTG EDH decks for cockatrice
 test_files: []