cockatrice_feeder 0.0.3 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b8351da4a5eff2051048c3b0c0610167361f97ee
-  data.tar.gz: 56f2632308cd7bd822a9daab16741d247fbbd274
+  metadata.gz: 35f49ec43fbddcb43336894609d1f110ed83b9c4
+  data.tar.gz: d43a4dc9a50c9f8e8c0dd0e8df2a1d08721ebe4d
 SHA512:
-  metadata.gz: dd03bf14ebbdf4731c7acb48147dd786c87668737e06f867d1048ff2241a79b39a34deb14f806ed1abb283a0c2d479a62d176f6f5124f90e6558ff5ad3688cb9
-  data.tar.gz: 4777f537c2a521625396116ee1b6bb9860f62d482c6e9e61815b39cdf5996fd2df7e43979656040250435e481ab791c3616dcf45df7a94853df6b450c64f698d
+  metadata.gz: 6f185893e9afa3c3cbed1c9217a55d58d7d456d6887c0078dac5b0d37f0c8ab54d5720983ce7e813ff95f2785c2caabf6e4b766f7cd90c6148213fbf230dd34d
+  data.tar.gz: 35bf4d00a14d7b2b7806d84ab8230915082da0f298dced2c0bd4acc3b4eef818b763292fa5b05305f9c04e5a055ca6b2a56badb051830f2c9cc5a752f2ad3c7c

data/lib/cockatrice_feeder.rb

@@ -2,7 +2,6 @@ module CockatriceFeeder
   require 'httparty'
   require 'nokogiri'
   require 'fileutils'
-  require 'descriptive_statistics'
 
   @@app_dir = Dir.pwd+"/"
   @@deck_dir = @@app_dir+"decks/"
@@ -55,7 +54,7 @@ module CockatriceFeeder
       Dir.mkdir(@@deck_dir)
       puts "Creating a folder at '#{@@deck_dir}' for generated decks."
 
-      folders = %w(edhrecavg mtgdecks tappedout deckstats)
+      folders = %w(edhrecavg mtgdecks tappedout deckstats archidekt)
 
       folders.each do |folder|
         unless File.directory?(@@deck_dir+folder)
@@ -146,6 +145,15 @@ module CockatriceFeeder
     tiers
   end
 
+  def self.update_full_data
+    File.open(@@meta_dir+"AllPrintings.json", "w") do |file|
+      file.binmode
+      HTTParty.get("https://mtgjson.com/api/v5/AllPrintings.json", stream_body: true) do |fragment|
+        file.write(fragment)
+      end
+    end
+  end
+
   def self.commanders
     unless File.exist?(@@meta_dir+"commanders.json")
       update_commanders()
@@ -168,6 +176,24 @@ module CockatriceFeeder
     JSON.parse(File.read(@@meta_dir+"tiers.json"))
   end
 
+  def self.full_data
+    unless File.exist?(@@meta_dir+"AllPrintings.json")
+      update_full_data()
+    end
+    JSON.parse(File.read(@@meta_dir+"AllPrintings.json"))
+  end
+
+  def self.deck_obj(link = "", name = "", commanders = [], date = nil, price = nil)
+    {
+      link: link,
+      name: name,
+      commanders: commanders,
+      date: date,
+      price: price,
+      cardlist: []
+    }
+  end
+
   def self.output_cod(deck, subfolder)
     comments = [
       deck[:name],
@@ -227,14 +253,7 @@ module CockatriceFeeder
       doc.css(".deck-wide-header a").each do |a|
         link = a.attribute("href").value
         if link.include?("/mtg-decks/")
-          decks << {
-            name: link.split("/").last,
-            commanders: [],
-            link: "https://tappedout.net"+link,
-            date: nil,
-            price: nil,
-            cardlist: []
-          }
+          decks << deck_obj("https://tappedout.net"+link, link.split("/").last)
         end
       end
     end
@@ -279,14 +298,7 @@ module CockatriceFeeder
 
   def self.edhrecavg_decklist
     commanders.map{|c| c["link"]}.map do |c|
-      {
-        name: c,
-        commanders: [c],
-        link: "https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json",
-        price: nil,
-        date: nil,
-        cardlist: []
-      }
+      deck_obj("https://edhrec-json.s3.amazonaws.com/en/decks/#{c}.json", c, [c])
     end
   end
 
@@ -300,7 +312,7 @@ module CockatriceFeeder
 
   #order ["views,desc", "price,desc", "likes,desc", "updated,desc"]
   #commander should be a name attribute from the commanders array of objects
-  def self.deckstats_decklist(commander = "nil", pages = (1..1), order = "likes,desc", price_min = "", price_max = "")
+  def self.deckstats_decklist(commander = "", pages = (1..1), order_by = "likes,desc", price_min = "", price_max = "")
     decklist = []
     pages.each do |page|
       url = [
@@ -317,7 +329,7 @@ module CockatriceFeeder
         "&search_number_cards_sideboard=",
         "&search_cards%5B%5D=",
         "&search_tags=",
-        "&search_order=#{URI.encode_www_form_component(order)}",
+        "&search_order=#{URI.encode_www_form_component(order_by)}",
         "&utf8=%E2%9C%94",
         "&page=#{page}"
       ].join("")
@@ -326,14 +338,7 @@ module CockatriceFeeder
 
       doc.css(".deck_row").each do |dr|
         link = dr.css("td")[1].css("a").first.attribute("href").value
-        decklist << {
-          link: link,
-          name: link.split("/")[-2],
-          commanders: [commander].reject(&:empty?),
-          price: nil,
-          date: nil,
-          cardlist: []
-        }
+        decklist << deck_obj(link,link.split("/")[-2],[commander].reject(&:empty?))
       end
     end
 
@@ -357,12 +362,91 @@ module CockatriceFeeder
         sec["cards"].map{|c| "#{c["amount"]} #{c["name"]}"}
       end.flatten
 
-      deck[:price] = doc.css(".deck_overview_price").first.content.gsub("$","").strip.split(".").first
+      deck[:price] = (
+        !doc.css(".deck_overview_price").first.nil? ?
+          doc.css(".deck_overview_price").first.content.gsub("$","").strip.split(".").first
+          : nil
+      )
 
       output_cod(deck,'deckstats')
     end
   end
 
+  #colors = "White,Blue,Black,Red,Green,Colorless"
+  #orderBy = "-updatedAt", "-createdAt", "-points", "-viewCount"
+  def self.archidekt_decklist(
+    andcolors = nil, colors = nil, commander = nil, owner = nil, formats = 3, orderBy = "-createdAt", size: 100, pageSize: 50
+  )
+
+    url = [
+      "https://www.archidekt.com/api/decks/cards/?",
+      [
+        (andcolors.nil? ? nil : "true"),
+        (colors.nil? ? nil : "colors=#{URI.encode_www_form_component(colors)}"),
+        (commander.nil? ? nil : "commanders=\"#{URI.encode_www_form_component(commander)}\""),
+        (owner.nil? ? nil : "owner=#{URI.encode_www_form_component(owner)}"),
+        "formats=#{formats}",
+        "orderBy=#{orderBy}",
+        "size=#{size}",
+        "pageSize=#{pageSize}"
+      ].compact.join("&")
+    ].join("")
+
+    puts url
+
+    decklist = []
+    data = JSON.parse(HTTParty.get(url).body)
+
+    data["results"].each do |r|
+      decklist << deck_obj("https://www.archidekt.com/decks/#{r["id"]}", r["name"])
+    end
+
+    decklist
+  end
+
+  # deck = CockatriceFeeder.deck_obj("https://www.archidekt.com/decks/992684#Rocking_that_equipment_Bro")
+  # CockatriceFeeder.archidekt_deck(deck)
+  def self.archidekt_deck(deck)
+    deck_id = deck[:link].split("/").last.split("#").first
+
+    api_url = "https://www.archidekt.com/api/decks/#{deck_id}/"
+
+    deck_data = JSON.parse(HTTParty.get(api_url).body)
+
+    included_categories = deck_data["categories"].select{|c| c["includedInDeck"]}.map{|c| c["name"] }
+    commander_categories = deck_data["categories"].select{|c| c["isPremier"]}.map{|c| c["name"] }
+    cardlist = []
+    tcg_price = 0.0
+    ck_price = 0.0
+    deck_data["cards"].each do |card|
+      cname = card["card"]["oracleCard"]["name"]
+
+      if card["card"]["oracleCard"]["layout"] != "split"
+        cname = cname.split(" // ").first
+      end
+
+      if (included_categories & card["categories"]).length > 0
+        cardlist << "#{card["quantity"]} #{cname}"
+
+        tcg_price += (card["card"]["prices"]["tcg"] * card["quantity"].to_f)
+        ck_price += (card["card"]["prices"]["ck"] * card["quantity"].to_f)
+      end
+
+      if (commander_categories & card["categories"]).length > 0
+        deck[:commanders] << cname
+      end
+    end
+
+    deck[:cardlist] = cardlist
+    deck[:price] = tcg_price.to_i.to_s
+
+    deck[:name] = deck_data["name"]
+
+    deck[:date] = deck_data["updatedAt"]
+
+    output_cod(deck,"archidekt")
+  end
+
   def self.mtgdecks_decklist(pages = (1..1))
     decks = []
     pages.each do |page|
@@ -370,16 +454,14 @@ module CockatriceFeeder
       doc = Nokogiri::HTML(HTTParty.get("https://mtgdecks.net/Commander/decklists/page:#{page}").body)
       doc.css(".decks tr.previewable").each do |r|
         if r.css("td")[0].css(".label-danger").length == 0
-          decks << {
-            name: "", # r.css("td")[1].css("a")[0].content,
-            link: "https://mtgdecks.net"+r.css("td")[1].css("a")[0].attribute("href").value,
-            date: r.css("td")[6].css("strong")[0].content.
-              gsub("<span class=\"hidden-xs\">","").
-              gsub("</span>","").gsub(/\s+/, ""),
-            price: r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, ""),
-            commanders: [],
-            cardlist: []
-          }
+          link = "https://mtgdecks.net"+r.css("td")[1].css("a")[0].attribute("href").value
+          name = link.split("/").last
+          date = r.css("td")[6].css("strong")[0].content.
+            gsub("<span class=\"hidden-xs\">","").
+            gsub("</span>","").gsub(/\s+/, "")
+          price = r.css("td")[7].css("span.paper")[0].content.gsub("$","").gsub(/\s+/, "")
+
+          decks << deck_obj(link,name,nil,date,price)
         end
       end
     end
@@ -408,6 +490,139 @@ module CockatriceFeeder
     output_cod(deck,"mtgdecks")
   end
 
+  def self.mtggoldfish_pricer(deck)
+    doc = Nokogiri::HTML(HTTParty.get("https://www.mtggoldfish.com/tools/deck_pricer#paper"))
+    csrf_token = nil
+    doc.css("meta").each do |m|
+      if !m.attribute("name").nil? && m.attribute("name").value == "csrf-token"
+        csrf_token = m.attribute("content").value
+      end
+    end
+
+    doc2 = Nokogiri::HTML(HTTParty.post("https://www.mtggoldfish.com/tools/deck_pricer#paper", {
+      body: {
+        utf8: "✓",
+        authenticity_token: csrf_token,
+        deck: deck[:cardlist].join("\n")
+      }
+    }))
+
+    deck[:price] = doc2.css(".deck-price-v2.paper").first.
+      content.strip.split(" ").last.split(".").first.gsub(",","")
+  end
+
+  def self.mtgapi_card(params = {multiverse_id: nil, name: nil, set:nil})
+    defaults = {multiverse_id: nil, name: nil}
+    params = defaults.merge(params)
+
+    puts "looking up #{params}"
+
+    card = nil
+    if params[:multiverse_id] != nil
+      card = JSON.parse(
+        HTTParty.get("https://api.magicthegathering.io/v1/cards/#{params[:multiverse_id]}").body
+      )["card"]
+    elsif params[:name] != nil
+      query = [
+        "name=#{URI.encode_www_form_component(params[:name])}",
+        (params[:set] != nil ? "set=#{params[:set]}" : nil)
+      ].compact.join("&")
+
+      card = JSON.parse(
+        HTTParty.get("https://api.magicthegathering.io/v1/cards?#{query}").body
+      )["cards"].first
+    end
+    return card
+  end
+
+  # CockatriceFeeder.build_deck_from_sets("Lazav, Dimir Mastermind", ["RAV", "GPT", "DIS", "RTR", "GTC", "DGM", "GRN", "RNA", "WAR"])
+  def self.build_deck_from_sets(commander, sets)
+    #lookup edhrecavg deck for commander
+    c = commanders.select{|c| c["name"] == commander}.first
+    d = deck_obj("https://edhrec-json.s3.amazonaws.com/en/decks/#{c["link"]}.json", c["name"]+" SCRAW", [c["name"]])
+    edhrecavg_deck(d)
+
+    #remember cards to avoid repeat lookups to mtgapi
+    mem = {}
+
+    #throw out any cards not in sets
+    d[:cardlist] = d[:cardlist].select do |cl|
+      name = cl.split(' ')[1..-1].join(" ")
+      unless mem.has_key?(name)
+        mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
+      end
+      card = mem[name]
+
+      (card["printings"] & sets).length > 0
+    end
+
+    #lookup archidekt decks for the commander to find other cards from the set
+    arch_decks = archidekt_decklist(nil, nil, c["name"])
+    arch_decks.each do |ad| archidekt_deck(ad) end
+    arch_decks.each do |ad|
+      in_set = ad[:cardlist].select do |cl|
+        name = cl.split(' ')[1..-1].join(" ")
+        unless mem.has_key?(name)
+          mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
+        end
+        card = mem[name]
+
+        #flip/dual cards don't always work
+        if card != nil
+          (card["printings"] & sets).length > 0
+        end
+      end
+
+      in_set.each do |cl|
+        name = cl.split(' ')[1..-1].join(" ")
+        cur = d[:cardlist].map{|x| x.split(' ')[1..-1].join(" ")}
+        curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
+        unless cur.include?(name) || curlen >= 100
+          d[:cardlist] << cl
+        end
+      end
+
+      curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
+      if curlen >= 100
+        break
+      end
+    end
+
+    #lookup deckstats decks for the commander
+    curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
+    if curlen < 100
+      dstat_decks = deckstats_decklist(c["name"],1..5)
+      dstat_decks.each do |ds| deckstats_deck(ds) end
+      dstat_decks.each do |ds|
+        in_set = ds[:cardlist].select do |cl|
+          name = cl.split(' ')[1..-1].join(" ")
+          unless mem.has_key?(name)
+            mem[name] = mtgapi_card(name: cl.split(' ')[1..-1].join(" "))
+          end
+          card = mem[name]
+
+          (card["printings"] & sets).length > 0
+        end
+
+        in_set.each do |cl|
+          name = cl.split(' ')[1..-1].join(" ")
+          cur = d[:cardlist].map{|x| x.split(' ')[1..-1].join(" ")}
+          curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
+          unless cur.include?(name) || curlen >= 100
+            d[:cardlist] << cl
+          end
+        end
+
+        curlen = d[:cardlist].map{|x| x.split(' ').first.to_i}.inject(0){|sum,x| sum + x }
+        if curlen >= 100
+          break
+        end
+      end
+    end
+
+    d
+  end
+
   def self.gobble
     setup(skip_meta = true)
     update_commanders()
@@ -435,7 +650,7 @@ module CockatriceFeeder
     puts "#{decks.length} decks found."
     decks.each {|d|
       CockatriceFeeder.mtgdecks_deck(d)
-      if d[:cardlist].length < 0
+      if d[:cardlist].length > 0
         total_decks += 1
       end
     }
@@ -445,7 +660,17 @@ module CockatriceFeeder
     puts "#{decks.length} decks found."
     decks.each {|d|
       CockatriceFeeder.deckstats_deck(d)
-      if d[:cardlist].length < 0
+      if d[:cardlist].length > 0
+        total_decks += 1
+      end
+    }
+
+    puts "Fetching the first page of edh decks from archidekt ordered by createdAt"
+    decks = CockatriceFeeder.archidekt_decklist()
+    puts "#{decks.length} decks found."
+    decks.each {|d|
+      CockatriceFeeder.archidekt_deck(d)
+      if d[:cardlist].length > 0
         total_decks += 1
       end
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cockatrice_feeder
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.8
 platform: ruby
 authors:
 - Matthew Steinwachs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2020-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -38,22 +38,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
-- !ruby/object:Gem::Dependency
-  name: descriptive_statistics
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-description: A tool to scrape MTG decks from the internet along with some meta information
-  and create Cockatrice compatible deck files.
+description: A tool to scrape MTG EDH decks from the internet along with some meta
+  information and create Cockatrice compatible deck files.
 email: matt.steinwachs@gmail.com
 executables:
 - gobble
@@ -85,5 +71,5 @@ rubyforge_project:
 rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
-summary: Scrape and generate decks for cockatrice
+summary: Scrape and generate MTG EDH decks for cockatrice
 test_files: []