gatherer_set_parser 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gatherer_set_parser.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 David Ratajczak
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# GathererSetParser
+
+This is a tool for parsing the official Magic the Gathering card database for 
+a complete collection of cards belonging to a specific set. The cards of the 
+given set will be returned in an array of hashes.
+
+This gem was extracted from a personal project, and hasn't been throughly 
+tested for every set in production. You'll likely want to refactor a lot of 
+the code before using it on a production server.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'gatherer_set_parser'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gatherer_set_parser
+
+## Usage
+
+This set requires you to pass in a case sensitive set name from the Gatherer
+website. Here's an example:
+
+```
+  # Create the array of cards from the given set name
+  cards = GathererSetParser::Fetch.new('Dark Ascension').cards
+```
+
+This will return an array of 171 cards. Here's an example of the output of 
+cards.first:
+
+```
+  { :title=> "Afflicted Deserter",
+  :card_type => "Creature",
+  :sub_type => "",
+  :power => 3,
+  :toughness => 2,
+  :cmc => 4,
+  :mana_cost => ["3", "R"],
+  :loyalty => nil,
+  :image_url => "http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid=262675&type=card",
+  :text => "\r\n At the beginning of each upkeep, if no spells were cast last turn, transform Afflicted Deserter.",
+  :gatherer_url => "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=262675",
+  :rarity => "Uncommon" }
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Contact
+
+[david@mockra.com](mailto:david@mockra.com)
+
+[@Mockra_](http://twitter.com/#!/mockra_)
+
+[mockra.com](http://mockra.com)

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/gatherer_set_parser.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/gatherer_set_parser/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["David Ratajczak"]
+  gem.email         = ["david@mockra.com"]
+  gem.description   = %q{Parses the MTG Gatherer website for a collection of
+                        cards from the provided set.}
+  gem.summary       = %q{This gem will return an array of card hashes that you
+                        can use to build a database of Magic the Gathering cards.}
+  gem.homepage      = "https://github.com/mockra/gatherer_set_parser"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "gatherer_set_parser"
+  gem.require_paths = ["lib"]
+  gem.version       = GathererSetParser::VERSION
+
+    gem.add_development_dependency 'rake'
+    gem.add_development_dependency 'rspec'
+    gem.add_development_dependency 'fakeweb'
+    gem.add_development_dependency 'vcr'
+    gem.add_dependency 'nokogiri'
+end

data/lib/gatherer_set_parser/card_parser.rb

@@ -0,0 +1,114 @@
+module GathererSetParser
+  class CardParser
+
+    attr_accessor :body
+
+    def initialize card
+      @body = card
+    end
+
+    def attributes
+      { title: title,
+        card_type: type,
+        sub_type: sub_type,
+        power: power,
+        toughness: toughness,
+        cmc: cmc,
+        mana_cost: mana_cost,
+        loyalty: loyalty,
+        image_url: image_url,
+        text: text,
+        gatherer_url: gatherer_url,
+        rarity: rarity }
+    end
+
+    def title
+      body.css('.cardTitle > a').text
+    end
+
+    def type_box
+      body.css('span.typeLine').text.
+        gsub(/\n/, '').gsub(/[ ]{2,}/, ' ').strip.lstrip
+    end
+
+    def type
+      type_box.match(/[a-z0-9 ]*/i).to_s.strip
+    end
+
+    def sub_type
+      type_box.gsub(/\(.*\)/, '').
+        match(/ ([a-z0-9 ]*)\z/i)[1].to_s.lstrip.strip
+    rescue
+      nil
+    end
+
+    def power
+      type_box.match(/\(([0-9]*)\//)[1].to_i rescue nil
+    end
+
+    def toughness
+      type_box.match(/\/([0-9]*)\)/)[1].to_i rescue nil
+    end
+
+    def cmc
+      body.css('.convertedManaCost').text.to_i
+    end
+
+    def loyalty
+      type_box.match(/\(([0-9]*)\)/)[1].to_i rescue nil
+    end
+
+    def rarity
+      body.css('.rightCol > a').first.children.attribute('alt').text.
+        match(/\((.*)\)/)[1].to_s
+    end
+
+    def text
+      body.xpath('//div[@class="rulesText"]//img[@alt]').each{ |i| i.swap( i['alt'].parse_mana_cost ) }
+      body.css('.rulesText').text.gsub(/[ ]{2,}/, ' ')
+    end
+
+    def mana_cost
+      array = []
+      body.css('.manaCost >img').each do |img|
+        array << img.attribute('alt').text.parse_mana_cost
+      end
+      array
+    end
+
+    def image_url
+      'http://gatherer.wizards.com/' +
+      body.css('.leftCol > a > img').attribute('src').
+        text.gsub(/\.\.\//, '')
+    end
+
+    def gatherer_url
+      'http://gatherer.wizards.com/Pages/' +
+      body.css('.cardTitle > a').attribute('href').
+        text.gsub(/\.\.\//, '')
+    end
+
+  end
+end
+
+class String
+  def parse_mana_cost
+    extract_phyrexia.extract_colors.extract_unique_costs
+  end
+
+  def extract_phyrexia
+    gsub(/Phyrexian Blue/, 'U/P').gsub(/Phyrexian Black/, 'B/P').
+    gsub(/Phyrexian White/, 'W/P').gsub(/Phyrexian Red/, 'R/P').
+    gsub(/Phyrexian Green/, 'G/P')
+  end
+
+  def extract_colors
+    gsub(/Blue/, 'U').gsub(/Black/, 'B').gsub(/Red/, 'R').gsub(/Green/, 'G').
+    gsub(/White/, 'W')
+  end
+
+  def extract_unique_costs
+    gsub(/Variable Colorless/, 'X').gsub(/Two/, '2').gsub(/or/, '/').
+    gsub(/Snow/, 'S').gsub(/Tap/, 'T').gsub(/Untap/, 'Q')
+  end
+end

data/lib/gatherer_set_parser/fetch.rb

@@ -0,0 +1,46 @@
+require 'open-uri'
+require 'nokogiri'
+
+module GathererSetParser
+  class Fetch
+
+    include GathererSetParser::QueryString
+
+    attr_reader :set_name
+
+    def response_page page
+      Nokogiri::HTML open(gatherer_url page), nil, 'UTF-8'
+    end
+
+    def complete_set
+      @complete_set ||= gather_set
+    end
+
+    def gather_set
+      set, page = [], 0
+      current_page = response_page page
+      total_pages = current_page.css('.paging > a').count
+      begin
+        set << current_page
+        page += 1
+        current_page = response_page page
+      end while page < total_pages - 1
+      set
+    end
+
+    def cards
+      @cards ||= collect_cards
+    end
+
+    def collect_cards
+      collection = []
+      complete_set.each do |page|
+        page.css('.cardItemTable > tr > td > table').each do |card|
+          collection << CardParser.new(card).attributes
+        end
+      end
+      collection
+    end
+
+  end
+end

data/lib/gatherer_set_parser/query_string.rb

@@ -0,0 +1,25 @@
+module GathererSetParser
+  module QueryString
+
+    def initialize set
+      @set_name = set
+    end
+
+    def escaped_set
+      @query_string ||= URI.escape set_name
+    end
+
+    def query_string
+      "output=standard&set=[%22#{escaped_set}%22]"
+    end
+
+    def url
+      "http://gatherer.wizards.com/Pages/Search/Default.aspx?"
+    end
+
+    def gatherer_url page
+      url + query_string + "&page=#{page}"
+    end
+
+  end
+end

data/lib/gatherer_set_parser/version.rb

@@ -0,0 +1,3 @@
+module GathererSetParser
+  VERSION = "0.0.1"
+end

data/lib/gatherer_set_parser.rb

@@ -0,0 +1,7 @@
+require "gatherer_set_parser/version"
+require 'gatherer_set_parser/query_string'
+require 'gatherer_set_parser/card_parser'
+require 'gatherer_set_parser/fetch'
+
+module GathererSetParser
+end

data/spec/gatherer_set_parser/card_parser_spec.rb

@@ -0,0 +1,181 @@
+require 'gatherer_set_parser_spec'
+require 'fakeweb'
+
+describe GathererSetParser::CardParser do
+
+  let(:parsed_card) { GathererSetParser::CardParser.new card }
+
+  describe '#parse_card' do
+    let(:card) { Nokogiri::HTML open('spec/support/flipped_creature.html') }
+
+    it 'should return a hash' do
+      parsed_card.attributes.should be_a Hash
+    end
+
+    it 'should have a title' do
+      parsed_card.attributes[:title].should_not be_nil
+    end
+  end
+
+  describe '#card_attributes' do
+    context 'back of flip card' do
+      let(:card) { Nokogiri::HTML open('spec/support/flipped_creature.html') }
+
+      it 'should return the correct title' do
+        parsed_card.title.should == "Tovolar's Magehunter"
+      end
+
+      it 'should return a card type' do
+        parsed_card.type.should == 'Creature'
+      end
+
+      it 'should match the species type' do
+        parsed_card.sub_type.should == 'Werewolf'
+      end
+
+      it 'should have the correct power' do
+        parsed_card.power.should == 5
+      end
+
+      it 'should have the correct toughness' do
+        parsed_card.toughness.should == 5
+      end
+
+      it 'should have no loyalty' do
+        parsed_card.loyalty.should be_nil
+      end
+
+      it 'should have a cmc' do
+        parsed_card.cmc.should == 0
+      end
+
+      it 'should have the correct rarity' do
+        parsed_card.rarity.should == 'Rare'
+      end
+
+      it 'should have text' do
+        parsed_card.text.should =~ /transform/i
+      end
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == []
+      end
+
+      it 'should have the correct image url' do
+        parsed_card.image_url.should =~
+          /\/Handlers\/Image\.ashx\?multiverseid\=253429\&type\=card/
+      end
+    end
+
+    context 'an instant/sorcery' do
+      let(:card) { Nokogiri::HTML open('spec/support/instant.html') }
+
+      it 'should have no power' do
+        parsed_card.power.should be_nil
+      end
+
+      it 'should have no loyalty' do
+        parsed_card.loyalty.should be_nil
+      end
+
+      it 'should have the correct type' do
+        parsed_card.type.should == 'Instant'
+      end
+
+      it 'should have the correct rarity' do
+        parsed_card.rarity.should == 'Uncommon'
+      end
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == %w[1 U]
+      end
+
+      it 'should have no sub type' do
+        parsed_card.sub_type.should be_nil
+      end
+    end
+
+    context 'land' do
+      let(:card) { Nokogiri::HTML open('spec/support/land.html') }
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == []
+      end
+
+      it 'should have no power' do
+        parsed_card.power.should be_nil
+      end
+    end
+
+    context 'planeswalker' do
+      let(:card) { Nokogiri::HTML open('spec/support/planeswalker.html') }
+
+      it 'should have a title' do
+        parsed_card.title.should =~ /Tamiyo/i
+      end
+
+      it 'should have no power' do
+        parsed_card.power.should be_nil
+      end
+
+      it 'should have no toughness' do
+        parsed_card.power.should be_nil
+      end
+
+      it 'should have loyalty' do
+        parsed_card.loyalty.should == 4
+      end
+
+      it 'should have the correct type' do
+        parsed_card.type.should == 'Planeswalker'
+      end
+
+      it 'should have the correct rarity' do
+        parsed_card.rarity.should == 'Mythic Rare'
+      end
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == %w[3 U U]
+      end
+
+      it 'should have correct gatherer url' do
+        parsed_card.gatherer_url.should =~
+          /Card\/Details\.aspx\?multiverseid\=240070/
+      end
+    end
+
+    context 'phyrexian' do
+      let(:card) { Nokogiri::HTML open('spec/support/phyrexia.html') }
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == %w[X G/P]
+      end
+    end
+
+    context 'enchant' do
+      let(:card) { Nokogiri::HTML open('spec/support/enchant.html') }
+
+      it 'should have the correct mana cost' do
+        parsed_card.mana_cost.should == %w[4 B B]
+      end
+    end
+
+    context 'gold' do
+    end
+
+    context 'dual_land' do
+      let(:card) { Nokogiri::HTML open('spec/support/dual_land.html') }
+
+      it 'should have the correct text' do
+        parsed_card.text.should =~ /U/i
+      end
+    end
+
+    context 'legendary' do
+    end
+
+    context 'artifact' do
+    end
+  end
+
+end

data/spec/gatherer_set_parser/fetch_spec.rb

@@ -0,0 +1,30 @@
+require 'gatherer_set_parser_spec'
+
+describe GathererSetParser::Fetch do
+
+  let(:gatherer) { GathererSetParser::Fetch.new 'Dark Ascension' }
+
+  describe '.new' do
+    it 'should assign the set_name' do
+      gatherer.set_name.should == 'Dark Ascension'
+    end
+  end
+
+  describe '#response_body' do
+    use_vcr_cassette 'dark_ascension'
+
+    it 'should contain every page' do
+      gatherer.complete_set.count.should == 7
+    end
+
+    it 'should contain a magic card' do
+      gatherer.complete_set.last.text.should =~ /wakedancer/i
+    end
+
+    it 'should have the complete set of cards' do
+      gatherer.cards.count.should == 171
+    end
+  end
+
+end
+

data/spec/gatherer_set_parser/query_string_spec.rb

@@ -0,0 +1,21 @@
+require 'gatherer_set_parser_spec'
+
+describe GathererSetParser::QueryString do
+
+  let(:gatherer) { GathererSetParser::Fetch.new 'Dark Ascension' }
+
+  describe '#escaped_set' do
+    it 'should escape set_name to query string' do
+      gatherer.escaped_set.should == 'Dark%20Ascension'
+    end
+  end
+
+  describe '#query_string' do
+    it 'should contain the entire query' do
+      gatherer.query_string.should ==
+        'output=standard&set=[%22Dark%20Ascension%22]'
+    end
+  end
+
+end
+

data/spec/gatherer_set_parser_spec.rb

@@ -0,0 +1,2 @@
+require 'gatherer_set_parser'
+require 'vcr_setup'

data/spec/support/artifact.html

@@ -0,0 +1,35 @@
+<table>
+                <tr class="cardItem evenItem">
+                    <td class="leftCol">
+                        <div class="clear"></div>
+                        <a href="../Card/Details.aspx?multiverseid=230070" id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl00_cardImageLink" onclick="return CardLinkAction(event, this, 'SameWindow');">
+                            <img src="../../Handlers/Image.ashx?multiverseid=230070&amp;type=card" id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl00_cardImage" width="95" height="132" alt="Conversion Chamber" border="0" />
+                        </a>
+                        <div class="clear"></div>
+                    </td>
+                    <td class="middleCol">
+                        <div class="clear"></div>
+                        <div class="cardInfo">
+                            <span class="cardTitle">
+                                <a id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl00_cardTitle" onclick="return CardLinkAction(event, this, 'SameWindow');" href="../Card/Details.aspx?multiverseid=230070">Conversion Chamber</a></span> <span class="manaCost">
+                                    <img src="/Handlers/Image.ashx?size=small&amp;name=3&amp;type=symbol" alt="3" align="absbottom" /></span> (<span class="convertedManaCost">3</span>)
+                            <br />
+                            <span class="typeLine">
+                                Artifact
+                                </span>
+                            <br />
+                            <div class="rulesText">
+                                <p><img src="/Handlers/Image.ashx?size=small&amp;name=2&amp;type=symbol" alt="2" align="absbottom" />, <img src="/Handlers/Image.ashx?size=small&amp;name=tap&amp;type=symbol" alt="Tap" align="absbottom" />: Exile target artifact card from a graveyard. Put a charge counter on Conversion Chamber.</p><p><img src="/Handlers/Image.ashx?size=small&amp;name=2&amp;type=symbol" alt="2" align="absbottom" />, <img src="/Handlers/Image.ashx?size=small&amp;name=tap&amp;type=symbol" alt="Tap" align="absbottom" />, Remove a charge counter from Conversion Chamber: Put a 3/3 colorless Golem artifact creature token onto the battlefield.</p></div>
+                        </div>
+                    </td>
+                    <td class="rightCol setVersions">
+                        <div class="clear"></div>
+                        <div>
+                            <div id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl00_cardSetCurrent" class="rightCol">
+	<a onclick="return CardLinkAction(event, this, 'SameWindow');" href="../Card/Details.aspx?multiverseid=230070"><img title="New Phyrexia (Uncommon)" src="../../Handlers/Image.ashx?type=symbol&amp;set=NPH&amp;size=small&amp;rarity=U" alt="New Phyrexia (Uncommon)" style="border-width:0px;" /></a>
+</div>
+                        </div>
+                        
+                    </td>
+                </tr>
+                </table>

data/spec/support/dual_land.html

@@ -0,0 +1,40 @@
+<table>
+                <tr class="cardItem oddItem">
+                    <td class="leftCol">
+                        <div class="clear"></div>
+                        <a href="../Card/Details.aspx?multiverseid=253682" id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_cardImageLink" onclick="return CardLinkAction(event, this, 'SameWindow');">
+                            <img src="../../Handlers/Image.ashx?multiverseid=253682&amp;type=card" id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_cardImage" width="95" height="132" alt="Steam Vents" border="0" />
+                        </a>
+                        <div class="clear"></div>
+                    </td>
+                    <td class="middleCol">
+                        <div class="clear"></div>
+                        <div class="cardInfo">
+                            <span class="cardTitle">
+                                <a id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_cardTitle" onclick="return CardLinkAction(event, this, 'SameWindow');" href="../Card/Details.aspx?multiverseid=253682">Steam Vents</a></span> <span class="manaCost">
+                                    </span> (<span class="convertedManaCost">0</span>)
+                            <br />
+                            <span class="typeLine">
+                                Land  — Island Mountain
+                                </span>
+                            <div class="rulesText">
+                                <p><i>(<img src="/Handlers/Image.ashx?size=small&amp;name=tap&amp;type=symbol" alt="Tap" align="absbottom" />: Add <img src="/Handlers/Image.ashx?size=small&amp;name=U&amp;type=symbol" alt="Blue" align="absbottom" /> or <img src="/Handlers/Image.ashx?size=small&amp;name=R&amp;type=symbol" alt="Red" align="absbottom" /> to your mana pool.)</i></p><p>As Steam Vents enters the battlefield, you may pay 2 life. If you don't, Steam Vents enters the battlefield tapped.</p></div>
+                        </div>
+                    </td>
+                    <td class="rightCol setVersions">
+                        <div class="clear"></div>
+                        <div>
+                            <div id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_cardSetCurrent" class="rightCol">
+	<a onclick="return CardLinkAction(event, this, 'SameWindow');" href="../Card/Details.aspx?multiverseid=253682"><img title="Return to Ravnica (Rare)" src="../../Handlers/Image.ashx?type=symbol&amp;set=RTR&amp;size=small&amp;rarity=R" alt="Return to Ravnica (Rare)" style="border-width:0px;" /></a>
+</div>
+                        </div>
+                        <div id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_otherSetSection" class="otherSetSection">
+                            <div>
+                                <i>Other Versions</i></div>
+                            <div id="ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_ctl00_listRepeater_ctl11_cardSets" class="rightCol">
+	<a onclick="return CardLinkAction(event, this, 'SameWindow');" href="../Card/Details.aspx?multiverseid=96923"><img title="Guildpact (Rare)" src="../../Handlers/Image.ashx?type=symbol&amp;set=GPT&amp;size=small&amp;rarity=R" alt="Guildpact (Rare)" style="border-width:0px;" /></a>
+</div>
+                        </div>
+                    </td>
+                </tr>
+                </table>