scapeshift 1.0.1rg0

data/lib/scapeshift/errors.rb

@@ -0,0 +1,85 @@
+module Scapeshift
+  
+  ##
+  # Contains any custom exceptions this gem might raise.
+  #
+  # @author Josh Lindsey
+  #
+  # @since 0.1.0
+  #
+  module Errors
+    
+    ##
+    # Raised when an unrecognized crawler is specified in
+    # {Scapeshift::Crawler.crawl}
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.1.0
+    #
+    class InvalidCrawlerType < StandardError; end
+
+    ##
+    # Raised when an unknown card attribute is encountered in
+    # {Scapeshift::Crawlers::Cards#_parse_row}
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.1.0
+    #
+    class UnknownCardAttribute < StandardError; end
+
+    ##
+    # Raised when an unknown metadata type is supplied to
+    # the Meta crawler.
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.1.4
+    #
+    class UnknownMetaType < StandardError; end
+
+    ##
+    # Raised when the card name supplied to the Single crawler
+    # resolves to a Gatherer search results page instead of a 
+    # Card detail page (implying that the Card name is ambiguous,
+    # or doesn't exist).
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.2.0
+    #
+    class CardNameAmbiguousOrNotFound < StandardError; end
+
+    ##
+    # Raised when insufficient options are passed to one of the
+    # Crawlers for it to continue execution.
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.2.0
+    #
+    class InsufficientOptions < StandardError; end
+
+
+    ##
+    # Raised when an unknown word is encountered in 
+    # {Scapeshift::Card.cost_symbol_from_str}.
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.2.0
+    #
+    class UnknownCostSymbol < StandardError; end
+
+    ##
+    # Raised when a subclass of {Crawlers::Base} doesn't
+    # override a required method (such as {Crawlers::Base#crawl}).
+    #
+    # @author Josh Lindsey
+    #
+    # @since 0.3.0
+    #
+    class InvalidSubclass < StandardError; end
+  end
+end

data/scapeshift.gemspec

@@ -0,0 +1,78 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{scapeshift}
+  s.version = "1.0.1rg0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Josh Lindsey"]
+  s.date = %q{2010-04-15}
+  s.description = %q{Magic: The Gathering Oracle web scraper.}
+  s.email = %q{joshua.s.lindsey@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     ".yardopts",
+     "Gemfile",
+     "Gemfile.lock",
+     "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "lib/scapeshift.rb",
+     "lib/scapeshift/card.rb",
+     "lib/scapeshift/crawler.rb",
+     "lib/scapeshift/crawlers.rb",
+     "lib/scapeshift/crawlers/base.rb",
+     "lib/scapeshift/crawlers/cards.rb",
+     "lib/scapeshift/crawlers/meta.rb",
+     "lib/scapeshift/crawlers/single.rb",
+     "lib/scapeshift/errors.rb",
+     "scapeshift.gemspec",
+     "test/helper.rb",
+     "test/test_base_crawler.rb",
+     "test/test_card.rb",
+     "test/test_card_crawler.rb",
+     "test/test_crawler_main.rb",
+     "test/test_meta_crawler.rb",
+     "test/test_single_crawler.rb"
+  ]
+  s.homepage = %q{http://github.com/Karunamon/scapeshift}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Magic: The Gathering Oracle web scraper}
+  s.test_files = [
+    "test/test_card.rb",
+     "test/helper.rb",
+     "test/test_crawler_main.rb",
+     "test/test_single_crawler.rb",
+     "test/test_meta_crawler.rb",
+     "test/test_card_crawler.rb",
+     "test/test_base_crawler.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+    else
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'fakeweb'
+require 'fakeweb_helper'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'scapeshift'
+require 'set'
+
+FakeWeb.allow_net_connect = false
+FakeWebHelper.fake_all_urls
+
+class Test::Unit::TestCase
+end

data/test/test_base_crawler.rb

@@ -0,0 +1,48 @@
+require 'helper'
+
+class TestBaseCrawler < Test::Unit::TestCase
+  context "The Crawler Base class" do
+    should "raise the correct exception when crawl is called" do
+      assert_raise Scapeshift::Errors::InvalidSubclass do
+        crawler = Scapeshift::Crawlers::Base.new :fake => :hash
+        crawler.crawl
+      end
+    end
+
+    should "raise the correct exception when instantiated without options" do
+      assert_raise Scapeshift::Errors::InsufficientOptions do
+        Scapeshift::Crawlers::Base.new
+      end
+    end
+
+    should "repond to has_callback_hook" do
+      assert_respond_to Scapeshift::Crawlers::Base, :has_callback_hook
+    end
+
+    context "with callbacks" do
+      setup do
+        class CallbackTester < Scapeshift::Crawlers::Base
+          has_callback_hook :before_foo
+
+          def foo
+            str = "Failure"
+            self.hook :before_foo, str
+            str
+          end
+        end
+
+        @tester = CallbackTester.new :fake => :hash
+      end
+
+      should "generate the instance method" do
+        assert_respond_to @tester, :before_foo
+      end
+
+      should "call the blocks for the hook" do
+        @tester.before_foo { |str| str.replace "Success" }
+        assert_equal "Success", @tester.foo
+      end
+    end
+  end
+end
+

data/test/test_card.rb

@@ -0,0 +1,80 @@
+require 'helper'
+
+class CardTest < Test::Unit::TestCase
+  context "The Card class" do
+    should "properly convert cost words to symbols" do
+      assert_equal "B", Scapeshift::Card.cost_symbol_from_str("Black")
+      assert_equal "W", Scapeshift::Card.cost_symbol_from_str("White")
+      assert_equal "U", Scapeshift::Card.cost_symbol_from_str("Blue")
+      assert_equal "R", Scapeshift::Card.cost_symbol_from_str("Red")
+      assert_equal "G", Scapeshift::Card.cost_symbol_from_str("Green")
+      assert_equal "X", Scapeshift::Card.cost_symbol_from_str("Variable Colorless")
+      assert_equal "3", Scapeshift::Card.cost_symbol_from_str("3")
+      assert_equal "[chaos]", Scapeshift::Card.cost_symbol_from_str("[chaos]")
+    end
+
+    should "raise the proper error when passed an invalid cost word" do
+      assert_raise Scapeshift::Errors::UnknownCostSymbol do
+        Scapeshift::Card.cost_symbol_from_str "Yellow"
+      end
+    end
+  end
+  
+  context "A Card object" do
+    context "with complex types" do
+      setup do
+        @types_str = "Legendary Artifact Creature - Human Wizard"
+        @card = Scapeshift::Card.new
+        @card.types = @types_str
+      end
+
+      should "parse the types string correctly" do
+        assert_equal %w{Legendary}, @card.supertypes
+        assert_equal %w{Artifact Creature}, @card.base_types
+        assert_equal %w{Human Wizard}, @card.subtypes
+      end
+
+      should "display the string representation of its types" do
+        assert_equal @types_str, @card.types
+      end
+    end
+
+    context "with several sets and rarities" do
+      setup do
+        printings = [["Magic 2010", "Rare"], ["Tenth Edition", "Uncommon"]]
+        @card = Scapeshift::Card.new :sets => printings
+      end
+
+      should "display its most recent set" do
+        assert_equal "Magic 2010", @card.set
+      end
+
+      should "display its most recent rarity" do
+        assert_equal "Rare", @card.rarity
+      end
+    end
+
+    context "that was instantiated with a Hash" do
+      setup do
+        @params = { :name => "Mind Spring", :cost => "XBB",
+          :types => "Sorcery", :text => "Draw X cards.",
+          :sets => [["Magic 2010", "Rare"], ["Morningtide", "Rare"]],
+          :image_id => 191323 }
+
+        @card = Scapeshift::Card.new @params
+      end
+
+      should "have its attributes set properly" do
+        assert_instance_of Scapeshift::Card, @card
+
+        assert_equal @params[:name], @card.name
+        assert_equal @params[:cost], @card.cost
+        assert_equal @params[:types], @card.types
+        assert_equal @params[:text], @card.text
+        assert_equal @params[:sets], @card.sets
+        assert_equal 'http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid=191323&type=card', @card.image_uri
+      end
+    end
+  end
+end
+

data/test/test_card_crawler.rb

@@ -0,0 +1,92 @@
+require 'helper'
+
+class TestCardCrawler < Test::Unit::TestCase
+  context "The Card crawler" do
+    should "respond to crawl" do
+      crawler = Scapeshift::Crawlers::Cards.new :set => "fake" 
+      assert_respond_to crawler, :crawl
+    end
+
+    context "when passed insufficient options" do
+      should "raise the appropriate exception" do
+        assert_raise Scapeshift::Errors::InsufficientOptions do
+          Scapeshift::Crawlers::Cards.new
+        end
+      end
+    end
+
+    context "when passed valid options" do
+      setup do
+        # Pull from a specific set instead of Type 2 so we don't have to 
+        # keep updating this test when new blocks cycle in.
+        @cards = Scapeshift::Crawler.crawl :cards, :set => "Darksteel"
+      end
+      
+      should "return a SortedSet of Card objects" do
+        assert_instance_of SortedSet, @cards
+        assert_instance_of Scapeshift::Card, @cards.to_a.first
+      end
+
+      should "pull from the correct set" do
+        check = Set.new %w(Coretapper Dismantle Eater\ of\ Days Soulscour Trinisphere Thunderstaff)
+        names = Set.new
+        @cards.each { |card| names << card.name }
+
+        assert check.proper_subset?(names)
+      end
+
+      should "have created the Card objects with the correct data" do
+        # Using the last entry here because the first one is 
+        # Æther Snap and fuck typing that out every time.
+        #
+        # Also, Set and SortedSet don't implement #first or #last.
+        # Which is why we're doing this little hack.
+        card = @cards.entries[@cards.size - 3]
+
+        check_card = Scapeshift::Card.new :name => "Wurm's Tooth",
+          :types => "Artifact", :sets => [["Magic 2012", "Uncommon"], ["Magic 2011", "Uncommon"],
+            ["Magic 2010", "Uncommon"], ["Tenth Edition", "Uncommon"], ["Ninth Edition", "Uncommon"],
+            ["Darksteel", "Uncommon"]],
+            :cost => "2", :text => "Whenever a player casts a green spell, you may gain 1 life.",
+            :image_uri => "http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid=72684&type=card",
+            :multiverse_id => "72684"
+
+            assert_equal check_card, card
+      end
+    end
+
+    context "when looking in sets with planeswalkers" do
+      setup do
+        # Pull from a specific set instead of Type 2 so we don't have to
+        # keep updating this test when new blocks cycle in.
+        @cards = Scapeshift::Crawler.crawl :cards, :set => "Shards of Alara"
+      end
+
+      should "return a SortedSet of Card objects" do
+        assert_instance_of SortedSet, @cards
+        assert_instance_of Scapeshift::Card, @cards.to_a.first
+      end
+
+      should "pull from the correct set" do
+        check = Set.new %w(Ajani\ Vengeant Archdemon\ of\ Unx Mindlock\ Orb Prince\ of\ Thralls)
+        names = Set.new
+        @cards.each { |card| names << card.name }
+
+        assert check.proper_subset?(names)
+      end
+
+      should "have created the Card objects with the correct data" do
+        card = @cards.entries[0]
+
+        check_card = Scapeshift::Card.new :name => "Ad Nauseam",
+          :types => "Instant", :sets => [["Shards of Alara", "Rare"]],
+            :cost => "3BB", :text => "Reveal the top card of your library and put that card into your hand. You lose life equal to its converted mana cost. You may repeat this process any number of times.",
+            :image_uri => "http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid=174915&type=card",
+            :multiverse_id => "174915"
+
+            assert_equal check_card, card
+      end
+    end
+  end
+end
+

data/test/test_crawler_main.rb

@@ -0,0 +1,16 @@
+require 'helper'
+
+class TestCrawlerMain < Test::Unit::TestCase
+  context "The Crawler router class" do
+    should "respond to crawl" do
+      assert_respond_to Scapeshift::Crawler, :crawl
+    end
+
+    should "raise an exception for an invalid crawler type" do
+      assert_raise Scapeshift::Errors::InvalidCrawlerType do
+        Scapeshift::Crawler.crawl :invalid
+      end
+    end
+  end
+end
+

data/test/test_meta_crawler.rb

@@ -0,0 +1,78 @@
+require 'helper'
+
+class TestMetaCrawler < Test::Unit::TestCase
+  context "The Meta crawler" do
+    should "respond to crawl" do
+      assert_respond_to Scapeshift::Crawlers::Meta.new(:type => "fake"), :crawl
+    end
+
+    context "when passed no options" do
+      should "raise the appropriate exception" do
+        assert_raise Scapeshift::Errors::InsufficientOptions do
+          Scapeshift::Crawlers::Meta.new
+        end
+      end
+    end
+
+    context "when passed a bad meta type" do
+      should "raise the appropiate exception" do
+        assert_raise Scapeshift::Errors::UnknownMetaType do
+          crawler = Scapeshift::Crawlers::Meta.new :type => :not_a_real_type
+          crawler.crawl
+        end
+      end
+    end
+
+    context "when crawling expansion Sets" do
+      setup do
+        @sets = Scapeshift::Crawler.crawl :meta, :type => :sets
+      end
+
+      should "return a SortedSet" do
+        assert_instance_of SortedSet, @sets
+      end
+
+      should "return valid expansion sets" do
+        check = Set.new %w{Darksteel Coldsnap Zendikar Shadowmoor Lorwyn Nemesis Onslaught}
+        assert check.proper_subset?(@sets)
+      end
+
+      should "not contain empty sets" do
+        @sets.each do |set|
+          assert !set.empty?
+        end
+      end
+    end
+
+    context "when crawling Formats" do
+      setup do
+        @formats = Scapeshift::Crawler.crawl :meta, :type => :formats
+      end
+
+      should "return a SortedSet" do
+        assert_instance_of SortedSet, @formats
+      end
+
+      should "return valid formats" do
+        check = Set.new %w{Standard Classic Legacy Extended Freeform Vintage}
+        assert check.proper_subset?(@formats)
+      end
+    end
+
+    context "when crawling Card Types" do
+      setup do
+        @types = Scapeshift::Crawler.crawl :meta, :type => :types
+      end
+
+      should "return a SortedSet" do
+        assert_instance_of SortedSet, @types
+      end
+
+      should "return valid card types" do
+        check = Set.new %w{Artifact World Tribal Plane Land}
+        assert check.proper_subset?(@types)
+      end
+    end
+  end
+end
+