earl 0.3.0 → 1.0.0

data/spec/integration/feed_spec.rb

@@ -0,0 +1,78 @@
+require 'spec_helper'
+
+# These tests hit real network endpoints - not included in the default test runs. Run with:
+#  rake spec:integration
+# or
+#  rake spec:all
+#
+describe Earl do
+  let(:instance) { Earl[url] }
+
+  subject { instance }
+
+  context "when page has no feeds associated" do
+    let(:url) { 'http://google.com/' }
+    its(:has_feed?) { should be_false }
+    its(:rss_feed) { should be_nil }
+    its(:feed) { should be_nil }
+  end
+
+  context "when page has rss feed associated" do
+    let(:url) { 'http://www.readwriteweb.com/' }
+    its(:has_feed?) { should be_true }
+    its(:rss_feed) { should eql('http://www.readwriteweb.com/rss.xml') }
+    its(:feed) { should eql('http://www.readwriteweb.com/rss.xml') }
+  end
+
+  # context "when page has atom feed associated" do
+  #   let(:url) { 'http:// still looking for a page that just has atom' }
+  #   its(:has_feed?) { should be_true }
+  #   its(:atom_feed) { should eql('http://www.readwriteweb.com/atom.xml') }
+  #   its(:feed) { should eql('http://www.readwriteweb.com/atom.xml') }
+  # end
+
+  context "when page has rss and atom feed associated" do
+    let(:url) { 'http://tardate.blogspot.com' }
+    let(:expected_rss_feed) { 'http://tardate.blogspot.com/feeds/posts/default?alt=rss' }
+    let(:expected_atom_feed) { 'http://tardate.blogspot.com/feeds/posts/default' }
+
+    its(:has_feed?) { should be_true }
+    its(:rss_feed) { should eql(expected_rss_feed) }
+    its(:atom_feed) { should eql(expected_atom_feed) }
+    describe "#feed" do
+      context "default (rss)" do
+        subject { instance.feed }
+        it { should eql(expected_rss_feed) }
+      end
+      context "rss prefered" do
+        subject { instance.feed(:rss) }
+        it { should eql(expected_rss_feed) }
+      end
+      context "atom prefered" do
+        subject { instance.feed(:atom) }
+        it { should eql(expected_atom_feed) }
+      end
+    end
+  end
+
+  context "when page IS an rss feed" do
+    let(:url) { 'http://www.readwriteweb.com/rss.xml' }
+    its(:url) { should eql(url) }
+    its(:base_url) { should eql('http://feeds.feedburner.com/readwriteweb') }
+    its(:content_type) { should eql('text/xml') }
+    its(:has_feed?) { should be_true }
+    its(:rss_feed) { should eql(url) }
+    its(:feed) { should eql(url) }
+  end
+
+  context "when page IS an atom feed" do
+    let(:url) { 'http://tardate.blogspot.com/feeds/posts/default' }
+    its(:url) { should eql(url) }
+    its(:base_url) { should eql('http://feeds.feedburner.com/tardate') }
+    its(:content_type) { should eql('text/xml') }
+    its(:has_feed?) { should be_true }
+    its(:atom_feed) { should eql(url) }
+    its(:feed) { should eql(url) }
+  end
+
+end

data/spec/integration/oembed_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+# These tests hit real network endpoints - not included in the default test runs. Run with:
+#  rake spec:integration
+# or
+#  rake spec:all
+#
+describe Earl do
+  let(:instance) { Earl[url] }
+
+  subject { instance }
+
+  context "when page does not support oembed" do
+    let(:url) { 'http://google.com/' }
+    its(:oembed) { should be_nil }
+    its(:oembed_html) { should be_nil }
+    describe "#metadata" do
+      subject { instance.metadata }
+      it { subject[:base_url].should match(/google/) }
+      it { subject[:content_type].should eql("text/html") }
+      it { subject[:html].should be_nil }
+    end
+  end
+
+  context "when page supports oembed" do
+    let(:url) { 'http://www.youtube.com/watch?v=g3DCEcSlfhw' }
+    let(:expected_oembed_html) { %(<iframe width="420" height="315" src="http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed" frameborder="0" allowfullscreen></iframe>) }
+
+    its(:oembed) { should be_a(Hash) }
+    its(:oembed_html) { should eql(expected_oembed_html) }
+    describe "#metadata" do
+      subject { instance.metadata }
+      it { subject[:base_url].should eql("http://www.youtube.com/watch?v=g3DCEcSlfhw") }
+      it { subject[:content_type].should eql("text/html") }
+      it { subject[:title].should eql("'Virtuosos of Guitar 2008' festival, Moscow. Marcin Dylla") }
+      it { subject[:html].should eql(expected_oembed_html) }
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,34 +1,24 @@
 require 'earl'
 
-# Configure RSpec
-RSpec.configure do |config|
-  # Use color
-  config.color_enabled = true
-  # Change the formatter
-  config.formatter = :documentation
-end
-
-# Custom entity manipulation matcher
-RSpec::Matchers.define :produce do |expected|
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 
-  match do |entity_class|
-    entity = entity_class.new @start
-    @changes.each do |key, value|
-      entity.send :"#{key}=", value
-    end
-    @actual = entity.to_s
-    @actual == expected
-  end
-
-  failure_message_for_should do |entity|
-    %Q{expected that "#{@start}" would become "#{expected}" when given #{@changes}, but was "#{@actual}"}
-  end
+RSpec.configure do |config|
+  # == Mock Framework
+  #
+  # If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  config.mock_with :rspec
 
-  chain :from do |start|
-    @start = start
-  end
+  # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+  # config.fixture_path = "#{::Rails.root}/spec/fixtures"
 
-  chain :when_given do |changes|
-    @changes = changes
-  end
+  # If you're not using ActiveRecord, or you'd prefer not to run each of your
+  # examples within a transaction, remove the following line or assign false
+  # instead of true.
+  # config.use_transactional_fixtures = true
 end

data/spec/support/fixtures.rb

@@ -0,0 +1,10 @@
+module FixtureHelper
+  def fixture(name)
+    # File.new( File.dirname(__FILE__) + "/../fixtures/#{name}.html" ).read
+    File.dirname(__FILE__) + "/../fixtures/#{name}.html"
+  end
+end
+
+RSpec.configure do |conf|
+  conf.include FixtureHelper
+end

data/spec/unit/earl/earl_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Earl do
+
+
+  describe '##new' do
+    subject { Earl.new('http://test.host/') }
+    its(:to_s) { should eql('http://test.host/') }
+  end
+
+  describe '[]=' do
+    subject { Earl['http://test.host/'] }
+    its(:to_s) { should eql('http://test.host/') }
+  end
+
+end

data/spec/unit/earl/feed_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Earl do
+  let(:instance) { Earl[url] }
+  let(:url) { 'http:://example.com' }
+
+  before { Earl.any_instance.stub(:uri).and_return(source) }
+  subject { instance }
+
+  context "when page has no feeds associated" do
+    let(:source) { fixture(:page_without_feeds) }
+    its(:has_feed?) { should be_false }
+    its(:rss_feed) { should be_nil }
+    its(:feed) { should be_nil }
+  end
+
+  context "when page has rss feed associated" do
+    let(:source) { fixture(:page_with_rss_feed) }
+    its(:has_feed?) { should be_true }
+    its(:rss_feed) { should eql('http://www.readwriteweb.com/rss.xml') }
+    its(:feed) { should eql('http://www.readwriteweb.com/rss.xml') }
+  end
+
+  context "when page has atom feed associated" do
+    let(:source) { fixture(:page_with_atom_feed) }
+    its(:has_feed?) { should be_true }
+    its(:atom_feed) { should eql('http://www.readwriteweb.com/atom.xml') }
+    its(:feed) { should eql('http://www.readwriteweb.com/atom.xml') }
+  end
+
+  context "when page has rss and atom feed associated" do
+    let(:source) { fixture(:page_with_rss_and_atom_feeds) }
+    its(:has_feed?) { should be_true }
+    its(:rss_feed) { should eql('http://www.readwriteweb.com/rss.xml') }
+    its(:atom_feed) { should eql('http://www.readwriteweb.com/atom.xml') }
+    describe "#feed" do
+      context "default (rss)" do
+        subject { instance.feed }
+        it { should eql('http://www.readwriteweb.com/rss.xml') }
+      end
+      context "rss prefered" do
+        subject { instance.feed(:rss) }
+        it { should eql('http://www.readwriteweb.com/rss.xml') }
+      end
+      context "atom prefered" do
+        subject { instance.feed(:atom) }
+        it { should eql('http://www.readwriteweb.com/atom.xml') }
+      end
+    end
+  end
+
+  # context "when page IS an rss feed" do
+  #   let(:source) { fixture(:page_as_rss) }
+  #   its(:has_feed?) { should be_true }
+  #   its(:rss_feed) { should eql(url) }
+  #   its(:feed) { should eql(url) }
+  # end
+
+end

data/spec/unit/earl/oembed_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+class MockOembedResponse
+  def body
+    { 'html' => '<iframe/>' }
+  end
+end
+
+describe Earl do
+  let(:instance) { Earl.new(url,options) }
+  let(:url) { 'http:://example.com' }
+  let(:options) { {} }
+
+  describe "#oembed_options" do
+    subject { instance.oembed_options }
+    context "with default options" do
+      let(:expected) { { :maxwidth => "560", :maxheight => "315" } }
+      it { should eql(expected) }
+    end
+    context "with custom options" do
+      let(:options) { { :oembed => { :maxwidth => "260" } } }
+      let(:expected) { { :maxwidth => "260", :maxheight => "315" } }
+      it { should eql(expected) }
+    end
+    context "with custom options passed to oembed" do
+      let(:expected) { { :maxwidth => "360", :maxheight => "315" } }
+      before do
+        Oembedr.stub(:fetch).and_return(nil)
+        instance.oembed({ :maxwidth => "360" })
+      end
+      it { should eql(expected) }
+    end
+  end
+
+  describe "#oembed" do
+    let(:dummy_response) { MockOembedResponse.new }
+    before do
+      instance.stub(:base_url).and_return('')
+      Oembedr.stub(:fetch).and_return(dummy_response)
+    end
+    subject { instance.oembed }
+    it { should eql({ :html => '<iframe/>' }) }
+    describe "#oembed_html" do
+      subject { instance.oembed_html }
+      it { should eql('<iframe/>') }
+    end
+  end
+
+end

data/spec/unit/earl/scraper_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe Earl::Scraper do
+
+  before :each do
+    Earl.any_instance.stub(:uri_response).and_return(<<-DOC
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+    <html></html>
+    DOC
+    )
+  end
+
+  describe 'When validating URLs' do
+    before :each do
+      class Earl::TestScraper < Earl::Scraper
+        match /^http\:\/\/www\.test\.com\/$/
+        define_attribute(:title) {|response| :test_title }
+      end
+    end
+
+    it 'should return the result if the URL matches the scraper regexp' do
+      Earl['http://www.test.com/'].title.should == :test_title
+    end
+  end
+
+  describe 'When retrieving the response' do
+    it 'should return a Nokogiri document' do
+      Earl['test'].response.css('html').size.should == 1
+    end
+  end
+
+  describe 'Scraper inheritance' do
+    class SubScraper < Earl::Scraper
+      define_attribute :some_attribute do |doc|
+        doc
+      end
+    end
+
+    it 'inherits all attributes from its superclass' do
+      scraper = SubScraper.new('foo.bar')
+      scraper.attributes.should include(:title)
+      scraper.attributes.should include(:description)
+      scraper.attributes.should include(:image)
+      scraper.attributes.should include(:some_attribute)
+    end
+  end
+
+end

data/spec/unit/earl_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+describe Earl do
+
+  before { Earl.any_instance.stub(:uri).and_return(source) }
+  let(:url) { 'http://mock.net' }
+  let(:instance) { Earl[url] }
+  subject { instance }
+
+  describe '#title' do
+    context "example with title tag" do
+      let(:source) { fixture(:bicycles) }
+      its(:title) { should eql('bicycles (bicycles) on Twitter') }
+    end
+    context "youtube example with title tag" do
+      let(:source) { fixture(:youtube) }
+      its(:title) { should eql('YouTube - Symptoms of Swine Flu') }
+    end
+  end
+
+  describe '#description' do
+    context "example with description tag" do
+      let(:source) { fixture(:bicycles) }
+      its(:description) { should eql('I write business plans for a living and fiction to stay sane.  I also try my hand at all sorts of creative side projects: painting, game dev, whiskey drinking..') }
+    end
+    context "example without description tag" do
+      let(:source) { fixture(:bicycles_without_description) }
+      its(:description) { should be_nil }
+    end
+  end
+
+  describe '#image' do
+    context "example with image tag" do
+      let(:source) { fixture(:bicycles) }
+      its(:image) { should eql('http://assets0.twitter.com/images/loader.gif') }
+    end
+    context "example without image tag" do
+      let(:source) { fixture(:bicycles_without_images) }
+      its(:image) { should be_nil }
+    end
+  end
+
+  describe '#attributes' do
+    let(:source) { fixture(:bicycles) }
+    its(:attributes) { should be_an(Array) }
+    [ :description, :title, :image ].each do |attribute|
+      its(:attributes) { should include(attribute) }
+    end
+  end
+
+  describe '#metadata' do
+    let(:source) { fixture(:bicycles) }
+    its(:metadata) { should be_a(Hash) }
+    its(:metadata) { should eql({
+      :title => 'bicycles (bicycles) on Twitter',
+      :description => 'I write business plans for a living and fiction to stay sane.  I also try my hand at all sorts of creative side projects: painting, game dev, whiskey drinking..',
+      :image => 'http://assets0.twitter.com/images/loader.gif',
+      :rss_feed => "/statuses/user_timeline/user_id.rss",
+      :base_url => "http://mock.net",
+      :feed => "/statuses/user_timeline/user_id.rss"
+    }) }
+  end
+
+end
+

metadata

@@ -1,83 +1,149 @@
 --- !ruby/object:Gem::Specification
 name: earl
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
-- Jeremy Ruppel
+- teejayvanslyke
+- Paul Gallagher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-25 00:00:00.000000000 Z
+date: 2012-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: treetop
-  requirement: !ruby/object:Gem::Requirement
+  name: nokogiri
+  requirement: &70141387965520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.10
+        version: 1.4.4
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  version_requirements: *70141387965520
+- !ruby/object:Gem::Dependency
+  name: oembedr
+  requirement: &70141387964980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.10
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70141387964980
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &70141387964420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: *70141387964420
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70141387963920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+  type: :development
+  prerelease: false
+  version_requirements: *70141387963920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70141387963380 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.9.0
+        version: 2.8.0
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  version_requirements: *70141387963380
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &70141387962880 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.11'
+  type: :development
+  prerelease: false
+  version_requirements: *70141387962880
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70141387962240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.9.0
-description: What URI wishes it could look like
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: *70141387962240
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  requirement: &70141387961480 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: *70141387961480
+description: URL metadata API
 email:
-- jeremy.ruppel@gmail.com
+- tj@elctech.com
+- gallagher.paul@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .document
 - .gitignore
-- .rvmrc
+- .rspec
+- .travis.yml
 - Gemfile
+- Gemfile.lock
+- Guardfile
 - LICENSE
-- README.md
+- README.rdoc
 - Rakefile
 - earl.gemspec
 - lib/earl.rb
-- lib/earl/email_assembler.rb
-- lib/earl/email_entity.rb
-- lib/earl/email_parser.tt
-- lib/earl/entity_base.rb
-- lib/earl/hash_inquirer.rb
-- lib/earl/string_inquirer.rb
-- lib/earl/url_assembler.rb
-- lib/earl/url_entity.rb
-- lib/earl/url_parser.tt
+- lib/earl/earl.rb
+- lib/earl/scraper.rb
 - lib/earl/version.rb
-- spec/earl/earl_spec.rb
-- spec/earl/email_entity_spec.rb
-- spec/earl/email_parser_spec.rb
-- spec/earl/entity_base_spec.rb
-- spec/earl/hash_inquirer_spec.rb
-- spec/earl/string_inquirer_spec.rb
-- spec/earl/url_entity_spec.rb
-- spec/earl/url_parser_spec.rb
+- script/console
+- spec/fixtures/bicycles.html
+- spec/fixtures/bicycles_without_description.html
+- spec/fixtures/bicycles_without_images.html
+- spec/fixtures/page_as_atom.html
+- spec/fixtures/page_as_rss.html
+- spec/fixtures/page_with_atom_feed.html
+- spec/fixtures/page_with_rss_and_atom_feeds.html
+- spec/fixtures/page_with_rss_feed.html
+- spec/fixtures/page_without_feeds.html
+- spec/fixtures/youtube.html
+- spec/integration/feed_spec.rb
+- spec/integration/oembed_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/remind101/earl
+- spec/support/fixtures.rb
+- spec/unit/earl/earl_spec.rb
+- spec/unit/earl/feed_spec.rb
+- spec/unit/earl/oembed_spec.rb
+- spec/unit/earl/scraper_spec.rb
+- spec/unit/earl_spec.rb
+homepage: https://github.com/evendis/earl
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -97,17 +163,28 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.19
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
-summary: What URI wishes it could look like
+summary: URL metadata API for scraping titles, descriptions, images, and videos from
+  URL's
 test_files:
-- spec/earl/earl_spec.rb
-- spec/earl/email_entity_spec.rb
-- spec/earl/email_parser_spec.rb
-- spec/earl/entity_base_spec.rb
-- spec/earl/hash_inquirer_spec.rb
-- spec/earl/string_inquirer_spec.rb
-- spec/earl/url_entity_spec.rb
-- spec/earl/url_parser_spec.rb
+- spec/fixtures/bicycles.html
+- spec/fixtures/bicycles_without_description.html
+- spec/fixtures/bicycles_without_images.html
+- spec/fixtures/page_as_atom.html
+- spec/fixtures/page_as_rss.html
+- spec/fixtures/page_with_atom_feed.html
+- spec/fixtures/page_with_rss_and_atom_feeds.html
+- spec/fixtures/page_with_rss_feed.html
+- spec/fixtures/page_without_feeds.html
+- spec/fixtures/youtube.html
+- spec/integration/feed_spec.rb
+- spec/integration/oembed_spec.rb
 - spec/spec_helper.rb
+- spec/support/fixtures.rb
+- spec/unit/earl/earl_spec.rb
+- spec/unit/earl/feed_spec.rb
+- spec/unit/earl/oembed_spec.rb
+- spec/unit/earl/scraper_spec.rb
+- spec/unit/earl_spec.rb