earl 0.3.0 → 1.0.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -1,17 +1,6 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
+*.sw?
+.DS_Store
 coverage
-doc/
-lib/bundler/man
-pkg
 rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
+pkg
+.rvmrc

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,11 @@
+# These are specific configuration settings required for travis-ci
+# see http://travis-ci.org/evendis/earl
+language: ruby
+rvm:
+  # - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  # - rbx-18mode
+  # - rbx-19mode
+  # - jruby-18mode
+  # - jruby-19mode

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source :rubygems
 
 # Specify your gem's dependencies in earl.gemspec
-gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,60 @@
+PATH
+  remote: .
+  specs:
+    earl (1.0.0)
+      nokogiri (>= 1.4.4)
+      oembedr (>= 1.0.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    faraday (0.8.4)
+      multipart-post (~> 1.1)
+    faraday_middleware (0.8.8)
+      faraday (>= 0.7.4, < 0.9)
+    ffi (1.1.5)
+    guard (1.4.0)
+      listen (>= 0.4.2)
+      thor (>= 0.14.6)
+    guard-rspec (1.2.1)
+      guard (>= 1.1)
+    json (1.7.5)
+    listen (0.5.2)
+    mime-types (1.19)
+    multi_json (1.3.6)
+    multipart-post (1.1.5)
+    nokogiri (1.5.5)
+    oembedr (1.0.0)
+      faraday
+      faraday_middleware
+      multi_json
+      typhoeus
+    rake (0.9.2.2)
+    rb-fsevent (0.9.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    thor (0.16.0)
+    typhoeus (0.4.2)
+      ffi (~> 1.0)
+      mime-types (~> 1.18)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (> 1.1.0)
+  earl!
+  guard-rspec (~> 1.2.0)
+  rake (~> 0.9.2.2)
+  rb-fsevent (~> 0.9.1)
+  rdoc (~> 3.11)
+  rspec (~> 2.8.0)

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2, :all_on_start => false, :all_after_pass => false do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch('spec/spec_helper.rb')  { "spec" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/unit/#{m[1]}_spec.rb" }
+
+end
+

data/LICENSE

@@ -1,6 +1,4 @@
-Copyright (c) 2012 Jeremy Ruppel
-
-MIT License
+Copyright (c) 2009 T.J. VanSlyke
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -19,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,145 @@
+= Earl {<img src="https://secure.travis-ci.org/evendis/earl.png" />}[http://travis-ci.org/evendis/earl]
+
+Earl wants to help you scrape all the relevant metadata for your favorite web pages so you can be as cool as
+Facebook when displaying user-submitted link content. Earl returns details like titles, descriptions, content type,
+associated feeds, and OEmbed definitions if available.
+
+Earl is based on an original source project called _earl_ by {teejayvanslyke}[https://github.com/teejayvanslyke/earl] (but never released as a gem).
+The revamp was done by {Paul Gallagher}[https://github.com/tardate], and master source is currently
+available at https://github.com/evendis/earl.
+
+The Earl gem is officially named _earl_. Big thanks go to {jeremyruppel}[https://github.com/jeremyruppel] who
+contributed the ownership of the _earl_ gem name. The original _earl_ gem had a somewhat similar purpose - it is now defunct, but still available up to version 0.3.0 via rubgems. Any _earl_ gem with version 1.0.0 or higher is the new gem release (and is in no way backwardly compatible with
+earlier versions).
+
+== The Earl Cookbook
+
+=== How do instantiate Earl?
+
+Pass any url-like string to Earl:
+
+    my_earl_instance = Earl.new('https://github.com/evendis/earl')
+    #
+    # or using the []= convenience method:
+    my_earl_instance = Earl['https://github.com/evendis/earl']
+
+
+=== How do I inspect details of the page?
+
+    earl = Earl['https://github.com/evendis/earl']
+    earl.title
+    =>  "evendis/earl · GitHub"
+    earl.description
+    => "earl - URL metadata API for scraping titles, descriptions, images, and videos from URL's."
+    earl.image
+    => "https://a248.e.akamai.net/assets.github.com/images/modules/header/logov7@4x.png?1340935010"
+
+=== How do I get oembed details for a link?
+
+Earl will get oembed details for a link if they are available.
+
+    earl = Earl['http://www.youtube.com/watch?v=g3DCEcSlfhw']
+    earl.oembed
+    => {"provider_url"=>"http://www.youtube.com/", "thumbnail_url"=>"http://i4.ytimg.com/vi/g3DCEcSlfhw/hqdefault.jpg", "title"=>"'Virtuosos of Guitar 2008' festival, Moscow. Marcin Dylla", "html"=>"<iframe width=\"420\" height=\"315\" src=\"http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed\" frameborder=\"0\" allowfullscreen></iframe>", "author_name"=>"guitarmagnet", "height"=>315, "thumbnail_width"=>480, "width"=>420, "version"=>"1.0", "author_url"=>"http://www.youtube.com/user/guitarmagnet", "provider_name"=>"YouTube", "type"=>"video", "thumbnail_height"=>360}
+    # to get the embed code:
+    earl.oembed_html
+    => "<iframe width=\"420\" height=\"315\" src=\"http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed\" frameborder=\"0\" allowfullscreen></iframe>"
+
+=== How do I customise the oembed link?
+
+Supported oembed parameters may be provided with to `Earl.new` or to the `oembed` call:
+
+    earl = Earl.new('http://www.youtube.com/watch?v=g3DCEcSlfhw', {:oembed => {:maxwidth => "200", :maxheight => "320"}})
+    earl.oembed_html
+    => <iframe width="200" height="150" src="http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed" frameborder="0" allowfullscreen></iframe>
+    earl = Earl.new('http://www.youtube.com/watch?v=g3DCEcSlfhw')
+    earl.oembed({:maxwidth => "100", :maxheight => "120"})
+    earl.oembed_html
+    => <iframe width="100" height="75" src="http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed" frameborder="0" allowfullscreen></iframe>
+
+
+=== How do I inspect what attributes are available for a page?
+
+To see all of the attributes a URL provides, simply ask:
+
+    earl = Earl['https://github.com/evendis/earl']
+    earl.attributes
+    => [:title, :image, :description, :rss_feed, :atom_feed, :content_type, :base_url, :charset, :content_encoding, :headers, :feed]
+
+
+=== How can I extend Earl to scrape additional page details?
+
+Need to scrape additional page details currently supported by Earl?  Implement your own scraper:
+
+    class QotdScraper < Earl::Scraper
+      match /^http\:\/\/www\.quotationspage\.com\/qotd\.html$/
+
+      define_attribute :qotd do |doc|
+        doc.at('dt.quote a').text
+      end
+    end
+
+The define_attribute method will supply you with a Nokogiri document which you can traverse to your heart's content.
+Use the match method to limit the scope of URLs that your scraper will apply to.
+
+Your new attribute is now available for use:
+
+    Earl['http://www.quotationspage.com/qotd.html'].qotd
+    => "Love is a snowmobile racing across the tundra and then suddenly it flips over, pinning you underneath. At night, the ice weasels come."
+
+
+=== How do I install it for normal use?
+
+If using bundler, add gem 'earl' your application's Gemfile and run `bundle`.
+
+Or install it from the command-line:
+
+    $ gem install earl
+
+=== How do I install it for gem development?
+
+To work on enhancements of fix bugs in Earl, fork and clone the github repository.
+If you are using bundler (recommended), run <tt>bundle</tt> to install development dependencies:
+
+    $ gem install bundler
+    $ bundle
+
+=== How do I run the tests?
+
+Once development dependencies are installed, all unit tests are run with just:
+
+    $ rake
+    # or..
+    $ rake spec
+
+Unit tests exclude a set of integration tests that actually hit the network (therefore not 'nice' to run all the time,
+and also subject to failures due to network availability or changes in the services accessed). To run integration tests:
+
+    $ rake spec:integration
+
+To run all tests (unit and integration):
+
+    $ rake spec:all
+
+
+=== How do I automatically run tests when I modify files?
+
+Guard is installed as part of the development dependencies. Start a guard process in a terminal window:
+
+    $ bundle exec guard
+
+It will run all the tests to start with by default. Then whenever you change a file, the associated tests will execute in this terminal window.
+
+== Contributing to Earl
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+See LICENSE for details.

data/Rakefile

@@ -1,5 +1,38 @@
 #!/usr/bin/env rake
-require 'bundler/gem_tasks'
+require "bundler/gem_tasks"
+
+require 'rspec'
 require 'rspec/core/rake_task'
 
-RSpec::Core::RakeTask.new :spec
+desc "Run only unit test examples"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/unit/**/*_spec.rb'
+end
+
+desc "Run only integration test examples"
+RSpec::Core::RakeTask.new(:'spec:integration') do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/integration/**/*_spec.rb'
+end
+
+desc "Run all test examples including integration tests"
+RSpec::Core::RakeTask.new(:'spec:all') do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.main = "README.rdoc"
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "earl"
+  rdoc.rdoc_files.include('README*', 'lib/**/*.rb')
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -I lib -r earl.rb"
+end

data/earl.gemspec

@@ -2,11 +2,11 @@
 require File.expand_path('../lib/earl/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Jeremy Ruppel"]
-  gem.email         = ["jeremy.ruppel@gmail.com"]
-  gem.description   = %q{What URI wishes it could look like}
-  gem.summary       = %q{What URI wishes it could look like}
-  gem.homepage      = "https://github.com/remind101/earl"
+  gem.authors       = ["teejayvanslyke", "Paul Gallagher"]
+  gem.email         = ["tj@elctech.com", "gallagher.paul@gmail.com"]
+  gem.description   = %q{URL metadata API}
+  gem.summary       = %q{URL metadata API for scraping titles, descriptions, images, and videos from URL's}
+  gem.homepage      = "https://github.com/evendis/earl"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -15,7 +15,13 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Earl::VERSION
 
-  gem.add_dependency 'treetop', '>= 1.4.10'
+  gem.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
+  gem.add_runtime_dependency(%q<oembedr>, [">= 1.0.0"])
+  gem.add_development_dependency(%q<bundler>, ["> 1.1.0"])
+  gem.add_development_dependency(%q<rake>, ["~> 0.9.2.2"])
+  gem.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+  gem.add_development_dependency(%q<rdoc>, ["~> 3.11"])
+  gem.add_development_dependency(%q<guard-rspec>, ["~> 1.2.0"])
+  gem.add_development_dependency(%q<rb-fsevent>, ["~> 0.9.1"])
 
-  gem.add_development_dependency 'rspec', '>= 2.9.0'
 end

data/lib/earl.rb

@@ -1,24 +1,9 @@
-require 'earl/version'
-
-module Earl
-  autoload :EntityBase,     'earl/entity_base'
-  autoload :URLEntity,      'earl/url_entity'
-  autoload :URLAssembler,   'earl/url_assembler'
-  autoload :EmailEntity,    'earl/email_entity'
-  autoload :EmailAssembler, 'earl/email_assembler'
-  autoload :HashInquirer,   'earl/hash_inquirer'
-  autoload :StringInquirer, 'earl/string_inquirer'
-
-  class << self
-    def URL( source )
-      Earl::URLEntity.new source
-    end
-    def Email( source )
-      Earl::EmailEntity.new source
-    end
-  end
+require 'nokogiri'
+require 'open-uri'
 
-  class EarlError       < StandardError; end
-  class InvalidURLError < EarlError; end
-  class SubclassError   < EarlError; end
+class Earl
 end
+
+require 'earl/version'
+require 'earl/scraper'
+require 'earl/earl'

data/lib/earl/earl.rb

@@ -0,0 +1,158 @@
+require 'open-uri'
+require 'oembedr'
+
+class Earl
+
+  attr_accessor :url, :options, :oembed
+
+  def initialize(url, options={})
+    @url = url
+    @options = options
+  end
+
+  def to_s
+    url
+  end
+
+  def uri
+    @uri ||= URI.parse(url)
+  end
+
+  def uri_response
+    @uri_response ||= open(uri)
+  end
+
+  # Returns
+  # :base_url - the actual url (resolved after possible redirect) as a String
+  # :content_type - mime type
+  # :charset - returns a charset parameter in Content-Type field. It is downcased for canonicalization.
+  # :content_encoding - returns a list of encodings in Content-Encoding field as an Array of String. The encodings are downcased for canonicalization.
+  # :headers - raw response header metadata
+  # (excluded since this generally returns not RFC 2616 compliant date :last_modified  - returns a Time which represents Last-Modified field.
+  def uri_response_attribute(name)
+    case name
+    when :base_url
+      (uri_response_attribute(:base_uri) || url).to_s
+    when :headers
+      uri_response_attribute(:meta)
+    else
+      uri_response && uri_response.respond_to?(name) && uri_response.send(name)
+    end
+  end
+  protected :uri_response_attribute
+
+  def uri_response_attributes
+    [:content_type,:base_url,:charset,:content_encoding,:headers]
+  end
+  protected :uri_response_attributes
+
+  def scraper
+    @scraper ||= Scraper.for(url,self)
+  end
+
+  def response
+    scraper && scraper.response
+  end
+
+  # Returns a hash of link meta data, including:
+  # :title, :description, :image (all attributes)
+  # :base_url
+  def metadata
+    data = oembed || {}
+    attributes.each do |attribute|
+      if attribute_value = self.send(attribute)
+        data[attribute] ||= attribute_value
+      end
+    end
+    data
+  end
+
+  # Dispatch missing methods if a match for:
+  # - uri_response_attributes
+  # - scraper attributes
+  def method_missing(method, *args)
+    if uri_response_attributes.include?(method)
+      return uri_response_attribute(method)
+    elsif scraper && scraper.has_attribute?(method)
+      return scraper.attribute(method)
+    end
+    super
+  end
+
+  # Returns a full array of attributes available for the link
+  def attributes
+    scraper.attributes.keys + uri_response_attributes + [:feed]
+  end
+
+  # Returns the options to be used for oembed
+  def oembed_options
+    { :maxwidth => "560", :maxheight => "315" }.merge(options[:oembed]||{})
+  end
+
+  # Returns the oembed meta data hash for the URL (or nil if not defined/available) e.g.
+  # For http://www.youtube.com/watch?v=g3DCEcSlfhw:
+  #   {
+  #     "provider_url"=>"http://www.youtube.com/",
+  #     "thumbnail_url"=>"http://i4.ytimg.com/vi/g3DCEcSlfhw/hqdefault.jpg",
+  #     "title"=>"'Virtuosos of Guitar 2008' festival, Moscow. Marcin Dylla",
+  #     "html"=>"<iframe width=\"459\" height=\"344\" src=\"http://www.youtube.com/embed/g3DCEcSlfhw?fs=1&feature=oembed\" frameborder=\"0\" allowfullscreen></iframe>",
+  #     "author_name"=>"guitarmagnet",
+  #     "height"=>344,
+  #     "thumbnail_width"=>480,
+  #     "width"=>459,
+  #     "version"=>"1.0",
+  #     "author_url"=>"http://www.youtube.com/user/guitarmagnet",
+  #     "provider_name"=>"YouTube",
+  #     "type"=>"video",
+  #     "thumbnail_height"=>360
+  #   }
+  #
+  # +options+ defines a custom oembed options hash and will cause a re-fetch of the oembed metadata
+  def oembed(options=nil)
+    if options # use custom options, refetch oembed metadata
+      @options[:oembed] = options
+      @oembed = nil
+    end
+    begin
+      @oembed ||= if h = Oembedr.fetch(base_url, :params => oembed_options).body
+        h.keys.each do |key| # symbolize_keys!
+          h[(key.to_sym rescue key) || key] = h.delete(key)
+        end
+        h
+      end
+    rescue
+    end
+    @oembed
+  end
+
+  # Returns the oembed code for the url (or nil if not defined/available)
+  def oembed_html
+    oembed && oembed[:html]
+  end
+
+  # Returns true if there is an ATOM or RSS feed associated with this URL.
+  def has_feed?
+    !feed.nil?
+  end
+
+  # Returns the feed URL associated with this URL.
+  # Returns RSS by default, or ATOM if +prefer+ is not :rss.
+  def feed(prefer = :rss)
+    rss = rss_feed
+    atom = atom_feed
+    if rss && atom
+      prefer == :rss ? rss : atom
+    else
+      rss || atom
+    end
+  end
+
+  class << self
+
+    def [](url)
+      new(url)
+    end
+
+  end
+
+end