url_info_extractor 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+spec/cassettes/*
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in url_info_extractor.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2011, 2012 Florian Dütsch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+UrlInfoExtractor
+================
+
+The library helps you extracting some metadata out of HTML URLs.
+
+[https://github.com/der-flo/url_info_extractor](github.com/der-flo/url_info_extractor)
+
+Usage
+-----
+    % gem install url_info_extractor
+
+    require 'url_info_extractor'
+    infos = UrlInfoExtractor.new('http://rubygems.org')
+    puts infos.title
+    # RubyGems.org | your community gem host
+    puts infos.favicon_url
+    # http://rubygems.org/favicon.ico
+    puts infos.favicon_exists?
+    # true
+    puts infos.description
+    #
+
+Note that the favicon handling determines the URL by link-rel-icon tag and
+base tag. `favicon_exists?` does an HTTP HEAD request to check whether the
+icon is available at the determined path.
+
+Fork me!
+--------
+If you find this library useful but miss some feature, please drop me a note
+or - even better - fork the project at GitHub. Along with some specs I'll
+accept your pull requests.
+
+Author and license
+------------------
+* Florian Dütsch (mail AT florian-duetsch.de)
+* MIT, see LICENSE file.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/TODO

@@ -0,0 +1,3 @@
+* Error handling necessary?
+* Nobody uses meta-keywords anymore. Should we support them?
+

data/lib/url_info_extractor/favicon.rb

@@ -0,0 +1,16 @@
+class UrlInfoExtractor
+  module Favicon
+    def favicon_url
+      base = fetched_data[:base_link_value]
+      uri = base ? URI.parse(base) : @uri
+      link = fetched_data[:favicon_link_value]
+      link ||= '/favicon.ico'
+      uri.merge(URI.parse(link)).to_s
+    end
+    def favicon_exists?
+      response = do_http_request(favicon_url, :head)
+      response.is_a? Net::HTTPSuccess
+    end
+  end
+end
+

data/lib/url_info_extractor/version.rb

@@ -0,0 +1,3 @@
+class UrlInfoExtractor
+  VERSION = "0.0.1"
+end

data/lib/url_info_extractor.rb

@@ -0,0 +1,65 @@
+require "url_info_extractor/version"
+require 'url_info_extractor/favicon'
+require 'nokogiri'
+require 'active_support'
+require 'active_support/core_ext/object/try'
+require 'active_support/core_ext/string/inflections'
+require 'net/https'
+
+class UrlInfoExtractor
+  include Favicon
+
+  class SchemeNotSupported < StandardError; end
+
+  def initialize(url)
+    @uri = URI.parse(url)
+    @path = @uri.path
+    @path = '/' if @path.empty?
+  end
+  def title
+    fetched_data[:title]
+  end
+
+  def description
+    fetched_data[:description]
+  end
+
+  def scheme_supported?
+    %w(http https).include? @uri.scheme
+  end
+
+  private
+
+  PROCESSABLE_MIME_TYPES = %w(application/xhtml+xml text/html)
+
+  def fetched_data
+    @fetched_data ||= fetch_data
+  end
+
+  def fetch_data
+    raise SchemeNotSupported unless scheme_supported?
+    response = do_http_request(@path, :get)
+
+    return {} unless PROCESSABLE_MIME_TYPES.include?(response.content_type)
+
+    doc = Nokogiri::HTML(response.body)
+    head = doc.css('html head')
+
+    { title: head.css('title').try(:first).try(:content),
+      description:
+        head.css('meta[name=description]').try(:first).try(:[], 'content'),
+      favicon_link_value:
+        head.css('link[rel~=icon]').try(:first).try(:[], 'href'),
+      base_link_value: head.css('base').try(:first).try(:[], 'href')
+    }
+  end
+
+  def do_http_request(path, method)
+    request = "Net::HTTP::#{method.to_s.camelize}".constantize.new(path)
+    Net::HTTP.start(@uri.host, @uri.port,
+                    use_ssl: @uri.scheme == 'https') do |http|
+      http.request request
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'url_info_extractor'
+require 'vcr'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/cassettes'
+  c.default_cassette_options = { record: :new_episodes }
+  c.hook_into :webmock
+end
+
+RSpec.configure do |c|
+  c.extend VCR::RSpec::Macros
+end
+

data/spec/url_info_extractor/favicon_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe UrlInfoExtractor::Favicon do
+  use_vcr_cassette
+  it "returns favicon of rubygems.org" do
+    infos = UrlInfoExtractor.new('http://rubygems.org')
+    infos.favicon_url.should == 'http://rubygems.org/favicon.ico'
+  end
+  it 'returns the favicon of der-flo.org' do
+    infos = UrlInfoExtractor.new('http://der-flo.org')
+    infos.favicon_url.should == 'http://der-flo.org/favicon.ico'
+  end
+
+  it 'handles pages without an specified favicon' do
+    infos = UrlInfoExtractor.new('https://github.com')
+    infos.favicon_url.should == 'https://github.com/favicon.ico'
+  end
+
+  context 'with non-existing favicon' do
+    use_vcr_cassette 'favicon does not exist', record: :none
+
+    it 'checks whether the icon really exists' do
+      infos = UrlInfoExtractor.new('http://www.favicon-missing.com/')
+      infos.favicon_exists?.should be_false
+    end
+  end
+
+  context 'with <base> sample' do
+    use_vcr_cassette 'with base sample', record: :none
+    it 'handles pages with a base specification correct' do
+      infos = UrlInfoExtractor.new('http://www.with-base.com/')
+      infos.favicon_url.should == 'http://www2.with-base.com/icon.ico'
+    end
+  end
+end
+

data/spec/url_info_extractor_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe UrlInfoExtractor do
+  use_vcr_cassette
+  it 'extracts the title from rubygems.org' do
+    infos = UrlInfoExtractor.new('http://rubygems.org')
+    infos.title.should =~ /rubygems/i
+  end
+  it 'extracts the title from rubyonrails.org' do
+    infos = UrlInfoExtractor.new('http://rubyonrails.org')
+    infos.title.should == 'Ruby on Rails'
+  end
+  it 'recognizes HTTPS url' do
+    infos = UrlInfoExtractor.new('https://github.com/')
+    infos.title.should == 'GitHub - Social Coding'
+  end
+  it 'verifies MIME type' do
+    infos = UrlInfoExtractor.new('http://rubyonrails.org/images/rails.png')
+    infos.title.should be_nil
+  end
+  it 'support only the HTTP and HTTPS protocol' do
+    infos = UrlInfoExtractor.new('ftp://test.com')
+    infos.scheme_supported?.should_not be
+    expect do
+      infos.title
+    end.to raise_error(UrlInfoExtractor::SchemeNotSupported)
+  end
+  it 'extracts the description out of the page' do
+    infos = UrlInfoExtractor.new('https://github.com/der-flo/dotfiles')
+    infos.description.should == 'dotfiles - My config files'
+  end
+end
+

data/url_info_extractor.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "url_info_extractor/version"
+
+Gem::Specification.new do |s|
+  s.name        = "url_info_extractor"
+  s.version     = UrlInfoExtractor::VERSION
+  s.authors     = ["Florian Dütsch"]
+  s.email       = ["mail@florian-duetsch.de"]
+  s.homepage    = ""
+  s.summary     = %q{Extract metadata out of a URL}
+  s.description = %q{Extract title, meta-description and favicon out of a URL}
+
+  s.rubyforge_project = "url_info_extractor"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map do |f|
+    File.basename(f)
+  end
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency 'guard-rspec'
+  s.add_development_dependency 'vcr', '2.0.0.rc1'
+  s.add_development_dependency 'webmock'
+  if RUBY_PLATFORM =~ /darwin/i
+    s.add_development_dependency 'rb-fsevent'
+    s.add_development_dependency 'growl'
+  end
+
+  s.add_runtime_dependency 'nokogiri', '~> 1.5'
+  s.add_runtime_dependency 'activesupport', '>= 3'
+end
+

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: url_info_extractor
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Florian Dütsch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70293470681620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70293470681620
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70293470681200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70293470681200
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: &70293470680700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: *70293470680700
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: &70293469848420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70293469848420
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  requirement: &70293469846880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70293469846880
+- !ruby/object:Gem::Dependency
+  name: growl
+  requirement: &70293469844900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70293469844900
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70293469843840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70293469843840
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70293469842780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70293469842780
+description: Extract title, meta-description and favicon out of a URL
+email:
+- mail@florian-duetsch.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- TODO
+- lib/url_info_extractor.rb
+- lib/url_info_extractor/favicon.rb
+- lib/url_info_extractor/version.rb
+- spec/spec_helper.rb
+- spec/url_info_extractor/favicon_spec.rb
+- spec/url_info_extractor_spec.rb
+- url_info_extractor.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: url_info_extractor
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Extract metadata out of a URL
+test_files:
+- spec/spec_helper.rb
+- spec/url_info_extractor/favicon_spec.rb
+- spec/url_info_extractor_spec.rb