trawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f2215c97872959bff772658f412a924f9d4f6357
+  data.tar.gz: 5ed5422791ad8fe45a668984c17ad84dba60d924
+SHA512:
+  metadata.gz: 1fe1b9aa090d52b281b38b1e948613eb37192c4872a0de922c8335ca9b6757c53ffa839ad462c083f9f31a6fcbc568412d8f414c57ea47f10854459002e4fdfe
+  data.tar.gz: b082e3234696d8bb0cd99346fa71816eb98e4062973e94e4e3cf7c2e6b42154200382507da7ef058348faf8c52a9baea8a0133be67be451766236c3dbe287e93

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.ruby-*
+.DS_Store

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in trawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jon Wheeler
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# Trawler
+
+Trawl the web for meta tags.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'trawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install trawler
+
+## Usage
+
+Trawler is designed to be simple to use. Give the Trawler a URL and some options
+and it will bring back a catch of Images, Meta Descriptions and Video URLS in a
+ParsedDocument object.
+
+```ruby
+  doc = Trawler.fetch("www.foobar.com") 
+  doc.url         # => "http://www.foobar.com"
+  doc.description # => "Descriptive meta tag"
+  doc.title       # => "Foobar.com The place of Foo"
+  doc.images      # => [
+                  #      http://www.foobar.com/assets/bar.png,
+                  #      http://www.foobar.com/assets/baz.jpg,
+                  #      http://www.foobar.com/assets/bat.gif
+                  #     ]
+  doc.video       # => [
+                  #      http://www.vimeo.com/354357349
+                  #     ]
+```
+
+## Testing
+
+```
+ rspec/
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/trawler/document.rb

@@ -0,0 +1,22 @@
+module Trawler
+  class Document
+    def initialize(url, options = {}) 
+      @url    = url
+      @parser = options.fetch(:parser, Parser)
+      @spider = options.fetch(:spider, Spider)
+      @image  = options.fetch(:image_size, "100")
+    end
+
+    def parse
+      ParsedDocument.new(doc.full_url, parsed_data)
+    end
+
+    def parsed_data
+      Parser.new(page: doc.page, url: doc.full_url, image_size: @image)
+    end
+
+    def doc
+      @page ||= @spider.new(@url).call
+    end
+  end
+end

data/lib/trawler/parsed_document.rb

@@ -0,0 +1,30 @@
+module Trawler
+  class ParsedDocument
+    attr_reader :url, :data
+
+    def initialize(host, data)
+      @url  = host
+      @data = data
+    end
+
+    def title
+      data.title
+    end
+  
+    def video
+      data.video
+    end
+
+    def images
+      data.images
+    end
+
+    def description
+      data.description
+    end
+
+    def raw_data
+      @raw_data ||= data.document
+    end
+  end
+end

data/lib/trawler/parser.rb

@@ -0,0 +1,94 @@
+require "nokogiri"
+require "hashr"
+
+module Trawler
+  class Parser
+    attr_reader :url
+    attr_reader :meta_data
+
+    def initialize(options)
+      @page           = options[:page]
+      @url            = options[:url]
+      @min_image_size = options[:image_size] 
+      @meta_data      = Hashr.new
+    end
+
+    def title
+      meta_title.nil? ? html_title : meta_title
+    end
+
+    def description
+      meta_description.nil? ? html_description : meta_description
+    end
+
+    def images
+      images = []
+      images << meta_image 
+      images << find_images
+      images.flatten.compact.map { |i| i.strip }.uniq
+    end
+
+    def video
+      meta_video
+    end
+
+    def document 
+      @document ||= Nokogiri::HTML(@page) 
+    end
+
+    private
+
+    def html_title
+      document.css("title").inner_text rescue nil
+    end
+
+    def html_description
+      first_long_paragraph = document.search("//p[string-length() >= 150]").first
+      first_long_paragraph ? first_long_paragraph.text : ''
+    end
+
+    def find_images
+      images = document.css('img')
+      images = images.select { |img| !img[:alt].nil? && !img[:alt].empty? }
+      images.reject! { |img| img[:alt] =~ /(loading|spinner)/i }
+      images.reject! { |img| img[:class] =~ /(loading|spinner|icon)/i }
+      images.reject! { |img| img[:width] && img[:width] < @min_image_size }
+      images.map { |img| img[:src] }
+    end
+
+    def meta_title
+      scrape_meta_data
+      meta_data.title rescue nil
+    end
+
+    def meta_description
+      scrape_meta_data
+      meta_data.description rescue nil
+    end
+
+    def meta_video
+      scrape_meta_data
+      meta_data.video rescue nil
+    end
+
+    def meta_image
+      scrape_meta_data
+      meta_data.image rescue nil
+    end
+
+    def scrape_meta_data
+      document.xpath("//meta").each do |element|
+        get_meta_data_name_or_property(element)
+      end
+    end
+
+    def get_meta_data_name_or_property(element)
+      name_or_property = element.attributes["name"] ? "name" : (element.attributes["property"] ? "property" : nil)
+      content_or_value = element.attributes["content"] ? "content" : (element.attributes["value"] ? "value" : nil)
+
+      if !name_or_property.nil? && !content_or_value.nil?
+        meta_data[element.attributes[name_or_property].value.downcase.gsub(/(og:|twitter:)/, "").gsub(/:/, "_").to_sym] = element.attributes[content_or_value].value
+      end
+    end
+  end
+end

data/lib/trawler/spider.rb

@@ -0,0 +1,24 @@
+require "open-uri"
+
+module Trawler
+  class Spider
+    attr_reader :page, :full_url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def call
+      get_page   
+      self
+    end
+
+    def full_url
+      @full_url ||= URI.parse(@url).scheme.nil? ? 'http://' + @url : @url
+    end
+
+    def get_page
+      @page ||= open(full_url)
+    end
+  end
+end

data/lib/trawler/version.rb

@@ -0,0 +1,3 @@
+module Trawler
+  VERSION = "0.0.1"
+end

data/lib/trawler.rb

@@ -0,0 +1,11 @@
+require "trawler/document"
+require "trawler/parsed_document"
+require "trawler/spider"
+require "trawler/parser"
+require "trawler/version"
+
+module Trawler
+  def self.fetch(url, options = {})
+    Document.new(url, options).parse
+  end
+end

data/spec/document_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe Trawler::Document do
+  let(:document) { Trawler::Document.new("http://www.dogshaming.com") }
+
+  before do
+    Trawler::Spider.any_instance.stub(:get_page) { fixture("sample_pages/tumblr.html") }
+  end
+
+  it "requires a URL" do
+    expect{ Trawler::Document.new }.to raise_error
+  end
+
+  describe "#doc" do
+    it "returns the crawled document object" do
+      expect(document.doc).to be_a Trawler::Spider
+    end
+  end
+
+  describe "#parse" do
+    it "returns a parsed document" do
+      expect(document.parse).to be_a Trawler::ParsedDocument
+    end
+  end
+
+  describe "#parsed_data" do
+    it "returns a parse object" do
+      expect(document.parsed_data).to be_a Trawler::Parser
+    end
+  end
+end

data/spec/fixtures/sample_pages/simple.html

@@ -0,0 +1,10 @@
+<html>
+  <head>
+    <title>Super simple html page</title>
+  </head>
+  <body>
+    <p>
+      Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed dapibus velit in lacus mollis vehicula nec a arcu. Vivamus in turpis ultricies, ornare quam vehicula, adipiscing metus. Nam mauris libero, dignissim quis sodales ac, dignissim sed risus. Quisque varius eget eros vel condimentum. Duis cursus pretium est vitae lobortis. Donec tincidunt lorem vel erat feugiat lobortis. Pellentesque scelerisque eu massa a bibendum. Curabitur aliquam vitae eros nec semper. Morbi eu dui dapibus, iaculis ipsum vulputate, mollis ante. Pellentesque quis ipsum dignissim, pharetra tortor non, fringilla diam. Morbi a justo nec neque vulputate tempor eget sed velit. Nunc imperdiet neque mi, ut tincidunt leo convallis id. Nam in gravida metus.
+    </p>
+  </body>
+</html>