RubyGems - trawler - Versions diffs - 0.0.1 - Mend

trawler 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

checksums.yaml +7 -0
data/.gitignore +19 -0
data/.rspec +1 -0
data/Gemfile +4 -0
data/LICENSE.txt +22 -0
data/README.md +52 -0
data/Rakefile +1 -0
data/lib/trawler/document.rb +22 -0
data/lib/trawler/parsed_document.rb +30 -0
data/lib/trawler/parser.rb +94 -0
data/lib/trawler/spider.rb +24 -0
data/lib/trawler/version.rb +3 -0
data/lib/trawler.rb +11 -0
data/spec/document_spec.rb +31 -0
data/spec/fixtures/sample_pages/simple.html +10 -0
data/spec/fixtures/sample_pages/tumblr.html +840 -0
data/spec/fixtures/sample_pages/youtube.html +1404 -0
data/spec/fixtures/vcr_cassettes/trawl_page.yml +57 -0
data/spec/parsed_document_spec.rb +32 -0
data/spec/parser_spec.rb +100 -0
data/spec/spec_helper.rb +26 -0
data/spec/spider_spec.rb +52 -0
data/spec/trawler_spec.rb +30 -0
data/trawler.gemspec +30 -0
metadata +189 -0

data/spec/fixtures/vcr_cassettes/trawl_page.yml ADDED Viewed

@@ -0,0 +1,57 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://www.cats.com/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - '*/*'
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - Apache
+      Set-Cookie:
+      - COOKIE=10.5.19.71.1382123959465997; path=/
+      - referrer=; path=/
+      - referrer=www.cats.com; path=/
+      - t=2fe697b0382a11e3a9020015c5e70714; path=/
+      Etag:
+      - '"AAAAUHDDtBw"'
+      Last-Modified:
+      - Wed, 16 Oct 2013 20:57:10 GMT
+      Vary:
+      - Accept-Encoding,User-Agent
+      Cartoon:
+      - aalander3
+      Content-Type:
+      - text/html; charset=UTF-8
+      Content-Length:
+      - '39'
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Fri, 18 Oct 2013 19:19:19 GMT
+      X-Varnish:
+      - '2126042946'
+      Age:
+      - '0'
+      Via:
+      - 1.1 varnish
+      Connection:
+      - keep-alive
+    body:
+      encoding: UTF-8
+      string: "<html>\r\n<body>\r\n</body>\r\n</html>\r\n"
+    http_version:
+  recorded_at: Fri, 18 Oct 2013 19:19:14 GMT
+recorded_with: VCR 2.6.0

data/spec/parsed_document_spec.rb ADDED Viewed

@@ -0,0 +1,32 @@
+require "spec_helper"
+describe Trawler::ParsedDocument do
+  let(:data)  { double("PARSER", images: ["foo"], title: "foobar", video: "bar", description: "foobarbaz", document: "") }
+  let(:doc)  { Trawler::ParsedDocument.new("www.cats.com", data) }
+  it "sets the url" do
+    expect(doc.url).to eq "www.cats.com"
+  end
+  it "has images" do
+    expect(doc.images).to be_a Array
+    expect(doc.images).to include "foo"
+  end
+  it "has a title" do
+    expect(doc.title).to eq "foobar"
+  end
+  it "has a video" do
+    expect(doc.video).to eq "bar"
+  end
+  it "has a description" do
+    expect(doc.description).to eq "foobarbaz"
+  end
+  it "has the raw data" do
+    expect(doc.raw_data).to eq ""
+  end
+end

data/spec/parser_spec.rb ADDED Viewed

@@ -0,0 +1,100 @@
+require "spec_helper"
+describe Trawler::Parser do
+  context "image rich page" do
+    let(:page) { fixture("sample_pages/tumblr.html") }
+    let(:parser) do
+      Trawler::Parser.new(
+        page:       page,
+        url:        "http://www.foo.com",
+        image_size: "100"
+      )
+    end
+    describe "#title" do
+      it "parses the page title" do
+        expect(parser.title).to eq "Dogshaming"
+      end
+    end
+    describe "#description" do
+      it "parses the description from the meta data" do
+        expect(parser.description).not_to be_empty
+      end
+    end
+    describe "#videos" do
+      it "returns nil if no videos are found" do
+        expect(parser.video).to be_nil
+      end
+    end
+    describe "#images" do
+      it "returns an array of images" do
+        expect(parser.images).to be_a Array
+      end
+      it "all the images that are larger than the min image size" do
+        expect(parser.images.size).to eq 10
+      end
+    end
+  end
+  context "video page" do
+    let(:page) { fixture("sample_pages/youtube.html") }
+    let(:parser) do
+      Trawler::Parser.new(
+        page:       page,
+        url:        "http://www.foo.com",
+        image_size: "100"
+      )
+    end
+    it "gets the title" do
+      expect(parser.title).to eq "Single Page Web Applications: JavaScript End-to-End (The Hard Stuff)"
+    end
+    it "gets the video" do
+      expect(parser.video).to eq "http://www.youtube.com/v/OrIFaWJ9Glo?version=3&autohide=1"
+    end
+    it "gets the description" do
+      expect(parser.description).to eq "In the old days, when websites were steam powered and exploded regularly, the web was simple, but slow. As it evolved it became more powerful, but harder on ..."
+    end
+    it "gets the images" do
+      expect(parser.images).to include "http://i1.ytimg.com/vi/OrIFaWJ9Glo/maxresdefault.jpg"
+    end
+  end
+  context "document style page" do
+    let(:page) { fixture("sample_pages/simple.html") }
+    let(:parser) do
+      Trawler::Parser.new(
+        page: page,
+        url: "foobar",
+        image_size: "100"
+      )
+    end
+    let(:description) do
+      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed dapibus velit in lacus mollis vehicula nec a arcu"
+    end
+    it "returns the description from the body of the text" do
+      expect(parser.description[0..120]).to match description
+    end
+    it "gets the title from the html" do
+      expect(parser.title).to eq "Super simple html page"
+    end
+    it "gets no images" do
+      expect(parser.images).to be_empty
+    end
+    it "gets no video" do
+      expect(parser.video).to be_nil
+    end
+  end
+end

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,26 @@
+require 'webmock/rspec'
+require "simplecov"
+require "json"
+require "vcr"
+SimpleCov.start do
+  add_filter "/spec/"
+end
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
+  c.hook_into :webmock
+end
+$LOAD_PATH.unshift(File.expand_path("../..", __FILE__))
+require 'lib/trawler'
+def fixture_path(filename=nil)
+  path = File.expand_path("../fixtures", __FILE__)
+  filename.nil? ? path : File.join(path, filename)
+end
+def fixture(file)
+  File.read(File.join(fixture_path, file))
+end

data/spec/spider_spec.rb ADDED Viewed

@@ -0,0 +1,52 @@
+require "spec_helper"
+describe Trawler::Spider do
+  let(:spider) { Trawler::Spider.new("www.cats.com") }
+  it "assigns the url" do
+    expect(spider.instance_variable_get("@url")).to eq "www.cats.com"
+  end
+  describe "#full_url" do
+    context "without a full scheme" do
+      it "adds the protocol to the url" do
+        expect(spider.full_url).to eq "http://www.cats.com"
+      end
+    end
+    context "with a full scheme" do
+      let(:spider) { Trawler::Spider.new("https://foo.com") }
+      it "returns the url" do
+        expect(spider.full_url).to eq "https://foo.com"
+      end
+    end
+  end
+  describe "#get_page" do
+    it "returns a string" do
+      VCR.use_cassette("trawl_page") do
+        expect(spider.get_page).to be_a StringIO
+      end
+    end
+  end
+  describe "#call" do
+    let(:crawled_spider) do
+      VCR.use_cassette("trawl_page") do
+        spider.call
+      end
+    end
+    it "returns the object" do
+      expect(crawled_spider).to be_a Trawler::Spider
+    end
+    it "has the full_url" do
+      expect(crawled_spider.full_url).to eq "http://www.cats.com"
+    end
+    it "has the page" do
+      expect(crawled_spider.page).not_to be_nil
+    end
+  end
+end

data/spec/trawler_spec.rb ADDED Viewed

@@ -0,0 +1,30 @@
+require "spec_helper"
+describe Trawler do
+  describe "#fetch" do
+    before do
+      Trawler::Spider.any_instance.stub(:get_page) { fixture("sample_pages/tumblr.html") }
+    end
+    let(:haul) { Trawler.fetch("http://www.dogshaming.com/") }
+    it "returns a ParsedDocument" do
+      expect(haul).to be_a(Trawler::ParsedDocument)
+    end
+    it "has the original url" do
+      expect(haul.url).to eq("http://www.dogshaming.com/")
+    end
+    [:video, :images, :description, :title].each do |attr|
+      it "has #{attr}" do
+        expect(haul).to respond_to(attr)
+      end
+    end
+    it "returns an array of images" do
+      expect(haul.images).to be_a Array
+    end
+  end
+end

data/trawler.gemspec ADDED Viewed

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'trawler/version'
+Gem::Specification.new do |spec|
+  spec.name          = "trawler"
+  spec.version       = Trawler::VERSION
+  spec.authors       = ["Jon Wheeler"]
+  spec.email         = ["jon@doejo.com"]
+  spec.description   = %q{Scrapes the web}
+  spec.summary       = %q{Scrape a url for images, video links and meta descriptions}
+  spec.homepage      = %q{https://github.com/Jonwheeler/trawler}
+  spec.license       = "MIT"
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.add_dependency "nokogiri",              "~> 1.6"
+  spec.add_dependency "hashr",                 "~> 0.0.22"
+  spec.add_development_dependency "bundler",   "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec",     "~> 2.14"
+  spec.add_development_dependency "webmock",   "~> 1.14"
+  spec.add_development_dependency "vcr",       "~> 2.6"
+  spec.add_development_dependency "simplecov", "~> 0.7"
+end

metadata ADDED Viewed

@@ -0,0 +1,189 @@
+--- !ruby/object:Gem::Specification
+name: trawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jon Wheeler
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-10-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: hashr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: Scrapes the web
+email:
+- jon@doejo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/trawler.rb
+- lib/trawler/document.rb
+- lib/trawler/parsed_document.rb
+- lib/trawler/parser.rb
+- lib/trawler/spider.rb
+- lib/trawler/version.rb
+- spec/document_spec.rb
+- spec/fixtures/sample_pages/simple.html
+- spec/fixtures/sample_pages/tumblr.html
+- spec/fixtures/sample_pages/youtube.html
+- spec/fixtures/vcr_cassettes/trawl_page.yml
+- spec/parsed_document_spec.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb
+- spec/spider_spec.rb
+- spec/trawler_spec.rb
+- trawler.gemspec
+homepage: https://github.com/Jonwheeler/trawler
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Scrape a url for images, video links and meta descriptions
+test_files:
+- spec/document_spec.rb
+- spec/fixtures/sample_pages/simple.html
+- spec/fixtures/sample_pages/tumblr.html
+- spec/fixtures/sample_pages/youtube.html
+- spec/fixtures/vcr_cassettes/trawl_page.yml
+- spec/parsed_document_spec.rb
+- spec/parser_spec.rb
+- spec/spec_helper.rb
+- spec/spider_spec.rb
+- spec/trawler_spec.rb