sagrone_scraper 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82fc9ba674d9d3398b5f596d513fbb8eeb8abe3b
-  data.tar.gz: a8acac43dc318b6dcad951d57d3e1ce59478c67d
+  metadata.gz: 80b3c30080aba0c8b1da8cfdcdefbb8e6ef527e1
+  data.tar.gz: c9992757e44377ed3081089f0348cdf1535e8a8e
 SHA512:
-  metadata.gz: 0dc5041f027f685ac241fcf0e103d3d8a6fa225002cdedd65a2f072fafb43904d3514a7cabad9a83bb0d3f02a3f7241c2c8dbeffdb48f133439f308319462870
-  data.tar.gz: 15644f0da27c4cb3f2452ac11958261acc7d0bd5e3e0539a7d7817b47f5c5096ea3f0d107b2f456f34f6fe8273fb7742128cd72f5c142c8416a521262ffb8e09
+  metadata.gz: 97477b7732ec3485aa7ba5ef2c7cb16ac130d6b1a1f6ee8b57deb5cf53fb6ae50bafdec13d1c575dbc382a748583a3e70b0da55660bd7065daaebc1479d466c4
+  data.tar.gz: fc8762b63b3429dcbd004bbdc39c6ef5bd7351e8b483ec8185c79b985ab5b2091601005a533270fc511e16d329f31a20616e1ea068f2b7ce7c3a0b3928087c5f

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ### HEAD
 
+### 0.0.3
+
+- add `SagroneScraper::Parser.can_parse?(url)` class method, which must be  implemented in subclasses
+- add `SagroneScraper` logic to _scrape_ a URL based on a set of _registered parsers_
+
 ### 0.0.2
 
 - add `SagroneScraper::Parser`

data/README.md

@@ -3,7 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/sagrone_scraper.svg)](http://badge.fury.io/rb/sagrone_scraper)
 [![Build Status](https://travis-ci.org/Sagrone/scraper.svg?branch=master)](https://travis-ci.org/Sagrone/scraper)
 
-Simple library to scrap web pages. Bellow you will find information on [how to use it](#usage).
+Simple library to scrap web pages. Bellow you will find information on [how to use it](#basic-usage).
 
 ## Table of Contents
 
@@ -12,6 +12,7 @@ Simple library to scrap web pages. Bellow you will find information on [how to u
 - [Modules](#modules)
   + [`SagroneScraper::Agent`](#sagronescraperagent)
   + [`SagroneScraper::Parser`](#sagronescraperparser)
+  + [`SagroneScraper.scrape`](#sagronescraperscrape)
 
 ## Installation
 
@@ -40,7 +41,7 @@ The agent is responsible for scraping a web page from a URL. Here is how you can
 1. one way is to pass it a `url` option
 
     ```ruby
-    require 'sagrone_scraper/agent'
+    require 'sagrone_scraper'
 
     agent = SagroneScraper::Agent.new(url: 'https://twitter.com/Milano_JS')
     agent.page
@@ -53,7 +54,7 @@ The agent is responsible for scraping a web page from a URL. Here is how you can
 2. another way is to pass a `page` option (`Mechanize::Page`)
 
     ```ruby
-    require 'sagrone_scraper/agent'
+    require 'sagrone_scraper'
 
     mechanize_agent = Mechanize.new { |agent| agent.user_agent_alias = 'Linux Firefox' }
     page = mechanize_agent.get('https://twitter.com/Milano_JS')
@@ -74,11 +75,16 @@ The _parser_ is responsible for extracting structured data from a _page_. The pa
 Example usage:
 
 ```ruby
-require 'sagrone_scraper/agent'
-require 'sagrone_scraper/parser'
+require 'sagrone_scraper'
 
 # 1) First define a custom parser, for example twitter.
 class TwitterParser < SagroneScraper::Parser
+  TWITTER_PROFILE_URL = /^https?:\/\/twitter.com\/(\w)+\/?$/i
+
+  def self.can_parse?(url)
+    url.match(TWITTER_PROFILE_URL)
+  end
+
   def bio
     page.at('.ProfileHeaderCard-bio').text
   end
@@ -100,6 +106,42 @@ parser.attributes
 # => {bio: "Javascript User Group Milano #milanojs", location: "Milan, Italy"}
 ```
 
+#### `SagroneScraper.scrape`
+
+This is the simplest way to scrape a web page:
+
+```ruby
+require 'sagrone_scraper'
+
+# 1) First we define a custom parser, for example twitter.
+class TwitterParser < SagroneScraper::Parser
+  TWITTER_PROFILE_URL = /^https?:\/\/twitter.com\/(\w)+\/?$/i
+
+  def self.can_parse?(url)
+    url.match(TWITTER_PROFILE_URL)
+  end
+
+  def bio
+    page.at('.ProfileHeaderCard-bio').text
+  end
+
+  def location
+    page.at('.ProfileHeaderCard-locationText').text
+  end
+end
+
+# 2) We register the parser.
+SagroneScraper.register_parser('TwitterParser')
+
+# 3) We can query for registered parsers.
+SagroneScraper.registered_parsers
+# => ['TwitterParser']
+
+# 4) We can now scrape twitter profile URLs.
+SagroneScraper.scrape(url: 'https://twitter.com/Milano_JS')
+# => {bio: "Javascript User Group Milano #milanojs", location: "Milan, Italy"}
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/sagrone_scraper/fork )

data/lib/sagrone_scraper.rb

@@ -1,7 +1,41 @@
 require "sagrone_scraper/version"
+require "sagrone_scraper/agent"
+require "sagrone_scraper/parser"
 
 module SagroneScraper
+  Error = Class.new(RuntimeError)
+
   def self.version
     VERSION
   end
+
+  def self.registered_parsers
+    @registered_parsers ||= []
+  end
+
+  def self.register_parser(name)
+    return if registered_parsers.include?(name)
+
+    parser_class = Object.const_get(name)
+    raise Error.new("Expected parser to be a SagroneScraper::Parser.") unless parser_class.ancestors.include?(SagroneScraper::Parser)
+
+    registered_parsers.push(name)
+  end
+
+  def self.scrape(options)
+    url = options.fetch(:url) do
+            raise Error.new('Option "url" must be provided.')
+          end
+
+    parser_class = registered_parsers
+                    .map { |parser_name| Object.const_get(parser_name) }
+                    .find { |parser_class| parser_class.can_parse?(url) }
+
+    raise Error.new("No registed parser can parse URL #{url}") unless parser_class
+
+    agent = SagroneScraper::Agent.new(url: url)
+    parser = parser_class.new(page: agent.page)
+    parser.parse_page!
+    parser.attributes
+  end
 end

data/lib/sagrone_scraper/parser.rb

@@ -4,22 +4,30 @@ module SagroneScraper
   class Parser
     Error = Class.new(RuntimeError)
 
-    attr_reader :page, :attributes
+    attr_reader :page, :page_url, :attributes
 
     def initialize(options = {})
       @page = options.fetch(:page) do
                 raise Error.new('Option "page" must be provided.')
               end
+      @page_url = @page.uri.to_s
       @attributes = {}
     end
 
     def parse_page!
+      return unless self.class.can_parse?(page_url)
+
       self.class.method_names.each do |name|
         attributes[name] = send(name)
       end
       nil
     end
 
+    def self.can_parse?(url)
+      class_with_method = "#{self}.can_parse?(url)"
+      raise NotImplementedError.new("Expected #{class_with_method} to be implemented.")
+    end
+
     private
 
     def self.method_names

data/lib/sagrone_scraper/version.rb

@@ -1,3 +1,3 @@
 module SagroneScraper
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/sagrone_scraper/agent_spec.rb

@@ -27,8 +27,10 @@ RSpec.describe SagroneScraper::Agent do
       end
 
       it 'when options is empty' do
-        expect { described_class.new }.to raise_error(SagroneScraper::Agent::Error,
-                                                      /Exactly one option must be provided: "url" or "page"/)
+        expect {
+          described_class.new
+        }.to raise_error(SagroneScraper::Agent::Error,
+                          'Exactly one option must be provided: "url" or "page"')
       end
 
       it 'when both options are present' do
@@ -37,7 +39,7 @@ RSpec.describe SagroneScraper::Agent do
         expect {
           described_class.new(url: 'http://example.com', page: page)
         }.to raise_error(SagroneScraper::Agent::Error,
-                          /Exactly one option must be provided: "url" or "page"/)
+                          'Exactly one option must be provided: "url" or "page"')
       end
     end
 
@@ -61,7 +63,7 @@ RSpec.describe SagroneScraper::Agent do
         @invalid_url = 'not-a-url'
 
         expect { agent }.to raise_error(SagroneScraper::Agent::Error,
-                                        /absolute URL needed \(not not-a-url\)/)
+                                        'absolute URL needed (not not-a-url)')
       end
 
       it 'should require absolute path' do

data/spec/sagrone_scraper/parser_spec.rb

@@ -4,7 +4,9 @@ require 'sagrone_scraper/parser'
 RSpec.describe SagroneScraper::Parser do
   describe '#initialize' do
     it 'requires a "page" option' do
-      expect { described_class.new }.to raise_error(SagroneScraper::Parser::Error, /Option "page" must be provided./)
+      expect {
+        described_class.new
+      }.to raise_error(SagroneScraper::Parser::Error, 'Option "page" must be provided.')
     end
   end
 
@@ -16,8 +18,17 @@ RSpec.describe SagroneScraper::Parser do
       it { expect(parser.page).to be_a(Mechanize::Page) }
     end
 
+    describe '#page_url' do
+      it { expect(parser.page_url).to be }
+      it { expect(parser.page_url).to eq page.uri.to_s }
+    end
+
     describe '#parse_page!' do
-      it { expect(parser.parse_page!).to eq nil }
+      it do
+        expect {
+          parser.parse_page!
+        }.to raise_error(NotImplementedError, "Expected #{described_class}.can_parse?(url) to be implemented.")
+      end
     end
 
     describe '#attributes' do
@@ -25,17 +36,17 @@ RSpec.describe SagroneScraper::Parser do
     end
   end
 
-  describe 'create custom TwitterParser from SagroneScraper::Parser' do
-    class TwitterParser < SagroneScraper::Parser
-      def bio
-        page.at('.ProfileHeaderCard-bio').text
-      end
-
-      def location
-        page.at('.ProfileHeaderCard-locationText').text
+  describe 'class methods' do
+    describe '.can_parse?(url)' do
+      it do
+        expect {
+          described_class.can_parse?('url')
+        }.to raise_error(NotImplementedError, "Expected #{described_class}.can_parse?(url) to be implemented.")
       end
     end
+  end
 
+  describe 'create custom TwitterParser from SagroneScraper::Parser' do
     before do
       stub_request_for('https://twitter.com/Milano_JS', 'twitter.com:Milano_JS')
     end

data/spec/sagrone_scraper_spec.rb

@@ -5,4 +5,77 @@ RSpec.describe SagroneScraper do
   describe '.version' do
     it { expect(SagroneScraper.version).to be_a(String) }
   end
+
+  context 'parsers registered' do
+    before do
+      described_class.registered_parsers.clear
+    end
+
+    describe '.registered_parsers' do
+      it { expect(described_class.registered_parsers).to be_empty }
+      it { expect(described_class.registered_parsers).to be_a(Array) }
+    end
+
+    describe '.register_parser(name)' do
+      TestParser = Class.new(SagroneScraper::Parser)
+      NotParser = Class.new
+
+      it 'should check parser name is an existing constant' do
+        expect {
+          described_class.register_parser('Unknown')
+        }.to raise_error(NameError, 'uninitialized constant Unknown')
+      end
+
+      it 'should check parser class inherits from SagroneScraper::Parser' do
+        expect {
+          described_class.register_parser('NotParser')
+        }.to raise_error(SagroneScraper::Error, 'Expected parser to be a SagroneScraper::Parser.')
+      end
+
+      it 'after adding a "parser" should have it registered' do
+        described_class.register_parser('TestParser')
+
+        expect(described_class.registered_parsers).to include('TestParser')
+        expect(described_class.registered_parsers.size).to eq 1
+      end
+
+      it 'adding same "parser" multiple times should register it once' do
+        described_class.register_parser('TestParser')
+        described_class.register_parser('TestParser')
+
+        expect(described_class.registered_parsers).to include('TestParser')
+        expect(described_class.registered_parsers.size).to eq 1
+      end
+    end
+  end
+
+  describe '.scrape' do
+    before do
+      SagroneScraper.registered_parsers.clear
+      SagroneScraper.register_parser('TwitterParser')
+
+      stub_request_for('https://twitter.com/Milano_JS', 'twitter.com:Milano_JS')
+    end
+
+    it 'should `url` option' do
+      expect {
+        described_class.scrape({})
+      }.to raise_error(SagroneScraper::Error, 'Option "url" must be provided.')
+    end
+
+    it 'should scrape URL if registered parser knows how to parse it' do
+      expected_attributes = {
+        bio: "Javascript User Group Milano #milanojs",
+        location: "Milan, Italy"
+      }
+
+      expect(described_class.scrape(url: 'https://twitter.com/Milano_JS')).to eq(expected_attributes)
+    end
+
+    it 'should return raise error if no registered paser can parse the URL' do
+      expect {
+        described_class.scrape(url: 'https://twitter.com/Milano_JS/media')
+      }.to raise_error(SagroneScraper::Error, "No registed parser can parse URL https://twitter.com/Milano_JS/media")
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,7 @@
 require 'stub_helper'
 
+Dir["./spec/support/**/*.rb"].sort.each { |file| require file }
+
 RSpec.configure do |config|
   config.include(StubHelper)
 

data/spec/stub_helper.rb

@@ -17,6 +17,6 @@ module StubHelper
   end
 
   def get_response_file(name)
-    IO.read(File.join('spec/test_responses', "#{name}"))
+    IO.read(File.join('spec/support/test_responses', "#{name}"))
   end
 end

data/spec/support/test_parsers/twitter_parser.rb

@@ -0,0 +1,17 @@
+require 'sagrone_scraper/parser'
+
+class TwitterParser < SagroneScraper::Parser
+  TWITTER_PROFILE_URL = /^https?:\/\/twitter.com\/(\w)+\/?$/i
+
+  def self.can_parse?(url)
+    url.match(TWITTER_PROFILE_URL)
+  end
+
+  def bio
+    page.at('.ProfileHeaderCard-bio').text
+  end
+
+  def location
+    page.at('.ProfileHeaderCard-locationText').text
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sagrone_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Marius Colacioiu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-06 00:00:00.000000000 Z
+date: 2015-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -121,8 +121,9 @@ files:
 - spec/sagrone_scraper_spec.rb
 - spec/spec_helper.rb
 - spec/stub_helper.rb
-- spec/test_responses/twitter.com:Milano_JS
-- spec/test_responses/www.example.com
+- spec/support/test_parsers/twitter_parser.rb
+- spec/support/test_responses/twitter.com:Milano_JS
+- spec/support/test_responses/www.example.com
 homepage: ''
 licenses:
 - Apache License 2.0
@@ -153,5 +154,6 @@ test_files:
 - spec/sagrone_scraper_spec.rb
 - spec/spec_helper.rb
 - spec/stub_helper.rb
-- spec/test_responses/twitter.com:Milano_JS
-- spec/test_responses/www.example.com
+- spec/support/test_parsers/twitter_parser.rb
+- spec/support/test_responses/twitter.com:Milano_JS
+- spec/support/test_responses/www.example.com