sagrone_scraper 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d067420377ca0e271b6ba7f8c00f5f6ae2198b85
-  data.tar.gz: cc93b626d827b17e7f16fa91b7fd00b13936c318
+  metadata.gz: 82fc9ba674d9d3398b5f596d513fbb8eeb8abe3b
+  data.tar.gz: a8acac43dc318b6dcad951d57d3e1ce59478c67d
 SHA512:
-  metadata.gz: 804b9c719e81d87b762f1cea45c3e1919d459d7520270ac1176907e3cb14efef3f992f24f2ed71db3baa85cae7e0fb3b4c5f18394785da396df772a6eeb59755
-  data.tar.gz: a9b2524b7029896731942e13483e736b45115739d8cc22a7176e9afc477f6b3d460fbe8b9b88c577ae706a976ad10c30d49deba45cdfb9d65206085dd4459f3d
+  metadata.gz: 0dc5041f027f685ac241fcf0e103d3d8a6fa225002cdedd65a2f072fafb43904d3514a7cabad9a83bb0d3f02a3f7241c2c8dbeffdb48f133439f308319462870
+  data.tar.gz: 15644f0da27c4cb3f2452ac11958261acc7d0bd5e3e0539a7d7817b47f5c5096ea3f0d107b2f456f34f6fe8273fb7742128cd72f5c142c8416a521262ffb8e09

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ### HEAD
 
+### 0.0.2
+
+- add `SagroneScraper::Parser`
+
 ### 0.0.1
 
 - add `SagroneScraper::Agent`

data/README.md

@@ -1,7 +1,18 @@
-# SagroneScraper
+# Sagrone scraper
+
+[![Gem Version](https://badge.fury.io/rb/sagrone_scraper.svg)](http://badge.fury.io/rb/sagrone_scraper)
+[![Build Status](https://travis-ci.org/Sagrone/scraper.svg?branch=master)](https://travis-ci.org/Sagrone/scraper)
 
 Simple library to scrap web pages. Bellow you will find information on [how to use it](#usage).
 
+## Table of Contents
+
+- [Installation](#installation)
+- [Basic Usage](#basic-usage)
+- [Modules](#modules)
+  + [`SagroneScraper::Agent`](#sagronescraperagent)
+  + [`SagroneScraper::Parser`](#sagronescraperparser)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -16,13 +27,15 @@ Or install it yourself as:
 
     $ gem install sagrone_scraper
 
-## Usage
+## Basic Usage
 
-#### `SagroneScraper::Agent`
+Comming soon...
 
-The agent is responsible for scraping a web page from a URL.
+## Modules
 
-Here is how you can create an `agent`:
+#### `SagroneScraper::Agent`
+
+The agent is responsible for scraping a web page from a URL. Here is how you can create an `agent`:
 
 1. one way is to pass it a `url` option
 
@@ -54,6 +67,38 @@ Here is how you can create an `agent`:
     # => "Milan, Italy"
     ```
 
+#### `SagroneScraper::Parser`
+
+The _parser_ is responsible for extracting structured data from a _page_. The page can be obtained by the _agent_.
+
+Example usage:
+
+```ruby
+require 'sagrone_scraper/agent'
+require 'sagrone_scraper/parser'
+
+# 1) First define a custom parser, for example twitter.
+class TwitterParser < SagroneScraper::Parser
+  def bio
+    page.at('.ProfileHeaderCard-bio').text
+  end
+
+  def location
+    page.at('.ProfileHeaderCard-locationText').text
+  end
+end
+
+# 2) Create an agent scraper, which will give us the page to parse.
+agent = SagroneScraper::Agent.new(url: 'https://twitter.com/Milano_JS')
+
+# 3) Instantiate the parser.
+parser = TwitterParser.new(page: agent.page)
+
+# 4) Parse page and extract attributes.
+parser.parse_page!
+parser.attributes
+# => {bio: "Javascript User Group Milano #milanojs", location: "Milan, Italy"}
+```
 
 ## Contributing
 

data/lib/sagrone_scraper/parser.rb

@@ -0,0 +1,34 @@
+require 'mechanize'
+
+module SagroneScraper
+  class Parser
+    Error = Class.new(RuntimeError)
+
+    attr_reader :page, :attributes
+
+    def initialize(options = {})
+      @page = options.fetch(:page) do
+                raise Error.new('Option "page" must be provided.')
+              end
+      @attributes = {}
+    end
+
+    def parse_page!
+      self.class.method_names.each do |name|
+        attributes[name] = send(name)
+      end
+      nil
+    end
+
+    private
+
+    def self.method_names
+      @method_names ||= []
+    end
+
+    def self.method_added(name)
+      puts "added #{name} to #{self}"
+      method_names.push(name)
+    end
+  end
+end

data/lib/sagrone_scraper/version.rb

@@ -1,3 +1,3 @@
 module SagroneScraper
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/sagrone_scraper/agent_spec.rb

@@ -78,7 +78,7 @@ RSpec.describe SagroneScraper::Agent do
 
         webmock_allow do
           expect { agent }.to raise_error(SagroneScraper::Agent::Error,
-                                          /getaddrinfo: nodename nor servname provided, or not known/)
+                                          /getaddrinfo/)
         end
       end
     end

data/spec/sagrone_scraper/parser_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+require 'sagrone_scraper/parser'
+
+RSpec.describe SagroneScraper::Parser do
+  describe '#initialize' do
+    it 'requires a "page" option' do
+      expect { described_class.new }.to raise_error(SagroneScraper::Parser::Error, /Option "page" must be provided./)
+    end
+  end
+
+  describe 'instance methods' do
+    let(:page) { Mechanize::Page.new }
+    let(:parser) { described_class.new(page: page) }
+
+    describe '#page' do
+      it { expect(parser.page).to be_a(Mechanize::Page) }
+    end
+
+    describe '#parse_page!' do
+      it { expect(parser.parse_page!).to eq nil }
+    end
+
+    describe '#attributes' do
+      it { expect(parser.attributes).to be_empty }
+    end
+  end
+
+  describe 'create custom TwitterParser from SagroneScraper::Parser' do
+    class TwitterParser < SagroneScraper::Parser
+      def bio
+        page.at('.ProfileHeaderCard-bio').text
+      end
+
+      def location
+        page.at('.ProfileHeaderCard-locationText').text
+      end
+    end
+
+    before do
+      stub_request_for('https://twitter.com/Milano_JS', 'twitter.com:Milano_JS')
+    end
+
+    let(:page) { Mechanize.new.get('https://twitter.com/Milano_JS') }
+    let(:twitter_parser) { TwitterParser.new(page: page) }
+    let(:expected_attributes) do
+      {
+        bio: "Javascript User Group Milano #milanojs",
+        location: "Milan, Italy"
+      }
+    end
+
+    describe 'should be able to parse page without errors' do
+      it { expect { twitter_parser.parse_page! }.to_not raise_error }
+    end
+
+    it 'should have attributes present after parsing' do
+      twitter_parser.parse_page!
+
+      expect(twitter_parser.attributes).to_not be_empty
+      expect(twitter_parser.attributes).to eq expected_attributes
+    end
+
+    it 'should have correct attributes event if parsing is done multiple times' do
+      twitter_parser.parse_page!
+      twitter_parser.parse_page!
+      twitter_parser.parse_page!
+
+      expect(twitter_parser.attributes).to_not be_empty
+      expect(twitter_parser.attributes).to eq expected_attributes
+    end
+  end
+end

data/spec/sagrone_scraper_spec.rb

@@ -5,4 +5,4 @@ RSpec.describe SagroneScraper do
   describe '.version' do
     it { expect(SagroneScraper.version).to be_a(String) }
   end
-end
+end