scraped 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7f71471b758c81074f1ed52e7d06ee9e2ee7df49
+  data.tar.gz: a61a2f95fcf2a889aa077fae49f38b387e08accf
+SHA512:
+  metadata.gz: c7d4c5948c39db02b97723fd0dec0b916e395526f1eadb62f455d0ab8875281c5c0111ec791fc3924c5605a37dfbc2cd5f635ba91e3200c62213bf648a0170d9
+  data.tar.gz: 78f1da053d76b752da56cc3da2d4f341e65ca5cf047bebe40f802fe2e1744d0fb86eaccca1177f5bdd756b6b77b934a4368874272f644c1c6bfad9450944a2d1

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.rubocop-https---raw-githubusercontent-com-everypolitician-everypolitician-data-master--rubocop-base-yml

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  TargetRubyVersion: 2.1
+
+inherit_from:
+  - https://raw.githubusercontent.com/everypolitician/everypolitician-data/master/.rubocop_base.yml
+
+Style/SingleLineBlockParams:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+before_install: gem install bundler -v 1.13.5

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## 0.1.0 - 2017-01-04
+
+### Added
+
+- Support for creating HTML scrapers.
+- Scraper classes can handle sections of a page.
+- Custom request logic via request strategies. This could be used to fetch
+  responses from an archive or a local cache.
+- Custom response decorators for altering the response status, headers and body
+  before it gets to the scraper class.
+- Built-in response decorator for making link and image urls absolute.
+- `String#tidy` method which cleans up various space characters and then strips
+  leading and trailing whitespace.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scraped.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 UK Citizens Online Democracy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,204 @@
+# Scraped
+
+Write declarative scrapers in Ruby
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'scraped'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scraped
+
+## Usage
+
+To write a standard HTML scraper, start by creating a subclass of
+`Scraped::HTML` for each _type_ of page you wish to scrape.
+
+For example if you were scraping a list of people you might have a
+`PeopleListPage` class for the list page and a `PersonPage` class for an
+individual person's page.
+
+```ruby
+require 'scraped'
+
+class ExamplePage < Scraped::HTML
+  field :title do
+    noko.at_css('h1').text
+  end
+
+  field :more_information do
+    noko.at_css('a')[:href]
+  end
+end
+```
+
+Then you can create a new instance and pass in a `Scraped::Response` instance.
+
+```ruby
+page = ExamplePage.new(response: Scraped::Request.new(url: 'http://example.com').response)
+
+page.title
+# => "Example Domain"
+
+page.more_information
+# => "http://www.iana.org/domains/reserved"
+
+page.to_h
+# => { :title => "Example Domain", :more_information => "http://www.iana.org/domains/reserved" }
+```
+
+### Dealing with sections of a page
+
+When writing an HTML scraper you'll often need to deal with just a part of the page.
+For example you might want to scrape a table containing a list of people and some
+associated data.
+
+To do this you can use the `fragment` method, passing it a hash with one entry
+where the key is the `noko` fragment you want to use and the value is the class
+that should handle that fragment.
+
+```ruby
+class MemberRow < Scraped::HTML
+  field :name do
+    noko.css('td')[2].text
+  end
+
+  field :party do
+    noko.css('td')[3].text
+  end
+end
+
+class AllMembersPage < Scraped::HTML
+  field :members do
+    noko.css('table.members-list tr').map do |row|
+      fragment row => MemberRow
+    end
+  end
+end
+```
+
+## Extending
+
+There are two main ways to extend `scraped` with your own custom logic - custom requests and decorated responses. Custom requests allow you to change where the scraper is getting its responses from, e.g. you might want to make requests to archive.org if the site you're scraping has disappeared. Decorated responses allow you to manipulate the response before it's passed to the scraper. Scraped comes with some [built in decorators](#built-in-decorators) for common tasks such as making all the link urls on the page absolute rather than relative.
+
+### Custom request strategies
+
+To make a custom request you'll need to create a class that subclasses `Scraped::Request::Strategy` and defines a `response` method.
+
+```ruby
+class FileOnDiskRequest < Scraped::Request::Strategy
+  def response
+    { body: open(filename).read }
+  end
+
+  private
+
+  def filename
+    @filename ||= File.join(URI.parse(url).host, Digest::SHA1.hexdigest(url))
+  end
+end
+```
+
+The `response` method should return a `Hash` which has at least a `body` key. You can also include `status` and `headers` parameters in the hash to fill out those fields in the response. If not given, status will default to `200` (OK) and headers will default to `{}`.
+
+To use a custom request strategy pass it to `Scraped::Request`:
+
+```ruby
+request = Scraped::Request.new(url: 'http://example.com', strategies: [FileOnDiskRequest, Scraped::Request::Strategy::LiveRequest])
+page = MyPersonPage.new(response: request.response)
+```
+
+### Decorated responses
+
+To manipulate the response before it is processed by the scraper create a class that subclasses `Scraped::Response::Decorator` and defines any of the following methods: `body`, `url`, `status`, `headers`.
+
+```ruby
+class AbsoluteLinks < Scraped::Response::Decorator
+  def body
+    doc = Nokogiri::HTML(super)
+    doc.css('a').each do |link|
+      link[:href] = URI.join(url, link[:href]).to_s
+    end
+    doc.to_s
+  end
+end
+```
+
+As well as the `body` method you can also supply your own `url`, `status` and `headers` methods. You can access the current request body by calling `super` from your method. You can also call `url`, `headers` or `status` to access those properties of the current response.
+
+To use a response decorator you need to use the `decorator` class method in a `Scraped::HTML` subclass:
+
+```ruby
+class PageWithRelativeLinks < Scraped::HTML
+  decorator AbsoluteLinks
+
+  # Other fields...
+end
+```
+
+### Configuring requests and responses
+
+When passing an array of request strategies or response decorators you should always pass the class, rather than the instance. If you want to configure an instance you can pass in a two element array where the first element is the class and the second element is the config:
+
+```ruby
+class CustomHeader < Scraped::Response::Decorator
+  def headers
+    response.headers.merge('X-Greeting' => config[:greeting])
+  end
+end
+
+class ExamplePage < Scraped::HTML
+  decorator CustomHeader, greeting: 'Hello, world'
+end
+```
+
+With the above code a custom header would be added to the response: `X-Greeting: Hello, world`.
+
+#### Inheritance with decorators
+
+When you inherit from a class that already has decorators the child class will also inherit the parent's decorators. There's currently no way to re-order or remove decorators in child classes, though that _may_ be added in the future.
+
+### Built in decorators
+
+#### Absolute link and image urls
+
+Very frequently you will find that you need to make links and images on the page
+you are scraping absolute rather than relative. Scraped comes with support for
+this out of the box via the `Scraped::Response::Decorator::AbsoluteUrls`
+decorator.
+
+```ruby
+require 'scraped'
+
+class MemberPage < Scraped::HTML
+  decorator Scraped::Response::Decorator::AbsoluteUrls
+
+  field :image do
+    # Image url will be absolute thanks to the decorator.
+    noko.at_css('.profile-picture/@src').text
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/everypolitician/scraped.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+task default: %i(test rubocop)

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'scraped'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/scraped.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require 'nokogiri'
+require 'field_serializer'
+require 'require_all'
+require_rel 'scraped'
+
+# Abstract class which scrapers can extend to implement their functionality.
+class Scraped
+  include FieldSerializer
+
+  def self.decorator(klass, config = {})
+    decorators << config.merge(decorator: klass)
+  end
+
+  def self.decorators
+    @decorators ||= []
+  end
+
+  def self.inherited(klass)
+    klass.decorators.concat(decorators)
+    super
+  end
+
+  def initialize(response:)
+    @original_response = response
+  end
+
+  private
+
+  attr_reader :original_response
+
+  def response
+    @response ||= ResponseDecorator.new(
+      response:   original_response,
+      decorators: self.class.decorators
+    ).response
+  end
+
+  def url
+    response.url
+  end
+end

data/lib/scraped/core_ext.rb

@@ -0,0 +1,5 @@
+class String
+  def tidy
+    gsub(/[[:space:]]+/, ' ').strip
+  end
+end

data/lib/scraped/html.rb

@@ -0,0 +1,19 @@
+class Scraped
+  class HTML < Scraped
+    private
+
+    def initialize(noko: nil, **args)
+      super(**args)
+      @noko = noko
+    end
+
+    def noko
+      @noko ||= Nokogiri::HTML(response.body)
+    end
+
+    def fragment(mapping)
+      noko_fragment, klass = mapping.to_a.first
+      klass.new(noko: noko_fragment, response: response)
+    end
+  end
+end

data/lib/scraped/request.rb

@@ -0,0 +1,32 @@
+require 'scraped/request/strategy/live_request'
+require 'scraped/response'
+
+class Scraped
+  class Request
+    def initialize(url:, strategies: [Strategy::LiveRequest])
+      @url = url
+      @strategies = strategies
+    end
+
+    def response(decorators: [])
+      abort "Failed to fetch #{url}" if first_successful_response.nil?
+      response = Response.new(first_successful_response.merge(url: url))
+      ResponseDecorator.new(response: response, decorators: decorators).response
+    end
+
+    private
+
+    attr_reader :url, :strategies
+
+    def first_successful_response
+      @first_successful_response ||=
+        strategies.lazy.map do |strategy_config|
+          unless strategy_config.respond_to?(:delete)
+            strategy_config = { strategy: strategy_config }
+          end
+          strategy_class = strategy_config.delete(:strategy)
+          strategy_class.new(url: url, config: strategy_config).response
+        end.reject(&:nil?).first
+    end
+  end
+end

data/lib/scraped/request/strategy.rb

@@ -0,0 +1,20 @@
+class Scraped
+  class Request
+    class Strategy
+      class NotImplementedError < StandardError; end
+
+      def initialize(url:, config: {})
+        @url = url
+        @config = config.to_h
+      end
+
+      def response
+        raise NotImplementedError, "No #{self.class}#response method found"
+      end
+
+      private
+
+      attr_reader :url, :config
+    end
+  end
+end

data/lib/scraped/request/strategy/live_request.rb

@@ -0,0 +1,26 @@
+require 'scraped/request/strategy'
+require 'open-uri'
+
+class Scraped
+  class Request
+    class Strategy
+      class LiveRequest < Strategy
+        def response
+          log "Fetching #{url}"
+          response = open(url)
+          {
+            status:  response.status.first.to_i,
+            headers: response.meta,
+            body:    response.read,
+          }
+        end
+
+        private
+
+        def log(message)
+          warn "[#{self.class}] #{message}" if ENV.key?('VERBOSE')
+        end
+      end
+    end
+  end
+end

data/lib/scraped/response.rb

@@ -0,0 +1,12 @@
+class Scraped
+  class Response
+    attr_reader :status, :headers, :body, :url
+
+    def initialize(body:, url:, status: 200, headers: {})
+      @status = status
+      @headers = headers
+      @body = body
+      @url = url
+    end
+  end
+end

data/lib/scraped/response/decorator.rb

@@ -0,0 +1,34 @@
+class Scraped
+  class Response
+    class Decorator
+      def initialize(response:, config: {})
+        @response = response
+        @config = config.to_h
+      end
+
+      def decorated_response
+        Response.new(url: url, body: body, headers: headers, status: status)
+      end
+
+      def url
+        response.url
+      end
+
+      def body
+        response.body
+      end
+
+      def headers
+        response.headers
+      end
+
+      def status
+        response.status
+      end
+
+      private
+
+      attr_reader :response, :config
+    end
+  end
+end

data/lib/scraped/response/decorator/absolute_urls.rb

@@ -0,0 +1,25 @@
+require 'nokogiri'
+require 'uri'
+
+class Scraped
+  class Response
+    class Decorator
+      class AbsoluteUrls < Decorator
+        def body
+          Nokogiri::HTML(super).tap do |doc|
+            doc.css('img').each { |img| img[:src] = absolute_url(img[:src]) }
+            doc.css('a').each { |a| a[:href] = absolute_url(a[:href]) }
+          end.to_s
+        end
+
+        private
+
+        def absolute_url(relative_url)
+          URI.join(url, relative_url) unless relative_url.to_s.empty?
+        rescue URI::InvalidURIError
+          relative_url
+        end
+      end
+    end
+  end
+end

data/lib/scraped/response_decorator.rb

@@ -0,0 +1,23 @@
+class Scraped
+  class ResponseDecorator
+    def initialize(response:, decorators:)
+      @original_response = response
+      @decorators = decorators.to_a
+    end
+
+    def response
+      decorators.reduce(original_response) do |r, decorator_config|
+        unless decorator_config.respond_to?(:[])
+          decorator_config = { decorator: decorator_config }
+        end
+        decorator_class = decorator_config[:decorator]
+        decorator_class.new(response: r, config: decorator_config)
+                       .decorated_response
+      end
+    end
+
+    private
+
+    attr_reader :original_response, :decorators
+  end
+end

data/lib/scraped/version.rb

@@ -0,0 +1,3 @@
+class Scraped
+  VERSION = '0.1.0'.freeze
+end

data/scraped.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scraped/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'scraped'
+  spec.version       = Scraped::VERSION
+  spec.authors       = ['EveryPolitician']
+  spec.email         = ['team@everypolitician.org']
+
+  spec.summary       = 'Write declarative scrapers in Ruby'
+  spec.homepage      = 'https://github.com/everypolitician/scraped'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'field_serializer', '>= 0.3.0'
+  spec.add_runtime_dependency 'require_all'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'pry', '~> 0.10'
+  spec.add_development_dependency 'rubocop', '~> 0.44'
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: scraped
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- EveryPolitician
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-01-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: field_serializer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.44'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.44'
+description: 
+email:
+- team@everypolitician.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/scraped.rb
+- lib/scraped/core_ext.rb
+- lib/scraped/html.rb
+- lib/scraped/request.rb
+- lib/scraped/request/strategy.rb
+- lib/scraped/request/strategy/live_request.rb
+- lib/scraped/response.rb
+- lib/scraped/response/decorator.rb
+- lib/scraped/response/decorator/absolute_urls.rb
+- lib/scraped/response_decorator.rb
+- lib/scraped/version.rb
+- scraped.gemspec
+homepage: https://github.com/everypolitician/scraped
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Write declarative scrapers in Ruby
+test_files: []