scraped 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e86824184d28b3141f65c48a5fc51e7502c7781b
-  data.tar.gz: d33f15a4b1e7094b2677dbf09c9573bbeb49fcff
+  metadata.gz: ffbf1466cb9e85f66f2c41dbbf9ef26f25ee568d
+  data.tar.gz: 7106cfa4f03a70452a3f53ac0b3f30de8d912162
 SHA512:
-  metadata.gz: 64f7fec521757f72ac2b927cddd5bf958d4d0fca01838ceb0e089bb40435e0a2d79ad6f4d737a0c83b4bba3cd26bed1dc252a2a0225169c1b382d4951157704a
-  data.tar.gz: 23368dd65b5773b317159343820bc9bd44525e42a38b32404616a17a3d83cb6a1fc908c2852c98607c0baf1d1f71ef5a6fc15ceaab7ce30753fbe6d74def3431
+  metadata.gz: 3193bab3627b954eb7db141fcf219eb6b92151d88d6829179a39410eff47c6b0c8aa4639c01ef799167c3d8b804f21e3711c5228bbfe26173df87dd80caa54e2
+  data.tar.gz: 12d9d2e59a564bfa823f4575c2c13471a0042e8d0fb02606ba8f142578bcdaff35481fa51ea8acf2d67e64bfc95a1cce1a89138b4c9b7b304ac370189d72fd27

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.5.0 - 2017-03-17
+
+### Added
+
+- Add `headers:` parameter to `Scraped::Request.new` for passing request
+  headers to the underlying strategy.
+
 ## 0.4.0 - 2017-03-14
 
 ### Changed

data/README.md

@@ -86,6 +86,15 @@ class AllMembersPage < Scraped::HTML
 end
 ```
 
+### Passing request headers
+
+To set request headers you can pass a `headers:` argument to `Scraped::Request.new`:
+
+```ruby
+response = Scraped::Request.new(url: 'http://example.com', headers: { 'Cookie' => 'user_id' => '42' }).response
+page = ExamplePage.new(response: response)
+```
+
 ## Extending
 
 There are two main ways to extend `scraped` with your own custom logic - custom requests and decorated responses. Custom requests allow you to change where the scraper is getting its responses from, e.g. you might want to make requests to archive.org if the site you're scraping has disappeared. Decorated responses allow you to manipulate the response before it's passed to the scraper. Scraped comes with some [built in decorators](#built-in-decorators) for common tasks such as making all the link urls on the page absolute rather than relative.

data/lib/scraped/request.rb

@@ -3,8 +3,9 @@ require 'scraped/response'
 
 module Scraped
   class Request
-    def initialize(url:, strategies: [Strategy::LiveRequest])
+    def initialize(url:, headers: {}, strategies: [Strategy::LiveRequest])
       @url = url
+      @headers = headers
       @strategies = strategies
     end
 
@@ -16,7 +17,7 @@ module Scraped
 
     private
 
-    attr_reader :url, :strategies
+    attr_reader :url, :headers, :strategies
 
     def first_successful_response
       @first_successful_response ||=
@@ -25,7 +26,7 @@ module Scraped
             strategy_config = { strategy: strategy_config }
           end
           strategy_class = strategy_config.delete(:strategy)
-          strategy_class.new(url: url, config: strategy_config).response
+          strategy_class.new(url: url, headers: headers, config: strategy_config).response
         end.reject(&:nil?).first
     end
   end

data/lib/scraped/request/strategy.rb

@@ -3,8 +3,9 @@ module Scraped
     class Strategy
       class NotImplementedError < StandardError; end
 
-      def initialize(url:, config: {})
+      def initialize(url:, headers: {}, config: {})
         @url = url
+        @headers = headers
         @config = config.to_h
       end
 
@@ -14,7 +15,7 @@ module Scraped
 
       private
 
-      attr_reader :url, :config
+      attr_reader :url, :headers, :config
     end
   end
 end

data/lib/scraped/request/strategy/live_request.rb

@@ -7,7 +7,7 @@ module Scraped
       class LiveRequest < Strategy
         def response
           log "Fetching #{url}"
-          response = open(url)
+          response = open(url, headers)
           {
             status:  response.status.first.to_i,
             headers: response.meta,

data/lib/scraped/version.rb

@@ -1,3 +1,3 @@
 module Scraped
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraped
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - EveryPolitician
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-15 00:00:00.000000000 Z
+date: 2017-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri