metainspector 4.6.1 → 4.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7cca73a64f172ba9906a1db7ba212780ffed0cdd
-  data.tar.gz: 0b2feb3de6af1562ce3fed0bfa35d3002015ed40
+  metadata.gz: 329fa61a1fb5c278adb8c07c58536cc84e774f31
+  data.tar.gz: 8c40e805a960cc5591bee0317307dad27fa46719
 SHA512:
-  metadata.gz: 5723c1dda401000aa4a54d517e4230a5f1559c9c099d0663c303896fa5ae00795eacd0dc686810dbd60ca4c64e278fbc8c0c5a1e1da6e51249ad450cd37aab46
-  data.tar.gz: c3f48b0a18962b9777d4e0ec81b80cd63b9b5be619b91a0485ce8fcd6656e76f66867aa50e78027973c47239c02bf359923cf4ce5f4224034ad1eb185364ac0f
+  metadata.gz: 0535544d927c37e963b567404764ff7af352c00afbfb896c44604e85d6e844155575d26b24a2234f478015f6f4aff9b75724d933c6cac9c3cceda6180545aa99
+  data.tar.gz: 9f7fa08ed92dcf775df62d97ab3a9d989c04ad4ff9ffe0cbaf6f83baf3c5bd052072d61e2db6b66ebef2643ca5de9de79e6aa78575a87bf432a21a3cb676a093

data/README.md

@@ -374,6 +374,15 @@ If you want to disable this, you can specify it like this:
 page = MetaInspector.new('http://example.com', download_images: false)
 ```
 
+### Caching responses
+
+MetaInspector can be configured to use [Faraday::HttpCache](https://github.com/plataformatec/faraday-http-cache) to cache page responses. For that you should pass the `faraday_http_cache` option with at least the `:store` key, for example:
+
+```ruby
+cache = ActiveSupport::Cache.lookup_store(:file_store, '/tmp/cache')
+page = MetaInspector.new('http://example.com', faraday_http_cache: { store: cache })
+```
+
 ## Exception Handling
 
 By default, MetaInspector will raise the exceptions found. We think that this is the safest default: in case the URL you're trying to scrape is unreachable, you should clearly be notified, and treat the exception as needed in your app.

data/lib/meta_inspector/document.rb

@@ -33,6 +33,7 @@ module MetaInspector
       @exception_log      = options[:exception_log] || MetaInspector::ExceptionLog.new(warn_level: warn_level)
       @normalize_url      = options[:normalize_url]
       @faraday_options    = options[:faraday_options]
+      @faraday_http_cache = options[:faraday_http_cache]
       @url                = MetaInspector::URL.new(initial_url, exception_log:      @exception_log,
                                                                 normalize:          @normalize_url)
       @request            = MetaInspector::Request.new(@url,    allow_redirections: @allow_redirections,
@@ -41,7 +42,8 @@ module MetaInspector
                                                                 retries:            @retries,
                                                                 exception_log:      @exception_log,
                                                                 headers:            @headers,
-                                                                faraday_options:    @faraday_options) unless @document
+                                                                faraday_options:    @faraday_options,
+                                                                faraday_http_cache: @faraday_http_cache) unless @document
       @parser             = MetaInspector::Parser.new(self,     exception_log:      @exception_log,
                                                                 download_images:    @download_images)
     end

data/lib/meta_inspector/request.rb

@@ -1,6 +1,7 @@
 require 'faraday'
 require 'faraday_middleware'
 require 'faraday-cookie_jar'
+require 'faraday-http-cache'
 
 module MetaInspector
 
@@ -18,6 +19,7 @@ module MetaInspector
       @exception_log      = options[:exception_log]
       @headers            = options[:headers]
       @faraday_options    = options[:faraday_options] || {}
+      @faraday_http_cache = options[:faraday_http_cache]
 
       response            # request early so we can fail early
     end
@@ -55,6 +57,11 @@ module MetaInspector
           faraday.use :cookie_jar
         end
 
+        if @faraday_http_cache.is_a?(Hash)
+          @faraday_http_cache[:serializer] ||= Marshal
+          faraday.use Faraday::HttpCache, @faraday_http_cache
+        end
+
         faraday.headers.merge!(@headers || {})
         faraday.adapter :net_http
       end

data/lib/meta_inspector/url.rb

@@ -31,18 +31,21 @@ module MetaInspector
 
     def tracked?
       u = parsed(url)
+      return false unless u.query_values
       found_tracking_params = WELL_KNOWN_TRACKING_PARAMS & u.query_values.keys
       return found_tracking_params.any?
     end
 
     def untracked_url
       u = parsed(url)
-      u.query_values = u.query_values.delete_if { |key, _| WELL_KNOWN_TRACKING_PARAMS.include? key }
+      return url unless u.query_values
+      query_values = u.query_values.delete_if { |key, _| WELL_KNOWN_TRACKING_PARAMS.include? key }
+      u.query_values = query_values.length > 0 ? query_values : nil
       u.to_s
     end
 
     def untrack!
-      self.url = untracked_url
+      self.url = untracked_url if tracked?
     end
 
     def url=(new_url)

data/lib/meta_inspector/version.rb

@@ -1,3 +1,3 @@
 module MetaInspector
-  VERSION = '4.6.1'
+  VERSION = '4.7.0'
 end

data/meta_inspector.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'faraday', '~> 0.9.0'
   gem.add_dependency 'faraday_middleware', '~> 0.10'
   gem.add_dependency 'faraday-cookie_jar', '~> 0.0.6'
+  gem.add_dependency 'faraday-http-cache', '~> 1.2.2'
   gem.add_dependency 'addressable', '~> 2.3.5'
   gem.add_dependency 'fastimage'
 

data/spec/meta_inspector/meta_inspector_spec.rb

@@ -4,4 +4,15 @@ describe MetaInspector do
   it "returns a Document" do
     expect(MetaInspector.new('http://example.com').class).to eq(MetaInspector::Document)
   end
+
+  it "cache request" do
+    # Creates a memory cache (a Hash that responds to #read, #write and #delete)
+    cache = Hash.new
+    def cache.read(k) self[k]; end
+    def cache.write(k, v) self[k] = v; end
+
+    expect(MetaInspector.new('http://example.com', warn_level: :store, faraday_http_cache: { store: cache })).to be_ok
+
+    expect(cache.keys).not_to be_empty
+  end
 end

data/spec/url_spec.rb

@@ -43,6 +43,8 @@ describe MetaInspector::URL do
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&utm_content=1234').untracked_url).to eq('http://example.com/foo?not_utm_thing=bar')
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&utm_campaign=1234').untracked_url).to eq('http://example.com/foo?not_utm_thing=bar')
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&utm_source=1234&utm_medium=5678&utm_term=4321&utm_content=9876&utm_campaign=5436').untracked_url).to eq('http://example.com/foo?not_utm_thing=bar')
+    expect(MetaInspector::URL.new('http://example.com/foo?utm_source=1234&utm_medium=5678&utm_term=4321&utm_content=9876&utm_campaign=5436').untracked_url).to eq('http://example.com/foo')
+    expect(MetaInspector::URL.new('http://example.com/foo').untracked_url).to eq('http://example.com/foo')
   end
 
   it "should remove tracking parameters from url" do
@@ -62,6 +64,18 @@ describe MetaInspector::URL do
     end
   end
 
+  it "should remove all query values when untrack url" do
+    url = MetaInspector::URL.new('http://example.com/foo?utm_campaign=1234')
+    url.untrack!
+    expect(url.url).to eq('http://example.com/foo')
+  end
+
+  it "should untrack untracked url" do
+    url = MetaInspector::URL.new('http://example.com/foo')
+    url.untrack!
+    expect(url.url).to eq('http://example.com/foo')
+  end
+
   it "should say if the url is tracked" do
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&utm_source=1234').tracked?).to be true
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&utm_medium=1234').tracked?).to be true
@@ -75,6 +89,8 @@ describe MetaInspector::URL do
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&not_utm_term=1234').tracked?).to be false
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&not_utm_content=1234').tracked?).to be false
     expect(MetaInspector::URL.new('http://example.com/foo?not_utm_thing=bar&not_utm_campaign=1234').tracked?).to be false
+
+    expect(MetaInspector::URL.new('http://example.com/foo').tracked?).to be false
   end
 
   describe "url=" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 4.6.1
+  version: 4.7.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-31 00:00:00.000000000 Z
+date: 2015-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.6
+- !ruby/object:Gem::Dependency
+  name: faraday-http-cache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -339,7 +353,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: MetaInspector is a ruby gem for web scraping purposes, that returns metadata