url_processor 0.5.5 → 0.5.6

data/lib/url_processor/base.rb

@@ -1,4 +1,22 @@
 module UrlProcessor
+  class Cache
+    def initialize
+      @memory = {}
+    end
+
+    def get(request)
+      @memory[request]
+    end
+
+    def set(request, response)
+      @memory[request] = response
+    end
+
+    def empty!
+      @memory.clear
+    end
+  end
+
   class Base
     attr_reader :config
 
@@ -78,6 +96,11 @@ module UrlProcessor
 
     def run
       processed_links = 0
+      
+      # use an in-memory cache of responses (per run)
+      cache = Cache.new
+      Typhoeus::Config.cache = cache
+      
       hydra = Typhoeus::Hydra.new(max_concurrency: config.max_concurrency, max_total_connections: config.max_total_connections)
 
       find_in_batches(config.links.call, config.batch_size) do |group|
@@ -135,6 +158,8 @@ module UrlProcessor
 
         hydra.run
       end
+
+      cache.empty!
     end
   end
 end

data/lib/url_processor/version.rb

@@ -1,3 +1,3 @@
 module UrlProcessor
-  VERSION = "0.5.5"
+  VERSION = "0.5.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: url_processor
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-07 00:00:00.000000000 Z
+date: 2014-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -186,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4358104254366073777
+      hash: 2447107549837569791
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -195,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4358104254366073777
+      hash: 2447107549837569791
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23.2