outlander 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b6951e27ebb022fbc2ec65a1b7394e84d876868
-  data.tar.gz: d77d4077c1cbdca9a6c09980c12016a9639b05d4
+  metadata.gz: 93eeac684cf867eacf0e54ee6d1967769e2a1c8e
+  data.tar.gz: 42d585420132fbdd11a665d9f89990e8ab3adfd6
 SHA512:
-  metadata.gz: 0a4ceb2e8a3e564db5ec088855428711e66a63e903f35c58ed18f127cea3eebea2fa7af673b872494dceffdbc54c7672f21dc78a8f5ff276d9df4fe5106ce2e4
-  data.tar.gz: 2da3b6e9e6939c0032004ef4fa71fea7d7ed2f65f6c20907bbfa318a7864beee8195c76821023f588669d8b1f355d8b3944c76d126584e3d1fe0f12caa010f27
+  metadata.gz: 88c97f88ec619bb42df78940234628d45cc2722b8a11257d02a441d99a0829a715bc5fbb1f56bf4f014275f07fc1b4daa8c426b02de9dce8fa5e7a03c8d72789
+  data.tar.gz: dbdfaa58aab3e29b194892204bc5b9b45dd35355af580e72d9dee2fc93a7f129ab42d451d174d929c044acc1786a5d3bade250fab5684b6aa64a1650daa180af

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.DS_Store

data/lib/outlander.rb

@@ -1,3 +1,2 @@
-require "outlander/version"
-
+require 'outlander/version'
 require 'outlander/crawler'

data/lib/outlander/agent.rb

@@ -5,37 +5,5 @@ module Outlander
   class Agent
 
     include HTTParty
-
-    class << self
-      attr_accessor :cache_storage
-    end
-
-    def self.fetch(path, options = {}, &block)
-      if cache_enabled? && options[:cache]
-        response_body = @cache_storage.get(path) || get(path, options, &block).body
-        @cache_storage.set(path, response_body)
-        response_body
-      else
-        get(path, options, &block).body
-      end
-    end
-
-    def self.method_missing(m, *args, &block)
-      if m.to_s.end_with? '_with_cache'
-        if cache_enabled?
-          response_body = @cache_storage.get(args[0]) || get(*args, &block).body
-          @cache_storage.set(args[0], response_body)
-          response_body
-        else
-          get(*args, &block).body
-        end
-      end
-    end
-
-    private
-
-    def self.cache_enabled?
-      !!@cache_storage
-    end
   end
 end

data/lib/outlander/crawler.rb

@@ -1,5 +1,5 @@
 require 'nokogiri'
-require 'logger'
+require 'thread'
 require 'outlander/agent'
 require 'outlander/threads_pool'
 
@@ -8,7 +8,8 @@ module Outlander
   module Crawler
 
     DEFAULT_OPTIONS = {
-      num_threads: 3
+      num_threads: 3,
+      pause: 1
     }
 
     class << self
@@ -24,9 +25,9 @@ module Outlander
 
     module ClassMethods
 
-      attr_reader :roots, :setup, :handlers
+      attr_reader :roots, :handlers, :setup
 
-      def entrypoint(url, handler = :process_root)
+      def entry_point(url, handler = :process_root)
         @roots[url] = handler
       end
 
@@ -44,13 +45,13 @@ module Outlander
     end
 
     def initialize(options = {})
-      agent.cache_storage = options.delete(:cache_storage)
-      @logger = Logger.new(options.fetch(:log_to, STDOUT))
-      @options = options.merge DEFAULT_OPTIONS
+      @cache = options.delete(:cache)
+      @options = DEFAULT_OPTIONS.merge options
       @history = {}
+      @mutex = Mutex.new
       @pool = ThreadsPool.new @options[:num_threads]
       self.class.roots.each do |url, handler|
-        enqueue url, handler
+        discover url, handler
       end
     end
 
@@ -62,27 +63,46 @@ module Outlander
 
     private
 
-    def record(data)
-      @result_handler.call data
+    def record(*args)
+      @result_handler.call *args
     end
 
-    def enqueue(url, handler, *args)
-      return if @history[url] == handler
+    def discover(url, handler, *args)
+      @mutex.synchronize {
+        return if @history[url] == handler
+        @history[url] = handler
+      }
       @pool.enqueue do
         begin
-          body = agent.get_with_cache(url)
+          body = fetch url do
+            sleep rand(@options[:pause])
+            @agent.get(url).body
+          end
           instance_exec Nokogiri::HTML(body), *args, &self.class.handlers[handler.to_sym]
         rescue => e
-          @logger.error "Failed to process #{url} with ##{handler} #{e.inspect}"
-        else
-          @logger.info "Processed #{url} with ##{handler}"
+          puts "[ERROR] Failed to process #{url} with ##{handler} #{e.inspect}"
         end
       end
-      @logger.info "Enqueued #{url} for ##{handler}"
     end
 
     def agent
-      @agent ||= Agent.dup
+      @agent ||= Class.new(Agent)
+    end
+
+    def fetch(url, &block)
+      if cache_enabled?
+        return @cache.get url if @cache.exists url
+
+        value = block.call
+        @cache.set url, value
+        value
+      else
+        block.call
+      end
+    end
+
+    def cache_enabled?
+      !!@cache
     end
   end
 end

data/lib/outlander/threads_pool.rb

@@ -15,7 +15,7 @@ module Outlander
 
     def start
       raise "Could not start with empty queue" if @queue.empty?
-      run_threads(@num_threads)
+      run_threads @num_threads
       sleep 1 until @queue.empty? && @queue.num_waiting == @num_threads
     end
 

data/lib/outlander/version.rb

@@ -1,3 +1,3 @@
 module Outlander
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/outlander.gemspec

@@ -24,5 +24,4 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "httparty"
   spec.add_dependency "nokogiri"
-  spec.add_dependency "dalli"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: outlander
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Terry Progetto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-08 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: dalli
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Multithreaded web crawler with transparent DSL and requests caching.
 email:
 - terryprogetto@gmail.com
@@ -87,16 +73,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".DS_Store"
 - ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
-- lib/.DS_Store
 - lib/outlander.rb
-- lib/outlander/.DS_Store
 - lib/outlander/agent.rb
 - lib/outlander/crawler.rb
 - lib/outlander/threads_pool.rb
@@ -121,7 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Web pages crawler.