vore 0.2.8-x86_64-darwin → 0.3.0-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4f96c051f27e8fd481213db0cf06e2c5bc02a61c159b820f0753796a378335a
-  data.tar.gz: 17219abae1846d6c2b48b8ceb92d3bdb7a3510c4a1d49b85eecdc1a81457c735
+  metadata.gz: 30fdc42af4a7181231ba9325a2f6803054d8d7cc801906a789df183fe106292c
+  data.tar.gz: 4d89ee0d331471992743b372cf7f64d8e30d7d166290fc8cf48b11246c41f5da
 SHA512:
-  metadata.gz: f73a3769ad439820018ced158e109871efcf74537c142240e6bb991dbd04cb32286d78393f3194f68ad4cb3f3f49b8a3962e27bc9f9cded1ec5ca5880d9d3573
-  data.tar.gz: 969e0c30ecce701f23ab32207a84599d18d986b82862cd2910069a6ec6a330ad5b62baf53c5483ae2c7ec7d11860cb3746841fd74216a4029b9dbcd4169ade69
+  metadata.gz: 8ba4d1f1cce56648d71155fd937af2e649d583169f44e07e529223166433dbb4b427d3f8b381a99c27e2958124af338b23e8f00dc6dc12ab037e0121b016d9ef
+  data.tar.gz: 9666321603fb722d77b8495e9b6b34515e29fd3636f7a420780359e778cc232dfe0dbaede172259ba523226fcde44cf93095b26268351f66f2d37da9d2acf0fb

data/lib/vore/configuration.rb

@@ -5,5 +5,9 @@ module Vole
     DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
       allow_doctype: false,
     })
+
+    DEFAULT_OPTIONS = {
+      delay: 3500,
+    }
   end
 end

data/lib/vore/crawler.rb

@@ -12,7 +12,7 @@ module Vore
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
-    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
+    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG, options: Vole::Configuration::DEFAULT_OPTIONS)
       @denylist_regexp = Regexp.union(denylist)
 
       @content_extractor = Vole::Handlers::ContentExtractor.new
@@ -20,6 +20,7 @@ module Vore
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @parent_output_dir = "tmp/vore"
+      @options = options
 
       return if File.exist?(@executable)
 
@@ -31,7 +32,7 @@ module Vore
       @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      output = run_command(website, @output_dir)
+      output = run_command(website, delay: @options[:delay])
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
@@ -86,14 +87,14 @@ module Vore
     #   crawl_site(site)
     # end
 
-    def run_command(website, output_dir)
+    def run_command(website, delay: 3500)
       %x(#{@executable} \
         --user-agent #{user_agent} \
-        --delay 3500 \
+        --delay #{delay} \
         --url #{website} \
         download \
         -t \
-        #{output_dir})
+        #{@output_dir})
     end
 
     def user_agent

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.8"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian