vore 0.2.6-arm64-darwin → 0.3.0-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 402d817b4979ae3cb7caee99653fc2ddfccf726b53f49c76dc078d54a9868af3
-  data.tar.gz: 1768208e84f98fbbcb5e8bb683528f6f476c928d79c1e006cb1b6a8bf521cae7
+  metadata.gz: 54a3f9525133d20d29eadd67c83ef0c996c3e85ae6843b3ea239e3cff0d9bea3
+  data.tar.gz: 065af90bf1234459fe430a2c49882d988e4b1367786dc506a7cf571804812339
 SHA512:
-  metadata.gz: '0091d5f55c923e91abfa57b01e7857072196c85e1053568b5799c715c6d188a27075a82c9393e9fdadade34ff02832c98780dcf9e1f70970f45db29a77fbc733'
-  data.tar.gz: 5915151dbdd4d055f5cf6305dada2a0a82d817a8563ddffab52582f6c580903a2f29269b81c3591c407775ab522d66e55c5a901d3afb4640aa0e959c56f88e2c
+  metadata.gz: 540ceeac482a7b9274161a5b64c7d5d0fd570dd3756934ae8a6dcaa760869b81c3f56d89b7c959f7a18c66fffe94c9b215efff8e648d5580aaeaa823e10405e3
+  data.tar.gz: f1d56bc655ac4e42e720837d00df756d497f55ceca844edc4fd7ceb887736458afb02532b90a1fe0f95a7e8b1fe3e551edd3ea196fa8d995c3bd3a5f42db7f32

data/lib/vore/configuration.rb

@@ -5,5 +5,9 @@ module Vole
     DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
       allow_doctype: false,
     })
+
+    DEFAULT_OPTIONS = {
+      delay: 3500,
+    }
   end
 end

data/lib/vore/crawler.rb

@@ -8,9 +8,11 @@ module Vore
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
     FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
+    attr_reader :output_dir
+
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
-    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
+    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG, options: Vole::Configuration::DEFAULT_OPTIONS)
       @denylist_regexp = Regexp.union(denylist)
 
       @content_extractor = Vole::Handlers::ContentExtractor.new
@@ -18,6 +20,7 @@ module Vore
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @parent_output_dir = "tmp/vore"
+      @options = options
 
       return if File.exist?(@executable)
 
@@ -26,16 +29,10 @@ module Vore
     end
 
     def scrape_each_page(website, &block)
-      output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      output = %x(#{@executable} \
-        --user-agent #{user_agent} \
-        --delay 3500 \
-        --url #{website} \
-        download \
-        -t \
-        #{output_dir})
+      output = run_command(website, delay: @options[:delay])
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
@@ -54,7 +51,6 @@ module Vore
         rewritten_html_file = ""
 
         if html_file.empty?
-          Vore.logger.warn("HTML file empty: #{path}")
           results[:pages_unprocessed] += 1
           results[:unprocessed_pages] << path
           next
@@ -91,6 +87,16 @@ module Vore
     #   crawl_site(site)
     # end
 
+    def run_command(website, delay: 3500)
+      %x(#{@executable} \
+        --user-agent #{user_agent} \
+        --delay #{delay} \
+        --url #{website} \
+        download \
+        -t \
+        #{@output_dir})
+    end
+
     def user_agent
       "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
     end

data/lib/vore/handlers/content_extractor.rb

@@ -20,7 +20,6 @@ module Vole
 
       def handle_element(element)
         if element.tag_name == "pre" ||
-            element.tag_name == "code" ||
             element.tag_name == "form" ||
             element.tag_name == "style" ||
             element.tag_name == "noscript" ||

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: arm64-darwin
 authors:
 - Garen J. Torikian