vore 0.2.6-x86_64-darwin → 0.2.8-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ede3f51d145b7ddb4dd0c042013f51b46d7b99268aec6d7fe013c0be027687f
-  data.tar.gz: 4bdeb20c4621e9611fec13d30c19f2c06ce71267867e5ff484b579b786ea113f
+  metadata.gz: a4f96c051f27e8fd481213db0cf06e2c5bc02a61c159b820f0753796a378335a
+  data.tar.gz: 17219abae1846d6c2b48b8ceb92d3bdb7a3510c4a1d49b85eecdc1a81457c735
 SHA512:
-  metadata.gz: 1be22361d6e06dce9c3e37494688f228f4bff053a75f974c0970601c87af6f184a2a1090a775ad61aa67eb284ee5bd14b4299991b63c030e607ee79c9470b60f
-  data.tar.gz: cb8e2cb3032554570ad0a3e94fda35deadfbe8ba81caa92d19018ca462e4e8226c27a155846a86fc701056548acff45cb160d6e49e286a4c0009ddc1df334778
+  metadata.gz: f73a3769ad439820018ced158e109871efcf74537c142240e6bb991dbd04cb32286d78393f3194f68ad4cb3f3f49b8a3962e27bc9f9cded1ec5ca5880d9d3573
+  data.tar.gz: 969e0c30ecce701f23ab32207a84599d18d986b82862cd2910069a6ec6a330ad5b62baf53c5483ae2c7ec7d11860cb3746841fd74216a4029b9dbcd4169ade69

data/lib/vore/crawler.rb

@@ -8,6 +8,8 @@ module Vore
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
     FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
+    attr_reader :output_dir
+
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
@@ -26,16 +28,10 @@ module Vore
     end
 
     def scrape_each_page(website, &block)
-      output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      output = %x(#{@executable} \
-        --user-agent #{user_agent} \
-        --delay 3500 \
-        --url #{website} \
-        download \
-        -t \
-        #{output_dir})
+      output = run_command(website, @output_dir)
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
@@ -54,7 +50,6 @@ module Vore
         rewritten_html_file = ""
 
         if html_file.empty?
-          Vore.logger.warn("HTML file empty: #{path}")
           results[:pages_unprocessed] += 1
           results[:unprocessed_pages] << path
           next
@@ -91,6 +86,16 @@ module Vore
     #   crawl_site(site)
     # end
 
+    def run_command(website, output_dir)
+      %x(#{@executable} \
+        --user-agent #{user_agent} \
+        --delay 3500 \
+        --url #{website} \
+        download \
+        -t \
+        #{output_dir})
+    end
+
     def user_agent
       "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
     end

data/lib/vore/handlers/content_extractor.rb

@@ -20,7 +20,6 @@ module Vole
 
       def handle_element(element)
         if element.tag_name == "pre" ||
-            element.tag_name == "code" ||
             element.tag_name == "form" ||
             element.tag_name == "style" ||
             element.tag_name == "noscript" ||

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.6"
+  VERSION = "0.2.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian