vore 0.2.4-x86_64-windows → 0.2.8-x86_64-windows

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 571c472b2c7e94a786883d6005c5a913e87581762f7ee90bdd915ec366cda335
-  data.tar.gz: 55cf52342b9b3b335469b7ac527f0c9bd9c13f86858591b015920106b5a7b70c
+  metadata.gz: 85a50437d0557e28ff3eeb155a8e527163aeb5f90fd98f0e5cb8e09e3d81bb6e
+  data.tar.gz: 61d1255a042db43b6e50bc749fccceb452080afd07e1eac04461ef4894fb8027
 SHA512:
-  metadata.gz: 3d8adc4b1cad88301ca10d57b654781ff4c7a66a75fb5e8cec08aa21469de51568162c22615001f60881a3a6b0efe72606ba1ca8d4846e77b2fd1527a2906eb2
-  data.tar.gz: 6db4625ac4e0d7c586c1e12b5012494eab0fb74a75a441cd38f49873e09b8d2d6b9b532d227fdd2807b7cf9f6edb153c3a4798e737b4913cd51aa3d855db4b0d
+  metadata.gz: d5c8a45adf9b4402d3600ae26728afb89b5e4d24953ad905462ad6b7c2d682a2806ed452673c486000621e5bf0373de6dd85a28ba592cb2e5eb1e86d83eb973f
+  data.tar.gz: 0077fcc1c2173c46be332da7727f6560a1cf8b2351491bc147b0d1111cefc82dc8bf8081e0f2370e2f7432d1905b2419e8bc168294dfef8a4387d609d512dae8

data/lib/vore/crawler.rb

@@ -8,6 +8,8 @@ module Vore
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
     FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
+    attr_reader :output_dir
+
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
@@ -26,16 +28,10 @@ module Vore
     end
 
     def scrape_each_page(website, &block)
-      output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      output = %x(#{@executable} \
-        --user-agent #{user_agent} \
-        --delay 3000 \
-        --url #{website} \
-        download \
-        -t \
-        #{output_dir})
+      output = run_command(website, @output_dir)
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
@@ -48,16 +44,25 @@ module Vore
       Dir.glob(File.join(output_dir, "**", "*")).each do |path|
         next unless File.file?(path)
 
+        results[:pages_visited] += 1
+
         html_file = File.read(path).force_encoding("UTF-8")
-        rewritten_html_file = @selma.rewrite(html_file)
+        rewritten_html_file = ""
 
-        results[:pages_visited] += 1
-        if rewritten_html_file.empty?
+        if html_file.empty?
           results[:pages_unprocessed] += 1
           results[:unprocessed_pages] << path
           next
         end
 
+        begin
+          rewritten_html_file = @selma.rewrite(html_file)
+        rescue StandardError => e
+          Vore.logger.warn("Error rewriting #{path}: #{e}")
+          results[:pages_unprocessed] += 1
+          next
+        end
+
         # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
         url_path = path.split(FILE_SEPERATOR)[3..].join("/")
 
@@ -81,6 +86,16 @@ module Vore
     #   crawl_site(site)
     # end
 
+    def run_command(website, output_dir)
+      %x(#{@executable} \
+        --user-agent #{user_agent} \
+        --delay 3500 \
+        --url #{website} \
+        download \
+        -t \
+        #{output_dir})
+    end
+
     def user_agent
       "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
     end

data/lib/vore/handlers/content_extractor.rb

@@ -20,7 +20,6 @@ module Vole
 
       def handle_element(element)
         if element.tag_name == "pre" ||
-            element.tag_name == "code" ||
             element.tag_name == "form" ||
             element.tag_name == "style" ||
             element.tag_name == "noscript" ||

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.4"
+  VERSION = "0.2.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.8
 platform: x86_64-windows
 authors:
 - Garen J. Torikian