vore 0.2.0-x86_64-darwin → 0.2.2-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c54c9d9a3d685f0545ebcec67d0aa30a3a6751d5ae31093197a021d24178040
-  data.tar.gz: d961d2d7c2bbcf3fad014e4bf1eea9f5d7a692901dcea50f66c3bdccf64066f9
+  metadata.gz: 5e3cf9efca051fe9cb002a0fbae78e520880a13d90192416875174e208a42a53
+  data.tar.gz: 3d9c3a6ca47d3d7c0b1f15bb6311d54ab7e172390e55c7fad83b76480be03a31
 SHA512:
-  metadata.gz: 1a9ae3ef5f6b227c86cb21ecef0377ab7582e96ca562505e83b127333c7cad7579c4e5b1525793cd54a4d97a42c626646fcfae4146b547e00373db339f4b9615
-  data.tar.gz: ec1173ad2572e8bf07760360941493045d9e1d39912651cab8403be4418da6d39df42e9bdcd3d2f8d61ddf576324fbb73de7d252ca301a75f83920e77a89de79
+  metadata.gz: 016c40c00eb7914370c2bfaeea8fffcd99c5449a6b786d6c5e97802e7ee504c4ebef4c08758dc13fb5b27cb7ecf57d71b91fe30f9bdc150929973a30135cb45a
+  data.tar.gz: edefee484a4de8340d2e9171c5b5e7a8d1e433daf4920bca860df892c3471f45a91e9e2b94ca000200a32ad98d0ef5e7ce683d19b6f77a9f27ea8e335f9f541e

data/lib/vore/crawler.rb

@@ -17,7 +17,7 @@ module Vore
       @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
-      @output_dir = "tmp/vore"
+      @parent_output_dir = "tmp/vore"
 
       return if File.exist?(@executable)
 
@@ -26,11 +26,12 @@ module Vore
     end
 
     def scrape_each_page(website, &block)
-      output_dir = "#{@output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
       output = %x(#{@executable} \
         --user-agent #{user_agent} \
+        --delay 3000
         --url #{website} \
         download \
         -t \
@@ -38,12 +39,25 @@ module Vore
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
-      Dir.glob("tmp/**/*").each do |path|
+      results = {
+        pages_visited: 0,
+        pages_unprocessed: 0,
+        unprocessed_pages: [],
+      }
+
+      Dir.glob(File.join(output_dir, "**", "*")).each do |path|
         next unless File.file?(path)
 
         html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
+        results[:pages_visited] += 1
+        if rewritten_html_file.empty?
+          results[:pages_unprocessed] += 1
+          results[:unprocessed_pages] << path
+          next
+        end
+
         # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
         url_path = path.split(FILE_SEPERATOR)[3..].join("/")
 
@@ -58,6 +72,8 @@ module Vore
       ensure
         File.delete(path) if File.file?(path)
       end
+
+      results
     end
 
     # def crawl(site, block)

data/lib/vore/handlers/content_extractor.rb

@@ -19,7 +19,13 @@ module Vole
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+        if element.tag_name == "pre" ||
+            element.tag_name == "code" ||
+            element.tag_name == "form" ||
+            element.tag_name == "style" ||
+            element.tag_name == "noscript" ||
+            element.tag_name == "script" ||
+            element.tag_name == "svg"
           element.remove
         elsif element.tag_name == "title"
           @within_title = true

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian