vore 0.2.1-x86_64-linux → 0.2.2-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3195a81c9fa1fa0fe6fa7020a88b364b31000dd777fb02f032857506cb21b161
-  data.tar.gz: e3e1d7f7f8077c8457a511c660349df50b5ced2401590146a7b21127ed2b8435
+  metadata.gz: c27c900fcf6aa58b097af96ed54fa00f9ec6457f96039328dd5745dd0be8e5b2
+  data.tar.gz: 97e9e59e081538976e39dad93d5a70638a487bfa5ed81f321a0eb73a505879de
 SHA512:
-  metadata.gz: 3832f2a5545fc2cf537bad43c1fccb71a6e0e0acf11bd4efcd1b4d172bb49a05b2f667182b5493af62b3a5e9110507618e0b94995eb0741a99be3d8dd13386e8
-  data.tar.gz: 66b48a21c7ae26152f901af9878847600e09c9e86a352984908f76de3175ab4353e55d01815809cd5b1ebb99a6c2551df8b60795375195b67dfaaa74e7faf6af
+  metadata.gz: 3b95950fef4cf650e97430ec7514b09aa9e2ef6009e8cf2589e870fe150e703cd6b5477278cea4ab99c3ffd429d126592a0c8c9f283f6369fa58f04fab8a7ced
+  data.tar.gz: 742ca3d1ad27ede5a8a2a4bdc469e438fa2113c65b7502f3a33dc542e272b3eddac9a91b79c5e130757fbe20ee6bb9809afe83b43a008ecf6fa7e0379536ef30

data/lib/vore/crawler.rb

@@ -31,6 +31,7 @@ module Vore
 
       output = %x(#{@executable} \
         --user-agent #{user_agent} \
+        --delay 3000
         --url #{website} \
         download \
         -t \
@@ -38,12 +39,25 @@ module Vore
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
+      results = {
+        pages_visited: 0,
+        pages_unprocessed: 0,
+        unprocessed_pages: [],
+      }
+
       Dir.glob(File.join(output_dir, "**", "*")).each do |path|
         next unless File.file?(path)
 
         html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
+        results[:pages_visited] += 1
+        if rewritten_html_file.empty?
+          results[:pages_unprocessed] += 1
+          results[:unprocessed_pages] << path
+          next
+        end
+
         # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
         url_path = path.split(FILE_SEPERATOR)[3..].join("/")
 
@@ -58,6 +72,8 @@ module Vore
       ensure
         File.delete(path) if File.file?(path)
       end
+
+      results
     end
 
     # def crawl(site, block)

data/lib/vore/handlers/content_extractor.rb

@@ -19,7 +19,13 @@ module Vole
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+        if element.tag_name == "pre" ||
+            element.tag_name == "code" ||
+            element.tag_name == "form" ||
+            element.tag_name == "style" ||
+            element.tag_name == "noscript" ||
+            element.tag_name == "script" ||
+            element.tag_name == "svg"
           element.remove
         elsif element.tag_name == "title"
           @within_title = true

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: x86_64-linux
 authors:
 - Garen J. Torikian