vore 0.2.1-arm64-linux → 0.2.3-arm64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46df38a7d14df27f53723a4ce83c9a3d34d9a3f516a32a62b1d2150665c4b622
-  data.tar.gz: 96e6ba5043de693565bd96815391f91f043390278a786abe951a662cc8f1c064
+  metadata.gz: e5b9fc7852a9bf9c1271f8b4ecba0a7144c2de434828ba5bd23c4ac27d7a12ad
+  data.tar.gz: 4aa0500aada9cb3ba7ec3a9f71872f97ec48203f9eb3f04648a6999110fb3d45
 SHA512:
-  metadata.gz: dcc8804b9f9cf57ecc7c87bfabec8416a8e12fc4987b713e2f3b16e6e2201a82b2be036590d37a9a6c4c8c58af9e6d52d9ed14efa4e84fd0472f7b0465290376
-  data.tar.gz: 5b2c0166e960100ca5f56d6df1c3f9223b9edcaf33a296338d640fd2bacdc4aab89eefcaa7799b46b97a2181d8e7f2cad9d0557e12ad54136f3efb9b7dfe0aa4
+  metadata.gz: 0faeb9c79b755d507c2ff2eac02dd3fb83a2c1482a97a2b2742be1d84ce989993615c3cf4172f48566cb2871d61532d09ce9849420779e4c06308806a50d6a37
+  data.tar.gz: cee36ddceb6c2baa131f8d7ff6da0a9f4f667f125f349a4fec86bb7ddfe3f65919bff0a9ca737728bad39165c05a9136af54b0267acf2ce83f39e68671de2dd1

data/lib/vore/crawler.rb

@@ -31,6 +31,7 @@ module Vore
 
       output = %x(#{@executable} \
         --user-agent #{user_agent} \
+        --delay 3000 \
         --url #{website} \
         download \
         -t \
@@ -38,12 +39,25 @@ module Vore
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
+      results = {
+        pages_visited: 0,
+        pages_unprocessed: 0,
+        unprocessed_pages: [],
+      }
+
       Dir.glob(File.join(output_dir, "**", "*")).each do |path|
         next unless File.file?(path)
 
         html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
+        results[:pages_visited] += 1
+        if rewritten_html_file.empty?
+          results[:pages_unprocessed] += 1
+          results[:unprocessed_pages] << path
+          next
+        end
+
         # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
         url_path = path.split(FILE_SEPERATOR)[3..].join("/")
 
@@ -58,6 +72,8 @@ module Vore
       ensure
         File.delete(path) if File.file?(path)
       end
+
+      results
     end
 
     # def crawl(site, block)

data/lib/vore/handlers/content_extractor.rb

@@ -19,7 +19,13 @@ module Vole
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+        if element.tag_name == "pre" ||
+            element.tag_name == "code" ||
+            element.tag_name == "form" ||
+            element.tag_name == "style" ||
+            element.tag_name == "noscript" ||
+            element.tag_name == "script" ||
+            element.tag_name == "svg"
           element.remove
         elsif element.tag_name == "title"
           @within_title = true

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: arm64-linux
 authors:
 - Garen J. Torikian