vore 0.4.0-x86_64-darwin → 0.5.0-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7231d77600a8372eda42123a340eac3eabdd707a95f018fcc112ae77934ad50d
-  data.tar.gz: b205e579cd2dd41dc8e816ac64f7fc40a829c4bd3ca455db670e72104ef94831
+  metadata.gz: 375d0bfc0c775c2be95de44e7d5a750da641f00332490deace0863b569331cb6
+  data.tar.gz: 9e39c8fbb2b9813e86138c76364e5de9539390d8569e76f35bdb7dcd25a18cae
 SHA512:
-  metadata.gz: bae9647133f39ea1ab8413991434ea2433e98433a2dcb386793edbbc2995c606338a4e12a7b83cb7b69f390d25a7bbdb8e9228403b3742b7b9d1a374e6d5fa61
-  data.tar.gz: 24461a245a4498dd4c1a2ef3d374510604186b64a57616d9a6d95aa9dd16a7ed95bb98cd336b5ec82c78ae90cbb44449c6731f6f905ffeb864b25a7b1075d4a3
+  metadata.gz: d8ed087d41652d299c514290f16dcf88e65f38d2c719dc3dc914b8ad1f9cd6674aca09053310b1cd08aeed256428fd98cabe2e9cdc5c93e0fa27838393c04759
+  data.tar.gz: 94cc7f34e9c8cccbf85e24b1ad38dd17e91fb0f736838422c61a09532aefa323973cc410c3f72d52939b6e75d909f703ab8c93e2b4da3195721102cedd4760e1

data/Cargo.lock

@@ -238,7 +238,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc229be27b394115abdc89e09500d5030407734d21a143a833eae5f136821bcd"
 dependencies = [
- "compact_str",
+ "compact_str 0.7.1",
  "serde",
 ]
 
@@ -346,6 +346,20 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "compact_str"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "static_assertions",
+]
+
 [[package]]
 name = "cookie"
 version = "0.18.1"
@@ -1418,18 +1432,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.98"
+version = "0.9.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8914b2e6af10bd50dd7aaac8c5146872d3924d6012929b4ff504e988f6badd24"
+checksum = "d83151cfea2b67db2444f68c53b119ff77cff235ad711c765072e4daf8f3185b"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.98"
+version = "0.9.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12af68c9757d419b82d65a12b5db538990dfe9416049fea3f0ba4b9a8ca108cd"
+checksum = "32d038214c118ad4a75db555ccb78672e17e1c5c10f344456cd129008dbaa7de"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -1786,14 +1800,14 @@ dependencies = [
 
 [[package]]
 name = "spider"
-version = "1.99.8"
+version = "1.99.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "525670cdc6aec8f4cb91da17ce0255050e89eb7c889272216d8a4fb644d67530"
+checksum = "e23ad22d5e55b09f480f849b37dd2fe315e3cf1df0f5261209aa5482483c617f"
 dependencies = [
  "ahash",
  "bytes",
  "case_insensitive_string",
- "compact_str",
+ "compact_str 0.8.0",
  "cssparser",
  "ego-tree",
  "fast_html5ever",
@@ -1817,9 +1831,9 @@ dependencies = [
 
 [[package]]
 name = "spider_cli"
-version = "1.99.8"
+version = "1.99.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bd9d95178dc0715608d5f28501c8321de3e14b40046c6584a12ffce96f0a676"
+checksum = "be5da7d570871156c08025bdc13de670807d36a90be94f8aa8342a04e5268662"
 dependencies = [
  "clap",
  "env_logger",

data/README.md

@@ -42,6 +42,29 @@ The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), s
 | `delete_after_yield`          | Whether the downloaded HTML files are deleted after the yield block finishes. | `true` |
 | `log_level`                   | The logging level. | `:warn` |
 
+### Processing pages
+
+Vore processes HTML using handlers. By default, there are two:
+
+* The `MetaExtractor`, which extracts information from your `title` and `meta` tags
+* The `TagRemover`, which removes unnecessary elements like `header`, `footer`, `script`
+
+If you wish to process the HTML further, you can provide your own handler:
+
+
+```ruby
+Vore::Crawler.new(handlers: [MySpecialHandler.new])
+```
+
+Handlers are defined using [Selma](https://github.com/gjtorikian/selma?tab=readme-ov-file#defining-handlers). Note that the `MetaExtractor` is always included and defined first, but if you pass in anything to the `handler` array, it'll overwrite Vore's other default handlers. You can of course choose to include them manually:
+
+
+```ruby
+# preserve Vore's default content handler while adding your own;
+# `MetaExtractor` is prefixed to the front
+Vore::Crawler.new(handlers: [Vore::Handlers::TagRemover.new, MySpecialHandler.new])
+```
+
 ### In tests
 
 Since the actual HTTP calls occur in a separate process, Vore will not integrate with libraries like VCR or Webmock by default. You'll need to `require "vore/minitest_helper"` to get a function that emulates the HTTP `GET` requests in a way Ruby can interpret.

data/lib/vore/configuration.rb

@@ -3,6 +3,7 @@
 module Vore
   class Configuration
     DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
+      allow_comments: false,
       allow_doctype: false,
     })
 

data/lib/vore/crawler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require_relative "handlers/content_extractor"
+require_relative "handlers/meta_extractor"
+require_relative "handlers/tag_remover"
 
 require "listen"
 
@@ -10,13 +11,20 @@ module Vore
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
     FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
-    attr_reader :output_dir
+    attr_reader :handlers, :output_dir
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
-    def initialize(sanitization_config: Vore::Configuration::DEFAULT_SANITIZATION_CONFIG, options: {})
-      @content_extractor = Vore::Handlers::ContentExtractor.new
-      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+    def initialize(sanitization_config: Vore::Configuration::DEFAULT_SANITIZATION_CONFIG, handlers: nil, options: {})
+      @meta_extractor = Vore::Handlers::MetaExtractor.new
+
+      @handlers = if handlers.nil?
+        [@meta_extractor, Vore::Handlers::TagRemover.new]
+      else
+        handlers.unshift(@meta_extractor)
+      end
+
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: @handlers)
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @options = Vore::Configuration::DEFAULT_OPTIONS.merge(options)
@@ -70,34 +78,36 @@ module Vore
       @results[:pages_visited] += 1
 
       html_file = File.read(path).force_encoding("UTF-8")
-      rewritten_html_file = ""
 
       if html_file.empty?
         @results[:unprocessed_pages] << path
         return
       end
 
-      begin
-        rewritten_html_file = @selma.rewrite(html_file)
-      rescue StandardError => e
-        Vore.logger.warn("Error rewriting #{path}: #{e}")
-        @results[:unprocessed_pages] << path
-        return
-      end
+      rewritten_html_file = @selma.rewrite(html_file)
+      return if rewritten_html_file.empty?
 
       # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
       url_path = path.split(FILE_SEPERATOR)[(@parent_output_dir_len + 1)..].join("/")
 
       page = Vore::PageData.new(
         content: rewritten_html_file,
-        title: @content_extractor.title,
-        meta: @content_extractor.meta,
+        title: @meta_extractor.title,
+        meta: @meta_extractor.meta,
         path: url_path,
       )
 
       yield page
     end
 
+    def rewrite(html_file)
+      @selma.rewrite(html_file)
+    rescue StandardError => e
+      Vore.logger.warn("Error rewriting #{path}: #{e}")
+      @results[:unprocessed_pages] << path
+      ""
+    end
+
     def run_command(website, delay: 0)
       pid = Process.spawn(
         @executable,

data/lib/vore/handlers/{content_extractor.rb → meta_extractor.rb}

@@ -2,7 +2,7 @@
 
 module Vore
   module Handlers
-    class ContentExtractor
+    class MetaExtractor
       SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
 
       attr_reader :title, :meta
@@ -19,22 +19,14 @@ module Vore
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" ||
-            element.tag_name == "form" ||
-            element.tag_name == "style" ||
-            element.tag_name == "noscript" ||
-            element.tag_name == "script" ||
-            element.tag_name == "svg"
-          element.remove
-        elsif element.tag_name == "title"
+        if element.tag_name == "title"
           @within_title = true
+
           element.remove
         elsif element.tag_name == "meta"
           return if element.attributes["name"].nil?
 
           @meta[element.attributes["name"]] = element.attributes["content"]
-        else
-          element.remove_and_keep_content
         end
       end
 

data/lib/vore/handlers/tag_remover.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Vore
+  module Handlers
+    class TagRemover
+      SELECTOR = Selma::Selector.new(match_element: "*")
+
+      def selector
+        SELECTOR
+      end
+
+      UNNECESSARY_TAGS = [
+        # Remove code elements
+        "pre",
+
+        # Remove unnecessary elements
+        "head",
+
+        "form",
+        "style",
+        "noscript",
+        "script",
+        "svg",
+
+        # Remove unnecessary nav elements
+        "header",
+        "footer",
+        "nav",
+        "aside",
+      ]
+
+      CONTENT_TO_KEEP = [
+        "html",
+        "body",
+      ]
+
+      def handle_element(element)
+        if UNNECESSARY_TAGS.include?(element.tag_name)
+          element.remove
+        elsif CONTENT_TO_KEEP.include?(element.tag_name)
+          element.remove_and_keep_content
+        end
+      end
+    end
+  end
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-22 00:00:00.000000000 Z
+date: 2024-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: listen
@@ -60,7 +60,8 @@ files:
 - lib/vore.rb
 - lib/vore/configuration.rb
 - lib/vore/crawler.rb
-- lib/vore/handlers/content_extractor.rb
+- lib/vore/handlers/meta_extractor.rb
+- lib/vore/handlers/tag_remover.rb
 - lib/vore/logger.rb
 - lib/vore/minitest_helper.rb
 - lib/vore/page.rb