vore 0.1.1.3-arm64-darwin → 0.2.0-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7f77800bde8d47acd14de4f7aca527d9a85f5837dca2028168e55403705ef11
-  data.tar.gz: d59098743ea871f46515378b77650672a315c18c7f8cdf42ee86b0a290c46b62
+  metadata.gz: 8d2efc3993d1d65cebb2a5576331bc20ece0520a49f59c0d7f995a179d2b7f4b
+  data.tar.gz: 9bfa5dd808324b2d52b540937f2a3cd7562e453adf79ff3c982ac8fbab4e6339
 SHA512:
-  metadata.gz: 169d2e60b0ee7c556ff5f6fe0c3f659a55e575e473e05e08cd9293e391e1ec8ec35e6ad35cfb7912fc6691e525f838adc0c46446aadea94a36f940d99b0a8710
-  data.tar.gz: 84a49048aaccaf6e5236c7d79536020a0e61a3982088ed6f2def78f2f5d0b1a2770f27c45a6e2b38d6c61eda28871b3d6fe4c08a1210f894df930da3e6f8cf8a
+  metadata.gz: a1364b00820082923ac4ca1679564a4e2ee44d89f27539022a1551d7dafca7df93817c71581021ab67ebfb2e044fcc4799ff1abd8f412cdab88c73fc658d52df
+  data.tar.gz: e95deb9ca83978c56bdd96211f806042f3c5907ffff9fcfb90dddc486b3b766f1c207a56fd484a1ee554ae3b26fb89fa7220e5cfa30434a23447bc08ee61bc8f

data/README.md

@@ -2,7 +2,7 @@
 
 ![Vore, by LewdBacon](https://github.com/user-attachments/assets/0923cc84-4cca-4d95-8a0e-4dad650525d2)
 
-Vore gobbles up webpages and spits out their content.
+Vore quickly crawls websites and spits out text sans tags. It's written in Ruby and powered by Rust.
 
 ## Installation
 

data/lib/vore/crawler.rb

@@ -6,14 +6,17 @@ module Vore
   # This is the class that starts and controls the crawling
   class Crawler
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+    FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
       @denylist_regexp = Regexp.union(denylist)
 
-      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [Vole::Handlers::ContentExtractor.new])
-      @executable = File.expand_path(File.join("exe", "vore-spider"))
+      @content_extractor = Vole::Handlers::ContentExtractor.new
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+      ext = PLATFORM.include?("windows") ? ".exe" : ""
+      @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @output_dir = "tmp/vore"
 
       return if File.exist?(@executable)
@@ -38,10 +41,20 @@ module Vore
       Dir.glob("tmp/**/*").each do |path|
         next unless File.file?(path)
 
-        html_file = File.read(path)
+        html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
-        yield rewritten_html_file
+        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+
+        page = Vore::PageData.new(
+          content: rewritten_html_file,
+          title: @content_extractor.title,
+          meta: @content_extractor.meta,
+          path: url_path,
+        )
+
+        yield page
       ensure
         File.delete(path) if File.file?(path)
       end

data/lib/vore/handlers/content_extractor.rb

@@ -3,19 +3,42 @@
 module Vole
   module Handlers
     class ContentExtractor
-      SELECTOR = Selma::Selector.new(match_element: "*")
+      SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
+
+      attr_reader :title, :meta
+
+      def initialize
+        super
+        @title = ""
+        @meta = {}
+        @within_title = false
+      end
 
       def selector
         SELECTOR
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code"
+        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+          element.remove
+        elsif element.tag_name == "title"
+          @within_title = true
           element.remove
+        elsif element.tag_name == "meta"
+          return if element.attributes["name"].nil?
+
+          @meta[element.attributes["name"]] = element.attributes["content"]
         else
           element.remove_and_keep_content
         end
       end
+
+      def handle_text_chunk(text)
+        if @within_title
+          @within_title = false
+          @title = text.to_s
+        end
+      end
     end
   end
 end

data/lib/vore/page_data.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vore
+  class PageData
+    attr_reader :title, :meta, :content, :path
+
+    def initialize(title:, meta:, content:, path:)
+      @title = title
+      @meta = meta
+      @content = content
+      @path = path
+    end
+  end
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.1.1.3"
+  VERSION = "0.2.0"
 end

data/lib/vore.rb

@@ -14,6 +14,9 @@ require_relative "vore/version"
 require_relative "vore/configuration"
 require_relative "vore/logger"
 require_relative "vore/crawler"
+require_relative "vore/page"
+require_relative "vore/page_data"
+require_relative "vore/website"
 
 module Vore
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.1.1.3
+  version: 0.2.0
 platform: arm64-darwin
 authors:
 - Garen J. Torikian
@@ -49,6 +49,7 @@ files:
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
 - lib/vore/page.rb
+- lib/vore/page_data.rb
 - lib/vore/version.rb
 - lib/vore/website.rb
 homepage: https://github.com/gjtorikian/vore
@@ -75,5 +76,5 @@ requirements: []
 rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
-summary: Quickly consume websites and spit out text. Powered by Rust.
+summary: Quickly crawls websites and spits out text sans tags. Powered by Rust.
 test_files: []