vore 0.1.1.4-x86_64-darwin → 0.2.0-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81fffac2120b7fe4284124ed91f01c7a9bc34324b60877b0f1d1b2789de9bd82
-  data.tar.gz: 4c9e4f94494a8ad9bff6613661857f5862679099302b7b1f3725ef79e484e16d
+  metadata.gz: 5c54c9d9a3d685f0545ebcec67d0aa30a3a6751d5ae31093197a021d24178040
+  data.tar.gz: d961d2d7c2bbcf3fad014e4bf1eea9f5d7a692901dcea50f66c3bdccf64066f9
 SHA512:
-  metadata.gz: 187fb799c3db652f85f70fd9676468b1f80efcd95dfd395cbc7b833951d605e55efbced8d1005d405836bae3ba36c87baa6565e761aa3f2b81fe9b9f447f9b07
-  data.tar.gz: 2346f4bca694fc780284e3e727ece3b3932a53e95d4b437b2b2c370cbbb6a9ba61dc8959d4b6045107d0b4635fbc8fca59b75984363ae6e18ca3d79e6f4f19a4
+  metadata.gz: 1a9ae3ef5f6b227c86cb21ecef0377ab7582e96ca562505e83b127333c7cad7579c4e5b1525793cd54a4d97a42c626646fcfae4146b547e00373db339f4b9615
+  data.tar.gz: ec1173ad2572e8bf07760360941493045d9e1d39912651cab8403be4418da6d39df42e9bdcd3d2f8d61ddf576324fbb73de7d252ca301a75f83920e77a89de79

data/README.md

@@ -2,7 +2,7 @@
 
 ![Vore, by LewdBacon](https://github.com/user-attachments/assets/0923cc84-4cca-4d95-8a0e-4dad650525d2)
 
-Vore gobbles up webpages and spits out their content.
+Vore quickly crawls websites and spits out text sans tags. It's written in Ruby and powered by Rust.
 
 ## Installation
 

data/lib/vore/crawler.rb

@@ -6,14 +6,17 @@ module Vore
   # This is the class that starts and controls the crawling
   class Crawler
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+    FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
       @denylist_regexp = Regexp.union(denylist)
 
-      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [Vole::Handlers::ContentExtractor.new])
-      @executable = File.expand_path(File.join("exe", "vore-spider"))
+      @content_extractor = Vole::Handlers::ContentExtractor.new
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+      ext = PLATFORM.include?("windows") ? ".exe" : ""
+      @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @output_dir = "tmp/vore"
 
       return if File.exist?(@executable)
@@ -38,10 +41,20 @@ module Vore
       Dir.glob("tmp/**/*").each do |path|
         next unless File.file?(path)
 
-        html_file = File.read(path)
+        html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
-        yield rewritten_html_file
+        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+
+        page = Vore::PageData.new(
+          content: rewritten_html_file,
+          title: @content_extractor.title,
+          meta: @content_extractor.meta,
+          path: url_path,
+        )
+
+        yield page
       ensure
         File.delete(path) if File.file?(path)
       end

data/lib/vore/handlers/content_extractor.rb

@@ -3,19 +3,42 @@
 module Vole
   module Handlers
     class ContentExtractor
-      SELECTOR = Selma::Selector.new(match_element: "*")
+      SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
+
+      attr_reader :title, :meta
+
+      def initialize
+        super
+        @title = ""
+        @meta = {}
+        @within_title = false
+      end
 
       def selector
         SELECTOR
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code"
+        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+          element.remove
+        elsif element.tag_name == "title"
+          @within_title = true
           element.remove
+        elsif element.tag_name == "meta"
+          return if element.attributes["name"].nil?
+
+          @meta[element.attributes["name"]] = element.attributes["content"]
         else
           element.remove_and_keep_content
         end
       end
+
+      def handle_text_chunk(text)
+        if @within_title
+          @within_title = false
+          @title = text.to_s
+        end
+      end
     end
   end
 end

data/lib/vore/page_data.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vore
+  class PageData
+    attr_reader :title, :meta, :content, :path
+
+    def initialize(title:, meta:, content:, path:)
+      @title = title
+      @meta = meta
+      @content = content
+      @path = path
+    end
+  end
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.1.1.4"
+  VERSION = "0.2.0"
 end

data/lib/vore.rb

@@ -14,6 +14,9 @@ require_relative "vore/version"
 require_relative "vore/configuration"
 require_relative "vore/logger"
 require_relative "vore/crawler"
+require_relative "vore/page"
+require_relative "vore/page_data"
+require_relative "vore/website"
 
 module Vore
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.1.1.4
+  version: 0.2.0
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian
@@ -49,6 +49,7 @@ files:
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
 - lib/vore/page.rb
+- lib/vore/page_data.rb
 - lib/vore/version.rb
 - lib/vore/website.rb
 homepage: https://github.com/gjtorikian/vore
@@ -75,5 +76,5 @@ requirements: []
 rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
-summary: Quickly consume websites and spit out text. Powered by Rust.
+summary: Quickly crawls websites and spits out text sans tags. Powered by Rust.
 test_files: []