vore 0.1.1.3-x86_64-linux → 0.2.0-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3a066d2ed47c281d2b9ccf74c5138b36b4e6f85160004d563df4932f6308849
-  data.tar.gz: 79753c3ff8f62347166b0e8c18d2c237b934226a48565e36b893d999e0fc0c13
+  metadata.gz: 7427980069e4d526d966f4e94f7f4f0e9f35831c2202d3ce6cdf93d7ed4db4cb
+  data.tar.gz: fd5b467cfa827e9c0b776b03982e5b2a683725b57be3d83c0e06e924634bfe7d
 SHA512:
-  metadata.gz: 1805d6b37a00b6b566c1b86fe2ce3b63179f8a59418df1e02702de522a5ddc0ca408171b980931396eb0cccdb493a831e45527186f024ed7839111a44f5c4002
-  data.tar.gz: 3b63fcb5ead63bda2fc83d4cef0a3a95f82bdf77d4c4835fca8d600c41022a84d17dc26502884d1c81ba743e4d8afdd505adf19ed2ab1b06233455847c08d3fe
+  metadata.gz: 1b252342c22884e71703595a880dc1d8c3f64daaaecbdf60744245ef8e741441ecb23de978f80531ed4a48d67283f343d2455992d88832ed555bf78c736232e7
+  data.tar.gz: 7b85dc417826abcac6d8d008e1b461aba77857d2fcf8a37953946e6a0b9f9014420270c3635c819e3a698df36a6ab58fba518135784f9b84887c9bf5ba8a2db6

data/README.md

@@ -2,7 +2,7 @@
 
 ![Vore, by LewdBacon](https://github.com/user-attachments/assets/0923cc84-4cca-4d95-8a0e-4dad650525d2)
 
-Vore gobbles up webpages and spits out their content.
+Vore quickly crawls websites and spits out text sans tags. It's written in Ruby and powered by Rust.
 
 ## Installation
 

data/lib/vore/crawler.rb

@@ -6,14 +6,17 @@ module Vore
   # This is the class that starts and controls the crawling
   class Crawler
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+    FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
       @denylist_regexp = Regexp.union(denylist)
 
-      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [Vole::Handlers::ContentExtractor.new])
-      @executable = File.expand_path(File.join("exe", "vore-spider"))
+      @content_extractor = Vole::Handlers::ContentExtractor.new
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+      ext = PLATFORM.include?("windows") ? ".exe" : ""
+      @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
       @output_dir = "tmp/vore"
 
       return if File.exist?(@executable)
@@ -38,10 +41,20 @@ module Vore
       Dir.glob("tmp/**/*").each do |path|
         next unless File.file?(path)
 
-        html_file = File.read(path)
+        html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
-        yield rewritten_html_file
+        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+
+        page = Vore::PageData.new(
+          content: rewritten_html_file,
+          title: @content_extractor.title,
+          meta: @content_extractor.meta,
+          path: url_path,
+        )
+
+        yield page
       ensure
         File.delete(path) if File.file?(path)
       end

data/lib/vore/handlers/content_extractor.rb

@@ -3,19 +3,42 @@
 module Vole
   module Handlers
     class ContentExtractor
-      SELECTOR = Selma::Selector.new(match_element: "*")
+      SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
+
+      attr_reader :title, :meta
+
+      def initialize
+        super
+        @title = ""
+        @meta = {}
+        @within_title = false
+      end
 
       def selector
         SELECTOR
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code"
+        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+          element.remove
+        elsif element.tag_name == "title"
+          @within_title = true
           element.remove
+        elsif element.tag_name == "meta"
+          return if element.attributes["name"].nil?
+
+          @meta[element.attributes["name"]] = element.attributes["content"]
         else
           element.remove_and_keep_content
         end
       end
+
+      def handle_text_chunk(text)
+        if @within_title
+          @within_title = false
+          @title = text.to_s
+        end
+      end
     end
   end
 end

data/lib/vore/page_data.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vore
+  class PageData
+    attr_reader :title, :meta, :content, :path
+
+    def initialize(title:, meta:, content:, path:)
+      @title = title
+      @meta = meta
+      @content = content
+      @path = path
+    end
+  end
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.1.1.3"
+  VERSION = "0.2.0"
 end

data/lib/vore.rb

@@ -14,6 +14,9 @@ require_relative "vore/version"
 require_relative "vore/configuration"
 require_relative "vore/logger"
 require_relative "vore/crawler"
+require_relative "vore/page"
+require_relative "vore/page_data"
+require_relative "vore/website"
 
 module Vore
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.1.1.3
+  version: 0.2.0
 platform: x86_64-linux
 authors:
 - Garen J. Torikian
@@ -49,6 +49,7 @@ files:
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
 - lib/vore/page.rb
+- lib/vore/page_data.rb
 - lib/vore/version.rb
 - lib/vore/website.rb
 homepage: https://github.com/gjtorikian/vore
@@ -75,5 +76,5 @@ requirements: []
 rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
-summary: Quickly consume websites and spit out text. Powered by Rust.
+summary: Quickly crawls websites and spits out text sans tags. Powered by Rust.
 test_files: []