vore 0.1.1.4-x86_64-darwin → 0.2.1-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81fffac2120b7fe4284124ed91f01c7a9bc34324b60877b0f1d1b2789de9bd82
-  data.tar.gz: 4c9e4f94494a8ad9bff6613661857f5862679099302b7b1f3725ef79e484e16d
+  metadata.gz: 31497cb9cec566256acca79e43bfb5b6fe8791bea446bba60ca895f2be3ab91b
+  data.tar.gz: 2252352ba0f823215117c143d0608431bfd6e0037f9b122f6711c191dc7f99c2
 SHA512:
-  metadata.gz: 187fb799c3db652f85f70fd9676468b1f80efcd95dfd395cbc7b833951d605e55efbced8d1005d405836bae3ba36c87baa6565e761aa3f2b81fe9b9f447f9b07
-  data.tar.gz: 2346f4bca694fc780284e3e727ece3b3932a53e95d4b437b2b2c370cbbb6a9ba61dc8959d4b6045107d0b4635fbc8fca59b75984363ae6e18ca3d79e6f4f19a4
+  metadata.gz: a0f51ccfc30cb95ff87b6275c033b343ca7c07994f9937fad18a670d8973dd2c0d2db8c40f5849d2e32a372c1e755e97b3489b46532bd9c0317e7bfdc88786e0
+  data.tar.gz: 9517a87119025f30bdb105fa89440990ec260b32db10e497a6775271fec547358a4552edf6033508d27225a12427bd61272b13e0585824802c4490d7187173de

data/README.md

@@ -2,7 +2,7 @@
 
 ![Vore, by LewdBacon](https://github.com/user-attachments/assets/0923cc84-4cca-4d95-8a0e-4dad650525d2)
 
-Vore gobbles up webpages and spits out their content.
+Vore quickly crawls websites and spits out text sans tags. It's written in Ruby and powered by Rust.
 
 ## Installation
 

data/lib/vore/crawler.rb

@@ -6,15 +6,18 @@ module Vore
   # This is the class that starts and controls the crawling
   class Crawler
     PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+    FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
     def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
       @denylist_regexp = Regexp.union(denylist)
 
-      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [Vole::Handlers::ContentExtractor.new])
-      @executable = File.expand_path(File.join("exe", "vore-spider"))
-      @output_dir = "tmp/vore"
+      @content_extractor = Vole::Handlers::ContentExtractor.new
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+      ext = PLATFORM.include?("windows") ? ".exe" : ""
+      @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
+      @parent_output_dir = "tmp/vore"
 
       return if File.exist?(@executable)
 
@@ -23,7 +26,7 @@ module Vore
     end
 
     def scrape_each_page(website, &block)
-      output_dir = "#{@output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
       Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
       output = %x(#{@executable} \
@@ -35,13 +38,23 @@ module Vore
 
       Vore.logger.info("Vore finished crawling #{website}: #{output}")
 
-      Dir.glob("tmp/**/*").each do |path|
+      Dir.glob(File.join(output_dir, "**", "*")).each do |path|
         next unless File.file?(path)
 
-        html_file = File.read(path)
+        html_file = File.read(path).force_encoding("UTF-8")
         rewritten_html_file = @selma.rewrite(html_file)
 
-        yield rewritten_html_file
+        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+
+        page = Vore::PageData.new(
+          content: rewritten_html_file,
+          title: @content_extractor.title,
+          meta: @content_extractor.meta,
+          path: url_path,
+        )
+
+        yield page
       ensure
         File.delete(path) if File.file?(path)
       end

data/lib/vore/handlers/content_extractor.rb

@@ -3,19 +3,42 @@
 module Vole
   module Handlers
     class ContentExtractor
-      SELECTOR = Selma::Selector.new(match_element: "*")
+      SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
+
+      attr_reader :title, :meta
+
+      def initialize
+        super
+        @title = ""
+        @meta = {}
+        @within_title = false
+      end
 
       def selector
         SELECTOR
       end
 
       def handle_element(element)
-        if element.tag_name == "pre" || element.tag_name == "code"
+        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+          element.remove
+        elsif element.tag_name == "title"
+          @within_title = true
           element.remove
+        elsif element.tag_name == "meta"
+          return if element.attributes["name"].nil?
+
+          @meta[element.attributes["name"]] = element.attributes["content"]
         else
           element.remove_and_keep_content
         end
       end
+
+      def handle_text_chunk(text)
+        if @within_title
+          @within_title = false
+          @title = text.to_s
+        end
+      end
     end
   end
 end

data/lib/vore/page_data.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vore
+  class PageData
+    attr_reader :title, :meta, :content, :path
+
+    def initialize(title:, meta:, content:, path:)
+      @title = title
+      @meta = meta
+      @content = content
+      @path = path
+    end
+  end
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.1.1.4"
+  VERSION = "0.2.1"
 end

data/lib/vore.rb

@@ -14,6 +14,9 @@ require_relative "vore/version"
 require_relative "vore/configuration"
 require_relative "vore/logger"
 require_relative "vore/crawler"
+require_relative "vore/page"
+require_relative "vore/page_data"
+require_relative "vore/website"
 
 module Vore
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.1.1.4
+  version: 0.2.1
 platform: x86_64-darwin
 authors:
 - Garen J. Torikian
@@ -49,6 +49,7 @@ files:
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
 - lib/vore/page.rb
+- lib/vore/page_data.rb
 - lib/vore/version.rb
 - lib/vore/website.rb
 homepage: https://github.com/gjtorikian/vore
@@ -75,5 +76,5 @@ requirements: []
 rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
-summary: Quickly consume websites and spit out text. Powered by Rust.
+summary: Quickly crawls websites and spits out text sans tags. Powered by Rust.
 test_files: []