vore 0.2.8-arm64-darwin → 0.4.0-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cea3bfc2d1ad6b14200a35b52eaa739bb14b2f6d5ace761be3d77a53366f19b2
-  data.tar.gz: '03861e5e9ccdbca488f9397bc59f480f48e6d62e03b4cf40ee652e3cc57745da'
+  metadata.gz: 214aba124a131e6567e5d6eeba171d3efb63d9b74f73efe19e2e63730da5f510
+  data.tar.gz: 971b1d9b5e5adc36ba7de5b3e6451443b4b3124ef3e73b63f44c3dc96d9effa1
 SHA512:
-  metadata.gz: b7c448fce4a1c6b26b6bff99fbe91053588f56b86b077567913902b4c3405d805d56e41ae430d302fbe2c8d834b50514a945f7361b8996c992d33355cdd7269a
-  data.tar.gz: aa7a8facb68f637b2642b4ee1fbb7437b4cf78ede8f31c8505e9d8141f45bdb4ef689f80ca498af4eb681cd6d5c6c3cbbdba8059f8890ad9fcb37eac5a480095
+  metadata.gz: c123316da7dfeba1c4f3b5f8c4a62411493770a3d726097ddefa23e06868a4ec7a38d853b8b973ec75ee7773383f302b671d83057a5ac0a343f801a4c90a3420
+  data.tar.gz: 6de3f0622a91ad15f63331075ed3b79746f4d894f9521c25dd8815779212d93492c10014e47cb36aa0c6bdaa1ae8c2feb5d6bafdc19cdc792f17eccc97db0e74

data/Cargo.lock

@@ -1786,9 +1786,9 @@ dependencies = [
 
 [[package]]
 name = "spider"
-version = "1.99.5"
+version = "1.99.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f62dc0e4f32d36a931471a1694d5b5b29e916c537121ca8028742c9acbe510d"
+checksum = "525670cdc6aec8f4cb91da17ce0255050e89eb7c889272216d8a4fb644d67530"
 dependencies = [
  "ahash",
  "bytes",
@@ -1817,9 +1817,9 @@ dependencies = [
 
 [[package]]
 name = "spider_cli"
-version = "1.99.5"
+version = "1.99.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6031d46576f5fdba52d5c054a8c69ba7ee17cc824563b764fce4c8471c15c3a1"
+checksum = "9bd9d95178dc0715608d5f28501c8321de3e14b40046c6584a12ffce96f0a676"
 dependencies = [
  "clap",
  "env_logger",

data/README.md

@@ -18,13 +18,75 @@ If bundler is not being used to manage dependencies, install the gem by executin
 ## Usage
 
 ```ruby
-    crawler = Vore::Crawler.new
-    crawler.scrape_each_page("https://choosealicense.com") do |page|
-      puts page
-    end
+crawler = Vore::Crawler.new
+crawler.scrape_each_page("https://choosealicense.com") do |page|
+  puts page
+end
 ```
 
-Each `page` is simply every text node. The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), so you know it's fast.
+Each `page` is a simple class consisting of the following values:
+
+* `content`: the text of the HTML document, sans tags
+* `title`: the title of the HTML document (if any)
+* `meta`: the document's meta tags (if any)
+* `path`: the document's path
+
+The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), so you know it's fast.
+
+### Configuration
+
+| Name                          | Description                                                                                                                                 | Default |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `delay`                       | A value (in milliseconds) which introduces an artifical delay when crawling. Useful for situations where there's rate limiting involved. | `0` |
+| `output_dir`                  | Where the resulting HTML files are stored. | `"tmp/vore"`    |
+| `delete_after_yield`          | Whether the downloaded HTML files are deleted after the yield block finishes. | `true` |
+| `log_level`                   | The logging level. | `:warn` |
+
+### In tests
+
+Since the actual HTTP calls occur in a separate process, Vore will not integrate with libraries like VCR or Webmock by default. You'll need to `require "vore/minitest_helper"` to get a function that emulates the HTTP `GET` requests in a way Ruby can interpret.
+
+Based on your needs, you can overwrite any of the existing methods to suit your application's needs. For example, if you prefer HTML to be generated by Faker, you can create and require a file that looks like the following:
+
+```ruby
+
+require "vore/minitest_helper"
+
+module Vore
+  module TestHelperExtension
+    DOCUMENT_TITLES = [
+      "Hello, I need help",
+      "I need to update my payment information",
+    ]
+    DOCUMENT_CONTENT = [
+      "Hey, I'm having trouble with my computer. Can you help me?",
+      # v--- always creates three page chunks
+      "I need to update my payment information. Like, now. Right now. Now. Can you help me? Please? Now?" + "Can you help me? Please? Now?" * 100,
+    ]
+
+    def content
+      @counter = -1 unless defined?(@counter)
+      @counter += 1
+
+      html = "<!DOCTYPE html><html><head><title>#{DOCUMENT_TITLES[@counter]}</title>"
+
+      meta_tag_count.times do # arbitrarily set to 5
+        html += "<meta name=\"#{Faker::Lorem.word}\" content=\"#{Faker::Lorem.word}\" />"
+      end
+
+      html += "</head><body>"
+
+      html += "<p>#{DOCUMENT_CONTENT[@counter]}</p>"
+
+      html += "</body></html>"
+
+      html
+    end
+  end
+
+  Vore::TestHelper.prepend(Vore::TestHelperExtension)
+end
+```
 
 ## Development
 

data/lib/vore/configuration.rb

@@ -1,9 +1,16 @@
 # frozen_string_literal: true
 
-module Vole
+module Vore
   class Configuration
     DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
       allow_doctype: false,
     })
+
+    DEFAULT_OPTIONS = {
+      delay: 0,
+      output_dir: "tmp/vore",
+      delete_after_yield: true,
+      log_level: :warn,
+    }
   end
 end

data/lib/vore/crawler.rb

@@ -2,6 +2,8 @@
 
 require_relative "handlers/content_extractor"
 
+require "listen"
+
 module Vore
   # This is the class that starts and controls the crawling
   class Crawler
@@ -12,14 +14,22 @@ module Vore
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
-    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
-      @denylist_regexp = Regexp.union(denylist)
-
-      @content_extractor = Vole::Handlers::ContentExtractor.new
+    def initialize(sanitization_config: Vore::Configuration::DEFAULT_SANITIZATION_CONFIG, options: {})
+      @content_extractor = Vore::Handlers::ContentExtractor.new
       @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
-      @parent_output_dir = "tmp/vore"
+      @options = Vore::Configuration::DEFAULT_OPTIONS.merge(options)
+      @parent_output_dir = @options[:output_dir]
+      @parent_output_dir_len = @parent_output_dir.to_s.split(FILE_SEPERATOR).size
+
+      Vore.logger.level = @options[:log_level]
+      Listen.logger = Vore.logger
+
+      @results = {
+        pages_visited: 0,
+        unprocessed_pages: [],
+      }
 
       return if File.exist?(@executable)
 
@@ -29,71 +39,82 @@ module Vore
 
     def scrape_each_page(website, &block)
       @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
-      Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
-
-      output = run_command(website, @output_dir)
+      FileUtils.rm_rf(@output_dir)
+      FileUtils.mkdir_p(@output_dir)
+
+      listener = Listen.to(@output_dir) do |_modified, added, _removed|
+        if added.any?
+          added.each do |path|
+            process_file(path, &block)
+            File.delete(path) if @options[:delete_after_yield]
+          end
+        end
+      end
+      listener.start
 
-      Vore.logger.info("Vore finished crawling #{website}: #{output}")
+      Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      results = {
-        pages_visited: 0,
-        pages_unprocessed: 0,
-        unprocessed_pages: [],
-      }
+      begin
+        run_command(website, delay: @options[:delay])
+      ensure
+        sleep(0.5) # give listener time to clean up
+        listener.stop
+      end
 
-      Dir.glob(File.join(output_dir, "**", "*")).each do |path|
-        next unless File.file?(path)
+      Vore.logger.info("Vore finished crawling #{website}")
 
-        results[:pages_visited] += 1
+      @results
+    end
 
-        html_file = File.read(path).force_encoding("UTF-8")
-        rewritten_html_file = ""
+    def process_file(path, &block)
+      @results[:pages_visited] += 1
 
-        if html_file.empty?
-          results[:pages_unprocessed] += 1
-          results[:unprocessed_pages] << path
-          next
-        end
+      html_file = File.read(path).force_encoding("UTF-8")
+      rewritten_html_file = ""
 
-        begin
-          rewritten_html_file = @selma.rewrite(html_file)
-        rescue StandardError => e
-          Vore.logger.warn("Error rewriting #{path}: #{e}")
-          results[:pages_unprocessed] += 1
-          next
-        end
+      if html_file.empty?
+        @results[:unprocessed_pages] << path
+        return
+      end
 
-        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
-        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+      begin
+        rewritten_html_file = @selma.rewrite(html_file)
+      rescue StandardError => e
+        Vore.logger.warn("Error rewriting #{path}: #{e}")
+        @results[:unprocessed_pages] << path
+        return
+      end
 
-        page = Vore::PageData.new(
-          content: rewritten_html_file,
-          title: @content_extractor.title,
-          meta: @content_extractor.meta,
-          path: url_path,
-        )
+      # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+      url_path = path.split(FILE_SEPERATOR)[(@parent_output_dir_len + 1)..].join("/")
 
-        yield page
-      ensure
-        File.delete(path) if File.file?(path)
-      end
+      page = Vore::PageData.new(
+        content: rewritten_html_file,
+        title: @content_extractor.title,
+        meta: @content_extractor.meta,
+        path: url_path,
+      )
 
-      results
+      yield page
     end
 
-    # def crawl(site, block)
-    #   Vore.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
-    #   crawl_site(site)
-    # end
-
-    def run_command(website, output_dir)
-      %x(#{@executable} \
-        --user-agent #{user_agent} \
-        --delay 3500 \
-        --url #{website} \
-        download \
-        -t \
-        #{output_dir})
+    def run_command(website, delay: 0)
+      pid = Process.spawn(
+        @executable,
+        "--user-agent",
+        user_agent,
+        "--delay",
+        delay.to_s,
+        "--url",
+        website,
+        "download",
+        "-t",
+        @output_dir,
+      )
+
+      _, _status = Process.waitpid2(pid)
+    rescue StandardError => e
+      Vore.logger.error(e)
     end
 
     def user_agent

data/lib/vore/handlers/content_extractor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Vole
+module Vore
   module Handlers
     class ContentExtractor
       SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")

data/lib/vore/logger.rb

@@ -23,6 +23,10 @@ module Vore
           instance
         end
       end
+
+      def level=(level)
+        instance.level = level
+      end
     end
   end
 end

data/lib/vore/minitest_helper.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "minitest/mock"
+require "net/http"
+
+module Vore
+  module TestHelper
+    def run_command(website, **options)
+      loop_times.times do |time|
+        net_http = ::Minitest::Mock.new
+        response = ::Minitest::Mock.new
+        response.expect(:is_a?, true, [::Net::HTTPSuccess])
+
+        # we need to trigger an HTTP call to pretend that we're making
+        # an external request. this way, the gem hooks into VCR/Webmock
+        net_http.expect(:get, response)
+        html = content
+        response.expect(:body, html)
+        time_s = time.to_s
+        uri = URI("#{website}/#{time_s}")
+        Net::HTTP.get(uri)
+
+        file = File.join(@output_dir, time_s)
+        File.write("#{file}.html", html)
+      end
+    end
+
+    def loop_times=(times)
+      @loop_times = times
+    end
+
+    def loop_times
+      @loop_times ||= 5
+    end
+
+    def meta_tag_count=(count)
+      @meta_tag_count = count
+    end
+
+    def meta_tag_count
+      @meta_tag_count ||= 5
+    end
+
+    def generate_word
+      ("a".."z").to_a.sample(8).join
+    end
+
+    def generate_sentence
+      Array.new((5..15).to_a.sample) { generate_word }.join(" ")
+    end
+
+    def generate_path
+      Array.new((1..3).to_a.sample) { generate_word }.join("/")
+    end
+
+    def content
+      html = "<!DOCTYPE html><html><head><title>#{generate_word}</title>"
+      meta_tag_count.times do
+        html += "<meta name=\"#{generate_word}\" content=\"#{generate_word}\" />"
+      end
+
+      html += "</head><body>"
+
+      50.times do
+        tagname = ["p", "h1", "h2", "h3", "h4", "h5", "h6"].sample
+        html += "<#{tagname}>#{generate_sentence}</#{tagname}>"
+      end
+
+      html += "</body></html>"
+      html
+    end
+  end
+
+  Vore::Crawler.prepend(Vore::TestHelper)
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.8"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,15 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.4.0
 platform: arm64-darwin
 authors:
 - Garen J. Torikian
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-07-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: selma
   requirement: !ruby/object:Gem::Requirement
@@ -47,6 +62,7 @@ files:
 - lib/vore/crawler.rb
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
+- lib/vore/minitest_helper.rb
 - lib/vore/page.rb
 - lib/vore/page_data.rb
 - lib/vore/version.rb