vore 0.2.8-x86_64-linux → 0.4.0-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 322dfb93cbfeae7bdf008ed00e0c7ac67affa00d787bd18fc483c3fe583c9215
-  data.tar.gz: d96c5a555ae20af0c4c967539256a528b8914c49ffe3322fc286d3f7684c1355
+  metadata.gz: 8bf0b076a7ec664aaf3b89dcc04ef818c0e001471e0030203ab740e43cf1f234
+  data.tar.gz: 86042da1109b675a3a0ed0aff3b59dc48bb2a066ee809cd9ab5535e9d137ed4b
 SHA512:
-  metadata.gz: bc438087f76029e4af3a9a28f25ac35800b72022299c63ecb57b3ddfccc96e17c625cad4a02d7e7209d7ac2123efa235163eb5bfa74f75d7105592c83dbc32db
-  data.tar.gz: 2874256c99d57c2269e04b604e716c00f3eba0bb7c287f11a2b96f5f0a58bd812a9d1740da419839c0e1000e96bea36c8f4e4bc130700ddb07293c3c9ecb26e5
+  metadata.gz: bbf8c117fe9ccea10de4048f279b8e95feb6e31ba99af751dfd19eb7a9dfab60a46ce1aad7801824891211dad07c6eb5f53fc7b332ef26758f9be71c3cd02a74
+  data.tar.gz: b8d4220844655b5fdd7ab033dcad91d76c9872629f1555542d6a6d2e5f706ea50fe32b70ec38587ed8147858f60e9c9bfb26768067dacd590a44d8732c5a2811

data/Cargo.lock

@@ -1786,9 +1786,9 @@ dependencies = [
 
 [[package]]
 name = "spider"
-version = "1.99.5"
+version = "1.99.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f62dc0e4f32d36a931471a1694d5b5b29e916c537121ca8028742c9acbe510d"
+checksum = "525670cdc6aec8f4cb91da17ce0255050e89eb7c889272216d8a4fb644d67530"
 dependencies = [
  "ahash",
  "bytes",
@@ -1817,9 +1817,9 @@ dependencies = [
 
 [[package]]
 name = "spider_cli"
-version = "1.99.5"
+version = "1.99.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6031d46576f5fdba52d5c054a8c69ba7ee17cc824563b764fce4c8471c15c3a1"
+checksum = "9bd9d95178dc0715608d5f28501c8321de3e14b40046c6584a12ffce96f0a676"
 dependencies = [
  "clap",
  "env_logger",

data/README.md

@@ -18,13 +18,75 @@ If bundler is not being used to manage dependencies, install the gem by executin
 ## Usage
 
 ```ruby
-    crawler = Vore::Crawler.new
-    crawler.scrape_each_page("https://choosealicense.com") do |page|
-      puts page
-    end
+crawler = Vore::Crawler.new
+crawler.scrape_each_page("https://choosealicense.com") do |page|
+  puts page
+end
 ```
 
-Each `page` is simply every text node. The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), so you know it's fast.
+Each `page` is a simple class consisting of the following values:
+
+* `content`: the text of the HTML document, sans tags
+* `title`: the title of the HTML document (if any)
+* `meta`: the document's meta tags (if any)
+* `path`: the document's path
+
+The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), so you know it's fast.
+
+### Configuration
+
+| Name                          | Description                                                                                                                                 | Default |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| `delay`                       | A value (in milliseconds) which introduces an artifical delay when crawling. Useful for situations where there's rate limiting involved. | `0` |
+| `output_dir`                  | Where the resulting HTML files are stored. | `"tmp/vore"`    |
+| `delete_after_yield`          | Whether the downloaded HTML files are deleted after the yield block finishes. | `true` |
+| `log_level`                   | The logging level. | `:warn` |
+
+### In tests
+
+Since the actual HTTP calls occur in a separate process, Vore will not integrate with libraries like VCR or Webmock by default. You'll need to `require "vore/minitest_helper"` to get a function that emulates the HTTP `GET` requests in a way Ruby can interpret.
+
+Based on your needs, you can overwrite any of the existing methods to suit your application's needs. For example, if you prefer HTML to be generated by Faker, you can create and require a file that looks like the following:
+
+```ruby
+
+require "vore/minitest_helper"
+
+module Vore
+  module TestHelperExtension
+    DOCUMENT_TITLES = [
+      "Hello, I need help",
+      "I need to update my payment information",
+    ]
+    DOCUMENT_CONTENT = [
+      "Hey, I'm having trouble with my computer. Can you help me?",
+      # v--- always creates three page chunks
+      "I need to update my payment information. Like, now. Right now. Now. Can you help me? Please? Now?" + "Can you help me? Please? Now?" * 100,
+    ]
+
+    def content
+      @counter = -1 unless defined?(@counter)
+      @counter += 1
+
+      html = "<!DOCTYPE html><html><head><title>#{DOCUMENT_TITLES[@counter]}</title>"
+
+      meta_tag_count.times do # arbitrarily set to 5
+        html += "<meta name=\"#{Faker::Lorem.word}\" content=\"#{Faker::Lorem.word}\" />"
+      end
+
+      html += "</head><body>"
+
+      html += "<p>#{DOCUMENT_CONTENT[@counter]}</p>"
+
+      html += "</body></html>"
+
+      html
+    end
+  end
+
+  Vore::TestHelper.prepend(Vore::TestHelperExtension)
+end
+```
 
 ## Development
 

data/lib/vore/configuration.rb

@@ -1,9 +1,16 @@
 # frozen_string_literal: true
 
-module Vole
+module Vore
   class Configuration
     DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
       allow_doctype: false,
     })
+
+    DEFAULT_OPTIONS = {
+      delay: 0,
+      output_dir: "tmp/vore",
+      delete_after_yield: true,
+      log_level: :warn,
+    }
   end
 end

data/lib/vore/crawler.rb

@@ -2,6 +2,8 @@
 
 require_relative "handlers/content_extractor"
 
+require "listen"
+
 module Vore
   # This is the class that starts and controls the crawling
   class Crawler
@@ -12,14 +14,22 @@ module Vore
 
     # Creates a crawler
     # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
-    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
-      @denylist_regexp = Regexp.union(denylist)
-
-      @content_extractor = Vole::Handlers::ContentExtractor.new
+    def initialize(sanitization_config: Vore::Configuration::DEFAULT_SANITIZATION_CONFIG, options: {})
+      @content_extractor = Vore::Handlers::ContentExtractor.new
       @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
       ext = PLATFORM.include?("windows") ? ".exe" : ""
       @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
-      @parent_output_dir = "tmp/vore"
+      @options = Vore::Configuration::DEFAULT_OPTIONS.merge(options)
+      @parent_output_dir = @options[:output_dir]
+      @parent_output_dir_len = @parent_output_dir.to_s.split(FILE_SEPERATOR).size
+
+      Vore.logger.level = @options[:log_level]
+      Listen.logger = Vore.logger
+
+      @results = {
+        pages_visited: 0,
+        unprocessed_pages: [],
+      }
 
       return if File.exist?(@executable)
 
@@ -29,71 +39,82 @@ module Vore
 
     def scrape_each_page(website, &block)
       @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
-      Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
-
-      output = run_command(website, @output_dir)
+      FileUtils.rm_rf(@output_dir)
+      FileUtils.mkdir_p(@output_dir)
+
+      listener = Listen.to(@output_dir) do |_modified, added, _removed|
+        if added.any?
+          added.each do |path|
+            process_file(path, &block)
+            File.delete(path) if @options[:delete_after_yield]
+          end
+        end
+      end
+      listener.start
 
-      Vore.logger.info("Vore finished crawling #{website}: #{output}")
+      Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
 
-      results = {
-        pages_visited: 0,
-        pages_unprocessed: 0,
-        unprocessed_pages: [],
-      }
+      begin
+        run_command(website, delay: @options[:delay])
+      ensure
+        sleep(0.5) # give listener time to clean up
+        listener.stop
+      end
 
-      Dir.glob(File.join(output_dir, "**", "*")).each do |path|
-        next unless File.file?(path)
+      Vore.logger.info("Vore finished crawling #{website}")
 
-        results[:pages_visited] += 1
+      @results
+    end
 
-        html_file = File.read(path).force_encoding("UTF-8")
-        rewritten_html_file = ""
+    def process_file(path, &block)
+      @results[:pages_visited] += 1
 
-        if html_file.empty?
-          results[:pages_unprocessed] += 1
-          results[:unprocessed_pages] << path
-          next
-        end
+      html_file = File.read(path).force_encoding("UTF-8")
+      rewritten_html_file = ""
 
-        begin
-          rewritten_html_file = @selma.rewrite(html_file)
-        rescue StandardError => e
-          Vore.logger.warn("Error rewriting #{path}: #{e}")
-          results[:pages_unprocessed] += 1
-          next
-        end
+      if html_file.empty?
+        @results[:unprocessed_pages] << path
+        return
+      end
 
-        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
-        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+      begin
+        rewritten_html_file = @selma.rewrite(html_file)
+      rescue StandardError => e
+        Vore.logger.warn("Error rewriting #{path}: #{e}")
+        @results[:unprocessed_pages] << path
+        return
+      end
 
-        page = Vore::PageData.new(
-          content: rewritten_html_file,
-          title: @content_extractor.title,
-          meta: @content_extractor.meta,
-          path: url_path,
-        )
+      # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+      url_path = path.split(FILE_SEPERATOR)[(@parent_output_dir_len + 1)..].join("/")
 
-        yield page
-      ensure
-        File.delete(path) if File.file?(path)
-      end
+      page = Vore::PageData.new(
+        content: rewritten_html_file,
+        title: @content_extractor.title,
+        meta: @content_extractor.meta,
+        path: url_path,
+      )
 
-      results
+      yield page
     end
 
-    # def crawl(site, block)
-    #   Vore.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
-    #   crawl_site(site)
-    # end
-
-    def run_command(website, output_dir)
-      %x(#{@executable} \
-        --user-agent #{user_agent} \
-        --delay 3500 \
-        --url #{website} \
-        download \
-        -t \
-        #{output_dir})
+    def run_command(website, delay: 0)
+      pid = Process.spawn(
+        @executable,
+        "--user-agent",
+        user_agent,
+        "--delay",
+        delay.to_s,
+        "--url",
+        website,
+        "download",
+        "-t",
+        @output_dir,
+      )
+
+      _, _status = Process.waitpid2(pid)
+    rescue StandardError => e
+      Vore.logger.error(e)
     end
 
     def user_agent

data/lib/vore/handlers/content_extractor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Vole
+module Vore
   module Handlers
     class ContentExtractor
       SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")

data/lib/vore/logger.rb

@@ -23,6 +23,10 @@ module Vore
           instance
         end
       end
+
+      def level=(level)
+        instance.level = level
+      end
     end
   end
 end

data/lib/vore/minitest_helper.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "minitest/mock"
+require "net/http"
+
+module Vore
+  module TestHelper
+    def run_command(website, **options)
+      loop_times.times do |time|
+        net_http = ::Minitest::Mock.new
+        response = ::Minitest::Mock.new
+        response.expect(:is_a?, true, [::Net::HTTPSuccess])
+
+        # we need to trigger an HTTP call to pretend that we're making
+        # an external request. this way, the gem hooks into VCR/Webmock
+        net_http.expect(:get, response)
+        html = content
+        response.expect(:body, html)
+        time_s = time.to_s
+        uri = URI("#{website}/#{time_s}")
+        Net::HTTP.get(uri)
+
+        file = File.join(@output_dir, time_s)
+        File.write("#{file}.html", html)
+      end
+    end
+
+    def loop_times=(times)
+      @loop_times = times
+    end
+
+    def loop_times
+      @loop_times ||= 5
+    end
+
+    def meta_tag_count=(count)
+      @meta_tag_count = count
+    end
+
+    def meta_tag_count
+      @meta_tag_count ||= 5
+    end
+
+    def generate_word
+      ("a".."z").to_a.sample(8).join
+    end
+
+    def generate_sentence
+      Array.new((5..15).to_a.sample) { generate_word }.join(" ")
+    end
+
+    def generate_path
+      Array.new((1..3).to_a.sample) { generate_word }.join("/")
+    end
+
+    def content
+      html = "<!DOCTYPE html><html><head><title>#{generate_word}</title>"
+      meta_tag_count.times do
+        html += "<meta name=\"#{generate_word}\" content=\"#{generate_word}\" />"
+      end
+
+      html += "</head><body>"
+
+      50.times do
+        tagname = ["p", "h1", "h2", "h3", "h4", "h5", "h6"].sample
+        html += "<#{tagname}>#{generate_sentence}</#{tagname}>"
+      end
+
+      html += "</body></html>"
+      html
+    end
+  end
+
+  Vore::Crawler.prepend(Vore::TestHelper)
+end

data/lib/vore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vore
-  VERSION = "0.2.8"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,15 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: vore
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.4.0
 platform: x86_64-linux
 authors:
 - Garen J. Torikian
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-07-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  force_ruby_platform: false
 - !ruby/object:Gem::Dependency
   name: selma
   requirement: !ruby/object:Gem::Requirement
@@ -47,6 +62,7 @@ files:
 - lib/vore/crawler.rb
 - lib/vore/handlers/content_extractor.rb
 - lib/vore/logger.rb
+- lib/vore/minitest_helper.rb
 - lib/vore/page.rb
 - lib/vore/page_data.rb
 - lib/vore/version.rb