vore 0.2.0-arm64-linux

data/Cargo.toml

@@ -0,0 +1,7 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/vore"]
+resolver = "2"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Garen J. Torikian
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Vore
+
+![Vore, by LewdBacon](https://github.com/user-attachments/assets/0923cc84-4cca-4d95-8a0e-4dad650525d2)
+
+Vore quickly crawls websites and spits out text sans tags. It's written in Ruby and powered by Rust.
+
+## Installation
+
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add vore
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install vore
+
+## Usage
+
+```ruby
+    crawler = Vore::Crawler.new
+    crawler.scrape_each_page("https://choosealicense.com") do |page|
+      puts page
+    end
+```
+
+Each `page` is simply every text node. The scraping is managed by [`spider-rs`](https://github.com/spider-rs/spider), so you know it's fast.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/gjtorikian/vore.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/ext/vore/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "vore"
+version = "0.1.0"
+edition = "2021"
+authors = ["Garen J. Torikian <gjtorikian@users.noreply.github.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.7", features = ["rb-sys"] }
+rb-sys = { version = "*", default-features = false, features = [
+    "stable-api-compiled-fallback",
+] }
+spider_cli = { version = "1.99" }

data/ext/vore/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("vore/vore")

data/ext/vore/src/lib.rs

@@ -0,0 +1,13 @@
+use magnus::{Error, Ruby};
+
+pub mod page;
+pub mod website;
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let m_vore = ruby.define_module("Vore")?;
+    website::init(m_vore).expect("cannot define Vore::Website class");
+    page::init(m_vore).expect("cannot define Vore::Page class");
+
+    Ok(())
+}

data/ext/vore/src/page.rs

@@ -0,0 +1,9 @@
+use magnus::{prelude::*, RModule};
+
+pub fn init(m_vore: RModule) -> Result<(), magnus::Error> {
+    let c_page = m_vore
+        .define_class("Page", magnus::class::object())
+        .expect("cannot define class Vore::Page");
+
+    Ok(())
+}

data/ext/vore/src/website.rs

@@ -0,0 +1,40 @@
+use std::{borrow::BorrowMut, cell::RefCell};
+
+use magnus::{function, prelude::*, scan_args, Error, RHash, RModule, Ruby, Value};
+
+#[derive(Clone)]
+pub struct Website {
+    website: String,
+}
+
+#[derive(Clone)]
+#[magnus::wrap(class = "Vore::Sanitizer")]
+pub struct VoreWebsite(RefCell<Website>);
+
+impl VoreWebsite {
+    pub fn new(arguments: &[Value]) -> Result<Self, magnus::Error> {
+        let args = scan_args::scan_args::<(String,), (), (), (), (), ()>(arguments)?;
+        let (website,): (String,) = args.required;
+
+        Ok(Self(RefCell::new(Website { website: website })))
+    }
+
+    // pub async fn scrape(&self) -> Result<(), magnus::Error> {
+    //     let mut binding = self.0.borrow_mut();
+
+    //     binding.website.scrape().await;
+
+    //     Ok(())
+    // }
+}
+
+pub fn init(m_vore: RModule) -> Result<(), magnus::Error> {
+    let c_website = m_vore
+        .define_class("Website", magnus::class::object())
+        .expect("cannot define class Vore::Website");
+
+    c_website.define_singleton_method("new", function!(VoreWebsite::new, -1))?;
+    // c_website.define_method("scrape!", function!(VoreWebsite::scrape, 1))?;
+
+    Ok(())
+}

data/lib/vore/configuration.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Vole
+  class Configuration
+    DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
+      allow_doctype: false,
+    })
+  end
+end

data/lib/vore/crawler.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative "handlers/content_extractor"
+
+module Vore
+  # This is the class that starts and controls the crawling
+  class Crawler
+    PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
+    FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
+
+    # Creates a crawler
+    # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
+    def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
+      @denylist_regexp = Regexp.union(denylist)
+
+      @content_extractor = Vole::Handlers::ContentExtractor.new
+      @selma = Selma::Rewriter.new(sanitizer: Selma::Sanitizer.new(sanitization_config), handlers: [@content_extractor])
+      ext = PLATFORM.include?("windows") ? ".exe" : ""
+      @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
+      @output_dir = "tmp/vore"
+
+      return if File.exist?(@executable)
+
+      warn("ERROR: Unsupported platform: `#{PLATFORM}`")
+      exit(1)
+    end
+
+    def scrape_each_page(website, &block)
+      output_dir = "#{@output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
+      Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
+
+      output = %x(#{@executable} \
+        --user-agent #{user_agent} \
+        --url #{website} \
+        download \
+        -t \
+        #{output_dir})
+
+      Vore.logger.info("Vore finished crawling #{website}: #{output}")
+
+      Dir.glob("tmp/**/*").each do |path|
+        next unless File.file?(path)
+
+        html_file = File.read(path).force_encoding("UTF-8")
+        rewritten_html_file = @selma.rewrite(html_file)
+
+        # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
+        url_path = path.split(FILE_SEPERATOR)[3..].join("/")
+
+        page = Vore::PageData.new(
+          content: rewritten_html_file,
+          title: @content_extractor.title,
+          meta: @content_extractor.meta,
+          path: url_path,
+        )
+
+        yield page
+      ensure
+        File.delete(path) if File.file?(path)
+      end
+    end
+
+    # def crawl(site, block)
+    #   Vore.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
+    #   crawl_site(site)
+    # end
+
+    def user_agent
+      "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
+    end
+  end
+end

data/lib/vore/handlers/content_extractor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Vole
+  module Handlers
+    class ContentExtractor
+      SELECTOR = Selma::Selector.new(match_element: "*", match_text_within: "title")
+
+      attr_reader :title, :meta
+
+      def initialize
+        super
+        @title = ""
+        @meta = {}
+        @within_title = false
+      end
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        if element.tag_name == "pre" || element.tag_name == "code" || element.tag_name == "script" || element.tag_name == "form"
+          element.remove
+        elsif element.tag_name == "title"
+          @within_title = true
+          element.remove
+        elsif element.tag_name == "meta"
+          return if element.attributes["name"].nil?
+
+          @meta[element.attributes["name"]] = element.attributes["content"]
+        else
+          element.remove_and_keep_content
+        end
+      end
+
+      def handle_text_chunk(text)
+        if @within_title
+          @within_title = false
+          @title = text.to_s
+        end
+      end
+    end
+  end
+end

data/lib/vore/logger.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "logger"
+require "forwardable"
+
+module Vore
+  class << self
+    attr_accessor :logger
+  end
+
+  class Logger
+    class << self
+      extend Forwardable
+      delegate [:debug, :info, :warn, :error] => :instance
+
+      attr_writer :instance
+
+      def instance
+        @instance ||= begin
+          $stdout.sync = true
+          instance = ::Logger.new($stdout)
+          instance.level = ::Logger::DEBUG
+          instance
+        end
+      end
+    end
+  end
+end
+
+Vore.logger = Vore::Logger

data/lib/vore/page.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Vore
+  class Page
+  end
+end

data/lib/vore/page_data.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Vore
+  class PageData
+    attr_reader :title, :meta, :content, :path
+
+    def initialize(title:, meta:, content:, path:)
+      @title = title
+      @meta = meta
+      @content = content
+      @path = path
+    end
+  end
+end

data/lib/vore/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Vore
+  VERSION = "0.2.0"
+end

data/lib/vore/website.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Vore
+  class Website
+  end
+end

data/lib/vore.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+if ENV.fetch("DEBUG", false)
+  begin
+    require "debug"
+    require "amazing_print"
+  rescue LoadError # rubocop:disable Lint/SuppressedException
+  end
+end
+
+require "selma"
+
+require_relative "vore/version"
+require_relative "vore/configuration"
+require_relative "vore/logger"
+require_relative "vore/crawler"
+require_relative "vore/page"
+require_relative "vore/page_data"
+require_relative "vore/website"
+
+module Vore
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: vore
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: arm64-linux
+authors:
+- Garen J. Torikian
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: selma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  force_ruby_platform: false
+description: 
+email:
+- gjtorikian@users.noreply.github.com
+executables: []
+extensions:
+- ext/vore/Cargo.toml
+extra_rdoc_files: []
+files:
+- Cargo.lock
+- Cargo.toml
+- LICENSE.txt
+- README.md
+- exe/vore-spider
+- ext/vore/Cargo.toml
+- ext/vore/extconf.rb
+- ext/vore/src/lib.rs
+- ext/vore/src/page.rs
+- ext/vore/src/website.rs
+- lib/vore.rb
+- lib/vore/configuration.rb
+- lib/vore/crawler.rb
+- lib/vore/handlers/content_extractor.rb
+- lib/vore/logger.rb
+- lib/vore/page.rb
+- lib/vore/page_data.rb
+- lib/vore/version.rb
+- lib/vore/website.rb
+homepage: https://github.com/gjtorikian/vore
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/gjtorikian/vore
+  source_code_uri: https://github.com/gjtorikian/vore
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.22
+requirements: []
+rubygems_version: 3.5.3
+signing_key: 
+specification_version: 4
+summary: Quickly crawls websites and spits out text sans tags. Powered by Rust.
+test_files: []