arachnid2 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 612ad5683bd7211e9be13fd1be854875d9ea54cd4e96680298185f3fe5a474cb
+  data.tar.gz: da911439fbd3d4be7741cb34a44b2adeeecf8fe920981f10621e992e203ec1be
+SHA512:
+  metadata.gz: 4a27c953917e51b2541ad9bdc3639d2273a7b4541933f9eb7151e6c0f684da366f4a17e366f25e4872013d5d22bd033f835d5d423bafbc085082d77cb968c104
+  data.tar.gz: f6fa3992a716a2cb2fa6cee3420629cd21069648e086846d5cd190a7fa3523e24a8adda0b7c779ecbe36b1d9effdde2903d8c07e5bcb4b9bc3f85cd1c4b9c987

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+arachnid2-*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at samuel.nissen@rakuten.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in arachnid2.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,56 @@
+PATH
+  remote: .
+  specs:
+    arachnid2 (0.1.0)
+      addressable
+      adomain
+      bloomfilter-rb
+      nokogiri
+      typhoeus
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    adomain (0.1.1)
+      addressable (~> 2.5)
+    bloomfilter-rb (2.1.1)
+      redis
+    diff-lcs (1.3)
+    ethon (0.11.0)
+      ffi (>= 1.3.0)
+    ffi (1.9.23)
+    mini_portile2 (2.3.0)
+    nokogiri (1.8.2)
+      mini_portile2 (~> 2.3.0)
+    public_suffix (3.0.2)
+    rake (10.5.0)
+    redis (4.0.1)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    typhoeus (1.3.0)
+      ethon (>= 0.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  arachnid2!
+  bundler (~> 1.16)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Sam Nissen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,139 @@
+# Arachnid2
+
+## About
+
+Arachnid2 is a simple, fast web-crawler written in Ruby.
+It uses [typhoeus](https://github.com/typhoeus/typhoeus)
+to get HTTP requests,
+[bloomfilter-rb](https://github.com/igrigorik/bloomfilter-rb)
+to store the URLs it will get and has gotten,
+and [nokogiri](https://github.com/sparklemotion/nokogiri)
+to find the URLs on each webpage.
+
+Arachnid2 is a successor to [Arachnid](https://github.com/dchuk/Arachnid),
+and was abstracted out of the [Tellurion Bot](https://github.com/samnissen/tellurion_bot).
+
+## Usage
+
+The basic use of Arachnid2 is surfacing the responses from a domains'
+URLs by visiting a URL, collecting any links to the same domain
+on that page, and visiting those to do the same.
+
+Hence, the simplest output would be to collect all of the responses
+while spidering from some URL.
+
+```ruby
+require "arachnid2"
+
+url = "http://www.maximumfun.org"
+spider = Arachnid2.new(url)
+responses = []
+
+spider.crawl { |response|
+  responses << response
+}
+```
+
+Obviously this could become unwieldy,
+so you can execute logic within the spidering to collect a narrow subset
+of the responses, transform or dissect the response,
+or both (or whatever you want).
+
+```ruby
+require "arachnid2"
+require "nokogiri"
+
+url = "https://daringfireball.net"
+spider = Arachnid2.new(url)
+responses = []
+
+spider.crawl { |response|
+  responses << Nokogiri::HTML(response.body) if response.effective_url =~ /.*amazon.*/
+  print '*'
+}
+```
+
+`Arachnid2#crawl` will return always `nil`.
+
+### Options
+
+```ruby
+require "arachnid2"
+
+url = "http://sixcolours.com"
+spider = Arachnid2.new(url)
+opts = {
+  time_box: 60,
+  max_urls: 50,
+  language: "en-UK",
+  user_agent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+}
+responses = []
+
+spider.crawl(opts) { |response|
+  responses << response
+}
+```
+
+#### `time_box`
+
+The crawler will time-bound your spidering. If no valid integer is provided,
+it will crawl for 15 seconds before exiting. 600 seconds (10 minutes)
+is the current maximum, and any value above it will be reduced to 600.
+
+#### `max_urls`
+
+The crawler will crawl a limited number of URLs before stopping.
+If no valid integer is provided, it will crawl for 50 URLs before exiting.
+10000 seconds is the current maximum,
+and any value above it will be reduced to 10000.
+
+#### `language`
+
+The language is a string mapped to the HTTP header Accept-Language. The
+default is
+`en-IE, en-UK;q=0.9, en-NL;q=0.8, en-MT;q=0.7, en-LU;q=0.6, en;q=0.5, \*;0.4`
+
+#### `user_agent`
+
+This user agent is a string mapped to the HTTP header User-Agent. The
+default is
+`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15`
+
+### Memory use in Docker
+
+In case you are operating the crawler within a container, Arachnid2
+will attempt to prevent the container from running out of memory,
+and crawls will end when the container has <= 20% of its available memory
+free.
+
+### Non-HTML links
+
+The crawler attempts to stop itself from returning data from
+links that are not indicative of HTML, as detailed in
+`Arachnid2::NON_HTML_EXTENSIONS`.
+
+## Development
+
+TODO: this
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/samnissen/arachnid2.
+This project is intended to be a safe,
+welcoming space for collaboration,
+and contributors are expected to adhere to the
+[Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Arachnid2 project’s codebases,
+issue trackers, chat rooms and mailing lists is expected
+to follow the
+[code of conduct](https://github.com/samnissen/arachnid2/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/arachnid2.gemspec

@@ -0,0 +1,33 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "arachnid2/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "arachnid2"
+  spec.version       = Arachnid2::VERSION
+  spec.authors       = ["Sam Nissen"]
+  spec.email         = ["scnissen@gmail.com"]
+
+  spec.summary       = %q{A simple, fast web crawler}
+  # spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "https://github.com/samnissen/arachnid2"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency "typhoeus"
+  spec.add_dependency "bloomfilter-rb"
+  spec.add_dependency "adomain"
+  spec.add_dependency "addressable"
+  spec.add_dependency "nokogiri"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "arachnid2"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/arachnid2/version.rb

@@ -0,0 +1,3 @@
+class Arachnid2
+  VERSION = "0.1.0"
+end

data/lib/arachnid2.rb

@@ -0,0 +1,237 @@
+require "arachnid2/version"
+
+require 'tempfile'
+require "typhoeus"
+require "bloomfilter-rb"
+require "adomain"
+require "addressable/uri"
+require "nokogiri"
+
+class Arachnid2
+
+  # META:
+  #   About the origins of this crawling approach
+  # The Crawler is heavily borrowed from by Arachnid.
+  # Original: https://github.com/dchuk/Arachnid
+  # Other iterations I've borrowed liberally from:
+  #   - https://github.com/matstc/Arachnid
+  #   - https://github.com/intrigueio/Arachnid
+  #   - https://github.com/jhulme/Arachnid
+  # And this was originally written as a part of Tellurion's bot
+  # https://github.com/samnissen/tellurion_bot
+
+  MAX_CRAWL_TIME = 600
+  BASE_CRAWL_TIME = 15
+  MAX_URLS = 10000
+  BASE_URLS = 50
+  DEFAULT_LANGUAGE = "en-IE, en-UK;q=0.9, en-NL;q=0.8, en-MT;q=0.7, en-LU;q=0.6, en;q=0.5, *;0.4"
+  DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15"
+
+  NON_HTML_EXTENSIONS = {
+    3 => ['.gz'],
+    4 => ['.jpg', '.png', '.m4a', '.mp3', '.mp4', '.pdf', '.zip',
+          '.wmv', '.gif', '.doc', '.xls', '.pps', '.ppt', '.tar',
+          '.iso', '.dmg', '.bin', '.ics', '.exe', '.wav', '.mid'],
+    5 => ['.xlsx', '.docx', '.pptx', '.tiff', '.zipx'],
+    8 => ['.torrent']
+  }
+  MEMORY_USE_FILE = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
+  MEMORY_LIMIT_FILE = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+  MAXIMUM_LOAD_RATE = 79.9
+
+  #
+  # Creates the object to execute the crawl
+  #
+  # @example
+  #   url = "https://daringfireball.net"
+  #   spider = Arachnid2.new(url)
+  #
+  # @param [String] url
+  #
+  # @return [Arachnid2] self
+  #
+  def initialize(url)
+    @url = url
+    @domain = Adomain[@url]
+  end
+
+  #
+  # Visits a URL, gathering links and visiting them,
+  # until running out of time, memory or attempts.
+  #
+  # @example
+  #   url = "https://daringfireball.net"
+  #   spider = Arachnid2.new(url)
+  #
+  #   opts = {
+  #     :time_box => 30,
+  #     :language => "es-IO",
+  #     :user_agent => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+  #   }
+  #   responses = []
+  #   spider.crawl(opts) { |response|
+  #     responses << response
+  #   }
+  #
+  # @param [Hash] opts
+  #
+  # @return nil
+  #
+  def crawl(opts = {})
+    preflight(opts)
+
+    until @global_queue.empty?
+      @global_queue.size.times do
+        begin
+          q = @global_queue.shift
+
+          break if @global_visited.size >= @crawl_options[:max_urls]
+          break if Time.now > @crawl_options[:time_limit]
+          break if memory_danger?
+
+          @global_visited.insert(q)
+
+          request = Typhoeus::Request.new(q, request_options)
+
+          request.on_complete do |response|
+            links = process(response)
+            next unless links
+
+            yield response
+
+            vacuum(links, response)
+          end
+
+          request.run
+        ensure
+          @cookie_file.close! if @cookie_file
+        end
+      end
+    end
+  end
+
+  private
+    def process(response)
+      return false unless Adomain["#{response.effective_url}"].include? @domain
+
+      elements = Nokogiri::HTML.parse(response.body).css('a')
+      return elements.map {|link| link.attribute('href').to_s}.uniq.sort.delete_if {|href| href.empty? }
+    end
+
+    def vacuum(links, response)
+      links.each do |link|
+        next if link.match(/^\(|^javascript:|^mailto:|^#|^\s*$|^about:/)
+
+        begin
+          absolute_link = make_absolute(link, response.effective_url)
+
+          next if skip_link?(absolute_link)
+
+          @global_queue << absolute_link
+        rescue Addressable::URI::InvalidURIError
+        end
+      end
+    end
+
+    def skip_link?(absolute_link)
+      internal  = internal_link?(absolute_link)
+      visited   = @global_visited.include?(absolute_link)
+      ignored   = extension_ignored?(absolute_link)
+      known     = @global_queue.include?(absolute_link)
+
+      !internal || visited || ignored || known
+    end
+
+    def preflight(opts)
+      @options = opts
+      @crawl_options = crawl_options
+      # TODO: write looping to take advantage of Hydra
+      # @hydra = Typhoeus::Hydra.new(:max_concurrency => 1)
+      @global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => true)
+      @global_queue = [@url]
+    end
+
+    def bound_time
+      boundary = "#{@options[:time_box]}".to_i
+      boundary = BASE_CRAWL_TIME if boundary <= 0
+      boundary = MAX_CRAWL_TIME  if boundary >  MAX_CRAWL_TIME
+
+      return Time.now + boundary
+    end
+
+    def bound_urls
+      amount = "#{@options[:max_urls]}".to_i
+      amount = BASE_URLS if amount <= 0
+      amount = MAX_URLS  if amount >  MAX_URLS
+
+      amount
+    end
+
+    def request_options
+      @cookie_file ||= Tempfile.new('cookies')
+
+      @request_options = {
+        timeout: 10000,
+        followlocation: true,
+        cookiefile: @cookie_file.path,
+        cookiejar: @cookie_file.path,
+        headers: {
+          'Accept-Language' => "#{language}",
+          'User-Agent' => "#{user_agent}"
+        }
+      }
+
+      @request_options
+    end
+
+    def language
+      @options[:language] || DEFAULT_LANGUAGE
+    end
+
+    def user_agent
+      @options[:user_agent] || DEFAULT_USER_AGENT
+    end
+
+    def crawl_options
+      { :max_urls => max_urls, :time_limit => time_limit }
+    end
+
+    def max_urls
+      bound_urls
+    end
+
+    def time_limit
+      bound_time
+    end
+
+    def make_absolute(href, root)
+      Addressable::URI.parse(root).join(Addressable::URI.parse(href)).to_s
+    end
+
+    def internal_link?(absolute_url)
+      Adomain[absolute_url].include? @domain
+    end
+
+    def extension_ignored?(url)
+      return false if url.empty?
+
+      !NON_HTML_EXTENSIONS.values.flatten.find { |e| url.downcase.end_with? e.downcase }.nil?
+    end
+
+    def memory_danger?
+      return false unless in_docker?
+
+      use      = "#{File.open(MEMORY_USE_FILE, "rb").read}".to_f
+      @limit ||= "#{File.open(MEMORY_LIMIT_FILE, "rb").read}".to_f
+
+      return false unless ( (use > 0.0) && (@limit > 0.0) )
+
+      return ( ( (use / @limit) * 100.0 ) >= MAXIMUM_LOAD_RATE )
+    end
+
+    def in_docker?
+      return false unless File.file?(MEMORY_USE_FILE)
+      true
+    end
+
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: arachnid2
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sam Nissen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-05-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: adomain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- scnissen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- arachnid2.gemspec
+- bin/console
+- bin/setup
+- lib/arachnid2.rb
+- lib/arachnid2/version.rb
+homepage: https://github.com/samnissen/arachnid2
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: A simple, fast web crawler
+test_files: []