RubyGems - medusa-crawler - Versions diffs - 1.0.0.pre.1 - Mend

medusa-crawler 1.0.0.pre.1

Files changed (35) hide show

checksums.yaml +7 -0
checksums.yaml.gz.sig +0 -0
data.tar.gz.sig +0 -0
data/CHANGELOG.md +20 -0
data/CONTRIBUTORS.md +22 -0
data/LICENSE.txt +20 -0
data/README.md +48 -0
data/Rakefile +24 -0
data/VERSION +1 -0
data/bin/medusa +4 -0
data/lib/medusa.rb +2 -0
data/lib/medusa/cli.rb +24 -0
data/lib/medusa/cli/count.rb +22 -0
data/lib/medusa/cli/cron.rb +90 -0
data/lib/medusa/cli/pagedepth.rb +32 -0
data/lib/medusa/cli/serialize.rb +35 -0
data/lib/medusa/cli/url_list.rb +41 -0
data/lib/medusa/cookie_store.rb +35 -0
data/lib/medusa/core.rb +305 -0
data/lib/medusa/exceptions.rb +5 -0
data/lib/medusa/http.rb +202 -0
data/lib/medusa/page.rb +229 -0
data/lib/medusa/page_store.rb +160 -0
data/lib/medusa/storage.rb +8 -0
data/lib/medusa/storage/base.rb +81 -0
data/lib/medusa/storage/exceptions.rb +15 -0
data/lib/medusa/storage/moneta.rb +42 -0
data/lib/medusa/tentacle.rb +39 -0
data/lib/medusa/version.rb +3 -0
data/spec/fakeweb_helper.rb +85 -0
data/spec/medusa_helper.rb +5 -0
data/spec/medusa_spec.rb +14 -0
data/spec/spec_helper.rb +104 -0
metadata +187 -0
metadata.gz.sig +0 -0

data/lib/medusa/storage/exceptions.rb ADDED

@@ -0,0 +1,15 @@
+module Medusa
+  module Storage
+    class GenericError < Error; end;
+    class ConnectionError < Error; end
+    class RetrievalError < Error; end
+    class InsertionError < Error; end
+    class CloseError < Error; end
+  end
+end

data/lib/medusa/storage/moneta.rb ADDED

@@ -0,0 +1,42 @@
+require 'moneta'
+require 'forwardable'
+module Medusa
+  module Storage
+    class Moneta
+      extend Forwardable
+      def_delegators :@moneta, :[], :[]=, :delete, :key?, :clear, :close
+      alias has_key? key?
+      def initialize(name, options = {})
+        default_options = { threadsafe: true, prefix: 'medusa' }
+        @moneta = ::Moneta.new(name, default_options.merge(options))
+      end
+      def each
+        @moneta.each_key do |k|
+          yield k, @moneta.fetch(k)
+        end
+        self
+      end
+      def size
+        current_size = @moneta.each_key.size
+        return @moneta.each_key.reduce(0) { |size, k|  size + 1 } if current_size.nil?
+        return current_size
+      end
+      def keys
+        @moneta.each_key.to_a.sort
+      end
+      def merge!(hash)
+        @moneta.merge!(hash) unless hash.empty?
+        self
+      end
+    end
+  end
+end

data/lib/medusa/tentacle.rb ADDED

@@ -0,0 +1,39 @@
+require 'medusa/http'
+module Medusa
+  class Tentacle
+    #
+    # Create a new Tentacle
+    #
+    def initialize(link_queue, page_queue, opts = {})
+      @link_queue = link_queue
+      @page_queue = page_queue
+      @http = Medusa::HTTP.new(opts)
+      @opts = opts
+    end
+    #
+    # Gets links from @link_queue, and returns the fetched
+    # Page objects into @page_queue
+    #
+    def run
+      loop do
+        link, referer, depth = @link_queue.deq
+        break if link == :END
+        @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }
+        delay
+      end
+    end
+    private
+    def delay
+      sleep @opts[:delay] if @opts[:delay] > 0
+    end
+  end
+end

data/lib/medusa/version.rb ADDED

@@ -0,0 +1,3 @@
+module Medusa
+  VERSION = '1.0.0.pre.1'
+end

data/spec/fakeweb_helper.rb ADDED

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+require 'webmock/rspec'
+WebMock.disable_net_connect!
+module Medusa
+  AUTH = ['user', 'pass']
+  SPEC_DOMAIN = "http://www.example.com/"
+  AUTH_SPEC_DOMAIN = "http://#{AUTH.join(':')}@#{URI.parse(SPEC_DOMAIN).host}/"
+  class FakePage
+    attr_accessor :links
+    attr_accessor :hrefs
+    attr_accessor :body
+    def initialize(name = '', options = {})
+      @name = name
+      @links = []
+      @hrefs = []
+      @redirect = nil
+      @auth = false
+      @base = ''
+      @links = [options[:links]].flatten if options.has_key?(:links)
+      @hrefs = [options[:hrefs]].flatten if options.has_key?(:hrefs)
+      @redirect = options[:redirect] if options.has_key?(:redirect)
+      @auth = options[:auth] if options.has_key?(:auth)
+      @base = options[:base] if options.has_key?(:base)
+      @content_type = options[:content_type] || "text/html"
+      @body = options[:body]
+      create_body unless @body
+      add_to_fakeweb
+    end
+    def url
+      SPEC_DOMAIN + @name
+    end
+    def auth_url
+      AUTH_SPEC_DOMAIN + @name
+    end
+    private
+    def create_body
+      if @base
+        @body = "<html><head><base href=\"#{@base}\"></head><body>"
+      else
+        @body = "<html><body>"
+      end
+      @links.each{|l| @body += "<a href=\"#{SPEC_DOMAIN}#{l}\"></a>"} if @links
+      @hrefs.each{|h| @body += "<a href=\"#{h}\"></a>"} if @hrefs
+      @body += "</body></html>"
+    end
+    def add_to_fakeweb
+      options = {body: @body, status: [200, 'OK'], headers: {'Content-Type' => @content_type}}
+      if @redirect
+        options[:status] = [301, 'Moved Permanently']
+        # only prepend SPEC_DOMAIN if a relative url (without an http scheme) was specified
+        redirect_url = (@redirect =~ /http/) ? @redirect : SPEC_DOMAIN + @redirect
+        options[:headers]['Location'] = redirect_url
+        # register the page this one redirects to
+        WebMock.stub_request(:get, redirect_url).to_return(body: '', status: [200, 'OK'], headers: {'Content-Type' => @content_type})
+      end
+      if @auth
+        unautorized_options = {body: 'Unauthorized', status: [401, 'Unauthorized']}
+        WebMock.stub_request(:get, url).to_return(unautorized_options)
+        WebMock.stub_request(:get, url).with(basic_auth: AUTH).to_return(options)
+      else
+        WebMock.stub_request(:get, url).to_return(options)
+      end
+    end
+  end
+end
+#default root
+Medusa::FakePage.new

data/spec/medusa_helper.rb ADDED

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+require 'medusa'
+SPEC_DOMAIN = 'http://www.example.com/'

data/spec/medusa_spec.rb ADDED

@@ -0,0 +1,14 @@
+RSpec.describe Medusa do
+  it "should have a version" do
+    expect(Medusa.const_defined?('VERSION')).to be true
+  end
+  it "should return a Medusa::Core from the crawl, which has a PageStore" do
+    result = Medusa.crawl(SPEC_DOMAIN)
+    expect(result).to be_an_instance_of(Medusa::Core)
+    expect(result.pages).to be_an_instance_of(Medusa::PageStore)
+  end
+end

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+require 'medusa_helper'
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  # config.warnings = true
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+end

metadata ADDED

@@ -0,0 +1,187 @@
+--- !ruby/object:Gem::Specification
+name: medusa-crawler
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre.1
+platform: ruby
+authors:
+- Mauro Asprea
+- Chris Kite
+autorequire:
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIERDCCAqygAwIBAgIBATANBgkqhkiG9w0BAQsFADAmMSQwIgYDVQQDDBttYXVy
+  b2FzcHJlYS9EQz1nbWFpbC9EQz1jb20wHhcNMjAwODA2MTEwNDAzWhcNMjEwODA2
+  MTEwNDAzWjAmMSQwIgYDVQQDDBttYXVyb2FzcHJlYS9EQz1nbWFpbC9EQz1jb20w
+  ggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDNsvJN5orxdj+eEUiSttpC
+  6N6IeTK/btk65TAbZlY7/9MVIHM/Ya3tUSWoQA6KURsksV+lUbJm0MC4pwm5y46T
+  p+Q6/eDz8gmofian4X+y3inyzYqmvbM2t2mPcO+Mj1NjSfGn56jqk0ey32xL7cz3
+  YZqx9LnkUUuqzn4QM+R8LfGUWXe9nlkMoLoNGw+X1XB8bPYoQo2YpofILEWcCLJl
+  NnFjcjIzQuE3NDaPIBX/cXPDpvuKNVXw51My35pQb5uW4Aa7sRZAz1MCDoFA1+XO
+  9czVf+zE+YsMtuqruWOI08TEPD2kMeiDqMc1fbPkLN9Rhe2Nxj8fTK1Ptyz0mQJL
+  x5cQZT2XK+nmZKiU/z+91XU17BNd+LEERU6ZmjkGQKUH/Gj060kQWMnw2sOsZJS9
+  VVU/raletJxhODeQbShcrxUR6VSGDkgQa4G/rqtPjUQ9AXYeNeVBg2aE4DRTRFhy
+  gYsyE9DbAluV7BcJWO+GKntyXqhu14dnATCfSFI3WMsCAwEAAaN9MHswCQYDVR0T
+  BAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFAjiCg2cEkNDWhG4P1PgDwATcmgR
+  MCAGA1UdEQQZMBeBFW1hdXJvYXNwcmVhQGdtYWlsLmNvbTAgBgNVHRIEGTAXgRVt
+  YXVyb2FzcHJlYUBnbWFpbC5jb20wDQYJKoZIhvcNAQELBQADggGBAG4S5uDseGtN
+  EqxvogGfJ+h7Pg5pdDsMVFWXOPN9IztSHE+88Ypv9b97zNk/vBztj3nwkUuhEczv
+  GZeI9/F72j1+8ARWdNgHalsrTHcFWaGEYNkm06EGh21dtO939Uqjg9auv9thDD/C
+  4jlEii1EvaGIaMhEdONAiRyOj0gEagw1AU5ItSxoWHFaaCEQSqXRskZMpw6fDZrE
+  jnJPYx5I8axku6D+/nlP3GeuMNwlqzTg1YblLDanzEdI7Yet41MgMwLcor6Z4BXb
+  xKDzHfGjkQfEpEvwzqSi0rAbrMv68FRybNxgB/3gwgSiEaeFeE4CFX0gm1F3m28m
+  Z9rgjNU1SA9dSTGLY14K3rjtn71PMlP64Ci+QJ0HA8V0+cpx1Lkn3WsywgILV16a
+  g4G6EZGbKCMwJDC0Wtmrygr7+THZVQlBs0ljTdrN8GXsuI9W52VlZctZQXEuoboH
+  mpXw1d3WewNciml1VaOG782DKqZvT0i19V5LnZzoGzmU2q3ZJw7jCw==
+  -----END CERTIFICATE-----
+date: 2020-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: moneta
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: robotex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: |-
+  == Medusa: a ruby crawler framework
+  Medusa is a ruby framework to crawl and collect useful information about the pages it visits.
+  It is versatile, allowing you to write your own specialized tasks quickly and easily.
+  #### Features
+  - Choose the links to follow on each page with `focus_crawl()`
+  - Multi-threaded design for high performance
+  - Tracks 301 HTTP redirects
+  - Allows exclusion of URLs based on regular expressions
+  - HTTPS support
+  - Records response time for each page
+  - Obey robots.txt
+  - In-memory or persistent storage of pages during crawl using Moneta adapters.
+  - Inherits OpenURI behavior (redirects, automatic charset and encoding detection, proxy configuration options).
+email:
+executables:
+- medusa
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- CHANGELOG.md
+- CONTRIBUTORS.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bin/medusa
+- lib/medusa.rb
+- lib/medusa/cli.rb
+- lib/medusa/cli/count.rb
+- lib/medusa/cli/cron.rb
+- lib/medusa/cli/pagedepth.rb
+- lib/medusa/cli/serialize.rb
+- lib/medusa/cli/url_list.rb
+- lib/medusa/cookie_store.rb
+- lib/medusa/core.rb
+- lib/medusa/exceptions.rb
+- lib/medusa/http.rb
+- lib/medusa/page.rb
+- lib/medusa/page_store.rb
+- lib/medusa/storage.rb
+- lib/medusa/storage/base.rb
+- lib/medusa/storage/exceptions.rb
+- lib/medusa/storage/moneta.rb
+- lib/medusa/tentacle.rb
+- lib/medusa/version.rb
+- spec/fakeweb_helper.rb
+- spec/medusa_helper.rb
+- spec/medusa_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/brutuscat/medusa-crawler
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/brutuscat/medusa-crawler/issues
+  source_code_uri: https://github.com/brutuscat/medusa-crawler/tree/v1.0.0.pre.1
+post_install_message:
+rdoc_options:
+- "-m"
+- README.md
+- "-t"
+- Medusa
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Medusa is a ruby crawler framework
+test_files:
+- spec/spec_helper.rb
+- spec/medusa_helper.rb
+- spec/fakeweb_helper.rb
+- spec/medusa_spec.rb