spidr 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46a2f2ad2ca789b83fac0e2519294403734e2ad6d647fbc3a612d429e57c1b43
-  data.tar.gz: b72f561e337c6a0fcdbca9f59562e06f0b5854b15d321f90be1a4168b352faca
+  metadata.gz: 6dee574a9edaa1949f5e018e3c47e41196379a097f46684d142dbeeb1a210441
+  data.tar.gz: f474aed8f7fa9277cdc5235870a9eadfb2d168fde8badeaa1d05ce67c581d275
 SHA512:
-  metadata.gz: ced221d8cdbeaf95df12d6c038de6539a5148657209137433cc82c5abc69779a13376a7e6becdf423d2f2bdd9ebfaf8c7b94a51dda70ffcbab932da4fc5260b3
-  data.tar.gz: f54bedf3648dd033b8a37388413ae4ab71b4b09f16cc508b8e43e72f2ef870c59fe325e3f36a841791d9d843acb08bb02009469168e9b231a9835a0249b55b6c
+  metadata.gz: 8bd271852427401fd04a3af6746216ebf36ca8bd3719dfb162b139272956db4e39758230da75ba462c187b4f9980c025ccee400d2e828a6db74b294bb11ee009
+  data.tar.gz: 87cfe842491566cd17e740461136900ce0e90e12f8ce07d3e6a1487bc5be74ad1bb234874eddc1f5cef99a9df7d495c358f8f3aab2ee29ebd38280837dfca6e9

data/.github/workflows/ruby.yml

@@ -9,18 +9,18 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - 2.7
           - '3.0'
           - '3.1'
+          - '3.2'
+          - '3.3'
           - jruby
     name: Ruby ${{ matrix.ruby }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-      - name: Install dependencies
-        run: bundle install --jobs 4 --retry 3
+          bundler-cache: true
       - name: Run tests
         run: bundle exec rake test

data/ChangeLog.md

@@ -1,3 +1,15 @@
+### 0.7.2 / 2025-02-02
+
+* Added the `base64` gem as a dependency to satisfy Bundler.
+
+### 0.7.1 / 2024-01-25
+
+* Switched to using `require_relative` to improve load-times.
+* Added `# frozen_string_literal: true` to all files.
+* Use keyword arguments for {Spidr.domain}.
+* Rescue `URI::Error` instead of `Exception` when calling `URI::HTTP#merge` in
+  {Spidr::Page#to_absolute}.
+
 ### 0.7.0 / 2022-12-31
 
 * Added {Spidr.domain} and {Spidr::Agent.domain}.

data/Gemfile

@@ -14,7 +14,7 @@ group :development do
 
   gem 'rspec',     '~> 3.0'
   gem 'webmock',   '~> 3.0'
-  gem 'sinatra',   '~> 2.0'
+  gem 'sinatra',   '~> 4.0'
   gem 'simplecov', '~> 0.20'
 
   gem 'kramdown'

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2008-2022 Hal Brodigan
+Copyright (c) 2008-2024 Hal Brodigan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -251,8 +251,6 @@ $ gem install spidr
 
 ## License
 
-Copyright (c) 2008-2022 Hal Brodigan
-
 See {file:LICENSE.txt} for license information.
 
 [ruby]: https://www.ruby-lang.org/

data/gemspec.yml

@@ -21,6 +21,7 @@ metadata:
 required_ruby_version: ">= 2.0.0"
 
 dependencies:
+  base64: ~> 0.1
   nokogiri: ~> 1.3
 
 development_dependencies:

data/lib/spidr/agent/actions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Agent
     module Actions

data/lib/spidr/agent/events.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Agent
     #

data/lib/spidr/agent/filters.rb

@@ -1,4 +1,6 @@
-require 'spidr/rules'
+# frozen_string_literal: true
+
+require_relative '../rules'
 
 module Spidr
   class Agent
@@ -170,7 +172,7 @@ module Spidr
     #
     # @yieldparam [String] link
     #   A link to accept or reject.
-    # 
+    #
     # @since 0.2.4
     #
     def visit_links_like(pattern=nil,&block)
@@ -238,7 +240,7 @@ module Spidr
     #
     # @yieldparam [URI::HTTP, URI::HTTPS] url
     #   A URL to accept or reject.
-    # 
+    #
     # @since 0.2.4
     #
     def visit_urls_like(pattern=nil,&block)

data/lib/spidr/agent/robots.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require 'robots'
 rescue LoadError

data/lib/spidr/agent/sanitizers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 module Spidr

data/lib/spidr/agent.rb

@@ -1,14 +1,16 @@
-require 'spidr/settings/user_agent'
-require 'spidr/agent/sanitizers'
-require 'spidr/agent/filters'
-require 'spidr/agent/events'
-require 'spidr/agent/actions'
-require 'spidr/agent/robots'
-require 'spidr/page'
-require 'spidr/session_cache'
-require 'spidr/cookie_jar'
-require 'spidr/auth_store'
-require 'spidr/spidr'
+# frozen_string_literal: true
+
+require_relative 'settings/user_agent'
+require_relative 'agent/sanitizers'
+require_relative 'agent/filters'
+require_relative 'agent/events'
+require_relative 'agent/actions'
+require_relative 'agent/robots'
+require_relative 'page'
+require_relative 'session_cache'
+require_relative 'cookie_jar'
+require_relative 'auth_store'
+require_relative 'spidr'
 
 require 'openssl'
 require 'net/http'

data/lib/spidr/auth_credential.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # Represents HTTP Authentication credentials for a website.

data/lib/spidr/auth_store.rb

@@ -1,6 +1,8 @@
-require 'spidr/extensions/uri'
-require 'spidr/auth_credential'
-require 'spidr/page'
+# frozen_string_literal: true
+
+require_relative 'extensions/uri'
+require_relative 'auth_credential'
+require_relative 'page'
 
 require 'base64'
 
@@ -20,7 +22,7 @@ module Spidr
       @credentials = {}
     end
 
-    # 
+    #
     # Given a URL, return the most specific matching auth credential.
     #
     # @param [URI] url
@@ -54,7 +56,7 @@ module Spidr
       return nil
     end
 
-    # 
+    #
     # Add an auth credential to the store for supplied base URL.
     #
     # @param [URI] url
@@ -122,7 +124,7 @@ module Spidr
       end
     end
 
-    # 
+    #
     # Clear the contents of the auth store.
     #
     # @return [AuthStore]

data/lib/spidr/cookie_jar.rb

@@ -1,4 +1,6 @@
-require 'spidr/page'
+# frozen_string_literal: true
+
+require_relative 'page'
 
 require 'set'
 
@@ -42,8 +44,8 @@ module Spidr
       @params.each(&block)
     end
 
-    # 
-    # Return all relevant cookies in a single string for the 
+    #
+    # Return all relevant cookies in a single string for the
     # named host or domain (in browser request format).
     #
     # @param [String] host
@@ -59,7 +61,7 @@ module Spidr
       @params[host] ||= {}
     end
 
-    # 
+    #
     # Add a cookie to the jar for a particular domain.
     #
     # @param [String] host
@@ -166,7 +168,7 @@ module Spidr
       return host_cookies
     end
 
-    # 
+    #
     # Clear out the jar, removing all stored cookies.
     #
     # @since 0.2.2

data/lib/spidr/extensions/uri.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'strscan'
 
@@ -58,7 +60,7 @@ module URI
     unless stack.empty?
       "#{leading_slash}#{stack.join('/')}#{trailing_slash}"
     else
-      '/'
+      String.new('/')
     end
   end
 end

data/lib/spidr/extensions.rb

@@ -1 +1,3 @@
-require 'spidr/extensions/uri'
+# frozen_string_literal: true
+
+require_relative 'extensions/uri'

data/lib/spidr/page/content_types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Page
     #

data/lib/spidr/page/cookies.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'set'
 
 module Spidr

data/lib/spidr/page/html.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
+require_relative '../extensions/uri'
+
 require 'nokogiri'
-require 'spidr/extensions/uri'
 
 module Spidr
   class Page
@@ -265,7 +268,7 @@ module Spidr
       link    = link.to_s
       new_url = begin
                   url.merge(link)
-                rescue Exception
+                rescue URI::Error
                   return
                 end
 

data/lib/spidr/page/status_codes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Page
     #
@@ -92,7 +94,7 @@ module Spidr
 
     #
     # Determines if the response code is `300`, `301`, `302`, `303`
-    # or `307`. Also checks for "soft" redirects added at the page 
+    # or `307`. Also checks for "soft" redirects added at the page
     # level by a meta refresh tag.
     #
     # @return [Boolean]

data/lib/spidr/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # Represents a requested page from a website.
@@ -142,7 +144,7 @@ module Spidr
 
       return super(name,*arguments,&block)
     end
-  
+
   end
 end
 

data/lib/spidr/proxy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # @since 0.6.0

data/lib/spidr/rules.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # The {Rules} class represents collections of acceptance and rejection

data/lib/spidr/session_cache.rb

@@ -1,6 +1,8 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/spidr'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'spidr'
 
 require 'net/http'
 require 'openssl'
@@ -135,7 +137,7 @@ module Spidr
       key = key_for(url)
 
       if (sess = @sessions[key])
-        begin 
+        begin
           sess.finish
         rescue IOError
         end

data/lib/spidr/settings/proxy.rb

@@ -1,4 +1,6 @@
-require 'spidr/proxy'
+# frozen_string_literal: true
+
+require_relative '../proxy'
 
 require 'uri/http'
 

data/lib/spidr/settings/timeouts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   module Settings
     #

data/lib/spidr/settings/user_agent.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   module Settings
     #

data/lib/spidr/settings.rb

@@ -1,3 +1,5 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/settings/user_agent'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'settings/user_agent'

data/lib/spidr/spidr.rb

@@ -1,7 +1,9 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/settings/user_agent'
-require 'spidr/agent'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'settings/user_agent'
+require_relative 'agent'
 
 module Spidr
   extend Settings::Proxy
@@ -52,8 +54,8 @@ module Spidr
   #
   # @since 0.7.0
   #
-  def self.domain(name,options={},&block)
-    Agent.domain(name,options,&block)
+  def self.domain(name,**kwargs,&block)
+    Agent.domain(name,**kwargs,&block)
   end
 
   #
@@ -63,7 +65,7 @@ module Spidr
     Agent.site(url,**kwargs,&block)
   end
 
-  # 
+  #
   # @abstract
   #
   def self.robots

data/lib/spidr/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 module Spidr
   # Spidr version
-  VERSION = '0.7.0'
+  VERSION = '0.7.2'
 end

data/lib/spidr.rb

@@ -1,3 +1,5 @@
-require 'spidr/agent'
-require 'spidr/spidr'
-require 'spidr/version'
+# frozen_string_literal: true
+
+require_relative 'spidr/agent'
+require_relative 'spidr/spidr'
+require_relative 'spidr/version'

data/spidr.gemspec

@@ -7,10 +7,7 @@ Gem::Specification.new do |gem|
 
   gem.name    = gemspec.fetch('name')
   gem.version = gemspec.fetch('version') do
-                  lib_dir = File.join(File.dirname(__FILE__),'lib')
-                  $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
-
-                  require 'spidr/version'
+                  require_relative 'lib/spidr/version'
                   Spidr::VERSION
                 end
 

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: spidr
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-01 00:00:00.000000000 Z
+date: 2025-02-03 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -128,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: A versatile Ruby web spidering library