spidr_cli 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c474f12ce4328ca292990e2e0132a436dca3e99f1a18ac3f3e0c2efbd780ba5b
+  data.tar.gz: 70a46ac79acc21bb34b2c78a442d0cc0299286f0a39946fdf017d92a2d789c19
+SHA512:
+  metadata.gz: 90a4939a6a532f7dfdb77ce61c9a250bbf222df5238de947594e35337d6365b0747ddd597073f02400a157aff5351baebfa54d414060dcbb37fc69555092e83e
+  data.tar.gz: 964c638ba0b24265507f6afca17ece303556e95f156ad00345e7bcaabce7b6f957d81177a0f2f15d29c0a8e688d725297ac0646a66fe7c733e3dbe4bb5fd25a0

data/.gitignore

@@ -0,0 +1,17 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# Ignore dependency lock files
+Gemfile.lock
+
+# Ignore byebug files
+.byebug_history

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+  - 2.4.0
+  - 2.5.0
+  - 2.6.0-preview1
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in spidr_cli.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Jacob Burenstam Linder
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+# SpidrCLI
+
+Command Line Interface (CLI) for the excellent [`spidr`](https://github.com/postmodern/spidr) gem.
+
+## Installation
+
+Install with
+
+    $ gem install spidr_cli
+
+## Usage
+
+Print all found pages on site
+
+```
+$ spidr https://jacoburenstam.com/
+```
+
+Print all HTML/JS/CSS pages
+```
+$ spidr --content-types=html,javascript,css https://jacoburenstam.com/
+```
+
+Max 10 pages
+```
+$ spidr --limit=10 https://jacoburenstam.com/
+```
+
+Spidr host
+```
+$ spidr host jacoburenstam.com
+```
+
+Spidr a single site (this is the default)
+```
+$ spidr site https://jacoburenstam.com
+```
+
+Start spidr from URL
+```
+$ spidr start_at https://jacoburenstam.com
+```
+
+Any method that [`Spidr::Page`](https://github.com/postmodern/spidr/blob/master/lib/spidr/page.rb) responds to you can output, you can also choose to include the header in the output (which is valid CSV)
+```
+$ spidr --columns=code,content_type,url \
+        --header                        \
+        https://jacoburenstam.com/
+```
+
+Full usage instructions
+
+```
+Usage: spidr [<strategy>] [options] <url>
+        --columns=[val1,val2]        Columns in output
+        --content-types=[val1,val2]  Formats to output (html, javascript, css, json, ..)
+        --[no-]header                Include the header
+        --ports=[80, 443]            Only spider links on certain ports
+        --ignore-ports=[8000, 8080, 3000]
+                                     Do not spider links on certain ports
+        --links=[/blog/]             Only spider links on certain link patterns
+        --ignore-links=[/blog/]      Do not spider links on certain link patterns
+        --urls=[/blog/]              Only spider links on certain urls
+        --ignore-urls=[/blog/]       Do not spider links on certain urls
+        --exts=[htm]                 Only spider links on certain extensions
+        --ignore-exts=[cfm]          Do not spider links on certain extensions
+        --open-timeout=val           Optional open timeout
+        --read-timeout=val           Optional read timeout
+        --ssl-timeout=val            Optional ssl timeout
+        --continue-timeout=val       Optional continue timeout
+        --keep-alive-timeout=val     Optional keep_alive timeout
+        --proxy-host=val             The host the proxy is running on
+        --proxy-port=val             The port the proxy is running on
+        --proxy-user=val             The user to authenticate as with the proxy
+        --proxy-password=val         The password to authenticate with
+        --default-headers=[key1=val1,key2=val2]
+                                     Default headers to set for every request
+        --host-header=val            The HTTP Host header to use with each request
+        --host-headers=[key1=val1,key2=val2]
+                                     The HTTP Host headers to use for specific hosts
+        --user-agent=val             The User-Agent string to send with each requests
+        --referer=val                The Referer URL to send with each request
+        --delay=val                  The number of seconds to pause between each request
+        --queue=[val1,val2]          The initial queue of URLs to visit
+        --history=[val1,val2]        The initial list of visited URLs
+        --limit=val                  The maximum number of pages to visit
+        --max-depth=val              The maximum link depth to follow
+        --[no-]robots                Respect Robots.txt
+    -h, --help                       How to use
+        --version                    Show version
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/buren/spidr_cli.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Thanks
+
+Huge thanks to [@postmodern](https://github.com/postmodern) for creating [`spidr`](https://github.com/postmodern/spidr) :star:

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'spidr_cli'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/spidr

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+# for dev purposes
+if ENV['SPIDR_CLI_GEM_DEV']
+  puts '[WARN] spidr_cli loaded in development mode'
+  require 'bundler/setup'
+end
+
+require 'spidr'
+require 'spidr_cli/options'
+
+require 'csv'
+require 'optparse'
+
+options = SpidrCLI::Options.new(ARGV)
+start_at = options.url
+header = options.header
+columns = options.columns
+content_types = options.content_types
+spidr_method = options.spidr_method
+spidr_options = options.spidr_options
+
+start_at = options.url
+if start_at.nil? || start_at.empty?
+  puts options.usage_doc
+  raise(ArgumentError, "<url> can't be blank")
+end
+
+# main
+puts CSV.generate_line(columns) if header
+Spidr.public_send(spidr_method, start_at, spidr_options) do |spider|
+  spider.every_page do |page|
+    next if content_types && content_types.all? { |type| !page.is_content_type?(type) }
+
+    row = columns.map { |column| page.public_send(column) }
+    puts CSV.generate_line(row)
+  end
+end

data/lib/spidr_cli/options.rb

@@ -0,0 +1,201 @@
+require 'optparse'
+
+module SpidrCLI
+  class Options
+    # Spidr method
+    METHODS = %w[site start_at host].map { |c| [c, c] }.to_h.freeze
+
+    attr_reader :url, :columns, :content_types, :header, :spidr_options, :usage_doc,
+                :spidr_method
+
+    def initialize(argv = ARGV)
+      @url = nil
+      @columns = %w[url]
+      @content_types = nil
+      @header = false
+      @usage_doc = nil
+      @spidr_method = 'site'
+      @spidr_options = {}
+
+      parse_options(argv)
+    end
+
+    private
+
+    def parse_options(argv)
+      proxy_options = {}
+      @spidr_method = METHODS[argv.first] if METHODS.key?(argv.first)
+
+      OptionParser.new do |parser|
+        @usage_doc = parser.to_s
+
+        parser.banner = 'Usage: spidr [<method>] [options] <url>'
+        parser.default_argv = argv
+
+        parser.on('--columns=[val1,val2]', Array, 'Columns in output') do |value|
+          @columns = value || columns
+        end
+
+        parser.on('--content-types=[val1,val2]', Array, 'Formats to output (html, javascript, css, json, ..)') do |value|
+          @content_types = value
+        end
+
+        parser.on('--[no-]header', 'Include the header') do |value|
+          @header = value
+        end
+
+        # Spidr::Rules options
+
+        # NOTE: --hosts and --ignore-hosts are overriden when using Spidr::site
+        #   @see https://github.com/postmodern/spidr/blob/master/lib/spidr/agent.rb#L273
+        # parser.on('--hosts=[example.com]', Array, 'Only spider links on certain hosts') do |value|
+        #   spidr_options[:hosts] = value.map { |v| Regexp.new(v) } if value
+        # end
+        #
+        # parser.on('--ignore-hosts=[www.example.com]', Array, 'Do not spider links on certain hosts') do |value|
+        #   spidr_options[:ignore_hosts] = value.map { |v| Regexp.new(v) } if value
+        # end
+
+        parser.on('--ports=[80, 443]', Array, 'Only spider links on certain ports') do |value|
+          spidr_options[:ports] = to_option_int_array(value) if value
+        end
+
+        parser.on('--ignore-ports=[8000, 8080, 3000]', Array, 'Do not spider links on certain ports') do |value|
+          spidr_options[:ignore_ports] = to_option_int_array(value) if value
+        end
+
+        parser.on('--links=[/blog/]', Array, 'Only spider links on certain link patterns') do |value|
+          spidr_options[:links] = to_option_regexp_array(value) if value
+        end
+
+        parser.on('--ignore-links=[/blog/]', Array, 'Do not spider links on certain link patterns') do |value|
+          spidr_options[:ignore_links] = to_option_regexp_array(value) if value
+        end
+
+        parser.on('--urls=[/blog/]', Array, 'Only spider links on certain urls') do |value|
+          spidr_options[:urls] = to_option_regexp_array(value) if value
+        end
+
+        parser.on('--ignore-urls=[/blog/]', Array, 'Do not spider links on certain urls') do |value|
+          spidr_options[:ignore_urls] = to_option_regexp_array(value) if value
+        end
+
+        parser.on('--exts=[htm]', Array, 'Only spider links on certain extensions') do |value|
+          spidr_options[:exts] = to_option_regexp_array(value) if value
+        end
+
+        parser.on('--ignore-exts=[cfm]', Array, 'Do not spider links on certain extensions') do |value|
+          spidr_options[:ignore_exts] = to_option_regexp_array(value) if value
+        end
+
+        # Spidr::Agent options
+        parser.on('--open-timeout=val', Integer, 'Optional open timeout') do |value|
+          spidr_options[:open_timeout] = value
+        end
+
+        parser.on('--read-timeout=val', Integer, 'Optional read timeout') do |value|
+          spidr_options[:read_timeout] = value
+        end
+
+        parser.on('--ssl-timeout=val', Integer, 'Optional ssl timeout') do |value|
+          spidr_options[:ssl_timeout] = value
+        end
+
+        parser.on('--continue-timeout=val', Integer, 'Optional continue timeout') do |value|
+          spidr_options[:continue_timeout] = value
+        end
+
+        parser.on('--keep-alive-timeout=val', Integer, 'Optional keep_alive timeout') do |value|
+          spidr_options[:keep_alive_timeout] = value
+        end
+
+        parser.on('--proxy-host=val', String, 'The host the proxy is running on') do |value|
+          proxy_options[:host] = value
+        end
+
+        parser.on('--proxy-port=val', Integer, 'The port the proxy is running on') do |value|
+          proxy_options[:port] = value
+        end
+
+        parser.on('--proxy-user=val', String, 'The user to authenticate as with the proxy') do |value|
+          proxy_options[:user] = value
+        end
+
+        parser.on('--proxy-password=val', String, 'The password to authenticate with') do |value|
+          proxy_options[:password] = value
+        end
+
+        parser.on('--default-headers=[key1=val1,key2=val2]', Array, 'Default headers to set for every request') do |value|
+          spidr_options[:default_headers] = option_hash(value || [])
+        end
+
+        parser.on('--host-header=val', String, 'The HTTP Host header to use with each request') do |value|
+          spidr_options[:host_header] = value
+        end
+
+        parser.on('--host-headers=[key1=val1,key2=val2]', Array, 'The HTTP Host headers to use for specific hosts') do |value|
+          spidr_options[:host_headers] = option_hash(value || [])
+        end
+
+        parser.on('--user-agent=val', String, 'The User-Agent string to send with each requests') do |value|
+          spidr_options[:user_agent] = value
+        end
+
+        parser.on('--referer=val', String, 'The Referer URL to send with each request') do |value|
+          spidr_options[:referer] = value
+        end
+
+        parser.on('--delay=val', Integer, 'The number of seconds to pause between each request') do |value|
+          spidr_options[:delay] = value
+        end
+
+        parser.on('--queue=[val1,val2]', Array, 'The initial queue of URLs to visit') do |value|
+          spidr_options[:queue] = value
+        end
+
+        parser.on('--history=[val1,val2]', Array, 'The initial list of visited URLs') do |value|
+          spidr_options[:history] = value
+        end
+
+        parser.on('--limit=val', Integer, 'The maximum number of pages to visit') do |value|
+          spidr_options[:limit] = value
+        end
+
+        parser.on('--max-depth=val', Integer, 'The maximum link depth to follow') do |value|
+          spidr_options[:max_depth] = value
+        end
+
+        parser.on('--[no-]robots', 'Respect Robots.txt') do |value|
+          spidr_options[:robots] = value
+        end
+
+        # Boilerplate CLI
+        parser.on('-h', '--help', 'How to use') do
+          puts parser
+          exit
+        end
+
+        parser.on_tail('--version', 'Show version') do
+          puts "Spidr version #{Spidr::VERSION}"
+          exit
+        end
+      end.parse!
+
+      spidr_options[:proxy] = proxy_options unless proxy_options.empty?
+
+      @url = argv.last
+    end
+
+    def to_option_int_array(value)
+      value.map { |v| Integer(v) }
+    end
+
+    def to_option_regexp_array(value)
+      value.map { |v| Regexp.new(v) }
+    end
+
+    def option_hash(value)
+      value.map { |v| v.split('=') }.to_h
+    end
+  end
+end

data/lib/spidr_cli/version.rb

@@ -0,0 +1,3 @@
+module SpidrCLI
+  VERSION = '0.1.0'.freeze
+end

data/lib/spidr_cli.rb

@@ -0,0 +1,6 @@
+require 'spidr_cli/version'
+require 'spidr_cli/options'
+
+# Spidr CLI
+module SpidrCLI
+end

data/spidr_cli.gemspec

@@ -0,0 +1,31 @@
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spidr_cli/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'spidr_cli'
+  spec.version       = SpidrCLI::VERSION
+  spec.authors       = ['Jacob Burenstam']
+  spec.email         = ['burenstam@gmail.com']
+
+  spec.summary       = 'CLI for the spidr gem'
+  spec.description   = 'Command Line Interface (CLI) for the spidr gem.'
+  spec.homepage      = 'https://github.com/buren/spidr_cli'
+  spec.license       = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'spidr', '~> 0.6'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'byebug', '~> 9.0'
+  spec.add_development_dependency 'simplecov'
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: spidr_cli
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jacob Burenstam
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-06-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: spidr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Command Line Interface (CLI) for the spidr gem.
+email:
+- burenstam@gmail.com
+executables:
+- spidr
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/spidr
+- lib/spidr_cli.rb
+- lib/spidr_cli/options.rb
+- lib/spidr_cli/version.rb
+- spidr_cli.gemspec
+homepage: https://github.com/buren/spidr_cli
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: CLI for the spidr gem
+test_files: []