spidr_cli 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 122d1ee2879d8333bae8fda51cbaa5adcc0a3810344a925593cfdc9dfc459916
-  data.tar.gz: 7f5471d05f5b977b531fccca29b07f3c5278b1b1b23e2c625f1ae832c0b57481
+  metadata.gz: e52c41017020452ca4bce9efdf8a4925941f2dd926fe9a896af32e5c02ba0350
+  data.tar.gz: 9007ef979a5d62185cdd7a9656921f53990946f997c0bb7535ae2276212ce60d
 SHA512:
-  metadata.gz: 4dfc79d60c5f5a6a3dd119eee44af28f31d92b915fe4ebe4fcb07a0a00781e111e260adacae6d0bb860223902478f6550e4ba39e6f82c1865237bf1f5a8de939
-  data.tar.gz: c577c737be56e1b1009a343354bef7aae2fc2dc3e1a829a358430e54772623d252ae24080a1800370f2c670a04c6f55598e91cd462e64ab81709682104cfaffb
+  metadata.gz: 9c80f0fbd9031f9d9c44192f610b7b5147411d53c7195b11dec733502299743fb760277e938653fb1e9cd64a8d10ea09e2fff4622262d0d022a512cc5bf5df1b
+  data.tar.gz: 2479ae42d872fc2541ccd938c01661ccabd11ebf3a195733851db5ede769eac14f478c869426757036b7e43d2ce2aec183f550c4c91b814bc3040c4754da2a0b

data/README.md

@@ -55,9 +55,13 @@ Usage: spidr [<method>] [options] <url>
         --columns=[val1,val2]        Columns in output
         --content-types=[val1,val2]  Formats to output (html, javascript, css, json, ..)
         --[no-]header                Include the header
-        --hosts=[example.com]        Only spider links on certain hosts
+        --[no-]strip-fragments       Specifies whether the Agent will strip URI fragments (default: true)
+        --[no-]strip-query           Specifies whether the Agent will strip URI query (default: false)
+        --schemes=[http,https]       Only spider links with certain scheme
+        --host=[example]             Only spider links on certain host
+        --hosts=[example.com]        Only spider links on certain hosts (ignored unless method is "start_at")
         --ignore-hosts=[www.example.com]
-                                     Do not spider links on certain hosts
+                                     Do not spider links on certain hosts (ignored unless method is "start_at")
         --ports=[80, 443]            Only spider links on certain ports
         --ignore-ports=[8000, 8080, 3000]
                                      Do not spider links on certain ports
@@ -67,15 +71,15 @@ Usage: spidr [<method>] [options] <url>
         --ignore-urls=[/blog/]       Do not spider links on certain urls
         --exts=[htm]                 Only spider links on certain extensions
         --ignore-exts=[cfm]          Do not spider links on certain extensions
-        --open-timeout=val           Optional open timeout
-        --read-timeout=val           Optional read timeout
-        --ssl-timeout=val            Optional ssl timeout
-        --continue-timeout=val       Optional continue timeout
-        --keep-alive-timeout=val     Optional keep_alive timeout
+        --open-timeout=val           Open timeout
+        --read-timeout=val           Read timeout
+        --ssl-timeout=val            SSL timeout
+        --continue-timeout=val       Continue timeout
+        --keep-alive-timeout=val     Keep alive timeout
         --proxy-host=val             The host the proxy is running on
         --proxy-port=val             The port the proxy is running on
-        --proxy-user=val             The user to authenticate as with the proxy
-        --proxy-password=val         The password to authenticate with
+        --proxy-user=val             The user to authenticate with the proxy
+        --proxy-password=val         The password to authenticate with the proxy
         --default-headers=[key1=val1,key2=val2]
                                      Default headers to set for every request
         --host-header=val            The HTTP Host header to use with each request

data/lib/spidr_cli.rb

@@ -1,3 +1,4 @@
+require 'spidr'
 require 'spidr_cli/version'
 require 'spidr_cli/options'
 

data/lib/spidr_cli/options.rb

@@ -2,7 +2,7 @@ require 'optparse'
 
 module SpidrCLI
   class Options
-    # Spidr method
+    # Spidr methods
     METHODS = %w[site start_at host].map { |c| [c, c] }.to_h.freeze
 
     attr_reader :url, :columns, :content_types, :header, :spidr_options, :usage_doc,
@@ -44,15 +44,33 @@ module SpidrCLI
           @header = value
         end
 
-        # Spidr::Rules options
+        # Spidr::Sanitizers options
+        parser.on('--[no-]strip-fragments', 'Specifies whether the Agent will strip URI fragments (default: true)') do |value|
+          spidr_options[:strip_fragments] = value
+        end
+
+        parser.on('--[no-]strip-query', 'Specifies whether the Agent will strip URI query (default: false)') do |value|
+          spidr_options[:strip_query] = value
+        end
+
+        # Spidr::Filters options
+        parser.on('--schemes=[http,https]', Array, 'Only spider links with certain scheme') do |value|
+          spidr_options[:schemes] = value if value
+        end
 
-        # NOTE: --hosts and --ignore-hosts are overriden when using Spidr::site
+        parser.on('--host=[example]', String, 'Only spider links on certain host') do |value|
+          spidr_options[:host] = value if value
+        end
+
+        # NOTE: --hosts is overriden
         #   @see https://github.com/postmodern/spidr/blob/master/lib/spidr/agent.rb#L273
-        parser.on('--hosts=[example.com]', Array, 'Only spider links on certain hosts') do |value|
+        parser.on('--hosts=[example.com]', Array, 'Only spider links on certain hosts (ignored unless method is "start_at")') do |value|
           spidr_options[:hosts] = to_option_regexp_array(value) if value
         end
 
-        parser.on('--ignore-hosts=[www.example.com]', Array, 'Do not spider links on certain hosts') do |value|
+        # NOTE: --ignore-hosts is overriden
+        #   @see https://github.com/postmodern/spidr/blob/master/lib/spidr/agent.rb#L273
+        parser.on('--ignore-hosts=[www.example.com]', Array, 'Do not spider links on certain hosts (ignored unless method is "start_at")') do |value|
           spidr_options[:ignore_hosts] = to_option_regexp_array(value) if value
         end
 
@@ -89,23 +107,23 @@ module SpidrCLI
         end
 
         # Spidr::Agent options
-        parser.on('--open-timeout=val', Integer, 'Optional open timeout') do |value|
+        parser.on('--open-timeout=val', Integer, 'Open timeout') do |value|
           spidr_options[:open_timeout] = value
         end
 
-        parser.on('--read-timeout=val', Integer, 'Optional read timeout') do |value|
+        parser.on('--read-timeout=val', Integer, 'Read timeout') do |value|
           spidr_options[:read_timeout] = value
         end
 
-        parser.on('--ssl-timeout=val', Integer, 'Optional ssl timeout') do |value|
+        parser.on('--ssl-timeout=val', Integer, 'SSL timeout') do |value|
           spidr_options[:ssl_timeout] = value
         end
 
-        parser.on('--continue-timeout=val', Integer, 'Optional continue timeout') do |value|
+        parser.on('--continue-timeout=val', Integer, 'Continue timeout') do |value|
           spidr_options[:continue_timeout] = value
         end
 
-        parser.on('--keep-alive-timeout=val', Integer, 'Optional keep_alive timeout') do |value|
+        parser.on('--keep-alive-timeout=val', Integer, 'Keep alive timeout') do |value|
           spidr_options[:keep_alive_timeout] = value
         end
 
@@ -117,11 +135,11 @@ module SpidrCLI
           proxy_options[:port] = value
         end
 
-        parser.on('--proxy-user=val', String, 'The user to authenticate as with the proxy') do |value|
+        parser.on('--proxy-user=val', String, 'The user to authenticate with the proxy') do |value|
           proxy_options[:user] = value
         end
 
-        parser.on('--proxy-password=val', String, 'The password to authenticate with') do |value|
+        parser.on('--proxy-password=val', String, 'The password to authenticate with the proxy') do |value|
           proxy_options[:password] = value
         end
 
@@ -176,11 +194,16 @@ module SpidrCLI
         end
 
         parser.on_tail('--version', 'Show version') do
-          puts "Spidr version #{Spidr::VERSION}"
+          puts "Spidr version #{Spidr::VERSION} (SpidrCLI version #{SpidrCLI::VERSION})"
           exit
         end
       end.parse!
 
+      if @spidr_method != 'start_at' &&
+          (spidr_options.key?(:hosts) || spidr_options.key?(:ignore_hosts))
+        raise(ArgumentError, '--hosts and --ignore-hosts argument are only valid if spidr method is "start_at"')
+      end
+
       spidr_options[:proxy] = proxy_options unless proxy_options.empty?
 
       @url = argv.last

data/lib/spidr_cli/version.rb

@@ -1,3 +1,3 @@
 module SpidrCLI
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidr_cli
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-30 00:00:00.000000000 Z
+date: 2018-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr