rangescan 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bacbfd805037d62c75718f21ef09116eb30a675606267959c9008b6a23c11607
-  data.tar.gz: 60c0236c232a57d66a72e7926e820b8a0940b20d4bdd5970c2dd5dccdef2bfd5
+  metadata.gz: 643ca26936dec4a00aabc6551e9cea7c87cecfda46d5055c26f1c3e3c967893b
+  data.tar.gz: 6f034dcd1e4d5c59b36baec5ea6e4911aba47d88cd462d92a33898194b871839
 SHA512:
-  metadata.gz: 3cf068f8884b6abf3231ae183a9190573d9371ec3054a66ee5fe85e9ed5babf5dcf3ff7ffc1f7e9ded672bc99e128fee3ec37023fb47f5e124c73afeefd30570
-  data.tar.gz: aa5cb7c38d41199fec63a3b04b0456e20fe36a6cf51a95fa8870fe5220e00c1ea11e277a5dc7810c8fb9c5a7e6528dbb40cd4fdfe7a7f6f05b9c5923427f9bbf
+  metadata.gz: e2ce5509d388bc2d8026a8d9534339c5428bb974751d4e48ca73964d517a6de38665b8fb070247ee28cab5203b3bbad495890b6576bee1a89bb0008437130a1b
+  data.tar.gz: '098beb8cbee3e0f21302a2cffbaa83f9bdae11b9d80270517e7c92210bb8bf0db8050e337f180ba31a128e2cb4d858405f6e0ea0e4604905c54790abfcb27bef'

data/.deepsource.toml

@@ -0,0 +1,5 @@
+version = 1
+
+[[analyzers]]
+name = "ruby"
+enabled = true

data/README.md

@@ -27,13 +27,14 @@ Usage:
 
 Options:
   [--host=HOST]                      # Host header
-  [--port=N]                         # Port to check (80 or 443)
-  [--scheme=SCHEME]                  # Scheme to use (http or https)
+  [--port=N]                         # Port
+  [--scheme=SCHEME]                  # Scheme (http or https)
   [--timeout=N]                      # Timeout in seconds
-  [--user-agent=USER_AGENT]          # User Agent header
-  [--veryfy-ssl], [--no-veryfy-ssl]  # Verify SSL or not
+  [--user-agent=USER_AGENT]          # User Agent
+  [--verify-ssl], [--no-verify-ssl]  # Whether to verify SSL or not
+  [--max-concurrency=N]              # Concurrency limit for HTTP requests to scan
 
-Scan an IP range & filter by a regexp
+Scan an IP range & filter by a regexp (default regexp = .)
 ```
 
 ## License

data/exe/rangescan

@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "rangescan"
 
-RangeScan::CLI.start
+ARGV.unshift(RangeScan::CLI.default_task) unless RangeScan::CLI.all_tasks.key?(ARGV[0])
+RangeScan::CLI.start(ARGV)

data/lib/rangescan.rb

@@ -3,6 +3,7 @@
 require "rangescan/version"
 
 require "rangescan/range"
+require "rangescan/utils"
 require "rangescan/scanner"
 require "rangescan/matcher"
 

data/lib/rangescan/cli.rb

@@ -5,14 +5,16 @@ require "json"
 
 module RangeScan
   class CLI < Thor
-    desc "scan [IP_WITH_SUBNET_MASK, REGEXP]", "Scan an IP range & filter by a regexp"
+    desc "scan [IP_WITH_SUBNET_MASK, REGEXP]", "Scan an IP range & filter by a regexp (default regexp = .)"
     method_option :host, type: :string, desc: "Host header"
     method_option :port, type: :numeric, desc: "Port"
     method_option :scheme, type: :string, desc: "Scheme (http or https)"
     method_option :timeout, type: :numeric, desc: "Timeout in seconds"
     method_option :user_agent, type: :string, desc: "User Agent"
     method_option :verify_ssl, type: :boolean, desc: "Whether to verify SSL or not"
-    def scan(ip_with_subnet_mask, regexp)
+    method_option :max_concurrency, type: :numeric, desc: "Concurrency limit for HTTP requests to scan"
+    method_option :headers, type: :hash, default: {}, desc: "Custom headers"
+    def scan(ip_with_subnet_mask, regexp = ".")
       symbolized_options = symbolize_hash_keys(options)
       range = Range.new(ip_with_subnet_mask)
 
@@ -25,10 +27,18 @@ module RangeScan
       puts JSON.pretty_generate(filtered)
     end
 
+    default_command :scan
+
     no_commands do
       def symbolize_hash_keys(hash)
         hash.map { |k, v| [k.to_sym, v] }.to_h
       end
     end
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/rangescan/matcher.rb

@@ -11,7 +11,11 @@ module RangeScan
     def filter(results)
       results.select do |result|
         body = result.dig(:body) || ""
-        body =~ regexp
+        begin
+          body =~ regexp
+        rescue ArgumentError, Encoding::CompatibilityError
+          false
+        end
       end
     end
   end

data/lib/rangescan/monkey_patch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "protocol/http1/connection"
+
+module Protocol
+  module HTTP1
+    class Connection
+      def write_request(authority, method, path, version, headers)
+        host = authority
+        if headers.include?("host")
+          host = headers["host"]
+          headers.delete "host"
+        end
+
+        @stream.write("#{method} #{path} #{version}\r\n")
+        @stream.write("host: #{host}\r\n")
+
+        write_headers(headers)
+      end
+    end
+  end
+end

data/lib/rangescan/scanner.rb

@@ -1,60 +1,98 @@
 # frozen_string_literal: true
 
-require "http"
-require "parallel"
+require "async"
+require "async/barrier"
+require "async/semaphore"
+require "async/http"
+require "etc"
+
+require "rangescan/monkey_patch"
 
 module RangeScan
   class Scanner
     attr_reader :context
+    attr_reader :headers
     attr_reader :host
+    attr_reader :max_concurrency
     attr_reader :port
+    attr_reader :processor_count
     attr_reader :scheme
     attr_reader :ssl_context
     attr_reader :timeout
     attr_reader :user_agent
+    attr_reader :verify_ssl
 
-    def initialize(host: nil, port: nil, scheme: "http", verify_ssl: true, timeout: 5, user_agent: nil)
+    def initialize(
+      headers: {},
+      host: nil,
+      max_concurrency: nil,
+      port: nil,
+      scheme: "http",
+      timeout: 5,
+      user_agent: nil,
+      verify_ssl: true
+    )
+      @headers = headers
       @host = host
       @port = port || (scheme == "http" ? 80 : 443)
-      @timeout = timeout
       @scheme = scheme
+      @timeout = timeout
       @user_agent = user_agent
+      @verify_ssl = verify_ssl
 
       @ssl_context = OpenSSL::SSL::SSLContext.new
       @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE unless verify_ssl
+
+      @max_concurrency = max_concurrency || Etc.nprocessors * 8
+    end
+
+    def url_for(ipv4)
+      return "#{scheme}://#{ipv4}" if (port == 80 && scheme == "http") || (port == 443 && scheme == "https")
+
+      "#{scheme}://#{ipv4}:#{port}"
     end
 
     def scan(ipv4s)
-      Parallel.map(ipv4s) do |ipv4|
-        get ipv4
-      end.compact
+      results = []
+      Async do
+        barrier = Async::Barrier.new
+        semaphore = Async::Semaphore.new(max_concurrency, parent: barrier)
+
+        ipv4s.each do |ipv4|
+          semaphore.async do
+            url = url_for(ipv4)
+
+            endpoint = Async::HTTP::Endpoint.parse(url, ssl_context: ssl_context, timeout: timeout)
+            client = Async::HTTP::Client.new(endpoint, retries: 0)
+            res = client.get(endpoint.path, default_request_headers)
+
+            headers = res.headers.fields.to_h
+            body = res.read || ""
+
+            results << {
+              url: url,
+              ipv4: ipv4,
+              code: res.status,
+              headers: Utils.to_utf8(headers),
+              body: Utils.to_utf8(body)
+            }
+          rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
+            next
+          end
+        end
+        barrier.wait
+      end
+      results.compact
     end
 
     private
 
-    def default_headers
-      { host: host, user_agent: user_agent }.compact
+    def default_request_headers
+      @default_request_headers ||= headers.merge({ "host" => host, "user-agent" => user_agent }.compact)
     end
 
     def ssl_options
       scheme == "http" ? {} : { ssl_context: ssl_context }
     end
-
-    def get(ipv4)
-      url = "#{scheme}://#{ipv4}:#{port}"
-
-      begin
-        res = HTTP.timeout(timeout).headers(default_headers).get(url, ssl_options)
-        {
-          url: url,
-          ipv4: ipv4,
-          code: res.code,
-          headers: res.headers.to_h,
-          body: res.body.to_s
-        }
-      rescue StandardError
-        nil
-      end
-    end
   end
 end

data/lib/rangescan/utils.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RangeScan
+  class Utils
+    class << self
+      def to_utf8(obj)
+        return obj.dup.force_encoding(Encoding::UTF_8) if obj.is_a?(String)
+
+        obj.map do |k, v|
+          k = k.dup.force_encoding(Encoding::UTF_8) if k.is_a?(String)
+          v = v.dup.force_encoding(Encoding::UTF_8) if v.is_a?(String)
+          [k, v]
+        end.to_h
+      end
+    end
+  end
+end

data/lib/rangescan/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RangeScan
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/rangescan.gemspec

@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.9"
 
-  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "async", "~> 1.26"
+  spec.add_dependency "async-http", "~> 0.52"
   spec.add_dependency "ipaddress", "~> 0.8"
-  spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "thor", "~> 1.0"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rangescan
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-16 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,56 +86,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '1.26'
 - !ruby/object:Gem::Dependency
-  name: ipaddress
+  name: async-http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.52'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.52'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: ipaddress
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -158,6 +158,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".deepsource.toml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -171,16 +172,19 @@ files:
 - lib/rangescan.rb
 - lib/rangescan/cli.rb
 - lib/rangescan/matcher.rb
+- lib/rangescan/monkey_patch.rb
 - lib/rangescan/range.rb
 - lib/rangescan/scanner.rb
+- lib/rangescan/utils.rb
 - lib/rangescan/version.rb
 - rangescan.gemspec
+- renovate.json
 homepage: https://github.com/ninoseki/rangescan
 licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/ninoseki/rangescan
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -196,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Scan websites on a specific IP range
 test_files: []