rangescan 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7560ddf707ab1d55befca7e8cdffd4274618845256d4da74575d99d613c4ab9
-  data.tar.gz: 296098c05eac8e1c7e6294a503aa6b591b2617d2fcdc3f6f4093299e9a174a53
+  metadata.gz: 4144b13853677049e63de5a4214bffd4140923b9627aa165793bd08a5fd3664f
+  data.tar.gz: 5e9a1489605be94188c2a4310013062451630b12764db065af20081c1bc2d2af
 SHA512:
-  metadata.gz: c8a7d6badbbaa041657f86015462f3049cce8e852073a268caadfcfc76fbfd555b31c69b1380ee4846bc92279ce98cb4758d33612d7a4656bad6ca5e2a1d06f0
-  data.tar.gz: 51537dc5b498beb430a225486b6b2ce259172e485d5bde54ed5400b191f2d909a7cb01bcf4372f1a6af72d8b1dd33e06da605449c53270d69bc60077b8e1748d
+  metadata.gz: e82b98bb97b2742ca54bcfd47cecde7a4bee65d7fefe841f7b76cc6dfec7832c397e8d2e422afc5f64a2533ff453bb3cf83c6ad1cbeb98948845ab5710eb84e6
+  data.tar.gz: 5e6e6173d6a858749fce649d25fd658a91a0d2941238f2ca730e07511e147e48440d713e07247c4a40c3299765760b403596b4122ac259a52d5e3847bfbfb4c2

data/.deepsource.toml

@@ -0,0 +1,5 @@
+version = 1
+
+[[analyzers]]
+name = "ruby"
+enabled = true

data/.gitignore

@@ -45,9 +45,9 @@ build-iPhoneSimulator/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
+Gemfile.lock
+.ruby-version
+.ruby-gemset
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc

data/README.md

@@ -1,5 +1,6 @@
 # rangescan
 
+[![Gem Version](https://badge.fury.io/rb/rangescan.svg)](https://badge.fury.io/rb/rangescan)
 [![Build Status](https://travis-ci.com/ninoseki/rangescan.svg?branch=master)](https://travis-ci.com/ninoseki/rangescan)
 [![Coverage Status](https://coveralls.io/repos/github/ninoseki/rangescan/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/rangescan?branch=master)
 [![CodeFactor](https://www.codefactor.io/repository/github/ninoseki/rangescan/badge)](https://www.codefactor.io/repository/github/ninoseki/rangescan)
@@ -26,13 +27,14 @@ Usage:
 
 Options:
   [--host=HOST]                      # Host header
-  [--port=N]                         # Port to check (80 or 443)
-  [--scheme=SCHEME]                  # Scheme to use (http or https)
+  [--port=N]                         # Port
+  [--scheme=SCHEME]                  # Scheme (http or https)
   [--timeout=N]                      # Timeout in seconds
-  [--user-agent=USER_AGENT]          # User Agent header
-  [--veryfy-ssl], [--no-veryfy-ssl]  # Verify SSL or not
+  [--user-agent=USER_AGENT]          # User Agent
+  [--verify-ssl], [--no-verify-ssl]  # Whether to verify SSL or not
+  [--max-concurrency=N]              # Concurrency limit for HTTP requests to scan
 
-Scan an IP range & filter by a regexp
+Scan an IP range & filter by a regexp (default regexp = .)
 ```
 
 ## License

data/lib/rangescan.rb

@@ -3,6 +3,7 @@
 require "rangescan/version"
 
 require "rangescan/range"
+require "rangescan/utils"
 require "rangescan/scanner"
 require "rangescan/matcher"
 

data/lib/rangescan/cli.rb

@@ -5,14 +5,15 @@ require "json"
 
 module RangeScan
   class CLI < Thor
-    desc "scan [IP_WITH_SUBNET_MASK, REGEXP]", "Scan an IP range & filter by a regexp"
+    desc "scan [IP_WITH_SUBNET_MASK, REGEXP]", "Scan an IP range & filter by a regexp (default regexp = .)"
     method_option :host, type: :string, desc: "Host header"
     method_option :port, type: :numeric, desc: "Port"
     method_option :scheme, type: :string, desc: "Scheme (http or https)"
     method_option :timeout, type: :numeric, desc: "Timeout in seconds"
     method_option :user_agent, type: :string, desc: "User Agent"
     method_option :verify_ssl, type: :boolean, desc: "Whether to verify SSL or not"
-    def scan(ip_with_subnet_mask, regexp)
+    method_option :max_concurrency, type: :numeric, desc: "Concurrency limit for HTTP requests to scan"
+    def scan(ip_with_subnet_mask, regexp = ".")
       symbolized_options = symbolize_hash_keys(options)
       range = Range.new(ip_with_subnet_mask)
 

data/lib/rangescan/matcher.rb

@@ -11,7 +11,11 @@ module RangeScan
     def filter(results)
       results.select do |result|
         body = result.dig(:body) || ""
-        body =~ regexp
+        begin
+          body =~ regexp
+        rescue ArgumentError, Encoding::CompatibilityError
+          false
+        end
       end
     end
   end

data/lib/rangescan/monkey_patch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "protocol/http1/connection"
+
+module Protocol
+  module HTTP1
+    class Connection
+      def write_request(authority, method, path, version, headers)
+        host = authority
+        if headers.include?("host")
+          host = headers["host"]
+          headers.delete "host"
+        end
+
+        @stream.write("#{method} #{path} #{version}\r\n")
+        @stream.write("host: #{host}\r\n")
+
+        write_headers(headers)
+      end
+    end
+  end
+end

data/lib/rangescan/scanner.rb

@@ -1,54 +1,88 @@
 # frozen_string_literal: true
 
-require "http"
-require "parallel"
+require "async"
+require "async/barrier"
+require "async/semaphore"
+require "async/http"
+require "etc"
+
+require "rangescan/monkey_patch"
 
 module RangeScan
   class Scanner
     attr_reader :context
     attr_reader :host
+    attr_reader :max_concurrency
     attr_reader :port
+    attr_reader :processor_count
     attr_reader :scheme
     attr_reader :ssl_context
     attr_reader :timeout
     attr_reader :user_agent
+    attr_reader :verify_ssl
 
-    def initialize(host: nil, port: nil, scheme: "http", verify_ssl: true, timeout: 5, user_agent: nil)
+    def initialize(host: nil, port: nil, scheme: "http", verify_ssl: true, timeout: 5, user_agent: nil, max_concurrency: nil)
       @host = host
       @port = port || (scheme == "http" ? 80 : 443)
       @timeout = timeout
       @scheme = scheme
       @user_agent = user_agent
 
+      @verify_ssl = verify_ssl
+
       @ssl_context = OpenSSL::SSL::SSLContext.new
       @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE unless verify_ssl
+
+      @max_concurrency = max_concurrency || Etc.nprocessors * 2
+    end
+
+    def url_for(ipv4)
+      return "#{scheme}://#{ipv4}" if (port == 80 && scheme == "http") || (port == 443 && scheme == "https")
+
+      "#{scheme}://#{ipv4}:#{port}"
     end
 
     def scan(ipv4s)
-      Parallel.map(ipv4s) do |ipv4|
-        get ipv4
-      end.compact
+      results = []
+      Async do
+        barrier = Async::Barrier.new
+        semaphore = Async::Semaphore.new(max_concurrency, parent: barrier)
+
+        ipv4s.each do |ipv4|
+          semaphore.async do
+            url = url_for(ipv4)
+
+            endpoint = Async::HTTP::Endpoint.parse(url, ssl_context: ssl_context, timeout: timeout)
+            client = Async::HTTP::Client.new(endpoint, retries: 0)
+            res = client.get(endpoint.path, default_request_headers)
+
+            headers = res.headers.fields.to_h
+            body = res.read || ""
+
+            results << {
+              url: url,
+              ipv4: ipv4,
+              code: res.status,
+              headers: Utils.to_utf8(headers),
+              body: Utils.to_utf8(body)
+            }
+          rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
+            next
+          end
+        end
+        barrier.wait
+      end
+      results.compact
     end
 
     private
 
-    def default_headers
-      { host: host, user_agent: user_agent }.compact
+    def default_request_headers
+      @default_request_headers ||= { "host" => host, "user-agent" => user_agent }.compact
     end
 
     def ssl_options
       scheme == "http" ? {} : { ssl_context: ssl_context }
     end
-
-    def get(ipv4)
-      url = "#{scheme}://#{ipv4}:#{port}"
-
-      begin
-        res = HTTP.timeout(timeout).headers(default_headers).get(url, ssl_options)
-        { ipv4: ipv4, code: res.code, body: res.body.to_s }
-      rescue StandardError
-        nil
-      end
-    end
   end
 end

data/lib/rangescan/utils.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RangeScan
+  class Utils
+    class << self
+      def to_utf8(obj)
+        return obj.dup.force_encoding(Encoding::UTF_8) if obj.is_a?(String)
+
+        obj.map do |k, v|
+          k = k.dup.force_encoding(Encoding::UTF_8) if k.is_a?(String)
+          v = v.dup.force_encoding(Encoding::UTF_8) if v.is_a?(String)
+          [k, v]
+        end.to_h
+      end
+    end
+  end
+end

data/lib/rangescan/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RangeScan
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/rangescan.gemspec

@@ -30,10 +30,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "webmock", "~> 3.9"
 
-  spec.add_dependency "http", "~> 4.4"
+  spec.add_dependency "async", "~> 1.26"
+  spec.add_dependency "async-http", "~> 0.52"
   spec.add_dependency "ipaddress", "~> 0.8"
-  spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "thor", "~> 1.0"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rangescan
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Manabu Niseki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-16 00:00:00.000000000 Z
+date: 2020-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,56 +86,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '1.26'
 - !ruby/object:Gem::Dependency
-  name: ipaddress
+  name: async-http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.52'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.52'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: ipaddress
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -158,11 +158,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".deepsource.toml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
@@ -172,16 +172,19 @@ files:
 - lib/rangescan.rb
 - lib/rangescan/cli.rb
 - lib/rangescan/matcher.rb
+- lib/rangescan/monkey_patch.rb
 - lib/rangescan/range.rb
 - lib/rangescan/scanner.rb
+- lib/rangescan/utils.rb
 - lib/rangescan/version.rb
 - rangescan.gemspec
+- renovate.json
 homepage: https://github.com/ninoseki/rangescan
 licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/ninoseki/rangescan
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -197,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Scan websites on a specific IP range
 test_files: []

data/Gemfile.lock

@@ -1,94 +0,0 @@
-PATH
-  remote: .
-  specs:
-    rangescan (0.1.0)
-      http (~> 4.4)
-      ipaddress (~> 0.8)
-      parallel (~> 1.19)
-      thor (~> 1.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    coveralls (0.8.23)
-      json (>= 1.8, < 3)
-      simplecov (~> 0.16.1)
-      term-ansicolor (~> 1.3)
-      thor (>= 0.19.4, < 2.0)
-      tins (~> 1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    diff-lcs (1.3)
-    docile (1.3.2)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    ffi (1.13.1)
-    ffi-compiler (1.0.1)
-      ffi (>= 1.0.0)
-      rake
-    glint (0.1.0)
-    hashdiff (1.0.1)
-    http (4.4.1)
-      addressable (~> 2.3)
-      http-cookie (~> 1.0)
-      http-form_data (~> 2.2)
-      http-parser (~> 1.2.0)
-    http-cookie (1.0.3)
-      domain_name (~> 0.5)
-    http-form_data (2.3.0)
-    http-parser (1.2.1)
-      ffi-compiler (>= 1.0, < 2.0)
-    ipaddress (0.8.3)
-    json (2.3.0)
-    parallel (1.19.1)
-    public_suffix (4.0.5)
-    rake (13.0.1)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    safe_yaml (1.0.5)
-    simplecov (0.16.1)
-      docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    sync (0.5.0)
-    term-ansicolor (1.7.1)
-      tins (~> 1.0)
-    thor (1.0.1)
-    tins (1.25.0)
-      sync
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.7)
-    webmock (3.8.3)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 2.1)
-  coveralls (~> 0.8)
-  glint (~> 0.1)
-  rake (~> 13.0)
-  rangescan!
-  rspec (~> 3.9)
-  webmock (~> 3.8)
-
-BUNDLED WITH
-   2.1.4