tansaku 0.3.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50036fe49b8faa2f534977c93c8352afdd2db88542bcf7b78323a0a410f4fb18
-  data.tar.gz: cad855e053c5cad7ebac3f8cef80e546b900a87dd30ad3bae53a9c7503e76c7d
+  metadata.gz: e7814e8eceaa52875d651e478c0e5fa000248441158ef3f59dda8a851c02d4bc
+  data.tar.gz: be7b44389bcf0cf1208336075dd5feaeecc11b88d1d8ffe3efb5f552206baa70
 SHA512:
-  metadata.gz: f4c0c1b4011de97cbbe3eeed76b2f0f07e17487ec0f00e092481ff9f37a0622d161eb1d37dcb938c16c131321da88e04c5a12e7cb1222dce729feb4a58efbf8a
-  data.tar.gz: 0d03f7cae1bde0eb320ca41a3af43c216e13a264fd61966a7bbf0bba09a0a4c630e82659a88d4c0525109ca148796098c895035bf16fa1b0a3a3fadb7c17aa85
+  metadata.gz: 36a13143014938b72714acaa0a103ba4eee8c040597297082f44062356ea3e9aa037fa547dd5d60461aaf1017e45b7727ec50eccc16c768ef39cb579c39eaf2a
+  data.tar.gz: bb4e92fa7b71ebb6878ae1a6bf4c28401ce0ac05d8a13626459e34f6f5a597c5108079c0141bc15ec759d430abb1e2c403986377d37131ce122fa9cb28a2074d

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0", 3.1]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Build and test with Rake
+        run: |
+          bundle exec rake
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ matrix.ruby-version }}
+          parallel: true
+
+  coverage:
+    name: Coverage
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true

data/README.md

@@ -25,22 +25,21 @@ gem install tansaku
 ### As a CLI
 
 ```sh
-$ tansaku
-Commands:
-  tansaku crawl URL       # Crawl a given URL
-  tansaku help [COMMAND]  # Describe available commands or one specific command
-
-$ tansaku help crawl
+$ tansaku help
 Usage:
   tansaku crawl URL
 
 Options:
-  [--additional-list=ADDITIONAL_LIST]  # Path to the file which includes additonal paths to crawl
-  [--host=HOST]                        # Host header to use
-  [--threads=N]                        # Number of threads to use
-  [--type=TYPE]                        # Type of a list to crawl (admin, backup, database, etc, log or all)
-                                       # Default: all
-  [--user-agent=USER_AGENT]            # User-Agent header to use
+  [--additional-list=ADDITIONAL_LIST]                              # Path to the file which includes additional paths to crawl
+  [--headers=key:value]                                            # HTTP headers to use
+  [--method=METHOD]                                                # HTTP method to use
+                                                                   # Default: HEAD
+  [--body=BODY]                                                    # HTTP request body to use
+  [--timeout=N]                                                    # Timeout in seconds
+  [--max-concurrent-requests=N]                                    # Max number of concurrent requests to use
+  [--ignore-certificate-errors], [--no-ignore-certificate-errors]  # Whether to ignore certificate errors or not
+  [--type=TYPE]                                                    # Type of a list to crawl (admin, backup, database, etc, log or all)
+                                                                   # Default: all
 
 Crawl a given URL
 ```

data/exe/tansaku

@@ -5,4 +5,14 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "tansaku"
 
-Tansaku::CLI.start
+def is_help?
+  ARGV.length == 1 && ARGV.first.casecmp("help").zero?
+end
+
+ARGV.unshift(Tansaku::CLI.default_task) unless Tansaku::CLI.all_tasks.key?(ARGV[0])
+
+if is_help?
+  Tansaku::CLI.start(["help", Tansaku::CLI.default_task] )
+else
+  Tansaku::CLI.start(ARGV)
+end

data/lib/tansaku/cli.rb

@@ -7,13 +7,16 @@ require "json"
 module Tansaku
   class CLI < Thor
     desc "crawl URL", "Crawl a given URL"
-    method_option :additional_list, desc: "Path to the file which includes additonal paths to crawl"
-    method_option :host, type: :string, desc: "Host header to use"
-    method_option :threads, type: :numeric, desc: "Number of threads to use"
+    method_option :additional_list, desc: "Path to the file which includes additional paths to crawl"
+    method_option :headers, type: :hash, default: {}, desc: "HTTP headers to use"
+    method_option :method, type: :string, default: "HEAD", desc: "HTTP method to use"
+    method_option :body, type: :string, required: false, default: nil, desc: "HTTP request body to use"
+    method_option :timeout, type: :numeric, required: false, default: nil, desc: "Timeout in seconds"
+    method_option :max_concurrent_requests, type: :numeric, desc: "Max number of concurrent requests to use"
+    method_option :ignore_certificate_errors, type: :boolean, default: false, desc: "Whether to ignore certificate errors or not"
     method_option :type, desc: "Type of a list to crawl (admin, backup, database, etc, log or all)", default: "all"
-    method_option :user_agent, type: :string, desc: "User-Agent header to use"
     def crawl(url)
-      params = options.compact.map { |k, v| [k.to_sym, v] }.to_h
+      params = options.compact.transform_keys(&:to_sym)
       begin
         crawler = Crawler.new(url, **params)
         results = crawler.crawl
@@ -22,5 +25,13 @@ module Tansaku
         puts e
       end
     end
+
+    default_command :crawl
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/tansaku/crawler.rb

@@ -1,60 +1,108 @@
 # frozen_string_literal: true
 
+require "async"
+require "async/barrier"
+require "async/semaphore"
 require "cgi"
-require "net/http"
-require "parallel"
+require "etc"
 require "uri"
 
+require "tansaku/monkey_patch"
+
 module Tansaku
   class Crawler
     DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
 
+    # @return [String]
     attr_reader :base_uri
 
     attr_reader :additional_list
-    attr_reader :host
-    attr_reader :threads
+
+    # @return [Integer]
+    attr_reader :max_concurrent_requests
+
+    # @return [String]
     attr_reader :type
-    attr_reader :user_agent
+
+    # @return [String]
+    attr_reader :method
+
+    # @return [String, nil]
+    attr_reader :body
+
+    # @return [Float, nil]
+    attr_reader :timeout
+
+    # @return [Boolean]
+    attr_reader :ignore_certificate_errors
 
     def initialize(
       base_uri,
       additional_list: nil,
-      host: nil,
-      threads: Parallel.processor_count,
-      type: "all",
-      user_agent: DEFAULT_USER_AGENT
+      headers: {},
+      method: "HEAD",
+      body: nil,
+      timeout: nil,
+      max_concurrent_requests: nil,
+      ignore_certificate_errors: false,
+      type: "all"
     )
-      @base_uri = URI.parse(base_uri)
+      @base_uri = URI.parse(base_uri.downcase)
       raise ArgumentError, "Invalid URI" unless valid_uri?
 
       @additional_list = additional_list
-      unless additional_list.nil?
-        raise ArgumentError, "Invalid path" unless valid_path?
-      end
+      raise ArgumentError, "Invalid path" if !additional_list.nil? && !valid_path?
 
-      @host = host
-      @threads = threads
-      @type = type
-      @user_agent = user_agent
-    end
+      @method = method.upcase
+      raise ArgumentError, "Invalid HTTP method" unless valid_method?
+
+      @headers = headers
+      @body = body
+
+      @timeout = timeout.nil? ? nil : timeout.to_f
 
-    def online?(url)
-      res = head(url)
-      [200, 401, 302].include? res.code.to_i
+      @max_concurrent_requests = max_concurrent_requests || (Etc.nprocessors * 8)
+
+      @ignore_certificate_errors = ignore_certificate_errors
+
+      @type = type
     end
 
     def crawl
-      results = Parallel.map(urls, in_threads: threads) do |url|
-        url if online?(url)
-      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => _e
-        nil
+      results = {}
+
+      Async do |task|
+        barrier = Async::Barrier.new
+        semaphore = Async::Semaphore.new(max_concurrent_requests, parent: barrier)
+        internet = Internet.new
+
+        paths.each do |path|
+          semaphore.async do
+            url = url_for(path)
+
+            res = dispatch_http_request(task, internet, url)
+            next unless online?(res.status)
+
+            log = [method, url, res.status].join(",")
+            Tansaku.logger.info(log)
+
+            results[url] = res.status
+          rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
+            next
+          end
+        end
+        barrier.wait
       end
-      results.compact
+
+      results
     end
 
     private
 
+    def online?(status)
+      [200, 204, 301, 302, 307, 401, 403].include? status.to_i
+    end
+
     def valid_uri?
       ["http", "https"].include? base_uri.scheme
     end
@@ -63,10 +111,14 @@ module Tansaku
       File.exist?(additional_list)
     end
 
+    def valid_method?
+      Protocol::HTTP::Methods.valid? method
+    end
+
     def paths
       paths = Path.get_by_type(type)
       paths += File.readlines(File.expand_path(additional_list, __dir__)) if additional_list
-      paths.map(&:chomp).compact
+      paths.filter_map(&:chomp)
     end
 
     def url_for(path)
@@ -77,16 +129,40 @@ module Tansaku
       paths.map { |path| url_for path }
     end
 
-    def request(req)
-      Net::HTTP.start(base_uri.host, base_uri.port) { |http| http.request(req) }
+    def request_headers
+      @request_headers ||= @headers.tap do |headers|
+        upcase_keys = headers.keys.map(&:downcase).map(&:to_s)
+        headers["user-agent"] = DEFAULT_USER_AGENT unless upcase_keys.include?("user-agent")
+      end.compact
     end
 
-    def head(url)
-      head = Net::HTTP::Head.new(url)
-      head["User-Agent"] = user_agent
-      head["Host"] = host unless host.nil?
+    def ssl_verify_mode
+      ignore_certificate_errors ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+    end
 
-      request(head)
+    def ssl_context
+      @ssl_context ||= OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.set_params(verify_mode: ssl_verify_mode)
+      end
+    end
+
+    #
+    # Dispatch an HTTP request
+    #
+    # @param [Async::Task] task
+    # @param [Tansaku::Internet] internet
+    # @param [String] url
+    #
+    # @return [Async::HTTP::Protocol::Response]
+    #
+    def dispatch_http_request(task, internet, url)
+      endpoint = Async::HTTP::Endpoint.parse(url, ssl_context: ssl_context)
+
+      return internet.call(method, endpoint, request_headers, body) if timeout.nil?
+
+      task.with_timeout(timeout) do
+        internet.call(method, endpoint, request_headers, body)
+      end
     end
   end
 end

data/lib/tansaku/internet.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "async/http/internet"
+
+module Tansaku
+  class Internet < Async::HTTP::Internet
+    def call(method, url_or_endpoint, headers = nil, body = nil)
+      endpoint = if url_or_endpoint.is_a?(Async::HTTP::Endpoint)
+                   url_or_endpoint
+                 else
+                   Async::HTTP::Endpoint.parse(url_or_endpoint)
+                 end
+      key = host_key(endpoint)
+
+      client = @clients.fetch(key) do
+        @clients[key] = client_for(endpoint)
+      end
+
+      body = Async::HTTP::Body::Buffered.wrap(body)
+      headers = ::Protocol::HTTP::Headers[headers]
+
+      request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body)
+
+      client.call(request)
+    end
+
+    ::Protocol::HTTP::Methods.each do |_name, verb|
+      define_method(verb.downcase) do |url_or_endpoint, headers = nil, body = nil|
+        url_or_endpoint = url_or_endpoint.to_str unless url_or_endpoint.is_a?(Async::HTTP::Endpoint)
+        call(verb, url_or_endpoint, headers, body)
+      end
+    end
+  end
+end

data/lib/tansaku/monkey_patch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "protocol/http1/connection"
+
+module Protocol
+  module HTTP1
+    class Connection
+      def write_request(authority, method, path, version, headers)
+        host = authority
+        if headers.include?("host")
+          host = headers["host"]
+          headers.delete "host"
+        end
+
+        @stream.write("#{method} #{path} #{version}\r\n")
+        @stream.write("host: #{host}\r\n")
+
+        write_headers(headers)
+      end
+    end
+  end
+end

data/lib/tansaku/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tansaku
-  VERSION = "0.3.0"
+  VERSION = "1.3.0"
 end

data/lib/tansaku.rb

@@ -1,8 +1,26 @@
 # frozen_string_literal: true
 
 require "tansaku/version"
+
+require "tansaku/internet"
 require "tansaku/path"
+
 require "tansaku/crawler"
+
 require "tansaku/cli"
 
-module Tansaku; end
+require "memist"
+require "semantic_logger"
+
+module Tansaku
+  class << self
+    include Memist::Memoizable
+
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Tansaku"]
+    end
+    memoize :logger
+  end
+end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

data/tansaku.gemspec

@@ -15,22 +15,29 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/ninoseki/tansaku"
   spec.license       = "MIT"
 
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 2.3"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.24"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
+  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "webrick", "~> 1.7"
 
-  spec.add_dependency "parallel", "~> 1.19"
-  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "async", "~> 1.30"
+  spec.add_dependency "async-http", "~> 0.56"
+  spec.add_dependency "memist", "~> 2.0"
+  spec.add_dependency "semantic_logger", "~> 4.10"
+  spec.add_dependency "thor", "~> 1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tansaku
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-30 00:00:00.000000000 Z
+date: 2022-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.24'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.24'
 - !ruby/object:Gem::Dependency
   name: glint
   requirement: !ruby/object:Gem::Requirement
@@ -72,56 +72,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.11'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.14'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.30'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.30'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.56'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.56'
+- !ruby/object:Gem::Dependency
+  name: memist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '4.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '4.10'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 description: Yet another dirbuster tool
 email:
 - manabu.niseki@gmail.com
@@ -131,9 +201,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitattributes"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -144,18 +214,22 @@ files:
 - lib/tansaku.rb
 - lib/tansaku/cli.rb
 - lib/tansaku/crawler.rb
+- lib/tansaku/internet.rb
 - lib/tansaku/lists/admin.txt
 - lib/tansaku/lists/backup.txt
 - lib/tansaku/lists/database.txt
 - lib/tansaku/lists/etc.txt
 - lib/tansaku/lists/log.txt
+- lib/tansaku/monkey_patch.rb
 - lib/tansaku/path.rb
 - lib/tansaku/version.rb
+- renovate.json
 - tansaku.gemspec
 homepage: https://github.com/ninoseki/tansaku
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -171,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Yet another dirbuster tool

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6
-  - 2.7
-before_install:
-  - gem update --system
-  - gem install bundler