tansaku 1.1.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a117357045f40d3f48c578da338c27d4c4e7c5945fb58f9ed5ee1ea5351d7230
-  data.tar.gz: c877f44ac8d888c25ed1c272e461bb53976c75bf191acf96dce0cc82493dd11f
+  metadata.gz: e7814e8eceaa52875d651e478c0e5fa000248441158ef3f59dda8a851c02d4bc
+  data.tar.gz: be7b44389bcf0cf1208336075dd5feaeecc11b88d1d8ffe3efb5f552206baa70
 SHA512:
-  metadata.gz: 7f67a140d2437f7431268890024e67f3bba2e10a5dc7519f6888107fef1c564c1dcc694573d42549ba5f8640a9e08c9fa20ba29024ce3fe167bf6e55d877cc54
-  data.tar.gz: 61785a2947f593c20b8074ad467b1e7326cbea5310fdec0c46a82d1c81139d8f446488766ea3a7f901494ede523c93740a9d67b4eba1b653888edbaf7e7cc2be
+  metadata.gz: 36a13143014938b72714acaa0a103ba4eee8c040597297082f44062356ea3e9aa037fa547dd5d60461aaf1017e45b7727ec50eccc16c768ef39cb579c39eaf2a
+  data.tar.gz: bb4e92fa7b71ebb6878ae1a6bf4c28401ce0ac05d8a13626459e34f6f5a597c5108079c0141bc15ec759d430abb1e2c403986377d37131ce122fa9cb28a2074d

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Ruby CI
+
+on: [pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, "3.0", 3.1]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Build and test with Rake
+        run: |
+          bundle exec rake
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: run-${{ matrix.ruby-version }}
+          parallel: true
+
+  coverage:
+    name: Coverage
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true

data/README.md

@@ -25,22 +25,21 @@ gem install tansaku
 ### As a CLI
 
 ```sh
-$ tansaku
-Commands:
-  tansaku crawl URL       # Crawl a given URL
-  tansaku help [COMMAND]  # Describe available commands or one specific command
-
-$ tansaku help crawl
+$ tansaku help
 Usage:
   tansaku crawl URL
 
 Options:
-  [--additional-list=ADDITIONAL_LIST]  # Path to the file which includes additonal paths to crawl
-  [--host=HOST]                        # Host header to use
-  [--max-concurrent-requests=N]        # Number of concurrent requests to use
-  [--type=TYPE]                        # Type of a list to crawl (admin, backup, database, etc, log or all)
-                                       # Default: all
-  [--user-agent=USER_AGENT]            # User-Agent header to use
+  [--additional-list=ADDITIONAL_LIST]                              # Path to the file which includes additional paths to crawl
+  [--headers=key:value]                                            # HTTP headers to use
+  [--method=METHOD]                                                # HTTP method to use
+                                                                   # Default: HEAD
+  [--body=BODY]                                                    # HTTP request body to use
+  [--timeout=N]                                                    # Timeout in seconds
+  [--max-concurrent-requests=N]                                    # Max number of concurrent requests to use
+  [--ignore-certificate-errors], [--no-ignore-certificate-errors]  # Whether to ignore certificate errors or not
+  [--type=TYPE]                                                    # Type of a list to crawl (admin, backup, database, etc, log or all)
+                                                                   # Default: all
 
 Crawl a given URL
 ```

data/exe/tansaku

@@ -5,5 +5,14 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "tansaku"
 
+def is_help?
+  ARGV.length == 1 && ARGV.first.casecmp("help").zero?
+end
+
 ARGV.unshift(Tansaku::CLI.default_task) unless Tansaku::CLI.all_tasks.key?(ARGV[0])
-Tansaku::CLI.start(ARGV)
+
+if is_help?
+  Tansaku::CLI.start(["help", Tansaku::CLI.default_task] )
+else
+  Tansaku::CLI.start(ARGV)
+end

data/lib/tansaku/cli.rb

@@ -7,14 +7,16 @@ require "json"
 module Tansaku
   class CLI < Thor
     desc "crawl URL", "Crawl a given URL"
-    method_option :additional_list, desc: "Path to the file which includes additonal paths to crawl"
-    method_option :headers, type: :hash, default: {}, desc: "Custom headers"
-    method_option :host, type: :string, desc: "Host header to use"
-    method_option :max_concurrent_requests, type: :numeric, desc: "Number of concurrent requests to use"
+    method_option :additional_list, desc: "Path to the file which includes additional paths to crawl"
+    method_option :headers, type: :hash, default: {}, desc: "HTTP headers to use"
+    method_option :method, type: :string, default: "HEAD", desc: "HTTP method to use"
+    method_option :body, type: :string, required: false, default: nil, desc: "HTTP request body to use"
+    method_option :timeout, type: :numeric, required: false, default: nil, desc: "Timeout in seconds"
+    method_option :max_concurrent_requests, type: :numeric, desc: "Max number of concurrent requests to use"
+    method_option :ignore_certificate_errors, type: :boolean, default: false, desc: "Whether to ignore certificate errors or not"
     method_option :type, desc: "Type of a list to crawl (admin, backup, database, etc, log or all)", default: "all"
-    method_option :user_agent, type: :string, desc: "User-Agent header to use"
     def crawl(url)
-      params = options.compact.map { |k, v| [k.to_sym, v] }.to_h
+      params = options.compact.transform_keys(&:to_sym)
       begin
         crawler = Crawler.new(url, **params)
         results = crawler.crawl

data/lib/tansaku/crawler.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "async/http/internet"
 require "async"
 require "async/barrier"
 require "async/semaphore"
@@ -14,58 +13,87 @@ module Tansaku
   class Crawler
     DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
 
+    # @return [String]
     attr_reader :base_uri
 
     attr_reader :additional_list
-    attr_reader :headers
-    attr_reader :host
+
+    # @return [Integer]
     attr_reader :max_concurrent_requests
+
+    # @return [String]
     attr_reader :type
-    attr_reader :user_agent
+
+    # @return [String]
+    attr_reader :method
+
+    # @return [String, nil]
+    attr_reader :body
+
+    # @return [Float, nil]
+    attr_reader :timeout
+
+    # @return [Boolean]
+    attr_reader :ignore_certificate_errors
 
     def initialize(
       base_uri,
       additional_list: nil,
       headers: {},
-      host: nil,
+      method: "HEAD",
+      body: nil,
+      timeout: nil,
       max_concurrent_requests: nil,
-      type: "all",
-      user_agent: DEFAULT_USER_AGENT
+      ignore_certificate_errors: false,
+      type: "all"
     )
-      @base_uri = URI.parse(base_uri)
+      @base_uri = URI.parse(base_uri.downcase)
       raise ArgumentError, "Invalid URI" unless valid_uri?
 
       @additional_list = additional_list
-      unless additional_list.nil?
-        raise ArgumentError, "Invalid path" unless valid_path?
-      end
+      raise ArgumentError, "Invalid path" if !additional_list.nil? && !valid_path?
+
+      @method = method.upcase
+      raise ArgumentError, "Invalid HTTP method" unless valid_method?
 
       @headers = headers
-      @host = host
-      @max_concurrent_requests = max_concurrent_requests || Etc.nprocessors * 8
+      @body = body
+
+      @timeout = timeout.nil? ? nil : timeout.to_f
+
+      @max_concurrent_requests = max_concurrent_requests || (Etc.nprocessors * 8)
+
+      @ignore_certificate_errors = ignore_certificate_errors
+
       @type = type
-      @user_agent = user_agent
     end
 
     def crawl
       results = {}
-      Async do
+
+      Async do |task|
         barrier = Async::Barrier.new
         semaphore = Async::Semaphore.new(max_concurrent_requests, parent: barrier)
-        internet = Async::HTTP::Internet.new
+        internet = Internet.new
 
         paths.each do |path|
           semaphore.async do
             url = url_for(path)
-            res = internet.head(url, default_request_headers)
 
-            results[url] = res.status if online?(res.status)
+            res = dispatch_http_request(task, internet, url)
+            next unless online?(res.status)
+
+            log = [method, url, res.status].join(",")
+            Tansaku.logger.info(log)
+
+            results[url] = res.status
           rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
             next
           end
         end
         barrier.wait
       end
+
       results
     end
 
@@ -83,10 +111,14 @@ module Tansaku
       File.exist?(additional_list)
     end
 
+    def valid_method?
+      Protocol::HTTP::Methods.valid? method
+    end
+
     def paths
       paths = Path.get_by_type(type)
       paths += File.readlines(File.expand_path(additional_list, __dir__)) if additional_list
-      paths.map(&:chomp).compact
+      paths.filter_map(&:chomp)
     end
 
     def url_for(path)
@@ -97,8 +129,40 @@ module Tansaku
       paths.map { |path| url_for path }
     end
 
-    def default_request_headers
-      @default_request_headers ||= headers.merge({ "host" => host, "user-agent" => user_agent }.compact)
+    def request_headers
+      @request_headers ||= @headers.tap do |headers|
+        upcase_keys = headers.keys.map(&:downcase).map(&:to_s)
+        headers["user-agent"] = DEFAULT_USER_AGENT unless upcase_keys.include?("user-agent")
+      end.compact
+    end
+
+    def ssl_verify_mode
+      ignore_certificate_errors ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+    end
+
+    def ssl_context
+      @ssl_context ||= OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.set_params(verify_mode: ssl_verify_mode)
+      end
+    end
+
+    #
+    # Dispatch an HTTP request
+    #
+    # @param [Async::Task] task
+    # @param [Tansaku::Internet] internet
+    # @param [String] url
+    #
+    # @return [Async::HTTP::Protocol::Response]
+    #
+    def dispatch_http_request(task, internet, url)
+      endpoint = Async::HTTP::Endpoint.parse(url, ssl_context: ssl_context)
+
+      return internet.call(method, endpoint, request_headers, body) if timeout.nil?
+
+      task.with_timeout(timeout) do
+        internet.call(method, endpoint, request_headers, body)
+      end
     end
   end
 end

data/lib/tansaku/internet.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "async/http/internet"
+
+module Tansaku
+  class Internet < Async::HTTP::Internet
+    def call(method, url_or_endpoint, headers = nil, body = nil)
+      endpoint = if url_or_endpoint.is_a?(Async::HTTP::Endpoint)
+                   url_or_endpoint
+                 else
+                   Async::HTTP::Endpoint.parse(url_or_endpoint)
+                 end
+      key = host_key(endpoint)
+
+      client = @clients.fetch(key) do
+        @clients[key] = client_for(endpoint)
+      end
+
+      body = Async::HTTP::Body::Buffered.wrap(body)
+      headers = ::Protocol::HTTP::Headers[headers]
+
+      request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body)
+
+      client.call(request)
+    end
+
+    ::Protocol::HTTP::Methods.each do |_name, verb|
+      define_method(verb.downcase) do |url_or_endpoint, headers = nil, body = nil|
+        url_or_endpoint = url_or_endpoint.to_str unless url_or_endpoint.is_a?(Async::HTTP::Endpoint)
+        call(verb, url_or_endpoint, headers, body)
+      end
+    end
+  end
+end

data/lib/tansaku/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tansaku
-  VERSION = "1.1.0"
+  VERSION = "1.3.0"
 end

data/lib/tansaku.rb

@@ -1,8 +1,26 @@
 # frozen_string_literal: true
 
 require "tansaku/version"
+
+require "tansaku/internet"
 require "tansaku/path"
+
 require "tansaku/crawler"
+
 require "tansaku/cli"
 
-module Tansaku; end
+require "memist"
+require "semantic_logger"
+
+module Tansaku
+  class << self
+    include Memist::Memoizable
+
+    def logger
+      SemanticLogger.default_level = :info
+      SemanticLogger.add_appender(io: $stderr, formatter: :color)
+      SemanticLogger["Tansaku"]
+    end
+    memoize :logger
+  end
+end

data/tansaku.gemspec

@@ -15,23 +15,29 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/ninoseki/tansaku"
   spec.license       = "MIT"
 
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "bundler", "~> 2.3"
+  spec.add_development_dependency "coveralls_reborn", "~> 0.24"
   spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "webmock", "~> 3.8"
+  spec.add_development_dependency "rspec", "~> 3.11"
+  spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
+  spec.add_development_dependency "webmock", "~> 3.14"
+  spec.add_development_dependency "webrick", "~> 1.7"
 
-  spec.add_dependency "async", "~> 1.26"
-  spec.add_dependency "async-http", "~> 0.52"
-  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "async", "~> 1.30"
+  spec.add_dependency "async-http", "~> 0.56"
+  spec.add_dependency "memist", "~> 2.0"
+  spec.add_dependency "semantic_logger", "~> 4.10"
+  spec.add_dependency "thor", "~> 1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tansaku
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2022-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: coveralls_reborn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.24'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.24'
 - !ruby/object:Gem::Dependency
   name: glint
   requirement: !ruby/object:Gem::Requirement
@@ -72,70 +72,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.11'
+- !ruby/object:Gem::Dependency
+  name: simplecov-lcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '3.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.14'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.30'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.30'
 - !ruby/object:Gem::Dependency
   name: async-http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.56'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.56'
+- !ruby/object:Gem::Dependency
+  name: memist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '4.10'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 description: Yet another dirbuster tool
 email:
 - manabu.niseki@gmail.com
@@ -145,9 +201,9 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitattributes"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -158,6 +214,7 @@ files:
 - lib/tansaku.rb
 - lib/tansaku/cli.rb
 - lib/tansaku/crawler.rb
+- lib/tansaku/internet.rb
 - lib/tansaku/lists/admin.txt
 - lib/tansaku/lists/backup.txt
 - lib/tansaku/lists/database.txt
@@ -171,7 +228,8 @@ files:
 homepage: https://github.com/ninoseki/tansaku
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -187,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Yet another dirbuster tool

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.6
-  - 2.7
-before_install:
-  - gem update --system
-  - gem install bundler