tansaku 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a179d2d95fc0e6c78d908ff8379ecefb609448f38284f77c21fcd92dd8a0b7ca
-  data.tar.gz: 0eaecd595420390290a4e5b9f8a759b229e096037e60d058d58fd6af02dfe9f0
+  metadata.gz: a117357045f40d3f48c578da338c27d4c4e7c5945fb58f9ed5ee1ea5351d7230
+  data.tar.gz: c877f44ac8d888c25ed1c272e461bb53976c75bf191acf96dce0cc82493dd11f
 SHA512:
-  metadata.gz: a4b252ff9c13cbe70109f8a15b77e9923672f15c5de2f93f26af6e6a0031ebdf0ed2c246e76968843e66f6e4eda0cdeae067b3886c8525237f5a412032780b63
-  data.tar.gz: '01952d73467ad3fafce9b3726675c7f145f7db359052cb6682e533f7faa2c42022f8e2f38e07a3f63fe132ee36d8a8dfcbfdc61563c84d9b55d3a8536b1b3146'
+  metadata.gz: 7f67a140d2437f7431268890024e67f3bba2e10a5dc7519f6888107fef1c564c1dcc694573d42549ba5f8640a9e08c9fa20ba29024ce3fe167bf6e55d877cc54
+  data.tar.gz: 61785a2947f593c20b8074ad467b1e7326cbea5310fdec0c46a82d1c81139d8f446488766ea3a7f901494ede523c93740a9d67b4eba1b653888edbaf7e7cc2be

data/exe/tansaku

@@ -5,4 +5,5 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
 
 require "tansaku"
 
-Tansaku::CLI.start
+ARGV.unshift(Tansaku::CLI.default_task) unless Tansaku::CLI.all_tasks.key?(ARGV[0])
+Tansaku::CLI.start(ARGV)

data/lib/tansaku/cli.rb

@@ -8,6 +8,7 @@ module Tansaku
   class CLI < Thor
     desc "crawl URL", "Crawl a given URL"
     method_option :additional_list, desc: "Path to the file which includes additonal paths to crawl"
+    method_option :headers, type: :hash, default: {}, desc: "Custom headers"
     method_option :host, type: :string, desc: "Host header to use"
     method_option :max_concurrent_requests, type: :numeric, desc: "Number of concurrent requests to use"
     method_option :type, desc: "Type of a list to crawl (admin, backup, database, etc, log or all)", default: "all"
@@ -22,5 +23,13 @@ module Tansaku
         puts e
       end
     end
+
+    default_command :crawl
+
+    class << self
+      def exit_on_failure?
+        true
+      end
+    end
   end
 end

data/lib/tansaku/crawler.rb

@@ -17,6 +17,7 @@ module Tansaku
     attr_reader :base_uri
 
     attr_reader :additional_list
+    attr_reader :headers
     attr_reader :host
     attr_reader :max_concurrent_requests
     attr_reader :type
@@ -25,8 +26,9 @@ module Tansaku
     def initialize(
       base_uri,
       additional_list: nil,
+      headers: {},
       host: nil,
-      max_concurrent_requests: Etc.nprocessors,
+      max_concurrent_requests: nil,
       type: "all",
       user_agent: DEFAULT_USER_AGENT
     )
@@ -38,8 +40,9 @@ module Tansaku
         raise ArgumentError, "Invalid path" unless valid_path?
       end
 
+      @headers = headers
       @host = host
-      @max_concurrent_requests = max_concurrent_requests
+      @max_concurrent_requests = max_concurrent_requests || Etc.nprocessors * 8
       @type = type
       @user_agent = user_agent
     end
@@ -95,7 +98,7 @@ module Tansaku
     end
 
     def default_request_headers
-      @default_request_headers ||= { "host" => host, "user-agent" => user_agent }.compact
+      @default_request_headers ||= headers.merge({ "host" => host, "user-agent" => user_agent }.compact)
     end
   end
 end

data/lib/tansaku/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tansaku
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/renovate.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "config:base"
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tansaku
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -166,6 +166,7 @@ files:
 - lib/tansaku/monkey_patch.rb
 - lib/tansaku/path.rb
 - lib/tansaku/version.rb
+- renovate.json
 - tansaku.gemspec
 homepage: https://github.com/ninoseki/tansaku
 licenses: