tansaku 0.3.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +41 -0
- data/README.md +11 -12
- data/exe/tansaku +11 -1
- data/lib/tansaku/cli.rb +16 -5
- data/lib/tansaku/crawler.rb +110 -34
- data/lib/tansaku/internet.rb +34 -0
- data/lib/tansaku/monkey_patch.rb +22 -0
- data/lib/tansaku/version.rb +1 -1
- data/lib/tansaku.rb +19 -1
- data/renovate.json +5 -0
- data/tansaku.gemspec +14 -7
- metadata +93 -19
- data/.travis.yml +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e7814e8eceaa52875d651e478c0e5fa000248441158ef3f59dda8a851c02d4bc
|
4
|
+
data.tar.gz: be7b44389bcf0cf1208336075dd5feaeecc11b88d1d8ffe3efb5f552206baa70
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 36a13143014938b72714acaa0a103ba4eee8c040597297082f44062356ea3e9aa037fa547dd5d60461aaf1017e45b7727ec50eccc16c768ef39cb579c39eaf2a
|
7
|
+
data.tar.gz: bb4e92fa7b71ebb6878ae1a6bf4c28401ce0ac05d8a13626459e34f6f5a597c5108079c0141bc15ec759d430abb1e2c403986377d37131ce122fa9cb28a2074d
|
@@ -0,0 +1,41 @@
|
|
1
|
+
name: Ruby CI
|
2
|
+
|
3
|
+
on: [pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
|
9
|
+
strategy:
|
10
|
+
fail-fast: false
|
11
|
+
matrix:
|
12
|
+
ruby: [2.7, "3.0", 3.1]
|
13
|
+
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v3
|
16
|
+
- name: Set up Ruby
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
bundler-cache: true
|
21
|
+
- name: Build and test with Rake
|
22
|
+
run: |
|
23
|
+
bundle exec rake
|
24
|
+
|
25
|
+
- name: Coveralls Parallel
|
26
|
+
uses: coverallsapp/github-action@master
|
27
|
+
with:
|
28
|
+
github-token: ${{ secrets.github_token }}
|
29
|
+
flag-name: run-${{ matrix.ruby-version }}
|
30
|
+
parallel: true
|
31
|
+
|
32
|
+
coverage:
|
33
|
+
name: Coverage
|
34
|
+
needs: test
|
35
|
+
runs-on: ubuntu-latest
|
36
|
+
steps:
|
37
|
+
- name: Coveralls Finished
|
38
|
+
uses: coverallsapp/github-action@master
|
39
|
+
with:
|
40
|
+
github-token: ${{ secrets.github_token }}
|
41
|
+
parallel-finished: true
|
data/README.md
CHANGED
@@ -25,22 +25,21 @@ gem install tansaku
|
|
25
25
|
### As a CLI
|
26
26
|
|
27
27
|
```sh
|
28
|
-
$ tansaku
|
29
|
-
Commands:
|
30
|
-
tansaku crawl URL # Crawl a given URL
|
31
|
-
tansaku help [COMMAND] # Describe available commands or one specific command
|
32
|
-
|
33
|
-
$ tansaku help crawl
|
28
|
+
$ tansaku help
|
34
29
|
Usage:
|
35
30
|
tansaku crawl URL
|
36
31
|
|
37
32
|
Options:
|
38
|
-
[--additional-list=ADDITIONAL_LIST]
|
39
|
-
[--
|
40
|
-
[--
|
41
|
-
|
42
|
-
|
43
|
-
[--
|
33
|
+
[--additional-list=ADDITIONAL_LIST] # Path to the file which includes additional paths to crawl
|
34
|
+
[--headers=key:value] # HTTP headers to use
|
35
|
+
[--method=METHOD] # HTTP method to use
|
36
|
+
# Default: HEAD
|
37
|
+
[--body=BODY] # HTTP request body to use
|
38
|
+
[--timeout=N] # Timeout in seconds
|
39
|
+
[--max-concurrent-requests=N] # Max number of concurrent requests to use
|
40
|
+
[--ignore-certificate-errors], [--no-ignore-certificate-errors] # Whether to ignore certificate errors or not
|
41
|
+
[--type=TYPE] # Type of a list to crawl (admin, backup, database, etc, log or all)
|
42
|
+
# Default: all
|
44
43
|
|
45
44
|
Crawl a given URL
|
46
45
|
```
|
data/exe/tansaku
CHANGED
@@ -5,4 +5,14 @@ $LOAD_PATH.unshift("#{__dir__}/../lib")
|
|
5
5
|
|
6
6
|
require "tansaku"
|
7
7
|
|
8
|
-
|
8
|
+
def is_help?
|
9
|
+
ARGV.length == 1 && ARGV.first.casecmp("help").zero?
|
10
|
+
end
|
11
|
+
|
12
|
+
ARGV.unshift(Tansaku::CLI.default_task) unless Tansaku::CLI.all_tasks.key?(ARGV[0])
|
13
|
+
|
14
|
+
if is_help?
|
15
|
+
Tansaku::CLI.start(["help", Tansaku::CLI.default_task] )
|
16
|
+
else
|
17
|
+
Tansaku::CLI.start(ARGV)
|
18
|
+
end
|
data/lib/tansaku/cli.rb
CHANGED
@@ -7,13 +7,16 @@ require "json"
|
|
7
7
|
module Tansaku
|
8
8
|
class CLI < Thor
|
9
9
|
desc "crawl URL", "Crawl a given URL"
|
10
|
-
method_option :additional_list, desc: "Path to the file which includes
|
11
|
-
method_option :
|
12
|
-
method_option :
|
10
|
+
method_option :additional_list, desc: "Path to the file which includes additional paths to crawl"
|
11
|
+
method_option :headers, type: :hash, default: {}, desc: "HTTP headers to use"
|
12
|
+
method_option :method, type: :string, default: "HEAD", desc: "HTTP method to use"
|
13
|
+
method_option :body, type: :string, required: false, default: nil, desc: "HTTP request body to use"
|
14
|
+
method_option :timeout, type: :numeric, required: false, default: nil, desc: "Timeout in seconds"
|
15
|
+
method_option :max_concurrent_requests, type: :numeric, desc: "Max number of concurrent requests to use"
|
16
|
+
method_option :ignore_certificate_errors, type: :boolean, default: false, desc: "Whether to ignore certificate errors or not"
|
13
17
|
method_option :type, desc: "Type of a list to crawl (admin, backup, database, etc, log or all)", default: "all"
|
14
|
-
method_option :user_agent, type: :string, desc: "User-Agent header to use"
|
15
18
|
def crawl(url)
|
16
|
-
params = options.compact.
|
19
|
+
params = options.compact.transform_keys(&:to_sym)
|
17
20
|
begin
|
18
21
|
crawler = Crawler.new(url, **params)
|
19
22
|
results = crawler.crawl
|
@@ -22,5 +25,13 @@ module Tansaku
|
|
22
25
|
puts e
|
23
26
|
end
|
24
27
|
end
|
28
|
+
|
29
|
+
default_command :crawl
|
30
|
+
|
31
|
+
class << self
|
32
|
+
def exit_on_failure?
|
33
|
+
true
|
34
|
+
end
|
35
|
+
end
|
25
36
|
end
|
26
37
|
end
|
data/lib/tansaku/crawler.rb
CHANGED
@@ -1,60 +1,108 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "async"
|
4
|
+
require "async/barrier"
|
5
|
+
require "async/semaphore"
|
3
6
|
require "cgi"
|
4
|
-
require "
|
5
|
-
require "parallel"
|
7
|
+
require "etc"
|
6
8
|
require "uri"
|
7
9
|
|
10
|
+
require "tansaku/monkey_patch"
|
11
|
+
|
8
12
|
module Tansaku
|
9
13
|
class Crawler
|
10
14
|
DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
|
11
15
|
|
16
|
+
# @return [String]
|
12
17
|
attr_reader :base_uri
|
13
18
|
|
14
19
|
attr_reader :additional_list
|
15
|
-
|
16
|
-
|
20
|
+
|
21
|
+
# @return [Integer]
|
22
|
+
attr_reader :max_concurrent_requests
|
23
|
+
|
24
|
+
# @return [String]
|
17
25
|
attr_reader :type
|
18
|
-
|
26
|
+
|
27
|
+
# @return [String]
|
28
|
+
attr_reader :method
|
29
|
+
|
30
|
+
# @return [String, nil]
|
31
|
+
attr_reader :body
|
32
|
+
|
33
|
+
# @return [Float, nil]
|
34
|
+
attr_reader :timeout
|
35
|
+
|
36
|
+
# @return [Boolean]
|
37
|
+
attr_reader :ignore_certificate_errors
|
19
38
|
|
20
39
|
def initialize(
|
21
40
|
base_uri,
|
22
41
|
additional_list: nil,
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
42
|
+
headers: {},
|
43
|
+
method: "HEAD",
|
44
|
+
body: nil,
|
45
|
+
timeout: nil,
|
46
|
+
max_concurrent_requests: nil,
|
47
|
+
ignore_certificate_errors: false,
|
48
|
+
type: "all"
|
27
49
|
)
|
28
|
-
@base_uri = URI.parse(base_uri)
|
50
|
+
@base_uri = URI.parse(base_uri.downcase)
|
29
51
|
raise ArgumentError, "Invalid URI" unless valid_uri?
|
30
52
|
|
31
53
|
@additional_list = additional_list
|
32
|
-
|
33
|
-
raise ArgumentError, "Invalid path" unless valid_path?
|
34
|
-
end
|
54
|
+
raise ArgumentError, "Invalid path" if !additional_list.nil? && !valid_path?
|
35
55
|
|
36
|
-
@
|
37
|
-
|
38
|
-
|
39
|
-
@
|
40
|
-
|
56
|
+
@method = method.upcase
|
57
|
+
raise ArgumentError, "Invalid HTTP method" unless valid_method?
|
58
|
+
|
59
|
+
@headers = headers
|
60
|
+
@body = body
|
61
|
+
|
62
|
+
@timeout = timeout.nil? ? nil : timeout.to_f
|
41
63
|
|
42
|
-
|
43
|
-
|
44
|
-
|
64
|
+
@max_concurrent_requests = max_concurrent_requests || (Etc.nprocessors * 8)
|
65
|
+
|
66
|
+
@ignore_certificate_errors = ignore_certificate_errors
|
67
|
+
|
68
|
+
@type = type
|
45
69
|
end
|
46
70
|
|
47
71
|
def crawl
|
48
|
-
results =
|
49
|
-
|
50
|
-
|
51
|
-
|
72
|
+
results = {}
|
73
|
+
|
74
|
+
Async do |task|
|
75
|
+
barrier = Async::Barrier.new
|
76
|
+
semaphore = Async::Semaphore.new(max_concurrent_requests, parent: barrier)
|
77
|
+
internet = Internet.new
|
78
|
+
|
79
|
+
paths.each do |path|
|
80
|
+
semaphore.async do
|
81
|
+
url = url_for(path)
|
82
|
+
|
83
|
+
res = dispatch_http_request(task, internet, url)
|
84
|
+
next unless online?(res.status)
|
85
|
+
|
86
|
+
log = [method, url, res.status].join(",")
|
87
|
+
Tansaku.logger.info(log)
|
88
|
+
|
89
|
+
results[url] = res.status
|
90
|
+
rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
|
91
|
+
next
|
92
|
+
end
|
93
|
+
end
|
94
|
+
barrier.wait
|
52
95
|
end
|
53
|
-
|
96
|
+
|
97
|
+
results
|
54
98
|
end
|
55
99
|
|
56
100
|
private
|
57
101
|
|
102
|
+
def online?(status)
|
103
|
+
[200, 204, 301, 302, 307, 401, 403].include? status.to_i
|
104
|
+
end
|
105
|
+
|
58
106
|
def valid_uri?
|
59
107
|
["http", "https"].include? base_uri.scheme
|
60
108
|
end
|
@@ -63,10 +111,14 @@ module Tansaku
|
|
63
111
|
File.exist?(additional_list)
|
64
112
|
end
|
65
113
|
|
114
|
+
def valid_method?
|
115
|
+
Protocol::HTTP::Methods.valid? method
|
116
|
+
end
|
117
|
+
|
66
118
|
def paths
|
67
119
|
paths = Path.get_by_type(type)
|
68
120
|
paths += File.readlines(File.expand_path(additional_list, __dir__)) if additional_list
|
69
|
-
paths.
|
121
|
+
paths.filter_map(&:chomp)
|
70
122
|
end
|
71
123
|
|
72
124
|
def url_for(path)
|
@@ -77,16 +129,40 @@ module Tansaku
|
|
77
129
|
paths.map { |path| url_for path }
|
78
130
|
end
|
79
131
|
|
80
|
-
def
|
81
|
-
|
132
|
+
def request_headers
|
133
|
+
@request_headers ||= @headers.tap do |headers|
|
134
|
+
upcase_keys = headers.keys.map(&:downcase).map(&:to_s)
|
135
|
+
headers["user-agent"] = DEFAULT_USER_AGENT unless upcase_keys.include?("user-agent")
|
136
|
+
end.compact
|
82
137
|
end
|
83
138
|
|
84
|
-
def
|
85
|
-
|
86
|
-
|
87
|
-
head["Host"] = host unless host.nil?
|
139
|
+
def ssl_verify_mode
|
140
|
+
ignore_certificate_errors ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
|
141
|
+
end
|
88
142
|
|
89
|
-
|
143
|
+
def ssl_context
|
144
|
+
@ssl_context ||= OpenSSL::SSL::SSLContext.new.tap do |context|
|
145
|
+
context.set_params(verify_mode: ssl_verify_mode)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
#
|
150
|
+
# Dispatch an HTTP request
|
151
|
+
#
|
152
|
+
# @param [Async::Task] task
|
153
|
+
# @param [Tansaku::Internet] internet
|
154
|
+
# @param [String] url
|
155
|
+
#
|
156
|
+
# @return [Async::HTTP::Protocol::Response]
|
157
|
+
#
|
158
|
+
def dispatch_http_request(task, internet, url)
|
159
|
+
endpoint = Async::HTTP::Endpoint.parse(url, ssl_context: ssl_context)
|
160
|
+
|
161
|
+
return internet.call(method, endpoint, request_headers, body) if timeout.nil?
|
162
|
+
|
163
|
+
task.with_timeout(timeout) do
|
164
|
+
internet.call(method, endpoint, request_headers, body)
|
165
|
+
end
|
90
166
|
end
|
91
167
|
end
|
92
168
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "async/http/internet"
|
4
|
+
|
5
|
+
module Tansaku
|
6
|
+
class Internet < Async::HTTP::Internet
|
7
|
+
def call(method, url_or_endpoint, headers = nil, body = nil)
|
8
|
+
endpoint = if url_or_endpoint.is_a?(Async::HTTP::Endpoint)
|
9
|
+
url_or_endpoint
|
10
|
+
else
|
11
|
+
Async::HTTP::Endpoint.parse(url_or_endpoint)
|
12
|
+
end
|
13
|
+
key = host_key(endpoint)
|
14
|
+
|
15
|
+
client = @clients.fetch(key) do
|
16
|
+
@clients[key] = client_for(endpoint)
|
17
|
+
end
|
18
|
+
|
19
|
+
body = Async::HTTP::Body::Buffered.wrap(body)
|
20
|
+
headers = ::Protocol::HTTP::Headers[headers]
|
21
|
+
|
22
|
+
request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body)
|
23
|
+
|
24
|
+
client.call(request)
|
25
|
+
end
|
26
|
+
|
27
|
+
::Protocol::HTTP::Methods.each do |_name, verb|
|
28
|
+
define_method(verb.downcase) do |url_or_endpoint, headers = nil, body = nil|
|
29
|
+
url_or_endpoint = url_or_endpoint.to_str unless url_or_endpoint.is_a?(Async::HTTP::Endpoint)
|
30
|
+
call(verb, url_or_endpoint, headers, body)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "protocol/http1/connection"
|
4
|
+
|
5
|
+
module Protocol
|
6
|
+
module HTTP1
|
7
|
+
class Connection
|
8
|
+
def write_request(authority, method, path, version, headers)
|
9
|
+
host = authority
|
10
|
+
if headers.include?("host")
|
11
|
+
host = headers["host"]
|
12
|
+
headers.delete "host"
|
13
|
+
end
|
14
|
+
|
15
|
+
@stream.write("#{method} #{path} #{version}\r\n")
|
16
|
+
@stream.write("host: #{host}\r\n")
|
17
|
+
|
18
|
+
write_headers(headers)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/tansaku/version.rb
CHANGED
data/lib/tansaku.rb
CHANGED
@@ -1,8 +1,26 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "tansaku/version"
|
4
|
+
|
5
|
+
require "tansaku/internet"
|
4
6
|
require "tansaku/path"
|
7
|
+
|
5
8
|
require "tansaku/crawler"
|
9
|
+
|
6
10
|
require "tansaku/cli"
|
7
11
|
|
8
|
-
|
12
|
+
require "memist"
|
13
|
+
require "semantic_logger"
|
14
|
+
|
15
|
+
module Tansaku
|
16
|
+
class << self
|
17
|
+
include Memist::Memoizable
|
18
|
+
|
19
|
+
def logger
|
20
|
+
SemanticLogger.default_level = :info
|
21
|
+
SemanticLogger.add_appender(io: $stderr, formatter: :color)
|
22
|
+
SemanticLogger["Tansaku"]
|
23
|
+
end
|
24
|
+
memoize :logger
|
25
|
+
end
|
26
|
+
end
|
data/tansaku.gemspec
CHANGED
@@ -15,22 +15,29 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.homepage = "https://github.com/ninoseki/tansaku"
|
16
16
|
spec.license = "MIT"
|
17
17
|
|
18
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
19
|
+
|
18
20
|
# Specify which files should be added to the gem when it is released.
|
19
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
20
|
-
spec.files
|
22
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
21
23
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
22
24
|
end
|
23
25
|
spec.bindir = "exe"
|
24
26
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
27
|
spec.require_paths = ["lib"]
|
26
28
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 2.
|
28
|
-
spec.add_development_dependency "
|
29
|
+
spec.add_development_dependency "bundler", "~> 2.3"
|
30
|
+
spec.add_development_dependency "coveralls_reborn", "~> 0.24"
|
29
31
|
spec.add_development_dependency "glint", "~> 0.1"
|
30
32
|
spec.add_development_dependency "rake", "~> 13.0"
|
31
|
-
spec.add_development_dependency "rspec", "~> 3.
|
32
|
-
spec.add_development_dependency "
|
33
|
+
spec.add_development_dependency "rspec", "~> 3.11"
|
34
|
+
spec.add_development_dependency "simplecov-lcov", "~> 0.8.0"
|
35
|
+
spec.add_development_dependency "webmock", "~> 3.14"
|
36
|
+
spec.add_development_dependency "webrick", "~> 1.7"
|
33
37
|
|
34
|
-
spec.add_dependency "
|
35
|
-
spec.add_dependency "
|
38
|
+
spec.add_dependency "async", "~> 1.30"
|
39
|
+
spec.add_dependency "async-http", "~> 0.56"
|
40
|
+
spec.add_dependency "memist", "~> 2.0"
|
41
|
+
spec.add_dependency "semantic_logger", "~> 4.10"
|
42
|
+
spec.add_dependency "thor", "~> 1.2"
|
36
43
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tansaku
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Niseki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,28 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '2.
|
19
|
+
version: '2.3'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '2.
|
26
|
+
version: '2.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: coveralls_reborn
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '0.
|
33
|
+
version: '0.24'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '0.
|
40
|
+
version: '0.24'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: glint
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,56 +72,126 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '3.
|
75
|
+
version: '3.11'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '3.
|
82
|
+
version: '3.11'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: simplecov-lcov
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 0.8.0
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 0.8.0
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: webmock
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
101
|
- - "~>"
|
88
102
|
- !ruby/object:Gem::Version
|
89
|
-
version: '3.
|
103
|
+
version: '3.14'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '3.14'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: webrick
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '1.7'
|
90
118
|
type: :development
|
91
119
|
prerelease: false
|
92
120
|
version_requirements: !ruby/object:Gem::Requirement
|
93
121
|
requirements:
|
94
122
|
- - "~>"
|
95
123
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
124
|
+
version: '1.7'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: async
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '1.30'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '1.30'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: async-http
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0.56'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0.56'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: memist
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '2.0'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '2.0'
|
97
167
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
168
|
+
name: semantic_logger
|
99
169
|
requirement: !ruby/object:Gem::Requirement
|
100
170
|
requirements:
|
101
171
|
- - "~>"
|
102
172
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
173
|
+
version: '4.10'
|
104
174
|
type: :runtime
|
105
175
|
prerelease: false
|
106
176
|
version_requirements: !ruby/object:Gem::Requirement
|
107
177
|
requirements:
|
108
178
|
- - "~>"
|
109
179
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
180
|
+
version: '4.10'
|
111
181
|
- !ruby/object:Gem::Dependency
|
112
182
|
name: thor
|
113
183
|
requirement: !ruby/object:Gem::Requirement
|
114
184
|
requirements:
|
115
185
|
- - "~>"
|
116
186
|
- !ruby/object:Gem::Version
|
117
|
-
version: '1.
|
187
|
+
version: '1.2'
|
118
188
|
type: :runtime
|
119
189
|
prerelease: false
|
120
190
|
version_requirements: !ruby/object:Gem::Requirement
|
121
191
|
requirements:
|
122
192
|
- - "~>"
|
123
193
|
- !ruby/object:Gem::Version
|
124
|
-
version: '1.
|
194
|
+
version: '1.2'
|
125
195
|
description: Yet another dirbuster tool
|
126
196
|
email:
|
127
197
|
- manabu.niseki@gmail.com
|
@@ -131,9 +201,9 @@ extensions: []
|
|
131
201
|
extra_rdoc_files: []
|
132
202
|
files:
|
133
203
|
- ".gitattributes"
|
204
|
+
- ".github/workflows/test.yml"
|
134
205
|
- ".gitignore"
|
135
206
|
- ".rspec"
|
136
|
-
- ".travis.yml"
|
137
207
|
- Gemfile
|
138
208
|
- LICENSE
|
139
209
|
- README.md
|
@@ -144,18 +214,22 @@ files:
|
|
144
214
|
- lib/tansaku.rb
|
145
215
|
- lib/tansaku/cli.rb
|
146
216
|
- lib/tansaku/crawler.rb
|
217
|
+
- lib/tansaku/internet.rb
|
147
218
|
- lib/tansaku/lists/admin.txt
|
148
219
|
- lib/tansaku/lists/backup.txt
|
149
220
|
- lib/tansaku/lists/database.txt
|
150
221
|
- lib/tansaku/lists/etc.txt
|
151
222
|
- lib/tansaku/lists/log.txt
|
223
|
+
- lib/tansaku/monkey_patch.rb
|
152
224
|
- lib/tansaku/path.rb
|
153
225
|
- lib/tansaku/version.rb
|
226
|
+
- renovate.json
|
154
227
|
- tansaku.gemspec
|
155
228
|
homepage: https://github.com/ninoseki/tansaku
|
156
229
|
licenses:
|
157
230
|
- MIT
|
158
|
-
metadata:
|
231
|
+
metadata:
|
232
|
+
rubygems_mfa_required: 'true'
|
159
233
|
post_install_message:
|
160
234
|
rdoc_options: []
|
161
235
|
require_paths:
|
@@ -171,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
171
245
|
- !ruby/object:Gem::Version
|
172
246
|
version: '0'
|
173
247
|
requirements: []
|
174
|
-
rubygems_version: 3.
|
248
|
+
rubygems_version: 3.3.7
|
175
249
|
signing_key:
|
176
250
|
specification_version: 4
|
177
251
|
summary: Yet another dirbuster tool
|