tansaku 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50036fe49b8faa2f534977c93c8352afdd2db88542bcf7b78323a0a410f4fb18
-  data.tar.gz: cad855e053c5cad7ebac3f8cef80e546b900a87dd30ad3bae53a9c7503e76c7d
+  metadata.gz: a179d2d95fc0e6c78d908ff8379ecefb609448f38284f77c21fcd92dd8a0b7ca
+  data.tar.gz: 0eaecd595420390290a4e5b9f8a759b229e096037e60d058d58fd6af02dfe9f0
 SHA512:
-  metadata.gz: f4c0c1b4011de97cbbe3eeed76b2f0f07e17487ec0f00e092481ff9f37a0622d161eb1d37dcb938c16c131321da88e04c5a12e7cb1222dce729feb4a58efbf8a
-  data.tar.gz: 0d03f7cae1bde0eb320ca41a3af43c216e13a264fd61966a7bbf0bba09a0a4c630e82659a88d4c0525109ca148796098c895035bf16fa1b0a3a3fadb7c17aa85
+  metadata.gz: a4b252ff9c13cbe70109f8a15b77e9923672f15c5de2f93f26af6e6a0031ebdf0ed2c246e76968843e66f6e4eda0cdeae067b3886c8525237f5a412032780b63
+  data.tar.gz: '01952d73467ad3fafce9b3726675c7f145f7db359052cb6682e533f7faa2c42022f8e2f38e07a3f63fe132ee36d8a8dfcbfdc61563c84d9b55d3a8536b1b3146'

data/README.md

@@ -37,7 +37,7 @@ Usage:
 Options:
   [--additional-list=ADDITIONAL_LIST]  # Path to the file which includes additonal paths to crawl
   [--host=HOST]                        # Host header to use
-  [--threads=N]                        # Number of threads to use
+  [--max-concurrent-requests=N]        # Number of concurrent requests to use
   [--type=TYPE]                        # Type of a list to crawl (admin, backup, database, etc, log or all)
                                        # Default: all
   [--user-agent=USER_AGENT]            # User-Agent header to use

data/lib/tansaku/cli.rb

@@ -9,7 +9,7 @@ module Tansaku
     desc "crawl URL", "Crawl a given URL"
     method_option :additional_list, desc: "Path to the file which includes additonal paths to crawl"
     method_option :host, type: :string, desc: "Host header to use"
-    method_option :threads, type: :numeric, desc: "Number of threads to use"
+    method_option :max_concurrent_requests, type: :numeric, desc: "Number of concurrent requests to use"
     method_option :type, desc: "Type of a list to crawl (admin, backup, database, etc, log or all)", default: "all"
     method_option :user_agent, type: :string, desc: "User-Agent header to use"
     def crawl(url)

data/lib/tansaku/crawler.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
+require "async/http/internet"
+require "async"
+require "async/barrier"
+require "async/semaphore"
 require "cgi"
-require "net/http"
-require "parallel"
+require "etc"
 require "uri"
 
+require "tansaku/monkey_patch"
+
 module Tansaku
   class Crawler
     DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
@@ -13,7 +18,7 @@ module Tansaku
 
     attr_reader :additional_list
     attr_reader :host
-    attr_reader :threads
+    attr_reader :max_concurrent_requests
     attr_reader :type
     attr_reader :user_agent
 
@@ -21,7 +26,7 @@ module Tansaku
       base_uri,
       additional_list: nil,
       host: nil,
-      threads: Parallel.processor_count,
+      max_concurrent_requests: Etc.nprocessors,
       type: "all",
       user_agent: DEFAULT_USER_AGENT
     )
@@ -34,27 +39,39 @@ module Tansaku
       end
 
       @host = host
-      @threads = threads
+      @max_concurrent_requests = max_concurrent_requests
       @type = type
       @user_agent = user_agent
     end
 
-    def online?(url)
-      res = head(url)
-      [200, 401, 302].include? res.code.to_i
-    end
-
     def crawl
-      results = Parallel.map(urls, in_threads: threads) do |url|
-        url if online?(url)
-      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => _e
-        nil
+      results = {}
+      Async do
+        barrier = Async::Barrier.new
+        semaphore = Async::Semaphore.new(max_concurrent_requests, parent: barrier)
+        internet = Async::HTTP::Internet.new
+
+        paths.each do |path|
+          semaphore.async do
+            url = url_for(path)
+            res = internet.head(url, default_request_headers)
+
+            results[url] = res.status if online?(res.status)
+          rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, OpenSSL::SSL::SSLError, Async::TimeoutError
+            next
+          end
+        end
+        barrier.wait
       end
-      results.compact
+      results
     end
 
     private
 
+    def online?(status)
+      [200, 204, 301, 302, 307, 401, 403].include? status.to_i
+    end
+
     def valid_uri?
       ["http", "https"].include? base_uri.scheme
     end
@@ -77,16 +94,8 @@ module Tansaku
       paths.map { |path| url_for path }
     end
 
-    def request(req)
-      Net::HTTP.start(base_uri.host, base_uri.port) { |http| http.request(req) }
-    end
-
-    def head(url)
-      head = Net::HTTP::Head.new(url)
-      head["User-Agent"] = user_agent
-      head["Host"] = host unless host.nil?
-
-      request(head)
+    def default_request_headers
+      @default_request_headers ||= { "host" => host, "user-agent" => user_agent }.compact
     end
   end
 end

data/lib/tansaku/monkey_patch.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "protocol/http1/connection"
+
+module Protocol
+  module HTTP1
+    class Connection
+      def write_request(authority, method, path, version, headers)
+        host = authority
+        if headers.include?("host")
+          host = headers["host"]
+          headers.delete "host"
+        end
+
+        @stream.write("#{method} #{path} #{version}\r\n")
+        @stream.write("host: #{host}\r\n")
+
+        write_headers(headers)
+      end
+    end
+  end
+end

data/lib/tansaku/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tansaku
-  VERSION = "0.3.0"
+  VERSION = "1.0.0"
 end

data/tansaku.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.9"
   spec.add_development_dependency "webmock", "~> 3.8"
 
-  spec.add_dependency "parallel", "~> 1.19"
+  spec.add_dependency "async", "~> 1.26"
+  spec.add_dependency "async-http", "~> 0.52"
   spec.add_dependency "thor", "~> 1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tansaku
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-30 00:00:00.000000000 Z
+date: 2020-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -95,19 +95,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.8'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: async
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1.26'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.52'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.52'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -149,6 +163,7 @@ files:
 - lib/tansaku/lists/database.txt
 - lib/tansaku/lists/etc.txt
 - lib/tansaku/lists/log.txt
+- lib/tansaku/monkey_patch.rb
 - lib/tansaku/path.rb
 - lib/tansaku/version.rb
 - tansaku.gemspec