apullo 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0827cde956b63c66bd88b8c6a37597626f355cd677a701e6590ab5996011c21
-  data.tar.gz: bb53dfba7e8297f85b3992a54efa8546e2d815247443430134bfbd1b15963acd
+  metadata.gz: 3fef54d32890b574081af9e64c6d76dc01287651779bb43eaecc046df9763e43
+  data.tar.gz: 07f3676d94fd54345a40a547c0a6b82ffa5d599ff34a21b7cf59270e608f2df1
 SHA512:
-  metadata.gz: 53469022402714f62e9561da863f35e0bf5032b29d7ec2754224d04b5dc44e40615d4f3fee980d23c8e60a3a27b84388561f3f00e7b14ab816e79aec63ec7acb
-  data.tar.gz: c67723532f368589ee043c700fa4fb3cc5a6dd500bdbbe29244f5a3f2c992a64181c08d7531c2ef2315c0b4854fa109052485ab81b64c9de09e845f369945f88
+  metadata.gz: 5bd0f54aebb6c6f7515fbc2979c22c3af2a3553535665f5a3ca7af5bd1358f0e5372b33a2db03c181e115d9aac87d7b819f57b49c281a8362c78860d175c2545
+  data.tar.gz: d5ac2f8806e242005d2bd6ee1d03de9126eed3b01b93a19583fe387e5842cce1b5a99ff05dcbfc6a3b58ba5b4d5ae38e11a48a68872953b03e47dcf2417d5e45

data/.travis.yml

@@ -4,4 +4,4 @@ language: ruby
 cache: bundler
 rvm:
   - 2.6
-before_install: gem install bundler -v 2.0.2
+before_install: gem install bundler -v 2.1

data/README.md

@@ -31,6 +31,7 @@ Commands:
 It takes basic network fingerprints of a target.
 
 - Hashes of an HTTP response body
+- Headers of an HTTP response
 - Hashes of an SSL certificate
 - Hashes of a favicon image
 - Hashes of an SSH host key
@@ -54,9 +55,46 @@ $ apullo check https://example.com
       "sha256": "9250711c54de546f4370e0c3d3a3ec45bc96092a25a4a71a1afa396af7047eb8"
     },
     "favicon": {
+    },
+    "headers": {
+      "accept-ranges": "bytes",
+      "cache-control": "max-age=604800",
+      "content-type": "text/html; charset=UTF-8",
+      "date": "Thu, 07 Nov 2019 23:21:43 GMT",
+      "etag": "\"3147526947+gzip\"",
+      "expires": "Thu, 14 Nov 2019 23:21:43 GMT",
+      "last-modified": "Thu, 17 Oct 2019 07:18:26 GMT",
+      "server": "ECS (sec/9739)",
+      "vary": "Accept-Encoding",
+      "x-cache": "HIT",
+      "content-length": "648"
+    },
+    "meta": {
+      "url": "https://example.com"
     }
   },
   "domain": {
+    "dns": {
+      "ns": [
+        "b.iana-servers.net",
+        "a.iana-servers.net"
+      ],
+      "cname": [
+
+      ],
+      "soa": [
+        "noc.dns.icann.org"
+      ],
+      "mx": [
+
+      ],
+      "a": [
+        "93.184.216.34"
+      ],
+      "aaaa": [
+        "2606:2800:220:1:248:1893:25C8:1946"
+      ]
+    },
     "whois": {
       "registrant_contacts": [
         {
@@ -84,27 +122,6 @@ $ apullo check https://example.com
       "technical_contacts": [
 
       ]
-    },
-    "dns": {
-      "ns": [
-        "a.iana-servers.net",
-        "b.iana-servers.net"
-      ],
-      "cname": [
-
-      ],
-      "soa": [
-        "noc.dns.icann.org"
-      ],
-      "mx": [
-
-      ],
-      "a": [
-        "93.184.216.34"
-      ],
-      "aaaa": [
-        "2606:2800:220:1:248:1893:25C8:1946"
-      ]
     }
   },
   "ssh": {
@@ -113,84 +130,12 @@ $ apullo check https://example.com
     "target": "https://example.com"
   }
 }
+```
 
-$ apullo check jppost-be.top
-{
-  "http": {
-    "body": {
-      "md5": "74ad15c4ab3f67eee1d546e22248931f",
-      "mmh3": -330759974,
-      "sha1": "c0280893956852b0c07ae4da752ee5d776d248b8",
-      "sha256": "28fa3b0beaf188d48b32557fa4df8f0aa451bd10f8e8bb26e919009d2d41b8fb"
-    },
-    "cert": {
-    },
-    "favicon": {
-      "md5": "ad184c25a1a01d97696dcb59a1ffef74",
-      "mmh3": 111036816,
-      "sha1": "cb4842a54c3e96408765290cb810793302c17f0b",
-      "sha256": "6949c58f841fa21a89e2e2375ae5645e1db62385f89a0218766f2b0a9c490fb8",
-      "meta": {
-        "url": "https://www.post.japanpost.jp/img/common/touch-icon.png"
-      }
-    }
-  },
-  "domain": {
-    "whois": {
-      "registrant_contacts": [
-
-      ],
-      "admin_contacts": [
-
-      ],
-      "technical_contacts": [
-
-      ]
-    },
-    "dns": {
-      "ns": [
-        "ns1.bdydns.cn",
-        "ns2.bdydns.cn"
-      ],
-      "cname": [
-
-      ],
-      "soa": [
-        "sa.dudns.com"
-      ],
-      "mx": [
+## Notes
 
-      ],
-      "a": [
-        "193.148.69.12"
-      ],
-      "aaaa": [
-
-      ]
-    }
-  },
-  "ssh": {
-    "rsa": {
-      "md5": "960bb068dbfb9aa9f9d6899c15844fca",
-      "sha1": "d36555028decde1f931b47c90e469fc52e8f364a",
-      "sha256": "cf3c7ea7b9442f71423f2253a9c0e448fd0d619e1abc7e519499cd789fac6e74"
-    },
-    "ecdsa-sha2-nistp256": {
-      "md5": "551222e53a38c10817653a723e6caf0c",
-      "sha1": "cd6044db29b30d35f32e26e74d66258570cd6527",
-      "sha256": "0664dbea7580f9430da6d0ba13e7a4bba0f1efd449c895a6adcc147abc958ce6"
-    },
-    "ed25519": {
-      "md5": "6da2245d9a211731c2d229ea7cce829b",
-      "sha1": "d86afd8fca1a052249ef3a0ee26a24f6cc644485",
-      "sha256": "7f0d4b642ea2c236eca4018a2dadff3b8a03c37745f9a9f741d9d246a420f358"
-    }
-  },
-  "meta": {
-    "target": "jppost-be.top"
-  }
-}
-```
+- `mmh3` is a 32 bit signed int value of MurmurHash3.
+- Keys of `http.headers` are downcased.
 
 ## License
 

data/apullo.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.9"
@@ -35,10 +35,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ipaddr", "~> 1.2"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "murmurhash3", "~> 0.1"
-  spec.add_dependency "oga", "~> 2.15"
+  spec.add_dependency "oga", "~> 3.0"
+  spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "ssh_scan", "~> 0.0"
-  spec.add_dependency "thor", "~> 0.20"
+  spec.add_dependency "thor", "~> 1.0"
   spec.add_dependency "whois", "~> 5.0"
   spec.add_dependency "whois-parser", "~> 1.2"
 end

data/lib/apullo/cli.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "parallel"
 require "thor"
 
 module Apullo
@@ -26,7 +27,7 @@ module Apullo
           }
         end
 
-        Apullo.fingerprints.map do |klass|
+        Parallel.map(Apullo.fingerprints) do |klass|
           fingerprint = klass.new(target)
           fingerprint.headers = headers if fingerprint.respond_to?(:headers=)
 

data/lib/apullo/fingerprints/base.rb

@@ -7,6 +7,7 @@ module Apullo
 
       def initialize(target)
         @target = target
+        @results = nil
       end
 
       def name
@@ -14,6 +15,23 @@ module Apullo
       end
 
       def results
+        return @results if @results
+
+        with_error_handling do
+          @results ||= build_results
+        end
+        @results
+      end
+
+      private
+
+      def with_error_handling
+        yield
+      rescue StandardError => e
+        @results = { error: e.to_s }
+      end
+
+      def build_results
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 

data/lib/apullo/fingerprints/domain.rb

@@ -15,15 +15,15 @@ module Apullo
         Resolv::DNS::Resource::IN::AAAA,
       ].freeze
 
-      def results
+      private
+
+      def build_results
         {
           dns: resources,
           whois: contacts,
         }
       end
 
-      private
-
       def dns
         @dns ||= Resolv::DNS.new
       end

data/lib/apullo/fingerprints/http.rb

@@ -10,23 +10,25 @@ module Apullo
       attr_writer :headers
 
       def initialize(target)
-        @target = target
+        super target
+
         @headers = {}
       end
 
-      def results
-        @results ||= [].tap do |out|
-          get(target.uri.path)
-
-          out << {
-            body: body,
-            cert: cert,
-            favicon: favicon,
-            meta: {
-              url: target.url
-            }
+      private
+
+      def build_results
+        get(target.uri.path)
+
+        {
+          body: body,
+          cert: cert,
+          favicon: favicon,
+          headers: response_headers,
+          meta: {
+            url: target.url
           }
-        end.first
+        }
       end
 
       def cert
@@ -61,7 +63,9 @@ module Apullo
         favicon.results
       end
 
-      private
+      def response_headers
+        @response_headers || {}
+      end
 
       def headers
         @headers.compact
@@ -71,10 +75,18 @@ module Apullo
         "#{target.uri.scheme}://#{target.uri.host}:#{target.uri.port}/favicon.ico"
       end
 
+      def build_doc
+        Oga.parse_html(@body)
+      rescue ArgumentError, LL::ParserError => _e
+        nil
+      end
+
       def favicon_url
         return nil unless @body
 
-        doc = Oga.parse_html(@body)
+        doc = build_doc
+        return nil unless doc
+
         icon = doc.at_css("link[rel='shortcut icon']") || doc.at_css("link[rel='icon']")
         return default_favicon_url unless icon
 
@@ -101,6 +113,7 @@ module Apullo
         else
           @peer_cert = http.peer_cert
           @body = response.body
+          @response_headers = response.each_header.to_h
           @path = path
         end
       rescue Errno::ECONNREFUSED, Net::HTTPError, OpenSSL::OpenSSLError, Timeout::Error => _e

data/lib/apullo/fingerprints/ssh.rb

@@ -8,26 +8,30 @@ module Apullo
       DEFAULT_OPTIONS = { "timeout" => 3 }.freeze
       DEFAULT_PORT = 22
 
-      def results
-        @results ||= pluck
-      end
-
       private
 
-      def pluck
+      def build_results
+        pluck_fingerprints
+      end
+
+      def pluck_fingerprints
         result = scan
         keys = result.dig("keys") || []
         keys.map do |cipher, data|
+          raw = data.dig("raw")
           fingerprints = data.dig("fingerprints") || []
-          normalized = fingerprints.map do |hash, value|
+          normalized_fingerprints = fingerprints.map do |hash, value|
             [hash, value.delete(":")]
           end.to_h
-          [cipher, normalized]
+          [
+            cipher,
+            { raw: raw, fingerprints: normalized_fingerprints }
+          ]
         end.to_h
       end
 
       def scan
-        return {} unless target.ipv4
+        return {} unless target.host
 
         engine = SSHScan::ScanEngine.new
         dest = "#{target.host}:#{DEFAULT_PORT}"

data/lib/apullo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Apullo
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apullo
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-07 00:00:00.000000000 Z
+date: 2019-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +156,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.15'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
@@ -198,14 +212,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: whois
   requirement: !ruby/object:Gem::Requirement