apullo 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96eb9917a6d5fcef270fe520a2312f7447896b2d1e6f0bfc61f966d554e8f9c6
-  data.tar.gz: ca4807677d7519969aae72fcc647cad01f3799e04cea1d3aee3433efbb4e77bc
+  metadata.gz: 1e817b368f4df9a10f01670a5319264d9fd1378f16e1a22590e587c8f75aba52
+  data.tar.gz: 52aa9310b175ee65cd82b1c9d800382edbf58081031bb2b16f88e63c9eba237b
 SHA512:
-  metadata.gz: 8d32ee78a83177d0dbca9a7d418f5e5e8af4894f516d05ed705a1ab8b082601663353f69e30571663015ddf0fbe9e91b19e6ac0b83431369a7866f51b147b6f7
-  data.tar.gz: 5e846ede99ffd718f5e75944eb1ef79b97be719f6cbbf62a43ebf05891e405142a1781d6676ec7e8fabdfe5d0530b7fddeedf7ef0573ca07165110828644a098
+  metadata.gz: a005b2e6d73cbac89b060e08e38147c27cc5d633eb6526e804f652ea813d9395236445c8674e8c41eec32a643f4c9cf1d550ffa79a4cc10244068e41cf3661f6
+  data.tar.gz: 9d8cb76e97fe4a7adc43618abbce2419e8c2b0549abe894611159e54486110a47c88be51bfd13eb86fb4bad52866ee99d7e048bececefb14c3cd92a2b9cf654f

data/README.md

@@ -50,34 +50,43 @@ $ apullo check https://example.com
     },
     "cert": {
       "md5": "3510c21c66bd62010fc547d3cd3f0ce6",
-      "serial": "21020869104500376438182461249190639870",
+      "serial": 21020869104500376438182461249190639870,
       "sha1": "7bb698386970363d2919cc5772846984ffd4a889",
       "sha256": "9250711c54de546f4370e0c3d3a3ec45bc96092a25a4a71a1afa396af7047eb8"
     },
     "favicon": {
     },
     "headers": {
-      "accept-ranges": "bytes",
       "cache-control": "max-age=604800",
       "content-type": "text/html; charset=UTF-8",
-      "date": "Thu, 07 Nov 2019 23:21:43 GMT",
+      "date": "Sat, 11 Jan 2020 10:47:09 GMT",
       "etag": "\"3147526947+gzip\"",
-      "expires": "Thu, 14 Nov 2019 23:21:43 GMT",
+      "expires": "Sat, 18 Jan 2020 10:47:09 GMT",
       "last-modified": "Thu, 17 Oct 2019 07:18:26 GMT",
-      "server": "ECS (sec/9739)",
+      "server": "ECS (oxr/830F)",
       "vary": "Accept-Encoding",
       "x-cache": "HIT",
       "content-length": "648"
     },
     "meta": {
-      "url": "https://example.com"
+      "url": "https://example.com",
+      "links": {
+        "shodan": {
+          "body": "https://www.shodan.io/search?query=http.html_hash%3A-2087618365",
+          "cert": "https://www.shodan.io/search?q=ssl.cert.serial%3A21020869104500376438182461249190639870"
+        },
+        "censys": {
+          "body": "https://censys.io/ipv4?q=ea8fac7c65fb589b0d53560f5251f74f9e9b243478dcb6b3ea79b5e36449c8d9",
+          "cert": "https://censys.io/ipv4?q=9250711c54de546f4370e0c3d3a3ec45bc96092a25a4a71a1afa396af7047eb8"
+        }
+      }
     }
   },
   "domain": {
     "dns": {
       "ns": [
-        "b.iana-servers.net",
-        "a.iana-servers.net"
+        "a.iana-servers.net",
+        "b.iana-servers.net"
       ],
       "cname": [
 
@@ -122,6 +131,11 @@ $ apullo check https://example.com
       "technical_contacts": [
 
       ]
+    },
+    "meta": {
+      "links": {
+        "securitytrails": "https://securitytrails.com/domain/example.com/dns"
+      }
     }
   },
   "ssh": {
@@ -130,6 +144,115 @@ $ apullo check https://example.com
     "target": "https://example.com"
   }
 }
+
+$ apullo check jppost-ku.com
+{
+  "http": {
+    "body": {
+      "md5": "0728450344e6ea95107ce8c3b00f10ae",
+      "mmh3": 421543491,
+      "sha1": "6fa29d366b33d5f3c54d62c95b23aa1cce2587a3",
+      "sha256": "7bc86f6a3d8877bd84d9917c3661658867af3fdb44842b973be2d299fe793dc2"
+    },
+    "cert": {
+    },
+    "favicon": {
+      "md5": "ad184c25a1a01d97696dcb59a1ffef74",
+      "mmh3": 111036816,
+      "sha1": "cb4842a54c3e96408765290cb810793302c17f0b",
+      "sha256": "6949c58f841fa21a89e2e2375ae5645e1db62385f89a0218766f2b0a9c490fb8",
+      "meta": {
+        "url": "https://www.post.japanpost.jp/img/common/touch-icon.png"
+      }
+    },
+    "headers": {
+      "server": "Apache-Coyote/1.1",
+      "accept-ranges": "bytes",
+      "etag": "W/\"54423-1577193448000\"",
+      "last-modified": "Tue, 24 Dec 2019 13:17:28 GMT",
+      "content-type": "text/html",
+      "content-length": "54423",
+      "date": "Sat, 11 Jan 2020 10:48:28 GMT"
+    },
+    "meta": {
+      "url": "http://jppost-ku.com",
+      "links": {
+        "shodan": {
+          "body": "https://www.shodan.io/search?query=http.html_hash%3A421543491",
+          "favicon": "https://www.shodan.io/search?query=http.favicon.hash%3A111036816"
+        },
+        "censys": {
+          "body": "https://censys.io/ipv4?q=7bc86f6a3d8877bd84d9917c3661658867af3fdb44842b973be2d299fe793dc2"
+        }
+      }
+    }
+  },
+  "domain": {
+    "dns": {
+      "ns": [
+        "ns2.bdydns.cn",
+        "ns1.bdydns.cn"
+      ],
+      "cname": [
+
+      ],
+      "soa": [
+        "sa.dudns.com"
+      ],
+      "mx": [
+
+      ],
+      "a": [
+        "45.10.90.113"
+      ],
+      "aaaa": [
+
+      ]
+    },
+    "whois": {
+      "registrant_contacts": [
+
+      ],
+      "admin_contacts": [
+
+      ],
+      "technical_contacts": [
+
+      ]
+    },
+    "meta": {
+      "links": {
+        "securitytrails": "https://securitytrails.com/domain/jppost-ku.com/dns"
+      }
+    }
+  },
+  "ssh": {
+    "rsa": {
+      "md5": "565c74c34ca3a4a44625e8cbf732bed5",
+      "sha1": "2fb4d2241f7b6dd83c376548a794d5e903ce2b64",
+      "sha256": "e97b6fa7a9c3cb00919fbe90d862b08c2b4b1ac8c09701a0bb063e47ae764160"
+    },
+    "ecdsa-sha2-nistp256": {
+      "md5": "59e75650c592742fbe54a56140965af6",
+      "sha1": "1cddc49647d0e3cd5fefcc15e41fa036651ba903",
+      "sha256": "54a7bcac7ac7c2ffc501396dd1ae68b0c7f7b3a627c813c0020822b7a01e6a69"
+    },
+    "ed25519": {
+      "md5": "5ca62c892f4cb1c3197b245b2e1b9254",
+      "sha1": "9bbcfec876f80c831a9ace061dfa7ba7d207c2d2",
+      "sha256": "e7c2073b8ae07dea059307eb4d1f435c92d25228e5def49075e8007f5cb44765"
+    },
+    "meta": {
+      "links": {
+        "shodan": "https://www.shodan.io/search?query=port%3A22+56%3A5c%3A74%3Ac3%3A4c%3Aa3%3Aa4%3Aa4%3A46%3A25%3Ae8%3Acb%3Af7%3A32%3Abe%3Ad5",
+        "censys": "https://censys.io/ipv4?q=54a7bcac7ac7c2ffc501396dd1ae68b0c7f7b3a627c813c0020822b7a01e6a69"
+      }
+    }
+  },
+  "meta": {
+    "target": "jppost-ku.com"
+  }
+}
 ```
 
 ## Notes

data/apullo.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ipaddr", "~> 1.2"
   spec.add_dependency "mem", "~> 0.1"
   spec.add_dependency "murmurhash3", "~> 0.1"
-  spec.add_dependency "oga", "~> 3.0"
+  spec.add_dependency "oga", "~> 3.2"
   spec.add_dependency "parallel", "~> 1.19"
   spec.add_dependency "public_suffix", "~> 4.0"
   spec.add_dependency "ssh_scan", "~> 0.0"

data/lib/apullo/fingerprints/domain.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "resolv"
+require "uri"
 require "whois-parser"
 
 module Apullo
@@ -21,6 +22,9 @@ module Apullo
         {
           dns: resources,
           whois: contacts,
+          meta: {
+            links: links
+          }
         }
       end
 
@@ -89,6 +93,16 @@ module Apullo
           technical_contacts: technical_contacts
         }
       end
+
+      def securitytrails_link
+        target.domain? ? "https://securitytrails.com/domain/#{target.host}/dns" : "https://securitytrails.com/list/ip/#{target.host}"
+      end
+
+      def links
+        {
+          securitytrails: securitytrails_link
+        }
+      end
     end
   end
 end

data/lib/apullo/fingerprints/favicon.rb

@@ -24,7 +24,7 @@ module Apullo
           sha1: hash.sha1,
           sha256: hash.sha256,
           meta: {
-            url: uri.to_s
+            url: uri.to_s,
           }
         }
       end

data/lib/apullo/fingerprints/http.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
+require "mem"
 require "net/http"
 require "oga"
 require "openssl"
+require "uri"
 
 module Apullo
   module Fingerprint
     class HTTP < Base
+      include Mem
+
       attr_writer :headers
 
       def initialize(target)
@@ -26,7 +30,8 @@ module Apullo
           favicon: favicon,
           headers: response_headers,
           meta: {
-            url: target.url
+            url: target.url,
+            links: links
           }
         }
       end
@@ -42,6 +47,7 @@ module Apullo
           sha256: hash.sha256,
         }
       end
+      memoize :cert
 
       def body
         return {} unless @body
@@ -54,6 +60,7 @@ module Apullo
           sha256: hash.sha256,
         }
       end
+      memoize :body
 
       def favicon
         url = favicon_url
@@ -62,6 +69,7 @@ module Apullo
         favicon = Favicon.new(url)
         favicon.results
       end
+      memoize :favicon
 
       def response_headers
         @response_headers || {}
@@ -133,6 +141,38 @@ module Apullo
       def rebuild_target(url)
         @target = Target.new(url)
       end
+
+      def shodan_link
+        uri = URI("https://www.shodan.io/search")
+        uri.query = URI.encode_www_form(query: "http.html_hash:#{body.dig(:mmh3)}")
+        body_link = body.empty? ? nil : uri.to_s
+
+        uri.query = URI.encode_www_form(q: "ssl.cert.serial:#{cert.dig(:serial)}")
+        cert_link = cert.empty? ? nil : uri.to_s
+
+        uri.query = URI.encode_www_form(query: "http.favicon.hash:#{favicon.dig(:mmh3)}")
+        favicon_link = favicon.empty? ? nil : uri.to_s
+
+        { body: body_link, cert: cert_link, favicon: favicon_link }.compact
+      end
+
+      def censys_link
+        uri = URI("https://censys.io/ipv4")
+        uri.query = URI.encode_www_form(q: body.dig(:sha256))
+        body_link = body.empty? ? nil : uri.to_s
+
+        uri.query = URI.encode_www_form(q: cert.dig(:sha256))
+        cert_link = cert.empty? ? nil : uri.to_s
+
+        { body: body_link, cert: cert_link }.compact
+      end
+
+      def links
+        {
+          shodan: shodan_link,
+          censys: censys_link
+        }
+      end
     end
   end
 end

data/lib/apullo/fingerprints/ssh.rb

@@ -1,34 +1,40 @@
 # frozen_string_literal: true
 
+require "mem"
 require "ssh_scan"
+require "uri"
 
 module Apullo
   module Fingerprint
     class SSH < Base
+      include Mem
+
       DEFAULT_OPTIONS = { "timeout" => 3 }.freeze
       DEFAULT_PORTS = [22, 2222].freeze
 
       private
 
       def build_results
-        pluck_fingerprints
+        results = fingerprints
+        results = results.merge(meta: { links: links }) unless results.empty?
+        results
       end
 
-      def pluck_fingerprints
+      def fingerprints
         result = scan
         keys = result.dig("keys") || {}
         keys.map do |cipher, data|
-          raw = data.dig("raw")
           fingerprints = data.dig("fingerprints") || []
           normalized_fingerprints = fingerprints.map do |hash, value|
             [hash, value.delete(":")]
           end.to_h
           [
             cipher,
-            { raw: raw, fingerprints: normalized_fingerprints }
+            normalized_fingerprints
           ]
         end.to_h
       end
+      memoize :fingerprints
 
       def _scan(target, port: 22)
         return nil unless target.host
@@ -47,6 +53,32 @@ module Apullo
         end
         {}
       end
+
+      def shodan_link
+        uri = URI("https://www.shodan.io/search")
+        fingerprint = fingerprints.dig("rsa", "md5") || fingerprints.dig("ecdsa-sha2-nistp256", "md5")
+        return nil unless fingerprint
+
+        fingerprint = fingerprint.to_s.chars.each_slice(2).map(&:join).join(":")
+        uri.query = URI.encode_www_form(query: "port:22 #{fingerprint}")
+        uri.to_s
+      end
+
+      def censys_link
+        uri = URI("https://censys.io/ipv4")
+        fingerprint = fingerprints.dig("ecdsa-sha2-nistp256", "sha256") || fingerprints.dig("rsa", "sha256")
+        return nil unless fingerprint
+
+        uri.query = URI.encode_www_form(q: fingerprint.to_s)
+        uri.to_s
+      end
+
+      def links
+        {
+          shodan: shodan_link,
+          censys: censys_link
+        }
+      end
     end
   end
 end

data/lib/apullo/target.rb

@@ -39,12 +39,6 @@ module Apullo
       uri && (ip? | domain?)
     end
 
-    private
-
-    def _url
-      @_url ||= id.start_with?("http://", "https://") ? id : "http://#{id}"
-    end
-
     def ip?
       IPAddr.new host
       true
@@ -58,6 +52,12 @@ module Apullo
       PublicSuffix.valid?(host, default_rule: nil)
     end
 
+    private
+
+    def _url
+      @_url ||= id.start_with?("http://", "https://") ? id : "http://#{id}"
+    end
+
     def resolve
       Resolv.getaddress uri&.host
     rescue Resolv::ResolvError => _e

data/lib/apullo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Apullo
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apullo
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-03 00:00:00.000000000 Z
+date: 2020-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -156,14 +156,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement