phisher_phinder 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfb338171a5bca59a9682dfe8157475e7ac1a6283d9410a8367fdbe78fed0be1
-  data.tar.gz: db30dccf14c2184cd01f4cb27e4d0cc305fda0ddb0b58205a6e3a077893e1eae
+  metadata.gz: bf9bd21584b11a5fe4ceff942aa6b7631fef9177682b775d9c0a0508a99c0643
+  data.tar.gz: b10859f0c80054eeeff83288dde7eb25ef5bdf8a32d83cd04cfff181a39864a2
 SHA512:
-  metadata.gz: 831b8c8a93dc8cf8105724960be8052c1aa47a16fb57336a683e9780b07ce89a72e2e081344826b5462582c1270ca3fcd8a359d8b5111f97c4925c37cd5c085e
-  data.tar.gz: c25e50d534e780eaeef883a8e6c4311c4c51176589ac80959adb880c0b9f190b5350dd752497f3b19eb8ede04c95bd527201cf18f5f96222890ba08893495dd9
+  metadata.gz: dcf8371fbe75476a791fd12c94be9e2c1fac881785b856a62a9aba67fdca39cdef6956aba9f5b772882f0b58f93ff810734f94f6c8abb2aff89dc0b12364f2c7
+  data.tar.gz: b44b1edcb3c1dc0732478d512f75ef599b2fa9b74db3ecc981c0125f5547703b983ddd9dddb205c745688631f79f0722548021da1bd42106ec1f6282bccae88c

data/Gemfile.lock

@@ -3,6 +3,8 @@ PATH
   specs:
     phisher_phinder (0.3.0)
       dotenv (~> 2.7.5)
+      excon (~> 0.78.1)
+      mail
       maxmind-geoip2 (~> 0.4.0)
       nokogiri (~> 1.11.0)
       sequel (~> 5.33)
@@ -14,7 +16,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.0)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -26,7 +28,7 @@ GEM
       bundler (>= 1.2.0, < 3)
       thor (>= 0.18, < 2)
     coderay (1.1.2)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.8)
     connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -38,6 +40,7 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     dotenv (2.7.6)
+    excon (0.78.1)
     ffi (1.14.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
@@ -51,18 +54,23 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.2)
-      ffi-compiler
-    i18n (1.8.7)
+    http-parser (1.2.3)
+      ffi-compiler (>= 1.0, < 2.0)
+    i18n (1.8.8)
       concurrent-ruby (~> 1.0)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
     maxmind-db (1.1.1)
     maxmind-geoip2 (0.4.0)
       connection_pool (~> 2.2)
       http (~> 4.3)
       maxmind-db (~> 1.1)
     method_source (1.0.0)
-    minitest (5.14.2)
-    nokogiri (1.11.0-x86_64-linux)
+    mini_mime (1.0.2)
+    mini_portile2 (2.5.0)
+    minitest (5.14.3)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     pry (0.13.1)
       coderay (~> 1.1)

data/bin/console

@@ -4,8 +4,8 @@ require "bundler/setup"
 require 'dotenv'
 Dotenv.load('.env')
 
-require'sequel'
-DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
+# require'sequel'
+# DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
 
 require "phisher_phinder"
 

data/lib/phisher_phinder.rb

@@ -8,7 +8,10 @@ require_relative './phisher_phinder/display'
 
 require_relative './phisher_phinder/body_hyperlink'
 require_relative './phisher_phinder/cached_geoip_client'
-require_relative './phisher_phinder/contact_finder'
+require_relative './phisher_phinder/host_information_finder'
+require_relative './phisher_phinder/host_response_policy'
+require_relative './phisher_phinder/link_explorer'
+require_relative './phisher_phinder/link_host'
 require_relative './phisher_phinder/whois_email_extractor'
 require_relative './phisher_phinder/null_lookup_client'
 require_relative './phisher_phinder/null_response'

data/lib/phisher_phinder/command.rb

@@ -14,11 +14,16 @@ module PhisherPhinder
       ip_factory = PhisherPhinder::ExtendedIpFactory.new(geoip_client: lookup_client)
       mail_parser = PhisherPhinder::MailParser::Parser.new(ip_factory, line_ending)
       whois_client = Whois::Client.new
+      host_information_finder = PhisherPhinder::HostInformationFinder.new(
+        whois_client: whois_client,
+        extractor: PhisherPhinder::WhoisEmailExtractor.new
+      )
       tracing_report = PhisherPhinder::TracingReport.new(
-        mail_parser.parse(contents),
-        PhisherPhinder::ContactFinder.new(
-          whois_client: whois_client,
-          extractor: PhisherPhinder::WhoisEmailExtractor.new
+        mail:  mail_parser.parse(contents),
+        host_information_finder: host_information_finder,
+        link_explorer: PhisherPhinder::LinkExplorer.new(
+          host_information_finder: host_information_finder,
+          host_response_policy: PhisherPhinder::HostResponsePolicy.new
         )
       )
       tracing_report.report

data/lib/phisher_phinder/display.rb

@@ -47,6 +47,22 @@ module PhisherPhinder
       )
 
       puts trace_table
+
+      puts "\n\n"
+
+      puts 'Body Content'
+      puts "\n"
+      puts "Linked URLs"
+      input_data[:content][:linked_urls].each do |link_set|
+        link_set.each_with_index do |link_host, tab_count|
+          puts "#{"\t"*tab_count}" +
+            "#{link_host.url.to_s} " +
+            "(#{display_creation_date(link_host)}) " +
+            "[#{display_email_addresses(link_host.host_information[:abuse_contacts])}]" +
+            "\n"
+        end
+        puts "\n"
+      end
     end
 
     private
@@ -66,5 +82,9 @@ module PhisherPhinder
     def display_email_addresses(email_addresses)
       email_addresses.map { |address| address.gsub(/[,<>]/, '') }.join(', ')
     end
+
+    def display_creation_date(link_host)
+      (date = link_host.host_information[:creation_date]) ? date.strftime('%Y-%m-%d %H:%M:%S') : nil
+    end
   end
 end

data/lib/phisher_phinder/host_information_finder.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class HostInformationFinder
+    def initialize(whois_client:, extractor:)
+      @whois_client = whois_client
+      @extractor = extractor
+    end
+
+    def information_for(address)
+      most_relevant_record = nil
+
+      begin
+        case address
+        when ExtendedIp
+          most_relevant_record = @whois_client.lookup(address.ip_address.to_s)
+        when String
+          hostname_parts = address.split('.')
+          until most_relevant_record do
+            address = hostname_parts.join('.')
+            whois_record = @whois_client.lookup(address)
+            if whois_record.parser.available?
+              hostname_parts = hostname_parts[1..-1]
+            else
+              most_relevant_record = whois_record
+            end
+          end
+        end
+      rescue Whois::ServerNotFound
+      rescue Whois::AttributeNotImplemented
+      end
+
+      {
+        abuse_contacts: most_relevant_record ? @extractor.abuse_contact_emails(most_relevant_record.content) : [],
+        creation_date: creation_date(most_relevant_record)
+      }
+    end
+
+    private
+
+    def creation_date(record)
+      return nil unless record
+
+      begin
+        record.parser.created_on
+      rescue Whois::AttributeNotImplemented, Whois::AttributeNotSupported
+      end
+    end
+  end
+end

data/lib/phisher_phinder/host_response_policy.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class HostResponsePolicy
+    def next_url(url, response)
+      location_header = response.headers['Location']
+
+      if [301, 302, 303, 307, 308].include?(response.status) && location_header && !location_header.empty?
+        if response.headers['Location'] =~ %r{\A/}
+          url.merge(response.headers['Location'])
+        else
+          URI.parse(response.headers['Location'])
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/link_explorer.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'excon'
+
+module PhisherPhinder
+  class LinkExplorer
+    def initialize(host_information_finder:, host_response_policy:)
+      @host_information_finder = host_information_finder
+      @host_response_policy = host_response_policy
+    end
+
+    def explore(hyperlink)
+      if hyperlink.type == :url
+        chain_terminated = false
+        url = hyperlink.href
+        output = []
+
+        until chain_terminated do
+          result = Excon.get(url.to_s)
+
+          output << LinkHost.new(
+            url: url,
+            body: result.body,
+            headers: result.headers,
+            status_code: result.status,
+            host_information: @host_information_finder.information_for("#{url.scheme}://#{url.host}"),
+          )
+
+          unless url = @host_response_policy.next_url(url, result)
+            chain_terminated = true
+          end
+        end
+
+        output
+      else
+        hyperlink.href =~ /mailto:(.+)/
+        ($1.split(';').map { |address| address.strip }).uniq
+      end
+    end
+  end
+end

data/lib/phisher_phinder/link_host.rb

@@ -0,0 +1,20 @@
+# frozen-string_literal: true
+
+module PhisherPhinder
+  class LinkHost
+    attr_accessor :url, :body, :status_code, :headers, :host_information
+
+    def initialize(url:, body:, status_code:, headers:, host_information:)
+      @url = url
+      @body = body
+      @status_code = status_code
+      @headers = headers
+      @host_information = host_information
+    end
+
+    def ==(other)
+      url == other.url && body == other.body && status_code == other.status_code && headers == other.headers &&
+        host_information == other.host_information
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser.rb

@@ -13,13 +13,13 @@ module PhisherPhinder
       def parse(contents)
         original_headers, original_body = separate(contents)
         headers = extract_headers(original_headers)
-        Mail.new(
+        PhisherPhinder::Mail.new(
           original_email: contents,
           original_headers: original_headers,
           original_body: original_body,
           headers: headers,
           tracing_headers: generate_tracing_headers(headers),
-          body: parse_body(original_body, headers),
+          body: MailParser::BodyParser.new.parse(contents),
           authentication_headers: generate_authentication_headers(headers)
         )
       end
@@ -86,14 +86,6 @@ module PhisherPhinder
         values.sort { |a,b| b[:sequence] <=> a[:sequence] }
       end
 
-      def parse_body(original_body, headers)
-        MailParser::BodyParser.new(@line_ending).parse(
-          body_contents: original_body,
-          content_type: content_type_data(headers),
-          content_transfer_encoding: content_transfer_encoding_data(headers),
-        )
-      end
-
       def valid_base64_decoded(text)
         if Base64.strict_encode64(Base64.decode64(text)) == text.gsub(/#{@line_ending}/, '')
           Base64.decode64(text)

data/lib/phisher_phinder/mail_parser/body_parser.rb

@@ -1,87 +1,36 @@
 # frozen_string_literal: true
+require 'mail'
 
 module PhisherPhinder
   module MailParser
     class BodyParser
-      def initialize(line_end)
-        @line_end = line_end
-      end
-
-      def parse(body_contents:, content_type:, content_transfer_encoding:)
-        if multipart_alternative?(content_type)
-          parse_multipart_alternative(content_type, body_contents)
-        else
-          classifier = Body::BlockClassifier.new(@line_end)
-          parser = Body::BlockParser.new(@line_end)
-
-          classification = classifier.classify_headers(
-            content_type: content_type, content_transfer_encoding: content_transfer_encoding
-          ).merge(content: body_contents)
-
-          contents = parser.parse(classification)
-
-          if classification[:content_type] == :html
-            {
-              html: contents,
-              text: nil
-            }
-          else
-            {
-              html: nil,
-              text: contents
-            }
-          end
-        end
+      def parse(contents)
+        mail = ::Mail.new(contents)
+        aggregate_body_parts(mail)
       end
 
       private
 
-      def html?(content_type)
-        content_type && content_type.split(';').first == 'text/html'
-      end
-
-      def decode_body(body_contents, content_transfer_encoding)
-        require 'base64'
-
-        content_transfer_encoding ? Base64.decode64(body_contents) : body_contents
-      end
-
-      def multipart_alternative?(content_type)
-        content_type =~ /\Amultipart\/alternative/
-      end
-
-      def parse_multipart_alternative(content_type, contents)
-        base_boundary = content_type.split(';').last.strip.gsub(/boundary=/, '').gsub(/"/, '')
-        start_boundary = '--' + base_boundary + @line_end
-        end_boundary = '--' + base_boundary + '--'
-
-        raw_blocks = contents.split(start_boundary)
-        trimmed_blocks = strip_epilogue(strip_prologue(raw_blocks), end_boundary)
-
-        categorise_blocks(trimmed_blocks).inject({html: '', text: ''}) do |memo, block|
-          memo.merge(block[:html] ? {html: memo[:html] + block[:contents]} : {text: memo[:text] + block[:contents]})
+      def aggregate_body_parts(mail)
+        accumulator = {text: [], html: []}
+        if mail.body.parts.any?
+          collapse_content(mail.body, accumulator)
+          accumulator.inject({}) { |accum, (type, parts)| accum.merge(type => parts.join) }
+        else
+          {
+            text: mail.body.decoded,
+            html: nil
+          }
         end
       end
 
-      def strip_prologue(blocks)
-        blocks[1..-1]
-      end
-
-      def strip_epilogue(blocks, end_boundary)
-        blocks[0..-2] << blocks[-1].split(end_boundary).first
-      end
-
-      def categorise_blocks(blocks)
-        classifier = Body::BlockClassifier.new(@line_end)
-        parser = Body::BlockParser.new(@line_end)
-        blocks.map do |block|
-          classification = classifier.classify_block(block)
-          contents = parser.parse(classification)
-
-          {
-            html: classification[:content_type] == :html,
-            contents: contents
-          }
+      def collapse_content(part, accumulator)
+        if part.parts.any?
+          part.parts.each { |p| collapse_content(p, accumulator) }
+        elsif part.content_type =~ %r{text/plain}
+          accumulator[:text] << part.decoded
+        elsif part.content_type =~ %r{text/html}
+          accumulator[:html] << part.decoded
         end
       end
     end

data/lib/phisher_phinder/tracing_report.rb

@@ -2,9 +2,10 @@
 
 module PhisherPhinder
   class TracingReport
-    def initialize(mail, contact_finder)
+    def initialize(mail:, host_information_finder:, link_explorer:)
       @mail = mail
-      @contact_finder = contact_finder
+      @host_information_finder = host_information_finder
+      @link_explorer = link_explorer
     end
 
     def report
@@ -19,7 +20,8 @@ module PhisherPhinder
           }
         },
         origin: extract_origin_headers(@mail.headers),
-        tracing: extract_tracing_headers(@mail.tracing_headers, latest_spf_entry)
+        tracing: extract_tracing_headers(@mail.tracing_headers, latest_spf_entry),
+        content: explore_hyperlinks(@mail.hypertext_links)
       }
     end
 
@@ -38,8 +40,16 @@ module PhisherPhinder
       received_headers[:received][start..-1].map do |h|
         h.merge(
           sender_contact_details: {
-            host: {email: @contact_finder.contacts_for(h[:sender][:host])},
-            ip: {email: @contact_finder.contacts_for(h[:sender][:ip])},
+            host: {
+              email: @host_information_finder.information_for(
+                h[:sender][:host]
+              )[:abuse_contacts]
+            },
+            ip: {
+              email: @host_information_finder.information_for(
+                h[:sender][:ip]
+              )[:abuse_contacts]
+            },
           }
         )
       end
@@ -51,5 +61,15 @@ module PhisherPhinder
         output.merge(header_type => entries.map { |h| h[:data] })
       end
     end
+
+    def explore_hyperlinks(hyperlinks)
+      url_hyperlinks = (hyperlinks.select{ |link| link.type == :url }).uniq { |link| link.href }
+      email_hyperlinks = hyperlinks.select { |link| link.type == :email_address }
+
+      {
+        linked_urls: url_hyperlinks.map { |hyperlink| @link_explorer.explore(hyperlink) },
+        linked_email_addresses: (email_hyperlinks.map { |hyperlink| @link_explorer.explore(hyperlink) }).flatten.uniq
+      }
+    end
   end
 end

data/lib/phisher_phinder/version.rb

@@ -1,3 +1,3 @@
 module PhisherPhinder
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/phisher_phinder.gemspec

@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "dotenv", "~> 2.7.5"
+  spec.add_dependency "excon", "~> 0.78.1"
+  spec.add_dependency "mail"
   spec.add_dependency "maxmind-geoip2", "~> 0.4.0"
   spec.add_dependency "nokogiri", "~> 1.11.0"
   spec.add_dependency "sequel", "~> 5.33"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: phisher_phinder
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Rory McKinley
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-09 00:00:00.000000000 Z
+date: 2021-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -24,6 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.7.5
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.78.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.78.1
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: maxmind-geoip2
   requirement: !ruby/object:Gem::Requirement
@@ -249,12 +277,15 @@ files:
 - lib/phisher_phinder/body_hyperlink.rb
 - lib/phisher_phinder/cached_geoip_client.rb
 - lib/phisher_phinder/command.rb
-- lib/phisher_phinder/contact_finder.rb
 - lib/phisher_phinder/display.rb
 - lib/phisher_phinder/expanded_data_processor.rb
 - lib/phisher_phinder/extended_ip.rb
 - lib/phisher_phinder/extended_ip_factory.rb
 - lib/phisher_phinder/geoip_ip_data.rb
+- lib/phisher_phinder/host_information_finder.rb
+- lib/phisher_phinder/host_response_policy.rb
+- lib/phisher_phinder/link_explorer.rb
+- lib/phisher_phinder/link_host.rb
 - lib/phisher_phinder/mail.rb
 - lib/phisher_phinder/mail_parser.rb
 - lib/phisher_phinder/mail_parser/authentication_headers/auth_results_parser.rb

data/lib/phisher_phinder/contact_finder.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module PhisherPhinder
-  class ContactFinder
-    def initialize(whois_client:, extractor:)
-      @whois_client = whois_client
-      @extractor = extractor
-    end
-
-    def contacts_for(address)
-      whois_content = nil
-      begin
-        whois_content = case address
-                        when ExtendedIp
-                          @whois_client.lookup(address.ip_address.to_s).content
-                        when String
-                          hostname_parts = address.split('.')
-                          whois_content = nil
-                          until whois_content do
-                            address = hostname_parts.join('.')
-                            whois_record = @whois_client.lookup(address)
-                            if whois_record.parser.available?
-                              hostname_parts = hostname_parts[1..-1]
-                            else
-                              whois_content = whois_record.content
-                            end
-                          end
-                          whois_content
-                        end
-      rescue Whois::ServerNotFound
-      rescue Whois::AttributeNotImplemented
-      end
-
-      whois_content ? @extractor.abuse_contact_emails(whois_content) : []
-    end
-  end
-end