phisher_phinder 0.1.0

data/lib/phisher_phinder/expanded_data_processor.rb

@@ -0,0 +1,61 @@
+module PhisherPhinder
+  class ExpandedDataProcessor
+    def process(mail)
+      {
+        linked_content: mail.hypertext_links.map { |l| lookup_content(l) },
+        mail: mail
+      }
+    end
+
+    private
+
+    def lookup_content(link)
+      base_output = {
+        href: link.href,
+        link_text: link.text,
+        content_requested: true,
+        response: nil,
+        error: nil
+      }
+
+      if link.supports_retrieval?
+        require 'net/http'
+
+        begin
+          response = Net::HTTP.get_response(link.href)
+
+          if response.is_a?(Net::HTTPOK)
+            base_output.merge({response: response_with_body(response)})
+          else
+            base_output.merge(response: response_status_only(response))
+          end
+        rescue => e
+          base_output.merge(
+            error: {
+              class: e.class,
+              message: e.message
+            }
+          )
+        end
+      else
+        base_output.merge(content_requested: false)
+      end
+    end
+
+    def response_with_body(response)
+      {
+        status: response.code.to_i,
+        body: response.body,
+        links_within_body: response.body.scan(/https?:\/\/[a-z0-9\/._?=,&#!*~();:@+$%\[\]-]+/i)
+      }
+    end
+
+    def response_status_only(response)
+      {
+        status: response.code.to_i,
+        body: nil,
+        links_within_body: []
+      }
+    end
+  end
+end

data/lib/phisher_phinder/extended_ip.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class ExtendedIp
+    attr_reader :ip_address, :geoip_ip_data
+
+    def initialize(ip_address:, geoip_ip_data:)
+      @ip_address = ip_address
+      @geoip_ip_data = geoip_ip_data
+    end
+
+    def ==(other)
+      ip_address == other.ip_address && geoip_ip_data == other.geoip_ip_data
+    end
+  end
+end

data/lib/phisher_phinder/extended_ip_factory.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+require 'ipaddr'
+
+module PhisherPhinder
+  class ExtendedIpFactory
+    def initialize(geoip_client:)
+      @geoip_client = geoip_client
+    end
+
+    def build(ip_string)
+      ip = IPAddr.new(ip_string)
+
+      if non_public_ip?(ip)
+        SimpleIp.new(ip_address: ip)
+      else
+        ExtendedIp.new(ip_address: ip, geoip_ip_data: geoip_data(ip_string))
+      end
+    rescue IPAddr::InvalidAddressError
+    end
+
+    private
+
+    def non_public_ip?(ip)
+      localhost_ip?(ip) ||
+        ipv4_class_a_private?(ip) ||
+        ipv4_class_b_private?(ip) ||
+        ipv4_class_c_private?(ip)
+    end
+
+    def localhost_ip?(ip)
+      ip.loopback?
+    end
+
+    def ipv4_class_a_private?(ip)
+      IPAddr.new('10.0.0.1/8').include?(ip)
+    end
+
+    def ipv4_class_b_private?(ip)
+      IPAddr.new('172.16.0.0/12').include?(ip)
+    end
+
+    def ipv4_class_c_private?(ip)
+      IPAddr.new('192.168.0.0/16').include?(ip)
+    end
+
+    def geoip_data(ip_string)
+      @geoip_client.lookup(ip_string)
+    rescue MaxMind::GeoIP2::AddressNotFoundError
+    end
+  end
+end

data/lib/phisher_phinder/geoip_ip_data.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class GeoipIpData < Sequel::Model(:geoip_ip_data)
+  end
+end

data/lib/phisher_phinder/mail.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class Mail
+    attr_reader :original_email, :original_headers, :original_body, :headers, :tracing_headers, :body
+
+    def initialize(
+      original_email:, original_headers:, original_body:, headers:, tracing_headers:, body:
+    )
+      @original_email = original_email
+      @original_headers = original_headers
+      @original_body = original_body
+      @headers = headers
+      @tracing_headers = tracing_headers
+      @body = body
+    end
+
+    def reply_to_addresses
+      @headers[:reply_to].map do |value_string|
+        value_string.split(",")
+      end.flatten.map do |email_address_string|
+        extract_email_address(email_address_string)
+      end.uniq
+    end
+
+    def hypertext_links
+      body_as_html.
+        xpath('//a').
+        select { |el| el.attributes['href'] }.
+        map { |el| BodyHyperlink.new(el.attributes['href'].value, el.text) }
+    end
+
+    private
+
+    def body_as_html
+      require 'nokogiri'
+
+      Nokogiri::HTML(body[:html])
+    end
+
+    def extract_email_address(email_address_string)
+      if email_address_string.include? '<'
+        email_address_string =~ /<([^>]+)>/
+        $1
+      else
+        email_address_string
+      end.downcase.strip
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+require_relative('mail_parser/body_parser')
+require_relative('mail_parser/header_value_parser')
+
+module PhisherPhinder
+  module MailParser
+    class Parser
+      def initialize(enriched_ip_factory, line_ending_type)
+        @line_end = line_ending_type == 'dos' ? "\r\n" : "\n"
+        @enriched_ip_factory = enriched_ip_factory
+      end
+
+      def parse(contents)
+        original_headers, original_body = separate(contents)
+        headers = extract_headers(original_headers)
+        Mail.new(
+          original_email: contents,
+          original_headers: original_headers,
+          original_body: original_body,
+          headers: headers,
+          tracing_headers: generate_tracing_headers(headers),
+          body: parse_body(original_body, headers)
+        )
+      end
+
+      private
+
+      def separate(contents)
+        contents.split("#{@line_end}#{@line_end}", 2)
+      end
+
+      def extract_headers(headers)
+        parse_headers(unfold_headers(headers).split(@line_end))
+      end
+
+      def unfold_headers(headers)
+        headers.gsub(/#{@line_end}[\s\t]+/, ' ')
+      end
+
+      def parse_headers(headers_array)
+        headers_array.each_with_index.inject({}) do |memo, (header_string, index)|
+          header, value = header_string.split(":", 2)
+          sequence = headers_array.length - index - 1
+          memo.merge(convert_header_name(header) => enrich_header_value(value, sequence)) do |_, existing, new|
+            if existing.is_a? Array
+              existing << new
+            else
+              [existing, new]
+            end
+          end
+        end
+      end
+
+      def convert_header_name(header)
+        header.gsub(/-/, '_').downcase.to_sym
+      end
+
+      def enrich_header_value(value, sequence)
+        {data: HeaderValueParser.new.parse(value), sequence: sequence}
+      end
+
+      def generate_tracing_headers(headers)
+        received_header_values = headers.inject([]) do |memo, (header_name, header_value)|
+          if [:received, :x_received].include? header_name
+            if header_value.is_a? Array
+              memo +=  header_value
+            else
+              memo << header_value
+            end
+          end
+
+          memo
+        end.flatten
+
+        {
+          received: restore_sequence(received_header_values).map { |v| parse_received_header(v[:data]) }
+        }
+      end
+
+      def parse_received_header(value)
+        parser = MailParser::ReceivedHeaders::Parser.new(
+          by_parser: MailParser::ReceivedHeaders::ByParser.new(@enriched_ip_factory),
+          for_parser: MailParser::ReceivedHeaders::ForParser.new,
+          from_parser: MailParser::ReceivedHeaders::FromParser.new(@enriched_ip_factory),
+          starttls_parser: MailParser::ReceivedHeaders::StarttlsParser.new,
+          timestamp_parser: MailParser::ReceivedHeaders::TimestampParser.new,
+          classifier: MailParser::ReceivedHeaders::Classifier.new
+        )
+        parser.parse(value)
+      end
+
+      def restore_sequence(values)
+        values.sort { |a,b| b[:sequence] <=> a[:sequence] }
+      end
+
+      def parse_body(original_body, headers)
+        MailParser::BodyParser.new(@line_end).parse(
+          body_contents: original_body,
+          content_type: headers.dig(:content_type, :data),
+          content_transfer_encoding: headers.dig(:content_transfer_encoding, :data),
+        )
+      end
+
+      def valid_base64_decoded(text)
+        if Base64.strict_encode64(Base64.decode64(text)) == text.gsub(/#{@line_end}/, '')
+          Base64.decode64(text)
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/body_parser.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    class BodyParser
+      def initialize(line_end)
+        @line_end = line_end
+      end
+
+      def parse(body_contents:, content_type:, content_transfer_encoding:)
+        if multipart_alternative?(content_type)
+          parse_multipart_alternative(content_type, body_contents)
+        elsif html?(content_type)
+          {
+            text: nil,
+            html: decode_body(body_contents, content_transfer_encoding)
+          }
+        else
+          {
+            text: body_contents,
+            html: nil
+          }
+        end
+      end
+
+      private
+
+      def html?(content_type)
+        content_type && content_type.split(';').first == 'text/html'
+      end
+
+      def decode_body(body_contents, content_transfer_encoding)
+        require 'base64'
+
+        content_transfer_encoding ? Base64.decode64(body_contents) : body_contents
+      end
+
+      def multipart_alternative?(content_type)
+        content_type =~ /\Amultipart\/alternative/
+      end
+
+      def parse_multipart_alternative(content_type, contents)
+        base_boundary = content_type.split(';').last.strip.split('=').last
+        start_boundary = '--' + base_boundary + @line_end
+        end_boundary = '--' + base_boundary + '--'
+
+        raw_blocks = contents.split(start_boundary)
+        trimmed_blocks = strip_epilogue(strip_prologue(raw_blocks), end_boundary)
+
+        categorise_blocks(trimmed_blocks).inject({html: '', text: ''}) do |memo, block|
+          memo.merge(block[:html] ? {html: memo[:html] + block[:contents]} : {text: memo[:text] + block[:contents]})
+        end
+      end
+
+      def strip_prologue(blocks)
+        blocks[1..-1]
+      end
+
+      def strip_epilogue(blocks, end_boundary)
+        blocks[0..-2] << blocks[-1].split(end_boundary).first
+      end
+
+      def categorise_blocks(blocks)
+        blocks.map do |block|
+          lines = block.split(@line_end)
+          processing_block_headers = true
+          html = false
+          base64_encoded = false
+
+          while processing_block_headers do
+            line = lines.shift.strip
+            if line.empty?
+              processing_block_headers = false
+            elsif line =~/\AContent-Type: text\/html/
+              html = true
+            elsif line =~ /\AContent-Transfer-Encoding: base64/
+              base64_encoded = true
+            end
+          end
+
+          contents = if base64_encoded
+                       (lines.map { |l| Base64.decode64(l) }).join
+                     else
+                       lines.join(@line_end)
+                     end
+          {
+            html: html,
+            contents: contents
+          }
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/header_value_parser.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    class HeaderValueParser
+      def parse(raw_value)
+        utf_8_preambles = raw_value.scan(/=\?UTF-8\?b\?/)
+        if raw_value.scan(/=\?UTF-8\?b\?/).any?
+          (raw_value.split(' ').map { |snippet| parse_utf8_base64(snippet) }).join
+        else
+          raw_value.strip
+        end
+      end
+
+      private
+
+      def parse_utf8_base64(raw_value)
+        require 'base64'
+
+        Base64.decode64(raw_value.strip.sub(/=\?UTF-8\?b\?/, '')).force_encoding('UTF-8')
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/received_headers/by_parser.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module ReceivedHeaders
+      class ByParser
+        def initialize(extended_ip_factory)
+          @extended_ip_factory = extended_ip_factory
+        end
+
+        def parse(component)
+          return {recipient: nil, protocol: nil, id: nil, recipient_additional: nil} unless component
+
+          patterns = [
+            /by\s(?<recipient>\S+)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\s(?<additional>.+)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\swith\s(?<protocol>.+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\swith\s(?<protocol>\S+)\sID\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\swith\s(?<protocol>.+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\swith\s(?<protocol>.+)/,
+          ]
+
+          matches = patterns.inject(nil) do |memo, pattern|
+            memo || component.match(pattern)
+          end
+
+          {
+            recipient: enrich_recipient(matches[:recipient]),
+            protocol: matches.names.include?('protocol') ? matches[:protocol]: nil,
+            id: matches.names.include?('id') ? matches[:id]: nil,
+            recipient_additional: matches.names.include?('additional') ? matches[:additional] : nil
+          }
+        end
+
+        private
+
+        def enrich_recipient(recipient)
+          @extended_ip_factory.build(recipient) || recipient
+        end
+      end
+    end
+  end
+end