phisher_phinder 0.1.0 → 0.2.0

data/lib/phisher_phinder/extended_ip.rb

@@ -12,5 +12,9 @@ module PhisherPhinder
     def ==(other)
       ip_address == other.ip_address && geoip_ip_data == other.geoip_ip_data
     end
+
+    def to_s
+      @ip_address.to_s
+    end
   end
 end

data/lib/phisher_phinder/extended_ip_factory.rb

@@ -8,9 +8,11 @@ module PhisherPhinder
     end
 
     def build(ip_string)
+      return nil unless ip_string
+
       ip = IPAddr.new(ip_string)
 
-      if non_public_ip?(ip)
+      if non_public_ip?(ip) || ip.ipv6?
         SimpleIp.new(ip_address: ip)
       else
         ExtendedIp.new(ip_address: ip, geoip_ip_data: geoip_data(ip_string))
@@ -45,7 +47,7 @@ module PhisherPhinder
 
     def geoip_data(ip_string)
       @geoip_client.lookup(ip_string)
-    rescue MaxMind::GeoIP2::AddressNotFoundError
+    rescue MaxMind::GeoIP2::AddressNotFoundError, MaxMind::GeoIP2::AddressReservedError
     end
   end
 end

data/lib/phisher_phinder/geoip_ip_data.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
-module PhisherPhinder
-  class GeoipIpData < Sequel::Model(:geoip_ip_data)
+if ENV['DATABASE_URL']
+  require 'sequel'
+  Sequel::Model.plugin :timestamps, update_on_create: true
+
+  DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
+
+  module PhisherPhinder
+    class GeoipIpData < Sequel::Model(:geoip_ip_data)
+    end
   end
 end

data/lib/phisher_phinder/mail.rb

@@ -2,21 +2,28 @@
 
 module PhisherPhinder
   class Mail
-    attr_reader :original_email, :original_headers, :original_body, :headers, :tracing_headers, :body
+    attr_reader :original_email,
+      :original_headers,
+      :original_body,
+      :headers,
+      :tracing_headers,
+      :body,
+      :authentication_headers
 
     def initialize(
-      original_email:, original_headers:, original_body:, headers:, tracing_headers:, body:
+      original_email:, original_headers:, original_body:, headers:, tracing_headers:, body:, authentication_headers:
     )
       @original_email = original_email
       @original_headers = original_headers
       @original_body = original_body
       @headers = headers
       @tracing_headers = tracing_headers
+      @authentication_headers = authentication_headers
       @body = body
     end
 
     def reply_to_addresses
-      @headers[:reply_to].map do |value_string|
+      (@headers[:reply_to] || []).map do |value_string|
         value_string.split(",")
       end.flatten.map do |email_address_string|
         extract_email_address(email_address_string)

data/lib/phisher_phinder/mail_parser.rb

@@ -5,8 +5,8 @@ require_relative('mail_parser/header_value_parser')
 module PhisherPhinder
   module MailParser
     class Parser
-      def initialize(enriched_ip_factory, line_ending_type)
-        @line_end = line_ending_type == 'dos' ? "\r\n" : "\n"
+      def initialize(enriched_ip_factory, line_ending)
+        @line_ending = line_ending
         @enriched_ip_factory = enriched_ip_factory
       end
 
@@ -19,34 +19,31 @@ module PhisherPhinder
           original_body: original_body,
           headers: headers,
           tracing_headers: generate_tracing_headers(headers),
-          body: parse_body(original_body, headers)
+          body: parse_body(original_body, headers),
+          authentication_headers: generate_authentication_headers(headers)
         )
       end
 
       private
 
       def separate(contents)
-        contents.split("#{@line_end}#{@line_end}", 2)
+        contents.split("#{@line_ending}#{@line_ending}", 2)
       end
 
       def extract_headers(headers)
-        parse_headers(unfold_headers(headers).split(@line_end))
+        parse_headers(unfold_headers(headers).split(@line_ending))
       end
 
       def unfold_headers(headers)
-        headers.gsub(/#{@line_end}[\s\t]+/, ' ')
+        headers.gsub(/#{@line_ending}[\s\t]+/, ' ')
       end
 
       def parse_headers(headers_array)
         headers_array.each_with_index.inject({}) do |memo, (header_string, index)|
           header, value = header_string.split(":", 2)
           sequence = headers_array.length - index - 1
-          memo.merge(convert_header_name(header) => enrich_header_value(value, sequence)) do |_, existing, new|
-            if existing.is_a? Array
-              existing << new
-            else
-              [existing, new]
-            end
+          memo.merge(convert_header_name(header) => [enrich_header_value(value, sequence)]) do |_, existing, new|
+            existing + new
           end
         end
       end
@@ -61,16 +58,8 @@ module PhisherPhinder
 
       def generate_tracing_headers(headers)
         received_header_values = headers.inject([]) do |memo, (header_name, header_value)|
-          if [:received, :x_received].include? header_name
-            if header_value.is_a? Array
-              memo +=  header_value
-            else
-              memo << header_value
-            end
-          end
-
-          memo
-        end.flatten
+          [:received, :x_received].include?(header_name) ? memo + header_value : memo
+        end
 
         {
           received: restore_sequence(received_header_values).map { |v| parse_received_header(v[:data]) }
@@ -78,11 +67,15 @@ module PhisherPhinder
       end
 
       def parse_received_header(value)
+        starttls_parser = MailParser::ReceivedHeaders::StarttlsParser.new
         parser = MailParser::ReceivedHeaders::Parser.new(
-          by_parser: MailParser::ReceivedHeaders::ByParser.new(@enriched_ip_factory),
-          for_parser: MailParser::ReceivedHeaders::ForParser.new,
-          from_parser: MailParser::ReceivedHeaders::FromParser.new(@enriched_ip_factory),
-          starttls_parser: MailParser::ReceivedHeaders::StarttlsParser.new,
+          by_parser: MailParser::ReceivedHeaders::ByParser.new(
+            ip_factory: @enriched_ip_factory, starttls_parser: starttls_parser
+          ),
+          for_parser: MailParser::ReceivedHeaders::ForParser.new(starttls_parser: starttls_parser),
+          from_parser: MailParser::ReceivedHeaders::FromParser.new(
+            ip_factory: @enriched_ip_factory, starttls_parser: starttls_parser
+          ),
           timestamp_parser: MailParser::ReceivedHeaders::TimestampParser.new,
           classifier: MailParser::ReceivedHeaders::Classifier.new
         )
@@ -94,18 +87,38 @@ module PhisherPhinder
       end
 
       def parse_body(original_body, headers)
-        MailParser::BodyParser.new(@line_end).parse(
+        MailParser::BodyParser.new(@line_ending).parse(
           body_contents: original_body,
-          content_type: headers.dig(:content_type, :data),
-          content_transfer_encoding: headers.dig(:content_transfer_encoding, :data),
+          content_type: content_type_data(headers),
+          content_transfer_encoding: content_transfer_encoding_data(headers),
         )
       end
 
       def valid_base64_decoded(text)
-        if Base64.strict_encode64(Base64.decode64(text)) == text.gsub(/#{@line_end}/, '')
+        if Base64.strict_encode64(Base64.decode64(text)) == text.gsub(/#{@line_ending}/, '')
           Base64.decode64(text)
         end
       end
+
+      def content_type_data(headers)
+        (headers[:content_type] && headers[:content_type].first[:data]) || nil
+      end
+
+      def content_transfer_encoding_data(headers)
+        (headers[:content_transfer_encoding] && headers[:content_transfer_encoding].first[:data]) || nil
+      end
+
+      def generate_authentication_headers(headers)
+        auth_parser = MailParser::AuthenticationHeaders::Parser.new(
+          authentication_results_parser: MailParser::AuthenticationHeaders::AuthResultsParser.new(
+            ip_factory: @enriched_ip_factory
+          ),
+          received_spf_parser: MailParser::AuthenticationHeaders::ReceivedSpfParser.new(
+            ip_factory: @enriched_ip_factory
+          ),
+        )
+        auth_parser.parse(headers)
+      end
     end
   end
 end

data/lib/phisher_phinder/mail_parser/authentication_headers/auth_results_parser.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module AuthenticationHeaders
+      class AuthResultsParser
+        def initialize(ip_factory:)
+          @ip_factory = ip_factory
+        end
+
+        def parse(value)
+          output_template = {authserv_id: nil, auth: [], dkim: [], dmarc: [], iprev: [], spf: []}
+
+          components(value).inject(output_template) do |output, component|
+            component_type, parsed_component = parse_component(component)
+            if output[component_type].respond_to?(:<<)
+              output.merge(component_type => (output[component_type] << parsed_component))
+            else
+              output.merge(component_type => parsed_component)
+            end
+          end
+        end
+
+        private
+
+        def components(value)
+          value.split(';').map { |component| component.strip }
+        end
+
+        def parse_component(component)
+          if component =~ /\Aspf=/
+            [:spf, spf_data(component)]
+          elsif component =~ /\Adkim=/
+            [:dkim, dkim_data(component)]
+          elsif component =~ /\Aiprev=/
+            [:iprev, iprev_data(component)]
+          elsif component =~ /\Aauth=/
+            [:auth, auth_data(component)]
+          elsif component =~ /\Admarc=/
+            [:dmarc, {}]
+          else
+            [:authserv_id, component]
+          end
+        end
+
+        def spf_data(value)
+          patterns = [
+            %r{
+              spf=(?<result>[\S]+)\s
+              \([^:]+:\s(?<ip>[\S]+)\sis\sneither\spermitted[^\)]+\)\s
+              smtp.mailfrom=(?<from>[^\s;]+)
+            }x,
+            %r{
+              spf=(?<result>[\S]+)\s
+              \(.+\s(?<ip>[\S]+)\sas\spermitted.+\)\s
+              smtp.mailfrom=(?<from>[^\s;]+)
+            }x,
+            %r{
+              spf=(?<result>[\S]+)\ssmtp.mailfrom=(?<from>[^\s;]+)
+            }x,
+          ]
+
+          matches = patterns.inject(nil) do |memo, pattern|
+            memo || value.match(pattern)
+          end
+
+          if matches
+            {
+              result: matches[:result].to_sym,
+              ip: matches.names.include?('ip') ? @ip_factory.build(matches[:ip]) : nil,
+              from: matches[:from]
+            }
+          else
+            {}
+          end
+        end
+
+        def dkim_data(value)
+          patterns = [
+            %r{
+              dkim=(?<result>[\S]+)\s.*?
+              header.i=(?<identity>[\S]+)\s
+              header.s=(?<selector>[\S]+)\s
+              header.b=(?<hash_snippet>.{8})
+            }x,
+            %r{
+              dkim=(?<result>[\S]+)\s.*?
+              header.i=(?<identity>[\S]+)\s
+              header.s=(?<selector>[\S]+)
+            }x,
+          ]
+
+          matches = patterns.inject(nil) do |memo, pattern|
+            memo || value.match(pattern)
+          end
+
+          if matches
+            {
+              result: matches[:result].to_sym,
+              identity: matches[:identity],
+              selector: matches[:selector],
+              hash_snippet: extract(matches, :hash_snippet)
+            }
+          else
+            {}
+          end
+        end
+
+        def iprev_data(value)
+          matches = value.match(
+            /
+            iprev=(?<result>[\S]+)\s
+            \((?<remote_host_name>[^\)]+)\)\s
+            smtp.remote-ip=(?<remote_ip>[^\s;]+)
+            /x
+          )
+
+          if matches
+            {
+              result: matches[:result].to_sym,
+              remote_host_name: matches[:remote_host_name],
+              remote_ip: @ip_factory.build(matches[:remote_ip]),
+            }
+          else
+            {}
+          end
+        end
+
+        def auth_data(value)
+          matches = value.match(/auth=(?<result>[\S]+)\s.+smtp.auth=(?<domain>[^\s;]+)/)
+
+          if matches
+            {
+              result: matches[:result].to_sym,
+              domain: matches[:domain],
+            }
+          else
+            {}
+          end
+        end
+
+        private
+
+        def extract(matches, key)
+          matches.names.include?(key.to_s) ? matches[key] : nil
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/authentication_headers/parser.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module AuthenticationHeaders
+      class Parser
+        def initialize(authentication_results_parser:, received_spf_parser:)
+          @authentication_results_parser = authentication_results_parser
+          @received_spf_parser = received_spf_parser
+        end
+
+        def parse(headers)
+          {
+            authentication_results: (headers[:authentication_results] || []).map do |header|
+              @authentication_results_parser.parse(header[:data])
+            end,
+            received_spf: (headers[:received_spf] || []).map do |header|
+              @received_spf_parser.parse(header[:data])
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/authentication_headers/received_spf_parser.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module AuthenticationHeaders
+      class ReceivedSpfParser
+        def initialize(ip_factory:)
+          @ip_factory = ip_factory
+        end
+
+        def parse(value)
+          matches = value.match(
+            %r{
+              \A(?<result>\S+)\s
+              \((?<additional_data>[^\)]+)\)
+              (?<attributes>.*)
+            }x
+          )
+
+          {
+            result: matches[:result].downcase.to_sym,
+          }.merge(parse_additional_data(matches[:additional_data])).merge(parse_attributes(matches[:attributes]))
+
+            # {
+            #   result: matches[:result].downcase.to_sym,
+            #   authserv_id: extract(matches, :authserv_id),
+            #   mailfrom: extract(matches, :mailfrom),
+            #   ip: @ip_factory.build(extract(matches, :ip)),
+            #   client_ip: expand_ip(extract(matches, :client_ip)),
+            #   receiver: extract(matches, :receiver),
+            #   helo: expand_ip(extract(matches, :helo)),
+            #   envelope_from: extract(matches, :envelope_from)
+            # }
+        end
+
+        # def parse(value)
+        #   patterns = [
+        #     /\A(?<result>\S+)\s\(domain\sof\s(?<mailfrom>\S+)\sdesignates\s(?<ip>\S+)\sas\spermitted\ssender\)/,
+        #     /\A(?<result>\S+)\s\(domain\sof\s(?<mailfrom>\S+)\sdoes\snot\sdesignate\spermitted\ssender\shosts\)/,
+        #     %r{
+        #       \A(?<result>\S+)\s\(mailfrom\)\s
+        #       identity=(?<identity>[^;]+);\s
+        #       client-ip=(?<client_ip>[^;]+);\s
+        #       helo=(?<helo>[^;]+);\s
+        #       envelope-from=(?<envelope_from>[^;]+);\s
+        #       receiver=(?<receiver>[^;]+)
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         transitioning\sdomain\sof\s(?<mailfrom>\S+)\s
+        #         .+?
+        #         \s(?<ip>\S+)\sas\spermitted\ssender
+        #       \)\s
+        #       client-ip=(?<client_ip>[^;]+);\s
+        #       envelope-from=(?<envelope_from>[^;]+);\s
+        #       helo=\[?(?<helo>[^;]+?)\]?;
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         best\sguess\srecord\sfor\sdomain\sof\s(?<mailfrom>\S+)\s
+        #         .+?
+        #         \s(?<ip>\S+)\sas\spermitted\ssender
+        #       \)\s
+        #       client-ip=(?<client_ip>[^;]+);
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         domain\sof\stransitioning\s(?<mailfrom>\S+)\s
+        #         .+?
+        #         \s(?<ip>\S+)\sas\spermitted\ssender
+        #       \)\s
+        #       client-ip=(?<client_ip>[^;]+);
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         domain\sof\s(?<mailfrom>\S+)\s
+        #         .+?
+        #         \s(?<ip>\S+)\sas\spermitted\ssender
+        #       \)\s
+        #       receiver=(?<receiver>[^;]+);\s
+        #       client-ip=(?<client_ip>[^;]+);\s
+        #       helo=\[?(?<helo>[^;]+?)\]?;
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         domain\sof\s(?<mailfrom>\S+)\s
+        #         .+?
+        #         \s(?<ip>\S+)\sas\spermitted\ssender
+        #       \)\s
+        #       client-ip=(?<client_ip>[^;]+);
+        #     }x,
+        #     %r{
+        #       \A(?<result>\S+)\s
+        #       \(
+        #         (?<authserv_id>[^:]+):\s
+        #         (?<ip>\S+)\sis\sneither\s
+        #         .+?
+        #         domain\sof\s(?<mailfrom>\S+)
+        #       \)\s
+        #       client-ip=(?<client_ip>[^;]+);
+        #     }x
+        #   ]
+        #
+        #   matches = patterns.inject(nil) do |memo, pattern|
+        #     memo || value.match(pattern)
+        #   end
+        #
+        #   if matches
+        #     {
+        #       result: matches[:result].downcase.to_sym,
+        #       authserv_id: extract(matches, :authserv_id),
+        #       mailfrom: extract(matches, :mailfrom),
+        #       ip: @ip_factory.build(extract(matches, :ip)),
+        #       client_ip: expand_ip(extract(matches, :client_ip)),
+        #       receiver: extract(matches, :receiver),
+        #       helo: expand_ip(extract(matches, :helo)),
+        #       envelope_from: extract(matches, :envelope_from)
+        #     }
+        #   end
+        # end
+
+        private
+
+        def parse_additional_data(data)
+          patterns = [
+            %r{
+              (?<authserv_id>[^:]+):\s
+              best\sguess\srecord\sfor\sdomain\sof\s(?<mailfrom>\S+)\s
+              .+?
+              \s(?<ip>\S+)\sas\spermitted\ssender
+            }x,
+            /domain\sof\s(?<mailfrom>\S+)\sdesignates\s(?<ip>\S+)\sas\spermitted\ssender/,
+            /domain\sof\s(?<mailfrom>\S+)\sdoes\snot\sdesignate\spermitted\ssender\shosts/,
+            %r{
+              (?<authserv_id>[^:]+):\s
+              transitioning\sdomain\sof\s(?<mailfrom>\S+)\s
+              .+?
+              \s(?<ip>\S+)\sas\spermitted\ssender
+            }x,
+            %r{
+              (?<authserv_id>[^:]+):\s
+              domain\sof\stransitioning\s(?<mailfrom>\S+)\s
+                .+?
+              \s(?<ip>\S+)\sas\spermitted\ssender
+            }x,
+            %r{
+              (?<authserv_id>[^:]+):\s
+              domain\sof\s(?<mailfrom>\S+)\s
+                .+?
+              \s(?<ip>\S+)\sas\spermitted\ssender
+            }x,
+            %r{
+              (?<authserv_id>[^:]+):\s
+              (?<ip>\S+)\sis\sneither\s
+                .+?
+              domain\sof\s(?<mailfrom>\S+)
+            }x
+          ]
+
+          matches = patterns.inject(nil) do |memo, pattern|
+            memo || data.match(pattern)
+          end
+
+          if matches
+            {
+              authserv_id: extract(matches, :authserv_id),
+              mailfrom: extract(matches, :mailfrom),
+              ip: @ip_factory.build(extract(matches, :ip)),
+              # client_ip: expand_ip(extract(matches, :client_ip)),
+              # receiver: extract(matches, :receiver),
+              # helo: expand_ip(extract(matches, :helo)),
+              # envelope_from: extract(matches, :envelope_from)
+            }
+          else
+            {
+              authserv_id: nil,
+              mailfrom: nil,
+              ip: nil,
+            }
+          end
+        end
+
+        def parse_attributes(attribute_data)
+          output_template = {
+            client_ip: nil, receiver: nil, helo: nil, envelope_from: nil, identity: nil
+          }
+          attribute_data.scan(/[^=]+=\S+/).inject(output_template) do |memo, attr_string|
+            attribute, value = parse_attr_value(attr_string)
+            if ['client_ip', 'helo'].include?(attribute)
+              memo.merge(attribute.to_sym => expand_ip(value))
+            else
+              memo.merge(attribute.to_sym => value)
+            end
+          end
+        end
+
+        def parse_attr_value(attr_value_string)
+          attribute, value = attr_value_string.strip.gsub(/[;\[\]]/, '').split('=')
+          [attribute.gsub(/-/, '_'), value]
+        end
+
+        def extract(matches, key)
+          matches.names.include?(key.to_s) ? matches[key] : nil
+        end
+
+        def expand_ip(ip)
+          ip ? (@ip_factory.build(ip) || ip) : nil
+        end
+      end
+    end
+  end
+end