phisher_phinder 0.1.0 → 0.2.0

data/lib/phisher_phinder/mail_parser/body/block_classifier.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module Body
+      class BlockClassifier
+        def initialize(line_end)
+          @line_end = line_end
+        end
+
+        def classify_block(contents)
+          lines = contents.split(@line_end)
+          processing_block_headers = true
+
+          output = {
+            content_type: :text,
+            character_set: :utf_8,
+            content_transfer_encoding: nil
+          }
+
+          while processing_block_headers && lines.any? do
+            line = lines.shift&.strip
+            if line && line.empty?
+              processing_block_headers = false
+            elsif line && line =~ /\AContent-Type:/
+              output.merge!(extract_content_type(line))
+
+              output.merge!(extract_character_set(line))
+            elsif line && line =~ /\AContent-Transfer-Encoding/
+              output.merge!(extract_encoding(line))
+            end
+          end
+
+          output[:content] = lines.join(@line_end)
+
+          output
+        end
+
+        def classify_headers(headers)
+          output = {
+            content_type: :text,
+            character_set: :utf_8,
+            content_transfer_encoding: nil
+          }
+
+          output.merge!(extract_content_type(headers[:content_type]))
+
+          output.merge!(extract_character_set(headers[:content_type]))
+
+          output.merge!(extract_encoding(headers[:content_transfer_encoding]))
+
+          output
+        end
+
+        private
+
+        def extract_content_type(content_type_string)
+          if content_type_string
+            if content_type_string.include?('text/plain')
+              {content_type: :text}
+            elsif content_type_string.include?('text/html')
+              {content_type: :html}
+            else
+              {}
+            end
+          else
+            {}
+          end
+        end
+
+        def extract_character_set(content_type_string)
+          if content_type_string
+            charset_matches = content_type_string.match(/charset="?(?<charset>.+?)"?\z/)
+            if charset_matches
+              if charset_matches[:charset].downcase == 'utf-8'
+                {character_set: :utf_8}
+              elsif charset_matches[:charset].downcase == 'windows-1251'
+                {character_set: :windows_1251}
+              elsif charset_matches[:charset].downcase == 'iso-8859-1'
+                {character_set: :iso_8859_1}
+              else
+                {}
+              end
+            else
+              {}
+            end
+          else
+            {}
+          end
+        end
+
+        def extract_encoding(encoding_string)
+          if encoding_string&.include? 'base64'
+            {content_transfer_encoding: :base64}
+          elsif encoding_string&.include? 'quoted-printable'
+            {content_transfer_encoding: :quoted_printable}
+          elsif encoding_string&.include? '7bit'
+            {content_transfer_encoding: :seven_bit}
+          else
+            {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/body/block_parser.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  module MailParser
+    module Body
+      class BlockParser
+        def initialize(line_end)
+          @line_end = line_end
+        end
+
+        def parse(block_data)
+          encoding = block_data[:content_transfer_encoding] || :seven_bit
+
+          case encoding
+          when :seven_bit
+            block_data[:content]
+          when :base64
+            decoded = Base64.decode64(block_data[:content])
+            if block_data[:character_set] == :utf_8
+              decoded.force_encoding('UTF-8')
+            elsif block_data[:character_set] == :windows_1251
+              decoded.force_encoding('cp1251').encode('UTF-8')
+            end
+          when :quoted_printable
+            remove_troublesome_sequences(block_data[:content]).unpack('M').first.force_encoding('UTF-8')
+          end
+        end
+
+        private
+
+        def remove_troublesome_sequences(content)
+          content.gsub(/=((?:[^a-f0-9#{@line_end}])|(?:[a-f0-9][^a-f0-9]))/i, '=3D\1')
+        end
+      end
+    end
+  end
+end

data/lib/phisher_phinder/mail_parser/body_parser.rb

@@ -10,16 +10,27 @@ module PhisherPhinder
       def parse(body_contents:, content_type:, content_transfer_encoding:)
         if multipart_alternative?(content_type)
           parse_multipart_alternative(content_type, body_contents)
-        elsif html?(content_type)
-          {
-            text: nil,
-            html: decode_body(body_contents, content_transfer_encoding)
-          }
         else
-          {
-            text: body_contents,
-            html: nil
-          }
+          classifier = Body::BlockClassifier.new(@line_end)
+          parser = Body::BlockParser.new(@line_end)
+
+          classification = classifier.classify_headers(
+            content_type: content_type, content_transfer_encoding: content_transfer_encoding
+          ).merge(content: body_contents)
+
+          contents = parser.parse(classification)
+
+          if classification[:content_type] == :html
+            {
+              html: contents,
+              text: nil
+            }
+          else
+            {
+              html: nil,
+              text: contents
+            }
+          end
         end
       end
 
@@ -40,7 +51,7 @@ module PhisherPhinder
       end
 
       def parse_multipart_alternative(content_type, contents)
-        base_boundary = content_type.split(';').last.strip.split('=').last
+        base_boundary = content_type.split(';').last.strip.gsub(/boundary=/, '').gsub(/"/, '')
         start_boundary = '--' + base_boundary + @line_end
         end_boundary = '--' + base_boundary + '--'
 
@@ -61,30 +72,14 @@ module PhisherPhinder
       end
 
       def categorise_blocks(blocks)
+        classifier = Body::BlockClassifier.new(@line_end)
+        parser = Body::BlockParser.new(@line_end)
         blocks.map do |block|
-          lines = block.split(@line_end)
-          processing_block_headers = true
-          html = false
-          base64_encoded = false
-
-          while processing_block_headers do
-            line = lines.shift.strip
-            if line.empty?
-              processing_block_headers = false
-            elsif line =~/\AContent-Type: text\/html/
-              html = true
-            elsif line =~ /\AContent-Transfer-Encoding: base64/
-              base64_encoded = true
-            end
-          end
+          classification = classifier.classify_block(block)
+          contents = parser.parse(classification)
 
-          contents = if base64_encoded
-                       (lines.map { |l| Base64.decode64(l) }).join
-                     else
-                       lines.join(@line_end)
-                     end
           {
-            html: html,
+            html: classification[:content_type] == :html,
             contents: contents
           }
         end

data/lib/phisher_phinder/mail_parser/header_value_parser.rb

@@ -4,20 +4,35 @@ module PhisherPhinder
   module MailParser
     class HeaderValueParser
       def parse(raw_value)
-        utf_8_preambles = raw_value.scan(/=\?UTF-8\?b\?/)
-        if raw_value.scan(/=\?UTF-8\?b\?/).any?
-          (raw_value.split(' ').map { |snippet| parse_utf8_base64(snippet) }).join
-        else
-          raw_value.strip
-        end
+        stripped_value = raw_value.strip
+        words = stripped_value.split(' ')
+        words.map do |word|
+          if encoded?(word)
+            matches = word.match(/\A=\?(?<character_set>.+)\?(?<encoding>.)\?(?<content>.+)\z/)
+
+            unencoded_content = if matches[:encoding].downcase == 'b'
+                                  Base64.decode64(matches[:content])
+                                elsif matches[:encoding].downcase == 'q'
+                                  matches[:content].unpack('M').first
+                                end
+
+            content = if matches[:character_set] =~ /iso-8859-1/i
+                        unencoded_content.force_encoding('ISO-8859-1').encode('UTF-8')
+                      elsif matches[:character_set] =~ /windows-1251/i
+                        unencoded_content.force_encoding('cp1251').encode('UTF-8')
+                      elsif matches[:character_set] =~ /utf-8/i
+                        unencoded_content.force_encoding('UTF-8')
+                      end
+          else
+            word
+          end
+        end.join(' ')
       end
 
       private
 
-      def parse_utf8_base64(raw_value)
-        require 'base64'
-
-        Base64.decode64(raw_value.strip.sub(/=\?UTF-8\?b\?/, '')).force_encoding('UTF-8')
+      def encoded?(raw_value)
+        raw_value =~ /=\?[a-z1-9-]+\?[bq]/i
       end
     end
   end

data/lib/phisher_phinder/mail_parser/received_headers/by_parser.rb

@@ -4,14 +4,34 @@ module PhisherPhinder
   module MailParser
     module ReceivedHeaders
       class ByParser
-        def initialize(extended_ip_factory)
-          @extended_ip_factory = extended_ip_factory
+        def initialize(ip_factory:, starttls_parser:)
+          @extended_ip_factory = ip_factory
+          @starttls_parser = starttls_parser
         end
 
         def parse(component)
-          return {recipient: nil, protocol: nil, id: nil, recipient_additional: nil} unless component
+          unless component
+            return {
+              recipient: nil,
+              protocol: nil,
+              id: nil,
+              recipient_additional: nil,
+              authenticated_as: nil
+            }.merge(@starttls_parser.parse(nil))
+          end
 
           patterns = [
+            %r{by\s(?<recipient>\S+)\s
+             \((?<additional>[^)]+)\)\s
+             with\sMicrosoft\sSMTP\sServer\s(?<starttls>\([^\)]+\))\s
+             id\s(?<id>\S+)\s
+             via\s(?<protocol>Frontend\sTransport)
+            }x,
+            %r{by\s(?<recipient>\S+)\s
+             \((?<additional>[^)]+)\)\s
+             with\sMicrosoft\sSMTP\sServer\s(?<starttls>\([^\)]+\))\s
+             id\s(?<id>\S+)
+            }x,
             /by\s(?<recipient>\S+)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
             /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
             /by\s(?<recipient>\S+)\s(?<additional>.+)\swith\s(?<protocol>\S+)\sid\s(?<id>\S+)/,
@@ -20,6 +40,9 @@ module PhisherPhinder
             /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\swith\s(?<protocol>\S+)\sID\s(?<id>\S+)/,
             /by\s(?<recipient>\S+)\swith\s(?<protocol>.+)\sid\s(?<id>\S+)/,
             /by\s(?<recipient>\S+)\swith\s(?<protocol>.+)/,
+            /by\s(?<recipient>\S+)\s\((?<additional>[^)]+)\)\s\(authenticated as (?<authenticated_as>[^\)]+)\)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)\sid\s(?<id>\S+)/,
+            /by\s(?<recipient>\S+)/
           ]
 
           matches = patterns.inject(nil) do |memo, pattern|
@@ -30,8 +53,15 @@ module PhisherPhinder
             recipient: enrich_recipient(matches[:recipient]),
             protocol: matches.names.include?('protocol') ? matches[:protocol]: nil,
             id: matches.names.include?('id') ? matches[:id]: nil,
-            recipient_additional: matches.names.include?('additional') ? matches[:additional] : nil
-          }
+            recipient_additional: matches.names.include?('additional') ? matches[:additional] : nil,
+            authenticated_as: matches.names.include?('authenticated_as') ? matches[:authenticated_as] : nil,
+          }.merge(
+            if matches.names.include?('starttls')
+              @starttls_parser.parse(matches[:starttls])
+            else
+              @starttls_parser.parse(nil)
+            end
+          )
         end
 
         private

data/lib/phisher_phinder/mail_parser/received_headers/for_parser.rb

@@ -4,18 +4,38 @@ module PhisherPhinder
   module MailParser
     module ReceivedHeaders
       class ForParser
+        def initialize(starttls_parser:)
+          @starttls_parser = starttls_parser
+        end
+
         def parse(component)
-          component =~ /\Afor\s(\S+)\z/
+          return {recipient_mailbox: nil}.merge(@starttls_parser.parse(nil)) unless component
+
+          patterns = [
+            /\Afor\s(?<recipient_mailbox>\S+)\s\(Google Transport Security\)\z/,
+            /\Afor\s(?<recipient_mailbox>\S+)\s(?<starttls>\([^\)]+\))\z/,
+            /\Afor\s(?<recipient_mailbox>.+)\z/,
+          ]
+
+          matches = patterns.inject(nil) do |memo, pattern|
+            memo || component.match(pattern)
+          end
 
-          {
-            recipient_mailbox: strip_angle_brackets($1)
-          }
+          output = {
+            recipient_mailbox: strip_angle_brackets(matches[:recipient_mailbox]),
+          }.merge(
+            if matches.names.include?('starttls')
+              @starttls_parser.parse(matches[:starttls])
+            else
+              @starttls_parser.parse(nil)
+            end
+          )
         end
 
         private
 
         def strip_angle_brackets(email_address_string)
-          email_address_string =~ /\<([^>]+)\>/ ? $1 : email_address_string
+          email_address_string =~ /\<\s?([^>]+?)\s?\>/ ? $1 : email_address_string
         end
       end
     end

data/lib/phisher_phinder/mail_parser/received_headers/from_parser.rb

@@ -4,14 +4,41 @@ module PhisherPhinder
   module MailParser
     module ReceivedHeaders
       class FromParser
-        def initialize(extended_ip_factory)
-          @extended_ip_factory = extended_ip_factory
+        def initialize(ip_factory:, starttls_parser:)
+          @extended_ip_factory = ip_factory
+          @starttls_parser = starttls_parser
         end
 
         def parse(component)
-          return {advertised_sender: nil, helo: nil, sender: nil} unless component
+          unless component
+            return {
+              advertised_authenticated_sender: nil,
+              advertised_sender: nil,
+              helo: nil,
+              sender: {
+                host: nil,
+                ip: nil
+              },
+            }.merge(@starttls_parser.parse(nil))
+          end
 
           patterns = [
+            %r{
+              from\s\[(?<advertised_sender>[\S]+)\]\s
+              \((?<sender_host>\S+?)\.?\s
+              \[(?<sender_ip>[^\]]+)\]\)\s
+              \(Authenticated\ssender:\s(?<advertised_authenticated_sender>[^\)]+)\)
+            }x,
+            /from\s\[(?<sender_ip>[^\]]+)\]\s\(helo=(?<helo>[^\)]+)\)/,
+            %r{
+              from\s\[(?<advertised_sender>[\S]+)\]\s
+              \((?<sender_host>\S+?)\.?\s
+              \[(?<sender_ip>[^\]]+)\]\)
+            }x,
+            /from\s(?<sender_ip>[^\]]+)\s\(EHLO\s(?<helo>[^\)]+)\)/,
+            /from\s(?<advertised_sender>[\S]+)\s\((?<sender_host>\S+?)\.?\s\[(?<sender_ip>[^\]]+)\]\) \((?<starttls>[^\)]+\))/,
+            /from\s(?<advertised_sender>[\S]+)\s\((?<sender_host>\S+?)\.?\s\[(?<sender_ip>[^\]]+)\]\) \((?<starttls>[^\)]+\))/,
+            /from\s(?<advertised_sender>[\S]+)\s\(HELO\s(?<helo>[^)]+)\)\s\(\)/,
             /from\s(?<advertised_sender>[\S]+)\s\(HELO\s(?<helo>[^)]+)\)\s\(\[(?<sender_ip>[^\]]+)\]\)/,
             /from\s(?<advertised_sender>[\S]+)\s\((?<sender_host>\S+?)\.?\s\[(?<sender_ip>[^\]]+)\]\)/,
             /from\s(?<advertised_sender>\S+)\s\((?<sender_host>\S+?)\.?\s(?<sender_ip>\S+?)\)/,
@@ -25,14 +52,31 @@ module PhisherPhinder
             memo || component.match(pattern)
           end
 
-          {
-            advertised_sender: matches[:advertised_sender],
+          output = {
+            advertised_sender: expand_advertised_sender(extract(matches, :advertised_sender)),
             helo: matches.names.include?('helo') ? matches[:helo] : nil,
             sender: {
               host: matches.names.include?('sender_host') ? matches[:sender_host] : nil,
               ip: matches.names.include?('sender_ip') ? @extended_ip_factory.build(matches[:sender_ip]) : nil
-            }
+            },
+            advertised_authenticated_sender: matches.names.include?('advertised_authenticated_sender') ? matches[:advertised_authenticated_sender] : nil
           }
+
+          if matches.names.include?('starttls')
+            output.merge(@starttls_parser.parse(matches[:starttls]))
+          else
+            output.merge(@starttls_parser.parse(nil))
+          end
+        end
+
+        private
+
+        def extract(matches, key)
+          matches.names.include?(key.to_s) ? matches[key] : nil
+        end
+
+        def expand_advertised_sender(sender)
+          sender ? (@extended_ip_factory.build(sender) || sender) : nil
         end
       end
     end