phisher_phinder 0.1.0 → 0.2.0

data/lib/phisher_phinder/mail_parser/received_headers/parser.rb

@@ -4,12 +4,11 @@ module PhisherPhinder
   module MailParser
     module ReceivedHeaders
       class Parser
-        def initialize(by_parser:, for_parser:, from_parser:, starttls_parser:, timestamp_parser:, classifier:)
+        def initialize(by_parser:, for_parser:, from_parser:, timestamp_parser:, classifier:)
           @parsers = {
             by: by_parser,
             for: for_parser,
             from: from_parser,
-            starttls: starttls_parser,
             time: timestamp_parser,
           }
           @classifier = classifier
@@ -24,7 +23,13 @@ module PhisherPhinder
           components = extract_value_components(header_value).merge(time: timestamp)
 
           output = @parsers.inject({}) do |memo, (component_name, parser)|
-            memo.merge(parser.parse(components[component_name]))
+            memo.merge(parser.parse(components[component_name])) do |key, old_value, new_value|
+              if key == :starttls && old_value
+                old_value
+              else
+                new_value
+              end
+            end
           end
 
           output.merge(@classifier.classify(output))
@@ -33,40 +38,56 @@ module PhisherPhinder
         private
 
         def extract_value_components(header_value)
-          require 'strscan'
 
-          scanner = StringScanner.new(header_value)
+          values = {
+            time: [],
+            for: [],
+            by: [],
+            from: []
+          }
+
+          cleaned_value = header_value.gsub(/\A\(([^\)]+)\)/, '\1')
 
-          output = {}
+          tokenised_value = cleaned_value.split(" ")
+          current_component = nil
 
-          if scanner.check(/\(from\s+[^)]+\)/)
-            from_part = scanner.scan(/\(from\s+[^)]+\)/)
-          elsif scanner.check(/from\s.+?\(HELO\s[^)]+\)\s\(\[[^\]]+\]\)/)
-            from_part = scanner.scan(/from.+?(?=by)/)
-          elsif scanner.check(/from\s[^)(]+\sby/)
-            from_part = scanner.scan(/from.+?(?=by)/)
-          else
-            from_part = scanner.scan(/from\s.+?\([^)]+?\)/)
+          tokenised_value.each do |val|
+            current_component = component_start(val, values) || current_component
+
+            values[current_component] << val if current_component
           end
 
-          if scanner.check(/.+\S+\s+by/)
-            starttls_part = scanner.scan(/.+\s(?=by)/)
-            by_part = scanner.scan(/\s?by.+?\sid\s[\S]+\s?/)
-            for_part = scanner.scan(/for\s+\S+/)
-          elsif scanner.check(/\s?by.+?\s(id|ID)\s[\S]+\s?/)
-            by_part = scanner.scan(/\s?by.+?\s(id|ID)\s[\S]+\s?/) unless scanner.eos?
-            for_part = scanner.scan(/for\s+\S+/) unless scanner.eos?
-            starttls_part = scanner.rest unless scanner.eos?
-          elsif scanner.check(/by.+(?!\sid)/)
-            by_part = scanner.scan(/.+/)
+          values.inject({}) do |memo, (component, values)|
+            memo.merge(component => (values.any? ? values.join(' ') : nil))
           end
+        end
 
-          {
-            by: by_part,
-            for: for_part,
-            from: from_part,
-            starttls: starttls_part
+        def for_scan(scanner)
+          return nil if scanner.eos?
+
+          scanner.check(/for\s<[^>]+>/) ? scanner.scan(/for\s<[^>]+>/) : scanner.scan(/for\s+\S+/)
+        end
+
+        def component_start(token, values)
+          markers = {
+            'from' => :from,
+            'by' => :by,
+            'for' => :for
+          }
+
+          return nil unless component = markers[token]
+
+          blocking_components = {
+            from: [:by, :for],
+            by: [:for],
+            for: []
           }
+
+          if blocking_components[component].any? { |blocking_comp| values[blocking_comp].any? }
+            nil
+          else
+            component
+          end
         end
       end
     end

data/lib/phisher_phinder/mail_parser/received_headers/starttls_parser.rb

@@ -9,6 +9,7 @@ module PhisherPhinder
 
           patterns = [
             /\(version=(?<version>\S+)\scipher=(?<cipher>\S+)\sbits=(?<bits>\S+)\)/,
+            /\(version=(?<version>\S+),\scipher=(?<cipher>\S+)\)/,
             /using\s(?<version>\S+)\swith cipher\s(?<cipher>\S+)\s\((?<bits>.+?) bits\)/
           ]
 
@@ -16,7 +17,13 @@ module PhisherPhinder
             memo || component.match(pattern)
           end
 
-          {starttls: {version: matches[:version], cipher: matches[:cipher], bits: matches[:bits]}}
+          {
+            starttls: {
+              version: matches[:version],
+              cipher: matches[:cipher],
+              bits: matches.names.include?('bits') ? matches[:bits] : nil,
+            }
+          }
         end
       end
     end

data/lib/phisher_phinder/null_lookup_client.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class NullLookupClient
+    def lookup(_ip_address)
+      NullResponse.new
+    end
+  end
+end

data/lib/phisher_phinder/null_response.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class NullResponse
+    def method_missing(*_args)
+    end
+
+    def ==(other)
+      other.instance_of? self.class
+    end
+  end
+end

data/lib/phisher_phinder/sender_extractor.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class SenderExtractor
+    def extract(mail)
+      auth_senders = {
+        hosts: [],
+        email_addresses: []
+      }
+
+      processed_authservs = []
+
+      authentication_results = mail.authentication_headers[:authentication_results]
+
+      if authentication_results.any?
+        trusted_auth_header = authentication_results.first
+        untrusted_auth_headers = authentication_results[1..-1]
+
+        auth_senders[:hosts] << {
+          entry_type: :ip,
+          host: trusted_auth_header[:spf].first[:ip],
+          spf: {present: true, trusted: true}
+        }
+        auth_senders[:email_addresses] << {
+          email_address: trusted_auth_header[:spf].first[:from],
+          spf: {present: true, trusted: true, result: trusted_auth_header[:spf].first[:result]},
+        }
+
+        processed_authservs << trusted_auth_header[:authserv_id]
+
+        untrusted_auth_headers.each do |header|
+          next if processed_authservs.include? header[:authserv_id]
+
+          auth_senders[:hosts] << {entry_type: :ip, host: header[:spf].first[:ip], spf: {present: true, trusted: false}}
+          unless auth_senders[:email_addresses].find { |entry| entry[:email_address] == header[:spf].first[:from] }
+            auth_senders[:email_addresses] << {
+              email_address: header[:spf].first[:from],
+              spf: {present: true, trusted: false, result: header[:spf].first[:result]},
+            }
+          end
+        end
+      end
+
+      tracing_senders = []
+
+      mail.tracing_headers[:received].each do |header|
+        if tracing_senders.empty?
+          if header[:sender] && header[:sender][:ip] == trusted_auth_sender_ip(auth_senders)
+            tracing_senders << header[:sender]
+          end
+
+          next
+        end
+
+        if header[:sender] && header[:recipient] == tracing_senders.last[:host]
+          tracing_senders << header[:sender]
+        else
+          break
+        end
+      end
+
+      {
+        authentication_senders: auth_senders,
+        tracing_senders: tracing_senders
+      }
+    end
+
+    private
+
+    def trusted_auth_sender_ip(authentication_senders)
+      (authentication_senders[:hosts].find { |e| e[:spf][:trusted] })[:host]
+    end
+  end
+end

data/lib/phisher_phinder/simple_ip.rb

@@ -11,5 +11,9 @@ module PhisherPhinder
     def ==(other)
       other.is_a?(SimpleIp) && ip_address == other.ip_address
     end
+
+    def to_s
+      @ip_address.to_s
+    end
   end
 end

data/lib/phisher_phinder/tracing_report.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class TracingReport
+    def initialize(mail)
+      @mail = mail
+    end
+
+    def report
+      {
+        authentication: {
+          mechanisms: [:spf],
+          spf: {
+            success: latest_spf_entry[:result] == :pass,
+            ip: latest_spf_entry[:ip],
+            from_address: latest_spf_entry[:mailfrom],
+            client_ip: latest_spf_entry[:client_ip],
+          }
+        },
+        origin: extract_origin_headers(@mail.headers),
+        tracing: extract_tracing_headers(@mail.tracing_headers, latest_spf_entry)
+      }
+    end
+
+    private
+
+    def latest_spf_entry
+      @mail.authentication_headers[:received_spf].first
+    end
+
+    def ip_address(spf_entry)
+      spf_entry[:ip] || spf_entry[:client_ip]
+    end
+
+    def extract_tracing_headers(received_headers, latest_spf_entry)
+      start = received_headers[:received].find_index { |h| h[:sender][:ip] == ip_address(latest_spf_entry) }
+      received_headers[:received][start..-1]
+    end
+
+    def extract_origin_headers(headers)
+      [:from, :return_path, :message_id].inject({}) do |output, header_type|
+        entries = headers[header_type] || []
+        output.merge(header_type => entries.map { |h| h[:data] })
+      end
+    end
+  end
+end

data/lib/phisher_phinder/version.rb

@@ -1,3 +1,3 @@
 module PhisherPhinder
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/phisher_phinder.gemspec

@@ -27,6 +27,20 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "dotenv", "~> 2.7.5"
+  spec.add_dependency "maxmind-geoip2", "~> 0.4.0"
+  spec.add_dependency "nokogiri", "~> 1.11.0"
+  spec.add_dependency "sequel", "~> 5.33"
+  spec.add_dependency "sqlite3", "~> 1.4.2"
+  spec.add_dependency "terminal-table", "~> 2.0.0"
+  spec.add_dependency "whois", "~> 5.0.1"
+  spec.add_dependency "whois-parser", "~> 1.2.0"
+
+  spec.add_development_dependency "bundler-audit", "~> 0.7.0.1"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "timecop", "~> 0.9.2"
+  spec.add_development_dependency 'database_cleaner-sequel', '1.8.0'
+  spec.add_development_dependency 'webmock', '~> 3.8.3'
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rspec", "3.9.0"
 end

metadata

@@ -1,47 +1,230 @@
 --- !ruby/object:Gem::Specification
 name: phisher_phinder
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Rory McKinley
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2021-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.7.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.5
+- !ruby/object:Gem::Dependency
+  name: maxmind-geoip2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.33'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.33'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: whois
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
+- !ruby/object:Gem::Dependency
+  name: whois-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.0.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: database_cleaner-sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 1.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.8.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A collection of tools to dissect and report on phishing emails
 email:
 - rorymckinley@gmail.com
-executables: []
+executables:
+- phisher_phinder
 extensions: []
 extra_rdoc_files: []
 files:
@@ -61,15 +244,23 @@ files:
 - bin/console
 - bin/setup
 - db/migrations/0001_create_geo_ip_cache.rb
+- exe/phisher_phinder
 - lib/phisher_phinder.rb
 - lib/phisher_phinder/body_hyperlink.rb
 - lib/phisher_phinder/cached_geoip_client.rb
+- lib/phisher_phinder/command.rb
+- lib/phisher_phinder/display.rb
 - lib/phisher_phinder/expanded_data_processor.rb
 - lib/phisher_phinder/extended_ip.rb
 - lib/phisher_phinder/extended_ip_factory.rb
 - lib/phisher_phinder/geoip_ip_data.rb
 - lib/phisher_phinder/mail.rb
 - lib/phisher_phinder/mail_parser.rb
+- lib/phisher_phinder/mail_parser/authentication_headers/auth_results_parser.rb
+- lib/phisher_phinder/mail_parser/authentication_headers/parser.rb
+- lib/phisher_phinder/mail_parser/authentication_headers/received_spf_parser.rb
+- lib/phisher_phinder/mail_parser/body/block_classifier.rb
+- lib/phisher_phinder/mail_parser/body/block_parser.rb
 - lib/phisher_phinder/mail_parser/body_parser.rb
 - lib/phisher_phinder/mail_parser/header_value_parser.rb
 - lib/phisher_phinder/mail_parser/received_headers/by_parser.rb
@@ -79,7 +270,11 @@ files:
 - lib/phisher_phinder/mail_parser/received_headers/parser.rb
 - lib/phisher_phinder/mail_parser/received_headers/starttls_parser.rb
 - lib/phisher_phinder/mail_parser/received_headers/timestamp_parser.rb
+- lib/phisher_phinder/null_lookup_client.rb
+- lib/phisher_phinder/null_response.rb
+- lib/phisher_phinder/sender_extractor.rb
 - lib/phisher_phinder/simple_ip.rb
+- lib/phisher_phinder/tracing_report.rb
 - lib/phisher_phinder/version.rb
 - phisher_phinder.gemspec
 homepage: https://capefox.co
@@ -90,7 +285,7 @@ metadata:
   homepage_uri: https://capefox.co
   source_code_uri: https://github.com/rorymckinley/phisher_phinder
   changelog_uri: https://github.com/rorymckinley/phisher_phinder/blob/master/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -106,7 +301,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A gem for dissecting phishing emails
 test_files: []