phisher_phinder 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee817dd4cbc016b47f7710714c4be4c5b07a81baa89b47d46c00ae9112fc04e0
-  data.tar.gz: 7fb874d90fbb94f310ccec195ee6901f77c193f92615cdb5d9c8f5764dfaac94
+  metadata.gz: c8063750bef71fc06792e7c10440f99e979891612540b06abf4c66f2f368022d
+  data.tar.gz: a94cd0e2438359bc626609216a91633deb589121ac92a2567c2a90625c0e30ae
 SHA512:
-  metadata.gz: c4d8660b672488b32ce4cb4ba1e3b7652e414b54afff9d2817a8461d570f7db5aac391424287953a4653ef3b00513dfcffa6666dc740aea823231bc4d6b5f685
-  data.tar.gz: 9e35efd983eb5961fac93c05036bf09e59919ecd1ce6ecd1c338d026e5c3489276f69edd135d2d81e47870daf3422df5ef07f8de570c3d5dfa0f14111c6cdce6
+  metadata.gz: 8cfc5618456ab2db9304232da36e6a5ae3e989edd1ba0993f5bd1f07feb27cfb4d1921052176630e002aa7847f0faee646f5f6690cdd650c34fd64ed843d0188
+  data.tar.gz: bb5ef0c6f6f601a8c7d42ec2a7a8c8de976ccc9a86e438c56cedf0e4853908edb9efb3dfecfae515d300e0f2671cf321c3eaebe0105dad874a52e9be2fff7050

data/.env.example

@@ -1,3 +1,4 @@
-DATABASE_URL=sqlite3://example.sqlite3
+DATABASE_URL=sqlite://example.sqlite3
 MAXMIND_USER_ID=12345
 MAXMIND_LICENCE_KEY=abcdef
+LINE_ENDING_TYPE=dos # or unix

data/.gitignore

@@ -10,9 +10,12 @@
 # rspec failure tracking
 .rspec_status
 
+*.gem
+
 *.sqlite3
 
 *.env
 *.env.test
 
 test_files
+icebox

data/Gemfile

@@ -1,14 +1,3 @@
 source "https://rubygems.org"
 
 gemspec
-
-gem 'dotenv', '~> 2.7.5'
-gem 'maxmind-geoip2', '~> 0.4.0'
-gem 'nokogiri', '~> 1.10.10'
-gem 'sequel', '~> 5.33'
-gem 'sqlite3', '~> 1.4.2'
-
-gem "rake", "~> 12.0"
-gem "rspec", "~> 3.0"
-gem 'database_cleaner-sequel', '1.8.0'
-gem 'webmock', '~> 3.8.3'

data/Gemfile.lock

@@ -1,14 +1,32 @@
 PATH
   remote: .
   specs:
-    phisher_phinder (0.1.0)
+    phisher_phinder (0.2.0)
+      dotenv (~> 2.7.5)
+      maxmind-geoip2 (~> 0.4.0)
+      nokogiri (~> 1.11.0)
+      sequel (~> 5.33)
+      sqlite3 (~> 1.4.2)
+      terminal-table (~> 2.0.0)
+      whois (~> 5.0.1)
+      whois-parser (~> 1.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (6.1.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    bundler-audit (0.7.0.1)
+      bundler (>= 1.2.0, < 3)
+      thor (>= 0.18, < 2)
     coderay (1.1.2)
+    concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -19,8 +37,8 @@ GEM
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    dotenv (2.7.5)
-    ffi (1.13.1)
+    dotenv (2.7.6)
+    ffi (1.14.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
@@ -33,21 +51,26 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.1)
-      ffi-compiler (>= 1.0, < 2.0)
+    http-parser (1.2.2)
+      ffi-compiler
+    i18n (1.8.7)
+      concurrent-ruby (~> 1.0)
     maxmind-db (1.1.1)
     maxmind-geoip2 (0.4.0)
       connection_pool (~> 2.2)
       http (~> 4.3)
       maxmind-db (~> 1.1)
     method_source (1.0.0)
-    mini_portile2 (2.4.0)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    mini_portile2 (2.5.0)
+    minitest (5.14.2)
+    nokogiri (1.11.0)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.5)
+    racc (1.5.2)
     rake (12.3.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -65,28 +88,37 @@ GEM
     safe_yaml (1.0.5)
     sequel (5.33.0)
     sqlite3 (1.4.2)
+    terminal-table (2.0.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    thor (1.0.1)
+    timecop (0.9.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
     webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    whois (5.0.1)
+    whois-parser (1.2.0)
+      activesupport (>= 4)
+      whois (>= 4.0.7)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  bundler-audit (~> 0.7.0.1)
   database_cleaner-sequel (= 1.8.0)
-  dotenv (~> 2.7.5)
-  maxmind-geoip2 (~> 0.4.0)
-  nokogiri (~> 1.10.10)
   phisher_phinder!
   pry
   rake (~> 12.0)
   rspec (~> 3.0)
-  sequel (~> 5.33)
-  sqlite3 (~> 1.4.2)
+  timecop (~> 0.9.2)
   webmock (~> 3.8.3)
 
 BUNDLED WITH

data/README.md

@@ -1,9 +1,22 @@
 # PhisherPhinder
 
-A simple gem to explore phishing emails
+PhisherPhinder is a small utility that extracts data from an email that can be useful when trying to determine the
+source of a phishing email.
+
+As of version 0.2.0 this functionality is very limited. There is a good chance that you will encounter sharp edges - if
+you do, feel free to file a GH issue and I will try to fix this.
+
+## Caution
+
+**When downloading an email for parsing, DO NOT CLICK on any links contained within the email. Rather download the 
+message source as text. As an example, here are the instructions for doing it using
+[GMail](https://www.lifewire.com/how-to-view-the-source-of-a-message-in-gmail-1172105 'Gmail Download Instructions')**
 
 ## Installation
 
+Note: Currently, Windows support is not guaranteed. If you would like to test it on Windows and tell me what does not
+work, I will try my best to address these issues.
+
 Add this line to your application's Gemfile:
 
 ```ruby
@@ -20,9 +33,102 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+At it's most simple - download the email as text to a location on th emahcine running PhihserPhinder
+(`/path/to/content.eml`), then:
+
+```ruby
+phisher_phinder /path/to/content.eml
+```
+
+By default, PhisherPhinder will assume that the file uses the dos-style line endings (`\r\n`), but you can specify the
+line ending type:
+
+```bash
+phisher_phinder -l unix /path/to/content.eml # unix line endings \n
+phisher_phinder -l dos /path/to/content.eml # windows line endings \r\n
+```
+
+PhisherPhinder can lookup IPV4 address data from the MaxMind GeoIP service which requires a Maxmind GeoIP account. 
+Lookup results are cached locally in a database (SQLITE will suffice). The URL for the database needs to be provided as
+an environment variable:
+
+```bash
+DATABASE_URL=sqlite://development.sqlite3 phisher_phinder -a foo -k bar -g /path/to/content.eml
+```
+
+Note: As of version 0.2.0, the GeoIP data is not used for much and the functionality may be removed in future releases.
+
+To see help instructions for the CLI usage:
+
+```bash
+phisher_phinder -h
+```
+
+```
+Usage: phisher_phinder [options] /path/to/email/contents
+    -a, --account_id ACCOUNT_ID      GeoIP account id
+    -k, --license_key LICENSE_KEY    GeoIP license key
+    -g, --geoip                      Enable lookup of GeoIP data for IP addresses (requires `DATABASE_URL` env variable to be defined)
+    -l, --line-ending TYPE           Select line ending type for file
+    -h, --help                       Prints help text
+
+```
+
+## Output
+
+The output of PhisherPhinder will produce something similar to the below:
+
+```
++-------------+---------------------------------------------+
+|                          Origin                           |
++-------------+---------------------------------------------+
+| From        | "Email Security Gateway" <test@test.zzz>    |
+| Message ID  | <ed1770$bbsq7@test.zzz>                     |
+| Return Path | <test@test.zzz>                             |
++-------------+---------------------------------------------+
+
+
++-----------+---------------+------------------+
+|                     SPF                      |
++-----------+---------------+------------------+
+| SPF Pass? | Sender Host   | From Address     |
++-----------+---------------+------------------+
+| Yes       | 10.0.0.1      | test@test.zzz    |
++-----------+---------------+------------------+
+
+
++---------------+------------------------------+------------------------------+--------------------------+
+|                                                 Trace                                                  |
++---------------+------------------------------+------------------------------+--------------------------+
+| Sender IP     | Sender Host                  | Advertised Sender            | Recipient                |
++---------------+------------------------------+------------------------------+--------------------------+
+| 10.0.0.1      | host1.test.zzz               | dodgyname.test.zzz           | mx.google.com            |
+| 10.0.0.2      |                              | othersdodgyname.text.zzz     | host1.test.zzz           |
++---------------+------------------------------+------------------------------+--------------------------+
+
+```
+
+The `Origin` section contains data relating to the email origin.
+
+With regard to the `SPF` and `Trace` sections, they are based on the assumption that the most recent SPF details
+provided in the headers can be trusted as they have been provided by the host of the recipient email and can, hopefully,
+be trusted.
+
+The `Trace` secion shows a subset of the `Received` headers from the original (advertised, but not necessarily actual) 
+origin (the last entry in the table) to the last external server to process the email before the recipient's mail host 
+received the email.
+
+## Dependencies
+1. [Maxmind GeoIP2 User Account](https://dev.maxmind.com/geoip/geoip2/web-services/) - pay as you go
+2. [Database Cleaner](https://github.com/DatabaseCleaner/database_cleaner#safeguards) - whitelist for database urls
 
 ## Development
+.env.example contains dependent environment variables for the .env.test file. You will need to change: DATABASE_URL=sqlite://test.sqlite3.  
+Prior to running the specs, run the 0001 migration to create the geo_ip_cache table.
+
+```
+# bundle exec rake db:migrate
+```
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 

data/exe/phisher_phinder

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'phisher_phinder'
+
+options = {
+  line_ending: "\r\n",
+  geoip_lookup: false,
+  geoip_settings: {
+    account_id: nil,
+    license_key: nil
+  }
+}
+
+OptionParser.new do |opts|
+  opts.banner = 'Usage: phisher_phinder [options] /path/to/email/contents'
+  line_endings = {
+    'windows' => "\r\n",
+    'dos' => "\r\n",
+    'unix' => "\n",
+  }
+
+  def geoip_credentials?(opts)
+    opts[:geoip_settings][:account_id] && opts[:geoip_settings][:license_key]
+  end
+
+  opts.on('-a ACCOUNT_ID', '--account_id ACCOUNT_ID', 'GeoIP account id') do |account_id|
+    options[:geoip_settings][:account_id] = account_id
+  end
+
+  opts.on('-k LICENSE_KEY', '--license_key LICENSE_KEY', 'GeoIP license key') do |license_key|
+    options[:geoip_settings][:license_key] = license_key
+  end
+
+  opts.on(
+    '-g',
+    '--geoip',
+    'Enable lookup of GeoIP data for IP addresses (requires `DATABASE_URL` env variable to be defined)'
+  ) do |geoip|
+    raise 'Please provide the GeoIP account id and license key' unless geoip_credentials?(options)
+    raise 'Please set the DATABASE_URL ENV variable' unless ENV['DATABASE_URL']
+    options[:geoip_lookup] = geoip
+  end
+
+  opts.on('-l TYPE', '--line-ending TYPE', line_endings, 'Select line ending type for file') do |ending|
+    options[:line_ending] = ending
+  end
+
+  opts.on('-h', '--help', 'Prints help text') do
+    puts opts
+    exit
+  end
+end.parse!
+
+file_contents = IO.read(ARGV.last)
+
+command = PhisherPhinder::Command.new
+
+PhisherPhinder::Display.new.display_report(
+  command.report(file_contents, **options)
+)

data/lib/phisher_phinder.rb

@@ -2,11 +2,13 @@ require "phisher_phinder/version"
 
 require 'maxmind/geoip2'
 
-require 'sequel'
-Sequel::Model.plugin :timestamps, update_on_create: true
+require_relative './phisher_phinder/command'
+require_relative './phisher_phinder/display'
 
 require_relative './phisher_phinder/body_hyperlink'
 require_relative './phisher_phinder/cached_geoip_client'
+require_relative './phisher_phinder/null_lookup_client'
+require_relative './phisher_phinder/null_response'
 require_relative './phisher_phinder/geoip_ip_data'
 require_relative './phisher_phinder/expanded_data_processor'
 require_relative './phisher_phinder/extended_ip'
@@ -14,6 +16,11 @@ require_relative './phisher_phinder/extended_ip_factory'
 require_relative './phisher_phinder/mail_parser'
 require_relative './phisher_phinder/mail'
 require_relative './phisher_phinder/simple_ip'
+require_relative './phisher_phinder/mail_parser/authentication_headers/parser'
+require_relative './phisher_phinder/mail_parser/authentication_headers/auth_results_parser'
+require_relative './phisher_phinder/mail_parser/authentication_headers/received_spf_parser'
+require_relative './phisher_phinder/mail_parser/body/block_classifier'
+require_relative './phisher_phinder/mail_parser/body/block_parser'
 require_relative './phisher_phinder/mail_parser/received_headers/parser'
 require_relative './phisher_phinder/mail_parser/received_headers/by_parser'
 require_relative './phisher_phinder/mail_parser/received_headers/classifier'
@@ -21,6 +28,8 @@ require_relative './phisher_phinder/mail_parser/received_headers/for_parser'
 require_relative './phisher_phinder/mail_parser/received_headers/from_parser'
 require_relative './phisher_phinder/mail_parser/received_headers/starttls_parser'
 require_relative './phisher_phinder/mail_parser/received_headers/timestamp_parser'
+require_relative './phisher_phinder/sender_extractor'
+require_relative './phisher_phinder/tracing_report'
 
 module PhisherPhinder
   class Error < StandardError; end

data/lib/phisher_phinder/command.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class Command
+    def report(contents, line_ending:, geoip_lookup:, geoip_settings: {})
+      lookup_client = if geoip_lookup
+                        PhisherPhinder::CachedGeoipClient.new(
+                          MaxMind::GeoIP2::Client.new(**geoip_settings),
+                          Time.now - 86400
+                        )
+                      else
+                        PhisherPhinder::NullLookupClient.new
+                      end
+      ip_factory = PhisherPhinder::ExtendedIpFactory.new(geoip_client: lookup_client)
+      mail_parser = PhisherPhinder::MailParser::Parser.new(ip_factory, line_ending)
+      tracing_report = PhisherPhinder::TracingReport.new(mail_parser.parse(contents))
+      tracing_report.report
+    end
+  end
+end

data/lib/phisher_phinder/display.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+require 'terminal-table'
+
+module PhisherPhinder
+  class Display
+    def display_report(input_data)
+      origin_table = Terminal::Table.new(
+        title: 'Origin',
+        rows: format_origin_data(input_data)
+      )
+
+      puts origin_table
+
+      puts "\n\n"
+
+      spf_table = Terminal::Table.new(
+        headings: ['SPF Pass?', 'Sender Host', 'From Address'],
+        title: 'SPF',
+        rows: [
+          [
+            input_data[:authentication][:spf][:success] ? 'Yes' : 'No',
+            input_data[:authentication][:spf][:ip],
+            input_data[:authentication][:spf][:from_address]
+          ]
+        ]
+      )
+
+      puts spf_table
+
+      puts "\n\n"
+
+      data = input_data[:tracing].map do |entry|
+        [
+          entry[:sender][:ip],
+          entry[:sender][:host],
+          entry[:advertised_sender] || entry[:helo],
+          entry[:recipient]
+        ]
+      end
+
+      trace_table = Terminal::Table.new(
+        headings: ['Sender IP', 'Sender Host', 'Advertised Sender', 'Recipient'],
+        title: 'Trace',
+        rows: data
+      )
+
+      puts trace_table
+    end
+
+    private
+
+    def format_origin_data(input_data)
+      types = [
+        ['From', :from],
+        ['Message ID', :message_id],
+        ['Return Path', :return_path],
+      ]
+
+      types.inject([]) do |output, (description, type)|
+        output << [description, input_data[:origin][type].join(', ')]
+      end
+    end
+  end
+end