phisher_phinder 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ee817dd4cbc016b47f7710714c4be4c5b07a81baa89b47d46c00ae9112fc04e0
+  data.tar.gz: 7fb874d90fbb94f310ccec195ee6901f77c193f92615cdb5d9c8f5764dfaac94
+SHA512:
+  metadata.gz: c4d8660b672488b32ce4cb4ba1e3b7652e414b54afff9d2817a8461d570f7db5aac391424287953a4653ef3b00513dfcffa6666dc740aea823231bc4d6b5f685
+  data.tar.gz: 9e35efd983eb5961fac93c05036bf09e59919ecd1ce6ecd1c338d026e5c3489276f69edd135d2d81e47870daf3422df5ef07f8de570c3d5dfa0f14111c6cdce6

data/.env.example

@@ -0,0 +1,3 @@
+DATABASE_URL=sqlite3://example.sqlite3
+MAXMIND_USER_ID=12345
+MAXMIND_LICENCE_KEY=abcdef

data/.gitignore

@@ -0,0 +1,18 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+*.sqlite3
+
+*.env
+*.env.test
+
+test_files

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+overphishing

data/.ruby-version

@@ -0,0 +1 @@
+2.7.1

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.1
+before_install: gem install bundler -v 2.1.4

data/CHANGELOG.md

@@ -0,0 +1 @@
+* CHANGELOG

data/Gemfile

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem 'dotenv', '~> 2.7.5'
+gem 'maxmind-geoip2', '~> 0.4.0'
+gem 'nokogiri', '~> 1.10.10'
+gem 'sequel', '~> 5.33'
+gem 'sqlite3', '~> 1.4.2'
+
+gem "rake", "~> 12.0"
+gem "rspec", "~> 3.0"
+gem 'database_cleaner-sequel', '1.8.0'
+gem 'webmock', '~> 3.8.3'

data/Gemfile.lock

@@ -0,0 +1,93 @@
+PATH
+  remote: .
+  specs:
+    phisher_phinder (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coderay (1.1.2)
+    connection_pool (2.2.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    database_cleaner (1.8.5)
+    database_cleaner-sequel (1.8.0)
+      database_cleaner (~> 1.8.0)
+      sequel
+    diff-lcs (1.3)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    dotenv (2.7.5)
+    ffi (1.13.1)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    hashdiff (1.0.1)
+    http (4.4.1)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    http-parser (1.2.1)
+      ffi-compiler (>= 1.0, < 2.0)
+    maxmind-db (1.1.1)
+    maxmind-geoip2 (0.4.0)
+      connection_pool (~> 2.2)
+      http (~> 4.3)
+      maxmind-db (~> 1.1)
+    method_source (1.0.0)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.5)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    safe_yaml (1.0.5)
+    sequel (5.33.0)
+    sqlite3 (1.4.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  database_cleaner-sequel (= 1.8.0)
+  dotenv (~> 2.7.5)
+  maxmind-geoip2 (~> 0.4.0)
+  nokogiri (~> 1.10.10)
+  phisher_phinder!
+  pry
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  sequel (~> 5.33)
+  sqlite3 (~> 1.4.2)
+  webmock (~> 3.8.3)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Rory McKinley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Rory McKinley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# PhisherPhinder
+
+A simple gem to explore phishing emails
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'phisher_phinder'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install phisher_phinder
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rorymckinley/phisher_phinder.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,38 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require 'dotenv/load'
+require 'sqlite3'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+namespace :db do
+  desc "Run migrations"
+  task :migrate, [:version] do |t, args|
+    require "sequel/core"
+    Sequel.extension :migration
+    version = args[:version].to_i if args[:version]
+    Sequel.connect(ENV.fetch("DATABASE_URL")) do |db|
+      Sequel::Migrator.run(db, "db/migrations", target: version)
+    end
+  end
+
+  desc "Create a migration"
+  task :create_migration, [:migration_name] do |_, args|
+    raise "Please provide a name for the migration" unless args[:migration_name]
+    in_use_sequence_numbers = Dir.glob('db/migrations/*.rb').map do |path|
+      matches = path.match('\Adb/migrations/(?<sequence>\d{4})_.+\.rb\z')
+      matches[:sequence].to_i
+    end
+    last_sequence = in_use_sequence_numbers.max || 0
+
+    next_migration_file_path = "db/migrations/%04d_#{args[:migration_name]}.rb" % [last_sequence + 1]
+
+    File.open(next_migration_file_path, 'w') do |f|
+      f.write("Sequel.migration do\n  change do\n  end\nend")
+    end
+
+    puts "Created #{next_migration_file_path}"
+  end
+end

data/bin/console

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require 'dotenv'
+Dotenv.load('.env')
+
+require'sequel'
+DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
+
+require "phisher_phinder"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/db/migrations/0001_create_geo_ip_cache.rb

@@ -0,0 +1,35 @@
+Sequel.migration do
+  change do
+    create_table(:geoip_ip_data) do
+      primary_key :id
+      String :ip_address
+      Integer :location_accuracy_radius
+      Float :latitude
+      Float :longitude
+      String :time_zone
+      String :city_name
+      String :city_geoname_id
+      Integer :city_confidence
+      String :country_name
+      String :country_geoname_id
+      String :country_iso_code
+      Integer :country_confidence
+      String :continent_name
+      String :continent_geoname_id
+      String :postal_code
+      Integer :postal_code_confidence
+      String :registered_country_name
+      String :registered_country_geoname_id
+      String :registered_country_iso_code
+      String :autonomous_system_organisation
+      Integer :autonomous_system_number
+      String :isp
+      String :network
+      String :organisation
+      String :static_ip_score
+      String :user_type
+      Time :created_at, null: false
+      Time :updated_at, null: false
+    end
+  end
+end

data/lib/phisher_phinder.rb

@@ -0,0 +1,28 @@
+require "phisher_phinder/version"
+
+require 'maxmind/geoip2'
+
+require 'sequel'
+Sequel::Model.plugin :timestamps, update_on_create: true
+
+require_relative './phisher_phinder/body_hyperlink'
+require_relative './phisher_phinder/cached_geoip_client'
+require_relative './phisher_phinder/geoip_ip_data'
+require_relative './phisher_phinder/expanded_data_processor'
+require_relative './phisher_phinder/extended_ip'
+require_relative './phisher_phinder/extended_ip_factory'
+require_relative './phisher_phinder/mail_parser'
+require_relative './phisher_phinder/mail'
+require_relative './phisher_phinder/simple_ip'
+require_relative './phisher_phinder/mail_parser/received_headers/parser'
+require_relative './phisher_phinder/mail_parser/received_headers/by_parser'
+require_relative './phisher_phinder/mail_parser/received_headers/classifier'
+require_relative './phisher_phinder/mail_parser/received_headers/for_parser'
+require_relative './phisher_phinder/mail_parser/received_headers/from_parser'
+require_relative './phisher_phinder/mail_parser/received_headers/starttls_parser'
+require_relative './phisher_phinder/mail_parser/received_headers/timestamp_parser'
+
+module PhisherPhinder
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/phisher_phinder/body_hyperlink.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class BodyHyperlink
+    attr_reader :href, :text, :type, :raw_href
+
+    def initialize(href, text)
+      @type = classify_href(href.strip)
+      @raw_href = href
+      @href = parse_href(href)
+      @text = text
+    end
+
+    def supports_retrieval?
+      @type == :url && href.is_a?(URI)
+    end
+
+    def ==(other)
+      href == other.href && text == other.text
+    end
+
+    private
+
+    def classify_href(href_value)
+      case href_value
+      when /\A#/
+        :url_fragment
+      when /\Amailto:/
+        :email_address
+      when /\Atel:/
+        :telephone_number
+      else
+        :url
+      end
+    end
+
+    def parse_href(href)
+      stripped_href = href.strip
+
+      (@type == :url && !stripped_href.empty?) ? URI.parse(strip_off_fragments(stripped_href)) : stripped_href
+    end
+
+    def strip_off_fragments(uri)
+      uri.split('#').first
+    end
+  end
+end

data/lib/phisher_phinder/cached_geoip_client.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module PhisherPhinder
+  class CachedGeoipClient
+    def initialize(client, expiry_time)
+      @client = client
+      @expiry_time = expiry_time
+      @lang = 'en'
+    end
+
+    def lookup(ip_address)
+      cached_record = retrieve_cached_record(ip_address)
+      if cached_record && cached_record_valid?(cached_record)
+        cached_record
+      else
+        refresh_cache(ip_address, cached_record)
+      end
+    end
+
+    private
+
+    def retrieve_cached_record(ip_address)
+      PhisherPhinder::GeoipIpData.first(ip_address: ip_address)
+    end
+
+    def cached_record_valid?(cached_record)
+      cached_record.updated_at > @expiry_time
+    end
+
+    def refresh_cache(ip_address, cached_record)
+      lookup_result = @client.insights(ip_address)
+
+      if cached_record
+        cached_record.update(
+          location_accuracy_radius: lookup_result.location.accuracy_radius,
+          latitude: lookup_result.location.latitude,
+          longitude: lookup_result.location.longitude,
+          time_zone: lookup_result.location.time_zone,
+          city_name: (lookup_result.city.names ? lookup_result.city.names[@lang] : nil),
+          city_geoname_id: lookup_result.city.geoname_id,
+          city_confidence: lookup_result.city.confidence,
+          country_name: lookup_result.country.names[@lang],
+          country_geoname_id: lookup_result.country.geoname_id,
+          country_iso_code: lookup_result.country.iso_code,
+          country_confidence: lookup_result.country.confidence,
+          continent_name: lookup_result.continent.names[@lang],
+          continent_geoname_id: lookup_result.continent.geoname_id,
+          postal_code: lookup_result.postal.code,
+          postal_code_confidence: lookup_result.postal.confidence,
+          registered_country_name: lookup_result.registered_country.names[@lang],
+          registered_country_geoname_id: lookup_result.registered_country.geoname_id,
+          registered_country_iso_code: lookup_result.registered_country.iso_code,
+          autonomous_system_organisation: lookup_result.traits.autonomous_system_organization,
+          autonomous_system_number: lookup_result.traits.autonomous_system_number,
+          isp: lookup_result.traits.isp,
+          network: lookup_result.traits.network,
+          organisation: lookup_result.traits.organization,
+          static_ip_score: lookup_result.traits.static_ip_score,
+          user_type: lookup_result.traits.user_type
+        )
+
+        cached_record
+      else
+        PhisherPhinder::GeoipIpData.create(
+          ip_address: ip_address,
+          location_accuracy_radius: lookup_result.location.accuracy_radius,
+          latitude: lookup_result.location.latitude,
+          longitude: lookup_result.location.longitude,
+          time_zone: lookup_result.location.time_zone,
+          city_name: (lookup_result.city.names ? lookup_result.city.names[@lang] : nil),
+          city_geoname_id: lookup_result.city.geoname_id,
+          city_confidence: lookup_result.city.confidence,
+          country_name: lookup_result.country.names[@lang],
+          country_geoname_id: lookup_result.country.geoname_id,
+          country_iso_code: lookup_result.country.iso_code,
+          country_confidence: lookup_result.country.confidence,
+          continent_name: lookup_result.continent.names[@lang],
+          continent_geoname_id: lookup_result.continent.geoname_id,
+          postal_code: lookup_result.postal.code,
+          postal_code_confidence: lookup_result.postal.confidence,
+          registered_country_name: lookup_result.registered_country.names[@lang],
+          registered_country_geoname_id: lookup_result.registered_country.geoname_id,
+          registered_country_iso_code: lookup_result.registered_country.iso_code,
+          autonomous_system_organisation: lookup_result.traits.autonomous_system_organization,
+          autonomous_system_number: lookup_result.traits.autonomous_system_number,
+          isp: lookup_result.traits.isp,
+          network: lookup_result.traits.network,
+          organisation: lookup_result.traits.organization,
+          static_ip_score: lookup_result.traits.static_ip_score,
+          user_type: lookup_result.traits.user_type
+        )
+      end
+    end
+  end
+end