phisher_phinder 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cfb338171a5bca59a9682dfe8157475e7ac1a6283d9410a8367fdbe78fed0be1
4
- data.tar.gz: db30dccf14c2184cd01f4cb27e4d0cc305fda0ddb0b58205a6e3a077893e1eae
3
+ metadata.gz: bf9bd21584b11a5fe4ceff942aa6b7631fef9177682b775d9c0a0508a99c0643
4
+ data.tar.gz: b10859f0c80054eeeff83288dde7eb25ef5bdf8a32d83cd04cfff181a39864a2
5
5
  SHA512:
6
- metadata.gz: 831b8c8a93dc8cf8105724960be8052c1aa47a16fb57336a683e9780b07ce89a72e2e081344826b5462582c1270ca3fcd8a359d8b5111f97c4925c37cd5c085e
7
- data.tar.gz: c25e50d534e780eaeef883a8e6c4311c4c51176589ac80959adb880c0b9f190b5350dd752497f3b19eb8ede04c95bd527201cf18f5f96222890ba08893495dd9
6
+ metadata.gz: dcf8371fbe75476a791fd12c94be9e2c1fac881785b856a62a9aba67fdca39cdef6956aba9f5b772882f0b58f93ff810734f94f6c8abb2aff89dc0b12364f2c7
7
+ data.tar.gz: b44b1edcb3c1dc0732478d512f75ef599b2fa9b74db3ecc981c0125f5547703b983ddd9dddb205c745688631f79f0722548021da1bd42106ec1f6282bccae88c
data/Gemfile.lock CHANGED
@@ -3,6 +3,8 @@ PATH
3
3
  specs:
4
4
  phisher_phinder (0.3.0)
5
5
  dotenv (~> 2.7.5)
6
+ excon (~> 0.78.1)
7
+ mail
6
8
  maxmind-geoip2 (~> 0.4.0)
7
9
  nokogiri (~> 1.11.0)
8
10
  sequel (~> 5.33)
@@ -14,7 +16,7 @@ PATH
14
16
  GEM
15
17
  remote: https://rubygems.org/
16
18
  specs:
17
- activesupport (6.1.0)
19
+ activesupport (6.1.1)
18
20
  concurrent-ruby (~> 1.0, >= 1.0.2)
19
21
  i18n (>= 1.6, < 2)
20
22
  minitest (>= 5.1)
@@ -26,7 +28,7 @@ GEM
26
28
  bundler (>= 1.2.0, < 3)
27
29
  thor (>= 0.18, < 2)
28
30
  coderay (1.1.2)
29
- concurrent-ruby (1.1.7)
31
+ concurrent-ruby (1.1.8)
30
32
  connection_pool (2.2.3)
31
33
  crack (0.4.3)
32
34
  safe_yaml (~> 1.0.0)
@@ -38,6 +40,7 @@ GEM
38
40
  domain_name (0.5.20190701)
39
41
  unf (>= 0.0.5, < 1.0.0)
40
42
  dotenv (2.7.6)
43
+ excon (0.78.1)
41
44
  ffi (1.14.2)
42
45
  ffi-compiler (1.0.1)
43
46
  ffi (>= 1.0.0)
@@ -51,18 +54,23 @@ GEM
51
54
  http-cookie (1.0.3)
52
55
  domain_name (~> 0.5)
53
56
  http-form_data (2.3.0)
54
- http-parser (1.2.2)
55
- ffi-compiler
56
- i18n (1.8.7)
57
+ http-parser (1.2.3)
58
+ ffi-compiler (>= 1.0, < 2.0)
59
+ i18n (1.8.8)
57
60
  concurrent-ruby (~> 1.0)
61
+ mail (2.7.1)
62
+ mini_mime (>= 0.1.1)
58
63
  maxmind-db (1.1.1)
59
64
  maxmind-geoip2 (0.4.0)
60
65
  connection_pool (~> 2.2)
61
66
  http (~> 4.3)
62
67
  maxmind-db (~> 1.1)
63
68
  method_source (1.0.0)
64
- minitest (5.14.2)
65
- nokogiri (1.11.0-x86_64-linux)
69
+ mini_mime (1.0.2)
70
+ mini_portile2 (2.5.0)
71
+ minitest (5.14.3)
72
+ nokogiri (1.11.1)
73
+ mini_portile2 (~> 2.5.0)
66
74
  racc (~> 1.4)
67
75
  pry (0.13.1)
68
76
  coderay (~> 1.1)
data/bin/console CHANGED
@@ -4,8 +4,8 @@ require "bundler/setup"
4
4
  require 'dotenv'
5
5
  Dotenv.load('.env')
6
6
 
7
- require'sequel'
8
- DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
7
+ # require'sequel'
8
+ # DB = Sequel.connect(ENV.fetch('DATABASE_URL'))
9
9
 
10
10
  require "phisher_phinder"
11
11
 
@@ -8,7 +8,10 @@ require_relative './phisher_phinder/display'
8
8
 
9
9
  require_relative './phisher_phinder/body_hyperlink'
10
10
  require_relative './phisher_phinder/cached_geoip_client'
11
- require_relative './phisher_phinder/contact_finder'
11
+ require_relative './phisher_phinder/host_information_finder'
12
+ require_relative './phisher_phinder/host_response_policy'
13
+ require_relative './phisher_phinder/link_explorer'
14
+ require_relative './phisher_phinder/link_host'
12
15
  require_relative './phisher_phinder/whois_email_extractor'
13
16
  require_relative './phisher_phinder/null_lookup_client'
14
17
  require_relative './phisher_phinder/null_response'
@@ -14,11 +14,16 @@ module PhisherPhinder
14
14
  ip_factory = PhisherPhinder::ExtendedIpFactory.new(geoip_client: lookup_client)
15
15
  mail_parser = PhisherPhinder::MailParser::Parser.new(ip_factory, line_ending)
16
16
  whois_client = Whois::Client.new
17
+ host_information_finder = PhisherPhinder::HostInformationFinder.new(
18
+ whois_client: whois_client,
19
+ extractor: PhisherPhinder::WhoisEmailExtractor.new
20
+ )
17
21
  tracing_report = PhisherPhinder::TracingReport.new(
18
- mail_parser.parse(contents),
19
- PhisherPhinder::ContactFinder.new(
20
- whois_client: whois_client,
21
- extractor: PhisherPhinder::WhoisEmailExtractor.new
22
+ mail: mail_parser.parse(contents),
23
+ host_information_finder: host_information_finder,
24
+ link_explorer: PhisherPhinder::LinkExplorer.new(
25
+ host_information_finder: host_information_finder,
26
+ host_response_policy: PhisherPhinder::HostResponsePolicy.new
22
27
  )
23
28
  )
24
29
  tracing_report.report
@@ -47,6 +47,22 @@ module PhisherPhinder
47
47
  )
48
48
 
49
49
  puts trace_table
50
+
51
+ puts "\n\n"
52
+
53
+ puts 'Body Content'
54
+ puts "\n"
55
+ puts "Linked URLs"
56
+ input_data[:content][:linked_urls].each do |link_set|
57
+ link_set.each_with_index do |link_host, tab_count|
58
+ puts "#{"\t"*tab_count}" +
59
+ "#{link_host.url.to_s} " +
60
+ "(#{display_creation_date(link_host)}) " +
61
+ "[#{display_email_addresses(link_host.host_information[:abuse_contacts])}]" +
62
+ "\n"
63
+ end
64
+ puts "\n"
65
+ end
50
66
  end
51
67
 
52
68
  private
@@ -66,5 +82,9 @@ module PhisherPhinder
66
82
  def display_email_addresses(email_addresses)
67
83
  email_addresses.map { |address| address.gsub(/[,<>]/, '') }.join(', ')
68
84
  end
85
+
86
+ def display_creation_date(link_host)
87
+ (date = link_host.host_information[:creation_date]) ? date.strftime('%Y-%m-%d %H:%M:%S') : nil
88
+ end
69
89
  end
70
90
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PhisherPhinder
4
+ class HostInformationFinder
5
+ def initialize(whois_client:, extractor:)
6
+ @whois_client = whois_client
7
+ @extractor = extractor
8
+ end
9
+
10
+ def information_for(address)
11
+ most_relevant_record = nil
12
+
13
+ begin
14
+ case address
15
+ when ExtendedIp
16
+ most_relevant_record = @whois_client.lookup(address.ip_address.to_s)
17
+ when String
18
+ hostname_parts = address.split('.')
19
+ until most_relevant_record do
20
+ address = hostname_parts.join('.')
21
+ whois_record = @whois_client.lookup(address)
22
+ if whois_record.parser.available?
23
+ hostname_parts = hostname_parts[1..-1]
24
+ else
25
+ most_relevant_record = whois_record
26
+ end
27
+ end
28
+ end
29
+ rescue Whois::ServerNotFound
30
+ rescue Whois::AttributeNotImplemented
31
+ end
32
+
33
+ {
34
+ abuse_contacts: most_relevant_record ? @extractor.abuse_contact_emails(most_relevant_record.content) : [],
35
+ creation_date: creation_date(most_relevant_record)
36
+ }
37
+ end
38
+
39
+ private
40
+
41
+ def creation_date(record)
42
+ return nil unless record
43
+
44
+ begin
45
+ record.parser.created_on
46
+ rescue Whois::AttributeNotImplemented, Whois::AttributeNotSupported
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PhisherPhinder
4
+ class HostResponsePolicy
5
+ def next_url(url, response)
6
+ location_header = response.headers['Location']
7
+
8
+ if [301, 302, 303, 307, 308].include?(response.status) && location_header && !location_header.empty?
9
+ if response.headers['Location'] =~ %r{\A/}
10
+ url.merge(response.headers['Location'])
11
+ else
12
+ URI.parse(response.headers['Location'])
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+ require 'excon'
3
+
4
+ module PhisherPhinder
5
+ class LinkExplorer
6
+ def initialize(host_information_finder:, host_response_policy:)
7
+ @host_information_finder = host_information_finder
8
+ @host_response_policy = host_response_policy
9
+ end
10
+
11
+ def explore(hyperlink)
12
+ if hyperlink.type == :url
13
+ chain_terminated = false
14
+ url = hyperlink.href
15
+ output = []
16
+
17
+ until chain_terminated do
18
+ result = Excon.get(url.to_s)
19
+
20
+ output << LinkHost.new(
21
+ url: url,
22
+ body: result.body,
23
+ headers: result.headers,
24
+ status_code: result.status,
25
+ host_information: @host_information_finder.information_for("#{url.scheme}://#{url.host}"),
26
+ )
27
+
28
+ unless url = @host_response_policy.next_url(url, result)
29
+ chain_terminated = true
30
+ end
31
+ end
32
+
33
+ output
34
+ else
35
+ hyperlink.href =~ /mailto:(.+)/
36
+ ($1.split(';').map { |address| address.strip }).uniq
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,20 @@
1
+ # frozen-string_literal: true
2
+
3
+ module PhisherPhinder
4
+ class LinkHost
5
+ attr_accessor :url, :body, :status_code, :headers, :host_information
6
+
7
+ def initialize(url:, body:, status_code:, headers:, host_information:)
8
+ @url = url
9
+ @body = body
10
+ @status_code = status_code
11
+ @headers = headers
12
+ @host_information = host_information
13
+ end
14
+
15
+ def ==(other)
16
+ url == other.url && body == other.body && status_code == other.status_code && headers == other.headers &&
17
+ host_information == other.host_information
18
+ end
19
+ end
20
+ end
@@ -13,13 +13,13 @@ module PhisherPhinder
13
13
  def parse(contents)
14
14
  original_headers, original_body = separate(contents)
15
15
  headers = extract_headers(original_headers)
16
- Mail.new(
16
+ PhisherPhinder::Mail.new(
17
17
  original_email: contents,
18
18
  original_headers: original_headers,
19
19
  original_body: original_body,
20
20
  headers: headers,
21
21
  tracing_headers: generate_tracing_headers(headers),
22
- body: parse_body(original_body, headers),
22
+ body: MailParser::BodyParser.new.parse(contents),
23
23
  authentication_headers: generate_authentication_headers(headers)
24
24
  )
25
25
  end
@@ -86,14 +86,6 @@ module PhisherPhinder
86
86
  values.sort { |a,b| b[:sequence] <=> a[:sequence] }
87
87
  end
88
88
 
89
- def parse_body(original_body, headers)
90
- MailParser::BodyParser.new(@line_ending).parse(
91
- body_contents: original_body,
92
- content_type: content_type_data(headers),
93
- content_transfer_encoding: content_transfer_encoding_data(headers),
94
- )
95
- end
96
-
97
89
  def valid_base64_decoded(text)
98
90
  if Base64.strict_encode64(Base64.decode64(text)) == text.gsub(/#{@line_ending}/, '')
99
91
  Base64.decode64(text)
@@ -1,87 +1,36 @@
1
1
  # frozen_string_literal: true
2
+ require 'mail'
2
3
 
3
4
  module PhisherPhinder
4
5
  module MailParser
5
6
  class BodyParser
6
- def initialize(line_end)
7
- @line_end = line_end
8
- end
9
-
10
- def parse(body_contents:, content_type:, content_transfer_encoding:)
11
- if multipart_alternative?(content_type)
12
- parse_multipart_alternative(content_type, body_contents)
13
- else
14
- classifier = Body::BlockClassifier.new(@line_end)
15
- parser = Body::BlockParser.new(@line_end)
16
-
17
- classification = classifier.classify_headers(
18
- content_type: content_type, content_transfer_encoding: content_transfer_encoding
19
- ).merge(content: body_contents)
20
-
21
- contents = parser.parse(classification)
22
-
23
- if classification[:content_type] == :html
24
- {
25
- html: contents,
26
- text: nil
27
- }
28
- else
29
- {
30
- html: nil,
31
- text: contents
32
- }
33
- end
34
- end
7
+ def parse(contents)
8
+ mail = ::Mail.new(contents)
9
+ aggregate_body_parts(mail)
35
10
  end
36
11
 
37
12
  private
38
13
 
39
- def html?(content_type)
40
- content_type && content_type.split(';').first == 'text/html'
41
- end
42
-
43
- def decode_body(body_contents, content_transfer_encoding)
44
- require 'base64'
45
-
46
- content_transfer_encoding ? Base64.decode64(body_contents) : body_contents
47
- end
48
-
49
- def multipart_alternative?(content_type)
50
- content_type =~ /\Amultipart\/alternative/
51
- end
52
-
53
- def parse_multipart_alternative(content_type, contents)
54
- base_boundary = content_type.split(';').last.strip.gsub(/boundary=/, '').gsub(/"/, '')
55
- start_boundary = '--' + base_boundary + @line_end
56
- end_boundary = '--' + base_boundary + '--'
57
-
58
- raw_blocks = contents.split(start_boundary)
59
- trimmed_blocks = strip_epilogue(strip_prologue(raw_blocks), end_boundary)
60
-
61
- categorise_blocks(trimmed_blocks).inject({html: '', text: ''}) do |memo, block|
62
- memo.merge(block[:html] ? {html: memo[:html] + block[:contents]} : {text: memo[:text] + block[:contents]})
14
+ def aggregate_body_parts(mail)
15
+ accumulator = {text: [], html: []}
16
+ if mail.body.parts.any?
17
+ collapse_content(mail.body, accumulator)
18
+ accumulator.inject({}) { |accum, (type, parts)| accum.merge(type => parts.join) }
19
+ else
20
+ {
21
+ text: mail.body.decoded,
22
+ html: nil
23
+ }
63
24
  end
64
25
  end
65
26
 
66
- def strip_prologue(blocks)
67
- blocks[1..-1]
68
- end
69
-
70
- def strip_epilogue(blocks, end_boundary)
71
- blocks[0..-2] << blocks[-1].split(end_boundary).first
72
- end
73
-
74
- def categorise_blocks(blocks)
75
- classifier = Body::BlockClassifier.new(@line_end)
76
- parser = Body::BlockParser.new(@line_end)
77
- blocks.map do |block|
78
- classification = classifier.classify_block(block)
79
- contents = parser.parse(classification)
80
-
81
- {
82
- html: classification[:content_type] == :html,
83
- contents: contents
84
- }
27
+ def collapse_content(part, accumulator)
28
+ if part.parts.any?
29
+ part.parts.each { |p| collapse_content(p, accumulator) }
30
+ elsif part.content_type =~ %r{text/plain}
31
+ accumulator[:text] << part.decoded
32
+ elsif part.content_type =~ %r{text/html}
33
+ accumulator[:html] << part.decoded
85
34
  end
86
35
  end
87
36
  end
@@ -2,9 +2,10 @@
2
2
 
3
3
  module PhisherPhinder
4
4
  class TracingReport
5
- def initialize(mail, contact_finder)
5
+ def initialize(mail:, host_information_finder:, link_explorer:)
6
6
  @mail = mail
7
- @contact_finder = contact_finder
7
+ @host_information_finder = host_information_finder
8
+ @link_explorer = link_explorer
8
9
  end
9
10
 
10
11
  def report
@@ -19,7 +20,8 @@ module PhisherPhinder
19
20
  }
20
21
  },
21
22
  origin: extract_origin_headers(@mail.headers),
22
- tracing: extract_tracing_headers(@mail.tracing_headers, latest_spf_entry)
23
+ tracing: extract_tracing_headers(@mail.tracing_headers, latest_spf_entry),
24
+ content: explore_hyperlinks(@mail.hypertext_links)
23
25
  }
24
26
  end
25
27
 
@@ -38,8 +40,16 @@ module PhisherPhinder
38
40
  received_headers[:received][start..-1].map do |h|
39
41
  h.merge(
40
42
  sender_contact_details: {
41
- host: {email: @contact_finder.contacts_for(h[:sender][:host])},
42
- ip: {email: @contact_finder.contacts_for(h[:sender][:ip])},
43
+ host: {
44
+ email: @host_information_finder.information_for(
45
+ h[:sender][:host]
46
+ )[:abuse_contacts]
47
+ },
48
+ ip: {
49
+ email: @host_information_finder.information_for(
50
+ h[:sender][:ip]
51
+ )[:abuse_contacts]
52
+ },
43
53
  }
44
54
  )
45
55
  end
@@ -51,5 +61,15 @@ module PhisherPhinder
51
61
  output.merge(header_type => entries.map { |h| h[:data] })
52
62
  end
53
63
  end
64
+
65
+ def explore_hyperlinks(hyperlinks)
66
+ url_hyperlinks = (hyperlinks.select{ |link| link.type == :url }).uniq { |link| link.href }
67
+ email_hyperlinks = hyperlinks.select { |link| link.type == :email_address }
68
+
69
+ {
70
+ linked_urls: url_hyperlinks.map { |hyperlink| @link_explorer.explore(hyperlink) },
71
+ linked_email_addresses: (email_hyperlinks.map { |hyperlink| @link_explorer.explore(hyperlink) }).flatten.uniq
72
+ }
73
+ end
54
74
  end
55
75
  end
@@ -1,3 +1,3 @@
1
1
  module PhisherPhinder
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
28
28
  spec.require_paths = ["lib"]
29
29
 
30
30
  spec.add_dependency "dotenv", "~> 2.7.5"
31
+ spec.add_dependency "excon", "~> 0.78.1"
32
+ spec.add_dependency "mail"
31
33
  spec.add_dependency "maxmind-geoip2", "~> 0.4.0"
32
34
  spec.add_dependency "nokogiri", "~> 1.11.0"
33
35
  spec.add_dependency "sequel", "~> 5.33"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: phisher_phinder
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rory McKinley
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-01-09 00:00:00.000000000 Z
11
+ date: 2021-02-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dotenv
@@ -24,6 +24,34 @@ dependencies:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 2.7.5
27
+ - !ruby/object:Gem::Dependency
28
+ name: excon
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 0.78.1
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.78.1
41
+ - !ruby/object:Gem::Dependency
42
+ name: mail
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
27
55
  - !ruby/object:Gem::Dependency
28
56
  name: maxmind-geoip2
29
57
  requirement: !ruby/object:Gem::Requirement
@@ -249,12 +277,15 @@ files:
249
277
  - lib/phisher_phinder/body_hyperlink.rb
250
278
  - lib/phisher_phinder/cached_geoip_client.rb
251
279
  - lib/phisher_phinder/command.rb
252
- - lib/phisher_phinder/contact_finder.rb
253
280
  - lib/phisher_phinder/display.rb
254
281
  - lib/phisher_phinder/expanded_data_processor.rb
255
282
  - lib/phisher_phinder/extended_ip.rb
256
283
  - lib/phisher_phinder/extended_ip_factory.rb
257
284
  - lib/phisher_phinder/geoip_ip_data.rb
285
+ - lib/phisher_phinder/host_information_finder.rb
286
+ - lib/phisher_phinder/host_response_policy.rb
287
+ - lib/phisher_phinder/link_explorer.rb
288
+ - lib/phisher_phinder/link_host.rb
258
289
  - lib/phisher_phinder/mail.rb
259
290
  - lib/phisher_phinder/mail_parser.rb
260
291
  - lib/phisher_phinder/mail_parser/authentication_headers/auth_results_parser.rb
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module PhisherPhinder
4
- class ContactFinder
5
- def initialize(whois_client:, extractor:)
6
- @whois_client = whois_client
7
- @extractor = extractor
8
- end
9
-
10
- def contacts_for(address)
11
- whois_content = nil
12
- begin
13
- whois_content = case address
14
- when ExtendedIp
15
- @whois_client.lookup(address.ip_address.to_s).content
16
- when String
17
- hostname_parts = address.split('.')
18
- whois_content = nil
19
- until whois_content do
20
- address = hostname_parts.join('.')
21
- whois_record = @whois_client.lookup(address)
22
- if whois_record.parser.available?
23
- hostname_parts = hostname_parts[1..-1]
24
- else
25
- whois_content = whois_record.content
26
- end
27
- end
28
- whois_content
29
- end
30
- rescue Whois::ServerNotFound
31
- rescue Whois::AttributeNotImplemented
32
- end
33
-
34
- whois_content ? @extractor.abuse_contact_emails(whois_content) : []
35
- end
36
- end
37
- end