osa 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c74cb930be462e745a7682afed8e2715e10bfb391f188d5f0f885c3b8150d435
-  data.tar.gz: e41a5966566a34b0f48f5eb40c9df7742fe56a8aa75299ce5db1cd0e64aeaa76
+  metadata.gz: 1008becc098b639309f218c9f763e57d4d01f201b28fd105c478cd46534de913
+  data.tar.gz: 2c285065f454aeaa7fcc1f9dd6f72e92d3423dbe292d06ef36afba6a8db6f9eb
 SHA512:
-  metadata.gz: 2a7fd8af344acc7f7839f4b9165cc39c913a9edca0e94155a68b231d773b918da69637a1c79dcd292654adf8248f30991c630f31b03588641ff1a9ca9e9bdc85
-  data.tar.gz: 9cc8eab12bacc5010f712ed0dbb5a2bbdf056c16ec97b98e1a90bcb93e9eede72af274bb91379ef3552a11022894dd2c4da0b853d843e43a9f2e6658e086e7bb
+  metadata.gz: 4b53ba423fc0b0da3bc8f49188d34370cf49d12e968a13a436ffaa6c849dceddedcf10c7e2b833d7316dbc0045fe487c44891b6eb50b98fb63477f63b8542a0e
+  data.tar.gz: ba063539fe384116422310df5a0d3a14c324a0560674932dd39998951d433236fb929a86aae981895adff04a29d71d0780c8fc4a2a9d11fa817359ad56c10ff0

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    osa (0.2.0)
+    osa (0.2.1)
       activerecord (~> 6.0)
       faraday (~> 1.1)
+      mail (~> 2.7.1)
       public_suffix (~> 4.0)
       sinatra (~> 2.1.0)
       sinatra-contrib (~> 2.1.0)
@@ -13,12 +14,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activerecord (6.1.2.1)
-      activemodel (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activesupport (6.1.2.1)
+    activemodel (6.1.3)
+      activesupport (= 6.1.3)
+    activerecord (6.1.3)
+      activemodel (= 6.1.3)
+      activesupport (= 6.1.3)
+    activesupport (6.1.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -33,6 +34,9 @@ GEM
     faraday-net_http (1.0.1)
     i18n (1.8.9)
       concurrent-ruby (~> 1.0)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    mini_mime (1.0.2)
     minitest (5.14.3)
     multi_json (1.15.0)
     multipart-post (2.1.1)

data/README.md

@@ -25,6 +25,13 @@ be blacklisted. However, to prevent millions of users to go blacklisted because
 list of free email providers (which includes domains like gmail.com, outlook.com among others). If the sender uses a free
 email provider, the full address is blacklisted.
 
+OSA also supports Domain Name System Blacklist. In fact it comes bundled with 3 DNSBL:
+1. [Spamcop Blocking List](https://www.spamcop.net/fom-serve/cache/297.html)
+2. [Spamhaus Block List](https://www.spamhaus.org/sbl)
+3. [Passive Spam Block List](https://psbl.org)
+
+You can remove these or add more blacklists, from the database, after you configure OSA.
+
 ## Installation
 
 You can install OSA from RubyGems:

data/lib/osa/clients/ms_graph_client.rb

@@ -6,6 +6,7 @@ require 'osa/util/paginated'
 require 'osa/clients/http_client'
 require 'active_support/core_ext/numeric/bytes'
 require 'active_support'
+require 'mail'
 
 module OSA
   class MSGraphClient
@@ -44,6 +45,12 @@ module OSA
       @authenticated.get("/v1.0/me/messages/#{mail_id}/$value")
     end
 
+    def sender_ip(mail_id)
+      content = raw_mail(mail_id)
+      mail = Mail.new(content)
+      mail.header['x-sender-ip']
+    end
+
     def forward_mail_as_attachment(mail_id, to)
       raw_mail = self.raw_mail(mail_id)
       forward_message = create_forward_message(mail_id)

data/lib/osa/migrations/00005_create_dns_blacklists.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+require 'active_record/migration'
+
+class CreateDnsBlacklists < ActiveRecord::Migration[5.0]
+  def change
+    create_table :dns_blacklists do |t|
+      t.text :name, null: false
+      t.text :server, null: false
+    end
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+        insert into dns_blacklists (name, server) values
+          ('spamcop', 'bl.spamcop.net'),
+          ('sbl', 'sbl.spamhaus.org'),
+          ('psbl', 'psbl.surriel.org');
+        SQL
+        add_column :reports, :blacklist, :string, null: true
+        execute <<~SQL
+          update reports set blacklist = 'db' where blacklisted = true;
+        SQL
+        remove_column :reports, :blacklisted
+      end
+
+      dir.down do
+        add_column :reports, :blacklisted, :boolean, default: false
+        execute <<~SQL
+          update reports set blacklisted = true where blacklist = 'db';
+        SQL
+        remove_column :reports, :blacklist
+      end
+    end
+  end
+end

data/lib/osa/scripts/dashboard_server.rb

@@ -11,6 +11,10 @@ class DashboardServer < Sinatra::Base
     erb :index
   end
 
+  get '/spammers/:spammer' do
+    erb :spammer
+  end
+
   get '/api/stats/summary' do
     blacklist_count = OSA::Blacklist.count
     total_reported = OSA::Report.count
@@ -31,13 +35,34 @@ class DashboardServer < Sinatra::Base
 
   end
 
+  get '/api/stats/spammers/:spammer/summary' do |spammer|
+    reports = OSA::Report.where(sender: spammer).or(OSA::Report.where(sender_domain: spammer))
+    total_reported = reports.count
+    today = DateTime.now.to_date
+    today_reported = reports.where('reported_at > ?', today).count
+    week = DateTime.now - 1.week
+    week_reported = reports.where('reported_at > ?', week).count
+    month = DateTime.now - 30.days
+    month_reported = reports.where('reported_at > ?', month).count
+    domains = reports.select(:sender).distinct.map { |e| e.sender.split('@', 2)[1] }.uniq
+
+    json total_reported: total_reported,
+         today_reported: today_reported,
+         week_reported: week_reported,
+         month_reported: month_reported,
+         domains: domains
+  end
+
   get '/api/stats/spammers' do
     spammers = OSA::Report.select('sender_domain as domain', 'COUNT(*) as count').limit(50).order(count: :desc).group(:sender_domain)
     json spammers
   end
 
   get '/api/stats/reports/historical' do
-    historical_data = OSA::Report.select("strftime('%Y-%m-%d', reported_at) as date", 'count(*) as count').group(:date)
-    json historical_data
+    historical_data = OSA::Report.select("strftime('%Y-%m-%d', reported_at) as date", 'count(*) as count')
+    unless params[:spammer].blank?
+      historical_data = historical_data.where(sender: params[:spammer]).or(OSA::Report.where(sender_domain: params[:spammer]))
+    end
+    json historical_data.group(:date)
   end
 end

data/lib/osa/scripts/scan_junk_folder.rb

@@ -9,6 +9,18 @@ context = OSA::Context.new
 
 continue = true
 
+dns_blacklists = OSA::DnsBlacklist.all
+
+def resolve_blacklist(mail_id, email_address, domain, context, dns_blacklists)
+  return 'db' if OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists?
+  ip = context.graph_client.sender_ip(mail_id)
+  if ip.nil?
+    puts "Couldn't detect ip for email from #{email_address}"
+    return nil
+  end
+  dns_blacklists.find { |bl| bl.blacklisted?(ip) }&.server
+end
+
 while continue
   mails = context.graph_client.mails(context.config.junk_folder_id)
   continue = false
@@ -20,13 +32,12 @@ while continue
       domain = PublicSuffix.domain(email_address.split('@', 2)[1])
 
       flagged = mail['flag']['flagStatus'] == 'flagged'
-      blacklisted = nil
-      blacklisted = OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists?
+      blacklist = (resolve_blacklist(mail['id'], email_address, domain, context, dns_blacklists) unless flagged)
 
       if flagged
         puts "Email from #{email_address} is flagged, reporting and deleting"
-      elsif blacklisted
-        puts "#{email_address} is blacklisted, reporting and deleting"
+      elsif !blacklist.nil?
+        puts "#{email_address} is blacklisted by #{blacklist}, reporting and deleting"
       else
         puts "Skipping mail from #{email_address}, its not blacklisted"
         next
@@ -39,8 +50,16 @@ while continue
       puts "deleting spam from #{email_address}"
       context.graph_client.delete_mail(mail['id'])
 
-      domain = PublicSuffix.domain(email_address.split('@', 2)[1])
+      OSA::Report.create!(sender: email_address,
+                          sender_domain: domain,
+                          subject: mail['subject'],
+                          flagged: flagged,
+                          blacklist: blacklist,
+                          received_at: Time.iso8601(mail['receivedDateTime']),
+                          reported_at: Time.now)
 
+      # Do not add to the blacklist if the it's blacklisted by the db (it's already present)
+      # or blacklisted by DNSBLs (these blacklists are only supposed to be temporary).
       if flagged
         is_free_provider = OSA::EmailProvider.where(value: domain).exists?
         if is_free_provider
@@ -51,15 +70,8 @@ while continue
           OSA::Blacklist.find_or_create_by(value: domain).save!
         end
       end
-
-      OSA::Report.create!(sender: email_address,
-                     sender_domain: domain,
-                     subject: mail['subject'],
-                     flagged: flagged,
-                     blacklisted: blacklisted,
-                     received_at: Time.iso8601(mail['receivedDateTime']),
-                     reported_at: Time.now)
     end
+
     mails = mails.next
   end
 end

data/lib/osa/util/db.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'active_record'
+require 'resolv'
 
 ActiveRecord::Base.establish_connection(
   adapter: 'sqlite3',
@@ -21,4 +22,12 @@ module OSA
 
   class Report < ActiveRecord::Base
   end
+
+  class DnsBlacklist < ActiveRecord::Base
+    def blacklisted?(ip)
+      ::Resolv.getaddress("#{ip}.#{server}") != "0.0.0.0"
+    rescue ::Resolv::ResolvError
+      return false
+    end
+  end
 end

data/lib/osa/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module OSA
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

data/lib/osa/views/index.erb

@@ -138,16 +138,13 @@
     let totalReportCount = 0;
 
     function sec2time(timeInSeconds) {
-        const pad = function (num, size) {
-            return ('000' + num).slice(size * -1);
-        }
         const time = parseFloat(timeInSeconds).toFixed(3);
         const hours = Math.floor(time / 60 / 60);
         const minutes = Math.floor(time / 60) % 60;
 
         const units = [{value: hours, unit: 'h'}, {value: minutes, unit: 'm'}].filter((value) => value.value > 0 );
 
-        return units.map((value) => `${pad(value.value, 2)}${value.unit}`).join(' ');
+        return units.map((value) => `${value.value}${value.unit}`).join(' ');
     }
 
     async function updateStats() {
@@ -194,7 +191,10 @@
                 tr.appendChild(indexTd);
 
                 const domainTd = document.createElement("td");
-                domainTd.innerText = spammer['domain'];
+                const domainRef = document.createElement("a");
+                domainRef.setAttribute("href", `/spammers/${encodeURI(spammer['domain'])}`);
+                domainRef.innerText = spammer['domain']
+                domainTd.appendChild(domainRef);
                 tr.appendChild(domainTd);
 
                 const reportCountTd = document.createElement("td");
@@ -208,7 +208,7 @@
                 tr.appendChild(spamRatioProgressTd);
 
                 const spamRatioTd = document.createElement("td");
-                spamRatioTd.innerText = `${spamRatio}%`;
+                spamRatioTd.innerText = `${spamRatio.toFixed(1)}%`;
                 tr.appendChild(spamRatioTd);
 
                 return tr;

data/lib/osa/views/layout.erb

@@ -13,7 +13,7 @@
   <!-- Main Sidebar Container -->
   <aside class="main-sidebar sidebar-dark-primary elevation-4">
     <!-- Brand Logo -->
-    <a href="index.html" class="brand-link">
+    <a href="/" class="brand-link">
       <span class="brand-text font-weight-light">Outlook Spam Automator</span>
     </a>
 
@@ -24,8 +24,7 @@
       <nav class="mt-2">
         <ul class="nav nav-pills nav-sidebar flex-column" data-widget="treeview" role="menu" data-accordion="false">
           <li class="nav-item">
-            <a href="#" class="nav-link">
-              <i class="nav-icon fas fa-th"></i>
+            <a href="/" class="nav-link">
               <p>Overview</p>
             </a>
           </li>
@@ -38,16 +37,8 @@
 
   <%= yield %>
 
-  <!-- Main Footer -->
-  <footer class="main-footer">
-    <!-- To the right -->
-    <div class="float-right d-none d-sm-inline">
-      Anything you want
-    </div>
-    <!-- Default to the left -->
-    <strong>Copyright © 2014-2020 <a href="https://adminlte.io">AdminLTE.io</a>.</strong> All rights reserved.
-  </footer>
-  <div id="sidebar-overlay"></div></div>
+  <div id="sidebar-overlay"></div>
+</div>
 <!-- ./wrapper -->
 
 </body>

data/lib/osa/views/spammer.erb

@@ -0,0 +1,209 @@
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper" style="min-height: 1233px;">
+  <!-- Content Header (Page header) -->
+  <div class="content-header">
+    <div class="container-fluid">
+      <div class="row mb-2">
+        <div class="col-sm-6">
+          <h1 id="title" class="m-0">Spammer Details</h1>
+        </div><!-- /.col -->
+      </div><!-- /.row -->
+    </div><!-- /.container-fluid -->
+  </div>
+  <!-- /.content-header -->
+
+  <!-- Main content -->
+  <div class="content">
+    <div class="container-fluid">
+      <div class="row">
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="today-count"></h3>
+              <p>emails reported today</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="week-count"></h3>
+              <p>emails reported in the last 7 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="month-count"></h3>
+              <p>emails reported in the last 30 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-warning">
+            <div class="inner">
+              <h3 id="total-count"></h3>
+              <p>emails reported in total</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Spams reported</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body">
+            <div class="chart">
+              <div class="chartjs-size-monitor">
+                <div class="chartjs-size-monitor-expand">
+                  <div class=""></div>
+                </div>
+                <div class="chartjs-size-monitor-shrink">
+                  <div class=""></div>
+                </div>
+              </div>
+              <canvas id="report-history-chart" style="min-height: 250px; height: 250px; max-height: 250px; max-width: 100%; display: block; width: 1732px;" width="1732" height="250" class="chartjs-render-monitor"></canvas>
+            </div>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Domains</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body p-0">
+            <table class="table table-striped">
+              <thead>
+              </thead>
+              <tbody id="domains-table-body">
+              </tbody>
+            </table>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+
+    </div> <!-- /.container-fluid -->
+    </div>
+  </div>
+  <!-- /.content -->
+</div>
+<!-- /.content-wrapper -->
+
+<script>
+    const pathComponents = window.location.pathname.split('/');
+    const spammer = pathComponents[pathComponents.length - 1];
+    document.getElementById("title").innerText = `${spammer} Details`;
+
+    async function updateStats() {
+        const response = await fetch(`/api/stats/spammers/${spammer}/summary`);
+        if (response.ok) {
+            const body = await response.json();
+            document.getElementById('today-count').innerText = body['today_reported'];
+            document.getElementById('week-count').innerText = body['week_reported'];
+            document.getElementById('month-count').innerText = body['month_reported'];
+            document.getElementById('total-count').innerText = body['total_reported'];
+            totalReportCount = body['total_reported'];
+
+            const tableBody = document.getElementById("domains-table-body");
+            tableBody.innerHTML = "";
+
+            body['domains'].forEach((domain) => {
+                const tr = document.createElement("tr");
+                const td = document.createElement("td");
+                td.innerText = domain;
+                tr.appendChild(td);
+                tableBody.appendChild(tr);
+            });
+        }
+    }
+
+    function createHistoricalChart() {
+        const options = {
+            maintainAspectRatio: false,
+            responsive: true,
+            legend: {
+                display: false
+            },
+            scales: {
+                xAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }],
+                yAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }]
+            }
+        }
+
+        const chartData = {
+            labels: [],
+            datasets: [
+                {
+                    label: 'Spams reported',
+                    backgroundColor: 'rgba(60,141,188,0.9)',
+                    borderColor: 'rgba(60,141,188,0.8)',
+                    pointColor: '#3b8bba',
+                    pointStrokeColor: 'rgba(60,141,188,1)',
+                    pointHighlightFill: '#fff',
+                    pointHighlightStroke: 'rgba(60,141,188,1)',
+                    data: []
+                }
+            ]
+        }
+        const ctx = document.getElementById('report-history-chart').getContext('2d');
+
+        return new Chart(ctx, {
+            type: 'line',
+            options: options,
+            data: chartData
+        });
+    }
+
+    const historicalChart = createHistoricalChart();
+
+    async function updateHistoricalChart() {
+        const response = await fetch(`/api/stats/reports/historical?spammer=${spammer}`);
+        if (response.ok) {
+            const body = await response.json();
+
+            const labels = body.map((data) => data['date']);
+            const values = body.map((data) => data['count']);
+
+            historicalChart.data.labels = labels
+            historicalChart.data.datasets[0].data = values
+            historicalChart.update();
+        }
+    }
+
+    function update() {
+        updateStats()
+            .then(updateHistoricalChart)
+            .then(() => {
+                setTimeout(update, 10 * 60 * 1000);
+            })
+    }
+
+    update();
+</script>

data/osa.gemspec

@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'tty-prompt', '~> 0.22'
   spec.add_dependency 'sinatra', '~> 2.1.0'
   spec.add_dependency 'sinatra-contrib', '~> 2.1.0'
+  spec.add_dependency 'mail', '~> 2.7.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osa
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Moray Baruh
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-14 00:00:00.000000000 Z
+date: 2021-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 description: Get rid of spam on your Outlook account
 email:
 - contact@moraybaruh.com
@@ -137,6 +151,7 @@ files:
 - lib/osa/migrations/00002_create_email_providers.rb
 - lib/osa/migrations/00003_create_config.rb
 - lib/osa/migrations/00004_create_reports.rb
+- lib/osa/migrations/00005_create_dns_blacklists.rb
 - lib/osa/migrations/free-email-providers.txt
 - lib/osa/scripts/dashboard_server.rb
 - lib/osa/scripts/scan_junk_folder.rb
@@ -149,6 +164,7 @@ files:
 - lib/osa/version.rb
 - lib/osa/views/index.erb
 - lib/osa/views/layout.erb
+- lib/osa/views/spammer.erb
 - osa.gemspec
 - release.sh
 - web/login.html
@@ -171,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Outlook Spam Automator