osa 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0fbdd1dceb350e48a1d593ec7590e5c1caba49136ae6c6a1afb3a41a86683d8
-  data.tar.gz: 962578a1b23e0879a5175f21a6c617812cec82e3f6577b58f401dc0cd78becc8
+  metadata.gz: c74cb930be462e745a7682afed8e2715e10bfb391f188d5f0f885c3b8150d435
+  data.tar.gz: e41a5966566a34b0f48f5eb40c9df7742fe56a8aa75299ce5db1cd0e64aeaa76
 SHA512:
-  metadata.gz: aabca2784a303f54c43eda8ef7f1491438c65658184eebc6821574785057b2b3ced909787cd66dca29b8db38aa85fdf60959041884373eb84c6bbeb72204d99e
-  data.tar.gz: befb3a9c9ac8ca252c5e23c84bb7fdb709be6765e092d2315472baccfda59736949d4bfcb1fa6e78b3216e09d638798c344150b18af26147c7eddd2a2307dd73
+  metadata.gz: 2a7fd8af344acc7f7839f4b9165cc39c913a9edca0e94155a68b231d773b918da69637a1c79dcd292654adf8248f30991c630f31b03588641ff1a9ca9e9bdc85
+  data.tar.gz: 9cc8eab12bacc5010f712ed0dbb5a2bbdf056c16ec97b98e1a90bcb93e9eede72af274bb91379ef3552a11022894dd2c4da0b853d843e43a9f2e6658e086e7bb

data/Gemfile.lock

@@ -1,44 +1,52 @@
 PATH
   remote: .
   specs:
-    osa (0.1.3)
+    osa (0.2.0)
       activerecord (~> 6.0)
       faraday (~> 1.1)
       public_suffix (~> 4.0)
+      sinatra (~> 2.1.0)
+      sinatra-contrib (~> 2.1.0)
       sqlite3 (~> 1.4)
       tty-prompt (~> 0.22)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.0)
-      activesupport (= 6.1.0)
-    activerecord (6.1.0)
-      activemodel (= 6.1.0)
-      activesupport (= 6.1.0)
-    activesupport (6.1.0)
+    activemodel (6.1.2.1)
+      activesupport (= 6.1.2.1)
+    activerecord (6.1.2.1)
+      activemodel (= 6.1.2.1)
+      activesupport (= 6.1.2.1)
+    activesupport (6.1.2.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
     ast (2.4.1)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.8)
     faraday (1.3.0)
       faraday-net_http (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords
-    faraday-net_http (1.0.0)
-    i18n (1.8.6)
+    faraday-net_http (1.0.1)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.2)
+    minitest (5.14.3)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     parallel (1.20.0)
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pastel (0.8.0)
       tty-color (~> 0.5)
     public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
     rainbow (3.0.0)
     regexp_parser (1.8.2)
     rexml (3.2.4)
@@ -57,8 +65,20 @@ GEM
       rubocop (>= 0.90.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.10.1)
-    ruby2_keywords (0.0.2)
+    ruby2_keywords (0.0.4)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    sinatra-contrib (2.1.0)
+      multi_json
+      mustermann (~> 1.0)
+      rack-protection (= 2.1.0)
+      sinatra (= 2.1.0)
+      tilt (~> 2.0)
     sqlite3 (1.4.2)
+    tilt (2.0.10)
     tty-color (0.6.0)
     tty-cursor (0.7.1)
     tty-prompt (0.23.0)
@@ -84,4 +104,4 @@ DEPENDENCIES
   rubocop-performance
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -65,6 +65,15 @@ Process your junk folder:
 osa scan-junk
 ```
 
+OSA also provides you a nice administration dashboard you. You can access the dashboard by running
+
+```sh
+osa dashboard
+```
+
+You are now able to access the dashboard on `http://localhost:8080`. You can also change the port of the server by
+providing the `SERVER_PORT` environment variable.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/moray95/osa.

data/exe/osa

@@ -12,7 +12,10 @@ when 'login'
   OSA::AuthService.login(OSA::Config.first || OSA::Config.new)
 when 'scan-junk'
   require 'osa/scripts/scan_junk_folder'
+when 'dashboard'
+  require 'osa/scripts/dashboard_server'
+  DashboardServer.start!
 else
-  $stderr.puts "Usage: #{File.basename($0)} [setup|login|scan-junk]"
+  $stderr.puts "Usage: #{File.basename($0)} [setup|login|scan-junk|dashboard]"
   exit 1
 end

data/lib/osa/migrations/00004_create_reports.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+require 'active_record/migration'
+
+class CreateReports < ActiveRecord::Migration[5.0]
+  def change
+    create_table :reports do |t|
+      t.text :sender, null: false
+      t.text :sender_domain, null: false
+      t.text :subject, null: false
+      t.boolean :flagged, null: false
+      t.boolean :blacklisted, null: false
+      t.datetime :received_at, null: false
+      t.datetime :reported_at, null: false
+    end
+  end
+end

data/lib/osa/scripts/dashboard_server.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+require 'sinatra'
+require 'sinatra/json'
+require 'osa/util/db'
+
+class DashboardServer < Sinatra::Base
+  set :views, File.absolute_path(File.dirname(__FILE__) + '/../views')
+  set :port, ENV['SERVER_PORT'] || 8080
+
+  get '/' do
+    erb :index
+  end
+
+  get '/api/stats/summary' do
+    blacklist_count = OSA::Blacklist.count
+    total_reported = OSA::Report.count
+    today = DateTime.now.to_date
+    today_reported = OSA::Report.where('reported_at > ?', today).count
+    week = DateTime.now - 1.week
+    week_reported = OSA::Report.where('reported_at > ?', week).count
+    month = DateTime.now - 30.days
+    month_reported = OSA::Report.where('reported_at > ?', month).count
+    mean_report_time = OSA::Report.select('avg(julianday(reported_at) - julianday(received_at)) * 86400.0 as avg').first['avg']
+
+    json blacklist_count: blacklist_count,
+         total_reported: total_reported,
+         today_reported: today_reported,
+         week_reported: week_reported,
+         month_reported: month_reported,
+         mean_report_time: mean_report_time
+
+  end
+
+  get '/api/stats/spammers' do
+    spammers = OSA::Report.select('sender_domain as domain', 'COUNT(*) as count').limit(50).order(count: :desc).group(:sender_domain)
+    json spammers
+  end
+
+  get '/api/stats/reports/historical' do
+    historical_data = OSA::Report.select("strftime('%Y-%m-%d', reported_at) as date", 'count(*) as count').group(:date)
+    json historical_data
+  end
+end

data/lib/osa/scripts/scan_junk_folder.rb

@@ -21,7 +21,7 @@ while continue
 
       flagged = mail['flag']['flagStatus'] == 'flagged'
       blacklisted = nil
-      blacklisted = OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists? unless flagged
+      blacklisted = OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists?
 
       if flagged
         puts "Email from #{email_address} is flagged, reporting and deleting"
@@ -39,9 +39,9 @@ while continue
       puts "deleting spam from #{email_address}"
       context.graph_client.delete_mail(mail['id'])
 
-      if flagged
-        domain = PublicSuffix.domain(email_address.split('@', 2)[1])
+      domain = PublicSuffix.domain(email_address.split('@', 2)[1])
 
+      if flagged
         is_free_provider = OSA::EmailProvider.where(value: domain).exists?
         if is_free_provider
           puts "#{email_address} is using a free provider, blacklisting full address"
@@ -51,6 +51,14 @@ while continue
           OSA::Blacklist.find_or_create_by(value: domain).save!
         end
       end
+
+      OSA::Report.create!(sender: email_address,
+                     sender_domain: domain,
+                     subject: mail['subject'],
+                     flagged: flagged,
+                     blacklisted: blacklisted,
+                     received_at: Time.iso8601(mail['receivedDateTime']),
+                     reported_at: Time.now)
     end
     mails = mails.next
   end

data/lib/osa/util/db.rb

@@ -18,4 +18,7 @@ module OSA
 
   class EmailProvider < ActiveRecord::Base
   end
+
+  class Report < ActiveRecord::Base
+  end
 end

data/lib/osa/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module OSA
-  VERSION = '0.1.3'
+  VERSION = '0.2.0'
 end

data/lib/osa/views/index.erb

@@ -0,0 +1,296 @@
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper" style="min-height: 1233px;">
+  <!-- Content Header (Page header) -->
+  <div class="content-header">
+    <div class="container-fluid">
+      <div class="row mb-2">
+        <div class="col-sm-6">
+          <h1 class="m-0">Overview</h1>
+        </div><!-- /.col -->
+      </div><!-- /.row -->
+    </div><!-- /.container-fluid -->
+  </div>
+  <!-- /.content-header -->
+
+  <!-- Main content -->
+  <div class="content">
+    <div class="container-fluid">
+      <div class="row">
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-info">
+            <div class="inner">
+              <h3 id="blacklist-count"></h3>
+              <p>entries in blacklist</p>
+            </div>
+          </div>
+        </div>
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-info">
+            <div class="inner">
+              <h3 id="reporting-time"></h3>
+              <p>average reporting time</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="today-count"></h3>
+              <p>emails reported today</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="week-count"></h3>
+              <p>emails reported in the last 7 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="month-count"></h3>
+              <p>emails reported in the last 30 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-warning">
+            <div class="inner">
+              <h3 id="total-count"></h3>
+              <p>emails reported in total</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Spams reported</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body">
+            <div class="chart">
+              <div class="chartjs-size-monitor">
+                <div class="chartjs-size-monitor-expand">
+                  <div class=""></div>
+                </div>
+                <div class="chartjs-size-monitor-shrink">
+                  <div class=""></div>
+                </div>
+              </div>
+              <canvas id="report-history-chart" style="min-height: 250px; height: 250px; max-height: 250px; max-width: 100%; display: block; width: 1732px;" width="1732" height="250" class="chartjs-render-monitor"></canvas>
+            </div>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Top spammers</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body p-0">
+            <table class="table table-striped">
+              <thead>
+              <tr>
+                <th style="width: 10px">#</th>
+                <th>Domain</th>
+                <th>Emails reported</th>
+                <th>Spam ratio</th>
+                <th style="width: 40px"></th>
+              </tr>
+              </thead>
+              <tbody id="top-spammers-table-body">
+              </tbody>
+            </table>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+
+    </div> <!-- /.container-fluid -->
+
+
+  </div>
+  <!-- /.content -->
+</div>
+<!-- /.content-wrapper -->
+
+<script>
+    let totalReportCount = 0;
+
+    function sec2time(timeInSeconds) {
+        const pad = function (num, size) {
+            return ('000' + num).slice(size * -1);
+        }
+        const time = parseFloat(timeInSeconds).toFixed(3);
+        const hours = Math.floor(time / 60 / 60);
+        const minutes = Math.floor(time / 60) % 60;
+
+        const units = [{value: hours, unit: 'h'}, {value: minutes, unit: 'm'}].filter((value) => value.value > 0 );
+
+        return units.map((value) => `${pad(value.value, 2)}${value.unit}`).join(' ');
+    }
+
+    async function updateStats() {
+        const response = await fetch('/api/stats/summary');
+        if (response.ok) {
+            const body = await response.json();
+            document.getElementById('blacklist-count').innerText = body['blacklist_count'];
+            document.getElementById('today-count').innerText = body['today_reported'];
+            document.getElementById('week-count').innerText = body['week_reported'];
+            document.getElementById('month-count').innerText = body['month_reported'];
+            document.getElementById('total-count').innerText = body['total_reported'];
+            const meanReportTime = body['mean_report_time'];
+            if (meanReportTime !== null) {
+                document.getElementById('reporting-time').innerText = sec2time(meanReportTime);
+            } else {
+                document.getElementById('reporting-time').innerText = 'N/A';
+            }
+            totalReportCount = body['total_reported'];
+        }
+    }
+
+    function createProgressDiv(fillRatio) {
+        const progressDiv = document.createElement("div");
+        progressDiv.className = "progress progress-xs";
+
+        const progressBarDiv = document.createElement("div");
+        progressBarDiv.className = "progress-bar progress-bar-danger";
+        progressBarDiv.style.width = `${fillRatio}%`;
+
+        progressDiv.appendChild(progressBarDiv);
+        return progressDiv;
+    }
+
+    async function updateSpammers() {
+        const response = await fetch('/api/stats/spammers');
+        if (response.ok) {
+            const body = await response.json();
+
+            const table = body.map((spammer, index) => {
+                const tr = document.createElement("tr");
+
+                const indexTd = document.createElement("td");
+                indexTd.innerText = index + 1;
+                tr.appendChild(indexTd);
+
+                const domainTd = document.createElement("td");
+                domainTd.innerText = spammer['domain'];
+                tr.appendChild(domainTd);
+
+                const reportCountTd = document.createElement("td");
+                reportCountTd.innerText = spammer['count'];
+                tr.appendChild(reportCountTd);
+
+                const spamRatio = spammer['count'] / totalReportCount * 100;
+
+                const spamRatioProgressTd = document.createElement("td");
+                spamRatioProgressTd.append(createProgressDiv(spamRatio));
+                tr.appendChild(spamRatioProgressTd);
+
+                const spamRatioTd = document.createElement("td");
+                spamRatioTd.innerText = `${spamRatio}%`;
+                tr.appendChild(spamRatioTd);
+
+                return tr;
+            });
+            const tableBody = document.getElementById("top-spammers-table-body")
+            tableBody.innerHTML = "";
+            table.forEach((row) => {
+                tableBody.appendChild(row);
+            });
+
+        }
+    }
+
+    function createHistoricalChart() {
+        const options = {
+            maintainAspectRatio: false,
+            responsive: true,
+            legend: {
+                display: false
+            },
+            scales: {
+                xAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }],
+                yAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }]
+            }
+        }
+
+        const chartData = {
+            labels: [],
+            datasets: [
+                {
+                    label: 'Spams reported',
+                    backgroundColor: 'rgba(60,141,188,0.9)',
+                    borderColor: 'rgba(60,141,188,0.8)',
+                    pointColor: '#3b8bba',
+                    pointStrokeColor: 'rgba(60,141,188,1)',
+                    pointHighlightFill: '#fff',
+                    pointHighlightStroke: 'rgba(60,141,188,1)',
+                    data: []
+                }
+            ]
+        }
+        const ctx = document.getElementById('report-history-chart').getContext('2d');
+
+        return new Chart(ctx, {
+            type: 'line',
+            options: options,
+            data: chartData
+        });
+    }
+
+    const historicalChart = createHistoricalChart();
+
+    async function updateHistoricalChart() {
+        const response = await fetch("/api/stats/reports/historical");
+        if (response.ok) {
+            const body = await response.json();
+
+            const labels = body.map((data) => data['date']);
+            const values = body.map((data) => data['count']);
+
+            historicalChart.data.labels = labels
+            historicalChart.data.datasets[0].data = values
+            historicalChart.update();
+        }
+    }
+
+    function update() {
+        updateStats()
+            .then(updateSpammers)
+            .then(updateHistoricalChart)
+            .then(() => {
+                setTimeout(update, 10 * 60 * 1000);
+            })
+    }
+
+    update();
+</script>

data/lib/osa/views/layout.erb

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Outlook Spam Automator Admin Panel</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous">
+  <script src="https://code.jquery.com/jquery-3.5.1.min.js" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js" integrity="sha512-d9xgZrVZpmmQlfonhQUvTR7lMPtO7NkZMkA0ABN3PHCbKA5nqylQ/yWlFAyY6hYgdF1Qh6nYiuADWwKB4C2WSw==" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/admin-lte@3.0/dist/js/adminlte.min.js" crossorigin="anonymous"></script>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/admin-lte@3.0/dist/css/adminlte.min.css">
+<body class="sidebar-mini" style="height: auto;">
+<div class="wrapper">
+  <!-- Main Sidebar Container -->
+  <aside class="main-sidebar sidebar-dark-primary elevation-4">
+    <!-- Brand Logo -->
+    <a href="index.html" class="brand-link">
+      <span class="brand-text font-weight-light">Outlook Spam Automator</span>
+    </a>
+
+    <!-- Sidebar -->
+    <div class="sidebar">
+
+      <!-- Sidebar Menu -->
+      <nav class="mt-2">
+        <ul class="nav nav-pills nav-sidebar flex-column" data-widget="treeview" role="menu" data-accordion="false">
+          <li class="nav-item">
+            <a href="#" class="nav-link">
+              <i class="nav-icon fas fa-th"></i>
+              <p>Overview</p>
+            </a>
+          </li>
+        </ul>
+      </nav>
+      <!-- /.sidebar-menu -->
+    </div>
+    <!-- /.sidebar -->
+  </aside>
+
+  <%= yield %>
+
+  <!-- Main Footer -->
+  <footer class="main-footer">
+    <!-- To the right -->
+    <div class="float-right d-none d-sm-inline">
+      Anything you want
+    </div>
+    <!-- Default to the left -->
+    <strong>Copyright © 2014-2020 <a href="https://adminlte.io">AdminLTE.io</a>.</strong> All rights reserved.
+  </footer>
+  <div id="sidebar-overlay"></div></div>
+<!-- ./wrapper -->
+
+</body>
+
+</html>

data/osa.gemspec

@@ -26,4 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'public_suffix', '~> 4.0'
   spec.add_dependency 'sqlite3', '~> 1.4'
   spec.add_dependency 'tty-prompt', '~> 0.22'
+  spec.add_dependency 'sinatra', '~> 2.1.0'
+  spec.add_dependency 'sinatra-contrib', '~> 2.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osa
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Moray Baruh
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-02 00:00:00.000000000 Z
+date: 2021-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,6 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 description: Get rid of spam on your Outlook account
 email:
 - contact@moraybaruh.com
@@ -108,7 +136,9 @@ files:
 - lib/osa/migrations/00001_create_blacklists.rb
 - lib/osa/migrations/00002_create_email_providers.rb
 - lib/osa/migrations/00003_create_config.rb
+- lib/osa/migrations/00004_create_reports.rb
 - lib/osa/migrations/free-email-providers.txt
+- lib/osa/scripts/dashboard_server.rb
 - lib/osa/scripts/scan_junk_folder.rb
 - lib/osa/services/auth_service.rb
 - lib/osa/services/setup_service.rb
@@ -117,6 +147,8 @@ files:
 - lib/osa/util/db.rb
 - lib/osa/util/paginated.rb
 - lib/osa/version.rb
+- lib/osa/views/index.erb
+- lib/osa/views/layout.erb
 - osa.gemspec
 - release.sh
 - web/login.html
@@ -139,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Outlook Spam Automator