osa 0.1.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22c6dd2dc64261089df031e75fee841e1416276244674dce617792aacbb12fca
-  data.tar.gz: f0442134bc13835edcbacea2aa594fdc95202e47beed2e71c29f5214bbae2335
+  metadata.gz: b3bc9c2b204260bd0cecd1650fdbfca0676c53e6e44c59aef8926caf9a5f0246
+  data.tar.gz: f32d6566bc6195847fc12fe680f8c22e6cbfe91f587bc7a35937efb1582eaccc
 SHA512:
-  metadata.gz: ca001d8b232f9bfa8bcfde6e75be2a6e1a7b2d025254e22af2feec25625d979e3bd10a88325cee888442b5686b22c7f93fd1866225d7bb6ca38e27e9c62e05af
-  data.tar.gz: 7d96e933b2c4b09d335f825c933ba01197399aea947e7ca2fca3f66a851c7f469f046fd37abbb59548c54fbb37835eef3684540172cf01bcf30489f2106e6208
+  metadata.gz: 76462c72b811ecb6b2224725ec954aa3ff3c6d2fc99a4cc91ae66698d72de709c740683ce6c4690095853199c7ef67c836ddc0bf6c0d3c4eb33751ee0fec3058
+  data.tar.gz: f4e79f61caa68f2c1ecd646ffa3d127070addf334f827bb33066bfdefc1e44f8b2986b6983c120c94867fa907e65184fd679ef73ad0eaa6b1b34f49956d5393d

data/Gemfile.lock

@@ -1,42 +1,56 @@
 PATH
   remote: .
   specs:
-    osa (0.1.1)
+    osa (0.2.2)
       activerecord (~> 6.0)
       faraday (~> 1.1)
+      mail (~> 2.7.1)
       public_suffix (~> 4.0)
+      sinatra (~> 2.1.0)
+      sinatra-contrib (~> 2.1.0)
       sqlite3 (~> 1.4)
       tty-prompt (~> 0.22)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.0.3.4)
-      activesupport (= 6.0.3.4)
-    activerecord (6.0.3.4)
-      activemodel (= 6.0.3.4)
-      activesupport (= 6.0.3.4)
-    activesupport (6.0.3.4)
+    activemodel (6.1.3)
+      activesupport (= 6.1.3)
+    activerecord (6.1.3)
+      activemodel (= 6.1.3)
+      activesupport (= 6.1.3)
+    activesupport (6.1.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     ast (2.4.1)
-    concurrent-ruby (1.1.7)
-    faraday (1.1.0)
+    concurrent-ruby (1.1.8)
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords
-    i18n (1.8.5)
+    faraday-net_http (1.0.1)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.2)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    mini_mime (1.0.2)
+    minitest (5.14.4)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     parallel (1.20.0)
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pastel (0.8.0)
       tty-color (~> 0.5)
     public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
     rainbow (3.0.0)
     regexp_parser (1.8.2)
     rexml (3.2.4)
@@ -55,24 +69,35 @@ GEM
       rubocop (>= 0.90.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.10.1)
-    ruby2_keywords (0.0.2)
+    ruby2_keywords (0.0.4)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    sinatra-contrib (2.1.0)
+      multi_json
+      mustermann (~> 1.0)
+      rack-protection (= 2.1.0)
+      sinatra (= 2.1.0)
+      tilt (~> 2.0)
     sqlite3 (1.4.2)
-    thread_safe (0.3.6)
+    tilt (2.0.10)
     tty-color (0.6.0)
     tty-cursor (0.7.1)
-    tty-prompt (0.22.0)
+    tty-prompt (0.23.0)
       pastel (~> 0.8)
       tty-reader (~> 0.8)
-    tty-reader (0.8.0)
+    tty-reader (0.9.0)
       tty-cursor (~> 0.7)
       tty-screen (~> 0.8)
       wisper (~> 2.0)
     tty-screen (0.8.1)
-    tzinfo (1.2.8)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
     wisper (2.0.1)
-    zeitwerk (2.4.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -83,4 +108,4 @@ DEPENDENCIES
   rubocop-performance
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -4,17 +4,18 @@
 
 ### Basics
 
-OSA uses asks you to choose two folders one "junk" folder and one "report" folder. Even though you can choose any folder
-for both, it's recommend to choose the default junk folder as the junk folder and create a custom folder for as the report
-folder.
+OSA asks you to choose your junk folder on first configuration. After the folder is selected, the `scan-junk` command
+allows you to scan the folder to report and delete any unwanted spam. It's important to chose the actual junk folder
+so your Outlook spam filters does not get broken. However, OSA will work with any folder. The processing for each email
+uses the following rules:
 
-These two folders will be used the following way:
-- When the report folder is scanned, all emails are reported to [Spamcop](https://spamcop.net), deleted and the senders blacklisted.
-- When the junk folder is scanned, all emails from the blacklist are reported to [Spamcop](https://spamcop.net) and deleted.
+1. If the email is flagged, the email is reported to Spamcop, then deleted and the sender is blacklisted.
+2. If the email's sender is blacklisted, the email is reported to Spamcop, then deleted.
+3. Otherwise, the email is left untouched. 
 
-*OSA will not touch any folder beside these two.*
+*OSA will not touch any folder beside the folder you've chosen.*
 
-*It's the user's responsibility to move junk mails to the report folder to build up the blacklist.*
+*It's the user's responsibility to move junk mails to the junk folder and flag them to build up the blacklist.*
 
 ### The blacklist
 
@@ -24,6 +25,13 @@ be blacklisted. However, to prevent millions of users to go blacklisted because
 list of free email providers (which includes domains like gmail.com, outlook.com among others). If the sender uses a free
 email provider, the full address is blacklisted.
 
+OSA also supports Domain Name System Blacklist. In fact it comes bundled with 3 DNSBL:
+1. [Spamcop Blocking List](https://www.spamcop.net/fom-serve/cache/297.html)
+2. [Spamhaus Block List](https://www.spamhaus.org/sbl)
+3. [Passive Spam Block List](https://psbl.org)
+
+You can remove these or add more blacklists, from the database, after you configure OSA.
+
 ## Installation
 
 You can install OSA from RubyGems:
@@ -58,18 +66,21 @@ osa setup
 
 Each time you run this command, your previous configuration will be erased, except for your blacklist.
 
-Process your report folder:
+Process your junk folder:
 
 ```sh
-osa scan-report
+osa scan-junk
 ```
 
-Process your junk folder:
+OSA also provides you a nice administration dashboard you. You can access the dashboard by running
 
 ```sh
-osa scan-junk
+osa dashboard
 ```
 
+You are now able to access the dashboard on `http://localhost:8080`. You can also change the port of the server by
+providing the `SERVER_PORT` environment variable.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/moray95/osa.

data/exe/osa

@@ -12,9 +12,10 @@ when 'login'
   OSA::AuthService.login(OSA::Config.first || OSA::Config.new)
 when 'scan-junk'
   require 'osa/scripts/scan_junk_folder'
-when 'scan-report'
-  require 'osa/scripts/scan_report_folder'
+when 'dashboard'
+  require 'osa/scripts/dashboard_server'
+  DashboardServer.start!
 else
-  $stderr.puts "Usage: #{File.basename($0)} [setup|login|scan-junk|scan-report]"
+  $stderr.puts "Usage: #{File.basename($0)} [setup|login|scan-junk|dashboard]"
   exit 1
 end

data/lib/osa/clients/ms_graph_client.rb

@@ -6,6 +6,7 @@ require 'osa/util/paginated'
 require 'osa/clients/http_client'
 require 'active_support/core_ext/numeric/bytes'
 require 'active_support'
+require 'mail'
 
 module OSA
   class MSGraphClient
@@ -33,21 +34,30 @@ module OSA
     end
 
     def folders
-      Paginated.new(@authenticated.get('/v1.0/me/mailFolders'), self)
+      Paginated.new(@authenticated.get('/v1.0/me/mailFolders'), @authenticated)
     end
 
     def mails(folder_id)
-      Paginated.new(@authenticated.get("/v1.0/me/mailFolders/#{folder_id}/messages"), self)
+      Paginated.new(@authenticated.get("/v1.0/me/mailFolders/#{folder_id}/messages"), @authenticated)
     end
 
     def raw_mail(mail_id)
       @authenticated.get("/v1.0/me/messages/#{mail_id}/$value")
     end
 
+    def sender_ip(mail_id)
+      content = raw_mail(mail_id)
+      mail = Mail.new(content)
+      mail.header['x-sender-ip']
+    end
+
     def forward_mail_as_attachment(mail_id, to)
       raw_mail = self.raw_mail(mail_id)
       forward_message = create_forward_message(mail_id)
       update = {
+        body: {
+          content: ''
+        },
         toRecipients: [
           {
             emailAddress: {

data/lib/osa/migrations/00004_create_reports.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+require 'active_record/migration'
+
+class CreateReports < ActiveRecord::Migration[5.0]
+  def change
+    create_table :reports do |t|
+      t.text :sender, null: false
+      t.text :sender_domain, null: false
+      t.text :subject, null: false
+      t.boolean :flagged, null: false
+      t.boolean :blacklisted, null: false
+      t.datetime :received_at, null: false
+      t.datetime :reported_at, null: false
+    end
+  end
+end

data/lib/osa/migrations/00005_create_dns_blacklists.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+require 'active_record/migration'
+
+class CreateDnsBlacklists < ActiveRecord::Migration[5.0]
+  def change
+    create_table :dns_blacklists do |t|
+      t.text :name, null: false
+      t.text :server, null: false
+    end
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+        insert into dns_blacklists (name, server) values
+          ('spamcop', 'bl.spamcop.net'),
+          ('sbl', 'sbl.spamhaus.org'),
+          ('psbl', 'psbl.surriel.org');
+        SQL
+        add_column :reports, :blacklist, :string, null: true
+        execute <<~SQL
+          update reports set blacklist = 'db' where blacklisted = true;
+        SQL
+        remove_column :reports, :blacklisted
+      end
+
+      dir.down do
+        add_column :reports, :blacklisted, :boolean, default: false
+        execute <<~SQL
+          update reports set blacklisted = true where blacklist = 'db';
+        SQL
+        remove_column :reports, :blacklist
+      end
+    end
+  end
+end

data/lib/osa/scripts/dashboard_server.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+require 'sinatra'
+require 'sinatra/json'
+require 'osa/util/db'
+
+class DashboardServer < Sinatra::Base
+  set :views, File.absolute_path(File.dirname(__FILE__) + '/../views')
+  set :port, ENV['SERVER_PORT'] || 8080
+  after { ActiveRecord::Base.connection.close }
+
+  get '/' do
+    erb :index
+  end
+
+  get '/spammers/:spammer' do
+    erb :spammer
+  end
+
+  get '/api/stats/summary' do
+    blacklist_count = OSA::Blacklist.count
+    total_reported = OSA::Report.count
+    today = DateTime.now.to_date
+    today_reported = OSA::Report.where('reported_at > ?', today).count
+    week = DateTime.now - 1.week
+    week_reported = OSA::Report.where('reported_at > ?', week).count
+    month = DateTime.now - 30.days
+    month_reported = OSA::Report.where('reported_at > ?', month).count
+    mean_report_time = OSA::Report.select('avg(julianday(reported_at) - julianday(received_at)) * 86400.0 as avg').first['avg']
+
+    json blacklist_count: blacklist_count,
+         total_reported: total_reported,
+         today_reported: today_reported,
+         week_reported: week_reported,
+         month_reported: month_reported,
+         mean_report_time: mean_report_time
+
+  end
+
+  get '/api/stats/spammers/:spammer/summary' do |spammer|
+    reports = OSA::Report.where(sender: spammer).or(OSA::Report.where(sender_domain: spammer))
+    total_reported = reports.count
+    today = DateTime.now.to_date
+    today_reported = reports.where('reported_at > ?', today).count
+    week = DateTime.now - 1.week
+    week_reported = reports.where('reported_at > ?', week).count
+    month = DateTime.now - 30.days
+    month_reported = reports.where('reported_at > ?', month).count
+    domains = reports.select(:sender).distinct.map { |e| e.sender.split('@', 2)[1] }.uniq
+
+    json total_reported: total_reported,
+         today_reported: today_reported,
+         week_reported: week_reported,
+         month_reported: month_reported,
+         domains: domains
+  end
+
+  get '/api/stats/spammers' do
+    spammers = OSA::Report.select('sender_domain as domain', 'COUNT(*) as count')
+    unless params[:interval].blank?
+      spammers = spammers.where('received_at > ?', Time.now - params[:interval].to_i.days)
+    end
+    spammers = spammers.limit(50).order(count: :desc).group(:sender_domain)
+    json spammers
+  end
+
+  get '/api/stats/reports/historical' do
+    historical_data = OSA::Report.select("strftime('%Y-%m-%d', reported_at) as date", 'count(*) as count')
+    unless params[:spammer].blank?
+      historical_data = historical_data.where(sender: params[:spammer]).or(OSA::Report.where(sender_domain: params[:spammer]))
+    end
+    unless params[:interval].blank?
+      historical_data = historical_data.where('received_at > ?', Time.now - params[:interval].to_i.days)
+    end
+    json historical_data.group(:date)
+  end
+end

data/lib/osa/scripts/scan_junk_folder.rb

@@ -3,24 +3,54 @@ require 'osa/util/constants'
 require 'public_suffix'
 require 'osa/util/db'
 require 'osa/util/context'
+require 'public_suffix'
 
 context = OSA::Context.new
 
 continue = true
 
+dns_blacklists = OSA::DnsBlacklist.all
+
+def resolve_blacklist(mail_id, email_address, domain, context, dns_blacklists)
+  return 'db' if OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists?
+  ip = context.graph_client.sender_ip(mail_id)
+  if ip.nil?
+    puts "Couldn't detect ip for email from #{email_address}"
+    return nil
+  end
+  dns_blacklists.find { |bl| bl.blacklisted?(ip) }&.server
+end
+
+def extract_email_address(mail)
+  email_address = mail['sender']['emailAddress']['address']
+  return email_address unless email_address.nil?
+
+  # Sometimes the SMTP From header is misformatted and the email address is
+  # parsed as part of the name by Outlook. Try to extract the email address
+  # from the name.
+  sender_name = mail['sender']['emailAddress']['name']
+  return nil if sender_name.nil?
+
+  sender_name.scan(/<(.+@.+)>/).first&.first
+end
+
 while continue
   mails = context.graph_client.mails(context.config.junk_folder_id)
   continue = false
   loop do
     break if mails.nil?
     mails['value'].each do |mail|
-      email_address = mail['sender']['emailAddress']['address']
+      email_address = extract_email_address(mail)
       next if email_address.nil?
       domain = PublicSuffix.domain(email_address.split('@', 2)[1])
 
-      blacklisted = OSA::Blacklist.where(value: email_address).or(OSA::Blacklist.where(value: domain)).exists?
-      if blacklisted
-        puts "#{email_address} is blacklisted, reporting and deleting"
+      flagged = mail['flag']['flagStatus'] == 'flagged'
+      blacklist = (resolve_blacklist(mail['id'], email_address, domain, context, dns_blacklists) unless flagged)
+
+      if flagged
+        puts "Email from #{email_address} is flagged, reporting and deleting"
+      elsif !blacklist.nil?
+        puts "#{email_address} is blacklisted by #{blacklist}, reporting and deleting"
       else
         puts "Skipping mail from #{email_address}, its not blacklisted"
         next
@@ -32,7 +62,29 @@ while continue
       context.graph_client.forward_mail_as_attachment(mail['id'], context.config.spamcop_report_email)
       puts "deleting spam from #{email_address}"
       context.graph_client.delete_mail(mail['id'])
+
+      OSA::Report.create!(sender: email_address,
+                          sender_domain: domain,
+                          subject: mail['subject'],
+                          flagged: flagged,
+                          blacklist: blacklist,
+                          received_at: Time.iso8601(mail['receivedDateTime']),
+                          reported_at: Time.now)
+
+      # Do not add to the blacklist if the it's blacklisted by the db (it's already present)
+      # or blacklisted by DNSBLs (these blacklists are only supposed to be temporary).
+      if flagged
+        is_free_provider = OSA::EmailProvider.where(value: domain).exists?
+        if is_free_provider
+          puts "#{email_address} is using a free provider, blacklisting full address"
+          OSA::Blacklist.find_or_create_by(value: email_address).save!
+        else
+          puts "Adding #{domain} to blacklist"
+          OSA::Blacklist.find_or_create_by(value: domain).save!
+        end
+      end
     end
+
     mails = mails.next
   end
 end

data/lib/osa/services/setup_service.rb

@@ -18,10 +18,9 @@ module OSA
       }.to_h
       prompt = TTY::Prompt.new
       junk_folder_id = prompt.select('Select junk folder:', choices)
-      report_folder_id = prompt.select('Select report folder:', choices)
       spamcop_email = prompt.ask('Spamcop report email:') { |q| q.validate :email }
 
-      config.update! junk_folder_id: junk_folder_id, report_folder_id: report_folder_id, spamcop_report_email: spamcop_email
+      config.update! junk_folder_id: junk_folder_id, spamcop_report_email: spamcop_email
     end
   end
 end

data/lib/osa/util/db.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'active_record'
+require 'resolv'
 
 ActiveRecord::Base.establish_connection(
   adapter: 'sqlite3',
@@ -18,4 +19,15 @@ module OSA
 
   class EmailProvider < ActiveRecord::Base
   end
+
+  class Report < ActiveRecord::Base
+  end
+
+  class DnsBlacklist < ActiveRecord::Base
+    def blacklisted?(ip)
+      ::Resolv.getaddress("#{ip}.#{server}") != '0.0.0.0'
+    rescue ::Resolv::ResolvError
+      return false
+    end
+  end
 end

data/lib/osa/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module OSA
-  VERSION = '0.1.1'
+  VERSION = '0.2.2'
 end

data/lib/osa/views/index.erb

@@ -0,0 +1,320 @@
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper" style="min-height: 1233px;">
+  <!-- Content Header (Page header) -->
+  <div class="content-header">
+    <div class="container-fluid">
+      <div class="row mb-2">
+        <div class="col-sm-6">
+          <h1 class="m-0">Overview</h1>
+        </div><!-- /.col -->
+      </div><!-- /.row -->
+    </div><!-- /.container-fluid -->
+  </div>
+  <!-- /.content-header -->
+
+  <!-- Main content -->
+  <div class="content">
+    <div class="container-fluid">
+      <div class="row">
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-info">
+            <div class="inner">
+              <h3 id="blacklist-count"></h3>
+              <p>entries in blacklist</p>
+            </div>
+          </div>
+        </div>
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-info">
+            <div class="inner">
+              <h3 id="reporting-time"></h3>
+              <p>average reporting time</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="today-count"></h3>
+              <p>emails reported today</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="week-count"></h3>
+              <p>emails reported in the last 7 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="month-count"></h3>
+              <p>emails reported in the last 30 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-warning">
+            <div class="inner">
+              <h3 id="total-count"></h3>
+              <p>emails reported in total</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+      </div>
+
+      <div class="row">
+        <div id="interval-btn-group"  class="btn-group">
+          <button onclick="updateInterval(this, '')" type="button" class="btn btn-info active">All time</button>
+          <button onclick="updateInterval(this, '1')" type="button" class="btn btn-info">1 day</button>
+          <button onclick="updateInterval(this, '7')" type="button" class="btn btn-info">7 days</button>
+          <button onclick="updateInterval(this, '30')" type="button" class="btn btn-info">30 days</button>
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Spams reported</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body">
+            <div class="chart">
+              <div class="chartjs-size-monitor">
+                <div class="chartjs-size-monitor-expand">
+                  <div class=""></div>
+                </div>
+                <div class="chartjs-size-monitor-shrink">
+                  <div class=""></div>
+                </div>
+              </div>
+              <canvas id="report-history-chart" style="min-height: 250px; height: 250px; max-height: 250px; max-width: 100%; display: block; width: 1732px;" width="1732" height="250" class="chartjs-render-monitor"></canvas>
+            </div>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Top spammers</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body p-0">
+            <table class="table table-striped">
+              <thead>
+              <tr>
+                <th style="width: 10px">#</th>
+                <th>Domain</th>
+                <th>Emails reported</th>
+                <th>Spam ratio</th>
+                <th style="width: 40px"></th>
+              </tr>
+              </thead>
+              <tbody id="top-spammers-table-body">
+              </tbody>
+            </table>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+
+    </div> <!-- /.container-fluid -->
+
+
+  </div>
+  <!-- /.content -->
+</div>
+<!-- /.content-wrapper -->
+
+<script>
+    let totalReportCount = 0;
+    let interval = "";
+
+    function updateInterval(element, newInterval) {
+        interval = newInterval;
+        let children = Array.from(document.getElementById("interval-btn-group").children);
+        console.log(children);
+        children.forEach((child) => {
+            if (child === element) {
+                child.classList.add("active");
+            } else {
+                child.classList.remove("active");
+            }
+        });
+        update();
+    }
+
+    function sec2time(timeInSeconds) {
+        const time = parseFloat(timeInSeconds).toFixed(3);
+        const hours = Math.floor(time / 60 / 60);
+        const minutes = Math.floor(time / 60) % 60;
+
+        const units = [{value: hours, unit: 'h'}, {value: minutes, unit: 'm'}].filter((value) => value.value > 0 );
+
+        return units.map((value) => `${value.value}${value.unit}`).join(' ');
+    }
+
+    async function updateStats() {
+        const response = await fetch('/api/stats/summary');
+        if (response.ok) {
+            const body = await response.json();
+            document.getElementById('blacklist-count').innerText = body['blacklist_count'];
+            document.getElementById('today-count').innerText = body['today_reported'];
+            document.getElementById('week-count').innerText = body['week_reported'];
+            document.getElementById('month-count').innerText = body['month_reported'];
+            document.getElementById('total-count').innerText = body['total_reported'];
+            const meanReportTime = body['mean_report_time'];
+            if (meanReportTime !== null) {
+                document.getElementById('reporting-time').innerText = sec2time(meanReportTime);
+            } else {
+                document.getElementById('reporting-time').innerText = 'N/A';
+            }
+            totalReportCount = body['total_reported'];
+        }
+    }
+
+    function createProgressDiv(fillRatio) {
+        const progressDiv = document.createElement("div");
+        progressDiv.className = "progress progress-xs";
+
+        const progressBarDiv = document.createElement("div");
+        progressBarDiv.className = "progress-bar progress-bar-danger";
+        progressBarDiv.style.width = `${fillRatio}%`;
+
+        progressDiv.appendChild(progressBarDiv);
+        return progressDiv;
+    }
+
+    async function updateSpammers() {
+        const response = await fetch(`/api/stats/spammers?interval=${interval}`);
+        if (response.ok) {
+            const body = await response.json();
+
+            const table = body.map((spammer, index) => {
+                const tr = document.createElement("tr");
+
+                const indexTd = document.createElement("td");
+                indexTd.innerText = index + 1;
+                tr.appendChild(indexTd);
+
+                const domainTd = document.createElement("td");
+                const domainRef = document.createElement("a");
+                domainRef.setAttribute("href", `/spammers/${encodeURI(spammer['domain'])}`);
+                domainRef.innerText = spammer['domain']
+                domainTd.appendChild(domainRef);
+                tr.appendChild(domainTd);
+
+                const reportCountTd = document.createElement("td");
+                reportCountTd.innerText = spammer['count'];
+                tr.appendChild(reportCountTd);
+
+                const spamRatio = spammer['count'] / totalReportCount * 100;
+
+                const spamRatioProgressTd = document.createElement("td");
+                spamRatioProgressTd.append(createProgressDiv(spamRatio));
+                tr.appendChild(spamRatioProgressTd);
+
+                const spamRatioTd = document.createElement("td");
+                spamRatioTd.innerText = `${spamRatio.toFixed(1)}%`;
+                tr.appendChild(spamRatioTd);
+
+                return tr;
+            });
+            const tableBody = document.getElementById("top-spammers-table-body")
+            tableBody.innerHTML = "";
+            table.forEach((row) => {
+                tableBody.appendChild(row);
+            });
+
+        }
+    }
+
+    function createHistoricalChart() {
+        const options = {
+            maintainAspectRatio: false,
+            responsive: true,
+            legend: {
+                display: false
+            },
+            scales: {
+                xAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }],
+                yAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }]
+            }
+        }
+
+        const chartData = {
+            labels: [],
+            datasets: [
+                {
+                    label: 'Spams reported',
+                    backgroundColor: 'rgba(60,141,188,0.9)',
+                    borderColor: 'rgba(60,141,188,0.8)',
+                    pointColor: '#3b8bba',
+                    pointStrokeColor: 'rgba(60,141,188,1)',
+                    pointHighlightFill: '#fff',
+                    pointHighlightStroke: 'rgba(60,141,188,1)',
+                    data: []
+                }
+            ]
+        }
+        const ctx = document.getElementById('report-history-chart').getContext('2d');
+
+        return new Chart(ctx, {
+            type: 'line',
+            options: options,
+            data: chartData
+        });
+    }
+
+    const historicalChart = createHistoricalChart();
+
+    async function updateHistoricalChart() {
+        const response = await fetch(`/api/stats/reports/historical?interval=${interval}`);
+        if (response.ok) {
+            const body = await response.json();
+
+            const labels = body.map((data) => data['date']);
+            const values = body.map((data) => data['count']);
+
+            historicalChart.data.labels = labels
+            historicalChart.data.datasets[0].data = values
+            historicalChart.update();
+        }
+    }
+
+    function update() {
+        updateStats()
+          .then(updateSpammers)
+          .then(updateHistoricalChart)
+          .then(() => {
+              setTimeout(update, 10 * 60 * 1000);
+          })
+    }
+
+    update();
+</script>

data/lib/osa/views/layout.erb

@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Outlook Spam Automator Admin Panel</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous">
+  <script src="https://code.jquery.com/jquery-3.5.1.min.js" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js" integrity="sha512-d9xgZrVZpmmQlfonhQUvTR7lMPtO7NkZMkA0ABN3PHCbKA5nqylQ/yWlFAyY6hYgdF1Qh6nYiuADWwKB4C2WSw==" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/admin-lte@3.0/dist/js/adminlte.min.js" crossorigin="anonymous"></script>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/admin-lte@3.0/dist/css/adminlte.min.css">
+<body class="sidebar-mini" style="height: auto;">
+<div class="wrapper">
+  <!-- Main Sidebar Container -->
+  <aside class="main-sidebar sidebar-dark-primary elevation-4">
+    <!-- Brand Logo -->
+    <a href="/" class="brand-link">
+      <span class="brand-text font-weight-light">Outlook Spam Automator</span>
+    </a>
+
+    <!-- Sidebar -->
+    <div class="sidebar">
+
+      <!-- Sidebar Menu -->
+      <nav class="mt-2">
+        <ul class="nav nav-pills nav-sidebar flex-column" data-widget="treeview" role="menu" data-accordion="false">
+          <li class="nav-item">
+            <a href="/" class="nav-link">
+              <p>Overview</p>
+            </a>
+          </li>
+        </ul>
+      </nav>
+      <!-- /.sidebar-menu -->
+    </div>
+    <!-- /.sidebar -->
+  </aside>
+
+  <%= yield %>
+
+  <div id="sidebar-overlay"></div>
+</div>
+<!-- ./wrapper -->
+
+</body>
+
+</html>

data/lib/osa/views/spammer.erb

@@ -0,0 +1,209 @@
+<!-- Content Wrapper. Contains page content -->
+<div class="content-wrapper" style="min-height: 1233px;">
+  <!-- Content Header (Page header) -->
+  <div class="content-header">
+    <div class="container-fluid">
+      <div class="row mb-2">
+        <div class="col-sm-6">
+          <h1 id="title" class="m-0">Spammer Details</h1>
+        </div><!-- /.col -->
+      </div><!-- /.row -->
+    </div><!-- /.container-fluid -->
+  </div>
+  <!-- /.content-header -->
+
+  <!-- Main content -->
+  <div class="content">
+    <div class="container-fluid">
+      <div class="row">
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="today-count"></h3>
+              <p>emails reported today</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="week-count"></h3>
+              <p>emails reported in the last 7 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-success">
+            <div class="inner">
+              <h3 id="month-count"></h3>
+              <p>emails reported in the last 30 days</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+        <div class="col-lg">
+          <!-- small card -->
+          <div class="small-box bg-warning">
+            <div class="inner">
+              <h3 id="total-count"></h3>
+              <p>emails reported in total</p>
+            </div>
+          </div>
+        </div>
+        <!-- ./col -->
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Spams reported</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body">
+            <div class="chart">
+              <div class="chartjs-size-monitor">
+                <div class="chartjs-size-monitor-expand">
+                  <div class=""></div>
+                </div>
+                <div class="chartjs-size-monitor-shrink">
+                  <div class=""></div>
+                </div>
+              </div>
+              <canvas id="report-history-chart" style="min-height: 250px; height: 250px; max-height: 250px; max-width: 100%; display: block; width: 1732px;" width="1732" height="250" class="chartjs-render-monitor"></canvas>
+            </div>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="card">
+          <div class="card-header">
+            <h3 class="card-title">Domains</h3>
+          </div>
+          <!-- /.card-header -->
+          <div class="card-body p-0">
+            <table class="table table-striped">
+              <thead>
+              </thead>
+              <tbody id="domains-table-body">
+              </tbody>
+            </table>
+          </div>
+          <!-- /.card-body -->
+        </div>
+      </div>
+
+
+    </div> <!-- /.container-fluid -->
+    </div>
+  </div>
+  <!-- /.content -->
+</div>
+<!-- /.content-wrapper -->
+
+<script>
+    const pathComponents = window.location.pathname.split('/');
+    const spammer = pathComponents[pathComponents.length - 1];
+    document.getElementById("title").innerText = `${spammer} Details`;
+
+    async function updateStats() {
+        const response = await fetch(`/api/stats/spammers/${spammer}/summary`);
+        if (response.ok) {
+            const body = await response.json();
+            document.getElementById('today-count').innerText = body['today_reported'];
+            document.getElementById('week-count').innerText = body['week_reported'];
+            document.getElementById('month-count').innerText = body['month_reported'];
+            document.getElementById('total-count').innerText = body['total_reported'];
+            totalReportCount = body['total_reported'];
+
+            const tableBody = document.getElementById("domains-table-body");
+            tableBody.innerHTML = "";
+
+            body['domains'].forEach((domain) => {
+                const tr = document.createElement("tr");
+                const td = document.createElement("td");
+                td.innerText = domain;
+                tr.appendChild(td);
+                tableBody.appendChild(tr);
+            });
+        }
+    }
+
+    function createHistoricalChart() {
+        const options = {
+            maintainAspectRatio: false,
+            responsive: true,
+            legend: {
+                display: false
+            },
+            scales: {
+                xAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }],
+                yAxes: [{
+                    gridLines: {
+                        display: false,
+                    }
+                }]
+            }
+        }
+
+        const chartData = {
+            labels: [],
+            datasets: [
+                {
+                    label: 'Spams reported',
+                    backgroundColor: 'rgba(60,141,188,0.9)',
+                    borderColor: 'rgba(60,141,188,0.8)',
+                    pointColor: '#3b8bba',
+                    pointStrokeColor: 'rgba(60,141,188,1)',
+                    pointHighlightFill: '#fff',
+                    pointHighlightStroke: 'rgba(60,141,188,1)',
+                    data: []
+                }
+            ]
+        }
+        const ctx = document.getElementById('report-history-chart').getContext('2d');
+
+        return new Chart(ctx, {
+            type: 'line',
+            options: options,
+            data: chartData
+        });
+    }
+
+    const historicalChart = createHistoricalChart();
+
+    async function updateHistoricalChart() {
+        const response = await fetch(`/api/stats/reports/historical?spammer=${spammer}`);
+        if (response.ok) {
+            const body = await response.json();
+
+            const labels = body.map((data) => data['date']);
+            const values = body.map((data) => data['count']);
+
+            historicalChart.data.labels = labels
+            historicalChart.data.datasets[0].data = values
+            historicalChart.update();
+        }
+    }
+
+    function update() {
+        updateStats()
+            .then(updateHistoricalChart)
+            .then(() => {
+                setTimeout(update, 10 * 60 * 1000);
+            })
+    }
+
+    update();
+</script>

data/osa.gemspec

@@ -26,4 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'public_suffix', '~> 4.0'
   spec.add_dependency 'sqlite3', '~> 1.4'
   spec.add_dependency 'tty-prompt', '~> 0.22'
+  spec.add_dependency 'sinatra', '~> 2.1.0'
+  spec.add_dependency 'sinatra-contrib', '~> 2.1.0'
+  spec.add_dependency 'mail', '~> 2.7.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osa
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Moray Baruh
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-28 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,6 +80,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 description: Get rid of spam on your Outlook account
 email:
 - contact@moraybaruh.com
@@ -108,9 +150,11 @@ files:
 - lib/osa/migrations/00001_create_blacklists.rb
 - lib/osa/migrations/00002_create_email_providers.rb
 - lib/osa/migrations/00003_create_config.rb
+- lib/osa/migrations/00004_create_reports.rb
+- lib/osa/migrations/00005_create_dns_blacklists.rb
 - lib/osa/migrations/free-email-providers.txt
+- lib/osa/scripts/dashboard_server.rb
 - lib/osa/scripts/scan_junk_folder.rb
-- lib/osa/scripts/scan_report_folder.rb
 - lib/osa/services/auth_service.rb
 - lib/osa/services/setup_service.rb
 - lib/osa/util/constants.rb
@@ -118,6 +162,9 @@ files:
 - lib/osa/util/db.rb
 - lib/osa/util/paginated.rb
 - lib/osa/version.rb
+- lib/osa/views/index.erb
+- lib/osa/views/layout.erb
+- lib/osa/views/spammer.erb
 - osa.gemspec
 - release.sh
 - web/login.html
@@ -140,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Outlook Spam Automator

data/lib/osa/scripts/scan_report_folder.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-require 'uri'
-require 'public_suffix'
-require 'osa/util/db'
-require 'osa/util/context'
-
-context = OSA::Context.new
-
-loop do
-  mails = context.graph_client.mails(context.config.report_folder_id)
-  break if mails['value'].empty?
-  mails['value'].each do |mail|
-    email_address = mail['sender']['emailAddress']['address']
-
-    puts "forwarding spam from #{email_address}"
-    context.graph_client.forward_mail_as_attachment(mail['id'], context.config.spamcop_report_email)
-    puts "deleting spam from #{email_address}"
-    context.graph_client.delete_mail(mail['id'])
-
-    next if email_address.nil?
-    domain = PublicSuffix.domain(email_address.split('@', 2)[1])
-
-    is_free_provider = OSA::EmailProvider.where(value: domain).exists?
-    if is_free_provider
-      puts "#{email_address} is using a free provider, blacklisting full address"
-      OSA::Blacklist.find_or_create_by(value: email_address).save!
-    else
-      puts "Adding #{domain} to blacklist"
-      OSA::Blacklist.find_or_create_by(value: domain).save!
-    end
-  end
-end