kimurai-dashboard 0.1.0

data/lib/kimurai/dashboard/models/run.rb

@@ -0,0 +1,74 @@
+module Kimurai::Dashboard
+  class Run < Sequel::Model(DB)
+    many_to_one :session
+    many_to_one :spider
+
+    plugin :json_serializer
+    plugin :serialization
+    plugin :serialization_modification_detection
+    serialize_attributes :json, :visits, :items, :server, :events
+
+    # scopes
+    dataset_module do
+      def spider_id(id)
+        filter(spider_id: id)
+      end
+
+      def session_id(id)
+        filter(session_id: id)
+      end
+    end
+
+    def latest?
+      Run.where(spider_name: spider_name).last&.id == id
+    end
+
+    def log_file
+      if latest?
+        session ? { available: true } : { available: false, reason: "run is not in session" }
+      else
+        { available: false, reason: "run is not latest" }
+      end
+    end
+
+    def difference_between_previous_run
+      previous_run = Run.where(spider_name: spider_name).reverse_order(:id).first(Sequel[:id] < id)
+      return unless previous_run
+
+      {
+        visits: {
+          requests: {
+            current: visits["requests"],
+            previous: previous_run.visits["requests"],
+            difference: calculate_difference(visits["requests"], previous_run.visits["requests"])
+          },
+          responses: {
+            current: visits["responses"],
+            previous: previous_run.visits["responses"],
+            difference: calculate_difference(visits["responses"], previous_run.visits["responses"])
+          }
+        },
+        items: {
+          sent: {
+            current: items["sent"],
+            previous: previous_run.items["sent"],
+            difference: calculate_difference(items["sent"], previous_run.items["sent"])
+          },
+          processed: {
+            current: items["processed"],
+            previous: previous_run.items["processed"],
+            difference: calculate_difference(items["processed"], previous_run.items["processed"])
+          }
+        },
+        previous_run_id: previous_run.id
+      }
+    end
+
+    private
+
+    def calculate_difference(current, previous)
+      return if current == 0 || previous == 0
+      (((current - previous).to_r / previous) * 100).to_f.round(1)
+    end
+  end
+end

data/lib/kimurai/dashboard/models/session.rb

@@ -0,0 +1,37 @@
+module Kimurai::Dashboard
+  class Session < Sequel::Model(DB)
+    one_to_many :runs
+
+    plugin :json_serializer
+    plugin :serialization
+    plugin :serialization_modification_detection
+    serialize_attributes :json, :spiders
+
+    unrestrict_primary_key
+
+    def total_time
+      (stop_time ? stop_time - start_time : Time.now - start_time).round(3)
+    end
+
+    def processing?
+      status == "processing"
+    end
+
+    def spiders_in_queue
+      return [] unless processing?
+      spiders - runs_dataset.select_map(:spider_name)
+    end
+
+    def running_runs
+      runs_dataset.where(status: "running").all
+    end
+
+    def failed_runs
+      runs_dataset.where(status: "failed").all
+    end
+
+    def completed_runs
+      runs_dataset.where(status: "completed").all
+    end
+  end
+end

data/lib/kimurai/dashboard/models/spider.rb

@@ -0,0 +1,20 @@
+module Kimurai::Dashboard
+  class Spider < Sequel::Model(DB)
+    one_to_many :runs
+    plugin :json_serializer
+
+    def current_session
+      session = Session.find(status: "processing")
+      return unless session
+      session if session.spiders.include?(name)
+    end
+
+    def running?
+      runs_dataset.first(status: "running") ? true : false
+    end
+
+    def current_state
+      running? ? "running" : "stopped"
+    end
+  end
+end

data/lib/kimurai/dashboard/public/application.css

@@ -0,0 +1,61 @@
+.alert-row {
+  padding-top: 1rem;
+}
+
+.footer {
+  position: absolute;
+  bottom: 0;
+  width: 100%;
+  height: 4rem; /* Set the fixed height of the footer here */
+  line-height: 4rem; /* Vertically center the text there */
+  background-color: #f5f5f5;
+  text-align: center;
+}
+html {
+  position: relative;
+  min-height: 100%;
+}
+body {
+  margin-bottom: 6rem;
+}
+
+main {
+  margin-top: 1.7rem;
+}
+
+.dotted {
+  text-decoration:underline;
+  text-decoration-style: dotted;
+  border: none;
+}
+
+.gray {
+  color: #6c757d;
+}
+
+.fancy-hr {
+  overflow: hidden;
+  text-align: center;
+  color: #6c757d;
+}
+
+.fancy-hr:before,
+.fancy-hr:after {
+  background-color: #dee2e6;
+  content: "";
+  display: inline-block;
+  height: 1px;
+  position: relative;
+  vertical-align: middle;
+  width: 50%;
+}
+
+.fancy-hr:before {
+  right: 0.5em;
+  margin-left: -50%;
+}
+
+.fancy-hr:after {
+  left: 0.5em;
+  margin-right: -50%;
+}

data/lib/kimurai/dashboard/public/application.js

@@ -0,0 +1,4 @@
+$(document).ready(function() {
+  let rootResourse = location.pathname.replace(/(^\/.*?)\/(.*)/, '$1')
+  $(`a[href="${rootResourse}"]`).closest('li').addClass('active');
+});

data/lib/kimurai/dashboard/runner.rb

@@ -0,0 +1,31 @@
+require 'kimurai/runner'
+
+module Kimurai
+  class Runner
+    alias_method :original_run!, :run!
+    def run!(exception_on_fail: true)
+      register_session(session_info)
+      _, error = original_run!(exception_on_fail: false)
+      update_session(session_info)
+
+      if error
+        exception_on_fail ? raise(error) : [session_info, error]
+      else
+        session_info
+      end
+    end
+
+    private
+
+    def register_session(session_info)
+      Dashboard::Session.create(session_info)
+    end
+
+    def update_session(session_info)
+      session = Dashboard::Session.find(session_info[:id]).first
+      session.set(session_info)
+      session.save
+    end
+  end
+end
+

data/lib/kimurai/dashboard/version.rb

@@ -0,0 +1,5 @@
+module Kimurai
+  module Dashboard
+    VERSION = "0.1.0"
+  end
+end

data/lib/kimurai/dashboard/views/_header.erb

@@ -0,0 +1,19 @@
+<nav class="navbar navbar-expand-lg navbar-light bg-light rounded" role="navigation">
+  <ul class="navbar-nav mr-auto">
+    <li class="nav-item">
+      <a class="nav-link" href="/spiders">Spiders</a>
+    </li>
+    <li class="nav-item">
+      <a class="nav-link" href="/runs">Runs</a>
+    </li>
+    <li class="nav-item">
+      <a class="nav-link" href="/sessions">Sessions</a>
+    </li>
+  </ul>
+  <ul class="navbar-nav ml-auto">
+    <li class="nav-item">
+      Environment: <%= Kimurai.env %>
+      <!-- <a class="nav-link" href="#" data-target="#login" data-toggle="modal"></a> -->
+    </li>
+  </ul>
+</nav>

data/lib/kimurai/dashboard/views/layout.erb

@@ -0,0 +1,29 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <title>Kimurai Dashboard</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+    <link rel="stylesheet" href="/application.css?<%= Time.now.to_i %>">
+  </head>
+  <body>
+    <header class="container">
+      <%= erb :'_header' %>
+    </header>
+
+    <main class="container" role="main">
+      <%= yield %>
+    </main>
+
+    <footer class="footer" class="container" role="contentinfo">
+      <span class="text-muted">Kimurai Framework Dashboard 2018</span>
+    </footer>
+
+    <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+    <script src="/application.js"></script>
+  </body>
+</html>

data/lib/kimurai/dashboard/views/runs/_errors.erb

@@ -0,0 +1,18 @@
+<!-- <div class="col-lg-12"> -->
+  <table class="table table-hover table-sm">
+    <thead>
+      <tr>
+        <th scope="col">Message</th>
+        <th scope="col">Count</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% errors.each do |error, count| %>
+        <tr>
+          <td><%= h(error) %></td>
+          <td><%= count %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<!-- </div> -->

data/lib/kimurai/dashboard/views/runs/_table.erb

@@ -0,0 +1,38 @@
+<div class="table-responsive">
+  <table class="table table-hover table-sm">
+    <thead>
+      <tr>
+        <th scope="col"># id</th>
+        <th scope="col">Spider</th>
+        <th scope="col">Status</th>
+        <th scope="col">Start time</th>
+        <th scope="col">Stop time</th>
+        <th scope="col">Running</th>
+        <th scope="col">Session</th>
+        <th scope="col" data-toggle="tooltip" title="Requests / Responses">Visits*</th>
+        <th scope="col" data-toggle="tooltip" title="Sent / Processed">Items*</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% runs.each do |run| %>
+        <tr>
+          <th scope="row">
+            <a href="/runs/<%= run.id %>"><%= run.id %></a>
+          </th>
+          <td>
+            <a href="/spiders/<%= run.spider_id %>"><%= run.spider_name %></a>
+          </td>
+          <td><%= get_badge(run.status) %></td>
+          <td><%= run.start_time %></td>
+          <td><%= run.stop_time %></td>
+          <td><%= run.running_time.duration %></td>
+          <td>
+            <a href="/sessions/<%= run.session_id %>"><%= run.session_id %></a>
+          </td>
+          <td><%= minimize_stats(run.visits).join("/") %></td>
+          <td><%= minimize_stats(run.items).join("/") %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/lib/kimurai/dashboard/views/runs/index.erb

@@ -0,0 +1,11 @@
+<%= breadcrumbs("/" => "Home", "" => "Runs") %>
+
+<div class="row">
+  <div class="col-lg-12">
+    <h1>Runs</h1>
+    <%= render_filters(filters) if filters.any? %>
+    <%= erb :'runs/_table', locals: { runs: @runs } %>
+
+    <%= pagy_nav_bootstrap(@pagy) if @pagy && @pagy.pages > 1 %>
+  </div>
+</div>

data/lib/kimurai/dashboard/views/runs/show.erb

@@ -0,0 +1,142 @@
+<%= breadcrumbs("/" => "Home", "/runs" => "Runs", "" => @run.id) %>
+
+<div class="row">
+  <div class="col-lg-12">
+    <h1>Run #<%= @run.id %></h1>
+
+    <table class="table table-bordered">
+      <tbody>
+        <tr>
+          <th>ID</th>
+          <td><%= @run.id %></td>
+        </tr>
+        <tr>
+          <th>Spider</th>
+          <td>
+            <strong><a href="/spiders/<%= @run.spider_id %>"><%= @run.spider_name %></a></strong>
+          </td>
+        </tr>
+        <tr>
+          <th>Session</th>
+          <td><a href="/sessions/<%= @run.session_id %>"><%= @run.session_id %></a></td>
+        </tr>
+        <tr>
+          <th>Status</th>
+          <td>
+            <%= get_badge(@run.status) %>
+            <% if error = @run.error %>
+              <i>(<%= h(error) %>)</i>
+            <% end %>
+          </td>
+        </tr>
+        <tr>
+          <th>Start time</th>
+          <td><%= @run.start_time %></td>
+        </tr>
+        <tr>
+          <th>Stop time</th>
+          <td><%= @run.stop_time %></td>
+        </tr>
+        <tr>
+          <th>Running time</th>
+          <td><%= @run.running_time.duration %></td>
+        </tr>
+        <tr>
+          <th>Environment</th>
+          <td><%= @run.environment %></td>
+        </tr>
+        <tr>
+          <th>Log file</th>
+          <td>
+            <% if @run.log_file[:available] == false %>
+              Not available (<%= @run.log_file[:reason] %>)
+            <% else %>
+              <a target="_blank" href="/runs/<%= @run.id %>/log"><%= @run.spider_name %>.log</a>
+            <% end %>
+          </td>
+        </tr>
+        <tr>
+          <th>Server hostname</th>
+          <td><%= @run.server["hostname"] %></td>
+        </tr>
+        <tr>
+          <th>Server IPV4</th>
+          <td><%= @run.server["ipv4"] %></td>
+        </tr>
+        <tr>
+          <th>Process pid</th>
+          <td><%= @run.server["process_pid"] %></td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+  <div class="col-lg-12">
+    <h2>Visits</h2>
+    <table class="table table-bordered table-sm">
+      <tbody>
+        <tr>
+          <th>Requests</th>
+          <td>
+            <strong><%= @run.visits["requests"] %></strong>
+            <% if difference %>
+              <i><%= format_difference(difference[:visits][:requests][:previous], difference[:visits][:requests][:difference], difference[:previous_run_id]) %></i>
+            <% end %>
+          </td>
+        </tr>
+        <tr>
+          <th>Responses</th>
+          <td>
+            <strong><%= @run.visits["responses"] %></strong>
+            <% if difference %>
+              <i><%= format_difference(difference[:visits][:responses][:previous], difference[:visits][:responses][:difference], difference[:previous_run_id]) %></i>
+            <% end %>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+
+  <div class="col-lg-12">
+    <h2>Items</h2>
+    <table class="table table-bordered table-sm">
+      <tbody>
+        <tr>
+          <th>Sent</th>
+          <td>
+            <strong><%= @run.items["sent"] %></strong>
+            <% if difference %>
+              <i><%= format_difference(difference[:items][:sent][:previous], difference[:items][:sent][:difference], difference[:previous_run_id]) %></i>
+            <% end %>
+          </td>
+        </tr>
+        <tr>
+          <th>Processed</th>
+          <td>
+            <strong><%= @run.items["processed"] %></strong>
+            <% if difference %>
+              <i><%= format_difference(difference[:items][:processed][:previous], difference[:items][:processed][:difference], difference[:previous_run_id]) %></i>
+            <% end %>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+
+  <div class="col-lg-12">
+    <h2>Events</h2>
+    <% if @run.events["requests_errors"]&.any? %>
+      <p class="fancy-hr" id="running">Requests Errors (<%= errors_count(@run.events["requests_errors"]) %>)</p>
+      <%= erb :'runs/_errors', locals: { errors: @run.events["requests_errors"] } %>
+    <% end %>
+
+    <% if @run.events["drop_items_errors"]&.any? %>
+      <p class="fancy-hr" id="running">Drop Items Errors (<%= errors_count(@run.events["drop_items_errors"]) %>)</p>
+      <%= erb :'runs/_errors', locals: { errors: @run.events["drop_items_errors"] } %>
+    <% end %>
+
+    <% if @run.events["custom"]&.any? %>
+      <p class="fancy-hr" id="running">Custom (<%= errors_count(@run.events["custom"]) %>)</p>
+      <%= erb :'runs/_errors', locals: { errors: @run.events["custom"] } %>
+    <% end %>
+  </div>
+</div>