kimurai-dashboard 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d45c04e24e2f570510b38e3027958d190e39cb8cc3991f03925691acc5c0a4cf
+  data.tar.gz: 8833f523ebc1f303d48a5ad9107c0ecc1e33f36d0e0aa987206f13f30508e26a
+SHA512:
+  metadata.gz: e12f971ce5e9c6dab110cae9e6975f4ace11ffac1c1a607d855e5d32b2662bdb3619fd08a4921b0f58f90dc30634a18ade29ead57273bcbc8b525c9786e6fc13
+  data.tar.gz: ff659649fb65ab2c7953e9544dc6ea3bc3a0196bd150d812316fb3f9ca8652a667c31c37e6d6a5643ff8804d5f29d3dbd57bc63b13ebc7eb5a05df2401f78833

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.16.2

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in kimurai-dashboard.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Victor Afanasev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+# Kimurai::Dashboard
+
+Simple Dashboard for [Kimurai web scraping framework](https://github.com/vifreefly/kimuraframework). Required version of Kimurai `>= 1.2.0`.
+
+## Installation
+Add this line to your Kimurai project's Gemfile:
+
+```ruby
+# add this line after `gem 'kimurai'`
+gem 'kimurai-dashboard', require: false
+```
+
+and then execute `$ bundle`.
+
+## Configuration
+You need to provide `stats_database_url` to enable stats and save info about project spiders runs and sessions to a database. Format for a database url: https://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html. You can use _sqlite_, _postgres_ or _mysql_ database (check Kimurai's project Gemfile and uncomment preferred gem).
+
+Example for SQlite:
+
+```ruby
+# Gemfile
+gem 'sqlite3'
+```
+
+**Note that dashboard should be required only after stats_database_url provided:**
+
+```ruby
+# config/boot.rb
+# ...
+
+Kimurai.configuration.stats_database_url = "sqlite://db/spiders_runs_#{Kimurai.env}.sqlite3"
+# Important: require dashboard ONLY after stats_database_url was provided:
+require 'kimurai/dashboard'
+```
+
+Also, there are optional settings for a dashboard:
+
+```ruby
+# config/application.rb
+
+Kimurai.configure do |config|
+  # ...
+
+  config.dashboard = {
+    bind_address: "0.0.0.0",
+    port: 3001,
+    basic_auth: { username: "admin", password: "123456" }
+  }
+end
+```
+
+## Usage
+After successful configuration, all spiders (running individually `kimurai start` or in queue `kimurai runner`) will save stats to the database.
+
+Run `$ bundle exec kimurai dashboard` and navigate to a dashboard url to see the stats.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kimurai/dashboard"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/kimurai-dashboard.gemspec

@@ -0,0 +1,30 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "kimurai/dashboard/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "kimurai-dashboard"
+  spec.version       = Kimurai::Dashboard::VERSION
+  spec.authors       = ["Victor Afanasev"]
+  spec.email         = ["vicfreefly@gmail.com"]
+
+  spec.summary       = "Simple dashboard for a Kimurai web scraping framework"
+  spec.homepage      = "https://github.com/vifreefly/kimurai-dashboard"
+  spec.license       = "MIT"
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 2.5.0"
+
+  spec.add_dependency "kimurai", "~> 1.2.0"
+  spec.add_dependency "sequel"
+  spec.add_dependency "sinatra-contrib"
+  spec.add_dependency "pagy"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/kimurai/dashboard.rb

@@ -0,0 +1,49 @@
+require 'sequel'
+require 'json'
+require 'kimurai/dashboard/version'
+
+require_relative 'dashboard/base'
+require_relative 'dashboard/runner'
+
+module Kimurai
+  module Dashboard
+    DB = Sequel.connect(Kimurai.configuration.stats_database_url ||= ENV["STATS_DATABASE_URL"])
+
+    DB.create_table?(:sessions) do
+      primary_key :id, type: :integer, auto_increment: false
+      string :status
+      datetime :start_time, empty: false
+      datetime :stop_time
+      string :environment
+      integer :concurrent_jobs
+      text :spiders
+      text :error
+    end
+
+    DB.create_table?(:runs) do
+      primary_key :id
+      string :spider_name, empty: false
+      string :status
+      string :environment
+      datetime :start_time, empty: false
+      datetime :stop_time
+      float :running_time
+      foreign_key :session_id, :sessions
+      foreign_key :spider_id, :spiders
+      text :visits
+      text :items
+      text :events
+      text :error
+      text :server
+    end
+
+    DB.create_table?(:spiders) do
+      primary_key :id
+      string :name, empty: false, unique: true
+    end
+  end
+end
+
+require_relative 'dashboard/models/session'
+require_relative 'dashboard/models/run'
+require_relative 'dashboard/models/spider'

data/lib/kimurai/dashboard/app.rb

@@ -0,0 +1,151 @@
+require 'sinatra/base'
+require 'sinatra/respond_with'
+require 'sinatra/json'
+require 'sinatra/namespace'
+require 'sinatra/reloader'
+require 'sinatra/streaming'
+require 'pagy'
+require_relative 'helpers'
+
+module Kimurai
+  module Dashboard
+    class App < Sinatra::Base
+      include Pagy::Backend
+
+      register Sinatra::RespondWith, Sinatra::Namespace
+      enable :logging
+      set :environment, Kimurai.env.to_sym
+
+      if bind_address = Kimurai.configuration.dashboard&.dig(:bind_address)
+        set :bind, bind_address
+      end
+
+      if port = Kimurai.configuration.dashboard&.dig(:port)
+        set :port, port
+      end
+
+      configure :development do
+        require 'pry'
+        register Sinatra::Reloader
+      end
+
+      helpers Sinatra::Streaming
+      helpers do
+        include Helpers
+        include Rack::Utils
+        alias_method :h, :escape_html
+      end
+
+      if auth = Kimurai.configuration.dashboard&.dig(:basic_auth)
+        use Rack::Auth::Basic, "Protected Area" do |username, password|
+          username == auth[:username] && password == auth[:password]
+        end
+      end
+
+      ###
+
+      get "/" do
+        redirect "/spiders"
+      end
+
+      namespace "/sessions" do
+        get do
+          @sessions = Session.reverse_order(:id)
+          @pagy, @sessions = pagy(@sessions) unless @sessions.count.zero?
+
+          respond_to do |f|
+            f.html { erb :'sessions/index' }
+          end
+        end
+
+        get "/:id" do
+          @session = Session.find(id: params[:id].to_i)
+          halt "Error, can't find session!" unless @session
+
+          respond_to do |f|
+            f.html { erb :'sessions/show' }
+          end
+        end
+      end
+
+      namespace "/runs" do
+        get do
+          @runs = Run.reverse_order(:id)
+
+          filters = params.slice("spider_id", "session_id")
+          filters.each do |filter_name, value|
+            @runs = @runs.send(filter_name, value)
+          end
+
+          @pagy, @runs = pagy(@runs) unless @runs.count.zero?
+          respond_to do |f|
+            f.html { erb :'runs/index', locals: { filters: filters }}
+          end
+        end
+
+        get "/:id" do
+          @run = Run.find(id: params[:id].to_i)
+          halt "Error, can't find session!" unless @run
+
+          respond_to do |f|
+            f.html { erb :'runs/show', locals: { difference: @run.difference_between_previous_run }}
+          end
+        end
+
+        get "/:id/log" do
+          @run = Run.find(id: params[:id].to_i)
+          halt "Error, can't find run with id: #{params[:id]}" unless @run
+
+          log_name = "./log/#{@run.spider_name}.log"
+
+          if @run.latest? && File.exists?(log_name)
+            content_type 'text/event-stream'
+            File.readlines(log_name)
+          else
+            halt "Log file is not available for this run"
+          end
+        end
+      end
+
+      namespace "/spiders" do
+        get do
+          @spiders = Spider
+          @pagy, @spiders = pagy(@spiders) unless @spiders.count.zero?
+
+          respond_to do |f|
+            f.html { erb :'spiders/index' }
+          end
+        end
+
+        get "/:id_or_name" do
+          @spider =
+            if params[:id_or_name].match?(/^(\d)+$/)
+              Spider.find(id: params[:id_or_name].to_i)
+            else
+              Spider.find(name: params[:id_or_name])
+            end
+
+          halt "Error, can't find spider!" unless @spider
+
+          @spider_runs = @spider.runs_dataset.reverse_order(:id)
+          @pagy, @spider_runs = pagy(@spider_runs, items: 15) unless @spider_runs.count.zero?
+
+          respond_to do |f|
+            f.html { erb :'spiders/show' }
+          end
+        end
+      end
+
+      private
+
+      def pagy_get_vars(collection, vars)
+        {
+          count: collection.count,
+          page_param: "page",
+          page: params["page"],
+          items: vars[:items] || 25
+        }
+      end
+    end
+  end
+end

data/lib/kimurai/dashboard/base.rb

@@ -0,0 +1,53 @@
+require 'kimurai/base'
+require 'socket'
+
+module Kimurai
+  class Base
+    class << self
+      alias_method :original_crawl!, :crawl!
+    end
+
+    def self.crawl!(continue: false, exception_on_fail: true)
+      logger.error "Spider: already running: #{name}" and return false if running?
+
+      spider = Dashboard::Spider.find_or_create(name: name)
+      run = Dashboard::Run.new(spider_id: spider.id)
+
+      updater = proc do |final_info|
+        if final_info
+          run.set(final_info)
+          run.save
+        elsif @run_info
+          unless @run_info[:server]
+            @run_info.merge!(
+              session_id: ENV["SESSION_ID"]&.to_i,
+              server: {
+                hostname: Socket.gethostname,
+                ipv4: Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }&.ip_address,
+                process_pid: Process.pid
+              }
+            )
+          end
+
+          running_time = (Time.now - run_info[:start_time]).round(3)
+          run.set(@run_info.merge!(running_time: running_time))
+          run.save
+        end
+      end
+
+      task = Thread.new do
+        loop { sleep 0.5 and updater.call and sleep 1.5 }
+      end
+
+      final_info, error = original_crawl!(continue: continue, exception_on_fail: false)
+      if error
+        exception_on_fail ? raise(error) : [final_info, error]
+      else
+        final_info
+      end
+    ensure
+      task.terminate if task
+      updater.call(final_info)# if final_info
+    end
+  end
+end

data/lib/kimurai/dashboard/helpers.rb

@@ -0,0 +1,78 @@
+require 'pagy'
+require 'pagy/extras/bootstrap'
+
+module Kimurai
+  module Dashboard
+    module Helpers
+      include Pagy::Frontend
+
+      def to_hash(object)
+        object.to_hash.merge(object.deserialized_values)
+      end
+
+      def convert_to_links(elements, base:)
+        elements.map { |element| %(<a href="#{base}/#{element}">#{element}</a>) }
+      end
+
+      def breadcrumbs(hash)
+        elements = hash.map do |path, value|
+          if path.empty?
+            %Q{<li class="breadcrumb-item active" aria-current="page">#{value}</li>}
+          else
+            %Q{<li class="breadcrumb-item"><a href="#{path}">#{value}</a></li>}
+          end
+        end
+        %Q{<nav aria-label="breadcrumb">
+          <ol class="breadcrumb"> #{elements.join}</ol>
+        </nav>}
+      end
+
+      def minimize_stats(stats)
+        stats.values.map { |stat| stat.class == Hash ? stat.size : stat }
+      end
+
+      def get_badge(status)
+        case status
+        when "running"
+          %Q{<span class="badge badge-primary">running</span>}
+        when "processing"
+          %Q{<span class="badge badge-primary">processing</span>}
+        when "completed"
+          %Q{<span class="badge badge-success">completed</span>}
+        when "failed"
+          %Q{<span class="badge badge-danger">failed</span>}
+        when "stopped"
+          %Q{<span class="badge badge-light">stopped</span>}
+        else
+          status
+        end
+      end
+
+      def render_filters(filters)
+        f = filters.map { |k,v| "#{k} = #{v}" }.join(", ")
+        %Q{<p class="text-muted"> Filters: #{f} </p>}
+      end
+
+      def format_difference(prev_value, prev_diff, prev_run_id)
+        previous =
+          %Q{previous <a href="/runs/#{prev_run_id}">#{prev_value}</a>}
+
+        formatted_diff = begin
+          str = prev_diff.to_s
+          str.insert(0, "+") if str !~ /^[-0]/i
+          "#{str}%"
+        end if prev_diff
+
+        if formatted_diff
+          "(#{previous}, #{formatted_diff})"
+        else
+          "(#{previous})"
+        end
+      end
+
+      def errors_count(hash)
+        hash.map { |_, count| count }.sum
+      end
+    end
+  end
+end