active_analytics 0.2.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8be54d2704a6f0c7141fd6c0c4d276992d13f8cd1126868a5e71d038ba4333e5
-  data.tar.gz: 4a6eb25b26894054f9a1b669a198a6f97d2e13bdf892d8b09b5f8da5251036db
+  metadata.gz: a008072c9c416bc367e4952deb03036e422368fc98a1da10e0e6921a59f45a09
+  data.tar.gz: a7a1999f79b95ba4007280e36be378dd2c6e00885f0254db8b9a3dfbc2e67dbd
 SHA512:
-  metadata.gz: efae89347949dc906204ee4188b278a4482e46a2575a83868c4ffc73915869c46f88517fc235c4976ebec9f7474bc26b6823066156aaa2ccffa9ffaeb1128164
-  data.tar.gz: 4b94f1808a3ce5da5e8050fda39270fb7f65990aecb708f0dc6e54174139fa5d9a4f4c87431917be607fed4ad3580710af84b8449f290bbebb78308086ed6f25
+  metadata.gz: 3ad8d068d845fbe22c00a3cafbbcc2cd4459934e4b2ba7790c394cb1c69e4858c1f58eb25ef76b5e0b77e3c71597ce1d1ac6f2caa8173a62b83f8b51f644226b
+  data.tar.gz: 256cc8d412ca06ea8cb71f522d4935fa212076e17a240deb03e075aa2ec8da37ea6d602dc6bd536d5a6f6b029e9c249757efdb185ac5394fb16d0f4b61ba1e7f

data/README.md

@@ -32,38 +32,109 @@ rails active_analytics:install:migrations
 rails db:migrate
 ```
 
-Your controllers have to call `ActiveAnalytics.record_request(request)` to record page views. The Rails way to achieve is to use a `before_action` :
+Add the route to ActiveAnalytics dashboard at the desired endpoint:
+
+```ruby
+# config/routes.rb
+mount ActiveAnalytics::Engine, at: "analytics"  # http://localhost:3000/analytics
+```
+By default ActiveAnalytics will extend `ActionController::Base`, but you can specify a custom base controller for the ActiveAnalytics dashboard:
+
+```ruby
+# config/initializers/active_analytics.rb
+Rails.application.configure do
+  ActiveAnalytics.base_controller_class = "AdminController"
+end
+```
+
+
+The next step is to collect trafic and there is 2 options.
+
+### Record requests synchronously
+
+This is the easiest way to start with.
+However it's less performant since it triggers a write into your database for each request.
+Your controllers have to call `ActiveAnalytics.record_request(request)` to record page views.
+The Rails way to achieve is to use `after_action` :
 
 ```ruby
 class ApplicationController < ActionController::Base
-  before_action :record_page_view
+  after_action :record_page_view
 
   def record_page_view
-    # Add a condition to record only your canonical domain
-    # and use a gem such as crawler_detect to skip bots.
-    ActiveAnalytics.record_request(request)
+    # This is a basic example, you might need to customize some conditions.
+    # For most sites, it makes no sense to record anything other than HTML.
+    if response.content_type && response.content_type.start_with?("text/html")
+      # Add a condition to record only your canonical domain
+      # and use a gem such as crawler_detect to skip bots.
+      ActiveAnalytics.record_request(request)
+    end
   end
 end
 ```
 
-In case you don't want to record all page views, because each application has sensitive URLs such as password reset and so on, simply define a `skip_before_action :record_page_view` in the relevant controller.
+In case you don't want to record all page views, because each application has sensitive URLs such as password reset and so on, simply define a `skip_after_action :record_page_view` in the relevant controller.
+
+### Queue requests asynchronously
 
-Finally, just add the route to ActiveAnalytics dashboard at the desired endpoint:
+It requires more work and it's relevant if your application handle a large trafic.
+The idea is to queue data into Redis because it does not require the database writing to the disk on each request.
+First you have to set the Redis URL or connection.
 
 ```ruby
-mount ActiveAnalytics::Engine, at: "analytics"  # http://localhost:3000/analytics
+# File lib/patches/active_analytics.rb or config/initializers/active_analytics.rb
+
+ActiveAnalytics.redis_url = "redis://user:password@host/1" # Default ENV["ACTIVE_ANALYTICS_REDIS_URL"] || ENV["REDIS_URL"] || "redis://localhost"
+
+# If you use special connection options you have to instantiate it yourself
+ActiveAnalytics.redis = Redis.new(
+  url: ENV["REDIS_URL"],
+  reconnect_attempts: 10,
+  ssl_params: {verify_mode: OpenSSL::SSL::VERIFY_NONE}
+)
+```
+
+Then your controllers have to call `ActiveAnalytics.queue_request(request)` to queue page views.
+The Rails way to achieve is to use `after_action` :
+
+```ruby
+class ApplicationController < ActionController::Base
+  after_action :queue_page_view
+
+  def queue_page_view
+    # This is a basic example, you might need to customize some conditions.
+    # For most sites, it makes no sense to record anything other than HTML.
+    if response.content_type && response.content_type.start_with?("text/html")
+      # Add a condition to record only your canonical domain
+      # and use a gem such as crawler_detect to skip bots.
+      ActiveAnalytics.queue_request(request)
+    end
+  end
+end
 ```
 
+Queued data need to be saved into the database in order to be viewable in the ActiveAnalytics dashboard.
+For that, call `ActiveAnalytics.flush_queue` from a cron task or a background job.
+
+It's up to you if you want to flush the queue every hour or every 10 minutes.
+I advise to execute the last flush of the day at 23:59.
+It prevents from shifting the trafic to the next day.
+In that case only the last minute will be shifted to the next day, even if the flush ends after midnight.
+This small imperfection allows a simpler implementation for now.
+Keep it simple !
+
+
 ## Authentication and permissions
 
-ActiveAnalytics cannot guess how you handle user authentication, because it is different for all Rails applications. So you have to monkey patch `ActiveAnalytics::ApplicationController` in order to inject your own mechanism. Create a file in `config/initializers/active_analytics.rb` to add a before action :
+ActiveAnalytics cannot guess how you handle user authentication, because it is different for all Rails applications.
+So you have to monkey patch `ActiveAnalytics::ApplicationController` in order to inject your own mechanism.
+The patch can be saved wherever you want.
+For example, I like to have all the patches in one place, so I put them in `lib/patches`.
 
 ```ruby
-# config/initializers/active_analytics.rb
-require_dependency "active_analytics/application_controller"
+# lib/patches/active_analytics.rb
 
-module ActiveAnalytics
-  class ApplicationController
+ActiveAnalytics::ApplicationController.class_eval do
     before_action :require_admin
 
     def require_admin
@@ -74,7 +145,18 @@ module ActiveAnalytics
 end
 ```
 
-If you have Devise, you can check the permission directly from routes.rb :
+Then you have to require the monkey patch.
+Because it's loaded via require, it won't be reloaded in development.
+Since you are not supposed to change this file often, it should not be an issue.
+
+```ruby
+# config/application.rb
+config.after_initialize do
+  require "patches/active_analytics"
+end
+```
+
+If you use Devise, you can check the permission directly from routes.rb :
 
 ```ruby
 # config/routes.rb
@@ -88,4 +170,4 @@ The gem is available as open-source under the terms of the [MIT License](https:/
 
 Made by [Base Secrète](https://basesecrete.com).
 
-Rails developer? Check out [RoRvsWild](https://rorvswild.com), our Ruby on Rails application monitoring tool.
+Rails developer? Check out [RoRvsWild](https://rorvswild.com), our Ruby on Rails application monitoring tool.

data/app/controllers/active_analytics/application_controller.rb

@@ -1,18 +1,41 @@
 module ActiveAnalytics
-  class ApplicationController < ActionController::Base
+  class ApplicationController < ActiveAnalytics.base_controller_class.constantize
+    layout "active_analytics/application"
+
+    helper PagesHelper
+    helper SitesHelper
+    helper BrowsersHelper
 
     private
 
+    def from_date
+      @from_date ||= Date.parse(params[:from])
+    end
+
+    def to_date
+      @to_date ||= Date.parse(params[:to])
+    end
+
     def require_date_range
-      if Date.parse(params[:from]) > Date.parse(params[:to])
-        redirect_to(params.to_unsafe_hash.merge(from: 7.days.ago.to_date, to: Date.today))
-      end
-    rescue TypeError, ArgumentError # Raise by Date.parse when invalid format
+      redirect_to(params.to_unsafe_hash.merge(from: to_date, to: from_date)) if from_date > to_date
+    rescue TypeError, ArgumentError # Raised by Date.parse when invalid format
       redirect_to(params.to_unsafe_hash.merge(from: 7.days.ago.to_date, to: Date.today))
     end
 
     def current_views_per_days
-      ViewsPerDay.where(site: params[:site]).between_dates(params[:from], params[:to])
+      ViewsPerDay.where(site: params[:site]).between_dates(from_date, to_date)
+    end
+
+    def previous_views_per_days
+      ViewsPerDay.where(site: params[:site]).between_dates(previous_from_date, previous_to_date)
+    end
+
+    def previous_from_date
+      from_date - (to_date - from_date)
+    end
+
+    def previous_to_date
+      to_date - (to_date - from_date)
     end
   end
 end

data/app/controllers/active_analytics/assets_controller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_dependency "active_analytics/application_controller"
+
+module ActiveAnalytics
+  class AssetsController < ApplicationController
+    protect_from_forgery except: :show
+
+    @@root = ActiveAnalytics::Engine.root.join("app/views", controller_path + "/").to_s
+
+    def self.endpoints
+      return @endpoints if @endpoints
+      paths = Dir["#{@@root}**/*"].keep_if { |path| File.file?(path) }
+      files = paths.map { |path| path.to_s.delete_prefix(@@root).delete_suffix(".erb") }
+      @endpoints = files.delete_if { |str| str.start_with?("_") }
+    end
+
+    def show
+      if self.class.endpoints.include?(params[:file])
+        expires_in(1.day, public: true)
+        render_asset(params[:file])
+      else
+        raise ActionController::RoutingError.new("Not found #{params[:file]}")
+      end
+    end
+
+    private
+
+    def render_asset(path)
+      ext = File.extname(params[:file])
+      path_without_ext = path.delete_suffix(ext)
+      mime_type = Mime::Type.lookup_by_extension(ext.delete_prefix("."))
+      render("#{controller_path}/#{path_without_ext}", mime_type: mime_type)
+    end
+  end
+end

data/app/controllers/active_analytics/browsers_controller.rb

@@ -0,0 +1,27 @@
+require_dependency "active_analytics/application_controller"
+
+module ActiveAnalytics
+  class BrowsersController < ApplicationController
+    def index
+      @histogram = Histogram.new(current_browsers_per_days.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_browsers_per_days.order_by_date.group_by_date, previous_from_date, previous_to_date)
+      @browsers = current_browsers_per_days.group_by_name.top(100)
+    end
+
+    def show
+      @histogram = Histogram.new(current_browsers_per_days.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_browsers_per_days.order_by_date.group_by_date, previous_from_date, previous_to_date)
+      @browsers = current_browsers_per_days.group_by_version.top(100)
+    end
+
+    private
+
+    def current_browsers_per_days
+      BrowsersPerDay.filter_by(params)
+    end
+
+    def previous_browsers_per_days
+      BrowsersPerDay.filter_by(params.merge(from: previous_from_date, to: previous_to_date))
+    end
+  end
+end

data/app/controllers/active_analytics/pages_controller.rb

@@ -7,19 +7,18 @@ module ActiveAnalytics
     before_action :require_date_range
 
     def index
-      scope = ViewsPerDay.where(site: params[:site]).between_dates(params[:from], params[:to])
-      @histogram = ViewsPerDay::Histogram.new(scope.order_by_date.group_by_date, params[:from], params[:to])
-      @pages = scope.top(100).group_by_page
+      @histogram = Histogram.new(current_views_per_days.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_views_per_days.order_by_date.group_by_date, previous_from_date, previous_to_date)
+      @pages = current_views_per_days.top(100).group_by_page
     end
 
     def show
-      dates_scopes = ViewsPerDay.between_dates(params[:from], params[:to])
-      page_scope = dates_scopes.where(site: params[:site], page: page_from_params)
-      @histogram = ViewsPerDay::Histogram.new(page_scope.order_by_date.group_by_date, params[:from], params[:to])
-      @referrers = page_scope.top.group_by_referrer_site
-
-      @next_pages = dates_scopes.where(referrer_host: params[:site], referrer_path: page_from_params).top(100).group_by_page
-      @previous_pages = dates_scopes.where(site: params[:site], page: page_from_params).where.not(referrer_path: nil).top(100).group_by_referrer_page
+      page_scope = current_views_per_days.where(page: page_from_params)
+      previous_page_scope = previous_views_per_days.where(page: page_from_params)
+      @histogram = Histogram.new(page_scope.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_page_scope.order_by_date.group_by_date, previous_from_date, previous_to_date)
+      @next_pages = current_views_per_days.where(referrer_host: params[:site], referrer_path: page_from_params).top(100).group_by_page
+      @previous_pages = page_scope.top(100).group_by_referrer_page
     end
   end
 end

data/app/controllers/active_analytics/referrers_controller.rb

@@ -5,14 +5,18 @@ module ActiveAnalytics
     before_action :require_date_range
 
     def index
-      scope = ViewsPerDay.where(site: params[:site]).between_dates(params[:from], params[:to])
-      @referrers = scope.top(100).group_by_referrer_site
-      @histogram = ViewsPerDay::Histogram.new(scope.order_by_date.group_by_date, params[:from], params[:to])
+      @referrers = current_views_per_days.top(100).group_by_referrer_site
+      @histogram = Histogram.new(current_views_per_days.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_views_per_days.order_by_date.group_by_date, previous_from_date, previous_to_date)
     end
 
     def show
-      scope = ViewsPerDay.where(site: params[:site], referrer_host: params[:referrer]).between_dates(params[:from], params[:to])
-      @histogram = ViewsPerDay::Histogram.new(scope.order_by_date.group_by_date, params[:from], params[:to])
+      referrer_host, referrer_path = params[:referrer].split("/", 2)
+      scope = current_views_per_days.where(referrer_host: referrer_host)
+      scope = scope.where(referrer_path: "/" + referrer_path) if referrer_path.present?
+      previous_scope = previous_views_per_days.where(referrer_host: params[:referrer])
+      @histogram = Histogram.new(scope.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_scope.order_by_date.group_by_date, previous_from_date, previous_to_date)
       @previous_pages = scope.top(100).group_by_referrer_page
       @next_pages = scope.top(100).group_by_page
     end

data/app/controllers/active_analytics/sites_controller.rb

@@ -10,10 +10,11 @@ module ActiveAnalytics
     end
 
     def show
-      scope = current_views_per_days
-      @histogram = ViewsPerDay::Histogram.new(scope.order_by_date.group_by_date, params[:from], params[:to])
-      @referrers = scope.top.group_by_referrer_site
-      @pages = scope.top.group_by_page
+      @histogram = Histogram.new(current_views_per_days.order_by_date.group_by_date, from_date, to_date)
+      @previous_histogram = Histogram.new(previous_views_per_days.order_by_date.group_by_date, previous_from_date, previous_to_date)
+      @referrers = current_views_per_days.top.group_by_referrer_site
+      @pages = current_views_per_days.top.group_by_page
+      @browsers = BrowsersPerDay.filter_by(params).group_by_name.top
     end
   end
 end

data/app/helpers/active_analytics/browsers_helper.rb

@@ -0,0 +1,12 @@
+module ActiveAnalytics
+  module BrowsersHelper
+    def browser_icon(browser_name)
+      path = "browsers/#{browser_name.to_s.parameterize(separator: "_")}.svg"
+      if AssetsController.endpoints.include?(path)
+        image_tag(asset_url(path, host: request.host), alt: browser_name, class: "referer-favicon", width: 16, height: 16)
+      else
+        image_tag(asset_url("browsers/default.svg", host: request.host), alt: browser_name, class: "referer-favicon", width: 16, height: 16)
+      end
+    end
+  end
+end

data/app/lib/active_analytics/histogram.rb

@@ -0,0 +1,47 @@
+module ActiveAnalytics
+  class Histogram
+    attr_reader :bars, :from_date, :to_date
+
+    def initialize(scope, from_date, to_date)
+      @scope = scope
+      @from_date, @to_date = from_date, to_date
+      @bars = scope.map { |record| Bar.new(record.date, record.total, self) }
+      fill_missing_days(@bars, @from_date, @to_date)
+    end
+
+    def fill_missing_days(bars, from, to)
+      i = 0
+      while (day = from + i) <= to
+        if !@bars[i] || @bars[i].label != day
+          @bars.insert(i, Bar.new(day, 0, self))
+        end
+        i += 1
+      end
+      @bars
+    end
+
+    def max_value
+      @max_total ||= bars.map(&:value).max
+    end
+
+    def total
+      @bars.reduce(0) { |sum, bar| sum += bar.value }
+    end
+
+    class Bar
+      attr_reader :label, :value, :histogram
+
+      def initialize(label, value, histogram)
+        @label, @value, @histogram = label, value, histogram
+      end
+
+      def height
+        if histogram.max_value > 0
+          (value.to_f / histogram.max_value).round(2)
+        else
+          0
+        end
+      end
+    end
+  end
+end

data/app/models/active_analytics/browsers_per_day.rb

@@ -0,0 +1,40 @@
+module ActiveAnalytics
+  class BrowsersPerDay < ApplicationRecord
+    # TODO: Deduplicate
+    scope :top, -> (n = 10) { order_by_totals.limit(n) }
+    scope :order_by_date, -> { order(:date) }
+    scope :order_by_totals, -> { order(Arel.sql("SUM(total) DESC")) }
+    scope :between_dates, -> (from, to) { where(date: from..to) }
+
+    def self.append(params)
+      total = params.delete(:total) || 1
+      params[:site] = params[:site].downcase if params[:site]
+      where(params).first.try(:increment!, :total, total) || create!(params.merge(total: total))
+    end
+
+    def self.group_by_name
+      group(:name).select("name, sum(total) AS total")
+    end
+
+    def self.group_by_version
+      group(:name, :version).select("version, sum(total) AS total")
+    end
+
+    def self.group_by_date
+      group(:date).select("date, sum(total) as total")
+    end
+
+    def self.filter_by(params)
+      scope = all
+      scope = scope.between_dates(params[:from], params[:to]) if params[:from].present? && params[:to].present?
+      scope = scope.where(site: params[:site]) if params[:site].present?
+      scope = scope.where(name: params[:id]) if params[:id].present?
+      scope = scope.where(version: params[:version]) if params[:version].present?
+      scope
+    end
+
+    def to_param
+      name
+    end
+  end
+end

data/app/models/active_analytics/views_per_day.rb

@@ -26,53 +26,6 @@ module ActiveAnalytics
       end
     end
 
-    class Day
-      attr_reader :day, :total
-      def initialize(day, total)
-        @day, @total = day, total
-      end
-    end
-
-    class Histogram
-      attr_reader :bars, :from_date, :to_date
-
-      def initialize(scope, from_date, to_date)
-        @bars = scope.map { |day| Bar.new(day.day, day.total, self) }
-        fill_missing_days(@bars, Date.parse(from_date.to_s), Date.parse(to_date.to_s))
-      end
-
-      def fill_missing_days(bars, from, to)
-        i = 0
-        while (day = from + i) <= to
-          if !@bars[i] || @bars[i].label != day
-            @bars.insert(i, Bar.new(day, 0, self))
-          end
-          i += 1
-        end
-        @bars
-      end
-
-      def max_value
-        @max_total ||= bars.map(&:value).max
-      end
-
-      def total
-        @bars.reduce(0) { |sum, bar| sum += bar.value }
-      end
-
-      class Bar
-        attr_reader :label, :value, :histogram
-
-        def initialize(label, value, histogram)
-          @label, @value, @histogram = label, value, histogram
-        end
-
-        def height
-          (value.to_f / histogram.max_value).round(2)
-        end
-      end
-    end
-
     def self.group_by_site
       group(:site).pluck("site, SUM(total)").map do |row|
         Site.new(row[0], row[1])
@@ -98,22 +51,32 @@ module ActiveAnalytics
     end
 
     def self.group_by_date
-      group(:date).pluck("date, SUM(total)").map do |row|
-        Day.new(row[0], row[1])
-      end
+      group(:date).select("date, sum(total) AS total")
     end
 
     def self.to_histogram
-      ViewsPerDay::Histogram.new(self)
+      Histogram.new(self)
     end
 
     def self.append(params)
+      total = params.delete(:total) || 1
       params[:site] = params[:site].downcase if params[:site]
-      params[:page] = params[:page].downcase if params[:page]
       params[:referrer_path] = nil if params[:referrer_path].blank?
       params[:referrer_path] = params[:referrer_path].downcase if params[:referrer_path]
       params[:referrer_host] = params[:referrer_host].downcase if params[:referrer_host]
-      find_or_create_by!(params) if where(params).update_all("total = total + 1") == 0
+      where(params).first.try(:increment!, :total, total) || create!(params.merge(total: total))
+    end
+
+    SLASH = "/"
+
+    def self.split_referrer(referrer)
+      return [nil, nil] if referrer.blank?
+      if (uri = URI(referrer)).host.present?
+        [uri.host, uri.path.presence]
+      else
+        strings = referrer.split(SLASH, 2)
+        [strings[0], strings[1] ? SLASH + strings[1] : nil]
+      end
     end
   end
 end