skadi 0.4.0.beta.1

data/app/models/skadi/helpers/sql.rb

@@ -0,0 +1,86 @@
+module Skadi
+  module Helpers
+    class Sql
+      class << self
+        def operator(model, operator)
+          return case operator
+            when "like"
+              ilike(model)
+            when "not like"
+              "NOT #{ilike(model)}"
+            else
+              operator
+          end
+        end
+
+        # Provides the keyword for case insensitive LIKE
+        def ilike(model)
+          case model.connection.adapter_name
+            when "PostgreSQL"
+              "ILIKE"
+            when "Mysql2", "SQLite"
+              "LIKE"
+            else
+              raise ::Skadi::Dashboard::UnsupportedDatabaseError.new("The database adapter #{model.connection.adapter_name} is not supported")
+          end
+        end
+
+        def string_before_separator(model, field, separator)
+          case model.connection.adapter_name
+            when "PostgreSQL"
+              "SPLIT_PART(#{model.table_name}.#{field}, '#{separator}', 1)"
+            when "Mysql2"
+              "SUBSTRING_INDEX(#{model.table_name}.#{field}, '#{separator}', 1)"
+            when "SQLite"
+              "SUBSTR(#{model.table_name}.#{field}, 1, INSTR(#{model.table_name}.#{field}, '#{separator}') - 1)"
+            else
+              raise ::Skadi::Dashboard::UnsupportedDatabaseError.new("The database adapter #{model.connection.adapter_name} is not supported")
+          end
+        end
+
+        def time_series(time_series, model, date_field)
+          case time_series
+            when "daily"
+              day_of_date(model, date_field)
+            when "weekly"
+              week_of_date(model, date_field)
+            when "monthly"
+              month_of_date(model, date_field)
+            else
+              raise ::Skadi::Dashboard::DatasetConfigurationError.new("The time_series #{time_series} is invalid")
+          end
+        end
+
+        def day_of_date(_model, date_field)
+          "DATE(#{date_field})"
+        end
+
+        def week_of_date(model, date_field)
+          case model.connection.adapter_name
+            when "PostgreSQL"
+              "DATE_TRUNC('week', #{date_field})::date"
+            when "Mysql2"
+              "DATE_SUB(DATE(#{date_field}), INTERVAL WEEKDAY(#{date_field}) DAY)"
+            when "SQLite"
+              "DATE(#{date_field}, '-' || ((CAST(STRFTIME('%w', #{date_field}) AS INTEGER) + 6) % 7) || ' days')"
+            else
+              raise ::Skadi::Dashboard::UnsupportedDatabaseError.new("The database adapter #{model.connection.adapter_name} is not supported")
+          end
+        end
+
+        def month_of_date(model, date_field)
+          case model.connection.adapter_name
+            when "PostgreSQL"
+              "DATE_TRUNC('month', #{date_field})::date"
+            when "Mysql2"
+              "DATE_FORMAT(#{date_field}, '%Y-%m-01')"
+            when "SQLite"
+              "DATE(#{date_field}, 'start of month')"
+            else
+              raise ::Skadi::Dashboard::UnsupportedDatabaseError.new("The database adapter #{model.connection.adapter_name} is not supported")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/skadi/schema.rb

@@ -0,0 +1,135 @@
+module Skadi
+  class Schema
+    class << self
+      # Extract out the most common field config into a const
+      FILTER_AND_SPLIT = { filter: true, split: true }
+      private_constant :FILTER_AND_SPLIT
+
+      # A hash where the key represents a dataset name and the value is its configuration
+      def database_schema
+        return @_database_schema if defined?(@_database_schema)
+
+        @_database_schema ||= {
+          # A hash with the following keys:
+          # model: The model for this dataset
+          # visit_key: used for chart-level visit filters
+          # count_sql: The SQL expression used to count this field. Defaults to COUNT().
+          # fields: A hash where the key represents a field and the value is its configuration
+          visits: {
+            model: Skadi::Visit,
+            visit_key: "id",
+            # If the table has a date field, the fields Hash expects a value for :date
+            fields: {
+              # A hash with the following keys:
+              # label: The label to show in the front end
+              # type: The datatype, one of :one_of, :date, :string, :number, :boolean. Defaults to :string.
+              # filter: True if this field can be filtered
+              # split: True if this field can be split
+              # sql: The SQL expression used to get this value for derived fields
+              # options: A list of possible values
+              # description: used in the front end as help text for this field
+              date: {
+                type: :date,
+                filter: true,
+                sql: "skadi_visits.created_at",
+              },
+              landing_page: FILTER_AND_SPLIT,
+              referrer_domain: {
+                **FILTER_AND_SPLIT,
+                sql: Helpers::Sql.string_before_separator(Skadi::Visit, "referrer", "/"),
+              },
+              utm_source: FILTER_AND_SPLIT,
+              utm_medium: FILTER_AND_SPLIT,
+              utm_term: FILTER_AND_SPLIT,
+              utm_content: FILTER_AND_SPLIT,
+              utm_campaign: FILTER_AND_SPLIT,
+            },
+          },
+          views: {
+            model: Skadi::View,
+            visit_key: "visit_id",
+            fields: {
+              date: {
+                type: :date,
+                filter: true,
+                sql: "skadi_views.created_at",
+              },
+              verified: {
+                type: :boolean,
+                filter: true,
+              },
+              controller: FILTER_AND_SPLIT,
+              action: {
+                filter: true,
+              },
+              controller_and_action: {
+                split: true,
+                sql: "CONCAT(skadi_views.controller, '::', skadi_views.action)",
+              },
+              path: FILTER_AND_SPLIT,
+              verb: {
+                type: :one_of,
+                filter: true,
+                split: true,
+                description: "Typically, GET requests are page views, and POST, PUT, PATCH and DELETE are form submissions.",
+                options: %w[GET POST PUT PATCH DELETE],
+              },
+              version: FILTER_AND_SPLIT,
+              exit_page: FILTER_AND_SPLIT,
+            },
+          },
+          events: {
+            model: Skadi::Event,
+            visit_key: "visit_id",
+            fields: {
+              date: {
+                type: :date,
+                filter: true,
+                sql: "skadi_events.created_at",
+              },
+              name: FILTER_AND_SPLIT,
+              **(Skadi.configuration.dashboard_custom_event_fields || {}),
+            },
+          },
+          demographics: {
+            model: Demographic,
+            count_sql: "COALESCE(SUM(skadi_demographics.count), 0)",
+            fields: {
+              date: {
+                type: :date,
+                filter: true,
+                sql: "skadi_demographics.recorded_on",
+              },
+              uri: FILTER_AND_SPLIT,
+              name: FILTER_AND_SPLIT,
+              value: FILTER_AND_SPLIT,
+              count: {
+                **FILTER_AND_SPLIT,
+                type: :number,
+              },
+            },
+          },
+          **(Skadi.configuration.dashboard_custom_schema || {}),
+        }
+      end
+
+      def frontend_schema
+        return @_frontend_schema if defined?(@_frontend_schema)
+
+        # Redact the SQL from the dashboard schema
+        schema = database_schema.deep_dup
+        schema.each_value do |dataset|
+          dataset.keys.each do |key|
+            dataset.delete(key) unless key == :fields
+          end
+
+          dataset[:fields].each_value do |field|
+            field.delete(:sql)
+          end
+        end
+
+        @_frontend_schema = schema
+      end
+    end
+  end
+end

data/app/models/skadi/view.rb

@@ -0,0 +1,9 @@
+module Skadi
+  class View < ApplicationRecord
+    belongs_to :visit, class_name: "Skadi::Visit", optional: true, inverse_of: :views
+
+    has_many :events, class_name: "Skadi::Event", inverse_of: :view
+
+    def token = view_token
+  end
+end

data/app/models/skadi/visit.rb

@@ -0,0 +1,59 @@
+module Skadi
+  class Visit < ApplicationRecord
+    has_many :views, class_name: "Skadi::View", inverse_of: :visit
+    has_many :events, class_name: "Skadi::Event", inverse_of: :visit
+
+    def token = visit_token
+
+    def self.find_active_visit_for(tracking_token, user)
+      return nil if tracking_token.nil? && user.nil?
+
+      visit_query = nil
+      if tracking_token
+        visit_query = where(tracking_token: tracking_token)
+          .and(where("created_at > ?", Skadi.configuration.visit_duration.ago))
+      end
+      if user&.persisted?
+        user_visit_query = where(user_id: user.id)
+          .and(where("created_at > ?", Skadi.configuration.visit_duration.ago))
+
+        visit_query = visit_query ? visit_query.or(user_visit_query) : user_visit_query
+      end
+
+      visit = visit_query&.order(created_at: :desc)&.limit(1)&.first
+
+      # If the user has changed since the last visit, create a new visit
+      return nil if visit&.user_id && user&.persisted? && visit.user_id != user.id
+
+      visit
+    end
+
+    # @param tracking_token [String, nil]
+    # @param user [ActiveModel::Model, nil]
+    # @param request [ActionDispatch::Request]
+    # @return [Skadi::Visit]
+    def self.build_from(tracking_token, user_id = nil, request = nil, cookies_enabled: nil)
+      if cookies_enabled.nil?
+        cookies_enabled = request&.cookie_jar&.key?("skadi_id") || false
+      end
+
+      new(
+        visit_token: SecureRandom.uuid_v7,
+        tracking_token: tracking_token,
+        user_id: user_id,
+
+        referrer: request ? Skadi::Url.redact_and_normalise_url(request.referrer) : nil,
+        landing_page: request ? Skadi::Url.view_path_from_request(request) : nil,
+
+        utm_source: request ? request.query_parameters["utm_source"] : nil,
+        utm_medium: request ? request.query_parameters["utm_medium"] : nil,
+        utm_term: request ? request.query_parameters["utm_term"] : nil,
+        utm_content: request ? request.query_parameters["utm_content"] : nil,
+        utm_campaign: request ? request.query_parameters["utm_campaign"] : nil,
+
+        verified: false,
+        cookies_enabled: cookies_enabled,
+      )
+    end
+  end
+end

data/app/views/skadi/dashboard/_layout.html.erb

@@ -0,0 +1,21 @@
+<%# locals: () %>
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>📈 Skadi Analytics</title>
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= yield :head %>
+  </head>
+
+  <body class="flex flex-col gap-4 min-h-screen">
+    <%= yield %>
+    <footer class="fixed bottom-2 right-6">
+      <p class="text-sm text-right text-dawn-900/65 font-semibold">
+        <a href="https://github.com/mwnciau/skadi" target="_blank">Powered by Skadi</a>
+      </p>
+    </footer>
+  </body>
+</html>

data/app/views/skadi/dashboard/show.html.erb

@@ -0,0 +1,18 @@
+<%# locals: (dashboard_configuration:, dataset_schema:, can_edit:, can_dangerously_use_sql:) %>
+
+<% content_for :head do %>
+  <%= stylesheet_link_tag "#{Skadi::Engine.routes.url_helpers.dashboard_css_path}?#{Skadi::VERSION}", media: "all" %>
+  <%= javascript_include_tag "#{Skadi::Engine.routes.url_helpers.dashboard_js_path}?#{Skadi::VERSION}",
+    data: {
+      "dashboard-config": dashboard_configuration.to_json,
+      "dataset-schema": dataset_schema.to_json,
+      "can-edit": can_edit ? "true" : "false",
+      "can-dangerously-use-sql": can_dangerously_use_sql ? "true" : "false",
+      "fetch-data-path": dashboard_data_path,
+      "update-dashboard-path": dashboard_update_path,
+    } %>
+<% end %>
+
+<%= render "layout" do %>
+  <div id="skadi-dashboard"></div>
+<% end %>

data/config/routes.rb

@@ -0,0 +1,14 @@
+Skadi::Engine.routes.draw do
+  root to: "dashboard#show"
+
+  post "/", to: "tracking#track", as: :tracking_endpoint
+
+  get "/skadi.js", to: "asset#tracking_script", as: :tracking_script
+  get "/dashboard.css", to: "asset#dashboard_css", as: :dashboard_css
+  get "/dashboard.js", to: "asset#dashboard_js", as: :dashboard_js
+
+  get "/dashboard", to: "dashboard#show", as: :dashboard
+  post "/dashboard/update", to: "dashboard#update", as: :dashboard_update
+
+  post "/data", to: "dashboard#data", as: :dashboard_data
+end

data/lib/generators/skadi/install/USAGE

@@ -0,0 +1,20 @@
+Description:
+    Creates a migration for the Skadi Analytics tables in your application.
+
+    Run `rails db:migrate` afterwards to apply the migration.
+
+Options:
+    --db-engine=ENGINE        # Database engine: postgres, mysql, sqlite
+                              # Default: postgres
+    --user-id-type=TYPE       # Column type of your users table primary key:
+                              # bigint, integer, uuid, string
+                              # Default: bigint
+
+Example:
+    rails generate skadi:install
+
+    This will create:
+        db/migrate/<timestamp>_install_skadi.rb
+
+    For a MySQL app with UUID user ids:
+        rails generate skadi:install --db-engine=mysql --user-id-type=uuid

data/lib/generators/skadi/install/install_generator.rb

@@ -0,0 +1,46 @@
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Skadi
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      class Error < StandardError; end
+
+      include Rails::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+      desc "Generates the Skadi analytics migration"
+
+      VALID_DB_ENGINES = [ :postgres, :mysql, :sqlite ]
+      class_option :db_engine,
+        type: :string,
+        default: "postgres",
+        desc: "Database engine (postgres, mysql, sqlite)"
+
+      VALID_USER_ID_TYPES = [ :bigint, :integer, :uuid, :string ]
+      class_option :user_id_type,
+        type: :string,
+        default: "bigint",
+        desc: "Column type of your users table primary key (bigint, integer, uuid, string)"
+
+      def self.next_migration_number(dirname) = ::ActiveRecord::Generators::Base.next_migration_number(dirname)
+
+      def create_migration_file
+        raise Error.new("Invalid database engine specified") unless VALID_DB_ENGINES.include?(db_engine)
+        raise Error.new("Invalid user id type specified") unless VALID_USER_ID_TYPES.include?(user_id_type)
+
+        migration_template "skadi_migration.rb.erb",
+          "db/migrate/install_skadi.rb"
+      end
+
+      # Thor requires us to use a private block rather than private on individual definitions
+      private
+
+      def user_id_type = options[:user_id_type].to_sym
+      def db_engine = options[:db_engine].to_sym
+      def uuid_column_type = (db_engine == :postgres) ? :uuid : :string
+      def uuid_column_options = (db_engine == :postgres) ? "" : ", limit: 36"
+      def json_column_type = (db_engine == :postgres) ? :jsonb : :json
+    end
+  end
+end

data/lib/generators/skadi/install/templates/skadi_migration.rb.erb

@@ -0,0 +1,111 @@
+class <%= migration_class_name %> < ActiveRecord::Migration[<%= "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}" %>]
+  def change
+    create_table :skadi_visits do |t|
+      # A random uuid identifying the visit. This will be sent to the front-end to allow updates to the visit javascript flag without exposing details about the site analytics to the user.
+      t.<%= uuid_column_type %> :visit_token<%= uuid_column_options %>, null: false
+
+      # A token identifying the user. This will either be a token generated from an anonymity set based on the user's IP and User Agent, or it will be sourced from a cookie.
+      t.<%= uuid_column_type %> :tracking_token<%= uuid_column_options %>
+
+      # The ID of a logged in user. The FK is intentionally absent in case the host app doesn't have a users table.
+      <%- if user_id_type == :uuid && db_engine != :postgres -%>
+      t.references :user, type: :string, limit: 36, index: false
+      <%- else -%>
+      t.references :user, type: :<%= user_id_type %>, index: false
+      <%- end -%>
+
+      t.text :referrer
+      t.text :landing_page
+
+      # Standard UTM parameters
+      t.text :utm_source
+      t.text :utm_medium
+      t.text :utm_term
+      t.text :utm_content
+      t.text :utm_campaign
+
+      # Whether the visit has been verified by the front-end
+      t.boolean :verified, null: false, default: false
+      
+      t.boolean :cookies_enabled, null: false, default: false
+
+      t.timestamps
+    end
+
+    add_index :skadi_visits, :visit_token, unique: true
+    add_index :skadi_visits, :created_at
+    add_index :skadi_visits, [:tracking_token, :created_at]
+    add_index :skadi_visits, [:user_id, :created_at]
+
+    create_table :skadi_views do |t|
+      # Intentionally left nullable as not all requests will have a visit in the case a user has requested no tracking.
+      t.references :visit, foreign_key: { to_table: :skadi_visits, on_delete: :cascade }, index: false
+
+      # A random uuid identifying the view. This will be sent to the front-end to allow updates to the view metrics without exposing details about the site analytics to the user.
+      t.<%= uuid_column_type %> :view_token<%= uuid_column_options %>, null: false
+
+      t.string :controller, null: false
+      t.string :action, null: false
+      t.string :verb, null: false
+      t.text :path, null: false
+      t.<%= json_column_type %> :query_params
+
+      # The page the user clicked on when leaving the page
+      t.text :exit_page
+
+      # Whether the view has been verified by the front-end
+      t.boolean :verified, null: false, default: false
+
+      # Version of the page presented to the user (e.g. branch A/B)
+      t.string :version
+
+      t.timestamps
+    end
+
+    add_index :skadi_views, :view_token, unique: true
+    add_index :skadi_views, :created_at
+    add_index :skadi_views, [:path, :created_at]
+    add_index :skadi_views, [:visit_id, :created_at]
+
+    create_table :skadi_events do |t|
+      # Intentionally left nullable as not all requests will have a visit in the case a user has requested no tracking.
+      t.references :visit, foreign_key: { to_table: :skadi_visits, on_delete: :cascade }, index: false
+
+      # Intentionally left nullable as not all requests will have a visit in the case a user has requested no tracking.
+      t.references :view, foreign_key: { to_table: :skadi_views, on_delete: :cascade }, index: false
+
+      t.string :name, null: false
+      t.<%= json_column_type %> :properties
+
+      t.datetime :created_at, null: false
+    end
+
+    add_index :skadi_events, :created_at
+    add_index :skadi_events, [:name, :created_at]
+    add_index :skadi_events, [:view_id, :created_at]
+    add_index :skadi_events, [:visit_id, :created_at]
+    <%- if db_engine == :postgres -%>
+    add_index :skadi_events, :properties, using: :gin, opclass: :jsonb_path_ops
+    <%- end -%>
+
+    # Store demographic data separately so that it cannot be used to identify users
+    # E.g. screen size, language, timezone, pointer type (mouse, touch), can-hover, prefers reduced motion, prefers contrast, forced colours, prefers dark mode
+    create_table :skadi_demographics do |t|
+      t.string :uri, null: false
+      t.string :name, null: false
+      t.string :value, null: false
+      t.date :recorded_on, null: false
+      t.integer :count, null: false, default: 0
+    end
+
+    add_index :skadi_demographics, [:uri, :name, :value, :recorded_on], unique: true
+
+    create_table :skadi_dashboards do |t|
+      t.string :name, null: false
+      t.text :description
+      t.<%= json_column_type %> :configuration, null: false
+
+      t.timestamps
+    end
+  end
+end

data/lib/skadi/analytics.rb

@@ -0,0 +1,35 @@
+# Adds Skadi Analytics helper methods to your controller, and enabled automatic tracking if configured.
+module Skadi
+  module Analytics
+    extend ActiveSupport::Concern
+
+    included do
+      around_action :skadi_track
+
+      # Make skadi available in the view for when we output the frontend script
+      helper_method :skadi
+
+      # Add the Skadi view helper methods
+      helper Skadi::ApplicationHelper
+
+      # Disable Skadi tracking for the current controller
+      def self.do_not_track!(**kwargs)
+        skip_around_action :skadi_track, **kwargs
+      end
+    end
+
+    def skadi
+      @_skadi ||= Skadi::ControllerDelegate.new(self)
+    end
+
+    def skadi_track
+      skadi._prepare
+
+      yield
+
+      skadi._persist
+    end
+
+    delegate :do_not_track!, to: :skadi
+  end
+end

data/lib/skadi/anonymity_set.rb

@@ -0,0 +1,38 @@
+module Skadi
+  module AnonymitySet
+    # Generates a unique token for the given IP and user agent
+    # @return [String]
+    def self.calculate(ip, user_agent)
+      user_fingerprint = "#{ip}|#{user_agent}"
+
+      hash = OpenSSL::HMAC.hexdigest("sha256", pepper, user_fingerprint)
+
+      # We want a UUID-like string to be compatible with the uuid type if the database is PostgreSQL, but don't need a valid UUID
+      "#{hash[0, 8]}-#{hash[8, 4]}-#{hash[12, 4]}-#{hash[16, 4]}-#{hash[20, 12]}"
+    end
+
+    # Generate a pepper to be used in the anonymity set hash
+    # @return [String]
+    def self.pepper
+      duration = Skadi.configuration.anonymity_set_duration
+      reset_hour = Skadi.configuration.anonymity_set_reset_hour
+
+      current_time = Time.current
+      pepper_expiry = current_time + duration
+
+      if reset_hour
+        # We always want to truncate the duration to the reset hour, so if the expiry hour is before the reset hour, go back to the previous day
+        pepper_expiry -= 1.day if pepper_expiry.hour < reset_hour
+
+        pepper_expiry = pepper_expiry.change(hour: reset_hour, min: 0, sec: 0)
+
+        # Handle the case where the duration is less than one day, so the expiry is in the past
+        pepper_expiry += 1.day if pepper_expiry < current_time
+      end
+
+      Rails.cache.fetch("#{Skadi.configuration.anonymity_set_cache_key}/v1", expires_in: [ 1, pepper_expiry.to_i - current_time.to_i ].max) do
+        SecureRandom.hex(32)
+      end
+    end
+  end
+end