skadi 0.4.0.beta.1

data/app/models/skadi/dashboard_query.rb

@@ -0,0 +1,314 @@
+module Skadi
+  class DashboardQuery
+    class << self
+      # Mimics app/frontend/dashboard/dashboardElements/DynamicTable.svelte:13
+      ITEMS_PER_PAGE = 10
+      PAGES_PER_CHUNK = 10
+
+      def chart_query(chart, untrusted_param_filters)
+        queries = chart["datasets"].filter_map do |dataset|
+          if dataset["type"] == "sql"
+            build_sql_query(dataset, chart, untrusted_param_filters)
+          elsif dataset["type"] == "percentage"
+            # Percentage charts are handled by the frontend using other datasets
+            next
+          elsif Schema.database_schema.key?(dataset["type"].to_sym)
+            schema = Schema.database_schema[dataset["type"].to_sym]
+
+            build_query_from_schema(dataset, chart, schema, untrusted_param_filters)
+          else
+            raise ::Skadi::Dashboard::DatasetConfigurationError.new("Unknown dataset type #{dataset["type"]}")
+          end
+        end
+
+        return { data: [] } unless queries.any?
+
+        result = {
+          data: [],
+        }
+        Skadi::ApplicationRecord.connection.transaction do
+          if chart["type"] == "table"
+            # Note that the page starts with 1
+            page = if untrusted_param_filters["page"].is_a?(Integer) && untrusted_param_filters["page"] > 0
+              untrusted_param_filters["page"]
+            else
+              1
+            end
+
+            # We cannot union more than one query when selecting multiple fields for the table dataset so just take the first one as a precaution
+            query = queries.first
+
+            result[:resultCount] = Skadi::Visit.connection.select_one("
+                SELECT COUNT(*) as count FROM (#{query}) q
+            ")["count"]
+
+            last_page = [ 1, (result[:resultCount].to_f / ITEMS_PER_PAGE).ceil ].max
+            offset = ([ page, last_page ].min - 1) * ITEMS_PER_PAGE
+            limit = ITEMS_PER_PAGE * PAGES_PER_CHUNK
+
+            result[:resultOffset] = offset
+
+            result[:data] = Skadi::Visit.connection.select_all("
+                SELECT * FROM (#{query}) q LIMIT #{limit} OFFSET #{offset}
+            ")
+          else
+            result[:data] = Skadi::ApplicationRecord.connection.unprepared_statement do
+              Skadi::Visit.connection.select_all(queries.join(" UNION ALL "))
+            end
+          end
+
+          # Rollback after the SQL is run, preventing some side effects
+          raise ActiveRecord::Rollback
+        end
+
+        return result
+      end
+
+      private def build_sql_query(dataset, chart, untrusted_param_filters)
+        model = Skadi::Visit
+
+        safe_group = [ "t.split" ]
+        safe_id = model.connection.quote(dataset["id"])
+
+        if chart["time_series"].present?
+          safe_aggregated_date = Helpers::Sql.time_series(chart["time_series"], model, "t.date")
+          safe_group << safe_aggregated_date
+        else
+          safe_aggregated_date = "NULL"
+        end
+
+        safe_where = build_sql_where(chart, model, untrusted_param_filters)
+
+        return "
+            SELECT
+              #{safe_id} as id,
+              #{safe_aggregated_date} as date,
+              t.split as split,
+              SUM(t.count) as count
+            FROM
+              (#{dataset["sql"]}) AS t
+            #{safe_where}
+            GROUP BY
+                #{safe_group.join(", ")}
+          "
+      end
+
+      private def build_sql_where(chart, model, untrusted_param_filters)
+        safe_where = []
+
+        url_date_from = untrusted_param_filters["date_from"] if valid_date_string?(untrusted_param_filters["date_from"])
+        date_from = combine_dates(chart["date_from"], url_date_from, type: :from)
+        safe_where << "DATE(t.date) >= #{model.connection.quote(date_from)}" unless date_from.nil?
+
+        url_date_to = untrusted_param_filters["date_to"] if valid_date_string?(untrusted_param_filters["date_to"])
+        date_to = combine_dates(chart["date_to"], url_date_to, type: :to)
+        safe_where << "DATE(t.date) <= #{model.connection.quote(date_to)}" unless date_to.nil?
+
+        return safe_where.any? ? "WHERE #{safe_where.join(" AND ")}" : ""
+      end
+
+      private def build_query_from_schema(dataset, chart, schema, untrusted_param_filters)
+        query = schema[:model].all
+
+        query = apply_schema_filters(dataset, schema, query)
+        query = apply_chart_filters(dataset, chart, schema, query, untrusted_param_filters)
+
+        query = if chart["type"] == "table"
+          select_table_query(schema, query)
+        else
+          select_split_and_group_chart_query(dataset, chart, schema, query)
+        end
+
+        return query.to_sql
+      end
+
+      POSITIVE_OPERATORS = [ "=", ">", ">=", "<=", "<", "like" ].freeze
+      NEGATED_OPERATORS = [ "!=", "not like" ].freeze
+
+      private def apply_schema_filters(dataset, schema, query)
+        model = schema[:model]
+
+        if dataset["filters"].is_a?(Array)
+          dataset["filters"].each do |filter|
+            next unless filter.is_a?(Hash)
+
+            field = filter["field"]
+            field_config = schema[:fields][field.to_s.to_sym]
+            operator = filter["operator"]
+            value = filter["value"]
+
+            next if field_config.nil? || operator.nil? || !field_config[:filter]
+
+            field_type = field_config[:type] || :string
+            field_sql = if field_config[:sql]
+              field_config[:sql]
+            else
+              "#{model.table_name}.#{model.connection.quote_column_name(field)}"
+            end
+
+            if field_type == :date
+              field_sql = "DATE(#{field_sql})"
+            end
+
+            if POSITIVE_OPERATORS.include?(operator)
+              operator = Helpers::Sql.operator(schema[:model], filter["operator"])
+
+              # For the non-negated operators, we consider an empty string the same as null
+              if field_type == :string && value == ""
+                query = query.where("#{field_sql} IS NULL OR #{field_sql} #{operator} ?", value)
+              else
+                query = query.where("#{field_sql} #{operator} ?", value)
+              end
+            elsif NEGATED_OPERATORS.include?(operator)
+              operator = Helpers::Sql.operator(schema[:model], filter["operator"])
+
+              # For the negated operators, we change the behaviour of SQL so that nulls are considered different (rather than defaulting to false)
+              if field_type == :string && value == ""
+                query = query.where("#{field_sql} #{operator} ?", value)
+              else
+                query = query.where("#{field_sql} IS NULL OR #{field_sql} #{operator} ?", value)
+              end
+            elsif operator == "empty"
+              if field_type == :string
+                query = query.where("#{field_sql} = '' OR #{field_sql} IS NULL")
+              else
+                query = query.where("#{field_sql} IS NULL")
+              end
+            elsif operator == "not empty"
+              if field_type == :string
+                query = query.where("#{field_sql} != '' AND #{field_sql} IS NOT NULL")
+              else
+                query = query.where("#{field_sql} IS NOT NULL")
+              end
+            else
+              raise Dashboard::UnsupportedOperatorError.new("Unknown operator #{operator}")
+            end
+          end
+        end
+
+        return query
+      end
+
+      private def apply_chart_filters(_dataset, chart, schema, query, untrusted_param_filters)
+        date_config = schema[:fields][:date]
+
+        if date_config
+          url_date_from = untrusted_param_filters["date_from"] if valid_date_string?(untrusted_param_filters["date_from"])
+          date_from = combine_dates(chart["date_from"], url_date_from, type: :from)
+          query = query.where("DATE(#{date_config[:sql]}) >= ?", date_from) unless date_from.nil?
+
+          url_date_to = untrusted_param_filters["date_to"] if valid_date_string?(untrusted_param_filters["date_to"])
+          date_to = combine_dates(chart["date_to"], url_date_to, type: :to)
+          query = query.where("DATE(#{date_config[:sql]}) <= ?", date_to) unless date_to.nil?
+        end
+
+        if schema[:visit_key]
+          # Join the visits if we need to
+          if chart["verified_visits"] == true || chart["visit_tracking"] || chart["unique_by"] == "visitor"
+            if schema[:model] != Skadi::Visit
+              query = query.joins(%(INNER JOIN skadi_visits ON skadi_visits.id = #{schema[:model].table_name}.#{schema[:visit_key]}))
+            end
+          end
+
+          if chart["verified_visits"] == true
+            query = query.where("skadi_visits.verified = TRUE")
+          end
+
+          if chart["visit_tracking"] == "any"
+            query = query.where("skadi_visits.tracking_token IS NOT NULL")
+          elsif chart["visit_tracking"] == "anonymity_set"
+            query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = FALSE")
+          elsif chart["visit_tracking"] == "cookie"
+            query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = TRUE")
+          end
+        end
+
+        return query
+      end
+
+      private def select_table_query(schema, query)
+        model = schema[:model]
+        select_fields = schema[:fields].map do |key, field_config|
+          quoted_field = model.connection.quote_column_name(key)
+
+          if field_config[:sql]
+            "#{field_config[:sql]} AS #{quoted_field}"
+          else
+            "#{model.table_name}.#{quoted_field}"
+          end
+        end
+
+        return query.select(*select_fields)
+      end
+
+      private def select_split_and_group_chart_query(dataset, chart, schema, query)
+        model = schema[:model]
+        split_columns = safe_split_columns(dataset, schema)
+
+        safe_id = model.connection.quote(dataset["id"])
+        safe_date = "NULL"
+        safe_split = "NULL"
+        safe_count = "COUNT(*)"
+        safe_group = split_columns.dup
+
+        if chart["time_series"].present? && schema[:fields][:date]
+          safe_date = Helpers::Sql.time_series(chart["time_series"], model, schema[:fields][:date][:sql])
+          safe_group << safe_date
+        end
+
+        # If there are split columns, we override the default value of split using them
+        if split_columns.length == 1
+          safe_split = split_columns.first
+        elsif split_columns.length > 1
+          # Concatenate the split columns, separating the columns with a token that can be processed in the front end
+          safe_split = "CONCAT(#{split_columns.join(", '|~|', ")})"
+        end
+
+        if schema[:count_sql]
+          safe_count = schema[:count_sql]
+        elsif chart["unique_by"] == "visit" && schema[:visit_key]
+          safe_count = "COUNT(DISTINCT #{schema[:model].table_name}.#{schema[:visit_key]})"
+        elsif chart["unique_by"] == "visitor" && schema[:visit_key]
+          safe_count = "COUNT(DISTINCT skadi_visits.tracking_token)"
+        end
+
+        return query
+            .select(
+              "#{safe_id} AS id",
+              "#{safe_date} AS date",
+              "#{safe_split} AS split",
+              "#{safe_count} AS count",
+            )
+            .group(safe_group)
+      end
+
+      private def safe_split_columns(dataset, schema)
+        safe_split = []
+
+        if dataset["split_by"].is_a?(Array)
+          dataset["split_by"].each do |field|
+            field_config = schema[:fields][field.to_sym]
+            if field_config && field_config[:split]
+              safe_field = schema[:model].connection.quote_column_name(field)
+
+              safe_split << (field_config[:sql] || %(#{schema[:model].table_name}.#{safe_field}))
+            end
+          end
+        end
+
+        return safe_split
+      end
+
+      private def valid_date_string?(date) = date.is_a?(String) && date.match(/\A\d{4}-[01]\d-[0-3]\d\z/)
+
+      # Combine dates for the date ranges, being conservative when combining time ranges
+      private def combine_dates(*dates, type:)
+        if type == :from
+          return dates.compact.max
+        end
+
+        return dates.compact.min
+      end
+    end
+  end
+end

data/app/models/skadi/dashboard_validator.rb

@@ -0,0 +1,269 @@
+module Skadi
+  # Validates that a Dashboard's configuration matches the DashboardConfig type in app/frontend/types.d.ts
+  class DashboardValidator < ActiveModel::Validator
+    OneOf = Struct.new(:allowed_values, :allow_missing)
+    ArrayOf = Struct.new(:element_type, :minimum_elements)
+    Filter = Struct.new(:filters)
+
+    FILTER_TYPES = {
+      boolean: {
+        operators: [ "=", "!=", "empty", "not empty" ],
+      },
+      date: {
+        operators: [ "=", ">", ">=", "<=", "<", "!=", "empty", "not empty" ],
+      },
+      number: {
+        operators: [ "=", ">", ">=", "<=", "<", "!=", "empty", "not empty" ],
+      },
+      one_of: {
+        operators: [ "=", "!=", "empty", "not empty" ],
+      },
+      string: {
+        operators: [ "=", "!=", "like", "not like", "empty", "not empty" ],
+      },
+    }
+
+    COMMON_DATASET_FIELDS = {
+      id: :string,
+      label: :string,
+      visible: :boolean?,
+      axis: OneOf.new(allowed_values: %w[left right].freeze, allow_missing: true),
+    }
+
+    def self.types
+      return @types if defined?(@types)
+
+      @types = {
+        DashboardTabConfig: {
+          id: :string,
+          title: :string,
+          description: :string?,
+          date_from: :date?,
+          date_to: :date?,
+          children: ArrayOf.new(:ChartConfig),
+        },
+        ChartConfig: {
+          id: :string,
+          type: OneOf.new(allowed_values: %w[bar line table].freeze).freeze,
+          title: :string,
+          description: :string?,
+          time_series: OneOf.new(allowed_values: %w[daily weekly monthly].freeze, allow_missing: true).freeze,
+          date_from: :date?,
+          date_to: :date?,
+          verified_visits: :boolean?,
+          unique_by: OneOf.new(allowed_values: %w[visit visitor].freeze, allow_missing: true).freeze,
+          visit_tracking: OneOf.new(allowed_values: %w[any anonymity_set cookie].freeze, allow_missing: true).freeze,
+          datasets: ArrayOf.new(:Dataset, 1),
+        },
+        percentageDataset: {
+          **COMMON_DATASET_FIELDS,
+          type: "percentage",
+
+          numerator: :dataset_id,
+          denominator: :dataset_id,
+        },
+        sqlDataset: {
+          **COMMON_DATASET_FIELDS,
+          type: "sql",
+
+          sql: :sql,
+        },
+      }
+
+      Schema.database_schema.each do |dataset_name, schema|
+        dataset_type = COMMON_DATASET_FIELDS.dup
+        dataset_type[:type] = dataset_name.to_s
+
+        split_by_fields = []
+        filters = {}
+
+        schema[:fields].each do |field, field_config|
+          split_by_fields << field.to_s if field_config[:split]
+          next unless field_config[:filter]
+
+          if field_config[:type] == :one_of
+            filters[field] = OneOf.new(allowed_values: field_config[:options], allow_missing: true)
+          else
+            filters[field] = field_config[:type] || :string
+          end
+        end
+
+        dataset_type[:split_by] = ArrayOf.new(OneOf.new(split_by_fields, false)) if split_by_fields.any?
+        dataset_type[:filters] = ArrayOf.new(Filter.new(filters))
+
+        @types[:"#{dataset_name}Dataset"] = dataset_type
+      end
+
+      return @types
+    end
+
+    Context = Struct.new(:record, :allowed_sql_strings, :dataset_ids)
+
+    def validate(record)
+      context = Context.new(record, nil, [])
+
+      validate_type(ArrayOf.new(:DashboardTabConfig, 1), record.configuration, "configuration", context:)
+    end
+
+    private def validate_type(type, value, path, context:)
+      return validate_array_of(type, value, path, context:) if type.is_a?(ArrayOf)
+      return validate_hash(type, value, path, context:) if type.is_a?(Hash)
+      return validate_filter(type, value, path, context:) if type.is_a?(Filter)
+      return validate_one_of(type, value, path, context:) if type.is_a?(OneOf)
+
+      if type.is_a?(String)
+        add_error(path, "must be #{type}", context:) unless type == value
+
+        return
+      end
+
+      if type.is_a?(Symbol) && type.end_with?("?")
+        return if value.nil?
+
+        type = type.to_s.delete_suffix("?").to_sym
+      end
+
+      case type
+        when :string
+          add_error(path, "must be a string", context:) unless value.is_a?(String)
+          return
+        when :boolean
+          add_error(path, "must be a boolean", context:) unless value == true || value == false
+          return
+        when :number
+          add_error(path, "must be a number", context:) unless value.is_a?(Numeric)
+          return
+        when :date
+          add_error(path, "must be a date", context:) unless value.is_a?(String) && value.match?(/\A\d{4}-[01]\d-[0-3]\d\z/)
+          return
+        when :dataset_id
+          add_error(path, "must be a dataset id", context:) unless context.dataset_ids.include?(value)
+          return
+        when :sql
+          return validate_sql(value, path, context:)
+        when :Dataset
+          return validate_dataset(value, path, context:)
+        else
+          return validate_custom_type(type, value, path, context:)
+      end
+    end
+
+    # Array of a particular type
+    # @param [ArrayOf] type
+    private def validate_array_of(type, value, path, context:)
+      return if type.minimum_elements.nil? && value.nil?
+      return add_error(path, "must be an array", context:) unless value.is_a?(Array)
+      return add_error(path, "must have at least one element", context:) if type.minimum_elements == 1 && value.empty?
+
+      return value.each_with_index do |item, index|
+        validate_type(type.element_type, item, "#{path}[#{index}]", context:)
+      end
+    end
+
+    private def validate_hash(type, value, path, context:)
+      return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
+
+      extra_keys = value.keys.map(&:to_sym) - type.keys
+      extra_keys.each do |key|
+        add_error("#{path}.#{key}", "is not a valid key", context:)
+      end
+
+      type.each do |item_key, item_type|
+        validate_type(item_type, value[item_key.to_s], "#{path}.#{item_key}", context:)
+      end
+    end
+
+    private def validate_filter(type, value, path, context:)
+      return add_error(path, "must be a Hash", context:) unless value.is_a?(Hash)
+
+      filters = type.filters
+      filter_field = value["field"].to_s.to_sym
+      return add_error("#{path}.field", "must be one of #{filters.keys.join(", ")}", context:) unless filters.key?(filter_field)
+
+      filter_type = filters[filter_field]
+      allowed_operators = FILTER_TYPES[filter_type.is_a?(OneOf) ? :one_of : filter_type][:operators]
+      filter_operator = value["operator"]
+
+      return add_error("#{path}.operator", "must be one of #{allowed_operators.join(", ")}", context:) unless allowed_operators.include?(filter_operator)
+
+      # These do not require a value to be specified
+      if [ "empty", "not empty" ].include?(filter_operator)
+        add_error("#{path}.value", "is not a valid key", context:) if value.key?("value")
+
+        return
+      end
+
+      return add_error("#{path}.value", "must be set", context:) if value["value"].nil?
+
+      return validate_type(filter_type, value["value"], "#{path}.value", context:)
+    end
+
+    private def validate_one_of(type, value, path, context:)
+        return if type.allow_missing && value.nil?
+        return if type.allowed_values.include?(value)
+
+        add_error(path, "#{value.inspect} must be one of #{type.allowed_values.map(&:inspect).join(", ")}", context:)
+    end
+
+    private def validate_sql(value, path, context:)
+      unless context.record.can_dangerously_use_sql
+        return add_error(path, "cannot be modified", context:) unless allowed_sql_strings(context:).include?(value)
+      end
+
+      unless value.is_a?(String)
+        return add_error(path, "must be a string", context:)
+      end
+
+      unless [ "date", "split", "count" ].all? { |it| value.include?(it) }
+        add_error(path, "must include the fields date, split and count", context:)
+      end
+    end
+
+    private def validate_dataset(value, path, context:)
+      return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
+
+      type = :"#{value["type"]}Dataset"
+      return add_error("#{path}.type", "is not a valid dataset type", context:) unless self.class.types.include?(type)
+
+      validate_type(type, value, path, context:)
+    end
+
+    private def validate_custom_type(type, value, path, context:)
+      return add_error(path, "Unknown type #{type.inspect}", context:) unless self.class.types.key?(type)
+
+      if type == :ChartConfig && value.is_a?(Hash)
+        context.dataset_ids = []
+
+        if value["datasets"].is_a?(Array)
+          # Special case for table chart type, limiting datasets to 1 and ruling out derived datasets
+          if value["type"] == "table"
+            if value["datasets"].is_a?(Array) && value["datasets"].length != 1
+              return add_error("#{path}.datasets", "must have one dataset for the table type", context:)
+            end
+            if value["datasets"].is_a?(Array) && value["datasets"].first.is_a?(Hash) && value["datasets"].first["type"] == "percentage"
+              return add_error("#{path}.datasets[0].type", "cannot be percentage for the table type", context:)
+            end
+          end
+
+          # Store dataset ids for the current chart for the :dataset_id type
+          value["datasets"].each do |dataset|
+            context.dataset_ids << dataset["id"] if dataset.is_a?(Hash) && dataset["id"].is_a?(String)
+          end
+        end
+      end
+
+      return validate_type(self.class.types[type], value, path, context:)
+    end
+
+    private def add_error(path, message, context:)
+      context.record.errors.add(:configuration, "#{path} #{message}")
+    end
+
+    private def allowed_sql_strings(context:)
+      # Return the cached SQL strings if already calculated
+      return context.allowed_sql_strings if context.allowed_sql_strings.is_a?(Array)
+
+      return context.allowed_sql_strings = context.record.sql_strings
+    end
+  end
+end

data/app/models/skadi/demographic.rb

@@ -0,0 +1,31 @@
+module Skadi
+  class Demographic < ApplicationRecord
+    self.implicit_order_column = :recorded_on
+
+    validates :name, presence: true
+    validates :value, presence: true
+
+    validates :recorded_on, presence: true
+
+    def self.create_or_increment_all(demographics)
+      demographics_to_upsert = demographics.map do |demographic|
+        {
+          name: demographic[:name].strip[0, 255],
+          value: demographic[:value].strip[0, 255],
+          # SQL specifies NULL values are not equal, so we need to default the URI to an empty string
+          # to ensure the unique index works correctly
+          uri: demographic[:uri]&.strip&.[](0, 255) || "",
+          recorded_on: Time.current,
+          count: 1,
+        }
+      end
+
+      Skadi::Demographic.upsert_all(
+        demographics_to_upsert,
+        unique_by: [ :uri, :name, :value, :recorded_on ],
+        on_duplicate: Arel.sql("count = skadi_demographics.count + 1"),
+        returning: false,
+      )
+    end
+  end
+end

data/app/models/skadi/event.rb

@@ -0,0 +1,33 @@
+module Skadi
+  class Event < ApplicationRecord
+    belongs_to :visit, class_name: "Skadi::Visit", optional: true, inverse_of: :events
+    belongs_to :view, class_name: "Skadi::View", optional: true, inverse_of: :events
+
+    validates :name, presence: true
+
+    # Redacts events that are marked as sensitive
+    def self.redact_and_insert(events, view:, visit:)
+      events.each do |event|
+        # Note: both paths must have the same set of keys, which is a requirement for upsert_all
+        if event[:sensitive]
+          event[:view_id] = nil
+          event[:visit_id] = nil
+
+          # Sensitive events should have their created date redacted so they cannot be linked to views/visits based on timings
+          event[:created_at] = Time.current.beginning_of_day
+        else
+          # Attach events to the current visit and view, only if it is not marked as sensitive
+          event[:view_id] = view.id
+          event[:visit_id] = visit&.id
+          event[:created_at] = Time.current
+        end
+
+        event[:name] = event[:name].strip[0, 255]
+
+        event.delete(:sensitive)
+      end
+
+      Event.insert_all(events)
+    end
+  end
+end