blazer 1.7.7 → 2.6.4

data/app/controllers/blazer/base_controller.rb

@@ -1,14 +1,12 @@
 module Blazer
   class BaseController < ApplicationController
-    # skip all filters
-    filters = _process_action_callbacks.map(&:filter)
-    if Rails::VERSION::MAJOR >= 5
-      skip_before_action(*filters, raise: false)
-      skip_after_action(*filters, raise: false)
-      skip_around_action(*filters, raise: false)
-    else
-      skip_action_callback *filters
-    end
+    # skip filters
+    filters = _process_action_callbacks.map(&:filter) - [:activate_authlogic]
+    skip_before_action(*filters, raise: false)
+    skip_after_action(*filters, raise: false)
+    skip_around_action(*filters, raise: false)
+
+    clear_helpers
 
     protect_from_forgery with: :exception
 
@@ -16,46 +14,52 @@ module Blazer
       http_basic_authenticate_with name: ENV["BLAZER_USERNAME"], password: ENV["BLAZER_PASSWORD"]
     end
 
+    if Blazer.settings["before_action"]
+      raise Blazer::Error, "The docs for protecting Blazer with a custom before_action had an incorrect example from August 2017 to June 2018. The example method had a boolean return value. However, you must render or redirect if a user is unauthorized rather than return a falsy value. Double check that your before_action works correctly for unauthorized users (if it worked when added, there should be no issue). Then, change before_action to before_action_method in config/blazer.yml."
+    end
+
     if Blazer.before_action
       before_action Blazer.before_action.to_sym
     end
 
+    if Blazer.override_csp
+      after_action do
+        response.headers['Content-Security-Policy'] = "default-src 'self' https: 'unsafe-inline' 'unsafe-eval' data:"
+      end
+    end
+
     layout "blazer/application"
 
     private
 
-      def process_vars(statement, data_source)
-        (@bind_vars ||= []).concat(extract_vars(statement)).uniq!
+      def process_vars(statement, var_params = nil)
+        var_params ||= request.query_parameters
+        (@bind_vars ||= []).concat(statement.variables).uniq!
+        # update in-place so populated in view and consistent across queries on dashboard
         @bind_vars.each do |var|
-          params[var] ||= Blazer.data_sources[data_source].variable_defaults[var]
-        end
-        @success = @bind_vars.all? { |v| params[v] }
-
-        if @success
-          @bind_vars.each do |var|
-            value = params[var].presence
-            if value
-              if ["start_time", "end_time"].include?(var)
-                value = value.to_s.gsub(" ", "+") # fix for Quip bug
-              end
-
-              if var.end_with?("_at")
-                begin
-                  value = Blazer.time_zone.parse(value)
-                rescue
-                  # do nothing
-                end
-              end
-
-              if value =~ /\A\d+\z/
-                value = value.to_i
-              elsif value =~ /\A\d+\.\d+\z/
-                value = value.to_f
-              end
-            end
-            statement.gsub!("{#{var}}", ActiveRecord::Base.connection.quote(value))
+          if !var_params[var]
+            default = statement.data_source.variable_defaults[var]
+            # only add if default exists
+            var_params[var] = default if default
           end
         end
+        runnable = @bind_vars.all? { |v| var_params[v] }
+        statement.add_values(var_params) if runnable
+        runnable
+      end
+
+      def refresh_query(query)
+        statement = query.statement_object
+        runnable = process_vars(statement)
+        cohort_analysis_statement(statement) if statement.cohort_analysis?
+        statement.clear_cache if runnable
+      end
+
+      def add_cohort_analysis_vars
+        @bind_vars << "cohort_period" unless @bind_vars.include?("cohort_period")
+        @smart_vars["cohort_period"] = ["day", "week", "month"] if @smart_vars
+        # TODO create var_params method
+        request.query_parameters["cohort_period"] ||= "week"
       end
 
       def parse_smart_variables(var, data_source)
@@ -79,20 +83,38 @@ module Blazer
         [smart_var, error]
       end
 
-      def extract_vars(statement)
-        # strip commented out lines
-        # and regex {1} or {1,2}
-        statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq
+      def cohort_analysis_statement(statement)
+        @cohort_period = params["cohort_period"] || "week"
+        @cohort_period = "week" unless ["day", "week", "month"].include?(@cohort_period)
+
+        # for now
+        @conversion_period = @cohort_period
+        @cohort_days =
+          case @cohort_period
+          when "day"
+            1
+          when "week"
+            7
+          when "month"
+            30
+          end
+
+        statement.apply_cohort_analysis(period: @cohort_period, days: @cohort_days)
       end
-      helper_method :extract_vars
 
-      def variable_params
-        params.except(:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id).permit!
+      # TODO allow all keys
+      # or show error message for disallowed keys
+      UNPERMITTED_KEYS = [:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id, :script_name, :original_script_name]
+
+      def variable_params(resource, var_params = nil)
+        permitted_keys = resource.variables - UNPERMITTED_KEYS.map(&:to_s)
+        var_params ||= request.query_parameters
+        var_params.slice(*permitted_keys)
       end
       helper_method :variable_params
 
       def blazer_user
-        send(Blazer.user_method) if Blazer.user_method && respond_to?(Blazer.user_method)
+        send(Blazer.user_method) if Blazer.user_method && respond_to?(Blazer.user_method, true)
       end
       helper_method :blazer_user
 
@@ -101,5 +123,10 @@ module Blazer
         action = resource.persisted? ? :edit : :new
         render action, status: :unprocessable_entity
       end
+
+      # do not inherit from ApplicationController - #120
+      def default_url_options
+        {}
+      end
   end
 end

data/app/controllers/blazer/checks_controller.rb

@@ -46,7 +46,7 @@ module Blazer
     private
 
       def check_params
-        params.require(:check).permit(:query_id, :emails, :invert, :check_type, :schedule)
+        params.require(:check).permit(:query_id, :emails, :slack_channels, :invert, :check_type, :schedule)
       end
 
       def set_check

data/app/controllers/blazer/dashboards_controller.rb

@@ -2,10 +2,6 @@ module Blazer
   class DashboardsController < BaseController
     before_action :set_dashboard, only: [:show, :edit, :update, :destroy, :refresh]
 
-    def index
-      redirect_to root_path(filter: "dashboards")
-    end
-
     def new
       @dashboard = Blazer::Dashboard.new
     end
@@ -26,7 +22,7 @@ module Blazer
     def show
       @queries = @dashboard.dashboard_queries.order(:position).preload(:query).map(&:query)
       @queries.each do |query|
-        process_vars(query.statement, query.data_source)
+        @success = process_vars(query.statement_object)
       end
       @bind_vars ||= []
 
@@ -40,6 +36,8 @@ module Blazer
           @sql_errors << error if error
         end
       end
+
+      add_cohort_analysis_vars if @queries.any?(&:cohort_analysis?)
     end
 
     def edit
@@ -47,7 +45,7 @@ module Blazer
 
     def update
       if update_dashboard(@dashboard)
-        redirect_to dashboard_path(@dashboard, variable_params)
+        redirect_to dashboard_path(@dashboard, params: variable_params(@dashboard))
       else
         render_errors @dashboard
       end
@@ -55,18 +53,14 @@ module Blazer
 
     def destroy
       @dashboard.destroy
-      redirect_to dashboards_path
+      redirect_to root_path
     end
 
     def refresh
       @dashboard.queries.each do |query|
-        data_source = Blazer.data_sources[query.data_source]
-        statement = query.statement.dup
-        process_vars(statement, query.data_source)
-        Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
-        data_source.clear_cache(statement)
+        refresh_query(query)
       end
-      redirect_to dashboard_path(@dashboard, variable_params)
+      redirect_to dashboard_path(@dashboard, params: variable_params(@dashboard))
     end
 
     private

data/app/controllers/blazer/queries_controller.rb

@@ -1,15 +1,12 @@
 module Blazer
   class QueriesController < BaseController
     before_action :set_query, only: [:show, :edit, :update, :destroy, :refresh]
+    before_action :set_data_source, only: [:tables, :docs, :schema, :cancel]
 
     def home
-      if params[:filter] == "dashboards"
-        @queries = []
-      else
-        set_queries(1000)
-      end
+      set_queries(1000)
 
-      if params[:filter] && params[:filter] != "dashboards"
+      if params[:filter]
         @dashboards = [] # TODO show my dashboards
       else
         @dashboards = Blazer::Dashboard.order(:name)
@@ -41,49 +38,72 @@ module Blazer
       if params[:fork_query_id]
         @query.statement ||= Blazer::Query.find(params[:fork_query_id]).try(:statement)
       end
+      if params[:upload_id]
+        upload = Blazer::Upload.find(params[:upload_id])
+        upload_settings = Blazer.settings["uploads"]
+        @query.data_source ||= upload_settings["data_source"]
+        @query.statement ||= "SELECT * FROM #{upload.table_name} LIMIT 10"
+      end
     end
 
     def create
       @query = Blazer::Query.new(query_params)
       @query.creator = blazer_user if @query.respond_to?(:creator)
+      @query.status = "active" if @query.respond_to?(:status)
 
       if @query.save
-        redirect_to query_path(@query, variable_params)
+        redirect_to query_path(@query, params: variable_params(@query))
       else
         render_errors @query
       end
     end
 
     def show
-      @statement = @query.statement.dup
-      process_vars(@statement, @query.data_source)
+      @statement = @query.statement_object
+      @success = process_vars(@statement)
 
       @smart_vars = {}
       @sql_errors = []
-      data_source = Blazer.data_sources[@query.data_source]
       @bind_vars.each do |var|
-        smart_var, error = parse_smart_variables(var, data_source)
+        smart_var, error = parse_smart_variables(var, @statement.data_source)
         @smart_vars[var] = smart_var if smart_var
         @sql_errors << error if error
       end
 
-      Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+      @query.update!(status: "active") if @query.respond_to?(:status) && @query.status.in?(["archived", nil])
+
+      add_cohort_analysis_vars if @query.cohort_analysis?
     end
 
     def edit
     end
 
     def run
-      @statement = params[:statement]
-      data_source = params[:data_source]
-      process_vars(@statement, data_source)
-      @only_chart = params[:only_chart]
-      @run_id = blazer_params[:run_id]
       @query = Query.find_by(id: params[:query_id]) if params[:query_id]
+
+      # use query data source when present
+      # need to update viewable? logic below if this changes
       data_source = @query.data_source if @query && @query.data_source
+      data_source ||= params[:data_source]
       @data_source = Blazer.data_sources[data_source]
 
-      if @run_id
+      @statement = Blazer::Statement.new(params[:statement], @data_source)
+      # before process_vars
+      @cohort_analysis = @statement.cohort_analysis?
+
+      # fallback for now for users with open tabs
+      # TODO remove fallback in future version
+      @var_params = request.request_parameters["variables"] || request.request_parameters
+      @success = process_vars(@statement, @var_params)
+      @only_chart = params[:only_chart]
+      @run_id = blazer_params[:run_id]
+
+      run_cohort_analysis if @cohort_analysis
+
+      # ensure viewable
+      if !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)
+        render_forbidden
+      elsif @run_id
         @timestamp = blazer_params[:timestamp].to_i
 
         @result = @data_source.run_results(@run_id)
@@ -113,16 +133,15 @@ module Blazer
 
         options = {user: blazer_user, query: @query, refresh_cache: params[:check], run_id: @run_id, async: Blazer.async}
         if Blazer.async && request.format.symbol != :csv
-          result = []
-          Blazer::RunStatementJob.perform_async(result, @data_source, @statement, options)
+          Blazer::RunStatementJob.perform_later(@data_source.id, @statement.statement, options.merge(values: @statement.values))
           wait_start = Time.now
           loop do
-            sleep(0.02)
-            break if result.any? || Time.now - wait_start > 3
+            sleep(0.1)
+            @result = @data_source.run_results(@run_id)
+            break if @result || Time.now - wait_start > 3
           end
-          @result = result.first
         else
-          @result = Blazer::RunStatement.new.perform(@data_source, @statement, options)
+          @result = Blazer::RunStatement.new.perform(@statement, options)
         end
 
         if @result
@@ -134,6 +153,13 @@ module Blazer
           @cached_at = @result.cached_at
           @just_cached = @result.just_cached
 
+          @forecast = @query && @result.forecastable? && params[:forecast]
+          if @forecast
+            @result.forecast
+            @forecast_error = @result.forecast_error
+            @forecast = @forecast_error.nil?
+          end
+
           render_run
         else
           @timestamp = Time.now.to_i
@@ -145,12 +171,8 @@ module Blazer
     end
 
     def refresh
-      data_source = Blazer.data_sources[@query.data_source]
-      @statement = @query.statement.dup
-      process_vars(@statement, @query.data_source)
-      Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
-      data_source.clear_cache(@statement)
-      redirect_to query_path(@query, variable_params)
+      refresh_query(@query)
+      redirect_to query_path(@query, params: variable_params(@query))
     end
 
     def update
@@ -158,11 +180,12 @@ module Blazer
         @query = Blazer::Query.new
         @query.creator = blazer_user if @query.respond_to?(:creator)
       end
+      @query.status = "active" if @query.respond_to?(:status)
       unless @query.editable?(blazer_user)
         @query.errors.add(:base, "Sorry, permission denied")
       end
       if @query.errors.empty? && @query.update(query_params)
-        redirect_to query_path(@query, variable_params)
+        redirect_to query_path(@query, params: variable_params(@query))
       else
         render_errors @query
       end
@@ -170,24 +193,38 @@ module Blazer
 
     def destroy
       @query.destroy if @query.editable?(blazer_user)
-      redirect_to root_url
+      redirect_to root_path
     end
 
     def tables
-      render json: Blazer.data_sources[params[:data_source]].tables
+      render json: @data_source.tables
+    end
+
+    def docs
+      @smart_variables = @data_source.smart_variables
+      @linked_columns = @data_source.linked_columns
+      @smart_columns = @data_source.smart_columns
     end
 
     def schema
-      @schema = Blazer.data_sources[params[:data_source]].schema
+      @schema = @data_source.schema
     end
 
     def cancel
-      Blazer.data_sources[params[:data_source]].cancel(blazer_run_id)
-      render json: {}
+      @data_source.cancel(blazer_run_id)
+      head :ok
     end
 
     private
 
+      def set_data_source
+        @data_source = Blazer.data_sources[params[:data_source]]
+
+        unless Query.new(data_source: @data_source.id).editable?(blazer_user)
+          render_forbidden
+        end
+      end
+
       def continue_run
         render json: {run_id: @run_id, timestamp: @timestamp}, status: :accepted
       end
@@ -198,7 +235,7 @@ module Blazer
         @first_row = @rows.first || []
         @column_types = []
         if @rows.any?
-          @columns.each_with_index do |column, i|
+          @columns.each_with_index do |_, i|
             @column_types << (
               case @first_row[i]
               when Integer
@@ -212,7 +249,6 @@ module Blazer
           end
         end
 
-        @filename = @query.name.parameterize if @query
         @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
 
         @boom = @result.boom if @result
@@ -229,7 +265,9 @@ module Blazer
                 r[lat_index] && r[lon_index]
               end.map do |r|
                 {
-                  title: r.each_with_index.map{ |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{@columns[i]}:</strong> #{v}" }.compact.join("<br />").truncate(140),
+                  # Mapbox.js does sanitization with https://github.com/mapbox/sanitize-caja
+                  # but we should do it here as well
+                  title: r.each_with_index.map { |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{ERB::Util.html_escape(@columns[i])}:</strong> #{ERB::Util.html_escape(v)}" }.compact.join("<br />").truncate(140),
                   latitude: r[lat_index],
                   longitude: r[lon_index]
                 }
@@ -237,24 +275,22 @@ module Blazer
           end
         end
 
+        render_cohort_analysis if @cohort_analysis && !@error
+
         respond_to do |format|
           format.html do
             render layout: false
           end
           format.csv do
+            # not ideal, but useful for testing
+            raise Error, @error if @error && Rails.env.test?
+
             send_data csv_data(@columns, @rows, @data_source), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
           end
         end
       end
 
       def set_queries(limit = nil)
-        @my_queries =
-          if limit && blazer_user && !params[:filter] && Blazer.audit
-            queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).where("created_at > ?", 30.days.ago).where("query_id IS NOT NULL").group(:query_id).order("count_all desc").count.keys)
-          else
-            []
-          end
-
         @queries = Blazer::Query.named.select(:id, :name, :creator_id, :statement)
         @queries = @queries.includes(:creator) if Blazer.user_class
 
@@ -263,15 +299,14 @@ module Blazer
         elsif blazer_user && params[:filter] == "viewed" && Blazer.audit
           @queries = queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).order(created_at: :desc).limit(500).pluck(:query_id).uniq)
         else
-          @queries = @queries.where("id NOT IN (?)", @my_queries.map(&:id)) if @my_queries.any?
           @queries = @queries.limit(limit) if limit
-          @queries = @queries.order(:name)
+          @queries = @queries.active.order(:name)
         end
         @queries = @queries.to_a
 
         @more = limit && @queries.size >= limit
 
-        @queries = (@my_queries + @queries).select { |q| !q.name.to_s.start_with?("#") || q.try(:creator).try(:id) == blazer_user.try(:id) }
+        @queries = @queries.select { |q| !q.name.to_s.start_with?("#") || q.try(:creator).try(:id) == blazer_user.try(:id) }
 
         @queries =
           @queries.map do |q|
@@ -279,14 +314,14 @@ module Blazer
               id: q.id,
               name: q.name,
               creator: blazer_user && q.try(:creator) == blazer_user ? "You" : q.try(:creator).try(Blazer.user_name),
-              vars: extract_vars(q.statement).join(", "),
+              vars: q.variables.join(", "),
               to_param: q.to_param
             }
           end
       end
 
       def queries_by_ids(favorite_query_ids)
-        queries = Blazer::Query.named.where(id: favorite_query_ids)
+        queries = Blazer::Query.active.named.where(id: favorite_query_ids)
         queries = queries.includes(:creator) if Blazer.user_class
         queries = queries.index_by(&:id)
         favorite_query_ids.map { |query_id| queries[query_id] }.compact
@@ -294,6 +329,14 @@ module Blazer
 
       def set_query
         @query = Blazer::Query.find(params[:id].to_s.split("-").first)
+
+        unless @query.viewable?(blazer_user)
+          render_forbidden
+        end
+      end
+
+      def render_forbidden
+        render plain: "Access denied", status: :forbidden
       end
 
       def query_params
@@ -321,5 +364,82 @@ module Blazer
       def blazer_run_id
         params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
       end
+
+      def run_cohort_analysis
+        unless @statement.data_source.supports_cohort_analysis?
+          @cohort_error = "This data source does not support cohort analysis"
+        end
+
+        @show_cohort_rows = !params[:query_id] || @cohort_error
+        cohort_analysis_statement(@statement) unless @show_cohort_rows
+      end
+
+      def render_cohort_analysis
+        if @show_cohort_rows
+          @cohort_analysis = false
+
+          @row_limit = 1000
+
+          # check results
+          unless @cohort_error
+            # check names
+            expected_columns = ["user_id", "conversion_time"]
+            missing_columns = expected_columns - @result.columns
+            if missing_columns.any?
+              @cohort_error = "Expected user_id and conversion_time columns"
+            end
+
+            # check types (user_id can be any type)
+            unless @cohort_error
+              column_types = @result.columns.zip(@result.column_types).to_h
+
+              if !column_types["cohort_time"].in?(["time", nil])
+                @cohort_error = "cohort_time must be time column"
+              elsif !column_types["conversion_time"].in?(["time", nil])
+                @cohort_error = "conversion_time must be time column"
+              end
+            end
+          end
+        else
+          @today = Blazer.time_zone.today
+          @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
+          @buckets = {}
+          @rows.each do |r|
+            @buckets[[r[0], r[1]]] = r[2]
+          end
+
+          @cohort_dates = []
+          current_date = @max_cohort_date
+          while current_date && current_date >= @min_cohort_date
+            @cohort_dates << current_date
+            current_date =
+              case @cohort_period
+              when "day"
+                current_date - 1
+              when "week"
+                current_date - 7
+              else
+                current_date.prev_month
+              end
+          end
+
+          num_cols = @cohort_dates.size
+          @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
+          rows = []
+          date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
+          @cohort_dates.each do |date|
+            row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
+
+            num_cols.times do |i|
+              if @today >= date + (@cohort_days * i)
+                row << (@buckets[[date, i + 1]] || 0)
+              end
+            end
+
+            rows << row
+          end
+          @rows = rows
+        end
+      end
   end
 end