blazer 2.2.6 → 2.4.0

data/app/controllers/blazer/base_controller.rb

@@ -6,6 +6,8 @@ module Blazer
     skip_after_action(*filters, raise: false)
     skip_around_action(*filters, raise: false)
 
+    clear_helpers
+
     protect_from_forgery with: :exception
 
     if ENV["BLAZER_PASSWORD"]
@@ -65,6 +67,12 @@ module Blazer
         end
       end
 
+      def add_cohort_analysis_vars
+        @bind_vars << "cohort_period" unless @bind_vars.include?("cohort_period")
+        @smart_vars["cohort_period"] = ["day", "week", "month"]
+        params[:cohort_period] ||= "week"
+      end
+
       def parse_smart_variables(var, data_source)
         smart_var_data_source =
           ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_variables[var] }
@@ -96,12 +104,12 @@ module Blazer
       # when permitted parameters are passed in Rails 6,
       # they appear to be added as GET parameters
       # root_url(params.permit(:host))
-      BLACKLISTED_KEYS = [:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id, :script_name, :original_script_name]
+      UNPERMITTED_KEYS = [:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id, :script_name, :original_script_name]
 
-      # remove blacklisted keys from both params and permitted keys for better sleep
+      # remove unpermitted keys from both params and permitted keys for better sleep
       def variable_params(resource)
-        permitted_keys = resource.variables - BLACKLISTED_KEYS.map(&:to_s)
-        params.except(*BLACKLISTED_KEYS).permit(*permitted_keys)
+        permitted_keys = resource.variables - UNPERMITTED_KEYS.map(&:to_s)
+        params.except(*UNPERMITTED_KEYS).slice(*permitted_keys).permit!
       end
       helper_method :variable_params
 

data/app/controllers/blazer/dashboards_controller.rb

@@ -21,8 +21,11 @@ module Blazer
 
     def show
       @queries = @dashboard.dashboard_queries.order(:position).preload(:query).map(&:query)
+      @statements = []
       @queries.each do |query|
-        process_vars(query.statement, query.data_source)
+        statement = query.statement.dup
+        process_vars(statement, query.data_source)
+        @statements << statement
       end
       @bind_vars ||= []
 
@@ -36,6 +39,8 @@ module Blazer
           @sql_errors << error if error
         end
       end
+
+      add_cohort_analysis_vars if @queries.any?(&:cohort_analysis?)
     end
 
     def edit
@@ -51,7 +56,7 @@ module Blazer
 
     def destroy
       @dashboard.destroy
-      redirect_to dashboards_path
+      redirect_to root_path
     end
 
     def refresh

data/app/controllers/blazer/queries_controller.rb

@@ -38,11 +38,18 @@ module Blazer
       if params[:fork_query_id]
         @query.statement ||= Blazer::Query.find(params[:fork_query_id]).try(:statement)
       end
+      if params[:upload_id]
+        upload = Blazer::Upload.find(params[:upload_id])
+        upload_settings = Blazer.settings["uploads"]
+        @query.data_source ||= upload_settings["data_source"]
+        @query.statement ||= "SELECT * FROM #{upload.table_name} LIMIT 10"
+      end
     end
 
     def create
       @query = Blazer::Query.new(query_params)
       @query.creator = blazer_user if @query.respond_to?(:creator)
+      @query.status = "active" if @query.respond_to?(:status)
 
       if @query.save
         redirect_to query_path(@query, variable_params(@query))
@@ -64,7 +71,11 @@ module Blazer
         @sql_errors << error if error
       end
 
+      @query.update!(status: "active") if @query.try(:status) == "archived"
+
       Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+
+      add_cohort_analysis_vars if @query.cohort_analysis?
     end
 
     def edit
@@ -72,6 +83,8 @@ module Blazer
 
     def run
       @statement = params[:statement]
+      # before process_vars
+      @cohort_analysis = Query.new(statement: @statement).cohort_analysis?
       data_source = params[:data_source]
       process_vars(@statement, data_source)
       @only_chart = params[:only_chart]
@@ -80,6 +93,8 @@ module Blazer
       data_source = @query.data_source if @query && @query.data_source
       @data_source = Blazer.data_sources[data_source]
 
+      run_cohort_analysis if @cohort_analysis
+
       # ensure viewable
       if !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)
         render_forbidden
@@ -155,6 +170,7 @@ module Blazer
       @statement = @query.statement.dup
       process_vars(@statement, @query.data_source)
       Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
+      @statement = cohort_analysis_statement(data_source, @statement) if @query.cohort_analysis?
       data_source.clear_cache(@statement)
       redirect_to query_path(@query, variable_params(@query))
     end
@@ -232,7 +248,6 @@ module Blazer
           end
         end
 
-        @filename = @query.name.parameterize if @query
         @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
 
         @boom = @result.boom if @result
@@ -259,6 +274,8 @@ module Blazer
           end
         end
 
+        render_cohort_analysis if @cohort_analysis && !@error
+
         respond_to do |format|
           format.html do
             render layout: false
@@ -279,7 +296,7 @@ module Blazer
           @queries = queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).order(created_at: :desc).limit(500).pluck(:query_id).uniq)
         else
           @queries = @queries.limit(limit) if limit
-          @queries = @queries.order(:name)
+          @queries = @queries.active.order(:name)
         end
         @queries = @queries.to_a
 
@@ -300,7 +317,7 @@ module Blazer
       end
 
       def queries_by_ids(favorite_query_ids)
-        queries = Blazer::Query.named.where(id: favorite_query_ids)
+        queries = Blazer::Query.active.named.where(id: favorite_query_ids)
         queries = queries.includes(:creator) if Blazer.user_class
         queries = queries.index_by(&:id)
         favorite_query_ids.map { |query_id| queries[query_id] }.compact
@@ -343,5 +360,104 @@ module Blazer
       def blazer_run_id
         params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
       end
+
+      def run_cohort_analysis
+        unless @data_source.supports_cohort_analysis?
+          @cohort_error = "This data source does not support cohort analysis"
+        end
+
+        @show_cohort_rows = !params[:query_id] || @cohort_error
+
+        unless @show_cohort_rows
+          @statement = cohort_analysis_statement(@data_source, @statement)
+        end
+      end
+
+      def cohort_analysis_statement(data_source, statement)
+        @cohort_period = params["cohort_period"] || "week"
+        @cohort_period = "week" unless ["day", "week", "month"].include?(@cohort_period)
+
+        # for now
+        @conversion_period = @cohort_period
+        @cohort_days =
+          case @cohort_period
+          when "day"
+            1
+          when "week"
+            7
+          when "month"
+            30
+          end
+
+        data_source.cohort_analysis_statement(statement, period: @cohort_period, days: @cohort_days)
+      end
+
+      def render_cohort_analysis
+        if @show_cohort_rows
+          @cohort_analysis = false
+
+          @row_limit = 1000
+
+          # check results
+          unless @cohort_error
+            # check names
+            expected_columns = ["user_id", "conversion_time"]
+            missing_columns = expected_columns - @result.columns
+            if missing_columns.any?
+              @cohort_error = "Expected user_id and conversion_time columns"
+            end
+
+            # check types (user_id can be any type)
+            unless @cohort_error
+              column_types = @result.columns.zip(@result.column_types).to_h
+
+              if !column_types["cohort_time"].in?(["time", nil])
+                @cohort_error = "cohort_time must be time column"
+              elsif !column_types["conversion_time"].in?(["time", nil])
+                @cohort_error = "conversion_time must be time column"
+              end
+            end
+          end
+        else
+          @today = Blazer.time_zone.today
+          @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
+          @buckets = {}
+          @rows.each do |r|
+            @buckets[[r[0], r[1]]] = r[2]
+          end
+
+          @cohort_dates = []
+          current_date = @max_cohort_date
+          while current_date && current_date >= @min_cohort_date
+            @cohort_dates << current_date
+            current_date =
+              case @cohort_period
+              when "day"
+                current_date - 1
+              when "week"
+                current_date - 7
+              else
+                current_date.prev_month
+              end
+          end
+
+          num_cols = @cohort_dates.size
+          @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
+          rows = []
+          date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
+          @cohort_dates.each do |date|
+            row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
+
+            num_cols.times do |i|
+              if @today >= date + (@cohort_days * i)
+                row << (@buckets[[date, i + 1]] || 0)
+              end
+            end
+
+            rows << row
+          end
+          @rows = rows
+        end
+      end
   end
 end

data/app/controllers/blazer/uploads_controller.rb

@@ -0,0 +1,147 @@
+module Blazer
+  class UploadsController < BaseController
+    before_action :ensure_uploads
+    before_action :set_upload, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @uploads = Blazer::Upload.order(:table)
+    end
+
+    def new
+      @upload = Blazer::Upload.new
+    end
+
+    def create
+      @upload = Blazer::Upload.new(upload_params)
+      # use creator_id instead of creator
+      # since we setup association without checking if column exists
+      @upload.creator = blazer_user if @upload.respond_to?(:creator_id=) && blazer_user
+
+      success = params.require(:upload).key?(:file)
+      if success
+        Blazer::Upload.transaction do
+          success = @upload.save
+          if success
+            begin
+              update_file(@upload)
+            rescue CSV::MalformedCSVError, Blazer::UploadError => e
+              @upload.errors.add(:base, e.message)
+              success = false
+              raise ActiveRecord::Rollback
+            end
+          end
+        end
+      else
+        @upload.errors.add(:base, "File can't be blank")
+      end
+
+      if success
+        redirect_to upload_path(@upload)
+      else
+        render_errors @upload
+      end
+    end
+
+    def show
+      redirect_to new_query_path(upload_id: @upload.id)
+    end
+
+    def edit
+    end
+
+    def update
+      original_table = @upload.table
+      @upload.assign_attributes(upload_params)
+
+      success = nil
+      Blazer::Upload.transaction do
+        success = @upload.save
+        if success
+          if params.require(:upload).key?(:file)
+            begin
+              update_file(@upload, drop: original_table)
+            rescue CSV::MalformedCSVError, Blazer::UploadError => e
+              @upload.errors.add(:base, e.message)
+              success = false
+              raise ActiveRecord::Rollback
+            end
+          elsif @upload.table != original_table
+            Blazer.uploads_connection.execute("ALTER TABLE #{Blazer.uploads_table_name(original_table)} RENAME TO #{Blazer.uploads_connection.quote_table_name(@upload.table)}")
+          end
+        end
+      end
+
+      if success
+        redirect_to upload_path(@upload)
+      else
+        render_errors @upload
+      end
+    end
+
+    def destroy
+      Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{@upload.table_name}")
+      @upload.destroy
+      redirect_to uploads_path
+    end
+
+    private
+
+      def update_file(upload, drop: nil)
+        file = params.require(:upload).fetch(:file)
+        raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"
+        raise Blazer::UploadError, "File is too large (maximum is 100MB)" if file.size > 100.megabytes
+
+        contents = file.read
+        rows = CSV.parse(contents, converters: %i[numeric date date_time])
+
+        # friendly column names
+        columns = rows.shift.map { |v| v.to_s.encode("UTF-8").gsub("%", " pct ").parameterize.gsub("-", "_") }
+        duplicate_column = columns.find { |c| columns.count(c) > 1 }
+        raise Blazer::UploadError, "Duplicate column name: #{duplicate_column}" if duplicate_column
+
+        column_types =
+          columns.size.times.map do |i|
+            values = rows.map { |r| r[i] }.uniq.compact
+            if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }
+              "bigint"
+            elsif values.all? { |v| v.is_a?(Numeric) }
+              "decimal"
+            elsif values.all? { |v| v.is_a?(DateTime) }
+              "timestamptz"
+            elsif values.all? { |v| v.is_a?(Date) }
+              "date"
+            else
+              "text"
+            end
+          end
+
+        begin
+          # maybe SET LOCAL statement_timeout = '30s'
+          # maybe regenerate CSV in Ruby to ensure consistent parsing
+          Blazer.uploads_connection.transaction do
+            Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop
+            Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")
+            Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do
+              Blazer.uploads_connection.raw_connection.put_copy_data(contents)
+            end
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          raise Blazer::UploadError, "Table already exists" if e.message.include?("PG::DuplicateTable")
+          raise e
+        end
+      end
+
+      def upload_params
+        params.require(:upload).except(:file).permit(:table, :description)
+      end
+
+      def set_upload
+        @upload = Blazer::Upload.find(params[:id])
+      end
+
+      # routes aren't added, but also check here
+      def ensure_uploads
+        render plain: "Uploads not enabled" unless Blazer.uploads?
+      end
+  end
+end

data/app/mailers/blazer/slack_notifier.rb

@@ -62,7 +62,7 @@ module Blazer
 
     # checks shouldn't have variables, but in any case,
     # avoid passing variable params to url helpers
-    # (known unsafe parameters are removed, but blacklist isn't ideal)
+    # (known unsafe parameters are removed, but still not ideal)
     def self.query_url(id)
       Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
     end

data/app/models/blazer/query.rb

@@ -8,6 +8,7 @@ module Blazer
 
     validates :statement, presence: true
 
+    scope :active, -> { column_names.include?("status") ? where(status: "active") : all }
     scope :named, -> { where("blazer_queries.name <> ''") }
 
     def to_param
@@ -34,7 +35,13 @@ module Blazer
     end
 
     def variables
-      Blazer.extract_vars(statement)
+      variables = Blazer.extract_vars(statement)
+      variables += ["cohort_period"] if cohort_analysis?
+      variables
+    end
+
+    def cohort_analysis?
+      /\/\*\s*cohort analysis\s*\*\//i.match?(statement)
     end
   end
 end

data/app/models/blazer/upload.rb

@@ -0,0 +1,11 @@
+module Blazer
+  class Upload < Record
+    belongs_to :creator, optional: true, class_name: Blazer.user_class.to_s if Blazer.user_class
+
+    validates :table, presence: true, uniqueness: true, format: {with: /\A[a-z0-9_]+\z/, message: "can only contain lowercase letters, numbers, and underscores"}, length: {maximum: 63}
+
+    def table_name
+      Blazer.uploads_table_name(table)
+    end
+  end
+end